Lines Matching refs:dl
148 const SDLoc &dl, in makeLibCall() argument
187 CLI.setDebugLoc(dl) in makeLibCall()
293 const SDLoc &dl, const SDValue OldLHS, in softenSetCCOperands() argument
296 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS, in softenSetCCOperands()
303 const SDLoc &dl, const SDValue OldLHS, in softenSetCCOperands() argument
409 auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain); in softenSetCCOperands()
411 NewRHS = DAG.getConstant(0, dl, RetVT); in softenSetCCOperands()
425 SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode); in softenSetCCOperands()
426 auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain); in softenSetCCOperands()
430 NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode); in softenSetCCOperands()
432 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second, in softenSetCCOperands()
434 NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl, in softenSetCCOperands()
476 SDValue TargetLowering::expandIndirectJTBranch(const SDLoc &dl, SDValue Value, in expandIndirectJTBranch() argument
482 Chain = DAG.getJumpTableDebugInfo(JTI, Chain, dl); in expandIndirectJTBranch()
484 return DAG.getNode(ISD::BRIND, dl, MVT::Other, Chain, Addr); in expandIndirectJTBranch()
582 SDLoc dl(Op); in ShrinkDemandedOp() local
603 Op.getOpcode(), dl, SmallVT, in ShrinkDemandedOp()
604 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)), in ShrinkDemandedOp()
605 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1))); in ShrinkDemandedOp()
607 SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, VT, X); in ShrinkDemandedOp()
1098 SDLoc dl(Op); in SimplifyDemandedBits() local
1270 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub, in SimplifyDemandedBits()
1297 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, in SimplifyDemandedBits()
1361 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask); in SimplifyDemandedBits()
1396 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1); in SimplifyDemandedBits()
1416 TLO.DAG.getNode(ISD::AND, dl, VT, Op0.getOperand(0), Op1); in SimplifyDemandedBits()
1418 TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, NewAnd, in SimplifyDemandedBits()
1441 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT)); in SimplifyDemandedBits()
1459 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1); in SimplifyDemandedBits()
1512 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1); in SimplifyDemandedBits()
1531 if (SDValue C12 = TLO.DAG.FoldConstantArithmetic(ISD::OR, dl, VT, in SimplifyDemandedBits()
1533 SDValue MaskX = TLO.DAG.getNode(ISD::AND, dl, VT, X, C12); in SimplifyDemandedBits()
1534 SDValue MaskY = TLO.DAG.getNode(ISD::AND, dl, VT, Y, C2); in SimplifyDemandedBits()
1536 Op, TLO.DAG.getNode(ISD::OR, dl, VT, MaskX, MaskY)); in SimplifyDemandedBits()
1574 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
1585 TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT); in SimplifyDemandedBits()
1586 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC)); in SimplifyDemandedBits()
1594 SDValue New = TLO.DAG.getNOT(dl, Op0, VT); in SimplifyDemandedBits()
1616 SDValue Not = TLO.DAG.getNOT(dl, Op0.getOperand(0), VT); in SimplifyDemandedBits()
1617 return TLO.CombineTo(Op, TLO.DAG.getNode(Op0Opcode, dl, VT, Not, in SimplifyDemandedBits()
1639 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1); in SimplifyDemandedBits()
1748 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT); in SimplifyDemandedBits()
1750 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA)); in SimplifyDemandedBits()
1765 ISD::SHL, dl, InnerVT, InnerOp, in SimplifyDemandedBits()
1766 TLO.DAG.getShiftAmountConstant(ShAmt, InnerVT, dl)); in SimplifyDemandedBits()
1768 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl)); in SimplifyDemandedBits()
1787 TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT); in SimplifyDemandedBits()
1788 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, in SimplifyDemandedBits()
1791 Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA)); in SimplifyDemandedBits()
1821 SDValue NewOp = TLO.DAG.getNode(ISD::SHL, dl, VT, DemandedOp0, Op1); in SimplifyDemandedBits()
1851 SDValue NewOp = TLO.DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Op0); in SimplifyDemandedBits()
1853 ShAmt, HalfVT, dl, TLO.LegalTypes()); in SimplifyDemandedBits()
1854 SDValue NewShift = TLO.DAG.getNode(ISD::SHL, dl, HalfVT, NewOp, in SimplifyDemandedBits()
1857 TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, NewShift); in SimplifyDemandedBits()
1928 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT); in SimplifyDemandedBits()
1930 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA)); in SimplifyDemandedBits()
1953 SDValue NewOp = TLO.DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Op0); in SimplifyDemandedBits()
1955 ShAmt, HalfVT, dl, TLO.LegalTypes()); in SimplifyDemandedBits()
1957 TLO.DAG.getNode(ISD::SRL, dl, HalfVT, NewOp, NewShiftAmt); in SimplifyDemandedBits()
1959 Op, TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, NewShift)); in SimplifyDemandedBits()
1978 SDValue NewOp = TLO.DAG.getNode(ISD::SRL, dl, VT, DemandedOp0, Op1); in SimplifyDemandedBits()
2006 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
2034 Op, TLO.DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT, in SimplifyDemandedBits()
2074 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags)); in SimplifyDemandedBits()
2080 SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT); in SimplifyDemandedBits()
2081 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA)); in SimplifyDemandedBits()
2093 SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1); in SimplifyDemandedBits()
2146 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedOp0, in SimplifyDemandedBits()
2190 Op1 = TLO.DAG.getConstant(IsROTL ? Amt : RevAmt, dl, Op1.getValueType()); in SimplifyDemandedBits()
2191 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
2195 Op1 = TLO.DAG.getConstant(IsROTL ? RevAmt : Amt, dl, Op1.getValueType()); in SimplifyDemandedBits()
2196 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
2293 SDValue ShAmt = TLO.DAG.getConstant(ShiftAmount, dl, ShiftAmtTy); in SimplifyDemandedBits()
2294 SDValue NewOp = TLO.DAG.getNode(ShiftOpcode, dl, VT, Src, ShAmt); in SimplifyDemandedBits()
2312 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT, in SimplifyDemandedBits()
2333 SDValue ShiftAmt = TLO.DAG.getConstant(BitWidth - ExVTBits, dl, in SimplifyDemandedBits()
2336 TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt)); in SimplifyDemandedBits()
2360 return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT)); in SimplifyDemandedBits()
2412 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src)); in SimplifyDemandedBits()
2433 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc)); in SimplifyDemandedBits()
2469 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src)); in SimplifyDemandedBits()
2490 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src, Flags)); in SimplifyDemandedBits()
2497 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc)); in SimplifyDemandedBits()
2529 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc)); in SimplifyDemandedBits()
2547 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc)); in SimplifyDemandedBits()
2578 ShVal, dl, getShiftAmountTy(VT, DL, TLO.LegalTypes())); in SimplifyDemandedBits()
2580 TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0)); in SimplifyDemandedBits()
2582 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt)); in SimplifyDemandedBits()
2636 TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx); in SimplifyDemandedBits()
2666 SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src); in SimplifyDemandedBits()
2669 Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign); in SimplifyDemandedBits()
2671 SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT); in SimplifyDemandedBits()
2673 TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt)); in SimplifyDemandedBits()
2757 SDValue AmtC = TLO.DAG.getConstant(CTZ, dl, ShiftAmtTy); in SimplifyDemandedBits()
2758 SDValue Shl = TLO.DAG.getNode(ISD::SHL, dl, VT, Op.getOperand(0), AmtC); in SimplifyDemandedBits()
2766 SDValue One = TLO.DAG.getConstant(1, dl, VT); in SimplifyDemandedBits()
2767 SDValue And1 = TLO.DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0), One); in SimplifyDemandedBits()
2813 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags); in SimplifyDemandedBits()
2827 SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT); in SimplifyDemandedBits()
2832 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags); in SimplifyDemandedBits()
2857 SDValue ShlAmtC = TLO.DAG.getConstant(ShlAmt, dl, ShiftAmtTy); in SimplifyDemandedBits()
2858 SDValue Shl = TLO.DAG.getNode(ISD::SHL, dl, VT, X, ShlAmtC); in SimplifyDemandedBits()
2859 SDValue Res = TLO.DAG.getNode(NT, dl, VT, Y, Shl); in SimplifyDemandedBits()
2920 return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT)); in SimplifyDemandedBits()
2925 APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT)); in SimplifyDemandedBits()
4218 ISD::CondCode Cond, const SDLoc &dl, in simplifySetCCWithCTPOP() argument
4249 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT); in simplifySetCCWithCTPOP()
4252 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne); in simplifySetCCWithCTPOP()
4253 Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add); in simplifySetCCWithCTPOP()
4256 return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC); in simplifySetCCWithCTPOP()
4265 SDValue Zero = DAG.getConstant(0, dl, CTVT); in simplifySetCCWithCTPOP()
4266 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT); in simplifySetCCWithCTPOP()
4268 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne); in simplifySetCCWithCTPOP()
4275 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add); in simplifySetCCWithCTPOP()
4276 SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond); in simplifySetCCWithCTPOP()
4282 SDValue Xor = DAG.getNode(ISD::XOR, dl, CTVT, CTOp, Add); in simplifySetCCWithCTPOP()
4284 return DAG.getSetCC(dl, VT, Xor, Add, CmpCond); in simplifySetCCWithCTPOP()
4291 ISD::CondCode Cond, const SDLoc &dl, in foldSetCCWithRotate() argument
4310 return DAG.getSetCC(dl, VT, R, N1, Cond); in foldSetCCWithRotate()
4321 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, R, N0.getOperand(1)); in foldSetCCWithRotate()
4322 return DAG.getSetCC(dl, VT, NewOr, N1, Cond); in foldSetCCWithRotate()
4325 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, R, N0.getOperand(0)); in foldSetCCWithRotate()
4326 return DAG.getSetCC(dl, VT, NewOr, N1, Cond); in foldSetCCWithRotate()
4334 ISD::CondCode Cond, const SDLoc &dl, in foldSetCCWithFunnelShift() argument
4383 SDValue NewShAmt = DAG.getConstant(ShAmt, dl, ShAmtVT); in foldSetCCWithFunnelShift()
4384 SDValue Shift = DAG.getNode(ISD::SHL, dl, OpVT, Y, NewShAmt); in foldSetCCWithFunnelShift()
4385 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, Shift, X); in foldSetCCWithFunnelShift()
4386 return DAG.getSetCC(dl, VT, NewOr, N1, Cond); in foldSetCCWithFunnelShift()
4390 SDValue NewShAmt = DAG.getConstant(BitWidth - ShAmt, dl, ShAmtVT); in foldSetCCWithFunnelShift()
4391 SDValue Shift = DAG.getNode(ISD::SRL, dl, OpVT, Y, NewShAmt); in foldSetCCWithFunnelShift()
4392 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, Shift, X); in foldSetCCWithFunnelShift()
4393 return DAG.getSetCC(dl, VT, NewOr, N1, Cond); in foldSetCCWithFunnelShift()
4404 const SDLoc &dl) const { in SimplifySetCC()
4411 if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl)) in SimplifySetCC()
4427 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC); in SimplifySetCC()
4438 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC); in SimplifySetCC()
4440 if (SDValue V = foldSetCCWithRotate(VT, N0, N1, Cond, dl, DAG)) in SimplifySetCC()
4443 if (SDValue V = foldSetCCWithFunnelShift(VT, N0, N1, Cond, dl, DAG)) in SimplifySetCC()
4450 if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG)) in SimplifySetCC()
4464 SDValue IsXZero = DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond); in SimplifySetCC()
4465 SDValue IsYZero = DAG.getSetCC(dl, VT, N0.getOperand(1), N1, Cond); in SimplifySetCC()
4467 return DAG.getNode(LogicOp, dl, VT, IsXZero, IsYZero); in SimplifySetCC()
4488 SDValue Zero = DAG.getConstant(0, dl, N0.getValueType()); in SimplifySetCC()
4489 return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero, in SimplifySetCC()
4545 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt); in SimplifySetCC()
4548 return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1), in SimplifySetCC()
4550 SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT); in SimplifySetCC()
4551 return DAG.getSetCC(dl, VT, Trunc, C, Cond); in SimplifySetCC()
4580 return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0), in SimplifySetCC()
4631 dl); in SimplifySetCC()
4633 DAG.getLoad(newVT, dl, Lod->getChain(), Ptr, in SimplifySetCC()
4636 return DAG.getSetCC(dl, VT, in SimplifySetCC()
4637 DAG.getNode(ISD::AND, dl, newVT, NewLoad, in SimplifySetCC()
4639 dl, newVT)), in SimplifySetCC()
4640 DAG.getConstant(0LL, dl, newVT), Cond); in SimplifySetCC()
4657 return DAG.getConstant(0, dl, VT); in SimplifySetCC()
4661 return DAG.getConstant(1, dl, VT); in SimplifySetCC()
4665 return DAG.getConstant(C1.isNegative(), dl, VT); in SimplifySetCC()
4669 return DAG.getConstant(C1.isNonNegative(), dl, VT); in SimplifySetCC()
4688 SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT); in SimplifySetCC()
4690 SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0), in SimplifySetCC()
4692 return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType()); in SimplifySetCC()
4711 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT); in SimplifySetCC()
4716 SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0), in SimplifySetCC()
4717 DAG.getConstant(Imm, dl, ExtDstTy)); in SimplifySetCC()
4721 return DAG.getSetCC(dl, VT, ZextOp, in SimplifySetCC()
4722 DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond); in SimplifySetCC()
4733 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0); in SimplifySetCC()
4739 return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC); in SimplifySetCC()
4761 Val = DAG.getNode(ISD::AND, dl, N0.getValueType(), in SimplifySetCC()
4766 return DAG.getSetCC(dl, VT, Val, N1, in SimplifySetCC()
4787 return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond); in SimplifySetCC()
4793 Op0 = DAG.getNode(ISD::AND, dl, VT, in SimplifySetCC()
4794 DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)), in SimplifySetCC()
4795 DAG.getConstant(1, dl, VT)); in SimplifySetCC()
4797 Op0 = DAG.getNode(ISD::AND, dl, VT, in SimplifySetCC()
4798 DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)), in SimplifySetCC()
4799 DAG.getConstant(1, dl, VT)); in SimplifySetCC()
4801 return DAG.getSetCC(dl, VT, Op0, in SimplifySetCC()
4802 DAG.getConstant(0, dl, Op0.getValueType()), in SimplifySetCC()
4807 return DAG.getSetCC(dl, VT, Op0, in SimplifySetCC()
4808 DAG.getConstant(0, dl, Op0.getValueType()), in SimplifySetCC()
4822 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond); in SimplifySetCC()
4831 return DAG.getSetCC(dl, VT, N0.getOperand(0), in SimplifySetCC()
4832 DAG.getConstant(0, dl, OpVT), in SimplifySetCC()
4837 optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl)) in SimplifySetCC()
4860 return DAG.getBoolConstant(true, dl, VT, OpVT); in SimplifySetCC()
4870 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
4871 DAG.getConstant(C, dl, N1.getValueType()), in SimplifySetCC()
4880 return DAG.getBoolConstant(true, dl, VT, OpVT); in SimplifySetCC()
4890 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
4891 DAG.getConstant(C, dl, N1.getValueType()), in SimplifySetCC()
4899 return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false in SimplifySetCC()
4905 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE); in SimplifySetCC()
4909 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
4910 DAG.getConstant(MinVal, dl, N0.getValueType()), in SimplifySetCC()
4917 return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false in SimplifySetCC()
4923 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE); in SimplifySetCC()
4927 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
4928 DAG.getConstant(MaxVal, dl, N0.getValueType()), in SimplifySetCC()
4937 VT, N0, N1, Cond, DCI, dl)) in SimplifySetCC()
4979 SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT); in SimplifySetCC()
4980 SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits); in SimplifySetCC()
4982 DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask); in SimplifySetCC()
4983 SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits; in SimplifySetCC()
4984 return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond); in SimplifySetCC()
4995 return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1), in SimplifySetCC()
4996 DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1)); in SimplifySetCC()
5010 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
5011 DAG.getConstant(0, dl, N1.getValueType()), in SimplifySetCC()
5018 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
5019 DAG.getAllOnesConstant(dl, N1.getValueType()), in SimplifySetCC()
5048 return DAG.getNode(ISD::TRUNCATE, dl, VT, in SimplifySetCC()
5049 DAG.getNode(ISD::SRL, dl, ShValTy, N0, in SimplifySetCC()
5050 DAG.getConstant(ShCt, dl, ShiftTy))); in SimplifySetCC()
5058 return DAG.getNode(ISD::TRUNCATE, dl, VT, in SimplifySetCC()
5059 DAG.getNode(ISD::SRL, dl, ShValTy, N0, in SimplifySetCC()
5060 DAG.getConstant(ShCt, dl, ShiftTy))); in SimplifySetCC()
5078 DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0), in SimplifySetCC()
5079 DAG.getConstant(ShiftBits, dl, ShiftTy)); in SimplifySetCC()
5080 SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy); in SimplifySetCC()
5081 return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond); in SimplifySetCC()
5106 SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0, in SimplifySetCC()
5107 DAG.getConstant(ShiftBits, dl, ShiftTy)); in SimplifySetCC()
5108 SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy); in SimplifySetCC()
5109 return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond); in SimplifySetCC()
5124 return DAG.getSetCC(dl, VT, N0, N0, Cond); in SimplifySetCC()
5131 SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1); in SimplifySetCC()
5132 return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond); in SimplifySetCC()
5146 return DAG.getNode(ISD::IS_FPCLASS, dl, VT, Op, in SimplifySetCC()
5147 DAG.getTargetConstant(Flag, dl, MVT::i32)); in SimplifySetCC()
5169 return DAG.getSetCC(dl, VT, N0, N1, NewCond); in SimplifySetCC()
5183 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT); in SimplifySetCC()
5185 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT); in SimplifySetCC()
5192 return DAG.getSetCC(dl, VT, N0, N1, NewCond); in SimplifySetCC()
5203 return DAG.getSetCC(dl, VT, N1.getOperand(0), N0.getOperand(0), Cond); in SimplifySetCC()
5207 SDValue Not = DAG.getNOT(dl, N1, OpVT); in SimplifySetCC()
5208 return DAG.getSetCC(dl, VT, Not, N0.getOperand(0), Cond); in SimplifySetCC()
5220 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond); in SimplifySetCC()
5222 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond); in SimplifySetCC()
5226 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0), in SimplifySetCC()
5229 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1), in SimplifySetCC()
5243 dl, VT, N0.getOperand(0), in SimplifySetCC()
5245 dl, N0.getValueType()), in SimplifySetCC()
5251 dl, VT, N0.getOperand(0), in SimplifySetCC()
5253 dl, N0.getValueType()), in SimplifySetCC()
5261 dl, VT, N0.getOperand(1), in SimplifySetCC()
5263 dl, N0.getValueType()), in SimplifySetCC()
5276 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI)) in SimplifySetCC()
5282 if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI)) in SimplifySetCC()
5285 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI)) in SimplifySetCC()
5296 if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl)) in SimplifySetCC()
5299 if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl)) in SimplifySetCC()
5311 Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1); in SimplifySetCC()
5312 N0 = DAG.getNOT(dl, Temp, OpVT); in SimplifySetCC()
5317 N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1); in SimplifySetCC()
5321 Temp = DAG.getNOT(dl, N0, OpVT); in SimplifySetCC()
5322 N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp); in SimplifySetCC()
5328 Temp = DAG.getNOT(dl, N1, OpVT); in SimplifySetCC()
5329 N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp); in SimplifySetCC()
5335 Temp = DAG.getNOT(dl, N0, OpVT); in SimplifySetCC()
5336 N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp); in SimplifySetCC()
5342 Temp = DAG.getNOT(dl, N1, OpVT); in SimplifySetCC()
5343 N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp); in SimplifySetCC()
5351 N0 = DAG.getNode(ExtendCode, dl, VT, N0); in SimplifySetCC()
6021 const SDLoc &dl, SelectionDAG &DAG, in BuildExactSDIV() argument
6047 Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT)); in BuildExactSDIV()
6048 Factors.push_back(DAG.getConstant(Factor, dl, SVT)); in BuildExactSDIV()
6058 Shift = DAG.getBuildVector(ShVT, dl, Shifts); in BuildExactSDIV()
6059 Factor = DAG.getBuildVector(VT, dl, Factors); in BuildExactSDIV()
6064 Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]); in BuildExactSDIV()
6065 Factor = DAG.getSplatVector(VT, dl, Factors[0]); in BuildExactSDIV()
6079 Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags); in BuildExactSDIV()
6083 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor); in BuildExactSDIV()
6157 SDLoc dl(N); in BuildSDIV() local
6185 return BuildExactSDIV(*this, N, dl, DAG, Created); in BuildSDIV()
6212 MagicFactors.push_back(DAG.getConstant(magics.Magic, dl, SVT)); in BuildSDIV()
6213 Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT)); in BuildSDIV()
6214 Shifts.push_back(DAG.getConstant(magics.ShiftAmount, dl, ShSVT)); in BuildSDIV()
6215 ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT)); in BuildSDIV()
6228 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors); in BuildSDIV()
6229 Factor = DAG.getBuildVector(VT, dl, Factors); in BuildSDIV()
6230 Shift = DAG.getBuildVector(ShVT, dl, Shifts); in BuildSDIV()
6231 ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks); in BuildSDIV()
6237 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]); in BuildSDIV()
6238 Factor = DAG.getSplatVector(VT, dl, Factors[0]); in BuildSDIV()
6239 Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]); in BuildSDIV()
6240 ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]); in BuildSDIV()
6255 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X); in BuildSDIV()
6256 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y); in BuildSDIV()
6257 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y); in BuildSDIV()
6258 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y, in BuildSDIV()
6259 DAG.getShiftAmountConstant(EltBits, MulVT, dl)); in BuildSDIV()
6260 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildSDIV()
6264 return DAG.getNode(ISD::MULHS, dl, VT, X, Y); in BuildSDIV()
6267 DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y); in BuildSDIV()
6277 X = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, X); in BuildSDIV()
6278 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, Y); in BuildSDIV()
6279 Y = DAG.getNode(ISD::MUL, dl, WideVT, X, Y); in BuildSDIV()
6280 Y = DAG.getNode(ISD::SRL, dl, WideVT, Y, in BuildSDIV()
6281 DAG.getShiftAmountConstant(EltBits, WideVT, dl)); in BuildSDIV()
6282 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildSDIV()
6294 Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor); in BuildSDIV()
6296 Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor); in BuildSDIV()
6300 Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift); in BuildSDIV()
6304 SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT); in BuildSDIV()
6305 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift); in BuildSDIV()
6307 T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask); in BuildSDIV()
6309 return DAG.getNode(ISD::ADD, dl, VT, Q, T); in BuildSDIV()
6319 SDLoc dl(N); in BuildUDIV() local
6379 MagicFactor = DAG.getConstant(magics.Magic, dl, SVT); in BuildUDIV()
6387 PreShift = DAG.getConstant(magics.PreShift, dl, ShSVT); in BuildUDIV()
6388 PostShift = DAG.getConstant(magics.PostShift, dl, ShSVT); in BuildUDIV()
6392 dl, SVT); in BuildUDIV()
6411 PreShift = DAG.getBuildVector(ShVT, dl, PreShifts); in BuildUDIV()
6412 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors); in BuildUDIV()
6413 NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors); in BuildUDIV()
6414 PostShift = DAG.getBuildVector(ShVT, dl, PostShifts); in BuildUDIV()
6419 PreShift = DAG.getSplatVector(ShVT, dl, PreShifts[0]); in BuildUDIV()
6420 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]); in BuildUDIV()
6421 NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]); in BuildUDIV()
6422 PostShift = DAG.getSplatVector(ShVT, dl, PostShifts[0]); in BuildUDIV()
6432 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift); in BuildUDIV()
6441 X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X); in BuildUDIV()
6442 Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y); in BuildUDIV()
6443 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y); in BuildUDIV()
6444 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y, in BuildUDIV()
6445 DAG.getShiftAmountConstant(EltBits, MulVT, dl)); in BuildUDIV()
6446 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildUDIV()
6450 return DAG.getNode(ISD::MULHU, dl, VT, X, Y); in BuildUDIV()
6453 DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y); in BuildUDIV()
6463 X = DAG.getNode(ISD::ZERO_EXTEND, dl, WideVT, X); in BuildUDIV()
6464 Y = DAG.getNode(ISD::ZERO_EXTEND, dl, WideVT, Y); in BuildUDIV()
6465 Y = DAG.getNode(ISD::MUL, dl, WideVT, X, Y); in BuildUDIV()
6466 Y = DAG.getNode(ISD::SRL, dl, WideVT, Y, in BuildUDIV()
6467 DAG.getShiftAmountConstant(EltBits, WideVT, dl)); in BuildUDIV()
6468 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildUDIV()
6481 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q); in BuildUDIV()
6489 NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT)); in BuildUDIV()
6493 Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q); in BuildUDIV()
6498 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift); in BuildUDIV()
6504 SDValue One = DAG.getConstant(1, dl, VT); in BuildUDIV()
6505 SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ); in BuildUDIV()
6506 return DAG.getSelect(dl, VT, IsOne, N0, Q); in BuildUDIV()
7399 bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, in expandMUL_LOHI() argument
7431 Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R); in expandMUL_LOHI()
7436 Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R); in expandMUL_LOHI()
7437 Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R); in expandMUL_LOHI()
7447 LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS); in expandMUL_LOHI()
7448 RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS); in expandMUL_LOHI()
7462 SDValue Zero = DAG.getConstant(0, dl, HiLoVT); in expandMUL_LOHI()
7483 SDValue Shift = DAG.getShiftAmountConstant(ShiftAmount, VT, dl); in expandMUL_LOHI()
7488 LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift); in expandMUL_LOHI()
7489 LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH); in expandMUL_LOHI()
7490 RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift); in expandMUL_LOHI()
7491 RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH); in expandMUL_LOHI()
7503 RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH); in expandMUL_LOHI()
7504 LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL); in expandMUL_LOHI()
7505 Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH); in expandMUL_LOHI()
7506 Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH); in expandMUL_LOHI()
7513 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); in expandMUL_LOHI()
7514 Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi); in expandMUL_LOHI()
7515 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); in expandMUL_LOHI()
7516 return DAG.getNode(ISD::OR, dl, VT, Lo, Hi); in expandMUL_LOHI()
7519 SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi); in expandMUL_LOHI()
7525 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi)); in expandMUL_LOHI()
7530 SDValue Zero = DAG.getConstant(0, dl, HiLoVT); in expandMUL_LOHI()
7536 Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next, in expandMUL_LOHI()
7539 Next = DAG.getNode(ISD::UADDO_CARRY, dl, DAG.getVTList(VT, BoolType), Next, in expandMUL_LOHI()
7540 Merge(Lo, Hi), DAG.getConstant(0, dl, BoolType)); in expandMUL_LOHI()
7543 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next)); in expandMUL_LOHI()
7544 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift); in expandMUL_LOHI()
7550 Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero, in expandMUL_LOHI()
7553 Hi = DAG.getNode(ISD::UADDO_CARRY, dl, DAG.getVTList(HiLoVT, BoolType), Hi, in expandMUL_LOHI()
7556 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi)); in expandMUL_LOHI()
7559 SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next, in expandMUL_LOHI()
7560 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL)); in expandMUL_LOHI()
7561 Next = DAG.getSelectCC(dl, LH, Zero, NextSub, Next, ISD::SETLT); in expandMUL_LOHI()
7563 NextSub = DAG.getNode(ISD::SUB, dl, VT, Next, in expandMUL_LOHI()
7564 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL)); in expandMUL_LOHI()
7565 Next = DAG.getSelectCC(dl, RH, Zero, NextSub, Next, ISD::SETLT); in expandMUL_LOHI()
7568 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next)); in expandMUL_LOHI()
7569 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift); in expandMUL_LOHI()
7570 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next)); in expandMUL_LOHI()
7661 SDLoc dl(N); in expandDIVREMByConstant() local
7672 std::tie(LL, LH) = DAG.SplitScalar(N->getOperand(0), dl, HiLoVT, HiLoVT); in expandDIVREMByConstant()
7680 PartialRem = DAG.getNode(ISD::AND, dl, HiLoVT, LL, in expandDIVREMByConstant()
7681 DAG.getConstant(Mask, dl, HiLoVT)); in expandDIVREMByConstant()
7685 ISD::OR, dl, HiLoVT, in expandDIVREMByConstant()
7686 DAG.getNode(ISD::SRL, dl, HiLoVT, LL, in expandDIVREMByConstant()
7687 DAG.getShiftAmountConstant(TrailingZeros, HiLoVT, dl)), in expandDIVREMByConstant()
7688 DAG.getNode(ISD::SHL, dl, HiLoVT, LH, in expandDIVREMByConstant()
7690 HiLoVT, dl))); in expandDIVREMByConstant()
7691 LH = DAG.getNode(ISD::SRL, dl, HiLoVT, LH, in expandDIVREMByConstant()
7692 DAG.getShiftAmountConstant(TrailingZeros, HiLoVT, dl)); in expandDIVREMByConstant()
7700 Sum = DAG.getNode(ISD::UADDO, dl, VTList, LL, LH); in expandDIVREMByConstant()
7701 Sum = DAG.getNode(ISD::UADDO_CARRY, dl, VTList, Sum, in expandDIVREMByConstant()
7702 DAG.getConstant(0, dl, HiLoVT), Sum.getValue(1)); in expandDIVREMByConstant()
7704 Sum = DAG.getNode(ISD::ADD, dl, HiLoVT, LL, LH); in expandDIVREMByConstant()
7705 SDValue Carry = DAG.getSetCC(dl, SetCCType, Sum, LL, ISD::SETULT); in expandDIVREMByConstant()
7710 Carry = DAG.getZExtOrTrunc(Carry, dl, HiLoVT); in expandDIVREMByConstant()
7712 Carry = DAG.getSelect(dl, HiLoVT, Carry, DAG.getConstant(1, dl, HiLoVT), in expandDIVREMByConstant()
7713 DAG.getConstant(0, dl, HiLoVT)); in expandDIVREMByConstant()
7714 Sum = DAG.getNode(ISD::ADD, dl, HiLoVT, Sum, Carry); in expandDIVREMByConstant()
7724 DAG.getNode(ISD::UREM, dl, HiLoVT, Sum, in expandDIVREMByConstant()
7725 DAG.getConstant(Divisor.trunc(HBitWidth), dl, HiLoVT)); in expandDIVREMByConstant()
7726 SDValue RemH = DAG.getConstant(0, dl, HiLoVT); in expandDIVREMByConstant()
7730 SDValue Dividend = DAG.getNode(ISD::BUILD_PAIR, dl, VT, LL, LH); in expandDIVREMByConstant()
7731 SDValue Rem = DAG.getNode(ISD::BUILD_PAIR, dl, VT, RemL, RemH); in expandDIVREMByConstant()
7733 Dividend = DAG.getNode(ISD::SUB, dl, VT, Dividend, Rem); in expandDIVREMByConstant()
7742 SDValue Quotient = DAG.getNode(ISD::MUL, dl, VT, Dividend, in expandDIVREMByConstant()
7743 DAG.getConstant(MulFactor, dl, VT)); in expandDIVREMByConstant()
7747 std::tie(QuotL, QuotH) = DAG.SplitScalar(Quotient, dl, HiLoVT, HiLoVT); in expandDIVREMByConstant()
7757 RemL = DAG.getNode(ISD::SHL, dl, HiLoVT, RemL, in expandDIVREMByConstant()
7758 DAG.getShiftAmountConstant(TrailingZeros, HiLoVT, dl)); in expandDIVREMByConstant()
7759 RemL = DAG.getNode(ISD::ADD, dl, HiLoVT, RemL, PartialRem); in expandDIVREMByConstant()
7762 Result.push_back(DAG.getConstant(0, dl, HiLoVT)); in expandDIVREMByConstant()
7992 SDLoc dl(Node); in expandShiftParts() local
7997 SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt, in expandShiftParts()
7998 DAG.getConstant(VTBits - 1, dl, ShAmtVT)); in expandShiftParts()
7999 SDValue Tmp1 = IsSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi, in expandShiftParts()
8000 DAG.getConstant(VTBits - 1, dl, ShAmtVT)) in expandShiftParts()
8001 : DAG.getConstant(0, dl, VT); in expandShiftParts()
8005 Tmp2 = DAG.getNode(ISD::FSHL, dl, VT, ShOpHi, ShOpLo, ShAmt); in expandShiftParts()
8006 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt); in expandShiftParts()
8008 Tmp2 = DAG.getNode(ISD::FSHR, dl, VT, ShOpHi, ShOpLo, ShAmt); in expandShiftParts()
8009 Tmp3 = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt); in expandShiftParts()
8015 SDValue AndNode = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt, in expandShiftParts()
8016 DAG.getConstant(VTBits, dl, ShAmtVT)); in expandShiftParts()
8017 SDValue Cond = DAG.getSetCC(dl, ShAmtCCVT, AndNode, in expandShiftParts()
8018 DAG.getConstant(0, dl, ShAmtVT), ISD::SETNE); in expandShiftParts()
8021 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2); in expandShiftParts()
8022 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3); in expandShiftParts()
8024 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2); in expandShiftParts()
8025 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3); in expandShiftParts()
8035 SDLoc dl(SDValue(Node, 0)); in expandFP_TO_SINT() local
8055 SDValue ExponentMask = DAG.getConstant(0x7F800000, dl, IntVT); in expandFP_TO_SINT()
8056 SDValue ExponentLoBit = DAG.getConstant(23, dl, IntVT); in expandFP_TO_SINT()
8057 SDValue Bias = DAG.getConstant(127, dl, IntVT); in expandFP_TO_SINT()
8058 SDValue SignMask = DAG.getConstant(APInt::getSignMask(SrcEltBits), dl, IntVT); in expandFP_TO_SINT()
8059 SDValue SignLowBit = DAG.getConstant(SrcEltBits - 1, dl, IntVT); in expandFP_TO_SINT()
8060 SDValue MantissaMask = DAG.getConstant(0x007FFFFF, dl, IntVT); in expandFP_TO_SINT()
8062 SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Src); in expandFP_TO_SINT()
8065 ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask), in expandFP_TO_SINT()
8066 DAG.getZExtOrTrunc(ExponentLoBit, dl, IntShVT)); in expandFP_TO_SINT()
8067 SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias); in expandFP_TO_SINT()
8069 SDValue Sign = DAG.getNode(ISD::SRA, dl, IntVT, in expandFP_TO_SINT()
8070 DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask), in expandFP_TO_SINT()
8071 DAG.getZExtOrTrunc(SignLowBit, dl, IntShVT)); in expandFP_TO_SINT()
8072 Sign = DAG.getSExtOrTrunc(Sign, dl, DstVT); in expandFP_TO_SINT()
8074 SDValue R = DAG.getNode(ISD::OR, dl, IntVT, in expandFP_TO_SINT()
8075 DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask), in expandFP_TO_SINT()
8076 DAG.getConstant(0x00800000, dl, IntVT)); in expandFP_TO_SINT()
8078 R = DAG.getZExtOrTrunc(R, dl, DstVT); in expandFP_TO_SINT()
8081 dl, Exponent, ExponentLoBit, in expandFP_TO_SINT()
8082 DAG.getNode(ISD::SHL, dl, DstVT, R, in expandFP_TO_SINT()
8084 DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit), in expandFP_TO_SINT()
8085 dl, IntShVT)), in expandFP_TO_SINT()
8086 DAG.getNode(ISD::SRL, dl, DstVT, R, in expandFP_TO_SINT()
8088 DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent), in expandFP_TO_SINT()
8089 dl, IntShVT)), in expandFP_TO_SINT()
8092 SDValue Ret = DAG.getNode(ISD::SUB, dl, DstVT, in expandFP_TO_SINT()
8093 DAG.getNode(ISD::XOR, dl, DstVT, R, Sign), Sign); in expandFP_TO_SINT()
8095 Result = DAG.getSelectCC(dl, Exponent, DAG.getConstant(0, dl, IntVT), in expandFP_TO_SINT()
8096 DAG.getConstant(0, dl, DstVT), Ret, ISD::SETLT); in expandFP_TO_SINT()
8103 SDLoc dl(SDValue(Node, 0)); in expandFP_TO_UINT() local
8130 Result = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other }, in expandFP_TO_UINT()
8134 Result = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src); in expandFP_TO_UINT()
8143 SDValue Cst = DAG.getConstantFP(APF, dl, SrcVT); in expandFP_TO_UINT()
8147 Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT, in expandFP_TO_UINT()
8151 Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT); in expandFP_TO_UINT()
8166 SDValue FltOfs = DAG.getSelect(dl, SrcVT, Sel, in expandFP_TO_UINT()
8167 DAG.getConstantFP(0.0, dl, SrcVT), Cst); in expandFP_TO_UINT()
8168 Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT); in expandFP_TO_UINT()
8169 SDValue IntOfs = DAG.getSelect(dl, DstVT, Sel, in expandFP_TO_UINT()
8170 DAG.getConstant(0, dl, DstVT), in expandFP_TO_UINT()
8171 DAG.getConstant(SignMask, dl, DstVT)); in expandFP_TO_UINT()
8174 SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl, { SrcVT, MVT::Other }, in expandFP_TO_UINT()
8176 SInt = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other }, in expandFP_TO_UINT()
8180 SDValue Val = DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FltOfs); in expandFP_TO_UINT()
8181 SInt = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Val); in expandFP_TO_UINT()
8183 Result = DAG.getNode(ISD::XOR, dl, DstVT, SInt, IntOfs); in expandFP_TO_UINT()
8190 SDValue True = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src); in expandFP_TO_UINT()
8192 SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, in expandFP_TO_UINT()
8193 DAG.getNode(ISD::FSUB, dl, SrcVT, Src, Cst)); in expandFP_TO_UINT()
8194 False = DAG.getNode(ISD::XOR, dl, DstVT, False, in expandFP_TO_UINT()
8195 DAG.getConstant(SignMask, dl, DstVT)); in expandFP_TO_UINT()
8196 Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT); in expandFP_TO_UINT()
8197 Result = DAG.getSelect(dl, DstVT, Sel, True, False); in expandFP_TO_UINT()
8226 SDLoc dl(SDValue(Node, 0)); in expandUINT_TO_FP() local
8234 SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000), dl, SrcVT); in expandUINT_TO_FP()
8236 llvm::bit_cast<double>(UINT64_C(0x4530000000100000)), dl, DstVT); in expandUINT_TO_FP()
8237 SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000), dl, SrcVT); in expandUINT_TO_FP()
8238 SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF), dl, SrcVT); in expandUINT_TO_FP()
8239 SDValue HiShift = DAG.getConstant(32, dl, ShiftVT); in expandUINT_TO_FP()
8241 SDValue Lo = DAG.getNode(ISD::AND, dl, SrcVT, Src, LoMask); in expandUINT_TO_FP()
8242 SDValue Hi = DAG.getNode(ISD::SRL, dl, SrcVT, Src, HiShift); in expandUINT_TO_FP()
8243 SDValue LoOr = DAG.getNode(ISD::OR, dl, SrcVT, Lo, TwoP52); in expandUINT_TO_FP()
8244 SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84); in expandUINT_TO_FP()
8248 DAG.getNode(ISD::FSUB, dl, DstVT, HiFlt, TwoP84PlusTwoP52); in expandUINT_TO_FP()
8249 Result = DAG.getNode(ISD::FADD, dl, DstVT, LoFlt, HiSub); in expandUINT_TO_FP()
8279 SDLoc dl(Node); in expandFMINNUM_FMAXNUM() local
8296 Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0, in expandFMINNUM_FMAXNUM()
8300 Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1, in expandFMINNUM_FMAXNUM()
8305 return DAG.getNode(NewOp, dl, VT, Quiet0, Quiet1, Node->getFlags()); in expandFMINNUM_FMAXNUM()
8320 return DAG.getNode(IEEE2018Op, dl, VT, Node->getOperand(0), in expandFMINNUM_FMAXNUM()
8627 SDLoc dl(Node); in expandCTPOP() local
8645 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT); in expandCTPOP()
8647 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT); in expandCTPOP()
8649 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT); in expandCTPOP()
8652 Op = DAG.getNode(ISD::SUB, dl, VT, Op, in expandCTPOP()
8653 DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8654 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8655 DAG.getConstant(1, dl, ShVT)), in expandCTPOP()
8658 Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, dl, VT, Op, Mask33), in expandCTPOP()
8659 DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8660 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8661 DAG.getConstant(2, dl, ShVT)), in expandCTPOP()
8664 Op = DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8665 DAG.getNode(ISD::ADD, dl, VT, Op, in expandCTPOP()
8666 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8667 DAG.getConstant(4, dl, ShVT))), in expandCTPOP()
8678 return DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8679 DAG.getNode(ISD::ADD, dl, VT, Op, in expandCTPOP()
8680 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8681 DAG.getConstant(8, dl, ShVT))), in expandCTPOP()
8682 DAG.getConstant(0xFF, dl, VT)); in expandCTPOP()
8687 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT); in expandCTPOP()
8688 return DAG.getNode(ISD::SRL, dl, VT, in expandCTPOP()
8689 DAG.getNode(ISD::MUL, dl, VT, Op, Mask01), in expandCTPOP()
8690 DAG.getConstant(Len - 8, dl, ShVT)); in expandCTPOP()
8694 SDLoc dl(Node); in expandVPCTPOP() local
8710 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT); in expandVPCTPOP()
8712 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT); in expandVPCTPOP()
8714 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT); in expandVPCTPOP()
8719 Tmp1 = DAG.getNode(ISD::VP_AND, dl, VT, in expandVPCTPOP()
8720 DAG.getNode(ISD::VP_LSHR, dl, VT, Op, in expandVPCTPOP()
8721 DAG.getConstant(1, dl, ShVT), Mask, VL), in expandVPCTPOP()
8723 Op = DAG.getNode(ISD::VP_SUB, dl, VT, Op, Tmp1, Mask, VL); in expandVPCTPOP()
8726 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Op, Mask33, Mask, VL); in expandVPCTPOP()
8727 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, in expandVPCTPOP()
8728 DAG.getNode(ISD::VP_LSHR, dl, VT, Op, in expandVPCTPOP()
8729 DAG.getConstant(2, dl, ShVT), Mask, VL), in expandVPCTPOP()
8731 Op = DAG.getNode(ISD::VP_ADD, dl, VT, Tmp2, Tmp3, Mask, VL); in expandVPCTPOP()
8734 Tmp4 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(4, dl, ShVT), in expandVPCTPOP()
8736 Tmp5 = DAG.getNode(ISD::VP_ADD, dl, VT, Op, Tmp4, Mask, VL); in expandVPCTPOP()
8737 Op = DAG.getNode(ISD::VP_AND, dl, VT, Tmp5, Mask0F, Mask, VL); in expandVPCTPOP()
8744 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT); in expandVPCTPOP()
8745 return DAG.getNode(ISD::VP_LSHR, dl, VT, in expandVPCTPOP()
8746 DAG.getNode(ISD::VP_MUL, dl, VT, Op, Mask01, Mask, VL), in expandVPCTPOP()
8747 DAG.getConstant(Len - 8, dl, ShVT), Mask, VL); in expandVPCTPOP()
8751 SDLoc dl(Node); in expandCTLZ() local
8760 return DAG.getNode(ISD::CTLZ, dl, VT, Op); in expandCTLZ()
8766 SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op); in expandCTLZ()
8767 SDValue Zero = DAG.getConstant(0, dl, VT); in expandCTLZ()
8768 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ); in expandCTLZ()
8769 return DAG.getSelect(dl, VT, SrcIsZero, in expandCTLZ()
8770 DAG.getConstant(NumBitsPerElt, dl, VT), CTLZ); in expandCTLZ()
8792 SDValue Tmp = DAG.getConstant(1ULL << i, dl, ShVT); in expandCTLZ()
8793 Op = DAG.getNode(ISD::OR, dl, VT, Op, in expandCTLZ()
8794 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp)); in expandCTLZ()
8796 Op = DAG.getNOT(dl, Op, VT); in expandCTLZ()
8797 return DAG.getNode(ISD::CTPOP, dl, VT, Op); in expandCTLZ()
8801 SDLoc dl(Node); in expandVPCTLZ() local
8817 SDValue Tmp = DAG.getConstant(1ULL << i, dl, ShVT); in expandVPCTLZ()
8818 Op = DAG.getNode(ISD::VP_OR, dl, VT, Op, in expandVPCTLZ()
8819 DAG.getNode(ISD::VP_LSHR, dl, VT, Op, Tmp, Mask, VL), Mask, in expandVPCTLZ()
8822 Op = DAG.getNode(ISD::VP_XOR, dl, VT, Op, DAG.getConstant(-1, dl, VT), Mask, in expandVPCTLZ()
8824 return DAG.getNode(ISD::VP_CTPOP, dl, VT, Op, Mask, VL); in expandVPCTLZ()
8872 SDLoc dl(Node); in expandCTTZ() local
8880 return DAG.getNode(ISD::CTTZ, dl, VT, Op); in expandCTTZ()
8886 SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op); in expandCTTZ()
8887 SDValue Zero = DAG.getConstant(0, dl, VT); in expandCTTZ()
8888 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ); in expandCTTZ()
8889 return DAG.getSelect(dl, VT, SrcIsZero, in expandCTTZ()
8890 DAG.getConstant(NumBitsPerElt, dl, VT), CTTZ); in expandCTTZ()
8907 if (SDValue V = CTTZTableLookup(Node, DAG, dl, VT, Op, NumBitsPerElt)) in expandCTTZ()
8915 ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT), in expandCTTZ()
8916 DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT))); in expandCTTZ()
8920 return DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT), in expandCTTZ()
8921 DAG.getNode(ISD::CTLZ, dl, VT, Tmp)); in expandCTTZ()
8924 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp); in expandCTTZ()
8931 SDLoc dl(Node); in expandVPCTTZ() local
8935 SDValue Not = DAG.getNode(ISD::VP_XOR, dl, VT, Op, in expandVPCTTZ()
8936 DAG.getConstant(-1, dl, VT), Mask, VL); in expandVPCTTZ()
8937 SDValue MinusOne = DAG.getNode(ISD::VP_SUB, dl, VT, Op, in expandVPCTTZ()
8938 DAG.getConstant(1, dl, VT), Mask, VL); in expandVPCTTZ()
8939 SDValue Tmp = DAG.getNode(ISD::VP_AND, dl, VT, Not, MinusOne, Mask, VL); in expandVPCTTZ()
8940 return DAG.getNode(ISD::VP_CTPOP, dl, VT, Tmp, Mask, VL); in expandVPCTTZ()
8945 SDLoc dl(N); in expandABS() local
8953 SDValue Zero = DAG.getConstant(0, dl, VT); in expandABS()
8954 return DAG.getNode(ISD::SMAX, dl, VT, Op, in expandABS()
8955 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
8961 SDValue Zero = DAG.getConstant(0, dl, VT); in expandABS()
8963 return DAG.getNode(ISD::UMIN, dl, VT, Op, in expandABS()
8964 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
8971 SDValue Zero = DAG.getConstant(0, dl, VT); in expandABS()
8972 return DAG.getNode(ISD::SMIN, dl, VT, Op, in expandABS()
8973 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
8986 DAG.getNode(ISD::SRA, dl, VT, Op, in expandABS()
8987 DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, ShVT)); in expandABS()
8988 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, Shift); in expandABS()
8992 return DAG.getNode(ISD::SUB, dl, VT, Xor, Shift); in expandABS()
8995 return DAG.getNode(ISD::SUB, dl, VT, Shift, Xor); in expandABS()
8999 SDLoc dl(N); in expandABD() local
9010 SDValue Max = DAG.getNode(MaxOpc, dl, VT, LHS, RHS); in expandABD()
9011 SDValue Min = DAG.getNode(MinOpc, dl, VT, LHS, RHS); in expandABD()
9012 return DAG.getNode(ISD::SUB, dl, VT, Max, Min); in expandABD()
9017 return DAG.getNode(ISD::OR, dl, VT, in expandABD()
9018 DAG.getNode(ISD::USUBSAT, dl, VT, LHS, RHS), in expandABD()
9019 DAG.getNode(ISD::USUBSAT, dl, VT, RHS, LHS)); in expandABD()
9025 SDValue Cmp = DAG.getSetCC(dl, CCVT, LHS, RHS, CC); in expandABD()
9026 return DAG.getSelect(dl, VT, Cmp, DAG.getNode(ISD::SUB, dl, VT, LHS, RHS), in expandABD()
9027 DAG.getNode(ISD::SUB, dl, VT, RHS, LHS)); in expandABD()
9031 SDLoc dl(N); in expandBSWAP() local
9045 return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9047 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9048 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9049 DAG.getConstant(0xFF00, dl, VT)); in expandBSWAP()
9050 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9051 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9052 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT)); in expandBSWAP()
9053 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9054 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); in expandBSWAP()
9055 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); in expandBSWAP()
9056 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); in expandBSWAP()
9058 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); in expandBSWAP()
9059 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9060 DAG.getConstant(255ULL<<8, dl, VT)); in expandBSWAP()
9061 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Tmp7, DAG.getConstant(40, dl, SHVT)); in expandBSWAP()
9062 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9063 DAG.getConstant(255ULL<<16, dl, VT)); in expandBSWAP()
9064 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Tmp6, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9065 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9066 DAG.getConstant(255ULL<<24, dl, VT)); in expandBSWAP()
9067 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Tmp5, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9068 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9069 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, in expandBSWAP()
9070 DAG.getConstant(255ULL<<24, dl, VT)); in expandBSWAP()
9071 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9072 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, in expandBSWAP()
9073 DAG.getConstant(255ULL<<16, dl, VT)); in expandBSWAP()
9074 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); in expandBSWAP()
9075 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, in expandBSWAP()
9076 DAG.getConstant(255ULL<<8, dl, VT)); in expandBSWAP()
9077 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); in expandBSWAP()
9078 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); in expandBSWAP()
9079 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); in expandBSWAP()
9080 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); in expandBSWAP()
9081 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); in expandBSWAP()
9082 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); in expandBSWAP()
9083 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); in expandBSWAP()
9084 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); in expandBSWAP()
9089 SDLoc dl(N); in expandVPBSWAP() local
9104 Tmp1 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9106 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9108 return DAG.getNode(ISD::VP_OR, dl, VT, Tmp1, Tmp2, Mask, EVL); in expandVPBSWAP()
9110 Tmp4 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9112 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Op, DAG.getConstant(0xFF00, dl, VT), in expandVPBSWAP()
9114 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9116 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9118 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBSWAP()
9119 DAG.getConstant(0xFF00, dl, VT), Mask, EVL); in expandVPBSWAP()
9120 Tmp1 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9122 Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp3, Mask, EVL); in expandVPBSWAP()
9123 Tmp2 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp1, Mask, EVL); in expandVPBSWAP()
9124 return DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp2, Mask, EVL); in expandVPBSWAP()
9126 Tmp8 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT), in expandVPBSWAP()
9128 Tmp7 = DAG.getNode(ISD::VP_AND, dl, VT, Op, in expandVPBSWAP()
9129 DAG.getConstant(255ULL << 8, dl, VT), Mask, EVL); in expandVPBSWAP()
9130 Tmp7 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp7, DAG.getConstant(40, dl, SHVT), in expandVPBSWAP()
9132 Tmp6 = DAG.getNode(ISD::VP_AND, dl, VT, Op, in expandVPBSWAP()
9133 DAG.getConstant(255ULL << 16, dl, VT), Mask, EVL); in expandVPBSWAP()
9134 Tmp6 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp6, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9136 Tmp5 = DAG.getNode(ISD::VP_AND, dl, VT, Op, in expandVPBSWAP()
9137 DAG.getConstant(255ULL << 24, dl, VT), Mask, EVL); in expandVPBSWAP()
9138 Tmp5 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp5, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9140 Tmp4 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9142 Tmp4 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp4, in expandVPBSWAP()
9143 DAG.getConstant(255ULL << 24, dl, VT), Mask, EVL); in expandVPBSWAP()
9144 Tmp3 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9146 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp3, in expandVPBSWAP()
9147 DAG.getConstant(255ULL << 16, dl, VT), Mask, EVL); in expandVPBSWAP()
9148 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(40, dl, SHVT), in expandVPBSWAP()
9150 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBSWAP()
9151 DAG.getConstant(255ULL << 8, dl, VT), Mask, EVL); in expandVPBSWAP()
9152 Tmp1 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(56, dl, SHVT), in expandVPBSWAP()
9154 Tmp8 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp7, Mask, EVL); in expandVPBSWAP()
9155 Tmp6 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp6, Tmp5, Mask, EVL); in expandVPBSWAP()
9156 Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp3, Mask, EVL); in expandVPBSWAP()
9157 Tmp2 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp1, Mask, EVL); in expandVPBSWAP()
9158 Tmp8 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp6, Mask, EVL); in expandVPBSWAP()
9159 Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp2, Mask, EVL); in expandVPBSWAP()
9160 return DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp4, Mask, EVL); in expandVPBSWAP()
9165 SDLoc dl(N); in expandBITREVERSE() local
9183 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op); in expandBITREVERSE()
9186 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT)); in expandBITREVERSE()
9187 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask4, dl, VT)); in expandBITREVERSE()
9188 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT)); in expandBITREVERSE()
9189 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT)); in expandBITREVERSE()
9190 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
9193 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT)); in expandBITREVERSE()
9194 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask2, dl, VT)); in expandBITREVERSE()
9195 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT)); in expandBITREVERSE()
9196 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT)); in expandBITREVERSE()
9197 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
9200 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT)); in expandBITREVERSE()
9201 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask1, dl, VT)); in expandBITREVERSE()
9202 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT)); in expandBITREVERSE()
9203 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT)); in expandBITREVERSE()
9204 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
9208 Tmp = DAG.getConstant(0, dl, VT); in expandBITREVERSE()
9212 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT)); in expandBITREVERSE()
9215 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT)); in expandBITREVERSE()
9218 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT)); in expandBITREVERSE()
9219 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2); in expandBITREVERSE()
9228 SDLoc dl(N); in expandVPBITREVERSE() local
9248 Tmp = (Sz > 8 ? DAG.getNode(ISD::VP_BSWAP, dl, VT, Op, Mask, EVL) : Op); in expandVPBITREVERSE()
9251 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT), in expandVPBITREVERSE()
9253 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBITREVERSE()
9254 DAG.getConstant(Mask4, dl, VT), Mask, EVL); in expandVPBITREVERSE()
9255 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT), in expandVPBITREVERSE()
9257 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT), in expandVPBITREVERSE()
9259 Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL); in expandVPBITREVERSE()
9262 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT), in expandVPBITREVERSE()
9264 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBITREVERSE()
9265 DAG.getConstant(Mask2, dl, VT), Mask, EVL); in expandVPBITREVERSE()
9266 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT), in expandVPBITREVERSE()
9268 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT), in expandVPBITREVERSE()
9270 Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL); in expandVPBITREVERSE()
9273 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT), in expandVPBITREVERSE()
9275 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBITREVERSE()
9276 DAG.getConstant(Mask1, dl, VT), Mask, EVL); in expandVPBITREVERSE()
9277 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT), in expandVPBITREVERSE()
9279 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT), in expandVPBITREVERSE()
9281 Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL); in expandVPBITREVERSE()
9463 SDLoc dl(LD); in expandUnalignedLoad() local
9477 SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr, in expandUnalignedLoad()
9479 SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad); in expandUnalignedLoad()
9482 ISD::ANY_EXTEND, dl, VT, Result); in expandUnalignedLoad()
9504 SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT); in expandUnalignedLoad()
9505 SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT); in expandUnalignedLoad()
9511 RegVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(Offset), in expandUnalignedLoad()
9516 Load.getValue(1), dl, Load, StackPtr, in expandUnalignedLoad()
9521 Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement); in expandUnalignedLoad()
9522 StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement); in expandUnalignedLoad()
9529 DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr, in expandUnalignedLoad()
9537 Load.getValue(1), dl, Load, StackPtr, in expandUnalignedLoad()
9541 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); in expandUnalignedLoad()
9544 Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase, in expandUnalignedLoad()
9573 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(), in expandUnalignedLoad()
9577 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize)); in expandUnalignedLoad()
9578 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, in expandUnalignedLoad()
9583 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(), in expandUnalignedLoad()
9587 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize)); in expandUnalignedLoad()
9588 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, in expandUnalignedLoad()
9596 DAG.getConstant(NumBits, dl, getShiftAmountTy(Hi.getValueType(), in expandUnalignedLoad()
9598 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount); in expandUnalignedLoad()
9599 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo); in expandUnalignedLoad()
9601 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), in expandUnalignedLoad()
9619 SDLoc dl(ST); in expandUnalignedStore() local
9632 SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val); in expandUnalignedStore()
9633 Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(), in expandUnalignedStore()
9653 Chain, dl, Val, StackPtr, in expandUnalignedStore()
9658 SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT); in expandUnalignedStore()
9659 SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT); in expandUnalignedStore()
9667 RegVT, dl, Store, StackPtr, in expandUnalignedStore()
9670 Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr, in expandUnalignedStore()
9676 StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement); in expandUnalignedStore()
9677 Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement); in expandUnalignedStore()
9688 ISD::EXTLOAD, dl, RegVT, Store, StackPtr, in expandUnalignedStore()
9692 DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr, in expandUnalignedStore()
9697 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); in expandUnalignedStore()
9710 NumBits, dl, getShiftAmountTy(Val.getValueType(), DAG.getDataLayout())); in expandUnalignedStore()
9717 ISD::AND, dl, VT, Lo, in expandUnalignedStore()
9718 DAG.getConstant(APInt::getLowBitsSet(VT.getSizeInBits(), NumBits), dl, in expandUnalignedStore()
9720 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount); in expandUnalignedStore()
9724 Store1 = DAG.getTruncStore(Chain, dl, in expandUnalignedStore()
9729 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize)); in expandUnalignedStore()
9731 Chain, dl, DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr, in expandUnalignedStore()
9736 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); in expandUnalignedStore()
9780 EVT VecVT, const SDLoc &dl, in clampDynamicVectorIndex() argument
9797 DAG.getVScale(dl, IdxVT, APInt(IdxVT.getFixedSizeInBits(), NElts)); in clampDynamicVectorIndex()
9799 SDValue Sub = DAG.getNode(SubOpcode, dl, IdxVT, VS, in clampDynamicVectorIndex()
9800 DAG.getConstant(NumSubElts, dl, IdxVT)); in clampDynamicVectorIndex()
9801 return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx, Sub); in clampDynamicVectorIndex()
9805 return DAG.getNode(ISD::AND, dl, IdxVT, Idx, in clampDynamicVectorIndex()
9806 DAG.getConstant(Imm, dl, IdxVT)); in clampDynamicVectorIndex()
9809 return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx, in clampDynamicVectorIndex()
9810 DAG.getConstant(MaxIndex, dl, IdxVT)); in clampDynamicVectorIndex()
9826 SDLoc dl(Index); in getVectorSubVecPointer() local
9828 Index = DAG.getZExtOrTrunc(Index, dl, VecPtr.getValueType()); in getVectorSubVecPointer()
9838 Index = clampDynamicVectorIndex(DAG, Index, VecVT, dl, in getVectorSubVecPointer()
9844 DAG.getNode(ISD::MUL, dl, IdxVT, Index, in getVectorSubVecPointer()
9845 DAG.getVScale(dl, IdxVT, APInt(IdxVT.getSizeInBits(), 1))); in getVectorSubVecPointer()
9847 Index = DAG.getNode(ISD::MUL, dl, IdxVT, Index, in getVectorSubVecPointer()
9848 DAG.getConstant(EltSize, dl, IdxVT)); in getVectorSubVecPointer()
9849 return DAG.getMemBasePlusOffset(VecPtr, Index, dl); in getVectorSubVecPointer()
9862 SDLoc dl(GA); in LowerToTLSEmulatedModel() local
9871 Entry.Node = DAG.getGlobalAddress(EmuTlsVar, dl, PtrVT); in LowerToTLSEmulatedModel()
9878 CLI.setDebugLoc(dl).setChain(DAG.getEntryNode()); in LowerToTLSEmulatedModel()
9899 SDLoc dl(Op); in lowerCmpEqZeroToCtlzSrl() local
9905 Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0)); in lowerCmpEqZeroToCtlzSrl()
9908 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext); in lowerCmpEqZeroToCtlzSrl()
9909 SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz, in lowerCmpEqZeroToCtlzSrl()
9910 DAG.getConstant(Log2b, dl, MVT::i32)); in lowerCmpEqZeroToCtlzSrl()
9911 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc); in lowerCmpEqZeroToCtlzSrl()
10002 SDLoc dl(Node); in expandAddSubSat() local
10009 SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS); in expandAddSubSat()
10010 return DAG.getNode(ISD::SUB, dl, VT, Max, RHS); in expandAddSubSat()
10015 SDValue InvRHS = DAG.getNOT(dl, RHS, VT); in expandAddSubSat()
10016 SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS); in expandAddSubSat()
10017 return DAG.getNode(ISD::ADD, dl, VT, Min, RHS); in expandAddSubSat()
10046 SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); in expandAddSubSat()
10049 SDValue Zero = DAG.getConstant(0, dl, VT); in expandAddSubSat()
10050 SDValue AllOnes = DAG.getAllOnesConstant(dl, VT); in expandAddSubSat()
10055 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT); in expandAddSubSat()
10056 return DAG.getNode(ISD::OR, dl, VT, SumDiff, OverflowMask); in expandAddSubSat()
10059 return DAG.getSelect(dl, VT, Overflow, AllOnes, SumDiff); in expandAddSubSat()
10065 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT); in expandAddSubSat()
10066 SDValue Not = DAG.getNOT(dl, OverflowMask, VT); in expandAddSubSat()
10067 return DAG.getNode(ISD::AND, dl, VT, SumDiff, Not); in expandAddSubSat()
10070 return DAG.getSelect(dl, VT, Overflow, Zero, SumDiff); in expandAddSubSat()
10091 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); in expandAddSubSat()
10092 return DAG.getSelect(dl, VT, Overflow, SatMax, SumDiff); in expandAddSubSat()
10099 SDValue SatMin = DAG.getConstant(MinVal, dl, VT); in expandAddSubSat()
10100 return DAG.getSelect(dl, VT, Overflow, SatMin, SumDiff); in expandAddSubSat()
10106 SDValue SatMin = DAG.getConstant(MinVal, dl, VT); in expandAddSubSat()
10107 SDValue Shift = DAG.getNode(ISD::SRA, dl, VT, SumDiff, in expandAddSubSat()
10108 DAG.getConstant(BitWidth - 1, dl, VT)); in expandAddSubSat()
10109 Result = DAG.getNode(ISD::XOR, dl, VT, Shift, SatMin); in expandAddSubSat()
10110 return DAG.getSelect(dl, VT, Overflow, Result, SumDiff); in expandAddSubSat()
10119 SDLoc dl(Node); in expandShlSat() local
10134 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, LHS, RHS); in expandShlSat()
10136 DAG.getNode(IsSigned ? ISD::SRA : ISD::SRL, dl, VT, Result, RHS); in expandShlSat()
10140 SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(BW), dl, VT); in expandShlSat()
10141 SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(BW), dl, VT); in expandShlSat()
10143 DAG.getSetCC(dl, BoolVT, LHS, DAG.getConstant(0, dl, VT), ISD::SETLT); in expandShlSat()
10144 SatVal = DAG.getSelect(dl, VT, Cond, SatMin, SatMax); in expandShlSat()
10146 SatVal = DAG.getConstant(APInt::getMaxValue(BW), dl, VT); in expandShlSat()
10148 SDValue Cond = DAG.getSetCC(dl, BoolVT, LHS, Orig, ISD::SETNE); in expandShlSat()
10149 return DAG.getSelect(dl, VT, Cond, SatVal, Result); in expandShlSat()
10160 SDLoc dl(Node); in expandFixedPointMul() local
10176 return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandFixedPointMul()
10179 DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); in expandFixedPointMul()
10182 SDValue Zero = DAG.getConstant(0, dl, VT); in expandFixedPointMul()
10186 SDValue SatMin = DAG.getConstant(MinVal, dl, VT); in expandFixedPointMul()
10187 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); in expandFixedPointMul()
10190 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); in expandFixedPointMul()
10191 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT); in expandFixedPointMul()
10192 Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax); in expandFixedPointMul()
10193 return DAG.getSelect(dl, VT, Overflow, Result, Product); in expandFixedPointMul()
10196 DAG.getNode(ISD::UMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); in expandFixedPointMul()
10201 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); in expandFixedPointMul()
10202 return DAG.getSelect(dl, VT, Overflow, SatMax, Product); in expandFixedPointMul()
10217 SDValue Result = DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS); in expandFixedPointMul()
10221 Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandFixedPointMul()
10222 Hi = DAG.getNode(HiOp, dl, VT, LHS, RHS); in expandFixedPointMul()
10239 SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo, in expandFixedPointMul()
10240 DAG.getConstant(Scale, dl, ShiftTy)); in expandFixedPointMul()
10252 dl, VT); in expandFixedPointMul()
10253 Result = DAG.getSelectCC(dl, Hi, LowMask, in expandFixedPointMul()
10254 DAG.getConstant(MaxVal, dl, VT), Result, in expandFixedPointMul()
10263 SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(VTSize), dl, VT); in expandFixedPointMul()
10264 SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(VTSize), dl, VT); in expandFixedPointMul()
10267 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, Lo, in expandFixedPointMul()
10268 DAG.getConstant(VTSize - 1, dl, ShiftTy)); in expandFixedPointMul()
10269 SDValue Overflow = DAG.getSetCC(dl, BoolVT, Hi, Sign, ISD::SETNE); in expandFixedPointMul()
10272 SDValue Zero = DAG.getConstant(0, dl, VT); in expandFixedPointMul()
10273 SDValue ResultIfOverflow = DAG.getSelectCC(dl, Hi, Zero, SatMin, SatMax, in expandFixedPointMul()
10276 return DAG.getSelect(dl, VT, Overflow, ResultIfOverflow, Result); in expandFixedPointMul()
10284 dl, VT); in expandFixedPointMul()
10285 Result = DAG.getSelectCC(dl, Hi, LowMask, SatMax, Result, ISD::SETGT); in expandFixedPointMul()
10290 dl, VT); in expandFixedPointMul()
10291 Result = DAG.getSelectCC(dl, Hi, HighMask, SatMin, Result, ISD::SETLT); in expandFixedPointMul()
10296 TargetLowering::expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, in expandFixedPointDiv() argument
10337 LHS = DAG.getNode(ISD::SHL, dl, VT, LHS, in expandFixedPointDiv()
10338 DAG.getConstant(LHSShift, dl, ShiftTy)); in expandFixedPointDiv()
10340 RHS = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, VT, RHS, in expandFixedPointDiv()
10341 DAG.getConstant(RHSShift, dl, ShiftTy)); in expandFixedPointDiv()
10354 Quot = DAG.getNode(ISD::SDIVREM, dl, in expandFixedPointDiv()
10360 Quot = DAG.getNode(ISD::SDIV, dl, VT, in expandFixedPointDiv()
10362 Rem = DAG.getNode(ISD::SREM, dl, VT, in expandFixedPointDiv()
10365 SDValue Zero = DAG.getConstant(0, dl, VT); in expandFixedPointDiv()
10366 SDValue RemNonZero = DAG.getSetCC(dl, BoolVT, Rem, Zero, ISD::SETNE); in expandFixedPointDiv()
10367 SDValue LHSNeg = DAG.getSetCC(dl, BoolVT, LHS, Zero, ISD::SETLT); in expandFixedPointDiv()
10368 SDValue RHSNeg = DAG.getSetCC(dl, BoolVT, RHS, Zero, ISD::SETLT); in expandFixedPointDiv()
10369 SDValue QuotNeg = DAG.getNode(ISD::XOR, dl, BoolVT, LHSNeg, RHSNeg); in expandFixedPointDiv()
10370 SDValue Sub1 = DAG.getNode(ISD::SUB, dl, VT, Quot, in expandFixedPointDiv()
10371 DAG.getConstant(1, dl, VT)); in expandFixedPointDiv()
10372 Quot = DAG.getSelect(dl, VT, in expandFixedPointDiv()
10373 DAG.getNode(ISD::AND, dl, BoolVT, RemNonZero, QuotNeg), in expandFixedPointDiv()
10376 Quot = DAG.getNode(ISD::UDIV, dl, VT, in expandFixedPointDiv()
10384 SDLoc dl(Node); in expandUADDSUBO() local
10392 SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1)); in expandUADDSUBO()
10393 SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(), in expandUADDSUBO()
10400 Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl, in expandUADDSUBO()
10414 DAG.getSetCC(dl, SetCCType, Result, in expandUADDSUBO()
10415 DAG.getConstant(0, dl, Node->getValueType(0)), ISD::SETEQ); in expandUADDSUBO()
10419 DAG.getSetCC(dl, SetCCType, LHS, in expandUADDSUBO()
10420 DAG.getConstant(0, dl, Node->getValueType(0)), ISD::SETNE); in expandUADDSUBO()
10423 SetCC = DAG.getSetCC(dl, SetCCType, Result, LHS, CC); in expandUADDSUBO()
10425 Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType); in expandUADDSUBO()
10430 SDLoc dl(Node); in expandSADDSUBO() local
10435 Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl, in expandSADDSUBO()
10445 SDValue Sat = DAG.getNode(OpcSat, dl, LHS.getValueType(), LHS, RHS); in expandSADDSUBO()
10446 SDValue SetCC = DAG.getSetCC(dl, OType, Result, Sat, ISD::SETNE); in expandSADDSUBO()
10447 Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType); in expandSADDSUBO()
10451 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType()); in expandSADDSUBO()
10459 SDValue ResultLowerThanLHS = DAG.getSetCC(dl, OType, Result, LHS, ISD::SETLT); in expandSADDSUBO()
10461 DAG.getSetCC(dl, OType, RHS, Zero, IsAdd ? ISD::SETLT : ISD::SETGT); in expandSADDSUBO()
10464 DAG.getNode(ISD::XOR, dl, OType, ConditionRHS, ResultLowerThanLHS), dl, in expandSADDSUBO()
10470 SDLoc dl(Node); in expandMULO() local
10485 SDValue ShiftAmt = DAG.getConstant(C.logBase2(), dl, ShiftAmtTy); in expandMULO()
10486 Result = DAG.getNode(ISD::SHL, dl, VT, LHS, ShiftAmt); in expandMULO()
10487 Overflow = DAG.getSetCC(dl, SetCCVT, in expandMULO()
10489 dl, VT, Result, ShiftAmt), in expandMULO()
10506 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandMULO()
10507 TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS); in expandMULO()
10509 BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS, in expandMULO()
10513 LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS); in expandMULO()
10514 RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS); in expandMULO()
10515 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); in expandMULO()
10516 BottomHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul); in expandMULO()
10517 SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits(), dl, in expandMULO()
10519 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, in expandMULO()
10520 DAG.getNode(ISD::SRL, dl, WideVT, Mul, ShiftAmt)); in expandMULO()
10547 DAG.getNode(ISD::SRA, dl, VT, LHS, in expandMULO()
10548 DAG.getConstant(LoSize - 1, dl, in expandMULO()
10551 DAG.getNode(ISD::SRA, dl, VT, RHS, in expandMULO()
10552 DAG.getConstant(LoSize - 1, dl, in expandMULO()
10555 HiLHS = DAG.getConstant(0, dl, VT); in expandMULO()
10556 HiRHS = DAG.getConstant(0, dl, VT); in expandMULO()
10573 Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first; in expandMULO()
10576 Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first; in expandMULO()
10593 VT.getScalarSizeInBits() - 1, dl, in expandMULO()
10595 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt); in expandMULO()
10596 Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, Sign, ISD::SETNE); in expandMULO()
10598 Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, in expandMULO()
10599 DAG.getConstant(0, dl, VT), ISD::SETNE); in expandMULO()
10605 Overflow = DAG.getNode(ISD::TRUNCATE, dl, RType, Overflow); in expandMULO()
10613 SDLoc dl(Node); in expandVecReduce() local
10630 std::tie(Lo, Hi) = DAG.SplitVector(Op, dl); in expandVecReduce()
10631 Op = DAG.getNode(BaseOpcode, dl, HalfVT, Lo, Hi); in expandVecReduce()
10644 Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Node->getFlags()); in expandVecReduce()
10648 Res = DAG.getNode(ISD::ANY_EXTEND, dl, Node->getValueType(0), Res); in expandVecReduce()
10653 SDLoc dl(Node); in expandVecReduceSeq() local
10674 Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Flags); in expandVecReduceSeq()
10682 SDLoc dl(Node); in expandREM() local
10690 Result = DAG.getNode(DivRemOpc, dl, VTs, Dividend, Divisor).getValue(1); in expandREM()
10695 SDValue Divide = DAG.getNode(DivOpc, dl, VT, Dividend, Divisor); in expandREM()
10696 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor); in expandREM()
10697 Result = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul); in expandREM()
10706 SDLoc dl(SDValue(Node, 0)); in expandFP_TO_INT_SAT() local
10733 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Src); in expandFP_TO_INT_SAT()
10747 SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT); in expandFP_TO_INT_SAT()
10748 SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT); in expandFP_TO_INT_SAT()
10759 Clamped = DAG.getNode(ISD::FMAXNUM, dl, SrcVT, Clamped, MinFloatNode); in expandFP_TO_INT_SAT()
10761 Clamped = DAG.getNode(ISD::FMINNUM, dl, SrcVT, Clamped, MaxFloatNode); in expandFP_TO_INT_SAT()
10764 dl, DstVT, Clamped); in expandFP_TO_INT_SAT()
10772 SDValue ZeroInt = DAG.getConstant(0, dl, DstVT); in expandFP_TO_INT_SAT()
10775 SDValue IsNan = DAG.getSetCC(dl, SetCCVT, Src, Src, ISD::CondCode::SETUO); in expandFP_TO_INT_SAT()
10776 return DAG.getSelect(dl, DstVT, IsNan, ZeroInt, FpToInt); in expandFP_TO_INT_SAT()
10779 SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT); in expandFP_TO_INT_SAT()
10780 SDValue MaxIntNode = DAG.getConstant(MaxInt, dl, DstVT); in expandFP_TO_INT_SAT()
10786 DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, DstVT, Src); in expandFP_TO_INT_SAT()
10795 SDValue ULT = DAG.getSetCC(dl, SetCCVT, Src, MinFloatNode, ISD::SETULT); in expandFP_TO_INT_SAT()
10796 Select = DAG.getSelect(dl, DstVT, ULT, MinIntNode, Select); in expandFP_TO_INT_SAT()
10798 SDValue OGT = DAG.getSetCC(dl, SetCCVT, Src, MaxFloatNode, ISD::SETOGT); in expandFP_TO_INT_SAT()
10799 Select = DAG.getSelect(dl, DstVT, OGT, MaxIntNode, Select); in expandFP_TO_INT_SAT()
10807 SDValue ZeroInt = DAG.getConstant(0, dl, DstVT); in expandFP_TO_INT_SAT()
10808 SDValue IsNan = DAG.getSetCC(dl, SetCCVT, Src, Src, ISD::CondCode::SETUO); in expandFP_TO_INT_SAT()
10809 return DAG.getSelect(dl, DstVT, IsNan, ZeroInt, Select); in expandFP_TO_INT_SAT()
10890 const SDLoc &dl, SDValue &Chain, in LegalizeSetCCCondCode() argument
11005 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1, Chain, IsSignaling); in LegalizeSetCCCondCode()
11006 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2, Chain, IsSignaling); in LegalizeSetCCCondCode()
11008 SetCC1 = DAG.getSetCCVP(dl, VT, LHS, RHS, CC1, Mask, EVL); in LegalizeSetCCCondCode()
11009 SetCC2 = DAG.getSetCCVP(dl, VT, LHS, RHS, CC2, Mask, EVL); in LegalizeSetCCCondCode()
11014 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1, Chain, IsSignaling); in LegalizeSetCCCondCode()
11015 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2, Chain, IsSignaling); in LegalizeSetCCCondCode()
11017 SetCC1 = DAG.getSetCCVP(dl, VT, LHS, LHS, CC1, Mask, EVL); in LegalizeSetCCCondCode()
11018 SetCC2 = DAG.getSetCCVP(dl, VT, RHS, RHS, CC2, Mask, EVL); in LegalizeSetCCCondCode()
11022 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, SetCC1.getValue(1), in LegalizeSetCCCondCode()
11025 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2); in LegalizeSetCCCondCode()
11030 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2, Mask, EVL); in LegalizeSetCCCondCode()