Lines Matching refs:ISD
98 if (Value->getOpcode() == ISD::AssertZext) in parametersInCSRMatch()
100 if (Value->getOpcode() != ISD::CopyFromReg) in parametersInCSRMatch()
242 if (isOperationLegalOrCustom(ISD::STORE, NewVT) && in findOptimalMemOpLowering()
246 isOperationLegalOrCustom(ISD::STORE, MVT::f64) && in findOptimalMemOpLowering()
292 ISD::CondCode &CCCode, in softenSetCCOperands()
302 ISD::CondCode &CCCode, in softenSetCCOperands()
318 case ISD::SETEQ: in softenSetCCOperands()
319 case ISD::SETOEQ: in softenSetCCOperands()
324 case ISD::SETNE: in softenSetCCOperands()
325 case ISD::SETUNE: in softenSetCCOperands()
330 case ISD::SETGE: in softenSetCCOperands()
331 case ISD::SETOGE: in softenSetCCOperands()
336 case ISD::SETLT: in softenSetCCOperands()
337 case ISD::SETOLT: in softenSetCCOperands()
342 case ISD::SETLE: in softenSetCCOperands()
343 case ISD::SETOLE: in softenSetCCOperands()
348 case ISD::SETGT: in softenSetCCOperands()
349 case ISD::SETOGT: in softenSetCCOperands()
354 case ISD::SETO: in softenSetCCOperands()
357 case ISD::SETUO: in softenSetCCOperands()
362 case ISD::SETONE: in softenSetCCOperands()
366 case ISD::SETUEQ: in softenSetCCOperands()
378 case ISD::SETULT: in softenSetCCOperands()
383 case ISD::SETULE: in softenSetCCOperands()
388 case ISD::SETUGT: in softenSetCCOperands()
393 case ISD::SETUGE: in softenSetCCOperands()
432 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second, in softenSetCCOperands()
434 NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl, in softenSetCCOperands()
484 return DAG.getNode(ISD::BRIND, dl, MVT::Other, Chain, Addr); in expandIndirectJTBranch()
532 case ISD::XOR: in ShrinkDemandedConstant()
533 case ISD::AND: in ShrinkDemandedConstant()
534 case ISD::OR: { in ShrinkDemandedConstant()
541 if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C)) in ShrinkDemandedConstant()
604 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)), in ShrinkDemandedOp()
605 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1))); in ShrinkDemandedOp()
607 SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, VT, X); in ShrinkDemandedOp()
686 case ISD::BITCAST: { in SimplifyMultipleUseDemandedBits()
745 case ISD::AND: { in SimplifyMultipleUseDemandedBits()
758 case ISD::OR: { in SimplifyMultipleUseDemandedBits()
771 case ISD::XOR: { in SimplifyMultipleUseDemandedBits()
783 case ISD::SHL: { in SimplifyMultipleUseDemandedBits()
798 case ISD::SETCC: { in SimplifyMultipleUseDemandedBits()
801 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); in SimplifyMultipleUseDemandedBits()
813 if (CC == ISD::SETLT && Op1.getValueType().isInteger() && in SimplifyMultipleUseDemandedBits()
814 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode()))) in SimplifyMultipleUseDemandedBits()
819 case ISD::SIGN_EXTEND_INREG: { in SimplifyMultipleUseDemandedBits()
833 case ISD::ANY_EXTEND_VECTOR_INREG: in SimplifyMultipleUseDemandedBits()
834 case ISD::SIGN_EXTEND_VECTOR_INREG: in SimplifyMultipleUseDemandedBits()
835 case ISD::ZERO_EXTEND_VECTOR_INREG: { in SimplifyMultipleUseDemandedBits()
851 case ISD::INSERT_VECTOR_ELT: { in SimplifyMultipleUseDemandedBits()
864 case ISD::INSERT_SUBVECTOR: { in SimplifyMultipleUseDemandedBits()
878 case ISD::VECTOR_SHUFFLE: { in SimplifyMultipleUseDemandedBits()
908 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) in SimplifyMultipleUseDemandedBits()
946 assert((Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SRA) && in combineShiftToAVG()
960 if (Add.getOpcode() != ISD::ADD) in combineShiftToAVG()
985 (ExtOpA.getOpcode() == ISD::ADD && in combineShiftToAVG()
987 (ExtOpB.getOpcode() == ISD::ADD && in combineShiftToAVG()
1011 case ISD::SRA: { in combineShiftToAVG()
1024 case ISD::SRL: { in combineShiftToAVG()
1039 unsigned AVGOpc = IsCeil ? (IsSigned ? ISD::AVGCEILS : ISD::AVGCEILU) in combineShiftToAVG()
1040 : (IsSigned ? ISD::AVGFLOORS : ISD::AVGFLOORU); in combineShiftToAVG()
1106 if (Op.getOpcode() == ISD::TargetConstant) in SimplifyDemandedBits()
1109 if (Op.getOpcode() == ISD::Constant) { in SimplifyDemandedBits()
1115 if (Op.getOpcode() == ISD::ConstantFP) { in SimplifyDemandedBits()
1143 case ISD::SCALAR_TO_VECTOR: { in SimplifyDemandedBits()
1162 case ISD::BUILD_VECTOR: in SimplifyDemandedBits()
1167 case ISD::SPLAT_VECTOR: { in SimplifyDemandedBits()
1179 case ISD::LOAD: { in SimplifyDemandedBits()
1185 if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { in SimplifyDemandedBits()
1194 case ISD::INSERT_VECTOR_ELT: { in SimplifyDemandedBits()
1232 case ISD::INSERT_SUBVECTOR: { in SimplifyDemandedBits()
1277 case ISD::EXTRACT_SUBVECTOR: { in SimplifyDemandedBits()
1304 case ISD::CONCAT_VECTORS: { in SimplifyDemandedBits()
1324 case ISD::VECTOR_SHUFFLE: { in SimplifyDemandedBits()
1367 case ISD::AND: { in SimplifyDemandedBits()
1396 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1); in SimplifyDemandedBits()
1403 if (Op0.getOpcode() == ISD::INSERT_SUBVECTOR && !VT.isScalableVector() && in SimplifyDemandedBits()
1405 ISD::isBuildVectorOfConstantSDNodes(Op0.getOperand(0).getNode())) && in SimplifyDemandedBits()
1416 TLO.DAG.getNode(ISD::AND, dl, VT, Op0.getOperand(0), Op1); in SimplifyDemandedBits()
1418 TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, NewAnd, in SimplifyDemandedBits()
1467 case ISD::OR: { in SimplifyDemandedBits()
1519 if (Op0.getOpcode() == ISD::AND && Op1.getOpcode() == ISD::AND && in SimplifyDemandedBits()
1527 if (Alt.getOpcode() == ISD::OR) { in SimplifyDemandedBits()
1531 if (SDValue C12 = TLO.DAG.FoldConstantArithmetic(ISD::OR, dl, VT, in SimplifyDemandedBits()
1533 SDValue MaskX = TLO.DAG.getNode(ISD::AND, dl, VT, X, C12); in SimplifyDemandedBits()
1534 SDValue MaskY = TLO.DAG.getNode(ISD::AND, dl, VT, Y, C2); in SimplifyDemandedBits()
1536 Op, TLO.DAG.getNode(ISD::OR, dl, VT, MaskX, MaskY)); in SimplifyDemandedBits()
1547 case ISD::XOR: { in SimplifyDemandedBits()
1574 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
1586 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC)); in SimplifyDemandedBits()
1599 if ((Op0Opcode == ISD::SRL || Op0Opcode == ISD::SHL) && Op0.hasOneUse()) { in SimplifyDemandedBits()
1607 Ones = Op0Opcode == ISD::SHL ? Ones.shl(ShiftAmt) in SimplifyDemandedBits()
1647 case ISD::SELECT: in SimplifyDemandedBits()
1664 case ISD::VSELECT: in SimplifyDemandedBits()
1677 case ISD::SELECT_CC: in SimplifyDemandedBits()
1694 case ISD::SETCC: { in SimplifyDemandedBits()
1697 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); in SimplifyDemandedBits()
1709 if (CC == ISD::SETLT && Op1.getValueType().isInteger() && in SimplifyDemandedBits()
1710 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode()))) in SimplifyDemandedBits()
1722 case ISD::SHL: { in SimplifyDemandedBits()
1737 if (Op0.getOpcode() == ISD::SRL) { in SimplifyDemandedBits()
1742 unsigned Opc = ISD::SHL; in SimplifyDemandedBits()
1746 Opc = ISD::SRL; in SimplifyDemandedBits()
1758 if (Op0.getOpcode() == ISD::ANY_EXTEND) { in SimplifyDemandedBits()
1763 isTypeDesirableForOp(ISD::SHL, InnerVT)) { in SimplifyDemandedBits()
1765 ISD::SHL, dl, InnerVT, InnerOp, in SimplifyDemandedBits()
1768 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl)); in SimplifyDemandedBits()
1777 if (InnerOp.getOpcode() == ISD::SRL && Op0.hasOneUse() && in SimplifyDemandedBits()
1788 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, in SimplifyDemandedBits()
1791 Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA)); in SimplifyDemandedBits()
1821 SDValue NewOp = TLO.DAG.getNode(ISD::SHL, dl, VT, DemandedOp0, Op1); in SimplifyDemandedBits()
1840 isTypeDesirableForOp(ISD::SHL, HalfVT) && in SimplifyDemandedBits()
1842 (!TLO.LegalOperations() || isOperationLegal(ISD::SHL, HalfVT))) { in SimplifyDemandedBits()
1851 SDValue NewOp = TLO.DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Op0); in SimplifyDemandedBits()
1854 SDValue NewShift = TLO.DAG.getNode(ISD::SHL, dl, HalfVT, NewOp, in SimplifyDemandedBits()
1857 TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, NewShift); in SimplifyDemandedBits()
1897 case ISD::SRL: { in SimplifyDemandedBits()
1917 if (Op0.getOpcode() == ISD::SHL) { in SimplifyDemandedBits()
1922 unsigned Opc = ISD::SRL; in SimplifyDemandedBits()
1926 Opc = ISD::SHL; in SimplifyDemandedBits()
1948 isTypeDesirableForOp(ISD::SRL, HalfVT) && in SimplifyDemandedBits()
1950 (!TLO.LegalOperations() || isOperationLegal(ISD::SRL, HalfVT)) && in SimplifyDemandedBits()
1953 SDValue NewOp = TLO.DAG.getNode(ISD::TRUNCATE, dl, HalfVT, Op0); in SimplifyDemandedBits()
1957 TLO.DAG.getNode(ISD::SRL, dl, HalfVT, NewOp, NewShiftAmt); in SimplifyDemandedBits()
1959 Op, TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, NewShift)); in SimplifyDemandedBits()
1978 SDValue NewOp = TLO.DAG.getNode(ISD::SRL, dl, VT, DemandedOp0, Op1); in SimplifyDemandedBits()
1989 case ISD::SRA: { in SimplifyDemandedBits()
2006 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
2021 if (Op0.getOpcode() == ISD::SHL) { in SimplifyDemandedBits()
2032 getOperationAction(ISD::SIGN_EXTEND_INREG, ExtVT) == Legal) in SimplifyDemandedBits()
2034 Op, TLO.DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT, in SimplifyDemandedBits()
2074 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags)); in SimplifyDemandedBits()
2081 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA)); in SimplifyDemandedBits()
2093 SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1); in SimplifyDemandedBits()
2100 case ISD::FSHL: in SimplifyDemandedBits()
2101 case ISD::FSHR: { in SimplifyDemandedBits()
2105 bool IsFSHL = (Op.getOpcode() == ISD::FSHL); in SimplifyDemandedBits()
2162 case ISD::ROTL: in SimplifyDemandedBits()
2163 case ISD::ROTR: { in SimplifyDemandedBits()
2166 bool IsROTL = (Op.getOpcode() == ISD::ROTL); in SimplifyDemandedBits()
2188 if ((!TLO.LegalOperations() || isOperationLegal(ISD::SHL, VT)) && in SimplifyDemandedBits()
2191 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
2193 if ((!TLO.LegalOperations() || isOperationLegal(ISD::SRL, VT)) && in SimplifyDemandedBits()
2196 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
2209 case ISD::SMIN: in SimplifyDemandedBits()
2210 case ISD::SMAX: in SimplifyDemandedBits()
2211 case ISD::UMIN: in SimplifyDemandedBits()
2212 case ISD::UMAX: { in SimplifyDemandedBits()
2219 (Opc == ISD::SMIN || Opc == ISD::UMAX) ? ISD::OR : ISD::AND; in SimplifyDemandedBits()
2231 case ISD::SMIN: in SimplifyDemandedBits()
2238 case ISD::SMAX: in SimplifyDemandedBits()
2245 case ISD::UMIN: in SimplifyDemandedBits()
2252 case ISD::UMAX: in SimplifyDemandedBits()
2262 case ISD::BITREVERSE: { in SimplifyDemandedBits()
2272 case ISD::BSWAP: { in SimplifyDemandedBits()
2289 unsigned ShiftOpcode = NLZ > NTZ ? ISD::SRL : ISD::SHL; in SimplifyDemandedBits()
2307 case ISD::CTPOP: { in SimplifyDemandedBits()
2312 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT, in SimplifyDemandedBits()
2318 case ISD::SIGN_EXTEND_INREG: { in SimplifyDemandedBits()
2336 TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt)); in SimplifyDemandedBits()
2372 case ISD::BUILD_PAIR: { in SimplifyDemandedBits()
2390 case ISD::ZERO_EXTEND_VECTOR_INREG: in SimplifyDemandedBits()
2394 case ISD::ZERO_EXTEND: { in SimplifyDemandedBits()
2399 bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG; in SimplifyDemandedBits()
2410 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND; in SimplifyDemandedBits()
2436 case ISD::SIGN_EXTEND_VECTOR_INREG: in SimplifyDemandedBits()
2440 case ISD::SIGN_EXTEND: { in SimplifyDemandedBits()
2445 bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG; in SimplifyDemandedBits()
2467 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND; in SimplifyDemandedBits()
2485 IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND; in SimplifyDemandedBits()
2500 case ISD::ANY_EXTEND_VECTOR_INREG: in SimplifyDemandedBits()
2504 case ISD::ANY_EXTEND: { in SimplifyDemandedBits()
2509 bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG; in SimplifyDemandedBits()
2532 case ISD::TRUNCATE: { in SimplifyDemandedBits()
2547 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc)); in SimplifyDemandedBits()
2554 case ISD::SRL: in SimplifyDemandedBits()
2557 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT)) in SimplifyDemandedBits()
2580 TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0)); in SimplifyDemandedBits()
2582 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt)); in SimplifyDemandedBits()
2591 case ISD::AssertZext: { in SimplifyDemandedBits()
2605 case ISD::EXTRACT_VECTOR_ELT: { in SimplifyDemandedBits()
2646 case ISD::BITCAST: { in SimplifyDemandedBits()
2658 bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT); in SimplifyDemandedBits()
2659 bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32); in SimplifyDemandedBits()
2666 SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src); in SimplifyDemandedBits()
2669 Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign); in SimplifyDemandedBits()
2673 TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt)); in SimplifyDemandedBits()
2748 case ISD::MUL: in SimplifyDemandedBits()
2758 SDValue Shl = TLO.DAG.getNode(ISD::SHL, dl, VT, Op.getOperand(0), AmtC); in SimplifyDemandedBits()
2767 SDValue And1 = TLO.DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0), One); in SimplifyDemandedBits()
2771 case ISD::ADD: in SimplifyDemandedBits()
2772 case ISD::SUB: { in SimplifyDemandedBits()
2797 if (Op.getOpcode() == ISD::SUB && DemandedBits.isOne() && in SimplifyDemandedBits()
2840 if (Mul.getOpcode() != ISD::MUL || !Mul.hasOneUse()) in SimplifyDemandedBits()
2855 auto foldMul = [&](ISD::NodeType NT, SDValue X, SDValue Y, unsigned ShlAmt) { in SimplifyDemandedBits()
2858 SDValue Shl = TLO.DAG.getNode(ISD::SHL, dl, VT, X, ShlAmtC); in SimplifyDemandedBits()
2863 if (isOperationLegalOrCustom(ISD::SHL, VT)) { in SimplifyDemandedBits()
2864 if (Op.getOpcode() == ISD::ADD) { in SimplifyDemandedBits()
2867 return foldMul(ISD::SUB, Op0.getOperand(0), Op1, ShAmt); in SimplifyDemandedBits()
2870 return foldMul(ISD::SUB, Op1.getOperand(0), Op0, ShAmt); in SimplifyDemandedBits()
2872 if (Op.getOpcode() == ISD::SUB) { in SimplifyDemandedBits()
2875 return foldMul(ISD::ADD, Op1.getOperand(0), Op0, ShAmt); in SimplifyDemandedBits()
2879 if (Op.getOpcode() == ISD::MUL) { in SimplifyDemandedBits()
2882 Known = KnownBits::computeForAddSub(Op.getOpcode() == ISD::ADD, in SimplifyDemandedBits()
2890 if (Op.getOpcode() >= ISD::BUILTIN_OP_END || in SimplifyDemandedBits()
2891 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN) { in SimplifyDemandedBits()
3067 case ISD::SCALAR_TO_VECTOR: { in SimplifyDemandedVectorElts()
3073 if (ScalarSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { in SimplifyDemandedVectorElts()
3096 case ISD::BITCAST: { in SimplifyDemandedVectorElts()
3190 case ISD::BUILD_VECTOR: { in SimplifyDemandedVectorElts()
3220 case ISD::CONCAT_VECTORS: { in SimplifyDemandedVectorElts()
3255 case ISD::INSERT_SUBVECTOR: { in SimplifyDemandedVectorElts()
3273 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in SimplifyDemandedVectorElts()
3299 case ISD::EXTRACT_SUBVECTOR: { in SimplifyDemandedVectorElts()
3327 case ISD::INSERT_VECTOR_ELT: { in SimplifyDemandedVectorElts()
3358 case ISD::VSELECT: { in SimplifyDemandedVectorElts()
3395 case ISD::VECTOR_SHUFFLE: { in SimplifyDemandedVectorElts()
3469 case ISD::ANY_EXTEND_VECTOR_INREG: in SimplifyDemandedVectorElts()
3470 case ISD::SIGN_EXTEND_VECTOR_INREG: in SimplifyDemandedVectorElts()
3471 case ISD::ZERO_EXTEND_VECTOR_INREG: { in SimplifyDemandedVectorElts()
3482 if (IsLE && Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG && in SimplifyDemandedVectorElts()
3489 if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) { in SimplifyDemandedVectorElts()
3497 if (IsLE && DemandedSrcElts == 1 && Src.getOpcode() == ISD::AND && in SimplifyDemandedVectorElts()
3508 ISD::AND, DL, SrcVT, {Src.getOperand(1), Mask})) { in SimplifyDemandedVectorElts()
3509 Fold = TLO.DAG.getNode(ISD::AND, DL, SrcVT, Src.getOperand(0), Fold); in SimplifyDemandedVectorElts()
3519 case ISD::ADD: { in SimplifyDemandedVectorElts()
3530 case ISD::OR: in SimplifyDemandedVectorElts()
3531 case ISD::XOR: in SimplifyDemandedVectorElts()
3532 case ISD::SUB: in SimplifyDemandedVectorElts()
3533 case ISD::FADD: in SimplifyDemandedVectorElts()
3534 case ISD::FSUB: in SimplifyDemandedVectorElts()
3535 case ISD::FMUL: in SimplifyDemandedVectorElts()
3536 case ISD::FDIV: in SimplifyDemandedVectorElts()
3537 case ISD::FREM: { in SimplifyDemandedVectorElts()
3560 case ISD::SHL: in SimplifyDemandedVectorElts()
3561 case ISD::SRL: in SimplifyDemandedVectorElts()
3562 case ISD::SRA: in SimplifyDemandedVectorElts()
3563 case ISD::ROTL: in SimplifyDemandedVectorElts()
3564 case ISD::ROTR: { in SimplifyDemandedVectorElts()
3587 case ISD::MUL: in SimplifyDemandedVectorElts()
3588 case ISD::MULHU: in SimplifyDemandedVectorElts()
3589 case ISD::MULHS: in SimplifyDemandedVectorElts()
3590 case ISD::AND: { in SimplifyDemandedVectorElts()
3628 case ISD::TRUNCATE: in SimplifyDemandedVectorElts()
3629 case ISD::SIGN_EXTEND: in SimplifyDemandedVectorElts()
3630 case ISD::ZERO_EXTEND: in SimplifyDemandedVectorElts()
3635 if (Op.getOpcode() == ISD::ZERO_EXTEND) { in SimplifyDemandedVectorElts()
3643 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) { in SimplifyDemandedVectorElts()
3674 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in computeKnownBitsForTargetNode()
3675 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in computeKnownBitsForTargetNode()
3676 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in computeKnownBitsForTargetNode()
3677 Op.getOpcode() == ISD::INTRINSIC_VOID) && in computeKnownBitsForTargetNode()
3708 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in ComputeNumSignBitsForTargetNode()
3709 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in ComputeNumSignBitsForTargetNode()
3710 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in ComputeNumSignBitsForTargetNode()
3711 Op.getOpcode() == ISD::INTRINSIC_VOID) && in ComputeNumSignBitsForTargetNode()
3726 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in SimplifyDemandedVectorEltsForTargetNode()
3727 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in SimplifyDemandedVectorEltsForTargetNode()
3728 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in SimplifyDemandedVectorEltsForTargetNode()
3729 Op.getOpcode() == ISD::INTRINSIC_VOID) && in SimplifyDemandedVectorEltsForTargetNode()
3738 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in SimplifyDemandedBitsForTargetNode()
3739 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in SimplifyDemandedBitsForTargetNode()
3740 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in SimplifyDemandedBitsForTargetNode()
3741 Op.getOpcode() == ISD::INTRINSIC_VOID) && in SimplifyDemandedBitsForTargetNode()
3752 (Op.getOpcode() >= ISD::BUILTIN_OP_END || in SimplifyMultipleUseDemandedBitsForTargetNode()
3753 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in SimplifyMultipleUseDemandedBitsForTargetNode()
3754 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in SimplifyMultipleUseDemandedBitsForTargetNode()
3755 Op.getOpcode() == ISD::INTRINSIC_VOID) && in SimplifyMultipleUseDemandedBitsForTargetNode()
3786 (Op.getOpcode() >= ISD::BUILTIN_OP_END || in isGuaranteedNotToBeUndefOrPoisonForTargetNode()
3787 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in isGuaranteedNotToBeUndefOrPoisonForTargetNode()
3788 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in isGuaranteedNotToBeUndefOrPoisonForTargetNode()
3789 Op.getOpcode() == ISD::INTRINSIC_VOID) && in isGuaranteedNotToBeUndefOrPoisonForTargetNode()
3798 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in canCreateUndefOrPoisonForTargetNode()
3799 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in canCreateUndefOrPoisonForTargetNode()
3800 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in canCreateUndefOrPoisonForTargetNode()
3801 Op.getOpcode() == ISD::INTRINSIC_VOID) && in canCreateUndefOrPoisonForTargetNode()
3812 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in isKnownNeverNaNForTargetNode()
3813 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in isKnownNeverNaNForTargetNode()
3814 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in isKnownNeverNaNForTargetNode()
3815 Op.getOpcode() == ISD::INTRINSIC_VOID) && in isKnownNeverNaNForTargetNode()
3826 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in isSplatValueForTargetNode()
3827 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in isSplatValueForTargetNode()
3828 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in isSplatValueForTargetNode()
3829 Op.getOpcode() == ISD::INTRINSIC_VOID) && in isSplatValueForTargetNode()
3913 ISD::CondCode Cond, const SDLoc &DL, in foldSetCCWithAnd()
3915 if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND) in foldSetCCWithAnd()
3920 if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() || in foldSetCCWithAnd()
3921 (Cond != ISD::SETEQ && Cond != ISD::SETNE)) in foldSetCCWithAnd()
3926 if (Cond == ISD::SETNE && isNullConstant(N1) && in foldSetCCWithAnd()
3951 Cond == ISD::SETEQ ? ISD::SETGE : ISD::SETLT); in foldSetCCWithAnd()
3980 Cond = ISD::getSetCCInverse(Cond, OpVT); in foldSetCCWithAnd()
3998 SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y); in foldSetCCWithAnd()
4016 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI, in optimizeSetCCOfSignedTruncationCheck()
4024 if (N0->getOpcode() != ISD::ADD) in optimizeSetCCOfSignedTruncationCheck()
4039 ISD::CondCode NewCond; in optimizeSetCCOfSignedTruncationCheck()
4040 if (Cond == ISD::CondCode::SETULT) { in optimizeSetCCOfSignedTruncationCheck()
4041 NewCond = ISD::CondCode::SETEQ; in optimizeSetCCOfSignedTruncationCheck()
4042 } else if (Cond == ISD::CondCode::SETULE) { in optimizeSetCCOfSignedTruncationCheck()
4043 NewCond = ISD::CondCode::SETEQ; in optimizeSetCCOfSignedTruncationCheck()
4046 } else if (Cond == ISD::CondCode::SETUGT) { in optimizeSetCCOfSignedTruncationCheck()
4047 NewCond = ISD::CondCode::SETNE; in optimizeSetCCOfSignedTruncationCheck()
4050 } else if (Cond == ISD::CondCode::SETUGE) { in optimizeSetCCOfSignedTruncationCheck()
4051 NewCond = ISD::CondCode::SETNE; in optimizeSetCCOfSignedTruncationCheck()
4096 SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt); in optimizeSetCCOfSignedTruncationCheck()
4097 SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt); in optimizeSetCCOfSignedTruncationCheck()
4105 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond, in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4109 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4125 case ISD::SHL: in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4126 NewShiftOpcode = ISD::SRL; in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4128 case ISD::SRL: in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4129 NewShiftOpcode = ISD::SHL; in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4150 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse()) in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4168 SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C); in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4177 ISD::CondCode Cond, const SDLoc &DL, in foldSetCCWithBinOp()
4180 assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && in foldSetCCWithBinOp()
4182 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode"); in foldSetCCWithBinOp()
4199 if (BOpcode == ISD::ADD || BOpcode == ISD::XOR) in foldSetCCWithBinOp()
4210 SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One); in foldSetCCWithBinOp()
4218 ISD::CondCode Cond, const SDLoc &dl, in simplifySetCCWithCTPOP()
4223 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() && in simplifySetCCWithCTPOP()
4227 if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse()) in simplifySetCCWithCTPOP()
4236 if (Cond == ISD::SETULT || Cond == ISD::SETUGT) { in simplifySetCCWithCTPOP()
4242 if (C1.ugt(CostLimit + (Cond == ISD::SETULT))) in simplifySetCCWithCTPOP()
4244 if (C1 == 0 && (Cond == ISD::SETULT)) in simplifySetCCWithCTPOP()
4247 unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT); in simplifySetCCWithCTPOP()
4252 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne); in simplifySetCCWithCTPOP()
4253 Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add); in simplifySetCCWithCTPOP()
4255 ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE; in simplifySetCCWithCTPOP()
4260 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) { in simplifySetCCWithCTPOP()
4268 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne); in simplifySetCCWithCTPOP()
4275 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add); in simplifySetCCWithCTPOP()
4282 SDValue Xor = DAG.getNode(ISD::XOR, dl, CTVT, CTOp, Add); in simplifySetCCWithCTPOP()
4283 ISD::CondCode CmpCond = Cond == ISD::SETEQ ? ISD::SETUGT : ISD::SETULE; in simplifySetCCWithCTPOP()
4291 ISD::CondCode Cond, const SDLoc &dl, in foldSetCCWithRotate()
4293 if (Cond != ISD::SETEQ && Cond != ISD::SETNE) in foldSetCCWithRotate()
4301 if (X.getOpcode() == ISD::ROTL || X.getOpcode() == ISD::ROTR) in foldSetCCWithRotate()
4319 if (N0.hasOneUse() && N0.getOpcode() == ISD::OR && C1->isZero()) { in foldSetCCWithRotate()
4321 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, R, N0.getOperand(1)); in foldSetCCWithRotate()
4325 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, R, N0.getOperand(0)); in foldSetCCWithRotate()
4334 ISD::CondCode Cond, const SDLoc &dl, in foldSetCCWithFunnelShift()
4338 if (Cond != ISD::SETEQ && Cond != ISD::SETNE) in foldSetCCWithFunnelShift()
4346 (N0.getOpcode() != ISD::FSHL && N0.getOpcode() != ISD::FSHR)) in foldSetCCWithFunnelShift()
4356 if (N0.getOpcode() == ISD::FSHR) in foldSetCCWithFunnelShift()
4362 if (Or.getOpcode() != ISD::OR || !Or.hasOneUse()) in foldSetCCWithFunnelShift()
4384 SDValue Shift = DAG.getNode(ISD::SHL, dl, OpVT, Y, NewShAmt); in foldSetCCWithFunnelShift()
4385 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, Shift, X); in foldSetCCWithFunnelShift()
4391 SDValue Shift = DAG.getNode(ISD::SRL, dl, OpVT, Y, NewShAmt); in foldSetCCWithFunnelShift()
4392 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, Shift, X); in foldSetCCWithFunnelShift()
4402 ISD::CondCode Cond, bool foldBooleans, in SimplifySetCC()
4423 ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond); in SimplifySetCC()
4436 DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) && in SimplifySetCC()
4437 !DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1})) in SimplifySetCC()
4459 if (C1.isZero() && (Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
4460 N0.getOpcode() == ISD::MUL && N0.hasOneUse() && in SimplifySetCC()
4466 unsigned LogicOp = Cond == ISD::SETEQ ? ISD::OR : ISD::AND; in SimplifySetCC()
4473 if (N0.getOpcode() == ISD::SRL && (C1.isZero() || C1.isOne()) && in SimplifySetCC()
4474 N0.getOperand(0).getOpcode() == ISD::CTLZ && in SimplifySetCC()
4477 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
4479 if ((C1 == 0) == (Cond == ISD::SETEQ)) { in SimplifySetCC()
4482 Cond = ISD::SETNE; in SimplifySetCC()
4486 Cond = ISD::SETEQ; in SimplifySetCC()
4502 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
4507 if (N0->getOpcode() == ISD::ZERO_EXTEND) { in SimplifySetCC()
4511 } else if (N0->getOpcode() == ISD::AND) { in SimplifySetCC()
4518 } else if (N0->getOpcode() == ISD::SIGN_EXTEND) { in SimplifySetCC()
4525 if (LN0->getExtensionType() == ISD::ZEXTLOAD) { in SimplifySetCC()
4528 } else if (LN0->getExtensionType() == ISD::SEXTLOAD) { in SimplifySetCC()
4543 if (isTypeDesirableForOp(ISD::SETCC, MinVT)) { in SimplifySetCC()
4545 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt); in SimplifySetCC()
4549 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ); in SimplifySetCC()
4564 bool SExt = (N0Opc == ISD::SIGN_EXTEND); in SimplifySetCC()
4566 TopSetCC.getOpcode() == ISD::SETCC && in SimplifySetCC()
4567 (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) && in SimplifySetCC()
4571 bool Inverse = (N1C->isZero() && Cond == ISD::SETEQ) || in SimplifySetCC()
4572 (!N1C->isZero() && Cond == ISD::SETNE); in SimplifySetCC()
4577 ISD::CondCode InvCond = ISD::getSetCCInverse( in SimplifySetCC()
4591 !ISD::isSignedIntSetCC(Cond) && in SimplifySetCC()
4592 N0.getOpcode() == ISD::AND && C1 == 0 && in SimplifySetCC()
4605 if (Lod->getExtensionType() != ISD::NON_EXTLOAD) in SimplifySetCC()
4627 shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) { in SimplifySetCC()
4637 DAG.getNode(ISD::AND, dl, newVT, NewLoad, in SimplifySetCC()
4646 if (N0.getOpcode() == ISD::ZERO_EXTEND) { in SimplifySetCC()
4654 case ISD::SETUGT: in SimplifySetCC()
4655 case ISD::SETUGE: in SimplifySetCC()
4656 case ISD::SETEQ: in SimplifySetCC()
4658 case ISD::SETULT: in SimplifySetCC()
4659 case ISD::SETULE: in SimplifySetCC()
4660 case ISD::SETNE: in SimplifySetCC()
4662 case ISD::SETGT: in SimplifySetCC()
4663 case ISD::SETGE: in SimplifySetCC()
4666 case ISD::SETLT: in SimplifySetCC()
4667 case ISD::SETLE: in SimplifySetCC()
4677 case ISD::SETEQ: in SimplifySetCC()
4678 case ISD::SETNE: in SimplifySetCC()
4679 case ISD::SETUGT: in SimplifySetCC()
4680 case ISD::SETUGE: in SimplifySetCC()
4681 case ISD::SETULT: in SimplifySetCC()
4682 case ISD::SETULE: { in SimplifySetCC()
4685 (isOperationLegal(ISD::SETCC, newVT) && in SimplifySetCC()
4699 } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG && in SimplifySetCC()
4700 (Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
4711 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT); in SimplifySetCC()
4716 SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0), in SimplifySetCC()
4724 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) { in SimplifySetCC()
4726 if (N0.getOpcode() == ISD::SETCC && in SimplifySetCC()
4731 bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne()); in SimplifySetCC()
4733 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0); in SimplifySetCC()
4735 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in SimplifySetCC()
4736 CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType()); in SimplifySetCC()
4742 if ((N0.getOpcode() == ISD::XOR || in SimplifySetCC()
4743 (N0.getOpcode() == ISD::AND && in SimplifySetCC()
4744 N0.getOperand(0).getOpcode() == ISD::XOR && in SimplifySetCC()
4755 if (N0.getOpcode() == ISD::XOR) { in SimplifySetCC()
4758 assert(N0.getOpcode() == ISD::AND && in SimplifySetCC()
4759 N0.getOperand(0).getOpcode() == ISD::XOR); in SimplifySetCC()
4761 Val = DAG.getNode(ISD::AND, dl, N0.getValueType(), in SimplifySetCC()
4767 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ); in SimplifySetCC()
4771 if (Op0.getOpcode() == ISD::TRUNCATE) in SimplifySetCC()
4774 if ((Op0.getOpcode() == ISD::XOR) && in SimplifySetCC()
4775 Op0.getOperand(0).getOpcode() == ISD::SETCC && in SimplifySetCC()
4776 Op0.getOperand(1).getOpcode() == ISD::SETCC) { in SimplifySetCC()
4786 Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ; in SimplifySetCC()
4790 if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) { in SimplifySetCC()
4793 Op0 = DAG.getNode(ISD::AND, dl, VT, in SimplifySetCC()
4794 DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)), in SimplifySetCC()
4797 Op0 = DAG.getNode(ISD::AND, dl, VT, in SimplifySetCC()
4798 DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)), in SimplifySetCC()
4803 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ); in SimplifySetCC()
4805 if (Op0.getOpcode() == ISD::AssertZext && in SimplifySetCC()
4809 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ); in SimplifySetCC()
4817 if (N0.getOpcode() == ISD::UREM && N1C->isZero() && in SimplifySetCC()
4818 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) { in SimplifySetCC()
4827 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
4828 N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(N0.getOperand(1)) && in SimplifySetCC()
4833 Cond == ISD::SETEQ ? ISD::SETLT : ISD::SETGE); in SimplifySetCC()
4848 if (ISD::isSignedIntSetCC(Cond)) { in SimplifySetCC()
4857 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) { in SimplifySetCC()
4865 ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT; in SimplifySetCC()
4877 if (Cond == ISD::SETLE || Cond == ISD::SETULE) { in SimplifySetCC()
4885 ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT; in SimplifySetCC()
4897 if (Cond == ISD::SETLT || Cond == ISD::SETULT) { in SimplifySetCC()
4905 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE); in SimplifySetCC()
4911 ISD::SETEQ); in SimplifySetCC()
4915 if (Cond == ISD::SETGT || Cond == ISD::SETUGT) { in SimplifySetCC()
4923 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE); in SimplifySetCC()
4929 ISD::SETEQ); in SimplifySetCC()
4933 if (Cond == ISD::SETEQ || Cond == ISD::SETNE) { in SimplifySetCC()
4950 if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0) in SimplifySetCC()
4956 if (RHS.getOpcode() == ISD::SHL && in SimplifySetCC()
4964 if (LHS.getOpcode() == ISD::SHL && in SimplifySetCC()
4980 SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits); in SimplifySetCC()
4982 DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask); in SimplifySetCC()
4991 if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) { in SimplifySetCC()
5008 if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) || in SimplifySetCC()
5009 (Cond == ISD::SETUGE && C1.isMinSignedValue())) in SimplifySetCC()
5012 ISD::SETLT); in SimplifySetCC()
5016 if ((Cond == ISD::SETULT && C1.isMinSignedValue()) || in SimplifySetCC()
5017 (Cond == ISD::SETULE && C1.isMaxSignedValue())) in SimplifySetCC()
5020 ISD::SETGT); in SimplifySetCC()
5037 (Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
5039 N0.getOpcode() == ISD::AND) { in SimplifySetCC()
5043 if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3 in SimplifySetCC()
5048 return DAG.getNode(ISD::TRUNCATE, dl, VT, in SimplifySetCC()
5049 DAG.getNode(ISD::SRL, dl, ShValTy, N0, in SimplifySetCC()
5052 } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) { in SimplifySetCC()
5058 return DAG.getNode(ISD::TRUNCATE, dl, VT, in SimplifySetCC()
5059 DAG.getNode(ISD::SRL, dl, ShValTy, N0, in SimplifySetCC()
5070 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
5071 N0.getOpcode() == ISD::AND && N0.hasOneUse()) { in SimplifySetCC()
5078 DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0), in SimplifySetCC()
5085 } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE || in SimplifySetCC()
5086 Cond == ISD::SETULE || Cond == ISD::SETUGT) { in SimplifySetCC()
5087 bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT); in SimplifySetCC()
5094 ISD::CondCode NewCond = Cond; in SimplifySetCC()
5098 NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE; in SimplifySetCC()
5106 SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0, in SimplifySetCC()
5123 if (Cond == ISD::SETO || Cond == ISD::SETUO) in SimplifySetCC()
5127 if (N0.getOpcode() == ISD::FNEG) { in SimplifySetCC()
5128 ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond); in SimplifySetCC()
5131 SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1); in SimplifySetCC()
5137 if (isOperationLegalOrCustom(ISD::IS_FPCLASS, N0.getValueType()) && in SimplifySetCC()
5139 bool IsFabs = N0.getOpcode() == ISD::FABS; in SimplifySetCC()
5141 if ((Cond == ISD::SETOEQ || Cond == ISD::SETUEQ) && CFP->isInfinity()) { in SimplifySetCC()
5144 if (Cond == ISD::SETUEQ) in SimplifySetCC()
5146 return DAG.getNode(ISD::IS_FPCLASS, dl, VT, Op, in SimplifySetCC()
5159 ISD::CondCode NewCond = ISD::SETCC_INVALID; in SimplifySetCC()
5161 case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break; in SimplifySetCC()
5162 case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break; in SimplifySetCC()
5163 case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break; in SimplifySetCC()
5164 case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break; in SimplifySetCC()
5167 if (NewCond != ISD::SETCC_INVALID && in SimplifySetCC()
5180 bool EqTrue = ISD::isTrueWhenEqual(Cond); in SimplifySetCC()
5181 unsigned UOF = ISD::getUnorderedFlavor(Cond); in SimplifySetCC()
5188 ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO; in SimplifySetCC()
5213 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
5215 if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB || in SimplifySetCC()
5216 N0.getOpcode() == ISD::XOR) { in SimplifySetCC()
5241 if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) in SimplifySetCC()
5249 if (N0.getOpcode() == ISD::XOR && N0.getNode()->hasOneUse()) in SimplifySetCC()
5259 if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) in SimplifySetCC()
5280 if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB || in SimplifySetCC()
5281 N1.getOpcode() == ISD::XOR) in SimplifySetCC()
5290 if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) && in SimplifySetCC()
5291 N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) { in SimplifySetCC()
5295 if (N0.getOpcode() == ISD::UREM) { in SimplifySetCC()
5298 } else if (N0.getOpcode() == ISD::SREM) { in SimplifySetCC()
5310 case ISD::SETEQ: // X == Y -> ~(X^Y) in SimplifySetCC()
5311 Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1); in SimplifySetCC()
5316 case ISD::SETNE: // X != Y --> (X^Y) in SimplifySetCC()
5317 N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1); in SimplifySetCC()
5319 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y in SimplifySetCC()
5320 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y in SimplifySetCC()
5322 N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp); in SimplifySetCC()
5326 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X in SimplifySetCC()
5327 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X in SimplifySetCC()
5329 N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp); in SimplifySetCC()
5333 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y in SimplifySetCC()
5334 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y in SimplifySetCC()
5336 N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp); in SimplifySetCC()
5340 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X in SimplifySetCC()
5341 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X in SimplifySetCC()
5343 N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp); in SimplifySetCC()
5350 ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT)); in SimplifySetCC()
5373 if (N->getOpcode() == ISD::ADD) { in isGAPlusOffset()
5496 ISD::NodeType ExtOpc = in LowerAsmOperandForConstraint()
5497 IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND; in LowerAsmOperandForConstraint()
5499 ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue(); in LowerAsmOperandForConstraint()
5523 if (OpCode == ISD::ADD || OpCode == ISD::SUB) { in LowerAsmOperandForConstraint()
5527 else if (OpCode == ISD::ADD && in LowerAsmOperandForConstraint()
5532 Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue(); in LowerAsmOperandForConstraint()
6053 if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern)) in BuildExactSDIV()
6057 if (Op1.getOpcode() == ISD::BUILD_VECTOR) { in BuildExactSDIV()
6060 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) { in BuildExactSDIV()
6079 Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags); in BuildExactSDIV()
6083 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor); in BuildExactSDIV()
6129 SDValue Cmp = DAG.getSetCC(DL, CCVT, N0, Zero, ISD::SETLT); in buildSDIVPow2WithCMov()
6130 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne); in buildSDIVPow2WithCMov()
6131 SDValue CMov = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0); in buildSDIVPow2WithCMov()
6139 DAG.getNode(ISD::SRA, DL, VT, CMov, DAG.getConstant(Lg2, DL, VT)); in buildSDIVPow2WithCMov()
6147 return DAG.getNode(ISD::SUB, DL, VT, Zero, SRA); in buildSDIVPow2WithCMov()
6179 !isOperationLegal(ISD::MUL, MulVT)) in BuildSDIV()
6223 if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern)) in BuildSDIV()
6227 if (N1.getOpcode() == ISD::BUILD_VECTOR) { in BuildSDIV()
6232 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) { in BuildSDIV()
6255 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X); in BuildSDIV()
6256 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y); in BuildSDIV()
6257 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y); in BuildSDIV()
6258 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y, in BuildSDIV()
6260 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildSDIV()
6263 if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization)) in BuildSDIV()
6264 return DAG.getNode(ISD::MULHS, dl, VT, X, Y); in BuildSDIV()
6265 if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) { in BuildSDIV()
6267 DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y); in BuildSDIV()
6276 if (isOperationLegalOrCustom(ISD::MUL, WideVT)) { in BuildSDIV()
6277 X = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, X); in BuildSDIV()
6278 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, Y); in BuildSDIV()
6279 Y = DAG.getNode(ISD::MUL, dl, WideVT, X, Y); in BuildSDIV()
6280 Y = DAG.getNode(ISD::SRL, dl, WideVT, Y, in BuildSDIV()
6282 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildSDIV()
6294 Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor); in BuildSDIV()
6296 Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor); in BuildSDIV()
6300 Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift); in BuildSDIV()
6305 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift); in BuildSDIV()
6307 T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask); in BuildSDIV()
6309 return DAG.getNode(ISD::ADD, dl, VT, Q, T); in BuildSDIV()
6341 !isOperationLegal(ISD::MUL, MulVT)) in BuildUDIV()
6406 if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern)) in BuildUDIV()
6410 if (N1.getOpcode() == ISD::BUILD_VECTOR) { in BuildUDIV()
6415 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) { in BuildUDIV()
6432 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift); in BuildUDIV()
6441 X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X); in BuildUDIV()
6442 Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y); in BuildUDIV()
6443 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y); in BuildUDIV()
6444 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y, in BuildUDIV()
6446 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildUDIV()
6449 if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization)) in BuildUDIV()
6450 return DAG.getNode(ISD::MULHU, dl, VT, X, Y); in BuildUDIV()
6451 if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) { in BuildUDIV()
6453 DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y); in BuildUDIV()
6462 if (isOperationLegalOrCustom(ISD::MUL, WideVT)) { in BuildUDIV()
6463 X = DAG.getNode(ISD::ZERO_EXTEND, dl, WideVT, X); in BuildUDIV()
6464 Y = DAG.getNode(ISD::ZERO_EXTEND, dl, WideVT, Y); in BuildUDIV()
6465 Y = DAG.getNode(ISD::MUL, dl, WideVT, X, Y); in BuildUDIV()
6466 Y = DAG.getNode(ISD::SRL, dl, WideVT, Y, in BuildUDIV()
6468 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildUDIV()
6481 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q); in BuildUDIV()
6489 NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT)); in BuildUDIV()
6493 Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q); in BuildUDIV()
6498 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift); in BuildUDIV()
6505 SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ); in BuildUDIV()
6544 ISD::CondCode Cond, in buildUREMEqFold()
6560 SDValue CompTargetNode, ISD::CondCode Cond, in prepareUREMEqFold()
6568 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in prepareUREMEqFold()
6579 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT)) in prepareUREMEqFold()
6675 if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern)) in prepareUREMEqFold()
6688 if (D.getOpcode() == ISD::BUILD_VECTOR) { in prepareUREMEqFold()
6702 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) { in prepareUREMEqFold()
6716 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT)) in prepareUREMEqFold()
6720 N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode); in prepareUREMEqFold()
6724 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal); in prepareUREMEqFold()
6731 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT)) in prepareUREMEqFold()
6734 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal); in prepareUREMEqFold()
6741 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT)); in prepareUREMEqFold()
6756 DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE); in prepareUREMEqFold()
6761 if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) { in prepareUREMEqFold()
6764 SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true, in prepareUREMEqFold()
6766 return DAG.getNode(ISD::VSELECT, DL, SETCCVT, TautologicalInvertedChannels, in prepareUREMEqFold()
6773 if (isOperationLegalOrCustom(ISD::XOR, SETCCVT)) in prepareUREMEqFold()
6774 return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC, in prepareUREMEqFold()
6787 ISD::CondCode Cond, in buildSREMEqFold()
6804 SDValue CompTargetNode, ISD::CondCode Cond, in prepareSREMEqFold()
6817 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in prepareSREMEqFold()
6829 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT)) in prepareSREMEqFold()
6929 if (!ISD::matchUnaryPredicate(D, BuildSREMPattern)) in prepareSREMEqFold()
6942 if (D.getOpcode() == ISD::BUILD_VECTOR) { in prepareSREMEqFold()
6961 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) { in prepareSREMEqFold()
6979 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal); in prepareSREMEqFold()
6984 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ADD, VT)) in prepareSREMEqFold()
6988 Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal); in prepareSREMEqFold()
6996 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT)) in prepareSREMEqFold()
6999 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal); in prepareSREMEqFold()
7006 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT)); in prepareSREMEqFold()
7020 if (!isOperationLegalOrCustom(ISD::SETCC, SETCCVT) || in prepareSREMEqFold()
7021 !isOperationLegalOrCustom(ISD::AND, VT) || in prepareSREMEqFold()
7023 !isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) in prepareSREMEqFold()
7036 SDValue DivisorIsIntMin = DAG.getSetCC(DL, SETCCVT, D, IntMin, ISD::SETEQ); in prepareSREMEqFold()
7040 SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax); in prepareSREMEqFold()
7049 SDValue Blended = DAG.getNode(ISD::VSELECT, DL, SETCCVT, DivisorIsIntMin, in prepareSREMEqFold()
7078 return DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ); in getSqrtInputTest()
7087 SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op); in getSqrtInputTest()
7088 return DAG.getSetCC(DL, CCVT, Fabs, NormC, ISD::SETLT); in getSqrtInputTest()
7096 if (Op.getOpcode() == ISD::FNEG || Op.getOpcode() == ISD::VP_FNEG) { in getNegatedExpression()
7113 if (!Op.hasOneUse() && Opcode != ISD::ConstantFP) { in getNegatedExpression()
7114 bool IsFreeExtend = Opcode == ISD::FP_EXTEND && in getNegatedExpression()
7133 case ISD::ConstantFP: { in getNegatedExpression()
7137 isOperationLegal(ISD::ConstantFP, VT) || in getNegatedExpression()
7155 case ISD::BUILD_VECTOR: { in getNegatedExpression()
7163 (isOperationLegal(ISD::ConstantFP, VT) && in getNegatedExpression()
7164 isOperationLegal(ISD::BUILD_VECTOR, VT)) || in getNegatedExpression()
7187 case ISD::FADD: { in getNegatedExpression()
7192 if (LegalOps && !isOperationLegalOrCustom(ISD::FSUB, VT)) in getNegatedExpression()
7215 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegX, Y, Flags); in getNegatedExpression()
7224 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegY, X, Flags); in getNegatedExpression()
7231 case ISD::FSUB: { in getNegatedExpression()
7246 return DAG.getNode(ISD::FSUB, DL, VT, Y, X, Flags); in getNegatedExpression()
7248 case ISD::FMUL: in getNegatedExpression()
7249 case ISD::FDIV: { in getNegatedExpression()
7279 if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL) in getNegatedExpression()
7292 case ISD::FMA: in getNegatedExpression()
7293 case ISD::FMAD: { in getNegatedExpression()
7344 case ISD::FP_EXTEND: in getNegatedExpression()
7345 case ISD::FSIN: in getNegatedExpression()
7350 case ISD::FP_ROUND: in getNegatedExpression()
7353 return DAG.getNode(ISD::FP_ROUND, DL, VT, NegV, Op.getOperand(1)); in getNegatedExpression()
7355 case ISD::SELECT: in getNegatedExpression()
7356 case ISD::VSELECT: { in getNegatedExpression()
7405 assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI || in expandMUL_LOHI()
7406 Opcode == ISD::SMUL_LOHI); in expandMUL_LOHI()
7409 isOperationLegalOrCustom(ISD::MULHS, HiLoVT); in expandMUL_LOHI()
7411 isOperationLegalOrCustom(ISD::MULHU, HiLoVT); in expandMUL_LOHI()
7413 isOperationLegalOrCustom(ISD::SMUL_LOHI, HiLoVT); in expandMUL_LOHI()
7415 isOperationLegalOrCustom(ISD::UMUL_LOHI, HiLoVT); in expandMUL_LOHI()
7431 Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R); in expandMUL_LOHI()
7436 Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R); in expandMUL_LOHI()
7437 Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R); in expandMUL_LOHI()
7446 isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) { in expandMUL_LOHI()
7447 LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS); in expandMUL_LOHI()
7448 RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS); in expandMUL_LOHI()
7461 if (Opcode != ISD::MUL) { in expandMUL_LOHI()
7470 if (!VT.isVector() && Opcode == ISD::MUL && in expandMUL_LOHI()
7486 isOperationLegalOrCustom(ISD::SRL, VT) && in expandMUL_LOHI()
7487 isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) { in expandMUL_LOHI()
7488 LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift); in expandMUL_LOHI()
7489 LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH); in expandMUL_LOHI()
7490 RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift); in expandMUL_LOHI()
7491 RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH); in expandMUL_LOHI()
7502 if (Opcode == ISD::MUL) { in expandMUL_LOHI()
7503 RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH); in expandMUL_LOHI()
7504 LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL); in expandMUL_LOHI()
7505 Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH); in expandMUL_LOHI()
7506 Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH); in expandMUL_LOHI()
7513 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); in expandMUL_LOHI()
7514 Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi); in expandMUL_LOHI()
7515 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); in expandMUL_LOHI()
7516 return DAG.getNode(ISD::OR, dl, VT, Lo, Hi); in expandMUL_LOHI()
7519 SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi); in expandMUL_LOHI()
7525 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi)); in expandMUL_LOHI()
7533 bool UseGlue = (isOperationLegalOrCustom(ISD::ADDC, VT) && in expandMUL_LOHI()
7534 isOperationLegalOrCustom(ISD::ADDE, VT)); in expandMUL_LOHI()
7536 Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next, in expandMUL_LOHI()
7539 Next = DAG.getNode(ISD::UADDO_CARRY, dl, DAG.getVTList(VT, BoolType), Next, in expandMUL_LOHI()
7543 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next)); in expandMUL_LOHI()
7544 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift); in expandMUL_LOHI()
7546 if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI)) in expandMUL_LOHI()
7550 Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero, in expandMUL_LOHI()
7553 Hi = DAG.getNode(ISD::UADDO_CARRY, dl, DAG.getVTList(HiLoVT, BoolType), Hi, in expandMUL_LOHI()
7556 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi)); in expandMUL_LOHI()
7558 if (Opcode == ISD::SMUL_LOHI) { in expandMUL_LOHI()
7559 SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next, in expandMUL_LOHI()
7560 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL)); in expandMUL_LOHI()
7561 Next = DAG.getSelectCC(dl, LH, Zero, NextSub, Next, ISD::SETLT); in expandMUL_LOHI()
7563 NextSub = DAG.getNode(ISD::SUB, dl, VT, Next, in expandMUL_LOHI()
7564 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL)); in expandMUL_LOHI()
7565 Next = DAG.getSelectCC(dl, RH, Zero, NextSub, Next, ISD::SETLT); in expandMUL_LOHI()
7568 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next)); in expandMUL_LOHI()
7569 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift); in expandMUL_LOHI()
7570 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next)); in expandMUL_LOHI()
7619 if (Opcode == ISD::SREM || Opcode == ISD::SDIV || Opcode == ISD::SDIVREM) in expandDIVREMByConstant()
7622 (Opcode == ISD::UREM || Opcode == ISD::UDIV || Opcode == ISD::UDIVREM) && in expandDIVREMByConstant()
7642 if (!isOperationLegalOrCustom(ISD::MULHU, HiLoVT) && in expandDIVREMByConstant()
7643 !isOperationLegalOrCustom(ISD::UMUL_LOHI, HiLoVT)) in expandDIVREMByConstant()
7678 if (Opcode != ISD::UDIV) { in expandDIVREMByConstant()
7680 PartialRem = DAG.getNode(ISD::AND, dl, HiLoVT, LL, in expandDIVREMByConstant()
7685 ISD::OR, dl, HiLoVT, in expandDIVREMByConstant()
7686 DAG.getNode(ISD::SRL, dl, HiLoVT, LL, in expandDIVREMByConstant()
7688 DAG.getNode(ISD::SHL, dl, HiLoVT, LH, in expandDIVREMByConstant()
7691 LH = DAG.getNode(ISD::SRL, dl, HiLoVT, LH, in expandDIVREMByConstant()
7698 if (isOperationLegalOrCustom(ISD::UADDO_CARRY, HiLoVT)) { in expandDIVREMByConstant()
7700 Sum = DAG.getNode(ISD::UADDO, dl, VTList, LL, LH); in expandDIVREMByConstant()
7701 Sum = DAG.getNode(ISD::UADDO_CARRY, dl, VTList, Sum, in expandDIVREMByConstant()
7704 Sum = DAG.getNode(ISD::ADD, dl, HiLoVT, LL, LH); in expandDIVREMByConstant()
7705 SDValue Carry = DAG.getSetCC(dl, SetCCType, Sum, LL, ISD::SETULT); in expandDIVREMByConstant()
7714 Sum = DAG.getNode(ISD::ADD, dl, HiLoVT, Sum, Carry); in expandDIVREMByConstant()
7724 DAG.getNode(ISD::UREM, dl, HiLoVT, Sum, in expandDIVREMByConstant()
7728 if (Opcode != ISD::UREM) { in expandDIVREMByConstant()
7730 SDValue Dividend = DAG.getNode(ISD::BUILD_PAIR, dl, VT, LL, LH); in expandDIVREMByConstant()
7731 SDValue Rem = DAG.getNode(ISD::BUILD_PAIR, dl, VT, RemL, RemH); in expandDIVREMByConstant()
7733 Dividend = DAG.getNode(ISD::SUB, dl, VT, Dividend, Rem); in expandDIVREMByConstant()
7742 SDValue Quotient = DAG.getNode(ISD::MUL, dl, VT, Dividend, in expandDIVREMByConstant()
7752 if (Opcode != ISD::UDIV) { in expandDIVREMByConstant()
7757 RemL = DAG.getNode(ISD::SHL, dl, HiLoVT, RemL, in expandDIVREMByConstant()
7759 RemL = DAG.getNode(ISD::ADD, dl, HiLoVT, RemL, PartialRem); in expandDIVREMByConstant()
7770 return ISD::matchUnaryPredicate( in isNonZeroModBitWidthOrUndef()
7787 bool IsFSHL = Node->getOpcode() == ISD::VP_FSHL; in expandVPFunnelShift()
7796 ShAmt = DAG.getNode(ISD::VP_UREM, DL, ShVT, Z, BitWidthC, Mask, VL); in expandVPFunnelShift()
7797 InvShAmt = DAG.getNode(ISD::VP_SUB, DL, ShVT, BitWidthC, ShAmt, Mask, VL); in expandVPFunnelShift()
7798 ShX = DAG.getNode(ISD::VP_SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt, Mask, in expandVPFunnelShift()
7800 ShY = DAG.getNode(ISD::VP_LSHR, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt, Mask, in expandVPFunnelShift()
7808 ShAmt = DAG.getNode(ISD::VP_AND, DL, ShVT, Z, BitMask, Mask, VL); in expandVPFunnelShift()
7810 SDValue NotZ = DAG.getNode(ISD::VP_XOR, DL, ShVT, Z, in expandVPFunnelShift()
7812 InvShAmt = DAG.getNode(ISD::VP_AND, DL, ShVT, NotZ, BitMask, Mask, VL); in expandVPFunnelShift()
7815 ShAmt = DAG.getNode(ISD::VP_UREM, DL, ShVT, Z, BitWidthC, Mask, VL); in expandVPFunnelShift()
7816 InvShAmt = DAG.getNode(ISD::VP_SUB, DL, ShVT, BitMask, ShAmt, Mask, VL); in expandVPFunnelShift()
7821 ShX = DAG.getNode(ISD::VP_SHL, DL, VT, X, ShAmt, Mask, VL); in expandVPFunnelShift()
7822 SDValue ShY1 = DAG.getNode(ISD::VP_LSHR, DL, VT, Y, One, Mask, VL); in expandVPFunnelShift()
7823 ShY = DAG.getNode(ISD::VP_LSHR, DL, VT, ShY1, InvShAmt, Mask, VL); in expandVPFunnelShift()
7825 SDValue ShX1 = DAG.getNode(ISD::VP_SHL, DL, VT, X, One, Mask, VL); in expandVPFunnelShift()
7826 ShX = DAG.getNode(ISD::VP_SHL, DL, VT, ShX1, InvShAmt, Mask, VL); in expandVPFunnelShift()
7827 ShY = DAG.getNode(ISD::VP_LSHR, DL, VT, Y, ShAmt, Mask, VL); in expandVPFunnelShift()
7830 return DAG.getNode(ISD::VP_OR, DL, VT, ShX, ShY, Mask, VL); in expandVPFunnelShift()
7840 if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) || in expandFunnelShift()
7841 !isOperationLegalOrCustom(ISD::SRL, VT) || in expandFunnelShift()
7842 !isOperationLegalOrCustom(ISD::SUB, VT) || in expandFunnelShift()
7843 !isOperationLegalOrCustomOrPromote(ISD::OR, VT))) in expandFunnelShift()
7851 bool IsFSHL = Node->getOpcode() == ISD::FSHL; in expandFunnelShift()
7857 unsigned RevOpcode = IsFSHL ? ISD::FSHR : ISD::FSHL; in expandFunnelShift()
7864 Z = DAG.getNode(ISD::SUB, DL, VT, Zero, Z); in expandFunnelShift()
7871 X = DAG.getNode(ISD::SRL, DL, VT, X, One); in expandFunnelShift()
7874 Y = DAG.getNode(ISD::SHL, DL, VT, Y, One); in expandFunnelShift()
7888 ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC); in expandFunnelShift()
7889 InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, ShAmt); in expandFunnelShift()
7890 ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt); in expandFunnelShift()
7891 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt); in expandFunnelShift()
7898 ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Z, Mask); in expandFunnelShift()
7900 InvShAmt = DAG.getNode(ISD::AND, DL, ShVT, DAG.getNOT(DL, Z, ShVT), Mask); in expandFunnelShift()
7903 ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC); in expandFunnelShift()
7904 InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, Mask, ShAmt); in expandFunnelShift()
7909 ShX = DAG.getNode(ISD::SHL, DL, VT, X, ShAmt); in expandFunnelShift()
7910 SDValue ShY1 = DAG.getNode(ISD::SRL, DL, VT, Y, One); in expandFunnelShift()
7911 ShY = DAG.getNode(ISD::SRL, DL, VT, ShY1, InvShAmt); in expandFunnelShift()
7913 SDValue ShX1 = DAG.getNode(ISD::SHL, DL, VT, X, One); in expandFunnelShift()
7914 ShX = DAG.getNode(ISD::SHL, DL, VT, ShX1, InvShAmt); in expandFunnelShift()
7915 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt); in expandFunnelShift()
7918 return DAG.getNode(ISD::OR, DL, VT, ShX, ShY); in expandFunnelShift()
7926 bool IsLeft = Node->getOpcode() == ISD::ROTL; in expandROT()
7935 unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL; in expandROT()
7938 SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1); in expandROT()
7943 (!isOperationLegalOrCustom(ISD::SHL, VT) || in expandROT()
7944 !isOperationLegalOrCustom(ISD::SRL, VT) || in expandROT()
7945 !isOperationLegalOrCustom(ISD::SUB, VT) || in expandROT()
7946 !isOperationLegalOrCustomOrPromote(ISD::OR, VT) || in expandROT()
7947 !isOperationLegalOrCustomOrPromote(ISD::AND, VT))) in expandROT()
7950 unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL; in expandROT()
7951 unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL; in expandROT()
7958 SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1); in expandROT()
7959 SDValue ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Op1, BitWidthMinusOneC); in expandROT()
7961 SDValue HsAmt = DAG.getNode(ISD::AND, DL, ShVT, NegOp1, BitWidthMinusOneC); in expandROT()
7967 SDValue ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Op1, BitWidthC); in expandROT()
7969 SDValue HsAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthMinusOneC, ShAmt); in expandROT()
7974 return DAG.getNode(ISD::OR, DL, VT, ShVal, HsVal); in expandROT()
7984 bool IsSHL = Node->getOpcode() == ISD::SHL_PARTS; in expandShiftParts()
7985 bool IsSRA = Node->getOpcode() == ISD::SRA_PARTS; in expandShiftParts()
7997 SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt, in expandShiftParts()
7999 SDValue Tmp1 = IsSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi, in expandShiftParts()
8005 Tmp2 = DAG.getNode(ISD::FSHL, dl, VT, ShOpHi, ShOpLo, ShAmt); in expandShiftParts()
8006 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt); in expandShiftParts()
8008 Tmp2 = DAG.getNode(ISD::FSHR, dl, VT, ShOpHi, ShOpLo, ShAmt); in expandShiftParts()
8009 Tmp3 = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt); in expandShiftParts()
8015 SDValue AndNode = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt, in expandShiftParts()
8018 DAG.getConstant(0, dl, ShAmtVT), ISD::SETNE); in expandShiftParts()
8021 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2); in expandShiftParts()
8022 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3); in expandShiftParts()
8024 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2); in expandShiftParts()
8025 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3); in expandShiftParts()
8062 SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Src); in expandFP_TO_SINT()
8065 ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask), in expandFP_TO_SINT()
8067 SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias); in expandFP_TO_SINT()
8069 SDValue Sign = DAG.getNode(ISD::SRA, dl, IntVT, in expandFP_TO_SINT()
8070 DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask), in expandFP_TO_SINT()
8074 SDValue R = DAG.getNode(ISD::OR, dl, IntVT, in expandFP_TO_SINT()
8075 DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask), in expandFP_TO_SINT()
8082 DAG.getNode(ISD::SHL, dl, DstVT, R, in expandFP_TO_SINT()
8084 DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit), in expandFP_TO_SINT()
8086 DAG.getNode(ISD::SRL, dl, DstVT, R, in expandFP_TO_SINT()
8088 DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent), in expandFP_TO_SINT()
8090 ISD::SETGT); in expandFP_TO_SINT()
8092 SDValue Ret = DAG.getNode(ISD::SUB, dl, DstVT, in expandFP_TO_SINT()
8093 DAG.getNode(ISD::XOR, dl, DstVT, R, Sign), Sign); in expandFP_TO_SINT()
8096 DAG.getConstant(0, dl, DstVT), Ret, ISD::SETLT); in expandFP_TO_SINT()
8115 unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT : in expandFP_TO_UINT()
8116 ISD::FP_TO_SINT; in expandFP_TO_UINT()
8118 !isOperationLegalOrCustomOrPromote(ISD::XOR, SrcVT))) in expandFP_TO_UINT()
8130 Result = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other }, in expandFP_TO_UINT()
8134 Result = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src); in expandFP_TO_UINT()
8140 Node->isStrictFPOpcode() ? ISD::STRICT_FSUB : ISD::FSUB, SrcVT)) in expandFP_TO_UINT()
8147 Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT, in expandFP_TO_UINT()
8151 Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT); in expandFP_TO_UINT()
8174 SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl, { SrcVT, MVT::Other }, in expandFP_TO_UINT()
8176 SInt = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other }, in expandFP_TO_UINT()
8180 SDValue Val = DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FltOfs); in expandFP_TO_UINT()
8181 SInt = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Val); in expandFP_TO_UINT()
8183 Result = DAG.getNode(ISD::XOR, dl, DstVT, SInt, IntOfs); in expandFP_TO_UINT()
8190 SDValue True = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src); in expandFP_TO_UINT()
8192 SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, in expandFP_TO_UINT()
8193 DAG.getNode(ISD::FSUB, dl, SrcVT, Src, Cst)); in expandFP_TO_UINT()
8194 False = DAG.getNode(ISD::XOR, dl, DstVT, False, in expandFP_TO_UINT()
8219 if (SrcVT.isVector() && (!isOperationLegalOrCustom(ISD::SRL, SrcVT) || in expandUINT_TO_FP()
8220 !isOperationLegalOrCustom(ISD::FADD, DstVT) || in expandUINT_TO_FP()
8221 !isOperationLegalOrCustom(ISD::FSUB, DstVT) || in expandUINT_TO_FP()
8222 !isOperationLegalOrCustomOrPromote(ISD::OR, SrcVT) || in expandUINT_TO_FP()
8223 !isOperationLegalOrCustomOrPromote(ISD::AND, SrcVT))) in expandUINT_TO_FP()
8241 SDValue Lo = DAG.getNode(ISD::AND, dl, SrcVT, Src, LoMask); in expandUINT_TO_FP()
8242 SDValue Hi = DAG.getNode(ISD::SRL, dl, SrcVT, Src, HiShift); in expandUINT_TO_FP()
8243 SDValue LoOr = DAG.getNode(ISD::OR, dl, SrcVT, Lo, TwoP52); in expandUINT_TO_FP()
8244 SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84); in expandUINT_TO_FP()
8248 DAG.getNode(ISD::FSUB, dl, DstVT, HiFlt, TwoP84PlusTwoP52); in expandUINT_TO_FP()
8249 Result = DAG.getNode(ISD::FADD, dl, DstVT, LoFlt, HiSub); in expandUINT_TO_FP()
8257 assert((Opcode == ISD::FMINNUM || Opcode == ISD::FMAXNUM || in createSelectForFMINNUM_FMAXNUM()
8258 Opcode == ISD::STRICT_FMINNUM || Opcode == ISD::STRICT_FMAXNUM) && in createSelectForFMINNUM_FMAXNUM()
8262 ISD::CondCode Pred = Opcode == ISD::FMINNUM ? ISD::SETLT : ISD::SETGT; in createSelectForFMINNUM_FMAXNUM()
8280 unsigned NewOp = Node->getOpcode() == ISD::FMINNUM ? in expandFMINNUM_FMAXNUM()
8281 ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE; in expandFMINNUM_FMAXNUM()
8296 Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0, in expandFMINNUM_FMAXNUM()
8300 Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1, in expandFMINNUM_FMAXNUM()
8318 Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM; in expandFMINNUM_FMAXNUM()
8370 Op = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::f64, Op, in expandIS_FPCLASS()
8392 isOperationLegalOrCustom(ISD::SETCC, OperandVT.getScalarType())) { in expandIS_FPCLASS()
8393 ISD::CondCode OrderedCmpOpcode = IsInverted ? ISD::SETUNE : ISD::SETOEQ; in expandIS_FPCLASS()
8394 ISD::CondCode UnorderedCmpOpcode = IsInverted ? ISD::SETONE : ISD::SETUEQ; in expandIS_FPCLASS()
8410 isCondCodeLegalOrCustom(IsInverted ? ISD::SETO : ISD::SETUO, in expandIS_FPCLASS()
8413 IsInverted ? ISD::SETO : ISD::SETUO); in expandIS_FPCLASS()
8417 isCondCodeLegalOrCustom(IsInverted ? ISD::SETUNE : ISD::SETOEQ, in expandIS_FPCLASS()
8419 isOperationLegalOrCustom(ISD::FABS, OperandVT.getScalarType())) { in expandIS_FPCLASS()
8421 SDValue Abs = DAG.getNode(ISD::FABS, DL, OperandVT, Op); in expandIS_FPCLASS()
8425 IsInverted ? ISD::SETUNE : ISD::SETOEQ); in expandIS_FPCLASS()
8461 Res = DAG.getNode(ISD::OR, DL, ResultVT, Res, PartialRes); in expandIS_FPCLASS()
8473 SDValue IntBitV = DAG.getNode(ISD::AND, DL, IntVT, OpAsInt, IntBitMaskV); in expandIS_FPCLASS()
8474 IntBitIsSetV = DAG.getSetCC(DL, ResultVT, IntBitV, ZeroV, ISD::SETNE); in expandIS_FPCLASS()
8480 SDValue AbsV = DAG.getNode(ISD::AND, DL, IntVT, OpAsInt, ValueMaskV); in expandIS_FPCLASS()
8482 DAG.getConstant(0.0, DL, IntVT), ISD::SETLT); in expandIS_FPCLASS()
8492 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, ExpMaskV, ISD::SETLT); in expandIS_FPCLASS()
8496 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, ExpMaskV, ISD::SETULT); in expandIS_FPCLASS()
8500 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, ExpMaskV, ISD::SETLT); in expandIS_FPCLASS()
8501 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, SignV); in expandIS_FPCLASS()
8510 SDValue ExpBits = DAG.getNode(ISD::AND, DL, IntVT, OpAsInt, ExpMaskV); in expandIS_FPCLASS()
8512 DAG.getSetCC(DL, ResultVT, ExpBits, ZeroV, ISD::SETEQ); in expandIS_FPCLASS()
8522 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, ZeroV, ISD::SETEQ); in expandIS_FPCLASS()
8524 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, ZeroV, ISD::SETEQ); in expandIS_FPCLASS()
8526 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, SignBitV, ISD::SETEQ); in expandIS_FPCLASS()
8536 DAG.getNode(ISD::SUB, DL, IntVT, V, DAG.getConstant(1, DL, IntVT)); in expandIS_FPCLASS()
8537 PartialRes = DAG.getSetCC(DL, ResultVT, VMinusOneV, MantissaV, ISD::SETULT); in expandIS_FPCLASS()
8539 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, SignV); in expandIS_FPCLASS()
8545 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, InfV, ISD::SETEQ); in expandIS_FPCLASS()
8547 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, InfV, ISD::SETEQ); in expandIS_FPCLASS()
8551 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, NegInfV, ISD::SETEQ); in expandIS_FPCLASS()
8561 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, InfV, ISD::SETGT); in expandIS_FPCLASS()
8565 SDValue ExpBits = DAG.getNode(ISD::AND, DL, IntVT, AbsV, ExpMaskV); in expandIS_FPCLASS()
8567 DAG.getSetCC(DL, ResultVT, ExpBits, ZeroV, ISD::SETEQ); in expandIS_FPCLASS()
8569 DAG.getSetCC(DL, ResultVT, getIntBitIsSet(), ExpIsZero, ISD::SETEQ); in expandIS_FPCLASS()
8570 PartialRes = DAG.getNode(ISD::OR, DL, ResultVT, PartialRes, IsPseudo); in expandIS_FPCLASS()
8575 DAG.getSetCC(DL, ResultVT, AbsV, InfWithQnanBitV, ISD::SETGE); in expandIS_FPCLASS()
8579 SDValue IsNan = DAG.getSetCC(DL, ResultVT, AbsV, InfV, ISD::SETGT); in expandIS_FPCLASS()
8581 DAG.getSetCC(DL, ResultVT, AbsV, InfWithQnanBitV, ISD::SETLT); in expandIS_FPCLASS()
8582 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, IsNan, IsNotQnan); in expandIS_FPCLASS()
8591 SDValue ExpMinus1 = DAG.getNode(ISD::SUB, DL, IntVT, AbsV, ExpLSBV); in expandIS_FPCLASS()
8594 PartialRes = DAG.getSetCC(DL, ResultVT, ExpMinus1, ExpLimitV, ISD::SETULT); in expandIS_FPCLASS()
8596 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, SignV); in expandIS_FPCLASS()
8599 DAG.getNode(ISD::XOR, DL, ResultVT, SignV, ResultInvertionMask); in expandIS_FPCLASS()
8600 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, PosSignV); in expandIS_FPCLASS()
8604 DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, getIntBitIsSet()); in expandIS_FPCLASS()
8611 Res = DAG.getNode(ISD::XOR, DL, ResultVT, Res, ResultInvertionMask); in expandIS_FPCLASS()
8619 return TLI.isOperationLegalOrCustom(ISD::ADD, VT) && in canExpandVectorCTPOP()
8620 TLI.isOperationLegalOrCustom(ISD::SUB, VT) && in canExpandVectorCTPOP()
8621 TLI.isOperationLegalOrCustom(ISD::SRL, VT) && in canExpandVectorCTPOP()
8622 (Len == 8 || TLI.isOperationLegalOrCustom(ISD::MUL, VT)) && in canExpandVectorCTPOP()
8623 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT); in canExpandVectorCTPOP()
8652 Op = DAG.getNode(ISD::SUB, dl, VT, Op, in expandCTPOP()
8653 DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8654 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8658 Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, dl, VT, Op, Mask33), in expandCTPOP()
8659 DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8660 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8664 Op = DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8665 DAG.getNode(ISD::ADD, dl, VT, Op, in expandCTPOP()
8666 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8678 return DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8679 DAG.getNode(ISD::ADD, dl, VT, Op, in expandCTPOP()
8680 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8688 return DAG.getNode(ISD::SRL, dl, VT, in expandCTPOP()
8689 DAG.getNode(ISD::MUL, dl, VT, Op, Mask01), in expandCTPOP()
8719 Tmp1 = DAG.getNode(ISD::VP_AND, dl, VT, in expandVPCTPOP()
8720 DAG.getNode(ISD::VP_LSHR, dl, VT, Op, in expandVPCTPOP()
8723 Op = DAG.getNode(ISD::VP_SUB, dl, VT, Op, Tmp1, Mask, VL); in expandVPCTPOP()
8726 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Op, Mask33, Mask, VL); in expandVPCTPOP()
8727 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, in expandVPCTPOP()
8728 DAG.getNode(ISD::VP_LSHR, dl, VT, Op, in expandVPCTPOP()
8731 Op = DAG.getNode(ISD::VP_ADD, dl, VT, Tmp2, Tmp3, Mask, VL); in expandVPCTPOP()
8734 Tmp4 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(4, dl, ShVT), in expandVPCTPOP()
8736 Tmp5 = DAG.getNode(ISD::VP_ADD, dl, VT, Op, Tmp4, Mask, VL); in expandVPCTPOP()
8737 Op = DAG.getNode(ISD::VP_AND, dl, VT, Tmp5, Mask0F, Mask, VL); in expandVPCTPOP()
8745 return DAG.getNode(ISD::VP_LSHR, dl, VT, in expandVPCTPOP()
8746 DAG.getNode(ISD::VP_MUL, dl, VT, Op, Mask01, Mask, VL), in expandVPCTPOP()
8758 if (Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF && in expandCTLZ()
8759 isOperationLegalOrCustom(ISD::CTLZ, VT)) in expandCTLZ()
8760 return DAG.getNode(ISD::CTLZ, dl, VT, Op); in expandCTLZ()
8763 if (isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) { in expandCTLZ()
8766 SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op); in expandCTLZ()
8768 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ); in expandCTLZ()
8776 (!isOperationLegalOrCustom(ISD::CTPOP, VT) && in expandCTLZ()
8778 !isOperationLegalOrCustom(ISD::SRL, VT) || in expandCTLZ()
8779 !isOperationLegalOrCustomOrPromote(ISD::OR, VT))) in expandCTLZ()
8793 Op = DAG.getNode(ISD::OR, dl, VT, Op, in expandCTLZ()
8794 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp)); in expandCTLZ()
8797 return DAG.getNode(ISD::CTPOP, dl, VT, Op); in expandCTLZ()
8818 Op = DAG.getNode(ISD::VP_OR, dl, VT, Op, in expandVPCTLZ()
8819 DAG.getNode(ISD::VP_LSHR, dl, VT, Op, Tmp, Mask, VL), Mask, in expandVPCTLZ()
8822 Op = DAG.getNode(ISD::VP_XOR, dl, VT, Op, DAG.getConstant(-1, dl, VT), Mask, in expandVPCTLZ()
8824 return DAG.getNode(ISD::VP_CTPOP, dl, VT, Op, Mask, VL); in expandVPCTLZ()
8838 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op); in CTTZTableLookup()
8840 ISD::SRL, DL, VT, in CTTZTableLookup()
8841 DAG.getNode(ISD::MUL, DL, VT, DAG.getNode(ISD::AND, DL, VT, Op, Neg), in CTTZTableLookup()
8857 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, DL, VT, DAG.getEntryNode(), in CTTZTableLookup()
8860 if (Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF) in CTTZTableLookup()
8866 SDValue SrcIsZero = DAG.getSetCC(DL, SetCCVT, Op, Zero, ISD::SETEQ); in CTTZTableLookup()
8878 if (Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF && in expandCTTZ()
8879 isOperationLegalOrCustom(ISD::CTTZ, VT)) in expandCTTZ()
8880 return DAG.getNode(ISD::CTTZ, dl, VT, Op); in expandCTTZ()
8883 if (isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) { in expandCTTZ()
8886 SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op); in expandCTTZ()
8888 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ); in expandCTTZ()
8896 (!isOperationLegalOrCustom(ISD::CTPOP, VT) && in expandCTTZ()
8897 !isOperationLegalOrCustom(ISD::CTLZ, VT) && in expandCTTZ()
8899 !isOperationLegalOrCustom(ISD::SUB, VT) || in expandCTTZ()
8900 !isOperationLegalOrCustomOrPromote(ISD::AND, VT) || in expandCTTZ()
8901 !isOperationLegalOrCustomOrPromote(ISD::XOR, VT))) in expandCTTZ()
8905 if (!VT.isVector() && isOperationExpand(ISD::CTPOP, VT) && in expandCTTZ()
8906 !isOperationLegal(ISD::CTLZ, VT)) in expandCTTZ()
8915 ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT), in expandCTTZ()
8916 DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT))); in expandCTTZ()
8919 if (isOperationLegal(ISD::CTLZ, VT) && !isOperationLegal(ISD::CTPOP, VT)) { in expandCTTZ()
8920 return DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT), in expandCTTZ()
8921 DAG.getNode(ISD::CTLZ, dl, VT, Tmp)); in expandCTTZ()
8924 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp); in expandCTTZ()
8935 SDValue Not = DAG.getNode(ISD::VP_XOR, dl, VT, Op, in expandVPCTTZ()
8937 SDValue MinusOne = DAG.getNode(ISD::VP_SUB, dl, VT, Op, in expandVPCTTZ()
8939 SDValue Tmp = DAG.getNode(ISD::VP_AND, dl, VT, Not, MinusOne, Mask, VL); in expandVPCTTZ()
8940 return DAG.getNode(ISD::VP_CTPOP, dl, VT, Tmp, Mask, VL); in expandVPCTTZ()
8951 if (!IsNegative && isOperationLegal(ISD::SUB, VT) && in expandABS()
8952 isOperationLegal(ISD::SMAX, VT)) { in expandABS()
8954 return DAG.getNode(ISD::SMAX, dl, VT, Op, in expandABS()
8955 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
8959 if (!IsNegative && isOperationLegal(ISD::SUB, VT) && in expandABS()
8960 isOperationLegal(ISD::UMIN, VT)) { in expandABS()
8963 return DAG.getNode(ISD::UMIN, dl, VT, Op, in expandABS()
8964 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
8968 if (IsNegative && isOperationLegal(ISD::SUB, VT) && in expandABS()
8969 isOperationLegal(ISD::SMIN, VT)) { in expandABS()
8972 return DAG.getNode(ISD::SMIN, dl, VT, Op, in expandABS()
8973 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
8978 (!isOperationLegalOrCustom(ISD::SRA, VT) || in expandABS()
8979 (!IsNegative && !isOperationLegalOrCustom(ISD::ADD, VT)) || in expandABS()
8980 (IsNegative && !isOperationLegalOrCustom(ISD::SUB, VT)) || in expandABS()
8981 !isOperationLegalOrCustomOrPromote(ISD::XOR, VT))) in expandABS()
8986 DAG.getNode(ISD::SRA, dl, VT, Op, in expandABS()
8988 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, Shift); in expandABS()
8992 return DAG.getNode(ISD::SUB, dl, VT, Xor, Shift); in expandABS()
8995 return DAG.getNode(ISD::SUB, dl, VT, Shift, Xor); in expandABS()
9003 bool IsSigned = N->getOpcode() == ISD::ABDS; in expandABD()
9007 unsigned MaxOpc = IsSigned ? ISD::SMAX : ISD::UMAX; in expandABD()
9008 unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN; in expandABD()
9012 return DAG.getNode(ISD::SUB, dl, VT, Max, Min); in expandABD()
9016 if (!IsSigned && isOperationLegal(ISD::USUBSAT, VT)) in expandABD()
9017 return DAG.getNode(ISD::OR, dl, VT, in expandABD()
9018 DAG.getNode(ISD::USUBSAT, dl, VT, LHS, RHS), in expandABD()
9019 DAG.getNode(ISD::USUBSAT, dl, VT, RHS, LHS)); in expandABD()
9024 ISD::CondCode CC = IsSigned ? ISD::CondCode::SETGT : ISD::CondCode::SETUGT; in expandABD()
9026 return DAG.getSelect(dl, VT, Cmp, DAG.getNode(ISD::SUB, dl, VT, LHS, RHS), in expandABD()
9027 DAG.getNode(ISD::SUB, dl, VT, RHS, LHS)); in expandABD()
9045 return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9047 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9048 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9050 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9051 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9052 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT)); in expandBSWAP()
9053 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9054 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); in expandBSWAP()
9055 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); in expandBSWAP()
9056 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); in expandBSWAP()
9058 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); in expandBSWAP()
9059 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9061 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Tmp7, DAG.getConstant(40, dl, SHVT)); in expandBSWAP()
9062 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9064 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Tmp6, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9065 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9067 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Tmp5, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9068 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9069 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, in expandBSWAP()
9071 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9072 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, in expandBSWAP()
9074 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); in expandBSWAP()
9075 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, in expandBSWAP()
9077 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); in expandBSWAP()
9078 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); in expandBSWAP()
9079 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); in expandBSWAP()
9080 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); in expandBSWAP()
9081 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); in expandBSWAP()
9082 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); in expandBSWAP()
9083 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); in expandBSWAP()
9084 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); in expandBSWAP()
9104 Tmp1 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9106 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9108 return DAG.getNode(ISD::VP_OR, dl, VT, Tmp1, Tmp2, Mask, EVL); in expandVPBSWAP()
9110 Tmp4 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9112 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Op, DAG.getConstant(0xFF00, dl, VT), in expandVPBSWAP()
9114 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9116 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9118 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBSWAP()
9120 Tmp1 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9122 Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp3, Mask, EVL); in expandVPBSWAP()
9123 Tmp2 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp1, Mask, EVL); in expandVPBSWAP()
9124 return DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp2, Mask, EVL); in expandVPBSWAP()
9126 Tmp8 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT), in expandVPBSWAP()
9128 Tmp7 = DAG.getNode(ISD::VP_AND, dl, VT, Op, in expandVPBSWAP()
9130 Tmp7 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp7, DAG.getConstant(40, dl, SHVT), in expandVPBSWAP()
9132 Tmp6 = DAG.getNode(ISD::VP_AND, dl, VT, Op, in expandVPBSWAP()
9134 Tmp6 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp6, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9136 Tmp5 = DAG.getNode(ISD::VP_AND, dl, VT, Op, in expandVPBSWAP()
9138 Tmp5 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp5, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9140 Tmp4 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9142 Tmp4 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp4, in expandVPBSWAP()
9144 Tmp3 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9146 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp3, in expandVPBSWAP()
9148 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(40, dl, SHVT), in expandVPBSWAP()
9150 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBSWAP()
9152 Tmp1 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(56, dl, SHVT), in expandVPBSWAP()
9154 Tmp8 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp7, Mask, EVL); in expandVPBSWAP()
9155 Tmp6 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp6, Tmp5, Mask, EVL); in expandVPBSWAP()
9156 Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp3, Mask, EVL); in expandVPBSWAP()
9157 Tmp2 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp1, Mask, EVL); in expandVPBSWAP()
9158 Tmp8 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp6, Mask, EVL); in expandVPBSWAP()
9159 Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp2, Mask, EVL); in expandVPBSWAP()
9160 return DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp4, Mask, EVL); in expandVPBSWAP()
9183 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op); in expandBITREVERSE()
9186 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT)); in expandBITREVERSE()
9187 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask4, dl, VT)); in expandBITREVERSE()
9188 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT)); in expandBITREVERSE()
9189 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT)); in expandBITREVERSE()
9190 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
9193 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT)); in expandBITREVERSE()
9194 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask2, dl, VT)); in expandBITREVERSE()
9195 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT)); in expandBITREVERSE()
9196 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT)); in expandBITREVERSE()
9197 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
9200 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT)); in expandBITREVERSE()
9201 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask1, dl, VT)); in expandBITREVERSE()
9202 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT)); in expandBITREVERSE()
9203 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT)); in expandBITREVERSE()
9204 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
9212 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT)); in expandBITREVERSE()
9215 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT)); in expandBITREVERSE()
9218 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT)); in expandBITREVERSE()
9219 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2); in expandBITREVERSE()
9226 assert(N->getOpcode() == ISD::VP_BITREVERSE); in expandVPBITREVERSE()
9248 Tmp = (Sz > 8 ? DAG.getNode(ISD::VP_BSWAP, dl, VT, Op, Mask, EVL) : Op); in expandVPBITREVERSE()
9251 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT), in expandVPBITREVERSE()
9253 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBITREVERSE()
9255 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT), in expandVPBITREVERSE()
9257 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT), in expandVPBITREVERSE()
9259 Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL); in expandVPBITREVERSE()
9262 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT), in expandVPBITREVERSE()
9264 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBITREVERSE()
9266 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT), in expandVPBITREVERSE()
9268 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT), in expandVPBITREVERSE()
9270 Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL); in expandVPBITREVERSE()
9273 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT), in expandVPBITREVERSE()
9275 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBITREVERSE()
9277 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT), in expandVPBITREVERSE()
9279 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT), in expandVPBITREVERSE()
9281 Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL); in expandVPBITREVERSE()
9295 ISD::LoadExtType ExtType = LD->getExtensionType(); in scalarizeVectorLoad()
9325 DAG.getExtLoad(ISD::EXTLOAD, SL, LoadVT, Chain, BasePTR, in scalarizeVectorLoad()
9336 SDValue ShiftedElt = DAG.getNode(ISD::SRL, SL, LoadVT, Load, ShiftAmount); in scalarizeVectorLoad()
9338 DAG.getNode(ISD::AND, SL, LoadVT, ShiftedElt, SrcEltBitMask); in scalarizeVectorLoad()
9339 SDValue Scalar = DAG.getNode(ISD::TRUNCATE, SL, SrcEltVT, Elt); in scalarizeVectorLoad()
9341 if (ExtType != ISD::NON_EXTLOAD) { in scalarizeVectorLoad()
9342 unsigned ExtendOp = ISD::getExtForLoadExtType(false, ExtType); in scalarizeVectorLoad()
9372 SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains); in scalarizeVectorLoad()
9412 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value, in scalarizeVectorStore()
9414 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MemSclVT, Elt); in scalarizeVectorStore()
9415 SDValue ExtElt = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Trunc); in scalarizeVectorStore()
9421 DAG.getNode(ISD::SHL, SL, IntVT, ExtElt, ShiftAmount); in scalarizeVectorStore()
9422 CurrVal = DAG.getNode(ISD::OR, SL, IntVT, CurrVal, ShiftedElt); in scalarizeVectorStore()
9437 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value, in scalarizeVectorStore()
9452 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Stores); in scalarizeVectorStore()
9457 assert(LD->getAddressingMode() == ISD::UNINDEXED && in expandUnalignedLoad()
9469 if (!isOperationLegalOrCustom(ISD::LOAD, intVT) && in expandUnalignedLoad()
9479 SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad); in expandUnalignedLoad()
9481 Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND : in expandUnalignedLoad()
9482 ISD::ANY_EXTEND, dl, VT, Result); in expandUnalignedLoad()
9529 DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr, in expandUnalignedLoad()
9541 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); in expandUnalignedLoad()
9564 ISD::LoadExtType HiExtType = LD->getExtensionType(); in expandUnalignedLoad()
9567 if (HiExtType == ISD::NON_EXTLOAD) in expandUnalignedLoad()
9568 HiExtType = ISD::ZEXTLOAD; in expandUnalignedLoad()
9573 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(), in expandUnalignedLoad()
9588 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, in expandUnalignedLoad()
9598 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount); in expandUnalignedLoad()
9599 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo); in expandUnalignedLoad()
9601 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), in expandUnalignedLoad()
9609 assert(ST->getAddressingMode() == ISD::UNINDEXED && in expandUnalignedStore()
9623 if (!isOperationLegalOrCustom(ISD::STORE, intVT) && in expandUnalignedStore()
9632 SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val); in expandUnalignedStore()
9688 ISD::EXTLOAD, dl, RegVT, Store, StackPtr, in expandUnalignedStore()
9697 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); in expandUnalignedStore()
9717 ISD::AND, dl, VT, Lo, in expandUnalignedStore()
9720 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount); in expandUnalignedStore()
9736 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); in expandUnalignedStore()
9758 MaskInIntReg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, MaskInIntReg); in IncrementMemoryAddress()
9763 Increment = DAG.getNode(ISD::CTPOP, DL, MaskIntVT, MaskInIntReg); in IncrementMemoryAddress()
9768 Increment = DAG.getNode(ISD::MUL, DL, AddrVT, Increment, Scale); in IncrementMemoryAddress()
9776 return DAG.getNode(ISD::ADD, DL, AddrVT, Addr, Increment); in IncrementMemoryAddress()
9798 unsigned SubOpcode = NumSubElts <= NElts ? ISD::SUB : ISD::USUBSAT; in clampDynamicVectorIndex()
9801 return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx, Sub); in clampDynamicVectorIndex()
9805 return DAG.getNode(ISD::AND, dl, IdxVT, Idx, in clampDynamicVectorIndex()
9809 return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx, in clampDynamicVectorIndex()
9844 DAG.getNode(ISD::MUL, dl, IdxVT, Index, in getVectorSubVecPointer()
9847 Index = DAG.getNode(ISD::MUL, dl, IdxVT, Index, in getVectorSubVecPointer()
9895 assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node."); in lowerCmpEqZeroToCtlzSrl()
9898 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); in lowerCmpEqZeroToCtlzSrl()
9900 if (isNullConstant(Op.getOperand(1)) && CC == ISD::SETEQ) { in lowerCmpEqZeroToCtlzSrl()
9905 Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0)); in lowerCmpEqZeroToCtlzSrl()
9908 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext); in lowerCmpEqZeroToCtlzSrl()
9909 SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz, in lowerCmpEqZeroToCtlzSrl()
9911 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc); in lowerCmpEqZeroToCtlzSrl()
9925 if (Opcode == ISD::UMAX && llvm::isOneOrOneSplat(Op1, true) && BoolVT == VT && in expandIntMINMAX()
9929 return DAG.getNode(ISD::SUB, DL, VT, Op0, in expandIntMINMAX()
9930 DAG.getSetCC(DL, VT, Op0, Zero, ISD::SETEQ)); in expandIntMINMAX()
9935 if (Opcode == ISD::UMIN && isOperationLegal(ISD::SUB, VT) && in expandIntMINMAX()
9936 isOperationLegal(ISD::USUBSAT, VT)) { in expandIntMINMAX()
9937 return DAG.getNode(ISD::SUB, DL, VT, Op0, in expandIntMINMAX()
9938 DAG.getNode(ISD::USUBSAT, DL, VT, Op0, Op1)); in expandIntMINMAX()
9943 if (Opcode == ISD::UMAX && isOperationLegal(ISD::ADD, VT) && in expandIntMINMAX()
9944 isOperationLegal(ISD::USUBSAT, VT)) { in expandIntMINMAX()
9945 return DAG.getNode(ISD::ADD, DL, VT, Op0, in expandIntMINMAX()
9946 DAG.getNode(ISD::USUBSAT, DL, VT, Op1, Op0)); in expandIntMINMAX()
9951 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT)) in expandIntMINMAX()
9957 auto buildMinMax = [&](ISD::CondCode PrefCC, ISD::CondCode AltCC, in expandIntMINMAX()
9958 ISD::CondCode PrefCommuteCC, in expandIntMINMAX()
9959 ISD::CondCode AltCommuteCC) { in expandIntMINMAX()
9961 for (ISD::CondCode CC : {PrefCC, AltCC}) { in expandIntMINMAX()
9962 if (DAG.doesNodeExist(ISD::SETCC, BoolVTList, in expandIntMINMAX()
9968 for (ISD::CondCode CC : {PrefCommuteCC, AltCommuteCC}) { in expandIntMINMAX()
9969 if (DAG.doesNodeExist(ISD::SETCC, BoolVTList, in expandIntMINMAX()
9984 case ISD::SMAX: in expandIntMINMAX()
9985 return buildMinMax(ISD::SETGT, ISD::SETGE, ISD::SETLT, ISD::SETLE); in expandIntMINMAX()
9986 case ISD::SMIN: in expandIntMINMAX()
9987 return buildMinMax(ISD::SETLT, ISD::SETLE, ISD::SETGT, ISD::SETGE); in expandIntMINMAX()
9988 case ISD::UMAX: in expandIntMINMAX()
9989 return buildMinMax(ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE); in expandIntMINMAX()
9990 case ISD::UMIN: in expandIntMINMAX()
9991 return buildMinMax(ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE); in expandIntMINMAX()
10008 if (Opcode == ISD::USUBSAT && isOperationLegal(ISD::UMAX, VT)) { in expandAddSubSat()
10009 SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS); in expandAddSubSat()
10010 return DAG.getNode(ISD::SUB, dl, VT, Max, RHS); in expandAddSubSat()
10014 if (Opcode == ISD::UADDSAT && isOperationLegal(ISD::UMIN, VT)) { in expandAddSubSat()
10016 SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS); in expandAddSubSat()
10017 return DAG.getNode(ISD::ADD, dl, VT, Min, RHS); in expandAddSubSat()
10022 case ISD::SADDSAT: in expandAddSubSat()
10023 OverflowOp = ISD::SADDO; in expandAddSubSat()
10025 case ISD::UADDSAT: in expandAddSubSat()
10026 OverflowOp = ISD::UADDO; in expandAddSubSat()
10028 case ISD::SSUBSAT: in expandAddSubSat()
10029 OverflowOp = ISD::SSUBO; in expandAddSubSat()
10031 case ISD::USUBSAT: in expandAddSubSat()
10032 OverflowOp = ISD::USUBO; in expandAddSubSat()
10041 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT)) in expandAddSubSat()
10052 if (Opcode == ISD::UADDSAT) { in expandAddSubSat()
10056 return DAG.getNode(ISD::OR, dl, VT, SumDiff, OverflowMask); in expandAddSubSat()
10062 if (Opcode == ISD::USUBSAT) { in expandAddSubSat()
10067 return DAG.getNode(ISD::AND, dl, VT, SumDiff, Not); in expandAddSubSat()
10073 if (Opcode == ISD::SADDSAT || Opcode == ISD::SSUBSAT) { in expandAddSubSat()
10088 bool RHSIsNonNegative = Opcode == ISD::SADDSAT ? KnownRHS.isNonNegative() in expandAddSubSat()
10096 bool RHSIsNegative = Opcode == ISD::SADDSAT ? KnownRHS.isNegative() in expandAddSubSat()
10107 SDValue Shift = DAG.getNode(ISD::SRA, dl, VT, SumDiff, in expandAddSubSat()
10109 Result = DAG.getNode(ISD::XOR, dl, VT, Shift, SatMin); in expandAddSubSat()
10115 bool IsSigned = Opcode == ISD::SSHLSAT; in expandShlSat()
10121 assert((Node->getOpcode() == ISD::SSHLSAT || in expandShlSat()
10122 Node->getOpcode() == ISD::USHLSAT) && in expandShlSat()
10127 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT)) in expandShlSat()
10134 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, LHS, RHS); in expandShlSat()
10136 DAG.getNode(IsSigned ? ISD::SRA : ISD::SRL, dl, VT, Result, RHS); in expandShlSat()
10143 DAG.getSetCC(dl, BoolVT, LHS, DAG.getConstant(0, dl, VT), ISD::SETLT); in expandShlSat()
10148 SDValue Cond = DAG.getSetCC(dl, BoolVT, LHS, Orig, ISD::SETNE); in expandShlSat()
10154 assert((Node->getOpcode() == ISD::SMULFIX || in expandFixedPointMul()
10155 Node->getOpcode() == ISD::UMULFIX || in expandFixedPointMul()
10156 Node->getOpcode() == ISD::SMULFIXSAT || in expandFixedPointMul()
10157 Node->getOpcode() == ISD::UMULFIXSAT) && in expandFixedPointMul()
10165 bool Saturating = (Node->getOpcode() == ISD::SMULFIXSAT || in expandFixedPointMul()
10166 Node->getOpcode() == ISD::UMULFIXSAT); in expandFixedPointMul()
10167 bool Signed = (Node->getOpcode() == ISD::SMULFIX || in expandFixedPointMul()
10168 Node->getOpcode() == ISD::SMULFIXSAT); in expandFixedPointMul()
10175 if (isOperationLegalOrCustom(ISD::MUL, VT)) in expandFixedPointMul()
10176 return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandFixedPointMul()
10177 } else if (Signed && isOperationLegalOrCustom(ISD::SMULO, VT)) { in expandFixedPointMul()
10179 DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); in expandFixedPointMul()
10190 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); in expandFixedPointMul()
10191 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT); in expandFixedPointMul()
10194 } else if (!Signed && isOperationLegalOrCustom(ISD::UMULO, VT)) { in expandFixedPointMul()
10196 DAG.getNode(ISD::UMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); in expandFixedPointMul()
10214 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI; in expandFixedPointMul()
10215 unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU; in expandFixedPointMul()
10221 Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandFixedPointMul()
10239 SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo, in expandFixedPointMul()
10255 ISD::SETUGT); in expandFixedPointMul()
10267 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, Lo, in expandFixedPointMul()
10269 SDValue Overflow = DAG.getSetCC(dl, BoolVT, Hi, Sign, ISD::SETNE); in expandFixedPointMul()
10274 ISD::SETLT); in expandFixedPointMul()
10285 Result = DAG.getSelectCC(dl, Hi, LowMask, SatMax, Result, ISD::SETGT); in expandFixedPointMul()
10291 Result = DAG.getSelectCC(dl, Hi, HighMask, SatMin, Result, ISD::SETLT); in expandFixedPointMul()
10299 assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT || in expandFixedPointDiv()
10300 Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) && in expandFixedPointDiv()
10304 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT; in expandFixedPointDiv()
10305 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT; in expandFixedPointDiv()
10337 LHS = DAG.getNode(ISD::SHL, dl, VT, LHS, in expandFixedPointDiv()
10340 RHS = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, VT, RHS, in expandFixedPointDiv()
10353 isOperationLegalOrCustom(ISD::SDIVREM, VT)) { in expandFixedPointDiv()
10354 Quot = DAG.getNode(ISD::SDIVREM, dl, in expandFixedPointDiv()
10360 Quot = DAG.getNode(ISD::SDIV, dl, VT, in expandFixedPointDiv()
10362 Rem = DAG.getNode(ISD::SREM, dl, VT, in expandFixedPointDiv()
10366 SDValue RemNonZero = DAG.getSetCC(dl, BoolVT, Rem, Zero, ISD::SETNE); in expandFixedPointDiv()
10367 SDValue LHSNeg = DAG.getSetCC(dl, BoolVT, LHS, Zero, ISD::SETLT); in expandFixedPointDiv()
10368 SDValue RHSNeg = DAG.getSetCC(dl, BoolVT, RHS, Zero, ISD::SETLT); in expandFixedPointDiv()
10369 SDValue QuotNeg = DAG.getNode(ISD::XOR, dl, BoolVT, LHSNeg, RHSNeg); in expandFixedPointDiv()
10370 SDValue Sub1 = DAG.getNode(ISD::SUB, dl, VT, Quot, in expandFixedPointDiv()
10373 DAG.getNode(ISD::AND, dl, BoolVT, RemNonZero, QuotNeg), in expandFixedPointDiv()
10376 Quot = DAG.getNode(ISD::UDIV, dl, VT, in expandFixedPointDiv()
10387 bool IsAdd = Node->getOpcode() == ISD::UADDO; in expandUADDSUBO()
10390 unsigned OpcCarry = IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY; in expandUADDSUBO()
10400 Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl, in expandUADDSUBO()
10415 DAG.getConstant(0, dl, Node->getValueType(0)), ISD::SETEQ); in expandUADDSUBO()
10420 DAG.getConstant(0, dl, Node->getValueType(0)), ISD::SETNE); in expandUADDSUBO()
10422 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT; in expandUADDSUBO()
10433 bool IsAdd = Node->getOpcode() == ISD::SADDO; in expandSADDSUBO()
10435 Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl, in expandSADDSUBO()
10443 unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT; in expandSADDSUBO()
10446 SDValue SetCC = DAG.getSetCC(dl, OType, Result, Sat, ISD::SETNE); in expandSADDSUBO()
10459 SDValue ResultLowerThanLHS = DAG.getSetCC(dl, OType, Result, LHS, ISD::SETLT); in expandSADDSUBO()
10461 DAG.getSetCC(dl, OType, RHS, Zero, IsAdd ? ISD::SETLT : ISD::SETGT); in expandSADDSUBO()
10464 DAG.getNode(ISD::XOR, dl, OType, ConditionRHS, ResultLowerThanLHS), dl, in expandSADDSUBO()
10475 bool isSigned = Node->getOpcode() == ISD::SMULO; in expandMULO()
10486 Result = DAG.getNode(ISD::SHL, dl, VT, LHS, ShiftAmt); in expandMULO()
10488 DAG.getNode(UseArithShift ? ISD::SRA : ISD::SRL, in expandMULO()
10490 LHS, ISD::SETNE); in expandMULO()
10503 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND }, in expandMULO()
10504 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }}; in expandMULO()
10506 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandMULO()
10515 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); in expandMULO()
10516 BottomHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul); in expandMULO()
10519 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, in expandMULO()
10520 DAG.getNode(ISD::SRL, dl, WideVT, Mul, ShiftAmt)); in expandMULO()
10547 DAG.getNode(ISD::SRA, dl, VT, LHS, in expandMULO()
10551 DAG.getNode(ISD::SRA, dl, VT, RHS, in expandMULO()
10578 assert(Ret.getOpcode() == ISD::MERGE_VALUES && in expandMULO()
10595 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt); in expandMULO()
10596 Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, Sign, ISD::SETNE); in expandMULO()
10599 DAG.getConstant(0, dl, VT), ISD::SETNE); in expandMULO()
10605 Overflow = DAG.getNode(ISD::TRUNCATE, dl, RType, Overflow); in expandMULO()
10614 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode()); in expandVecReduce()
10648 Res = DAG.getNode(ISD::ANY_EXTEND, dl, Node->getValueType(0), Res); in expandVecReduce()
10670 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode()); in expandVecReduceSeq()
10683 bool isSigned = Node->getOpcode() == ISD::SREM; in expandREM()
10684 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV; in expandREM()
10685 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; in expandREM()
10696 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor); in expandREM()
10697 Result = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul); in expandREM()
10705 bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT; in expandFP_TO_INT_SAT()
10733 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Src); in expandFP_TO_INT_SAT()
10753 bool MinMaxLegal = isOperationLegal(ISD::FMINNUM, SrcVT) && in expandFP_TO_INT_SAT()
10754 isOperationLegal(ISD::FMAXNUM, SrcVT); in expandFP_TO_INT_SAT()
10759 Clamped = DAG.getNode(ISD::FMAXNUM, dl, SrcVT, Clamped, MinFloatNode); in expandFP_TO_INT_SAT()
10761 Clamped = DAG.getNode(ISD::FMINNUM, dl, SrcVT, Clamped, MaxFloatNode); in expandFP_TO_INT_SAT()
10763 SDValue FpToInt = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, in expandFP_TO_INT_SAT()
10775 SDValue IsNan = DAG.getSetCC(dl, SetCCVT, Src, Src, ISD::CondCode::SETUO); in expandFP_TO_INT_SAT()
10786 DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, DstVT, Src); in expandFP_TO_INT_SAT()
10795 SDValue ULT = DAG.getSetCC(dl, SetCCVT, Src, MinFloatNode, ISD::SETULT); in expandFP_TO_INT_SAT()
10798 SDValue OGT = DAG.getSetCC(dl, SetCCVT, Src, MaxFloatNode, ISD::SETOGT); in expandFP_TO_INT_SAT()
10808 SDValue IsNan = DAG.getSetCC(dl, SetCCVT, Src, Src, ISD::CondCode::SETUO); in expandFP_TO_INT_SAT()
10814 assert(Node->getOpcode() == ISD::VECTOR_SPLICE && "Unexpected opcode!"); in expandVectorSplice()
10851 SDValue StackPtr2 = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, OffsetToV2); in expandVectorSplice()
10875 TrailingBytes = DAG.getNode(ISD::UMIN, DL, PtrVT, TrailingBytes, VLBytes); in expandVectorSplice()
10879 StackPtr2 = DAG.getNode(ISD::SUB, DL, PtrVT, StackPtr2, TrailingBytes); in expandVectorSplice()
10894 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); in LegalizeSetCCCondCode()
10905 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode); in LegalizeSetCCCondCode()
10917 InvCC = ISD::getSetCCSwappedOperands(InvCC); in LegalizeSetCCCondCode()
10928 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; in LegalizeSetCCCondCode()
10933 case ISD::SETUO: in LegalizeSetCCCondCode()
10934 if (TLI.isCondCodeLegal(ISD::SETUNE, OpVT)) { in LegalizeSetCCCondCode()
10935 CC1 = ISD::SETUNE; in LegalizeSetCCCondCode()
10936 CC2 = ISD::SETUNE; in LegalizeSetCCCondCode()
10937 Opc = ISD::OR; in LegalizeSetCCCondCode()
10940 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && in LegalizeSetCCCondCode()
10944 case ISD::SETO: in LegalizeSetCCCondCode()
10945 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && in LegalizeSetCCCondCode()
10947 CC1 = ISD::SETOEQ; in LegalizeSetCCCondCode()
10948 CC2 = ISD::SETOEQ; in LegalizeSetCCCondCode()
10949 Opc = ISD::AND; in LegalizeSetCCCondCode()
10951 case ISD::SETONE: in LegalizeSetCCCondCode()
10952 case ISD::SETUEQ: in LegalizeSetCCCondCode()
10957 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO; in LegalizeSetCCCondCode()
10959 (TLI.isCondCodeLegal(ISD::SETOGT, OpVT) || in LegalizeSetCCCondCode()
10960 TLI.isCondCodeLegal(ISD::SETOLT, OpVT))) { in LegalizeSetCCCondCode()
10961 CC1 = ISD::SETOGT; in LegalizeSetCCCondCode()
10962 CC2 = ISD::SETOLT; in LegalizeSetCCCondCode()
10963 Opc = ISD::OR; in LegalizeSetCCCondCode()
10968 case ISD::SETOEQ: in LegalizeSetCCCondCode()
10969 case ISD::SETOGT: in LegalizeSetCCCondCode()
10970 case ISD::SETOGE: in LegalizeSetCCCondCode()
10971 case ISD::SETOLT: in LegalizeSetCCCondCode()
10972 case ISD::SETOLE: in LegalizeSetCCCondCode()
10973 case ISD::SETUNE: in LegalizeSetCCCondCode()
10974 case ISD::SETUGT: in LegalizeSetCCCondCode()
10975 case ISD::SETUGE: in LegalizeSetCCCondCode()
10976 case ISD::SETULT: in LegalizeSetCCCondCode()
10977 case ISD::SETULE: in LegalizeSetCCCondCode()
10982 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO; in LegalizeSetCCCondCode()
10983 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND; in LegalizeSetCCCondCode()
10984 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10); in LegalizeSetCCCondCode()
10989 case ISD::SETLE: in LegalizeSetCCCondCode()
10990 case ISD::SETGT: in LegalizeSetCCCondCode()
10991 case ISD::SETGE: in LegalizeSetCCCondCode()
10992 case ISD::SETLT: in LegalizeSetCCCondCode()
10993 case ISD::SETNE: in LegalizeSetCCCondCode()
10994 case ISD::SETEQ: in LegalizeSetCCCondCode()
11001 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) { in LegalizeSetCCCondCode()
11022 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, SetCC1.getValue(1), in LegalizeSetCCCondCode()
11028 assert((Opc == ISD::OR || Opc == ISD::AND) && "Unexpected opcode"); in LegalizeSetCCCondCode()
11029 Opc = Opc == ISD::OR ? ISD::VP_OR : ISD::VP_AND; in LegalizeSetCCCondCode()