Lines Matching refs:ISD
96 if (Value->getOpcode() == ISD::AssertZext) in parametersInCSRMatch()
98 if (Value->getOpcode() != ISD::CopyFromReg) in parametersInCSRMatch()
240 if (isOperationLegalOrCustom(ISD::STORE, NewVT) && in findOptimalMemOpLowering()
244 isOperationLegalOrCustom(ISD::STORE, MVT::f64) && in findOptimalMemOpLowering()
290 ISD::CondCode &CCCode, in softenSetCCOperands()
300 ISD::CondCode &CCCode, in softenSetCCOperands()
316 case ISD::SETEQ: in softenSetCCOperands()
317 case ISD::SETOEQ: in softenSetCCOperands()
322 case ISD::SETNE: in softenSetCCOperands()
323 case ISD::SETUNE: in softenSetCCOperands()
328 case ISD::SETGE: in softenSetCCOperands()
329 case ISD::SETOGE: in softenSetCCOperands()
334 case ISD::SETLT: in softenSetCCOperands()
335 case ISD::SETOLT: in softenSetCCOperands()
340 case ISD::SETLE: in softenSetCCOperands()
341 case ISD::SETOLE: in softenSetCCOperands()
346 case ISD::SETGT: in softenSetCCOperands()
347 case ISD::SETOGT: in softenSetCCOperands()
352 case ISD::SETO: in softenSetCCOperands()
355 case ISD::SETUO: in softenSetCCOperands()
360 case ISD::SETONE: in softenSetCCOperands()
364 case ISD::SETUEQ: in softenSetCCOperands()
376 case ISD::SETULT: in softenSetCCOperands()
381 case ISD::SETULE: in softenSetCCOperands()
386 case ISD::SETUGT: in softenSetCCOperands()
391 case ISD::SETUGE: in softenSetCCOperands()
430 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second, in softenSetCCOperands()
432 NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl, in softenSetCCOperands()
514 case ISD::XOR: in ShrinkDemandedConstant()
515 case ISD::AND: in ShrinkDemandedConstant()
516 case ISD::OR: { in ShrinkDemandedConstant()
523 if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C)) in ShrinkDemandedConstant()
587 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)), in ShrinkDemandedOp()
588 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1))); in ShrinkDemandedOp()
590 SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X); in ShrinkDemandedOp()
682 case ISD::BITCAST: { in SimplifyMultipleUseDemandedBits()
738 case ISD::AND: { in SimplifyMultipleUseDemandedBits()
751 case ISD::OR: { in SimplifyMultipleUseDemandedBits()
764 case ISD::XOR: { in SimplifyMultipleUseDemandedBits()
776 case ISD::SHL: { in SimplifyMultipleUseDemandedBits()
791 case ISD::SETCC: { in SimplifyMultipleUseDemandedBits()
794 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); in SimplifyMultipleUseDemandedBits()
806 if (CC == ISD::SETLT && Op1.getValueType().isInteger() && in SimplifyMultipleUseDemandedBits()
807 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode()))) in SimplifyMultipleUseDemandedBits()
812 case ISD::SIGN_EXTEND_INREG: { in SimplifyMultipleUseDemandedBits()
825 case ISD::ANY_EXTEND_VECTOR_INREG: in SimplifyMultipleUseDemandedBits()
826 case ISD::SIGN_EXTEND_VECTOR_INREG: in SimplifyMultipleUseDemandedBits()
827 case ISD::ZERO_EXTEND_VECTOR_INREG: { in SimplifyMultipleUseDemandedBits()
840 case ISD::INSERT_VECTOR_ELT: { in SimplifyMultipleUseDemandedBits()
850 case ISD::INSERT_SUBVECTOR: { in SimplifyMultipleUseDemandedBits()
868 case ISD::VECTOR_SHUFFLE: { in SimplifyMultipleUseDemandedBits()
892 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) in SimplifyMultipleUseDemandedBits()
934 assert((Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SRA) && in combineShiftToAVG()
948 if (Add.getOpcode() != ISD::ADD) in combineShiftToAVG()
976 (ExtOpA.getOpcode() == ISD::ADD && in combineShiftToAVG()
978 (ExtOpB.getOpcode() == ISD::ADD && in combineShiftToAVG()
1002 case ISD::SRA: { in combineShiftToAVG()
1015 case ISD::SRL: { in combineShiftToAVG()
1030 unsigned AVGOpc = IsCeil ? (IsSigned ? ISD::AVGCEILS : ISD::AVGCEILU) in combineShiftToAVG()
1031 : (IsSigned ? ISD::AVGFLOORS : ISD::AVGFLOORU); in combineShiftToAVG()
1047 DAG.getNode(AVGOpc, DL, NVT, DAG.getNode(ISD::TRUNCATE, DL, NVT, ExtOpA), in combineShiftToAVG()
1048 DAG.getNode(ISD::TRUNCATE, DL, NVT, ExtOpB)); in combineShiftToAVG()
1049 return DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL, VT, in combineShiftToAVG()
1092 if (Op.getOpcode() == ISD::Constant) { in SimplifyDemandedBits()
1098 if (Op.getOpcode() == ISD::ConstantFP) { in SimplifyDemandedBits()
1127 case ISD::TargetConstant: in SimplifyDemandedBits()
1129 case ISD::SCALAR_TO_VECTOR: { in SimplifyDemandedBits()
1146 case ISD::BUILD_VECTOR: in SimplifyDemandedBits()
1151 case ISD::LOAD: { in SimplifyDemandedBits()
1157 if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { in SimplifyDemandedBits()
1166 case ISD::INSERT_VECTOR_ELT: { in SimplifyDemandedBits()
1202 case ISD::INSERT_SUBVECTOR: { in SimplifyDemandedBits()
1245 case ISD::EXTRACT_SUBVECTOR: { in SimplifyDemandedBits()
1270 case ISD::CONCAT_VECTORS: { in SimplifyDemandedBits()
1288 case ISD::VECTOR_SHUFFLE: { in SimplifyDemandedBits()
1345 case ISD::AND: { in SimplifyDemandedBits()
1374 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1); in SimplifyDemandedBits()
1381 if (Op0.getOpcode() == ISD::INSERT_SUBVECTOR && in SimplifyDemandedBits()
1383 ISD::isBuildVectorOfConstantSDNodes(Op0.getOperand(0).getNode())) && in SimplifyDemandedBits()
1394 TLO.DAG.getNode(ISD::AND, dl, VT, Op0.getOperand(0), Op1); in SimplifyDemandedBits()
1396 TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, NewAnd, in SimplifyDemandedBits()
1445 case ISD::OR: { in SimplifyDemandedBits()
1487 if (Op0.getOpcode() == ISD::AND && Op1.getOpcode() == ISD::AND && in SimplifyDemandedBits()
1495 if (Alt.getOpcode() == ISD::OR) { in SimplifyDemandedBits()
1499 if (SDValue C12 = TLO.DAG.FoldConstantArithmetic(ISD::OR, dl, VT, in SimplifyDemandedBits()
1501 SDValue MaskX = TLO.DAG.getNode(ISD::AND, dl, VT, X, C12); in SimplifyDemandedBits()
1502 SDValue MaskY = TLO.DAG.getNode(ISD::AND, dl, VT, Y, C2); in SimplifyDemandedBits()
1504 Op, TLO.DAG.getNode(ISD::OR, dl, VT, MaskX, MaskY)); in SimplifyDemandedBits()
1515 case ISD::XOR: { in SimplifyDemandedBits()
1542 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
1554 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC)); in SimplifyDemandedBits()
1567 if ((Op0Opcode == ISD::SRL || Op0Opcode == ISD::SHL) && Op0.hasOneUse()) { in SimplifyDemandedBits()
1575 Ones = Op0Opcode == ISD::SHL ? Ones.shl(ShiftAmt) in SimplifyDemandedBits()
1615 case ISD::SELECT: in SimplifyDemandedBits()
1632 case ISD::VSELECT: in SimplifyDemandedBits()
1645 case ISD::SELECT_CC: in SimplifyDemandedBits()
1662 case ISD::SETCC: { in SimplifyDemandedBits()
1665 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); in SimplifyDemandedBits()
1677 if (CC == ISD::SETLT && Op1.getValueType().isInteger() && in SimplifyDemandedBits()
1678 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode()))) in SimplifyDemandedBits()
1690 case ISD::SHL: { in SimplifyDemandedBits()
1705 if (Op0.getOpcode() == ISD::SRL) { in SimplifyDemandedBits()
1710 unsigned Opc = ISD::SHL; in SimplifyDemandedBits()
1714 Opc = ISD::SRL; in SimplifyDemandedBits()
1726 if (Op0.getOpcode() == ISD::ANY_EXTEND) { in SimplifyDemandedBits()
1731 isTypeDesirableForOp(ISD::SHL, InnerVT)) { in SimplifyDemandedBits()
1736 TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp, in SimplifyDemandedBits()
1739 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl)); in SimplifyDemandedBits()
1748 if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL && in SimplifyDemandedBits()
1759 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, in SimplifyDemandedBits()
1762 Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA)); in SimplifyDemandedBits()
1783 SDValue NewOp = TLO.DAG.getNode(ISD::SHL, dl, VT, DemandedOp0, Op1); in SimplifyDemandedBits()
1828 case ISD::SRL: { in SimplifyDemandedBits()
1848 if (Op0.getOpcode() == ISD::SHL) { in SimplifyDemandedBits()
1853 unsigned Opc = ISD::SRL; in SimplifyDemandedBits()
1857 Opc = ISD::SHL; in SimplifyDemandedBits()
1885 case ISD::SRA: { in SimplifyDemandedBits()
1902 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
1941 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags)); in SimplifyDemandedBits()
1948 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA)); in SimplifyDemandedBits()
1960 SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1); in SimplifyDemandedBits()
1967 case ISD::FSHL: in SimplifyDemandedBits()
1968 case ISD::FSHR: { in SimplifyDemandedBits()
1972 bool IsFSHL = (Op.getOpcode() == ISD::FSHL); in SimplifyDemandedBits()
2030 case ISD::ROTL: in SimplifyDemandedBits()
2031 case ISD::ROTR: { in SimplifyDemandedBits()
2034 bool IsROTL = (Op.getOpcode() == ISD::ROTL); in SimplifyDemandedBits()
2056 if ((!TLO.LegalOperations() || isOperationLegal(ISD::SHL, VT)) && in SimplifyDemandedBits()
2059 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
2061 if ((!TLO.LegalOperations() || isOperationLegal(ISD::SRL, VT)) && in SimplifyDemandedBits()
2064 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
2077 case ISD::UMIN: { in SimplifyDemandedBits()
2090 case ISD::UMAX: { in SimplifyDemandedBits()
2103 case ISD::BITREVERSE: { in SimplifyDemandedBits()
2113 case ISD::BSWAP: { in SimplifyDemandedBits()
2130 unsigned ShiftOpcode = NLZ > NTZ ? ISD::SRL : ISD::SHL; in SimplifyDemandedBits()
2148 case ISD::CTPOP: { in SimplifyDemandedBits()
2153 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT, in SimplifyDemandedBits()
2159 case ISD::SIGN_EXTEND_INREG: { in SimplifyDemandedBits()
2177 TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt)); in SimplifyDemandedBits()
2213 case ISD::BUILD_PAIR: { in SimplifyDemandedBits()
2235 case ISD::ZERO_EXTEND: in SimplifyDemandedBits()
2236 case ISD::ZERO_EXTEND_VECTOR_INREG: { in SimplifyDemandedBits()
2241 bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG; in SimplifyDemandedBits()
2252 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND; in SimplifyDemandedBits()
2272 case ISD::SIGN_EXTEND: in SimplifyDemandedBits()
2273 case ISD::SIGN_EXTEND_VECTOR_INREG: { in SimplifyDemandedBits()
2278 bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG; in SimplifyDemandedBits()
2289 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND; in SimplifyDemandedBits()
2313 IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND; in SimplifyDemandedBits()
2324 case ISD::ANY_EXTEND: in SimplifyDemandedBits()
2325 case ISD::ANY_EXTEND_VECTOR_INREG: { in SimplifyDemandedBits()
2330 bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG; in SimplifyDemandedBits()
2353 case ISD::TRUNCATE: { in SimplifyDemandedBits()
2368 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc)); in SimplifyDemandedBits()
2376 case ISD::SRL: in SimplifyDemandedBits()
2379 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT)) in SimplifyDemandedBits()
2401 TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0)); in SimplifyDemandedBits()
2403 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt)); in SimplifyDemandedBits()
2412 case ISD::AssertZext: { in SimplifyDemandedBits()
2425 case ISD::EXTRACT_VECTOR_ELT: { in SimplifyDemandedBits()
2466 case ISD::BITCAST: { in SimplifyDemandedBits()
2476 bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT); in SimplifyDemandedBits()
2477 bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32); in SimplifyDemandedBits()
2484 SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src); in SimplifyDemandedBits()
2487 Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign); in SimplifyDemandedBits()
2491 TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt)); in SimplifyDemandedBits()
2557 case ISD::MUL: in SimplifyDemandedBits()
2567 SDValue Shl = TLO.DAG.getNode(ISD::SHL, dl, VT, Op.getOperand(0), AmtC); in SimplifyDemandedBits()
2576 SDValue And1 = TLO.DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0), One); in SimplifyDemandedBits()
2580 case ISD::ADD: in SimplifyDemandedBits()
2581 case ISD::SUB: { in SimplifyDemandedBits()
2643 if (Mul.getOpcode() != ISD::MUL || !Mul.hasOneUse()) in SimplifyDemandedBits()
2658 auto foldMul = [&](ISD::NodeType NT, SDValue X, SDValue Y, unsigned ShlAmt) { in SimplifyDemandedBits()
2661 SDValue Shl = TLO.DAG.getNode(ISD::SHL, dl, VT, X, ShlAmtC); in SimplifyDemandedBits()
2666 if (isOperationLegalOrCustom(ISD::SHL, VT)) { in SimplifyDemandedBits()
2667 if (Op.getOpcode() == ISD::ADD) { in SimplifyDemandedBits()
2670 return foldMul(ISD::SUB, Op0.getOperand(0), Op1, ShAmt); in SimplifyDemandedBits()
2673 return foldMul(ISD::SUB, Op1.getOperand(0), Op0, ShAmt); in SimplifyDemandedBits()
2675 if (Op.getOpcode() == ISD::SUB) { in SimplifyDemandedBits()
2678 return foldMul(ISD::ADD, Op1.getOperand(0), Op0, ShAmt); in SimplifyDemandedBits()
2685 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) { in SimplifyDemandedBits()
2850 case ISD::SCALAR_TO_VECTOR: { in SimplifyDemandedVectorElts()
2856 if (ScalarSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { in SimplifyDemandedVectorElts()
2879 case ISD::BITCAST: { in SimplifyDemandedVectorElts()
2973 case ISD::BUILD_VECTOR: { in SimplifyDemandedVectorElts()
3003 case ISD::CONCAT_VECTORS: { in SimplifyDemandedVectorElts()
3038 case ISD::INSERT_SUBVECTOR: { in SimplifyDemandedVectorElts()
3056 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in SimplifyDemandedVectorElts()
3082 case ISD::EXTRACT_SUBVECTOR: { in SimplifyDemandedVectorElts()
3110 case ISD::INSERT_VECTOR_ELT: { in SimplifyDemandedVectorElts()
3141 case ISD::VSELECT: { in SimplifyDemandedVectorElts()
3178 case ISD::VECTOR_SHUFFLE: { in SimplifyDemandedVectorElts()
3251 case ISD::ANY_EXTEND_VECTOR_INREG: in SimplifyDemandedVectorElts()
3252 case ISD::SIGN_EXTEND_VECTOR_INREG: in SimplifyDemandedVectorElts()
3253 case ISD::ZERO_EXTEND_VECTOR_INREG: { in SimplifyDemandedVectorElts()
3264 if (IsLE && Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG && in SimplifyDemandedVectorElts()
3271 if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) { in SimplifyDemandedVectorElts()
3279 if (IsLE && DemandedSrcElts == 1 && Src.getOpcode() == ISD::AND && in SimplifyDemandedVectorElts()
3290 ISD::AND, DL, SrcVT, {Src.getOperand(1), Mask})) { in SimplifyDemandedVectorElts()
3291 Fold = TLO.DAG.getNode(ISD::AND, DL, SrcVT, Src.getOperand(0), Fold); in SimplifyDemandedVectorElts()
3301 case ISD::ADD: { in SimplifyDemandedVectorElts()
3312 case ISD::OR: in SimplifyDemandedVectorElts()
3313 case ISD::XOR: in SimplifyDemandedVectorElts()
3314 case ISD::SUB: in SimplifyDemandedVectorElts()
3315 case ISD::FADD: in SimplifyDemandedVectorElts()
3316 case ISD::FSUB: in SimplifyDemandedVectorElts()
3317 case ISD::FMUL: in SimplifyDemandedVectorElts()
3318 case ISD::FDIV: in SimplifyDemandedVectorElts()
3319 case ISD::FREM: { in SimplifyDemandedVectorElts()
3342 case ISD::SHL: in SimplifyDemandedVectorElts()
3343 case ISD::SRL: in SimplifyDemandedVectorElts()
3344 case ISD::SRA: in SimplifyDemandedVectorElts()
3345 case ISD::ROTL: in SimplifyDemandedVectorElts()
3346 case ISD::ROTR: { in SimplifyDemandedVectorElts()
3369 case ISD::MUL: in SimplifyDemandedVectorElts()
3370 case ISD::AND: { in SimplifyDemandedVectorElts()
3402 case ISD::TRUNCATE: in SimplifyDemandedVectorElts()
3403 case ISD::SIGN_EXTEND: in SimplifyDemandedVectorElts()
3404 case ISD::ZERO_EXTEND: in SimplifyDemandedVectorElts()
3409 if (Op.getOpcode() == ISD::ZERO_EXTEND) { in SimplifyDemandedVectorElts()
3417 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) { in SimplifyDemandedVectorElts()
3448 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in computeKnownBitsForTargetNode()
3449 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in computeKnownBitsForTargetNode()
3450 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in computeKnownBitsForTargetNode()
3451 Op.getOpcode() == ISD::INTRINSIC_VOID) && in computeKnownBitsForTargetNode()
3482 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in ComputeNumSignBitsForTargetNode()
3483 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in ComputeNumSignBitsForTargetNode()
3484 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in ComputeNumSignBitsForTargetNode()
3485 Op.getOpcode() == ISD::INTRINSIC_VOID) && in ComputeNumSignBitsForTargetNode()
3500 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in SimplifyDemandedVectorEltsForTargetNode()
3501 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in SimplifyDemandedVectorEltsForTargetNode()
3502 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in SimplifyDemandedVectorEltsForTargetNode()
3503 Op.getOpcode() == ISD::INTRINSIC_VOID) && in SimplifyDemandedVectorEltsForTargetNode()
3512 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in SimplifyDemandedBitsForTargetNode()
3513 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in SimplifyDemandedBitsForTargetNode()
3514 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in SimplifyDemandedBitsForTargetNode()
3515 Op.getOpcode() == ISD::INTRINSIC_VOID) && in SimplifyDemandedBitsForTargetNode()
3526 (Op.getOpcode() >= ISD::BUILTIN_OP_END || in SimplifyMultipleUseDemandedBitsForTargetNode()
3527 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in SimplifyMultipleUseDemandedBitsForTargetNode()
3528 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in SimplifyMultipleUseDemandedBitsForTargetNode()
3529 Op.getOpcode() == ISD::INTRINSIC_VOID) && in SimplifyMultipleUseDemandedBitsForTargetNode()
3560 (Op.getOpcode() >= ISD::BUILTIN_OP_END || in isGuaranteedNotToBeUndefOrPoisonForTargetNode()
3561 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in isGuaranteedNotToBeUndefOrPoisonForTargetNode()
3562 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in isGuaranteedNotToBeUndefOrPoisonForTargetNode()
3563 Op.getOpcode() == ISD::INTRINSIC_VOID) && in isGuaranteedNotToBeUndefOrPoisonForTargetNode()
3573 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in isKnownNeverNaNForTargetNode()
3574 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in isKnownNeverNaNForTargetNode()
3575 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in isKnownNeverNaNForTargetNode()
3576 Op.getOpcode() == ISD::INTRINSIC_VOID) && in isKnownNeverNaNForTargetNode()
3586 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END || in isSplatValueForTargetNode()
3587 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in isSplatValueForTargetNode()
3588 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || in isSplatValueForTargetNode()
3589 Op.getOpcode() == ISD::INTRINSIC_VOID) && in isSplatValueForTargetNode()
3673 ISD::CondCode Cond, const SDLoc &DL, in foldSetCCWithAnd()
3675 if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND) in foldSetCCWithAnd()
3680 if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() || in foldSetCCWithAnd()
3681 (Cond != ISD::SETEQ && Cond != ISD::SETNE)) in foldSetCCWithAnd()
3686 if (Cond == ISD::SETNE && isNullConstant(N1) && in foldSetCCWithAnd()
3716 Cond = ISD::getSetCCInverse(Cond, OpVT); in foldSetCCWithAnd()
3735 SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y); in foldSetCCWithAnd()
3753 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI, in optimizeSetCCOfSignedTruncationCheck()
3761 if (N0->getOpcode() != ISD::ADD) in optimizeSetCCOfSignedTruncationCheck()
3776 ISD::CondCode NewCond; in optimizeSetCCOfSignedTruncationCheck()
3777 if (Cond == ISD::CondCode::SETULT) { in optimizeSetCCOfSignedTruncationCheck()
3778 NewCond = ISD::CondCode::SETEQ; in optimizeSetCCOfSignedTruncationCheck()
3779 } else if (Cond == ISD::CondCode::SETULE) { in optimizeSetCCOfSignedTruncationCheck()
3780 NewCond = ISD::CondCode::SETEQ; in optimizeSetCCOfSignedTruncationCheck()
3783 } else if (Cond == ISD::CondCode::SETUGT) { in optimizeSetCCOfSignedTruncationCheck()
3784 NewCond = ISD::CondCode::SETNE; in optimizeSetCCOfSignedTruncationCheck()
3787 } else if (Cond == ISD::CondCode::SETUGE) { in optimizeSetCCOfSignedTruncationCheck()
3788 NewCond = ISD::CondCode::SETNE; in optimizeSetCCOfSignedTruncationCheck()
3833 SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt); in optimizeSetCCOfSignedTruncationCheck()
3834 SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt); in optimizeSetCCOfSignedTruncationCheck()
3842 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond, in optimizeSetCCByHoistingAndByConstFromLogicalShift()
3847 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in optimizeSetCCByHoistingAndByConstFromLogicalShift()
3863 case ISD::SHL: in optimizeSetCCByHoistingAndByConstFromLogicalShift()
3864 NewShiftOpcode = ISD::SRL; in optimizeSetCCByHoistingAndByConstFromLogicalShift()
3866 case ISD::SRL: in optimizeSetCCByHoistingAndByConstFromLogicalShift()
3867 NewShiftOpcode = ISD::SHL; in optimizeSetCCByHoistingAndByConstFromLogicalShift()
3888 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse()) in optimizeSetCCByHoistingAndByConstFromLogicalShift()
3906 SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C); in optimizeSetCCByHoistingAndByConstFromLogicalShift()
3915 ISD::CondCode Cond, const SDLoc &DL, in foldSetCCWithBinOp()
3918 assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && in foldSetCCWithBinOp()
3920 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode"); in foldSetCCWithBinOp()
3937 if (BOpcode == ISD::ADD || BOpcode == ISD::XOR) in foldSetCCWithBinOp()
3948 SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One); in foldSetCCWithBinOp()
3956 ISD::CondCode Cond, const SDLoc &dl, in simplifySetCCWithCTPOP()
3961 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() && in simplifySetCCWithCTPOP()
3965 if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse()) in simplifySetCCWithCTPOP()
3973 if (VT.isVector() && TLI.isOperationLegal(ISD::CTPOP, CTVT)) in simplifySetCCWithCTPOP()
3978 if (Cond == ISD::SETULT || Cond == ISD::SETUGT) { in simplifySetCCWithCTPOP()
3980 if (C1.ugt(CostLimit + (Cond == ISD::SETULT))) in simplifySetCCWithCTPOP()
3982 if (C1 == 0 && (Cond == ISD::SETULT)) in simplifySetCCWithCTPOP()
3985 unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT); in simplifySetCCWithCTPOP()
3990 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne); in simplifySetCCWithCTPOP()
3991 Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add); in simplifySetCCWithCTPOP()
3993 ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE; in simplifySetCCWithCTPOP()
3998 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) { in simplifySetCCWithCTPOP()
4000 if (!VT.isVector() && TLI.isOperationLegalOrCustom(ISD::CTPOP, CTVT)) in simplifySetCCWithCTPOP()
4010 ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT); in simplifySetCCWithCTPOP()
4011 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne); in simplifySetCCWithCTPOP()
4012 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add); in simplifySetCCWithCTPOP()
4015 unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR; in simplifySetCCWithCTPOP()
4023 ISD::CondCode Cond, const SDLoc &dl, in foldSetCCWithRotate()
4025 if (Cond != ISD::SETEQ && Cond != ISD::SETNE) in foldSetCCWithRotate()
4033 if (X.getOpcode() == ISD::ROTL || X.getOpcode() == ISD::ROTR) in foldSetCCWithRotate()
4051 if (N0.hasOneUse() && N0.getOpcode() == ISD::OR && C1->isZero()) { in foldSetCCWithRotate()
4053 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, R, N0.getOperand(1)); in foldSetCCWithRotate()
4057 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, R, N0.getOperand(0)); in foldSetCCWithRotate()
4066 ISD::CondCode Cond, const SDLoc &dl, in foldSetCCWithFunnelShift()
4070 if (Cond != ISD::SETEQ && Cond != ISD::SETNE) in foldSetCCWithFunnelShift()
4078 (N0.getOpcode() != ISD::FSHL && N0.getOpcode() != ISD::FSHR)) in foldSetCCWithFunnelShift()
4088 if (N0.getOpcode() == ISD::FSHR) in foldSetCCWithFunnelShift()
4094 if (Or.getOpcode() != ISD::OR || !Or.hasOneUse()) in foldSetCCWithFunnelShift()
4116 SDValue Shift = DAG.getNode(ISD::SHL, dl, OpVT, Y, NewShAmt); in foldSetCCWithFunnelShift()
4117 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, Shift, X); in foldSetCCWithFunnelShift()
4123 SDValue Shift = DAG.getNode(ISD::SRL, dl, OpVT, Y, NewShAmt); in foldSetCCWithFunnelShift()
4124 SDValue NewOr = DAG.getNode(ISD::OR, dl, OpVT, Shift, X); in foldSetCCWithFunnelShift()
4134 ISD::CondCode Cond, bool foldBooleans, in SimplifySetCC()
4154 ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond); in SimplifySetCC()
4167 DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) && in SimplifySetCC()
4168 !DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1})) in SimplifySetCC()
4187 if (N0.getOpcode() == ISD::SRL && (C1.isZero() || C1.isOne()) && in SimplifySetCC()
4188 N0.getOperand(0).getOpcode() == ISD::CTLZ && in SimplifySetCC()
4191 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
4193 if ((C1 == 0) == (Cond == ISD::SETEQ)) { in SimplifySetCC()
4196 Cond = ISD::SETNE; in SimplifySetCC()
4200 Cond = ISD::SETEQ; in SimplifySetCC()
4216 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
4221 if (N0->getOpcode() == ISD::ZERO_EXTEND) { in SimplifySetCC()
4225 } else if (N0->getOpcode() == ISD::AND) { in SimplifySetCC()
4232 } else if (N0->getOpcode() == ISD::SIGN_EXTEND) { in SimplifySetCC()
4239 if (LN0->getExtensionType() == ISD::ZEXTLOAD) { in SimplifySetCC()
4242 } else if (LN0->getExtensionType() == ISD::SEXTLOAD) { in SimplifySetCC()
4257 if (isTypeDesirableForOp(ISD::SETCC, MinVT)) { in SimplifySetCC()
4259 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt); in SimplifySetCC()
4263 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ); in SimplifySetCC()
4278 bool SExt = (N0Opc == ISD::SIGN_EXTEND); in SimplifySetCC()
4280 TopSetCC.getOpcode() == ISD::SETCC && in SimplifySetCC()
4281 (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) && in SimplifySetCC()
4285 bool Inverse = (N1C->isZero() && Cond == ISD::SETEQ) || in SimplifySetCC()
4286 (!N1C->isZero() && Cond == ISD::SETNE); in SimplifySetCC()
4291 ISD::CondCode InvCond = ISD::getSetCCInverse( in SimplifySetCC()
4305 !ISD::isSignedIntSetCC(Cond) && in SimplifySetCC()
4306 N0.getOpcode() == ISD::AND && C1 == 0 && in SimplifySetCC()
4319 if (Lod->getExtensionType() != ISD::NON_EXTLOAD) in SimplifySetCC()
4341 shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) { in SimplifySetCC()
4351 DAG.getNode(ISD::AND, dl, newVT, NewLoad, in SimplifySetCC()
4360 if (N0.getOpcode() == ISD::ZERO_EXTEND) { in SimplifySetCC()
4368 case ISD::SETUGT: in SimplifySetCC()
4369 case ISD::SETUGE: in SimplifySetCC()
4370 case ISD::SETEQ: in SimplifySetCC()
4372 case ISD::SETULT: in SimplifySetCC()
4373 case ISD::SETULE: in SimplifySetCC()
4374 case ISD::SETNE: in SimplifySetCC()
4376 case ISD::SETGT: in SimplifySetCC()
4377 case ISD::SETGE: in SimplifySetCC()
4380 case ISD::SETLT: in SimplifySetCC()
4381 case ISD::SETLE: in SimplifySetCC()
4391 case ISD::SETEQ: in SimplifySetCC()
4392 case ISD::SETNE: in SimplifySetCC()
4393 case ISD::SETUGT: in SimplifySetCC()
4394 case ISD::SETUGE: in SimplifySetCC()
4395 case ISD::SETULT: in SimplifySetCC()
4396 case ISD::SETULE: { in SimplifySetCC()
4399 (isOperationLegal(ISD::SETCC, newVT) && in SimplifySetCC()
4413 } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG && in SimplifySetCC()
4414 (Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
4425 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT); in SimplifySetCC()
4430 SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0), in SimplifySetCC()
4438 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) { in SimplifySetCC()
4440 if (N0.getOpcode() == ISD::SETCC && in SimplifySetCC()
4445 bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne()); in SimplifySetCC()
4447 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0); in SimplifySetCC()
4449 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in SimplifySetCC()
4450 CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType()); in SimplifySetCC()
4456 if ((N0.getOpcode() == ISD::XOR || in SimplifySetCC()
4457 (N0.getOpcode() == ISD::AND && in SimplifySetCC()
4458 N0.getOperand(0).getOpcode() == ISD::XOR && in SimplifySetCC()
4469 if (N0.getOpcode() == ISD::XOR) { in SimplifySetCC()
4472 assert(N0.getOpcode() == ISD::AND && in SimplifySetCC()
4473 N0.getOperand(0).getOpcode() == ISD::XOR); in SimplifySetCC()
4475 Val = DAG.getNode(ISD::AND, dl, N0.getValueType(), in SimplifySetCC()
4481 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ); in SimplifySetCC()
4485 if (Op0.getOpcode() == ISD::TRUNCATE) in SimplifySetCC()
4488 if ((Op0.getOpcode() == ISD::XOR) && in SimplifySetCC()
4489 Op0.getOperand(0).getOpcode() == ISD::SETCC && in SimplifySetCC()
4490 Op0.getOperand(1).getOpcode() == ISD::SETCC) { in SimplifySetCC()
4500 Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ; in SimplifySetCC()
4504 if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) { in SimplifySetCC()
4507 Op0 = DAG.getNode(ISD::AND, dl, VT, in SimplifySetCC()
4508 DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)), in SimplifySetCC()
4511 Op0 = DAG.getNode(ISD::AND, dl, VT, in SimplifySetCC()
4512 DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)), in SimplifySetCC()
4517 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ); in SimplifySetCC()
4519 if (Op0.getOpcode() == ISD::AssertZext && in SimplifySetCC()
4523 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ); in SimplifySetCC()
4531 if (N0.getOpcode() == ISD::UREM && N1C->isZero() && in SimplifySetCC()
4532 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) { in SimplifySetCC()
4541 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
4542 N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(N0.getOperand(1)) && in SimplifySetCC()
4547 Cond == ISD::SETEQ ? ISD::SETLT : ISD::SETGE); in SimplifySetCC()
4562 if (ISD::isSignedIntSetCC(Cond)) { in SimplifySetCC()
4571 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) { in SimplifySetCC()
4579 ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT; in SimplifySetCC()
4591 if (Cond == ISD::SETLE || Cond == ISD::SETULE) { in SimplifySetCC()
4599 ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT; in SimplifySetCC()
4611 if (Cond == ISD::SETLT || Cond == ISD::SETULT) { in SimplifySetCC()
4619 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE); in SimplifySetCC()
4625 ISD::SETEQ); in SimplifySetCC()
4629 if (Cond == ISD::SETGT || Cond == ISD::SETUGT) { in SimplifySetCC()
4637 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE); in SimplifySetCC()
4643 ISD::SETEQ); in SimplifySetCC()
4647 if (Cond == ISD::SETEQ || Cond == ISD::SETNE) { in SimplifySetCC()
4664 if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0) in SimplifySetCC()
4670 if (RHS.getOpcode() == ISD::SHL && in SimplifySetCC()
4678 if (LHS.getOpcode() == ISD::SHL && in SimplifySetCC()
4694 SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits); in SimplifySetCC()
4696 DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask); in SimplifySetCC()
4705 if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) { in SimplifySetCC()
4722 if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) || in SimplifySetCC()
4723 (Cond == ISD::SETUGE && C1.isMinSignedValue())) in SimplifySetCC()
4726 ISD::SETLT); in SimplifySetCC()
4730 if ((Cond == ISD::SETULT && C1.isMinSignedValue()) || in SimplifySetCC()
4731 (Cond == ISD::SETULE && C1.isMaxSignedValue())) in SimplifySetCC()
4734 ISD::SETGT); in SimplifySetCC()
4751 (Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
4753 N0.getOpcode() == ISD::AND) { in SimplifySetCC()
4757 if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3 in SimplifySetCC()
4762 return DAG.getNode(ISD::TRUNCATE, dl, VT, in SimplifySetCC()
4763 DAG.getNode(ISD::SRL, dl, ShValTy, N0, in SimplifySetCC()
4766 } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) { in SimplifySetCC()
4772 return DAG.getNode(ISD::TRUNCATE, dl, VT, in SimplifySetCC()
4773 DAG.getNode(ISD::SRL, dl, ShValTy, N0, in SimplifySetCC()
4784 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
4785 N0.getOpcode() == ISD::AND && N0.hasOneUse()) { in SimplifySetCC()
4792 DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0), in SimplifySetCC()
4799 } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE || in SimplifySetCC()
4800 Cond == ISD::SETULE || Cond == ISD::SETUGT) { in SimplifySetCC()
4801 bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT); in SimplifySetCC()
4808 ISD::CondCode NewCond = Cond; in SimplifySetCC()
4812 NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE; in SimplifySetCC()
4820 SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0, in SimplifySetCC()
4837 if (Cond == ISD::SETO || Cond == ISD::SETUO) in SimplifySetCC()
4841 if (N0.getOpcode() == ISD::FNEG) { in SimplifySetCC()
4842 ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond); in SimplifySetCC()
4845 SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1); in SimplifySetCC()
4858 ISD::CondCode NewCond = ISD::SETCC_INVALID; in SimplifySetCC()
4860 case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break; in SimplifySetCC()
4861 case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break; in SimplifySetCC()
4862 case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break; in SimplifySetCC()
4863 case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break; in SimplifySetCC()
4866 if (NewCond != ISD::SETCC_INVALID && in SimplifySetCC()
4879 bool EqTrue = ISD::isTrueWhenEqual(Cond); in SimplifySetCC()
4880 unsigned UOF = ISD::getUnorderedFlavor(Cond); in SimplifySetCC()
4887 ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO; in SimplifySetCC()
4894 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in SimplifySetCC()
4896 if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB || in SimplifySetCC()
4897 N0.getOpcode() == ISD::XOR) { in SimplifySetCC()
4922 if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) in SimplifySetCC()
4930 if (N0.getOpcode() == ISD::XOR && N0.getNode()->hasOneUse()) in SimplifySetCC()
4940 if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) in SimplifySetCC()
4961 if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB || in SimplifySetCC()
4962 N1.getOpcode() == ISD::XOR) in SimplifySetCC()
4971 if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) && in SimplifySetCC()
4972 N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) { in SimplifySetCC()
4978 if (N0.getOpcode() == ISD::UREM) { in SimplifySetCC()
4981 } else if (N0.getOpcode() == ISD::SREM) { in SimplifySetCC()
4993 case ISD::SETEQ: // X == Y -> ~(X^Y) in SimplifySetCC()
4994 Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1); in SimplifySetCC()
4999 case ISD::SETNE: // X != Y --> (X^Y) in SimplifySetCC()
5000 N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1); in SimplifySetCC()
5002 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y in SimplifySetCC()
5003 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y in SimplifySetCC()
5005 N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp); in SimplifySetCC()
5009 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X in SimplifySetCC()
5010 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X in SimplifySetCC()
5012 N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp); in SimplifySetCC()
5016 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y in SimplifySetCC()
5017 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y in SimplifySetCC()
5019 N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp); in SimplifySetCC()
5023 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X in SimplifySetCC()
5024 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X in SimplifySetCC()
5026 N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp); in SimplifySetCC()
5033 ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT)); in SimplifySetCC()
5056 if (N->getOpcode() == ISD::ADD) { in isGAPlusOffset()
5178 ISD::NodeType ExtOpc = in LowerAsmOperandForConstraint()
5179 IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND; in LowerAsmOperandForConstraint()
5181 ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue(); in LowerAsmOperandForConstraint()
5205 if (OpCode == ISD::ADD || OpCode == ISD::SUB) { in LowerAsmOperandForConstraint()
5209 else if (OpCode == ISD::ADD && in LowerAsmOperandForConstraint()
5214 Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue(); in LowerAsmOperandForConstraint()
5712 if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern)) in BuildExactSDIV()
5716 if (Op1.getOpcode() == ISD::BUILD_VECTOR) { in BuildExactSDIV()
5719 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) { in BuildExactSDIV()
5738 Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags); in BuildExactSDIV()
5742 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor); in BuildExactSDIV()
5795 !isOperationLegal(ISD::MUL, MulVT)) in BuildSDIV()
5839 if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern)) in BuildSDIV()
5843 if (N1.getOpcode() == ISD::BUILD_VECTOR) { in BuildSDIV()
5848 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) { in BuildSDIV()
5871 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X); in BuildSDIV()
5872 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y); in BuildSDIV()
5873 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y); in BuildSDIV()
5874 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y, in BuildSDIV()
5876 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildSDIV()
5879 if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization)) in BuildSDIV()
5880 return DAG.getNode(ISD::MULHS, dl, VT, X, Y); in BuildSDIV()
5881 if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) { in BuildSDIV()
5883 DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y); in BuildSDIV()
5896 Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor); in BuildSDIV()
5898 Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor); in BuildSDIV()
5902 Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift); in BuildSDIV()
5907 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift); in BuildSDIV()
5909 T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask); in BuildSDIV()
5911 return DAG.getNode(ISD::ADD, dl, VT, Q, T); in BuildSDIV()
5943 !isOperationLegal(ISD::MUL, MulVT)) in BuildUDIV()
5996 if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern)) in BuildUDIV()
6000 if (N1.getOpcode() == ISD::BUILD_VECTOR) { in BuildUDIV()
6005 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) { in BuildUDIV()
6021 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift); in BuildUDIV()
6029 X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X); in BuildUDIV()
6030 Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y); in BuildUDIV()
6031 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y); in BuildUDIV()
6032 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y, in BuildUDIV()
6034 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildUDIV()
6037 if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization)) in BuildUDIV()
6038 return DAG.getNode(ISD::MULHU, dl, VT, X, Y); in BuildUDIV()
6039 if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) { in BuildUDIV()
6041 DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y); in BuildUDIV()
6055 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q); in BuildUDIV()
6063 NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT)); in BuildUDIV()
6067 Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q); in BuildUDIV()
6071 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift); in BuildUDIV()
6077 SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ); in BuildUDIV()
6116 ISD::CondCode Cond, in buildUREMEqFold()
6132 SDValue CompTargetNode, ISD::CondCode Cond, in prepareUREMEqFold()
6140 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in prepareUREMEqFold()
6151 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT)) in prepareUREMEqFold()
6247 if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern)) in prepareUREMEqFold()
6260 if (D.getOpcode() == ISD::BUILD_VECTOR) { in prepareUREMEqFold()
6274 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) { in prepareUREMEqFold()
6288 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT)) in prepareUREMEqFold()
6292 N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode); in prepareUREMEqFold()
6296 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal); in prepareUREMEqFold()
6303 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT)) in prepareUREMEqFold()
6306 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal); in prepareUREMEqFold()
6313 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT)); in prepareUREMEqFold()
6328 DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE); in prepareUREMEqFold()
6333 if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) { in prepareUREMEqFold()
6336 SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true, in prepareUREMEqFold()
6338 return DAG.getNode(ISD::VSELECT, DL, SETCCVT, TautologicalInvertedChannels, in prepareUREMEqFold()
6345 if (isOperationLegalOrCustom(ISD::XOR, SETCCVT)) in prepareUREMEqFold()
6346 return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC, in prepareUREMEqFold()
6359 ISD::CondCode Cond, in buildSREMEqFold()
6376 SDValue CompTargetNode, ISD::CondCode Cond, in prepareSREMEqFold()
6389 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && in prepareSREMEqFold()
6401 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT)) in prepareSREMEqFold()
6501 if (!ISD::matchUnaryPredicate(D, BuildSREMPattern)) in prepareSREMEqFold()
6514 if (D.getOpcode() == ISD::BUILD_VECTOR) { in prepareSREMEqFold()
6533 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) { in prepareSREMEqFold()
6551 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal); in prepareSREMEqFold()
6556 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ADD, VT)) in prepareSREMEqFold()
6560 Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal); in prepareSREMEqFold()
6568 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT)) in prepareSREMEqFold()
6571 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal); in prepareSREMEqFold()
6578 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT)); in prepareSREMEqFold()
6592 if (!isOperationLegalOrCustom(ISD::SETEQ, VT) || in prepareSREMEqFold()
6593 !isOperationLegalOrCustom(ISD::AND, VT) || in prepareSREMEqFold()
6595 !isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) in prepareSREMEqFold()
6608 SDValue DivisorIsIntMin = DAG.getSetCC(DL, SETCCVT, D, IntMin, ISD::SETEQ); in prepareSREMEqFold()
6612 SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax); in prepareSREMEqFold()
6621 SDValue Blended = DAG.getNode(ISD::VSELECT, DL, SETCCVT, DivisorIsIntMin, in prepareSREMEqFold()
6653 SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op); in getSqrtInputTest()
6654 return DAG.getSetCC(DL, CCVT, Fabs, NormC, ISD::SETLT); in getSqrtInputTest()
6657 return DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ); in getSqrtInputTest()
6665 if (Op.getOpcode() == ISD::FNEG) { in getNegatedExpression()
6682 if (!Op.hasOneUse() && Opcode != ISD::ConstantFP) { in getNegatedExpression()
6683 bool IsFreeExtend = Opcode == ISD::FP_EXTEND && in getNegatedExpression()
6702 case ISD::ConstantFP: { in getNegatedExpression()
6706 isOperationLegal(ISD::ConstantFP, VT) || in getNegatedExpression()
6724 case ISD::BUILD_VECTOR: { in getNegatedExpression()
6732 (isOperationLegal(ISD::ConstantFP, VT) && in getNegatedExpression()
6733 isOperationLegal(ISD::BUILD_VECTOR, VT)) || in getNegatedExpression()
6756 case ISD::FADD: { in getNegatedExpression()
6761 if (LegalOps && !isOperationLegalOrCustom(ISD::FSUB, VT)) in getNegatedExpression()
6784 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegX, Y, Flags); in getNegatedExpression()
6793 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegY, X, Flags); in getNegatedExpression()
6800 case ISD::FSUB: { in getNegatedExpression()
6815 return DAG.getNode(ISD::FSUB, DL, VT, Y, X, Flags); in getNegatedExpression()
6817 case ISD::FMUL: in getNegatedExpression()
6818 case ISD::FDIV: { in getNegatedExpression()
6848 if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL) in getNegatedExpression()
6861 case ISD::FMA: in getNegatedExpression()
6862 case ISD::FMAD: { in getNegatedExpression()
6913 case ISD::FP_EXTEND: in getNegatedExpression()
6914 case ISD::FSIN: in getNegatedExpression()
6919 case ISD::FP_ROUND: in getNegatedExpression()
6922 return DAG.getNode(ISD::FP_ROUND, DL, VT, NegV, Op.getOperand(1)); in getNegatedExpression()
6939 assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI || in expandMUL_LOHI()
6940 Opcode == ISD::SMUL_LOHI); in expandMUL_LOHI()
6943 isOperationLegalOrCustom(ISD::MULHS, HiLoVT); in expandMUL_LOHI()
6945 isOperationLegalOrCustom(ISD::MULHU, HiLoVT); in expandMUL_LOHI()
6947 isOperationLegalOrCustom(ISD::SMUL_LOHI, HiLoVT); in expandMUL_LOHI()
6949 isOperationLegalOrCustom(ISD::UMUL_LOHI, HiLoVT); in expandMUL_LOHI()
6965 Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R); in expandMUL_LOHI()
6970 Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R); in expandMUL_LOHI()
6971 Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R); in expandMUL_LOHI()
6980 isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) { in expandMUL_LOHI()
6981 LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS); in expandMUL_LOHI()
6982 RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS); in expandMUL_LOHI()
6995 if (Opcode != ISD::MUL) { in expandMUL_LOHI()
7004 if (!VT.isVector() && Opcode == ISD::MUL && in expandMUL_LOHI()
7021 isOperationLegalOrCustom(ISD::SRL, VT) && in expandMUL_LOHI()
7022 isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) { in expandMUL_LOHI()
7023 LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift); in expandMUL_LOHI()
7024 LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH); in expandMUL_LOHI()
7025 RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift); in expandMUL_LOHI()
7026 RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH); in expandMUL_LOHI()
7037 if (Opcode == ISD::MUL) { in expandMUL_LOHI()
7038 RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH); in expandMUL_LOHI()
7039 LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL); in expandMUL_LOHI()
7040 Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH); in expandMUL_LOHI()
7041 Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH); in expandMUL_LOHI()
7048 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); in expandMUL_LOHI()
7049 Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi); in expandMUL_LOHI()
7050 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); in expandMUL_LOHI()
7051 return DAG.getNode(ISD::OR, dl, VT, Lo, Hi); in expandMUL_LOHI()
7054 SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi); in expandMUL_LOHI()
7060 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi)); in expandMUL_LOHI()
7068 bool UseGlue = (isOperationLegalOrCustom(ISD::ADDC, VT) && in expandMUL_LOHI()
7069 isOperationLegalOrCustom(ISD::ADDE, VT)); in expandMUL_LOHI()
7071 Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next, in expandMUL_LOHI()
7074 Next = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(VT, BoolType), Next, in expandMUL_LOHI()
7078 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next)); in expandMUL_LOHI()
7079 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift); in expandMUL_LOHI()
7081 if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI)) in expandMUL_LOHI()
7085 Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero, in expandMUL_LOHI()
7088 Hi = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(HiLoVT, BoolType), Hi, in expandMUL_LOHI()
7091 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi)); in expandMUL_LOHI()
7093 if (Opcode == ISD::SMUL_LOHI) { in expandMUL_LOHI()
7094 SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next, in expandMUL_LOHI()
7095 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL)); in expandMUL_LOHI()
7096 Next = DAG.getSelectCC(dl, LH, Zero, NextSub, Next, ISD::SETLT); in expandMUL_LOHI()
7098 NextSub = DAG.getNode(ISD::SUB, dl, VT, Next, in expandMUL_LOHI()
7099 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL)); in expandMUL_LOHI()
7100 Next = DAG.getSelectCC(dl, RH, Zero, NextSub, Next, ISD::SETLT); in expandMUL_LOHI()
7103 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next)); in expandMUL_LOHI()
7104 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift); in expandMUL_LOHI()
7105 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next)); in expandMUL_LOHI()
7127 return ISD::matchUnaryPredicate( in isNonZeroModBitWidthOrUndef()
7137 if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) || in expandFunnelShift()
7138 !isOperationLegalOrCustom(ISD::SRL, VT) || in expandFunnelShift()
7139 !isOperationLegalOrCustom(ISD::SUB, VT) || in expandFunnelShift()
7140 !isOperationLegalOrCustomOrPromote(ISD::OR, VT))) in expandFunnelShift()
7148 bool IsFSHL = Node->getOpcode() == ISD::FSHL; in expandFunnelShift()
7154 unsigned RevOpcode = IsFSHL ? ISD::FSHR : ISD::FSHL; in expandFunnelShift()
7161 Z = DAG.getNode(ISD::SUB, DL, VT, Zero, Z); in expandFunnelShift()
7168 X = DAG.getNode(ISD::SRL, DL, VT, X, One); in expandFunnelShift()
7171 Y = DAG.getNode(ISD::SHL, DL, VT, Y, One); in expandFunnelShift()
7185 ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC); in expandFunnelShift()
7186 InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, ShAmt); in expandFunnelShift()
7187 ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt); in expandFunnelShift()
7188 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt); in expandFunnelShift()
7195 ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Z, Mask); in expandFunnelShift()
7197 InvShAmt = DAG.getNode(ISD::AND, DL, ShVT, DAG.getNOT(DL, Z, ShVT), Mask); in expandFunnelShift()
7200 ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC); in expandFunnelShift()
7201 InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, Mask, ShAmt); in expandFunnelShift()
7206 ShX = DAG.getNode(ISD::SHL, DL, VT, X, ShAmt); in expandFunnelShift()
7207 SDValue ShY1 = DAG.getNode(ISD::SRL, DL, VT, Y, One); in expandFunnelShift()
7208 ShY = DAG.getNode(ISD::SRL, DL, VT, ShY1, InvShAmt); in expandFunnelShift()
7210 SDValue ShX1 = DAG.getNode(ISD::SHL, DL, VT, X, One); in expandFunnelShift()
7211 ShX = DAG.getNode(ISD::SHL, DL, VT, ShX1, InvShAmt); in expandFunnelShift()
7212 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt); in expandFunnelShift()
7215 return DAG.getNode(ISD::OR, DL, VT, ShX, ShY); in expandFunnelShift()
7223 bool IsLeft = Node->getOpcode() == ISD::ROTL; in expandROT()
7232 unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL; in expandROT()
7235 SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1); in expandROT()
7240 (!isOperationLegalOrCustom(ISD::SHL, VT) || in expandROT()
7241 !isOperationLegalOrCustom(ISD::SRL, VT) || in expandROT()
7242 !isOperationLegalOrCustom(ISD::SUB, VT) || in expandROT()
7243 !isOperationLegalOrCustomOrPromote(ISD::OR, VT) || in expandROT()
7244 !isOperationLegalOrCustomOrPromote(ISD::AND, VT))) in expandROT()
7247 unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL; in expandROT()
7248 unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL; in expandROT()
7255 SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1); in expandROT()
7256 SDValue ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Op1, BitWidthMinusOneC); in expandROT()
7258 SDValue HsAmt = DAG.getNode(ISD::AND, DL, ShVT, NegOp1, BitWidthMinusOneC); in expandROT()
7264 SDValue ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Op1, BitWidthC); in expandROT()
7266 SDValue HsAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthMinusOneC, ShAmt); in expandROT()
7271 return DAG.getNode(ISD::OR, DL, VT, ShVal, HsVal); in expandROT()
7281 bool IsSHL = Node->getOpcode() == ISD::SHL_PARTS; in expandShiftParts()
7282 bool IsSRA = Node->getOpcode() == ISD::SRA_PARTS; in expandShiftParts()
7294 SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt, in expandShiftParts()
7296 SDValue Tmp1 = IsSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi, in expandShiftParts()
7302 Tmp2 = DAG.getNode(ISD::FSHL, dl, VT, ShOpHi, ShOpLo, ShAmt); in expandShiftParts()
7303 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt); in expandShiftParts()
7305 Tmp2 = DAG.getNode(ISD::FSHR, dl, VT, ShOpHi, ShOpLo, ShAmt); in expandShiftParts()
7306 Tmp3 = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt); in expandShiftParts()
7312 SDValue AndNode = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt, in expandShiftParts()
7315 DAG.getConstant(0, dl, ShAmtVT), ISD::SETNE); in expandShiftParts()
7318 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2); in expandShiftParts()
7319 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3); in expandShiftParts()
7321 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2); in expandShiftParts()
7322 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3); in expandShiftParts()
7359 SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Src); in expandFP_TO_SINT()
7362 ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask), in expandFP_TO_SINT()
7364 SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias); in expandFP_TO_SINT()
7366 SDValue Sign = DAG.getNode(ISD::SRA, dl, IntVT, in expandFP_TO_SINT()
7367 DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask), in expandFP_TO_SINT()
7371 SDValue R = DAG.getNode(ISD::OR, dl, IntVT, in expandFP_TO_SINT()
7372 DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask), in expandFP_TO_SINT()
7379 DAG.getNode(ISD::SHL, dl, DstVT, R, in expandFP_TO_SINT()
7381 DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit), in expandFP_TO_SINT()
7383 DAG.getNode(ISD::SRL, dl, DstVT, R, in expandFP_TO_SINT()
7385 DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent), in expandFP_TO_SINT()
7387 ISD::SETGT); in expandFP_TO_SINT()
7389 SDValue Ret = DAG.getNode(ISD::SUB, dl, DstVT, in expandFP_TO_SINT()
7390 DAG.getNode(ISD::XOR, dl, DstVT, R, Sign), Sign); in expandFP_TO_SINT()
7393 DAG.getConstant(0, dl, DstVT), Ret, ISD::SETLT); in expandFP_TO_SINT()
7412 unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT : in expandFP_TO_UINT()
7413 ISD::FP_TO_SINT; in expandFP_TO_UINT()
7415 !isOperationLegalOrCustomOrPromote(ISD::XOR, SrcVT))) in expandFP_TO_UINT()
7427 Result = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other }, in expandFP_TO_UINT()
7431 Result = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src); in expandFP_TO_UINT()
7437 Node->isStrictFPOpcode() ? ISD::STRICT_FSUB : ISD::FSUB, SrcVT)) in expandFP_TO_UINT()
7444 Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT, in expandFP_TO_UINT()
7448 Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT); in expandFP_TO_UINT()
7471 SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl, { SrcVT, MVT::Other }, in expandFP_TO_UINT()
7473 SInt = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other }, in expandFP_TO_UINT()
7477 SDValue Val = DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FltOfs); in expandFP_TO_UINT()
7478 SInt = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Val); in expandFP_TO_UINT()
7480 Result = DAG.getNode(ISD::XOR, dl, DstVT, SInt, IntOfs); in expandFP_TO_UINT()
7487 SDValue True = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src); in expandFP_TO_UINT()
7489 SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, in expandFP_TO_UINT()
7490 DAG.getNode(ISD::FSUB, dl, SrcVT, Src, Cst)); in expandFP_TO_UINT()
7491 False = DAG.getNode(ISD::XOR, dl, DstVT, False, in expandFP_TO_UINT()
7516 if (SrcVT.isVector() && (!isOperationLegalOrCustom(ISD::SRL, SrcVT) || in expandUINT_TO_FP()
7517 !isOperationLegalOrCustom(ISD::FADD, DstVT) || in expandUINT_TO_FP()
7518 !isOperationLegalOrCustom(ISD::FSUB, DstVT) || in expandUINT_TO_FP()
7519 !isOperationLegalOrCustomOrPromote(ISD::OR, SrcVT) || in expandUINT_TO_FP()
7520 !isOperationLegalOrCustomOrPromote(ISD::AND, SrcVT))) in expandUINT_TO_FP()
7538 SDValue Lo = DAG.getNode(ISD::AND, dl, SrcVT, Src, LoMask); in expandUINT_TO_FP()
7539 SDValue Hi = DAG.getNode(ISD::SRL, dl, SrcVT, Src, HiShift); in expandUINT_TO_FP()
7540 SDValue LoOr = DAG.getNode(ISD::OR, dl, SrcVT, Lo, TwoP52); in expandUINT_TO_FP()
7541 SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84); in expandUINT_TO_FP()
7545 DAG.getNode(ISD::FSUB, dl, DstVT, HiFlt, TwoP84PlusTwoP52); in expandUINT_TO_FP()
7546 Result = DAG.getNode(ISD::FADD, dl, DstVT, LoFlt, HiSub); in expandUINT_TO_FP()
7554 assert((Opcode == ISD::FMINNUM || Opcode == ISD::FMAXNUM || in createSelectForFMINNUM_FMAXNUM()
7555 Opcode == ISD::STRICT_FMINNUM || Opcode == ISD::STRICT_FMAXNUM) && in createSelectForFMINNUM_FMAXNUM()
7559 ISD::CondCode Pred = Opcode == ISD::FMINNUM ? ISD::SETLT : ISD::SETGT; in createSelectForFMINNUM_FMAXNUM()
7577 unsigned NewOp = Node->getOpcode() == ISD::FMINNUM ? in expandFMINNUM_FMAXNUM()
7578 ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE; in expandFMINNUM_FMAXNUM()
7593 Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0, in expandFMINNUM_FMAXNUM()
7597 Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1, in expandFMINNUM_FMAXNUM()
7609 Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM; in expandFMINNUM_FMAXNUM()
7638 Op = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::f64, Op, in expandIS_FPCLASS()
7660 isOperationLegalOrCustom(ISD::SETCC, OperandVT.getScalarType())) { in expandIS_FPCLASS()
7664 IsInverted ? ISD::SETUNE : ISD::SETOEQ); in expandIS_FPCLASS()
7667 IsInverted ? ISD::SETO : ISD::SETUO); in expandIS_FPCLASS()
7702 Res = DAG.getNode(ISD::OR, DL, ResultVT, Res, PartialRes); in expandIS_FPCLASS()
7714 SDValue IntBitV = DAG.getNode(ISD::AND, DL, IntVT, OpAsInt, IntBitMaskV); in expandIS_FPCLASS()
7715 IntBitIsSetV = DAG.getSetCC(DL, ResultVT, IntBitV, ZeroV, ISD::SETNE); in expandIS_FPCLASS()
7721 SDValue AbsV = DAG.getNode(ISD::AND, DL, IntVT, OpAsInt, ValueMaskV); in expandIS_FPCLASS()
7723 DAG.getConstant(0.0, DL, IntVT), ISD::SETLT); in expandIS_FPCLASS()
7733 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, ExpMaskV, ISD::SETLT); in expandIS_FPCLASS()
7737 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, ExpMaskV, ISD::SETULT); in expandIS_FPCLASS()
7741 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, ExpMaskV, ISD::SETLT); in expandIS_FPCLASS()
7742 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, SignV); in expandIS_FPCLASS()
7751 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, ZeroV, ISD::SETEQ); in expandIS_FPCLASS()
7753 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, ZeroV, ISD::SETEQ); in expandIS_FPCLASS()
7755 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, SignBitV, ISD::SETEQ); in expandIS_FPCLASS()
7761 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, InfV, ISD::SETEQ); in expandIS_FPCLASS()
7763 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, InfV, ISD::SETEQ); in expandIS_FPCLASS()
7767 PartialRes = DAG.getSetCC(DL, ResultVT, OpAsInt, NegInfV, ISD::SETEQ); in expandIS_FPCLASS()
7777 PartialRes = DAG.getSetCC(DL, ResultVT, AbsV, InfV, ISD::SETGT); in expandIS_FPCLASS()
7781 SDValue ExpBits = DAG.getNode(ISD::AND, DL, IntVT, AbsV, ExpMaskV); in expandIS_FPCLASS()
7783 DAG.getSetCC(DL, ResultVT, ExpBits, ZeroV, ISD::SETEQ); in expandIS_FPCLASS()
7785 DAG.getSetCC(DL, ResultVT, getIntBitIsSet(), ExpIsZero, ISD::SETEQ); in expandIS_FPCLASS()
7786 PartialRes = DAG.getNode(ISD::OR, DL, ResultVT, PartialRes, IsPseudo); in expandIS_FPCLASS()
7791 DAG.getSetCC(DL, ResultVT, AbsV, InfWithQnanBitV, ISD::SETGE); in expandIS_FPCLASS()
7795 SDValue IsNan = DAG.getSetCC(DL, ResultVT, AbsV, InfV, ISD::SETGT); in expandIS_FPCLASS()
7797 DAG.getSetCC(DL, ResultVT, AbsV, InfWithQnanBitV, ISD::SETLT); in expandIS_FPCLASS()
7798 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, IsNan, IsNotQnan); in expandIS_FPCLASS()
7809 DAG.getNode(ISD::SUB, DL, IntVT, V, DAG.getConstant(1, DL, IntVT)); in expandIS_FPCLASS()
7810 PartialRes = DAG.getSetCC(DL, ResultVT, VMinusOneV, MantissaV, ISD::SETULT); in expandIS_FPCLASS()
7812 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, SignV); in expandIS_FPCLASS()
7820 SDValue ExpMinus1 = DAG.getNode(ISD::SUB, DL, IntVT, AbsV, ExpLSBV); in expandIS_FPCLASS()
7823 PartialRes = DAG.getSetCC(DL, ResultVT, ExpMinus1, ExpLimitV, ISD::SETULT); in expandIS_FPCLASS()
7825 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, SignV); in expandIS_FPCLASS()
7828 DAG.getNode(ISD::XOR, DL, ResultVT, SignV, ResultInvertionMask); in expandIS_FPCLASS()
7829 PartialRes = DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, PosSignV); in expandIS_FPCLASS()
7833 DAG.getNode(ISD::AND, DL, ResultVT, PartialRes, getIntBitIsSet()); in expandIS_FPCLASS()
7840 Res = DAG.getNode(ISD::XOR, DL, ResultVT, Res, ResultInvertionMask); in expandIS_FPCLASS()
7848 return TLI.isOperationLegalOrCustom(ISD::ADD, VT) && in canExpandVectorCTPOP()
7849 TLI.isOperationLegalOrCustom(ISD::SUB, VT) && in canExpandVectorCTPOP()
7850 TLI.isOperationLegalOrCustom(ISD::SRL, VT) && in canExpandVectorCTPOP()
7851 (Len == 8 || TLI.isOperationLegalOrCustom(ISD::MUL, VT)) && in canExpandVectorCTPOP()
7852 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT); in canExpandVectorCTPOP()
7881 Op = DAG.getNode(ISD::SUB, dl, VT, Op, in expandCTPOP()
7882 DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
7883 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
7887 Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, dl, VT, Op, Mask33), in expandCTPOP()
7888 DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
7889 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
7893 Op = DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
7894 DAG.getNode(ISD::ADD, dl, VT, Op, in expandCTPOP()
7895 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
7907 return DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
7908 DAG.getNode(ISD::ADD, dl, VT, Op, in expandCTPOP()
7909 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
7917 return DAG.getNode(ISD::SRL, dl, VT, in expandCTPOP()
7918 DAG.getNode(ISD::MUL, dl, VT, Op, Mask01), in expandCTPOP()
7930 if (Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF && in expandCTLZ()
7931 isOperationLegalOrCustom(ISD::CTLZ, VT)) in expandCTLZ()
7932 return DAG.getNode(ISD::CTLZ, dl, VT, Op); in expandCTLZ()
7935 if (isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) { in expandCTLZ()
7938 SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op); in expandCTLZ()
7940 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ); in expandCTLZ()
7948 (!isOperationLegalOrCustom(ISD::CTPOP, VT) && in expandCTLZ()
7950 !isOperationLegalOrCustom(ISD::SRL, VT) || in expandCTLZ()
7951 !isOperationLegalOrCustomOrPromote(ISD::OR, VT))) in expandCTLZ()
7965 Op = DAG.getNode(ISD::OR, dl, VT, Op, in expandCTLZ()
7966 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp)); in expandCTLZ()
7969 return DAG.getNode(ISD::CTPOP, dl, VT, Op); in expandCTLZ()
7979 if (Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF && in expandCTTZ()
7980 isOperationLegalOrCustom(ISD::CTTZ, VT)) in expandCTTZ()
7981 return DAG.getNode(ISD::CTTZ, dl, VT, Op); in expandCTTZ()
7984 if (isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) { in expandCTTZ()
7987 SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op); in expandCTTZ()
7989 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ); in expandCTTZ()
7997 (!isOperationLegalOrCustom(ISD::CTPOP, VT) && in expandCTTZ()
7998 !isOperationLegalOrCustom(ISD::CTLZ, VT) && in expandCTTZ()
8000 !isOperationLegalOrCustom(ISD::SUB, VT) || in expandCTTZ()
8001 !isOperationLegalOrCustomOrPromote(ISD::AND, VT) || in expandCTTZ()
8002 !isOperationLegalOrCustomOrPromote(ISD::XOR, VT))) in expandCTTZ()
8010 ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT), in expandCTTZ()
8011 DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT))); in expandCTTZ()
8014 if (isOperationLegal(ISD::CTLZ, VT) && !isOperationLegal(ISD::CTPOP, VT)) { in expandCTTZ()
8015 return DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT), in expandCTTZ()
8016 DAG.getNode(ISD::CTLZ, dl, VT, Tmp)); in expandCTTZ()
8019 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp); in expandCTTZ()
8030 if (!IsNegative && isOperationLegal(ISD::SUB, VT) && in expandABS()
8031 isOperationLegal(ISD::SMAX, VT)) { in expandABS()
8033 return DAG.getNode(ISD::SMAX, dl, VT, Op, in expandABS()
8034 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
8038 if (!IsNegative && isOperationLegal(ISD::SUB, VT) && in expandABS()
8039 isOperationLegal(ISD::UMIN, VT)) { in expandABS()
8042 return DAG.getNode(ISD::UMIN, dl, VT, Op, in expandABS()
8043 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
8047 if (IsNegative && isOperationLegal(ISD::SUB, VT) && in expandABS()
8048 isOperationLegal(ISD::SMIN, VT)) { in expandABS()
8051 return DAG.getNode(ISD::SMIN, dl, VT, Op, in expandABS()
8052 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
8057 (!isOperationLegalOrCustom(ISD::SRA, VT) || in expandABS()
8058 (!IsNegative && !isOperationLegalOrCustom(ISD::ADD, VT)) || in expandABS()
8059 (IsNegative && !isOperationLegalOrCustom(ISD::SUB, VT)) || in expandABS()
8060 !isOperationLegalOrCustomOrPromote(ISD::XOR, VT))) in expandABS()
8065 DAG.getNode(ISD::SRA, dl, VT, Op, in expandABS()
8067 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, Shift); in expandABS()
8071 return DAG.getNode(ISD::SUB, dl, VT, Xor, Shift); in expandABS()
8074 return DAG.getNode(ISD::SUB, dl, VT, Shift, Xor); in expandABS()
8092 return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
8094 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
8095 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
8096 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
8097 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
8098 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, in expandBSWAP()
8100 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT)); in expandBSWAP()
8101 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); in expandBSWAP()
8102 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); in expandBSWAP()
8103 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); in expandBSWAP()
8105 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); in expandBSWAP()
8106 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); in expandBSWAP()
8107 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
8108 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
8109 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
8110 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
8111 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); in expandBSWAP()
8112 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); in expandBSWAP()
8113 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, in expandBSWAP()
8115 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, in expandBSWAP()
8117 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, in expandBSWAP()
8119 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, in expandBSWAP()
8121 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, in expandBSWAP()
8123 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, in expandBSWAP()
8125 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); in expandBSWAP()
8126 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); in expandBSWAP()
8127 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); in expandBSWAP()
8128 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); in expandBSWAP()
8129 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); in expandBSWAP()
8130 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); in expandBSWAP()
8131 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); in expandBSWAP()
8154 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op); in expandBITREVERSE()
8157 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT)); in expandBITREVERSE()
8158 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask4, dl, VT)); in expandBITREVERSE()
8159 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT)); in expandBITREVERSE()
8160 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT)); in expandBITREVERSE()
8161 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
8164 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT)); in expandBITREVERSE()
8165 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask2, dl, VT)); in expandBITREVERSE()
8166 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT)); in expandBITREVERSE()
8167 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT)); in expandBITREVERSE()
8168 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
8171 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT)); in expandBITREVERSE()
8172 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask1, dl, VT)); in expandBITREVERSE()
8173 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT)); in expandBITREVERSE()
8174 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT)); in expandBITREVERSE()
8175 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
8183 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT)); in expandBITREVERSE()
8186 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT)); in expandBITREVERSE()
8190 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT)); in expandBITREVERSE()
8191 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2); in expandBITREVERSE()
8205 ISD::LoadExtType ExtType = LD->getExtensionType(); in scalarizeVectorLoad()
8235 DAG.getExtLoad(ISD::EXTLOAD, SL, LoadVT, Chain, BasePTR, in scalarizeVectorLoad()
8246 SDValue ShiftedElt = DAG.getNode(ISD::SRL, SL, LoadVT, Load, ShiftAmount); in scalarizeVectorLoad()
8248 DAG.getNode(ISD::AND, SL, LoadVT, ShiftedElt, SrcEltBitMask); in scalarizeVectorLoad()
8249 SDValue Scalar = DAG.getNode(ISD::TRUNCATE, SL, SrcEltVT, Elt); in scalarizeVectorLoad()
8251 if (ExtType != ISD::NON_EXTLOAD) { in scalarizeVectorLoad()
8252 unsigned ExtendOp = ISD::getExtForLoadExtType(false, ExtType); in scalarizeVectorLoad()
8282 SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains); in scalarizeVectorLoad()
8322 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value, in scalarizeVectorStore()
8324 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MemSclVT, Elt); in scalarizeVectorStore()
8325 SDValue ExtElt = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Trunc); in scalarizeVectorStore()
8331 DAG.getNode(ISD::SHL, SL, IntVT, ExtElt, ShiftAmount); in scalarizeVectorStore()
8332 CurrVal = DAG.getNode(ISD::OR, SL, IntVT, CurrVal, ShiftedElt); in scalarizeVectorStore()
8347 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value, in scalarizeVectorStore()
8362 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Stores); in scalarizeVectorStore()
8367 assert(LD->getAddressingMode() == ISD::UNINDEXED && in expandUnalignedLoad()
8379 if (!isOperationLegalOrCustom(ISD::LOAD, intVT) && in expandUnalignedLoad()
8389 SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad); in expandUnalignedLoad()
8391 Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND : in expandUnalignedLoad()
8392 ISD::ANY_EXTEND, dl, VT, Result); in expandUnalignedLoad()
8439 DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr, in expandUnalignedLoad()
8451 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); in expandUnalignedLoad()
8474 ISD::LoadExtType HiExtType = LD->getExtensionType(); in expandUnalignedLoad()
8477 if (HiExtType == ISD::NON_EXTLOAD) in expandUnalignedLoad()
8478 HiExtType = ISD::ZEXTLOAD; in expandUnalignedLoad()
8483 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(), in expandUnalignedLoad()
8498 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, in expandUnalignedLoad()
8508 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount); in expandUnalignedLoad()
8509 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo); in expandUnalignedLoad()
8511 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), in expandUnalignedLoad()
8519 assert(ST->getAddressingMode() == ISD::UNINDEXED && in expandUnalignedStore()
8533 if (!isOperationLegalOrCustom(ISD::STORE, intVT) && in expandUnalignedStore()
8542 SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val); in expandUnalignedStore()
8598 ISD::EXTLOAD, dl, RegVT, Store, StackPtr, in expandUnalignedStore()
8607 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); in expandUnalignedStore()
8622 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount); in expandUnalignedStore()
8638 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); in expandUnalignedStore()
8660 MaskInIntReg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, MaskInIntReg); in IncrementMemoryAddress()
8665 Increment = DAG.getNode(ISD::CTPOP, DL, MaskIntVT, MaskInIntReg); in IncrementMemoryAddress()
8670 Increment = DAG.getNode(ISD::MUL, DL, AddrVT, Increment, Scale); in IncrementMemoryAddress()
8678 return DAG.getNode(ISD::ADD, DL, AddrVT, Addr, Increment); in IncrementMemoryAddress()
8700 unsigned SubOpcode = NumSubElts <= NElts ? ISD::SUB : ISD::USUBSAT; in clampDynamicVectorIndex()
8703 return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx, Sub); in clampDynamicVectorIndex()
8707 return DAG.getNode(ISD::AND, dl, IdxVT, Idx, in clampDynamicVectorIndex()
8711 return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx, in clampDynamicVectorIndex()
8746 DAG.getNode(ISD::MUL, dl, IdxVT, Index, in getVectorSubVecPointer()
8749 Index = DAG.getNode(ISD::MUL, dl, IdxVT, Index, in getVectorSubVecPointer()
8797 assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node."); in lowerCmpEqZeroToCtlzSrl()
8800 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); in lowerCmpEqZeroToCtlzSrl()
8803 if (C->isZero() && CC == ISD::SETEQ) { in lowerCmpEqZeroToCtlzSrl()
8808 Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0)); in lowerCmpEqZeroToCtlzSrl()
8811 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext); in lowerCmpEqZeroToCtlzSrl()
8812 SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz, in lowerCmpEqZeroToCtlzSrl()
8814 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc); in lowerCmpEqZeroToCtlzSrl()
8828 if (Opcode == ISD::UMIN && isOperationLegal(ISD::SUB, VT) && in expandIntMINMAX()
8829 isOperationLegal(ISD::USUBSAT, VT)) { in expandIntMINMAX()
8830 return DAG.getNode(ISD::SUB, DL, VT, Op0, in expandIntMINMAX()
8831 DAG.getNode(ISD::USUBSAT, DL, VT, Op0, Op1)); in expandIntMINMAX()
8835 if (Opcode == ISD::UMAX && isOperationLegal(ISD::ADD, VT) && in expandIntMINMAX()
8836 isOperationLegal(ISD::USUBSAT, VT)) { in expandIntMINMAX()
8837 return DAG.getNode(ISD::ADD, DL, VT, Op0, in expandIntMINMAX()
8838 DAG.getNode(ISD::USUBSAT, DL, VT, Op1, Op0)); in expandIntMINMAX()
8842 ISD::CondCode CC; in expandIntMINMAX()
8845 case ISD::SMAX: CC = ISD::SETGT; break; in expandIntMINMAX()
8846 case ISD::SMIN: CC = ISD::SETLT; break; in expandIntMINMAX()
8847 case ISD::UMAX: CC = ISD::SETUGT; break; in expandIntMINMAX()
8848 case ISD::UMIN: CC = ISD::SETULT; break; in expandIntMINMAX()
8853 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT)) in expandIntMINMAX()
8872 if (Opcode == ISD::USUBSAT && isOperationLegal(ISD::UMAX, VT)) { in expandAddSubSat()
8873 SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS); in expandAddSubSat()
8874 return DAG.getNode(ISD::SUB, dl, VT, Max, RHS); in expandAddSubSat()
8878 if (Opcode == ISD::UADDSAT && isOperationLegal(ISD::UMIN, VT)) { in expandAddSubSat()
8880 SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS); in expandAddSubSat()
8881 return DAG.getNode(ISD::ADD, dl, VT, Min, RHS); in expandAddSubSat()
8886 case ISD::SADDSAT: in expandAddSubSat()
8887 OverflowOp = ISD::SADDO; in expandAddSubSat()
8889 case ISD::UADDSAT: in expandAddSubSat()
8890 OverflowOp = ISD::UADDO; in expandAddSubSat()
8892 case ISD::SSUBSAT: in expandAddSubSat()
8893 OverflowOp = ISD::SSUBO; in expandAddSubSat()
8895 case ISD::USUBSAT: in expandAddSubSat()
8896 OverflowOp = ISD::USUBO; in expandAddSubSat()
8905 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT)) in expandAddSubSat()
8916 if (Opcode == ISD::UADDSAT) { in expandAddSubSat()
8920 return DAG.getNode(ISD::OR, dl, VT, SumDiff, OverflowMask); in expandAddSubSat()
8926 if (Opcode == ISD::USUBSAT) { in expandAddSubSat()
8931 return DAG.getNode(ISD::AND, dl, VT, SumDiff, Not); in expandAddSubSat()
8940 SDValue Shift = DAG.getNode(ISD::SRA, dl, VT, SumDiff, in expandAddSubSat()
8942 Result = DAG.getNode(ISD::XOR, dl, VT, Shift, SatMin); in expandAddSubSat()
8948 bool IsSigned = Opcode == ISD::SSHLSAT; in expandShlSat()
8954 assert((Node->getOpcode() == ISD::SSHLSAT || in expandShlSat()
8955 Node->getOpcode() == ISD::USHLSAT) && in expandShlSat()
8963 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, LHS, RHS); in expandShlSat()
8965 DAG.getNode(IsSigned ? ISD::SRA : ISD::SRL, dl, VT, Result, RHS); in expandShlSat()
8972 SatMin, SatMax, ISD::SETLT); in expandShlSat()
8976 Result = DAG.getSelectCC(dl, LHS, Orig, SatVal, Result, ISD::SETNE); in expandShlSat()
8983 assert((Node->getOpcode() == ISD::SMULFIX || in expandFixedPointMul()
8984 Node->getOpcode() == ISD::UMULFIX || in expandFixedPointMul()
8985 Node->getOpcode() == ISD::SMULFIXSAT || in expandFixedPointMul()
8986 Node->getOpcode() == ISD::UMULFIXSAT) && in expandFixedPointMul()
8994 bool Saturating = (Node->getOpcode() == ISD::SMULFIXSAT || in expandFixedPointMul()
8995 Node->getOpcode() == ISD::UMULFIXSAT); in expandFixedPointMul()
8996 bool Signed = (Node->getOpcode() == ISD::SMULFIX || in expandFixedPointMul()
8997 Node->getOpcode() == ISD::SMULFIXSAT); in expandFixedPointMul()
9004 if (isOperationLegalOrCustom(ISD::MUL, VT)) in expandFixedPointMul()
9005 return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandFixedPointMul()
9006 } else if (Signed && isOperationLegalOrCustom(ISD::SMULO, VT)) { in expandFixedPointMul()
9008 DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); in expandFixedPointMul()
9019 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); in expandFixedPointMul()
9020 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT); in expandFixedPointMul()
9023 } else if (!Signed && isOperationLegalOrCustom(ISD::UMULO, VT)) { in expandFixedPointMul()
9025 DAG.getNode(ISD::UMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); in expandFixedPointMul()
9043 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI; in expandFixedPointMul()
9044 unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU; in expandFixedPointMul()
9050 Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandFixedPointMul()
9068 SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo, in expandFixedPointMul()
9084 ISD::SETUGT); in expandFixedPointMul()
9096 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, Lo, in expandFixedPointMul()
9098 SDValue Overflow = DAG.getSetCC(dl, BoolVT, Hi, Sign, ISD::SETNE); in expandFixedPointMul()
9103 ISD::SETLT); in expandFixedPointMul()
9114 Result = DAG.getSelectCC(dl, Hi, LowMask, SatMax, Result, ISD::SETGT); in expandFixedPointMul()
9120 Result = DAG.getSelectCC(dl, Hi, HighMask, SatMin, Result, ISD::SETLT); in expandFixedPointMul()
9128 assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT || in expandFixedPointDiv()
9129 Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) && in expandFixedPointDiv()
9133 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT; in expandFixedPointDiv()
9134 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT; in expandFixedPointDiv()
9166 LHS = DAG.getNode(ISD::SHL, dl, VT, LHS, in expandFixedPointDiv()
9169 RHS = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, VT, RHS, in expandFixedPointDiv()
9182 isOperationLegalOrCustom(ISD::SDIVREM, VT)) { in expandFixedPointDiv()
9183 Quot = DAG.getNode(ISD::SDIVREM, dl, in expandFixedPointDiv()
9189 Quot = DAG.getNode(ISD::SDIV, dl, VT, in expandFixedPointDiv()
9191 Rem = DAG.getNode(ISD::SREM, dl, VT, in expandFixedPointDiv()
9195 SDValue RemNonZero = DAG.getSetCC(dl, BoolVT, Rem, Zero, ISD::SETNE); in expandFixedPointDiv()
9196 SDValue LHSNeg = DAG.getSetCC(dl, BoolVT, LHS, Zero, ISD::SETLT); in expandFixedPointDiv()
9197 SDValue RHSNeg = DAG.getSetCC(dl, BoolVT, RHS, Zero, ISD::SETLT); in expandFixedPointDiv()
9198 SDValue QuotNeg = DAG.getNode(ISD::XOR, dl, BoolVT, LHSNeg, RHSNeg); in expandFixedPointDiv()
9199 SDValue Sub1 = DAG.getNode(ISD::SUB, dl, VT, Quot, in expandFixedPointDiv()
9202 DAG.getNode(ISD::AND, dl, BoolVT, RemNonZero, QuotNeg), in expandFixedPointDiv()
9205 Quot = DAG.getNode(ISD::UDIV, dl, VT, in expandFixedPointDiv()
9216 bool IsAdd = Node->getOpcode() == ISD::UADDO; in expandUADDSUBO()
9219 unsigned OpcCarry = IsAdd ? ISD::ADDCARRY : ISD::SUBCARRY; in expandUADDSUBO()
9229 Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl, in expandUADDSUBO()
9244 DAG.getConstant(0, dl, Node->getValueType(0)), ISD::SETEQ); in expandUADDSUBO()
9246 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT; in expandUADDSUBO()
9257 bool IsAdd = Node->getOpcode() == ISD::SADDO; in expandSADDSUBO()
9259 Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl, in expandSADDSUBO()
9267 unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT; in expandSADDSUBO()
9270 SDValue SetCC = DAG.getSetCC(dl, OType, Result, Sat, ISD::SETNE); in expandSADDSUBO()
9283 SDValue ResultLowerThanLHS = DAG.getSetCC(dl, OType, Result, LHS, ISD::SETLT); in expandSADDSUBO()
9285 DAG.getSetCC(dl, OType, RHS, Zero, IsAdd ? ISD::SETLT : ISD::SETGT); in expandSADDSUBO()
9288 DAG.getNode(ISD::XOR, dl, OType, ConditionRHS, ResultLowerThanLHS), dl, in expandSADDSUBO()
9299 bool isSigned = Node->getOpcode() == ISD::SMULO; in expandMULO()
9310 Result = DAG.getNode(ISD::SHL, dl, VT, LHS, ShiftAmt); in expandMULO()
9312 DAG.getNode(UseArithShift ? ISD::SRA : ISD::SRL, in expandMULO()
9314 LHS, ISD::SETNE); in expandMULO()
9327 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND }, in expandMULO()
9328 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }}; in expandMULO()
9330 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandMULO()
9339 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); in expandMULO()
9340 BottomHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul); in expandMULO()
9343 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, in expandMULO()
9344 DAG.getNode(ISD::SRL, dl, WideVT, Mul, ShiftAmt)); in expandMULO()
9371 DAG.getNode(ISD::SRA, dl, VT, LHS, in expandMULO()
9375 DAG.getNode(ISD::SRA, dl, VT, RHS, in expandMULO()
9402 assert(Ret.getOpcode() == ISD::MERGE_VALUES && in expandMULO()
9419 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt); in expandMULO()
9420 Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, Sign, ISD::SETNE); in expandMULO()
9423 DAG.getConstant(0, dl, VT), ISD::SETNE); in expandMULO()
9429 Overflow = DAG.getNode(ISD::TRUNCATE, dl, RType, Overflow); in expandMULO()
9438 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode()); in expandVecReduce()
9472 Res = DAG.getNode(ISD::ANY_EXTEND, dl, Node->getValueType(0), Res); in expandVecReduce()
9494 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode()); in expandVecReduceSeq()
9507 bool isSigned = Node->getOpcode() == ISD::SREM; in expandREM()
9508 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV; in expandREM()
9509 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; in expandREM()
9520 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor); in expandREM()
9521 Result = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul); in expandREM()
9529 bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT; in expandFP_TO_INT_SAT()
9557 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Src); in expandFP_TO_INT_SAT()
9577 bool MinMaxLegal = isOperationLegal(ISD::FMINNUM, SrcVT) && in expandFP_TO_INT_SAT()
9578 isOperationLegal(ISD::FMAXNUM, SrcVT); in expandFP_TO_INT_SAT()
9583 Clamped = DAG.getNode(ISD::FMAXNUM, dl, SrcVT, Clamped, MinFloatNode); in expandFP_TO_INT_SAT()
9585 Clamped = DAG.getNode(ISD::FMINNUM, dl, SrcVT, Clamped, MaxFloatNode); in expandFP_TO_INT_SAT()
9587 SDValue FpToInt = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, in expandFP_TO_INT_SAT()
9598 ISD::CondCode::SETUO); in expandFP_TO_INT_SAT()
9608 DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, DstVT, Src); in expandFP_TO_INT_SAT()
9615 ISD::CondCode::SETULT); in expandFP_TO_INT_SAT()
9618 ISD::CondCode::SETOGT); in expandFP_TO_INT_SAT()
9627 return DAG.getSelectCC(dl, Src, Src, ZeroInt, Select, ISD::CondCode::SETUO); in expandFP_TO_INT_SAT()
9632 assert(Node->getOpcode() == ISD::VECTOR_SPLICE && "Unexpected opcode!"); in expandVectorSplice()
9669 SDValue StackPtr2 = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, OffsetToV2); in expandVectorSplice()
9692 TrailingBytes = DAG.getNode(ISD::UMIN, DL, PtrVT, TrailingBytes, VLBytes); in expandVectorSplice()
9696 StackPtr2 = DAG.getNode(ISD::SUB, DL, PtrVT, StackPtr2, TrailingBytes); in expandVectorSplice()
9711 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); in LegalizeSetCCCondCode()
9722 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode); in LegalizeSetCCCondCode()
9734 InvCC = ISD::getSetCCSwappedOperands(InvCC); in LegalizeSetCCCondCode()
9745 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; in LegalizeSetCCCondCode()
9750 case ISD::SETUO: in LegalizeSetCCCondCode()
9751 if (TLI.isCondCodeLegal(ISD::SETUNE, OpVT)) { in LegalizeSetCCCondCode()
9752 CC1 = ISD::SETUNE; in LegalizeSetCCCondCode()
9753 CC2 = ISD::SETUNE; in LegalizeSetCCCondCode()
9754 Opc = ISD::OR; in LegalizeSetCCCondCode()
9757 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && in LegalizeSetCCCondCode()
9761 case ISD::SETO: in LegalizeSetCCCondCode()
9762 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && in LegalizeSetCCCondCode()
9764 CC1 = ISD::SETOEQ; in LegalizeSetCCCondCode()
9765 CC2 = ISD::SETOEQ; in LegalizeSetCCCondCode()
9766 Opc = ISD::AND; in LegalizeSetCCCondCode()
9768 case ISD::SETONE: in LegalizeSetCCCondCode()
9769 case ISD::SETUEQ: in LegalizeSetCCCondCode()
9774 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO; in LegalizeSetCCCondCode()
9776 (TLI.isCondCodeLegal(ISD::SETOGT, OpVT) || in LegalizeSetCCCondCode()
9777 TLI.isCondCodeLegal(ISD::SETOLT, OpVT))) { in LegalizeSetCCCondCode()
9778 CC1 = ISD::SETOGT; in LegalizeSetCCCondCode()
9779 CC2 = ISD::SETOLT; in LegalizeSetCCCondCode()
9780 Opc = ISD::OR; in LegalizeSetCCCondCode()
9785 case ISD::SETOEQ: in LegalizeSetCCCondCode()
9786 case ISD::SETOGT: in LegalizeSetCCCondCode()
9787 case ISD::SETOGE: in LegalizeSetCCCondCode()
9788 case ISD::SETOLT: in LegalizeSetCCCondCode()
9789 case ISD::SETOLE: in LegalizeSetCCCondCode()
9790 case ISD::SETUNE: in LegalizeSetCCCondCode()
9791 case ISD::SETUGT: in LegalizeSetCCCondCode()
9792 case ISD::SETUGE: in LegalizeSetCCCondCode()
9793 case ISD::SETULT: in LegalizeSetCCCondCode()
9794 case ISD::SETULE: in LegalizeSetCCCondCode()
9799 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO; in LegalizeSetCCCondCode()
9800 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND; in LegalizeSetCCCondCode()
9801 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10); in LegalizeSetCCCondCode()
9806 case ISD::SETLE: in LegalizeSetCCCondCode()
9807 case ISD::SETGT: in LegalizeSetCCCondCode()
9808 case ISD::SETGE: in LegalizeSetCCCondCode()
9809 case ISD::SETLT: in LegalizeSetCCCondCode()
9810 case ISD::SETNE: in LegalizeSetCCCondCode()
9811 case ISD::SETEQ: in LegalizeSetCCCondCode()
9818 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) { in LegalizeSetCCCondCode()
9839 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, SetCC1.getValue(1), in LegalizeSetCCCondCode()
9845 assert((Opc == ISD::OR || Opc == ISD::AND) && "Unexpected opcode"); in LegalizeSetCCCondCode()
9846 Opc = Opc == ISD::OR ? ISD::VP_OR : ISD::VP_AND; in LegalizeSetCCCondCode()