Lines Matching refs:ISD
139 bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) { in isConstantSplatVector()
140 if (N->getOpcode() == ISD::SPLAT_VECTOR) { in isConstantSplatVector()
169 bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) { in isConstantSplatVectorAllOnes()
171 while (N->getOpcode() == ISD::BITCAST) in isConstantSplatVectorAllOnes()
174 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { in isConstantSplatVectorAllOnes()
179 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; in isConstantSplatVectorAllOnes()
218 bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) { in isConstantSplatVectorAllZeros()
220 while (N->getOpcode() == ISD::BITCAST) in isConstantSplatVectorAllZeros()
223 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) { in isConstantSplatVectorAllZeros()
228 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; in isConstantSplatVectorAllZeros()
260 bool ISD::isBuildVectorAllOnes(const SDNode *N) { in isBuildVectorAllOnes()
264 bool ISD::isBuildVectorAllZeros(const SDNode *N) { in isBuildVectorAllZeros()
268 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) { in isBuildVectorOfConstantSDNodes()
269 if (N->getOpcode() != ISD::BUILD_VECTOR) in isBuildVectorOfConstantSDNodes()
281 bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) { in isBuildVectorOfConstantFPSDNodes()
282 if (N->getOpcode() != ISD::BUILD_VECTOR) in isBuildVectorOfConstantFPSDNodes()
294 bool ISD::allOperandsUndef(const SDNode *N) { in allOperandsUndef()
303 bool ISD::matchUnaryPredicate(SDValue Op, in matchUnaryPredicate()
311 if (ISD::BUILD_VECTOR != Op.getOpcode() && in matchUnaryPredicate()
312 ISD::SPLAT_VECTOR != Op.getOpcode()) in matchUnaryPredicate()
330 bool ISD::matchBinaryPredicate( in matchBinaryPredicate()
344 (LHS.getOpcode() != ISD::BUILD_VECTOR && in matchBinaryPredicate()
345 LHS.getOpcode() != ISD::SPLAT_VECTOR)) in matchBinaryPredicate()
367 ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) { in getVecReduceBaseOpcode()
371 case ISD::VECREDUCE_FADD: in getVecReduceBaseOpcode()
372 case ISD::VECREDUCE_SEQ_FADD: in getVecReduceBaseOpcode()
373 case ISD::VP_REDUCE_FADD: in getVecReduceBaseOpcode()
374 case ISD::VP_REDUCE_SEQ_FADD: in getVecReduceBaseOpcode()
375 return ISD::FADD; in getVecReduceBaseOpcode()
376 case ISD::VECREDUCE_FMUL: in getVecReduceBaseOpcode()
377 case ISD::VECREDUCE_SEQ_FMUL: in getVecReduceBaseOpcode()
378 case ISD::VP_REDUCE_FMUL: in getVecReduceBaseOpcode()
379 case ISD::VP_REDUCE_SEQ_FMUL: in getVecReduceBaseOpcode()
380 return ISD::FMUL; in getVecReduceBaseOpcode()
381 case ISD::VECREDUCE_ADD: in getVecReduceBaseOpcode()
382 case ISD::VP_REDUCE_ADD: in getVecReduceBaseOpcode()
383 return ISD::ADD; in getVecReduceBaseOpcode()
384 case ISD::VECREDUCE_MUL: in getVecReduceBaseOpcode()
385 case ISD::VP_REDUCE_MUL: in getVecReduceBaseOpcode()
386 return ISD::MUL; in getVecReduceBaseOpcode()
387 case ISD::VECREDUCE_AND: in getVecReduceBaseOpcode()
388 case ISD::VP_REDUCE_AND: in getVecReduceBaseOpcode()
389 return ISD::AND; in getVecReduceBaseOpcode()
390 case ISD::VECREDUCE_OR: in getVecReduceBaseOpcode()
391 case ISD::VP_REDUCE_OR: in getVecReduceBaseOpcode()
392 return ISD::OR; in getVecReduceBaseOpcode()
393 case ISD::VECREDUCE_XOR: in getVecReduceBaseOpcode()
394 case ISD::VP_REDUCE_XOR: in getVecReduceBaseOpcode()
395 return ISD::XOR; in getVecReduceBaseOpcode()
396 case ISD::VECREDUCE_SMAX: in getVecReduceBaseOpcode()
397 case ISD::VP_REDUCE_SMAX: in getVecReduceBaseOpcode()
398 return ISD::SMAX; in getVecReduceBaseOpcode()
399 case ISD::VECREDUCE_SMIN: in getVecReduceBaseOpcode()
400 case ISD::VP_REDUCE_SMIN: in getVecReduceBaseOpcode()
401 return ISD::SMIN; in getVecReduceBaseOpcode()
402 case ISD::VECREDUCE_UMAX: in getVecReduceBaseOpcode()
403 case ISD::VP_REDUCE_UMAX: in getVecReduceBaseOpcode()
404 return ISD::UMAX; in getVecReduceBaseOpcode()
405 case ISD::VECREDUCE_UMIN: in getVecReduceBaseOpcode()
406 case ISD::VP_REDUCE_UMIN: in getVecReduceBaseOpcode()
407 return ISD::UMIN; in getVecReduceBaseOpcode()
408 case ISD::VECREDUCE_FMAX: in getVecReduceBaseOpcode()
409 case ISD::VP_REDUCE_FMAX: in getVecReduceBaseOpcode()
410 return ISD::FMAXNUM; in getVecReduceBaseOpcode()
411 case ISD::VECREDUCE_FMIN: in getVecReduceBaseOpcode()
412 case ISD::VP_REDUCE_FMIN: in getVecReduceBaseOpcode()
413 return ISD::FMINNUM; in getVecReduceBaseOpcode()
417 bool ISD::isVPOpcode(unsigned Opcode) { in isVPOpcode()
422 case ISD::VPSD: \ in isVPOpcode()
428 bool ISD::isVPBinaryOp(unsigned Opcode) { in isVPBinaryOp()
432 #define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD: in isVPBinaryOp()
440 bool ISD::isVPReduction(unsigned Opcode) { in isVPReduction()
444 #define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD: in isVPReduction()
453 Optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) { in getVPMaskIdx()
458 case ISD::VPSD: \ in getVPMaskIdx()
465 Optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) { in getVPExplicitVectorLengthIdx()
470 case ISD::VPSD: \ in getVPExplicitVectorLengthIdx()
476 ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) { in getExtForLoadExtType()
478 case ISD::EXTLOAD: in getExtForLoadExtType()
479 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND; in getExtForLoadExtType()
480 case ISD::SEXTLOAD: in getExtForLoadExtType()
481 return ISD::SIGN_EXTEND; in getExtForLoadExtType()
482 case ISD::ZEXTLOAD: in getExtForLoadExtType()
483 return ISD::ZERO_EXTEND; in getExtForLoadExtType()
491 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { in getSetCCSwappedOperands()
496 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits in getSetCCSwappedOperands()
501 static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) { in getSetCCInverseImpl()
508 if (Operation > ISD::SETTRUE2) in getSetCCInverseImpl()
511 return ISD::CondCode(Operation); in getSetCCInverseImpl()
514 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) { in getSetCCInverse()
518 ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op, in getSetCCInverse()
526 static int isSignedOp(ISD::CondCode Opcode) { in isSignedOp()
529 case ISD::SETEQ: in isSignedOp()
530 case ISD::SETNE: return 0; in isSignedOp()
531 case ISD::SETLT: in isSignedOp()
532 case ISD::SETLE: in isSignedOp()
533 case ISD::SETGT: in isSignedOp()
534 case ISD::SETGE: return 1; in isSignedOp()
535 case ISD::SETULT: in isSignedOp()
536 case ISD::SETULE: in isSignedOp()
537 case ISD::SETUGT: in isSignedOp()
538 case ISD::SETUGE: return 2; in isSignedOp()
542 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, in getSetCCOrOperation()
547 return ISD::SETCC_INVALID; in getSetCCOrOperation()
553 if (Op > ISD::SETTRUE2) in getSetCCOrOperation()
557 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT in getSetCCOrOperation()
558 Op = ISD::SETNE; in getSetCCOrOperation()
560 return ISD::CondCode(Op); in getSetCCOrOperation()
563 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, in getSetCCAndOperation()
568 return ISD::SETCC_INVALID; in getSetCCAndOperation()
571 ISD::CondCode Result = ISD::CondCode(Op1 & Op2); in getSetCCAndOperation()
577 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT in getSetCCAndOperation()
578 case ISD::SETOEQ: // SETEQ & SETU[LG]E in getSetCCAndOperation()
579 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE in getSetCCAndOperation()
580 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE in getSetCCAndOperation()
581 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE in getSetCCAndOperation()
631 case ISD::TargetExternalSymbol: in AddNodeIDCustom()
632 case ISD::ExternalSymbol: in AddNodeIDCustom()
633 case ISD::MCSymbol: in AddNodeIDCustom()
636 case ISD::TargetConstant: in AddNodeIDCustom()
637 case ISD::Constant: { in AddNodeIDCustom()
643 case ISD::TargetConstantFP: in AddNodeIDCustom()
644 case ISD::ConstantFP: in AddNodeIDCustom()
647 case ISD::TargetGlobalAddress: in AddNodeIDCustom()
648 case ISD::GlobalAddress: in AddNodeIDCustom()
649 case ISD::TargetGlobalTLSAddress: in AddNodeIDCustom()
650 case ISD::GlobalTLSAddress: { in AddNodeIDCustom()
657 case ISD::BasicBlock: in AddNodeIDCustom()
660 case ISD::Register: in AddNodeIDCustom()
663 case ISD::RegisterMask: in AddNodeIDCustom()
666 case ISD::SRCVALUE: in AddNodeIDCustom()
669 case ISD::FrameIndex: in AddNodeIDCustom()
670 case ISD::TargetFrameIndex: in AddNodeIDCustom()
673 case ISD::LIFETIME_START: in AddNodeIDCustom()
674 case ISD::LIFETIME_END: in AddNodeIDCustom()
680 case ISD::PSEUDO_PROBE: in AddNodeIDCustom()
685 case ISD::JumpTable: in AddNodeIDCustom()
686 case ISD::TargetJumpTable: in AddNodeIDCustom()
690 case ISD::ConstantPool: in AddNodeIDCustom()
691 case ISD::TargetConstantPool: { in AddNodeIDCustom()
702 case ISD::TargetIndex: { in AddNodeIDCustom()
709 case ISD::LOAD: { in AddNodeIDCustom()
717 case ISD::STORE: { in AddNodeIDCustom()
725 case ISD::VP_LOAD: { in AddNodeIDCustom()
733 case ISD::VP_STORE: { in AddNodeIDCustom()
741 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: { in AddNodeIDCustom()
748 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: { in AddNodeIDCustom()
755 case ISD::VP_GATHER: { in AddNodeIDCustom()
763 case ISD::VP_SCATTER: { in AddNodeIDCustom()
771 case ISD::MLOAD: { in AddNodeIDCustom()
779 case ISD::MSTORE: { in AddNodeIDCustom()
787 case ISD::MGATHER: { in AddNodeIDCustom()
795 case ISD::MSCATTER: { in AddNodeIDCustom()
803 case ISD::ATOMIC_CMP_SWAP: in AddNodeIDCustom()
804 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: in AddNodeIDCustom()
805 case ISD::ATOMIC_SWAP: in AddNodeIDCustom()
806 case ISD::ATOMIC_LOAD_ADD: in AddNodeIDCustom()
807 case ISD::ATOMIC_LOAD_SUB: in AddNodeIDCustom()
808 case ISD::ATOMIC_LOAD_AND: in AddNodeIDCustom()
809 case ISD::ATOMIC_LOAD_CLR: in AddNodeIDCustom()
810 case ISD::ATOMIC_LOAD_OR: in AddNodeIDCustom()
811 case ISD::ATOMIC_LOAD_XOR: in AddNodeIDCustom()
812 case ISD::ATOMIC_LOAD_NAND: in AddNodeIDCustom()
813 case ISD::ATOMIC_LOAD_MIN: in AddNodeIDCustom()
814 case ISD::ATOMIC_LOAD_MAX: in AddNodeIDCustom()
815 case ISD::ATOMIC_LOAD_UMIN: in AddNodeIDCustom()
816 case ISD::ATOMIC_LOAD_UMAX: in AddNodeIDCustom()
817 case ISD::ATOMIC_LOAD: in AddNodeIDCustom()
818 case ISD::ATOMIC_STORE: { in AddNodeIDCustom()
826 case ISD::PREFETCH: { in AddNodeIDCustom()
832 case ISD::VECTOR_SHUFFLE: { in AddNodeIDCustom()
839 case ISD::TargetBlockAddress: in AddNodeIDCustom()
840 case ISD::BlockAddress: { in AddNodeIDCustom()
847 case ISD::AssertAlign: in AddNodeIDCustom()
885 case ISD::HANDLENODE: in doNotCSE()
886 case ISD::EH_LABEL: in doNotCSE()
929 if (N->getOpcode() == ISD::DELETED_NODE) in RemoveDeadNodes()
1016 N->NodeType = ISD::DELETED_NODE; in DeallocateNode()
1029 case ISD::BUILD_PAIR: { in VerifySDNode()
1043 case ISD::BUILD_VECTOR: { in VerifySDNode()
1084 case ISD::HANDLENODE: return false; // noop. in RemoveNodeFromCSEMaps()
1085 case ISD::CONDCODE: in RemoveNodeFromCSEMaps()
1091 case ISD::ExternalSymbol: in RemoveNodeFromCSEMaps()
1094 case ISD::TargetExternalSymbol: { in RemoveNodeFromCSEMaps()
1100 case ISD::MCSymbol: { in RemoveNodeFromCSEMaps()
1105 case ISD::VALUETYPE: { in RemoveNodeFromCSEMaps()
1117 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"); in RemoveNodeFromCSEMaps()
1118 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"); in RemoveNodeFromCSEMaps()
1233 EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)), in SelectionDAG()
1285 case ISD::Constant: in FindNodeOrInsertPos()
1286 case ISD::ConstantFP: in FindNodeOrInsertPos()
1299 case ISD::Constant: in FindNodeOrInsertPos()
1300 case ISD::ConstantFP: in FindNodeOrInsertPos()
1343 ? getNode(ISD::FP_EXTEND, DL, VT, Op) in getFPExtendOrRound()
1344 : getNode(ISD::FP_ROUND, DL, VT, Op, getIntPtrConstant(0, DL)); in getFPExtendOrRound()
1354 ? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op}) in getStrictFPExtendOrRound()
1355 : getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other}, in getStrictFPExtendOrRound()
1363 getNode(ISD::ANY_EXTEND, DL, VT, Op) : in getAnyExtOrTrunc()
1364 getNode(ISD::TRUNCATE, DL, VT, Op); in getAnyExtOrTrunc()
1369 getNode(ISD::SIGN_EXTEND, DL, VT, Op) : in getSExtOrTrunc()
1370 getNode(ISD::TRUNCATE, DL, VT, Op); in getSExtOrTrunc()
1375 getNode(ISD::ZERO_EXTEND, DL, VT, Op) : in getZExtOrTrunc()
1376 getNode(ISD::TRUNCATE, DL, VT, Op); in getZExtOrTrunc()
1382 return getNode(ISD::TRUNCATE, SL, VT, Op); in getBoolExtOrTrunc()
1403 return getNode(ISD::AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT)); in getZeroExtendInReg()
1420 return getNode(ISD::XOR, DL, VT, Val, getAllOnesConstant(DL, VT)); in getNOT()
1425 return getNode(ISD::XOR, DL, VT, Val, TrueValue); in getLogicalNOT()
1431 return getNode(ISD::VP_XOR, DL, VT, Val, TrueValue, Mask, EVL); in getVPLogicalNOT()
1505 return getNode(ISD::SPLAT_VECTOR_PARTS, DL, VT, ScalarParts); in getConstant()
1539 getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops)); in getConstant()
1545 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; in getConstant()
1603 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; in getConstantFP()
1659 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress; in getGlobalAddress()
1661 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress; in getGlobalAddress()
1680 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; in getFrameIndex()
1698 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable; in getJumpTable()
1722 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; in getConstantPool()
1749 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; in getConstantPool()
1770 AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None); in getTargetIndex()
1786 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None); in getBasicBlock()
1839 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { in getCondCode()
1861 ISD::STEP_VECTOR, DL, ResVT, in getStepVector()
1983 while (V.getOpcode() == ISD::BITCAST) in getVectorShuffle()
2018 NewBV = getNode(ISD::BITCAST, dl, VT, NewBV); in getVectorShuffle()
2026 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops); in getVectorShuffle()
2063 AddNodeIDNode(ID, ISD::Register, getVTList(VT), None); in getRegister()
2078 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None); in getRegisterMask()
2092 return getLabelNode(ISD::EH_LABEL, dl, Root, Label); in getEHLabel()
2117 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; in getBlockAddress()
2136 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None); in getSrcValue()
2151 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None); in getMDNode()
2168 return getNode(ISD::BITCAST, SDLoc(V), VT, V); in getBitcast()
2175 AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops); in getAddrSpaceCast()
2193 return getNode(ISD::FREEZE, SDLoc(V), V.getValueType(), V); in getFreeze()
2220 VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList, in expandVAArg()
2224 getNode(ISD::AND, dl, VAList.getValueType(), VAList, in expandVAArg()
2229 Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList, in expandVAArg()
2319 ISD::CondCode Cond, const SDLoc &dl) { in FoldSetCC()
2325 case ISD::SETFALSE: in FoldSetCC()
2326 case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT); in FoldSetCC()
2327 case ISD::SETTRUE: in FoldSetCC()
2328 case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT); in FoldSetCC()
2330 case ISD::SETOEQ: in FoldSetCC()
2331 case ISD::SETOGT: in FoldSetCC()
2332 case ISD::SETOGE: in FoldSetCC()
2333 case ISD::SETOLT: in FoldSetCC()
2334 case ISD::SETOLE: in FoldSetCC()
2335 case ISD::SETONE: in FoldSetCC()
2336 case ISD::SETO: in FoldSetCC()
2337 case ISD::SETUO: in FoldSetCC()
2338 case ISD::SETUEQ: in FoldSetCC()
2339 case ISD::SETUNE: in FoldSetCC()
2350 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) in FoldSetCC()
2361 return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT); in FoldSetCC()
2381 case ISD::SETEQ: if (R==APFloat::cmpUnordered) in FoldSetCC()
2384 case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT, in FoldSetCC()
2386 case ISD::SETNE: if (R==APFloat::cmpUnordered) in FoldSetCC()
2389 case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan || in FoldSetCC()
2392 case ISD::SETLT: if (R==APFloat::cmpUnordered) in FoldSetCC()
2395 case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT, in FoldSetCC()
2397 case ISD::SETGT: if (R==APFloat::cmpUnordered) in FoldSetCC()
2400 case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl, in FoldSetCC()
2402 case ISD::SETLE: if (R==APFloat::cmpUnordered) in FoldSetCC()
2405 case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan || in FoldSetCC()
2408 case ISD::SETGE: if (R==APFloat::cmpUnordered) in FoldSetCC()
2411 case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan || in FoldSetCC()
2413 case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT, in FoldSetCC()
2415 case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT, in FoldSetCC()
2417 case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered || in FoldSetCC()
2420 case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT, in FoldSetCC()
2422 case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered || in FoldSetCC()
2425 case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan || in FoldSetCC()
2428 case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl, in FoldSetCC()
2430 case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT, in FoldSetCC()
2435 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond); in FoldSetCC()
2446 switch (ISD::getUnorderedFlavor(Cond)) { in FoldSetCC()
2475 case ISD::Constant: { in GetDemandedBits()
2482 case ISD::SRL: in GetDemandedBits()
2496 return getNode(ISD::SRL, SDLoc(V), V.getValueType(), SimplifyLHS, in GetDemandedBits()
2560 case ISD::SPLAT_VECTOR: in isSplatValue()
2565 case ISD::ADD: in isSplatValue()
2566 case ISD::SUB: in isSplatValue()
2567 case ISD::AND: in isSplatValue()
2568 case ISD::XOR: in isSplatValue()
2569 case ISD::OR: { in isSplatValue()
2580 case ISD::ABS: in isSplatValue()
2581 case ISD::TRUNCATE: in isSplatValue()
2582 case ISD::SIGN_EXTEND: in isSplatValue()
2583 case ISD::ZERO_EXTEND: in isSplatValue()
2586 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN || in isSplatValue()
2587 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) in isSplatValue()
2602 case ISD::BUILD_VECTOR: { in isSplatValue()
2618 case ISD::VECTOR_SHUFFLE: { in isSplatValue()
2656 case ISD::EXTRACT_SUBVECTOR: { in isSplatValue()
2672 case ISD::ANY_EXTEND_VECTOR_INREG: in isSplatValue()
2673 case ISD::SIGN_EXTEND_VECTOR_INREG: in isSplatValue()
2674 case ISD::ZERO_EXTEND_VECTOR_INREG: { in isSplatValue()
2689 case ISD::BITCAST: { in isSplatValue()
2772 case ISD::SPLAT_VECTOR: in getSplatSourceVector()
2775 case ISD::VECTOR_SHUFFLE: { in getSplatSourceVector()
2807 return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), LegalSVT, SrcVector, in getSplatValue()
2816 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || in getValidShiftAmountConstant()
2817 V.getOpcode() == ISD::SRA) && in getValidShiftAmountConstant()
2831 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || in getValidMinimumShiftAmountConstant()
2832 V.getOpcode() == ISD::SRA) && in getValidMinimumShiftAmountConstant()
2860 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL || in getValidMaximumShiftAmountConstant()
2861 V.getOpcode() == ISD::SRA) && in getValidMaximumShiftAmountConstant()
2943 case ISD::MERGE_VALUES: in computeKnownBits()
2946 case ISD::BUILD_VECTOR: in computeKnownBits()
2971 case ISD::VECTOR_SHUFFLE: { in computeKnownBits()
3013 case ISD::CONCAT_VECTORS: { in computeKnownBits()
3033 case ISD::INSERT_SUBVECTOR: { in computeKnownBits()
3057 case ISD::EXTRACT_SUBVECTOR: { in computeKnownBits()
3069 case ISD::SCALAR_TO_VECTOR: { in computeKnownBits()
3082 case ISD::BITCAST: { in computeKnownBits()
3147 case ISD::AND: in computeKnownBits()
3153 case ISD::OR: in computeKnownBits()
3159 case ISD::XOR: in computeKnownBits()
3165 case ISD::MUL: { in computeKnownBits()
3184 case ISD::MULHU: { in computeKnownBits()
3190 case ISD::MULHS: { in computeKnownBits()
3196 case ISD::UMUL_LOHI: { in computeKnownBits()
3207 case ISD::SMUL_LOHI: { in computeKnownBits()
3218 case ISD::AVGCEILU: { in computeKnownBits()
3228 case ISD::SELECT: in computeKnownBits()
3229 case ISD::VSELECT: in computeKnownBits()
3239 case ISD::SELECT_CC: in computeKnownBits()
3249 case ISD::SMULO: in computeKnownBits()
3250 case ISD::UMULO: in computeKnownBits()
3262 case ISD::SETCC: in computeKnownBits()
3263 case ISD::SETCCCARRY: in computeKnownBits()
3264 case ISD::STRICT_FSETCC: in computeKnownBits()
3265 case ISD::STRICT_FSETCCS: { in computeKnownBits()
3274 case ISD::SHL: in computeKnownBits()
3284 case ISD::SRL: in computeKnownBits()
3294 case ISD::SRA: in computeKnownBits()
3300 case ISD::FSHL: in computeKnownBits()
3301 case ISD::FSHR: in computeKnownBits()
3308 Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1), in computeKnownBits()
3317 if (Opcode == ISD::FSHL) { in computeKnownBits()
3332 case ISD::SHL_PARTS: in computeKnownBits()
3333 case ISD::SRA_PARTS: in computeKnownBits()
3334 case ISD::SRL_PARTS: { in computeKnownBits()
3349 if (Opcode == ISD::SHL_PARTS) in computeKnownBits()
3351 else if (Opcode == ISD::SRA_PARTS) in computeKnownBits()
3364 case ISD::SIGN_EXTEND_INREG: { in computeKnownBits()
3370 case ISD::CTTZ: in computeKnownBits()
3371 case ISD::CTTZ_ZERO_UNDEF: { in computeKnownBits()
3379 case ISD::CTLZ: in computeKnownBits()
3380 case ISD::CTLZ_ZERO_UNDEF: { in computeKnownBits()
3388 case ISD::CTPOP: { in computeKnownBits()
3395 case ISD::PARITY: { in computeKnownBits()
3400 case ISD::LOAD: { in computeKnownBits()
3403 if (ISD::isNON_EXTLoad(LD) && Cst) { in computeKnownBits()
3450 } else if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { in computeKnownBits()
3456 if (LD->getExtensionType() == ISD::NON_EXTLOAD) in computeKnownBits()
3461 case ISD::ZERO_EXTEND_VECTOR_INREG: { in computeKnownBits()
3468 case ISD::ZERO_EXTEND: { in computeKnownBits()
3473 case ISD::SIGN_EXTEND_VECTOR_INREG: { in computeKnownBits()
3482 case ISD::SIGN_EXTEND: { in computeKnownBits()
3489 case ISD::ANY_EXTEND_VECTOR_INREG: { in computeKnownBits()
3496 case ISD::ANY_EXTEND: { in computeKnownBits()
3501 case ISD::TRUNCATE: { in computeKnownBits()
3506 case ISD::AssertZext: { in computeKnownBits()
3514 case ISD::AssertAlign: { in computeKnownBits()
3525 case ISD::FGETSIGN: in computeKnownBits()
3529 case ISD::USUBO: in computeKnownBits()
3530 case ISD::SSUBO: in computeKnownBits()
3531 case ISD::SUBCARRY: in computeKnownBits()
3532 case ISD::SSUBO_CARRY: in computeKnownBits()
3542 case ISD::SUB: in computeKnownBits()
3543 case ISD::SUBC: { in computeKnownBits()
3548 if (Opcode == ISD::SUBCARRY || Opcode == ISD::SSUBO_CARRY) in computeKnownBits()
3557 case ISD::UADDO: in computeKnownBits()
3558 case ISD::SADDO: in computeKnownBits()
3559 case ISD::ADDCARRY: in computeKnownBits()
3560 case ISD::SADDO_CARRY: in computeKnownBits()
3570 case ISD::ADD: in computeKnownBits()
3571 case ISD::ADDC: in computeKnownBits()
3572 case ISD::ADDE: { in computeKnownBits()
3577 if (Opcode == ISD::ADDE) in computeKnownBits()
3580 else if (Opcode == ISD::ADDCARRY || Opcode == ISD::SADDO_CARRY) in computeKnownBits()
3595 case ISD::UDIV: { in computeKnownBits()
3601 case ISD::SREM: { in computeKnownBits()
3607 case ISD::UREM: { in computeKnownBits()
3613 case ISD::EXTRACT_ELEMENT: { in computeKnownBits()
3626 case ISD::EXTRACT_VECTOR_ELT: { in computeKnownBits()
3654 case ISD::INSERT_VECTOR_ELT: { in computeKnownBits()
3681 case ISD::BITREVERSE: { in computeKnownBits()
3686 case ISD::BSWAP: { in computeKnownBits()
3691 case ISD::ABS: { in computeKnownBits()
3696 case ISD::USUBSAT: { in computeKnownBits()
3702 case ISD::UMIN: { in computeKnownBits()
3708 case ISD::UMAX: { in computeKnownBits()
3714 case ISD::SMIN: in computeKnownBits()
3715 case ISD::SMAX: { in computeKnownBits()
3718 bool IsMax = (Opcode == ISD::SMAX); in computeKnownBits()
3721 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX)) in computeKnownBits()
3766 case ISD::FP_TO_UINT_SAT: { in computeKnownBits()
3772 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: in computeKnownBits()
3785 case ISD::ATOMIC_CMP_SWAP: in computeKnownBits()
3786 case ISD::ATOMIC_SWAP: in computeKnownBits()
3787 case ISD::ATOMIC_LOAD_ADD: in computeKnownBits()
3788 case ISD::ATOMIC_LOAD_SUB: in computeKnownBits()
3789 case ISD::ATOMIC_LOAD_AND: in computeKnownBits()
3790 case ISD::ATOMIC_LOAD_CLR: in computeKnownBits()
3791 case ISD::ATOMIC_LOAD_OR: in computeKnownBits()
3792 case ISD::ATOMIC_LOAD_XOR: in computeKnownBits()
3793 case ISD::ATOMIC_LOAD_NAND: in computeKnownBits()
3794 case ISD::ATOMIC_LOAD_MIN: in computeKnownBits()
3795 case ISD::ATOMIC_LOAD_MAX: in computeKnownBits()
3796 case ISD::ATOMIC_LOAD_UMIN: in computeKnownBits()
3797 case ISD::ATOMIC_LOAD_UMAX: in computeKnownBits()
3798 case ISD::ATOMIC_LOAD: { in computeKnownBits()
3803 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND) in computeKnownBits()
3808 case ISD::FrameIndex: in computeKnownBits()
3809 case ISD::TargetFrameIndex: in computeKnownBits()
3815 if (Opcode < ISD::BUILTIN_OP_END) in computeKnownBits()
3818 case ISD::INTRINSIC_WO_CHAIN: in computeKnownBits()
3819 case ISD::INTRINSIC_W_CHAIN: in computeKnownBits()
3820 case ISD::INTRINSIC_VOID: in computeKnownBits()
3847 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 && in computeOverflowKind()
3851 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1) { in computeOverflowKind()
3871 if (Val.getOpcode() == ISD::SHL) { in isKnownToBeAPowerOfTwo()
3879 if (Val.getOpcode() == ISD::SRL) { in isKnownToBeAPowerOfTwo()
3886 if (Val.getOpcode() == ISD::BUILD_VECTOR) in isKnownToBeAPowerOfTwo()
3895 if (Val.getOpcode() == ISD::SPLAT_VECTOR) in isKnownToBeAPowerOfTwo()
3901 if (Val.getOpcode() == ISD::VSCALE && in isKnownToBeAPowerOfTwo()
3950 case ISD::AssertSext: in ComputeNumSignBits()
3953 case ISD::AssertZext: in ComputeNumSignBits()
3956 case ISD::MERGE_VALUES: in ComputeNumSignBits()
3959 case ISD::BUILD_VECTOR: in ComputeNumSignBits()
3979 case ISD::VECTOR_SHUFFLE: { in ComputeNumSignBits()
4012 case ISD::BITCAST: { in ComputeNumSignBits()
4056 case ISD::FP_TO_SINT_SAT: in ComputeNumSignBits()
4060 case ISD::SIGN_EXTEND: in ComputeNumSignBits()
4063 case ISD::SIGN_EXTEND_INREG: in ComputeNumSignBits()
4069 case ISD::SIGN_EXTEND_VECTOR_INREG: { in ComputeNumSignBits()
4076 case ISD::SRA: in ComputeNumSignBits()
4083 case ISD::SHL: in ComputeNumSignBits()
4092 case ISD::AND: in ComputeNumSignBits()
4093 case ISD::OR: in ComputeNumSignBits()
4094 case ISD::XOR: // NOT is handled here. in ComputeNumSignBits()
4106 case ISD::SELECT: in ComputeNumSignBits()
4107 case ISD::VSELECT: in ComputeNumSignBits()
4112 case ISD::SELECT_CC: in ComputeNumSignBits()
4118 case ISD::SMIN: in ComputeNumSignBits()
4119 case ISD::SMAX: { in ComputeNumSignBits()
4122 bool IsMax = (Opcode == ISD::SMAX); in ComputeNumSignBits()
4125 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX)) in ComputeNumSignBits()
4145 case ISD::UMIN: in ComputeNumSignBits()
4146 case ISD::UMAX: in ComputeNumSignBits()
4152 case ISD::SADDO: in ComputeNumSignBits()
4153 case ISD::UADDO: in ComputeNumSignBits()
4154 case ISD::SADDO_CARRY: in ComputeNumSignBits()
4155 case ISD::ADDCARRY: in ComputeNumSignBits()
4156 case ISD::SSUBO: in ComputeNumSignBits()
4157 case ISD::USUBO: in ComputeNumSignBits()
4158 case ISD::SSUBO_CARRY: in ComputeNumSignBits()
4159 case ISD::SUBCARRY: in ComputeNumSignBits()
4160 case ISD::SMULO: in ComputeNumSignBits()
4161 case ISD::UMULO: in ComputeNumSignBits()
4172 case ISD::SETCC: in ComputeNumSignBits()
4173 case ISD::SETCCCARRY: in ComputeNumSignBits()
4174 case ISD::STRICT_FSETCC: in ComputeNumSignBits()
4175 case ISD::STRICT_FSETCCS: { in ComputeNumSignBits()
4183 case ISD::ROTL: in ComputeNumSignBits()
4184 case ISD::ROTR: in ComputeNumSignBits()
4196 if (Opcode == ISD::ROTR) in ComputeNumSignBits()
4204 case ISD::ADD: in ComputeNumSignBits()
4205 case ISD::ADDC: in ComputeNumSignBits()
4232 case ISD::SUB: in ComputeNumSignBits()
4260 case ISD::MUL: { in ComputeNumSignBits()
4272 case ISD::SREM: in ComputeNumSignBits()
4278 case ISD::TRUNCATE: { in ComputeNumSignBits()
4286 case ISD::EXTRACT_ELEMENT: { in ComputeNumSignBits()
4299 case ISD::INSERT_VECTOR_ELT: { in ComputeNumSignBits()
4329 case ISD::EXTRACT_VECTOR_ELT: { in ComputeNumSignBits()
4356 case ISD::EXTRACT_SUBVECTOR: { in ComputeNumSignBits()
4367 case ISD::CONCAT_VECTORS: { in ComputeNumSignBits()
4385 case ISD::INSERT_SUBVECTOR: { in ComputeNumSignBits()
4409 case ISD::ATOMIC_CMP_SWAP: in ComputeNumSignBits()
4410 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: in ComputeNumSignBits()
4411 case ISD::ATOMIC_SWAP: in ComputeNumSignBits()
4412 case ISD::ATOMIC_LOAD_ADD: in ComputeNumSignBits()
4413 case ISD::ATOMIC_LOAD_SUB: in ComputeNumSignBits()
4414 case ISD::ATOMIC_LOAD_AND: in ComputeNumSignBits()
4415 case ISD::ATOMIC_LOAD_CLR: in ComputeNumSignBits()
4416 case ISD::ATOMIC_LOAD_OR: in ComputeNumSignBits()
4417 case ISD::ATOMIC_LOAD_XOR: in ComputeNumSignBits()
4418 case ISD::ATOMIC_LOAD_NAND: in ComputeNumSignBits()
4419 case ISD::ATOMIC_LOAD_MIN: in ComputeNumSignBits()
4420 case ISD::ATOMIC_LOAD_MAX: in ComputeNumSignBits()
4421 case ISD::ATOMIC_LOAD_UMIN: in ComputeNumSignBits()
4422 case ISD::ATOMIC_LOAD_UMAX: in ComputeNumSignBits()
4423 case ISD::ATOMIC_LOAD: { in ComputeNumSignBits()
4429 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND) in ComputeNumSignBits()
4431 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND) in ComputeNumSignBits()
4445 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known. in ComputeNumSignBits()
4448 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known. in ComputeNumSignBits()
4451 case ISD::NON_EXTLOAD: in ComputeNumSignBits()
4487 if (Opcode >= ISD::BUILTIN_OP_END || in ComputeNumSignBits()
4488 Opcode == ISD::INTRINSIC_WO_CHAIN || in ComputeNumSignBits()
4489 Opcode == ISD::INTRINSIC_W_CHAIN || in ComputeNumSignBits()
4490 Opcode == ISD::INTRINSIC_VOID) { in ComputeNumSignBits()
4519 if (Op.getOpcode() == ISD::FREEZE) in isGuaranteedNotToBeUndefOrPoison()
4540 if (Opcode == ISD::FREEZE) in isGuaranteedNotToBeUndefOrPoison()
4550 case ISD::UNDEF: in isGuaranteedNotToBeUndefOrPoison()
4553 case ISD::BUILD_VECTOR: in isGuaranteedNotToBeUndefOrPoison()
4571 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN || in isGuaranteedNotToBeUndefOrPoison()
4572 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) in isGuaranteedNotToBeUndefOrPoison()
4582 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) || in isBaseWithConstantOffset()
4586 if (Op.getOpcode() == ISD::OR && in isBaseWithConstantOffset()
4610 case ISD::FADD: in isKnownNeverNaN()
4611 case ISD::FSUB: in isKnownNeverNaN()
4612 case ISD::FMUL: in isKnownNeverNaN()
4613 case ISD::FDIV: in isKnownNeverNaN()
4614 case ISD::FREM: in isKnownNeverNaN()
4615 case ISD::FSIN: in isKnownNeverNaN()
4616 case ISD::FCOS: { in isKnownNeverNaN()
4622 case ISD::FCANONICALIZE: in isKnownNeverNaN()
4623 case ISD::FEXP: in isKnownNeverNaN()
4624 case ISD::FEXP2: in isKnownNeverNaN()
4625 case ISD::FTRUNC: in isKnownNeverNaN()
4626 case ISD::FFLOOR: in isKnownNeverNaN()
4627 case ISD::FCEIL: in isKnownNeverNaN()
4628 case ISD::FROUND: in isKnownNeverNaN()
4629 case ISD::FROUNDEVEN: in isKnownNeverNaN()
4630 case ISD::FRINT: in isKnownNeverNaN()
4631 case ISD::FNEARBYINT: { in isKnownNeverNaN()
4636 case ISD::FABS: in isKnownNeverNaN()
4637 case ISD::FNEG: in isKnownNeverNaN()
4638 case ISD::FCOPYSIGN: { in isKnownNeverNaN()
4641 case ISD::SELECT: in isKnownNeverNaN()
4644 case ISD::FP_EXTEND: in isKnownNeverNaN()
4645 case ISD::FP_ROUND: { in isKnownNeverNaN()
4650 case ISD::SINT_TO_FP: in isKnownNeverNaN()
4651 case ISD::UINT_TO_FP: in isKnownNeverNaN()
4653 case ISD::FMA: in isKnownNeverNaN()
4654 case ISD::FMAD: { in isKnownNeverNaN()
4661 case ISD::FSQRT: // Need is known positive in isKnownNeverNaN()
4662 case ISD::FLOG: in isKnownNeverNaN()
4663 case ISD::FLOG2: in isKnownNeverNaN()
4664 case ISD::FLOG10: in isKnownNeverNaN()
4665 case ISD::FPOWI: in isKnownNeverNaN()
4666 case ISD::FPOW: { in isKnownNeverNaN()
4672 case ISD::FMINNUM: in isKnownNeverNaN()
4673 case ISD::FMAXNUM: { in isKnownNeverNaN()
4679 case ISD::FMINNUM_IEEE: in isKnownNeverNaN()
4680 case ISD::FMAXNUM_IEEE: { in isKnownNeverNaN()
4690 case ISD::FMINIMUM: in isKnownNeverNaN()
4691 case ISD::FMAXIMUM: { in isKnownNeverNaN()
4696 case ISD::EXTRACT_VECTOR_ELT: { in isKnownNeverNaN()
4700 if (Opcode >= ISD::BUILTIN_OP_END || in isKnownNeverNaN()
4701 Opcode == ISD::INTRINSIC_WO_CHAIN || in isKnownNeverNaN()
4702 Opcode == ISD::INTRINSIC_W_CHAIN || in isKnownNeverNaN()
4703 Opcode == ISD::INTRINSIC_VOID) { in isKnownNeverNaN()
4727 if (ISD::matchUnaryPredicate(Op, in isKnownNeverZero()
4734 case ISD::OR: in isKnownNeverZero()
4765 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND) in getBitwiseNotOperand()
4771 ExtArg.getOperand(0).getOpcode() == ISD::TRUNCATE && in getBitwiseNotOperand()
4786 if (Other->getOpcode() == ISD::AND) in haveNoCommonBitsSetCommutative()
4792 if (A->getOpcode() == ISD::AND) in haveNoCommonBitsSetCommutative()
4835 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT || in FoldBUILD_VECTOR()
4881 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR || in foldCONCAT_VECTORS()
4911 else if (Op.getOpcode() == ISD::BUILD_VECTOR) in foldCONCAT_VECTORS()
4966 assert(Operand.getOpcode() != ISD::DELETED_NODE && in getNode()
4977 case ISD::SIGN_EXTEND: in getNode()
4980 case ISD::TRUNCATE: in getNode()
4984 case ISD::ZERO_EXTEND: in getNode()
4987 case ISD::ANY_EXTEND: in getNode()
4994 case ISD::UINT_TO_FP: in getNode()
4995 case ISD::SINT_TO_FP: { in getNode()
4999 Opcode==ISD::SINT_TO_FP, in getNode()
5003 case ISD::BITCAST: in getNode()
5013 case ISD::ABS: in getNode()
5016 case ISD::BITREVERSE: in getNode()
5019 case ISD::BSWAP: in getNode()
5022 case ISD::CTPOP: in getNode()
5025 case ISD::CTLZ: in getNode()
5026 case ISD::CTLZ_ZERO_UNDEF: in getNode()
5029 case ISD::CTTZ: in getNode()
5030 case ISD::CTTZ_ZERO_UNDEF: in getNode()
5033 case ISD::FP16_TO_FP: in getNode()
5034 case ISD::BF16_TO_FP: { in getNode()
5036 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf() in getNode()
5046 case ISD::STEP_VECTOR: { in getNode()
5058 case ISD::FNEG: in getNode()
5061 case ISD::FABS: in getNode()
5064 case ISD::FCEIL: { in getNode()
5070 case ISD::FTRUNC: { in getNode()
5076 case ISD::FFLOOR: { in getNode()
5082 case ISD::FP_EXTEND: { in getNode()
5090 case ISD::FP_TO_SINT: in getNode()
5091 case ISD::FP_TO_UINT: { in getNode()
5093 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT); in getNode()
5101 case ISD::BITCAST: in getNode()
5111 case ISD::FP_TO_FP16: in getNode()
5112 case ISD::FP_TO_BF16: { in getNode()
5116 (void)V.convert(Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf() in getNode()
5130 case ISD::FNEG: in getNode()
5131 case ISD::FABS: in getNode()
5132 case ISD::FCEIL: in getNode()
5133 case ISD::FTRUNC: in getNode()
5134 case ISD::FFLOOR: in getNode()
5135 case ISD::FP_EXTEND: in getNode()
5136 case ISD::FP_TO_SINT: in getNode()
5137 case ISD::FP_TO_UINT: in getNode()
5138 case ISD::TRUNCATE: in getNode()
5139 case ISD::ANY_EXTEND: in getNode()
5140 case ISD::ZERO_EXTEND: in getNode()
5141 case ISD::SIGN_EXTEND: in getNode()
5142 case ISD::UINT_TO_FP: in getNode()
5143 case ISD::SINT_TO_FP: in getNode()
5144 case ISD::ABS: in getNode()
5145 case ISD::BITREVERSE: in getNode()
5146 case ISD::BSWAP: in getNode()
5147 case ISD::CTLZ: in getNode()
5148 case ISD::CTLZ_ZERO_UNDEF: in getNode()
5149 case ISD::CTTZ: in getNode()
5150 case ISD::CTTZ_ZERO_UNDEF: in getNode()
5151 case ISD::CTPOP: { in getNode()
5160 case ISD::STEP_VECTOR: in getNode()
5163 assert(OpOpcode == ISD::TargetConstant && in getNode()
5167 case ISD::FREEZE: in getNode()
5172 case ISD::TokenFactor: in getNode()
5173 case ISD::MERGE_VALUES: in getNode()
5174 case ISD::CONCAT_VECTORS: in getNode()
5176 case ISD::BUILD_VECTOR: { in getNode()
5183 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node"); in getNode()
5184 case ISD::FP_EXTEND: in getNode()
5197 case ISD::FP_TO_SINT: in getNode()
5198 case ISD::FP_TO_UINT: in getNode()
5202 case ISD::SINT_TO_FP: in getNode()
5203 case ISD::UINT_TO_FP: in getNode()
5208 case ISD::SIGN_EXTEND: in getNode()
5221 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) in getNode()
5223 if (OpOpcode == ISD::UNDEF) in getNode()
5227 case ISD::ZERO_EXTEND: in getNode()
5240 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x) in getNode()
5241 return getNode(ISD::ZERO_EXTEND, DL, VT, Operand.getOperand(0)); in getNode()
5242 if (OpOpcode == ISD::UNDEF) in getNode()
5246 case ISD::ANY_EXTEND: in getNode()
5260 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || in getNode()
5261 OpOpcode == ISD::ANY_EXTEND) in getNode()
5264 if (OpOpcode == ISD::UNDEF) in getNode()
5268 if (OpOpcode == ISD::TRUNCATE) { in getNode()
5276 case ISD::TRUNCATE: in getNode()
5289 if (OpOpcode == ISD::TRUNCATE) in getNode()
5290 return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0)); in getNode()
5291 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || in getNode()
5292 OpOpcode == ISD::ANY_EXTEND) { in getNode()
5298 return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0)); in getNode()
5301 if (OpOpcode == ISD::UNDEF) in getNode()
5303 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes) in getNode()
5306 case ISD::ANY_EXTEND_VECTOR_INREG: in getNode()
5307 case ISD::ZERO_EXTEND_VECTOR_INREG: in getNode()
5308 case ISD::SIGN_EXTEND_VECTOR_INREG: in getNode()
5316 case ISD::ABS: in getNode()
5319 if (OpOpcode == ISD::UNDEF) in getNode()
5322 case ISD::BSWAP: in getNode()
5327 if (OpOpcode == ISD::UNDEF) in getNode()
5330 if (OpOpcode == ISD::BSWAP) in getNode()
5333 case ISD::BITREVERSE: in getNode()
5336 if (OpOpcode == ISD::UNDEF) in getNode()
5339 case ISD::BITCAST: in getNode()
5343 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x) in getNode()
5344 return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0)); in getNode()
5345 if (OpOpcode == ISD::UNDEF) in getNode()
5348 case ISD::SCALAR_TO_VECTOR: in getNode()
5355 if (OpOpcode == ISD::UNDEF) in getNode()
5358 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT && in getNode()
5364 case ISD::FNEG: in getNode()
5366 if (OpOpcode == ISD::UNDEF) in getNode()
5369 if (OpOpcode == ISD::FNEG) // --X -> X in getNode()
5372 case ISD::FABS: in getNode()
5373 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X) in getNode()
5374 return getNode(ISD::FABS, DL, VT, Operand.getOperand(0)); in getNode()
5376 case ISD::VSCALE: in getNode()
5379 case ISD::CTPOP: in getNode()
5383 case ISD::CTLZ: in getNode()
5384 case ISD::CTTZ: in getNode()
5388 case ISD::VECREDUCE_ADD: in getNode()
5390 return getNode(ISD::VECREDUCE_XOR, DL, VT, Operand); in getNode()
5392 case ISD::VECREDUCE_SMIN: in getNode()
5393 case ISD::VECREDUCE_UMAX: in getNode()
5395 return getNode(ISD::VECREDUCE_OR, DL, VT, Operand); in getNode()
5397 case ISD::VECREDUCE_SMAX: in getNode()
5398 case ISD::VECREDUCE_UMIN: in getNode()
5400 return getNode(ISD::VECREDUCE_AND, DL, VT, Operand); in getNode()
5434 case ISD::ADD: return C1 + C2; in FoldValue()
5435 case ISD::SUB: return C1 - C2; in FoldValue()
5436 case ISD::MUL: return C1 * C2; in FoldValue()
5437 case ISD::AND: return C1 & C2; in FoldValue()
5438 case ISD::OR: return C1 | C2; in FoldValue()
5439 case ISD::XOR: return C1 ^ C2; in FoldValue()
5440 case ISD::SHL: return C1 << C2; in FoldValue()
5441 case ISD::SRL: return C1.lshr(C2); in FoldValue()
5442 case ISD::SRA: return C1.ashr(C2); in FoldValue()
5443 case ISD::ROTL: return C1.rotl(C2); in FoldValue()
5444 case ISD::ROTR: return C1.rotr(C2); in FoldValue()
5445 case ISD::SMIN: return C1.sle(C2) ? C1 : C2; in FoldValue()
5446 case ISD::SMAX: return C1.sge(C2) ? C1 : C2; in FoldValue()
5447 case ISD::UMIN: return C1.ule(C2) ? C1 : C2; in FoldValue()
5448 case ISD::UMAX: return C1.uge(C2) ? C1 : C2; in FoldValue()
5449 case ISD::SADDSAT: return C1.sadd_sat(C2); in FoldValue()
5450 case ISD::UADDSAT: return C1.uadd_sat(C2); in FoldValue()
5451 case ISD::SSUBSAT: return C1.ssub_sat(C2); in FoldValue()
5452 case ISD::USUBSAT: return C1.usub_sat(C2); in FoldValue()
5453 case ISD::SSHLSAT: return C1.sshl_sat(C2); in FoldValue()
5454 case ISD::USHLSAT: return C1.ushl_sat(C2); in FoldValue()
5455 case ISD::UDIV: in FoldValue()
5459 case ISD::UREM: in FoldValue()
5463 case ISD::SDIV: in FoldValue()
5467 case ISD::SREM: in FoldValue()
5471 case ISD::MULHS: { in FoldValue()
5477 case ISD::MULHU: { in FoldValue()
5483 case ISD::AVGFLOORS: { in FoldValue()
5489 case ISD::AVGFLOORU: { in FoldValue()
5495 case ISD::AVGCEILS: { in FoldValue()
5501 case ISD::AVGCEILU: { in FoldValue()
5514 if (GA->getOpcode() != ISD::GlobalAddress) in FoldSymbolOffset()
5523 case ISD::ADD: break; in FoldSymbolOffset()
5524 case ISD::SUB: Offset = -uint64_t(Offset); break; in FoldSymbolOffset()
5533 case ISD::SDIV: in isUndef()
5534 case ISD::UDIV: in isUndef()
5535 case ISD::SREM: in isUndef()
5536 case ISD::UREM: { in isUndef()
5544 return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) && in isUndef()
5564 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS) in FoldConstantArithmetic()
5614 Ops[0].getOpcode() == ISD::BITCAST && in FoldConstantArithmetic()
5615 Ops[1].getOpcode() == ISD::BITCAST) { in FoldConstantArithmetic()
5660 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) && in FoldConstantArithmetic()
5661 Ops[0].getOpcode() == ISD::STEP_VECTOR) { in FoldConstantArithmetic()
5663 if (ISD::isConstantSplatVector(Ops[1].getNode(), RHSVal)) { in FoldConstantArithmetic()
5664 APInt NewStep = Opcode == ISD::MUL in FoldConstantArithmetic()
5677 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE || in FoldConstantArithmetic()
5678 Op.getOpcode() == ISD::BUILD_VECTOR || in FoldConstantArithmetic()
5679 Op.getOpcode() == ISD::SPLAT_VECTOR; in FoldConstantArithmetic()
5692 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType()); in FoldConstantArithmetic()
5693 ISD::NodeType ExtendCode = in FoldConstantArithmetic()
5694 (Opcode == ISD::SETCC && SVT != VT.getScalarType()) in FoldConstantArithmetic()
5696 : ISD::SIGN_EXTEND; in FoldConstantArithmetic()
5718 if (Op.getOpcode() != ISD::BUILD_VECTOR && in FoldConstantArithmetic()
5719 Op.getOpcode() != ISD::SPLAT_VECTOR) { in FoldConstantArithmetic()
5728 Op.getOperand(Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I); in FoldConstantArithmetic()
5743 ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp); in FoldConstantArithmetic()
5757 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant && in FoldConstantArithmetic()
5758 ScalarResult.getOpcode() != ISD::ConstantFP) in FoldConstantArithmetic()
5781 case ISD::FADD: in foldConstantFPMath()
5784 case ISD::FSUB: in foldConstantFPMath()
5787 case ISD::FMUL: in foldConstantFPMath()
5790 case ISD::FDIV: in foldConstantFPMath()
5793 case ISD::FREM: in foldConstantFPMath()
5796 case ISD::FCOPYSIGN: in foldConstantFPMath()
5799 case ISD::FMINNUM: in foldConstantFPMath()
5801 case ISD::FMAXNUM: in foldConstantFPMath()
5803 case ISD::FMINIMUM: in foldConstantFPMath()
5805 case ISD::FMAXIMUM: in foldConstantFPMath()
5810 if (N1CFP && Opcode == ISD::FP_ROUND) { in foldConstantFPMath()
5821 case ISD::FSUB: in foldConstantFPMath()
5828 case ISD::FADD: in foldConstantFPMath()
5829 case ISD::FMUL: in foldConstantFPMath()
5830 case ISD::FDIV: in foldConstantFPMath()
5831 case ISD::FREM: in foldConstantFPMath()
5851 AddNodeIDNode(ID, ISD::AssertAlign, getVTList(Val.getValueType()), {Val}); in getAssertAlign()
5894 else if (N1.getOpcode() == ISD::SPLAT_VECTOR && in canonicalizeCommutativeBinop()
5895 N2.getOpcode() == ISD::STEP_VECTOR) in canonicalizeCommutativeBinop()
5901 assert(N1.getOpcode() != ISD::DELETED_NODE && in getNode()
5902 N2.getOpcode() != ISD::DELETED_NODE && in getNode()
5917 case ISD::TokenFactor: in getNode()
5921 if (N1.getOpcode() == ISD::EntryToken) return N2; in getNode()
5922 if (N2.getOpcode() == ISD::EntryToken) return N1; in getNode()
5925 case ISD::BUILD_VECTOR: { in getNode()
5932 case ISD::CONCAT_VECTORS: { in getNode()
5938 case ISD::AND: in getNode()
5949 case ISD::OR: in getNode()
5950 case ISD::XOR: in getNode()
5951 case ISD::ADD: in getNode()
5952 case ISD::SUB: in getNode()
5960 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) && VT.isVector() && in getNode()
5962 return getNode(ISD::XOR, DL, VT, N1, N2); in getNode()
5964 case ISD::MUL: in getNode()
5969 return getNode(ISD::AND, DL, VT, N1, N2); in getNode()
5970 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) { in getNode()
5976 case ISD::UDIV: in getNode()
5977 case ISD::UREM: in getNode()
5978 case ISD::MULHU: in getNode()
5979 case ISD::MULHS: in getNode()
5980 case ISD::SDIV: in getNode()
5981 case ISD::SREM: in getNode()
5982 case ISD::SADDSAT: in getNode()
5983 case ISD::SSUBSAT: in getNode()
5984 case ISD::UADDSAT: in getNode()
5985 case ISD::USUBSAT: in getNode()
5991 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT) in getNode()
5992 return getNode(ISD::OR, DL, VT, N1, N2); in getNode()
5994 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT) in getNode()
5995 return getNode(ISD::AND, DL, VT, N1, getNOT(DL, N2, VT)); in getNode()
5998 case ISD::SMIN: in getNode()
5999 case ISD::UMAX: in getNode()
6004 return getNode(ISD::OR, DL, VT, N1, N2); in getNode()
6006 case ISD::SMAX: in getNode()
6007 case ISD::UMIN: in getNode()
6012 return getNode(ISD::AND, DL, VT, N1, N2); in getNode()
6014 case ISD::FADD: in getNode()
6015 case ISD::FSUB: in getNode()
6016 case ISD::FMUL: in getNode()
6017 case ISD::FDIV: in getNode()
6018 case ISD::FREM: in getNode()
6025 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. in getNode()
6031 case ISD::SHL: in getNode()
6032 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) { in getNode()
6038 case ISD::SRA: in getNode()
6039 case ISD::SRL: in getNode()
6043 case ISD::ROTL: in getNode()
6044 case ISD::ROTR: in getNode()
6067 case ISD::FP_ROUND: in getNode()
6075 case ISD::AssertSext: in getNode()
6076 case ISD::AssertZext: { in getNode()
6088 case ISD::SIGN_EXTEND_INREG: { in getNode()
6114 if (ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) { in getNode()
6131 case ISD::FP_TO_SINT_SAT: in getNode()
6132 case ISD::FP_TO_UINT_SAT: { in getNode()
6147 case ISD::EXTRACT_VECTOR_ELT: in getNode()
6168 N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0) { in getNode()
6171 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in getNode()
6178 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR || in getNode()
6179 N1.getOpcode() == ISD::SPLAT_VECTOR)) { in getNode()
6180 assert((N1.getOpcode() != ISD::BUILD_VECTOR || in getNode()
6184 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0; in getNode()
6198 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) { in getNode()
6215 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); in getNode()
6227 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR && in getNode()
6230 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), in getNode()
6234 case ISD::EXTRACT_ELEMENT: in getNode()
6244 if (N1.getOpcode() == ISD::BUILD_PAIR) in getNode()
6255 case ISD::EXTRACT_SUBVECTOR: { in getNode()
6285 if (N1.getOpcode() == ISD::CONCAT_VECTORS && N1.getNumOperands() > 0 && in getNode()
6293 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) && in getNode()
6310 case ISD::SUB: in getNode()
6312 case ISD::SIGN_EXTEND_INREG: in getNode()
6313 case ISD::UDIV: in getNode()
6314 case ISD::SDIV: in getNode()
6315 case ISD::UREM: in getNode()
6316 case ISD::SREM: in getNode()
6317 case ISD::SSUBSAT: in getNode()
6318 case ISD::USUBSAT: in getNode()
6327 case ISD::XOR: in getNode()
6333 case ISD::ADD: in getNode()
6334 case ISD::SUB: in getNode()
6335 case ISD::UDIV: in getNode()
6336 case ISD::SDIV: in getNode()
6337 case ISD::UREM: in getNode()
6338 case ISD::SREM: in getNode()
6340 case ISD::MUL: in getNode()
6341 case ISD::AND: in getNode()
6342 case ISD::SSUBSAT: in getNode()
6343 case ISD::USUBSAT: in getNode()
6345 case ISD::OR: in getNode()
6346 case ISD::SADDSAT: in getNode()
6347 case ISD::UADDSAT: in getNode()
6391 assert(N1.getOpcode() != ISD::DELETED_NODE && in getNode()
6392 N2.getOpcode() != ISD::DELETED_NODE && in getNode()
6393 N3.getOpcode() != ISD::DELETED_NODE && in getNode()
6397 case ISD::FMA: { in getNode()
6413 case ISD::BUILD_VECTOR: { in getNode()
6420 case ISD::CONCAT_VECTORS: { in getNode()
6426 case ISD::SETCC: { in getNode()
6446 case ISD::SELECT: in getNode()
6447 case ISD::VSELECT: in getNode()
6451 case ISD::VECTOR_SHUFFLE: in getNode()
6453 case ISD::VECTOR_SPLICE: { in getNode()
6458 case ISD::INSERT_VECTOR_ELT: { in getNode()
6477 case ISD::INSERT_SUBVECTOR: { in getNode()
6509 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR && in getNode()
6514 case ISD::BITCAST: in getNode()
6580 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); in getStackArgumentTokenFactor()
6607 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value); in getMemsetValue()
6612 Value = DAG.getNode(ISD::MUL, dl, IntVT, Value, in getMemsetValue()
6639 return DAG.getNode(ISD::BITCAST, dl, VT, in getMemsetStringVal()
6690 return getNode(ISD::ADD, DL, BasePtrVT, Ptr, Offset, Flags); in getMemBasePlusOffset()
6697 if (Src.getOpcode() == ISD::GlobalAddress) in isMemSrcFromConstant()
6699 else if (Src.getOpcode() == ISD::ADD && in isMemSrcFromConstant()
6700 Src.getOperand(0).getOpcode() == ISD::GlobalAddress && in isMemSrcFromConstant()
6701 Src.getOperand(1).getOpcode() == ISD::Constant) { in isMemSrcFromConstant()
6734 SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in chainLoadsAndStoresForMemcpy()
6885 ISD::EXTLOAD, dl, NVT, Chain, in getMemcpyLoadsAndStores()
6945 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in getMemcpyLoadsAndStores()
7027 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); in getMemmoveLoadsAndStores()
7042 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in getMemmoveLoadsAndStores()
7137 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue); in getMemsetStores()
7153 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in getMemsetStores()
7539 assert(Opcode == ISD::ATOMIC_CMP_SWAP || in getAtomicCmpSwap()
7540 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); in getAtomicCmpSwap()
7550 assert((Opcode == ISD::ATOMIC_LOAD_ADD || in getAtomic()
7551 Opcode == ISD::ATOMIC_LOAD_SUB || in getAtomic()
7552 Opcode == ISD::ATOMIC_LOAD_AND || in getAtomic()
7553 Opcode == ISD::ATOMIC_LOAD_CLR || in getAtomic()
7554 Opcode == ISD::ATOMIC_LOAD_OR || in getAtomic()
7555 Opcode == ISD::ATOMIC_LOAD_XOR || in getAtomic()
7556 Opcode == ISD::ATOMIC_LOAD_NAND || in getAtomic()
7557 Opcode == ISD::ATOMIC_LOAD_MIN || in getAtomic()
7558 Opcode == ISD::ATOMIC_LOAD_MAX || in getAtomic()
7559 Opcode == ISD::ATOMIC_LOAD_UMIN || in getAtomic()
7560 Opcode == ISD::ATOMIC_LOAD_UMAX || in getAtomic()
7561 Opcode == ISD::ATOMIC_LOAD_FADD || in getAtomic()
7562 Opcode == ISD::ATOMIC_LOAD_FSUB || in getAtomic()
7563 Opcode == ISD::ATOMIC_LOAD_FMAX || in getAtomic()
7564 Opcode == ISD::ATOMIC_LOAD_FMIN || in getAtomic()
7565 Opcode == ISD::ATOMIC_SWAP || in getAtomic()
7566 Opcode == ISD::ATOMIC_STORE) && in getAtomic()
7571 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : in getAtomic()
7580 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); in getAtomic()
7596 return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops); in getMergeValues()
7619 assert((Opcode == ISD::INTRINSIC_VOID || in getMemIntrinsicNode()
7620 Opcode == ISD::INTRINSIC_W_CHAIN || in getMemIntrinsicNode()
7621 Opcode == ISD::PREFETCH || in getMemIntrinsicNode()
7623 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && in getMemIntrinsicNode()
7660 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END; in getLifetimeNode()
7690 const unsigned Opcode = ISD::PSEUDO_PROBE; in getPseudoProbeNode()
7724 if (Ptr.getOpcode() != ISD::ADD || in InferPointerInfo()
7750 SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, in getLoad()
7774 SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, in getLoad()
7779 ExtType = ISD::NON_EXTLOAD; in getLoad()
7780 } else if (ExtType == ISD::NON_EXTLOAD) { in getLoad()
7795 bool Indexed = AM != ISD::UNINDEXED; in getLoad()
7802 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops); in getLoad()
7830 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, in getLoad()
7837 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, in getLoad()
7841 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, in getExtLoad()
7848 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo, in getExtLoad()
7852 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, in getExtLoad()
7856 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, in getExtLoad()
7862 ISD::MemIndexedMode AM) { in getIndexedLoad()
7904 AddNodeIDNode(ID, ISD::STORE, VTs, Ops); in getStore()
7907 dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO)); in getStore()
7916 ISD::UNINDEXED, false, VT, MMO); in getStore()
7971 AddNodeIDNode(ID, ISD::STORE, VTs, Ops); in getTruncStore()
7974 dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO)); in getTruncStore()
7983 ISD::UNINDEXED, true, SVT, MMO); in getTruncStore()
7995 ISD::MemIndexedMode AM) { in getIndexedStore()
8001 AddNodeIDNode(ID, ISD::STORE, VTs, Ops); in getIndexedStore()
8023 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, in getLoadVP()
8045 SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM, in getLoadVP()
8046 ISD::LoadExtType ExtType, EVT VT, in getLoadVP()
8051 bool Indexed = AM != ISD::UNINDEXED; in getLoadVP()
8058 AddNodeIDNode(ID, ISD::VP_LOAD, VTs, Ops); in getLoadVP()
8088 return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, in getLoadVP()
8097 return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, in getLoadVP()
8101 SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, in getExtLoadVP()
8109 return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask, in getExtLoadVP()
8114 SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, in getExtLoadVP()
8119 return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask, in getExtLoadVP()
8125 ISD::MemIndexedMode AM) { in getIndexedLoadVP()
8142 ISD::MemIndexedMode AM, bool IsTruncating, in getStoreVP()
8145 bool Indexed = AM != ISD::UNINDEXED; in getStoreVP()
8151 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops); in getStoreVP()
8206 EVL, VT, MMO, ISD::UNINDEXED, in getTruncStoreVP()
8222 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops); in getTruncStoreVP()
8225 dl.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO)); in getTruncStoreVP()
8235 ISD::UNINDEXED, true, IsCompressing, SVT, MMO); in getTruncStoreVP()
8247 ISD::MemIndexedMode AM) { in getIndexedStoreVP()
8254 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops); in getIndexedStoreVP()
8276 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, in getStridedLoadVP()
8299 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, in getStridedLoadVP()
8302 bool Indexed = AM != ISD::UNINDEXED; in getStridedLoadVP()
8309 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTs, Ops); in getStridedLoadVP()
8338 return getStridedLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, DL, Chain, Ptr, in getStridedLoadVP()
8349 return getStridedLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, DL, Chain, Ptr, in getStridedLoadVP()
8354 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain, in getExtStridedLoadVP()
8360 return getStridedLoadVP(ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Undef, in getExtStridedLoadVP()
8366 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain, in getExtStridedLoadVP()
8370 return getStridedLoadVP(ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Undef, in getExtStridedLoadVP()
8376 ISD::MemIndexedMode AM) { in getIndexedStridedLoadVP()
8396 ISD::MemIndexedMode AM, in getStridedStoreVP()
8399 bool Indexed = AM != ISD::UNINDEXED; in getStridedStoreVP()
8405 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops); in getStridedStoreVP()
8458 Stride, Mask, EVL, VT, MMO, ISD::UNINDEXED, in getTruncStridedStoreVP()
8474 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops); in getTruncStridedStoreVP()
8477 DL.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO)); in getTruncStridedStoreVP()
8485 VTs, ISD::UNINDEXED, true, in getTruncStridedStoreVP()
8499 ISD::MemIndexedMode AM) { in getIndexedStridedStoreVP()
8508 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops); in getIndexedStridedStoreVP()
8530 ISD::MemIndexType IndexType) { in getGatherVP()
8534 AddNodeIDNode(ID, ISD::VP_GATHER, VTs, Ops); in getGatherVP()
8574 ISD::MemIndexType IndexType) { in getScatterVP()
8578 AddNodeIDNode(ID, ISD::VP_SCATTER, VTs, Ops); in getScatterVP()
8619 ISD::MemIndexedMode AM, in getMaskedLoad()
8620 ISD::LoadExtType ExtTy, bool isExpanding) { in getMaskedLoad()
8621 bool Indexed = AM != ISD::UNINDEXED; in getMaskedLoad()
8628 AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops); in getMaskedLoad()
8652 ISD::MemIndexedMode AM) { in getIndexedMaskedLoad()
8665 ISD::MemIndexedMode AM, bool IsTruncating, in getMaskedStore()
8669 bool Indexed = AM != ISD::UNINDEXED; in getMaskedStore()
8676 AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops); in getMaskedStore()
8701 ISD::MemIndexedMode AM) { in getIndexedMaskedStore()
8713 ISD::MemIndexType IndexType, in getMaskedGather()
8714 ISD::LoadExtType ExtTy) { in getMaskedGather()
8718 AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops); in getMaskedGather()
8760 ISD::MemIndexType IndexType, in getMaskedScatter()
8765 AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops); in getMaskedScatter()
8851 if (ISD::matchUnaryPredicate(Y, isShiftTooBig, true)) in simplifyShift()
8879 if (Opcode == ISD::FADD) in simplifyFPBinop()
8884 if (Opcode == ISD::FSUB) in simplifyFPBinop()
8890 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV) in simplifyFPBinop()
8895 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros()) in simplifyFPBinop()
8905 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops); in getVAArg()
8945 assert(Op.getOpcode() != ISD::DELETED_NODE && in getNode()
8951 case ISD::BUILD_VECTOR: in getNode()
8956 case ISD::CONCAT_VECTORS: in getNode()
8960 case ISD::SELECT_CC: in getNode()
8974 case ISD::BR_CC: in getNode()
8979 case ISD::VP_ADD: in getNode()
8980 case ISD::VP_SUB: in getNode()
8983 Opcode = ISD::VP_XOR; in getNode()
8985 case ISD::VP_MUL: in getNode()
8988 Opcode = ISD::VP_AND; in getNode()
8990 case ISD::VP_REDUCE_MUL: in getNode()
8993 Opcode = ISD::VP_REDUCE_AND; in getNode()
8995 case ISD::VP_REDUCE_ADD: in getNode()
8998 Opcode = ISD::VP_REDUCE_XOR; in getNode()
9000 case ISD::VP_REDUCE_SMAX: in getNode()
9001 case ISD::VP_REDUCE_UMIN: in getNode()
9005 Opcode = ISD::VP_REDUCE_AND; in getNode()
9007 case ISD::VP_REDUCE_SMIN: in getNode()
9008 case ISD::VP_REDUCE_UMAX: in getNode()
9012 Opcode = ISD::VP_REDUCE_OR; in getNode()
9064 assert(Op.getOpcode() != ISD::DELETED_NODE && in getNode()
9069 case ISD::SADDO: in getNode()
9070 case ISD::UADDO: in getNode()
9071 case ISD::SSUBO: in getNode()
9072 case ISD::USUBO: { in getNode()
9087 return getNode(ISD::MERGE_VALUES, DL, VTList, {N1, ZeroOverFlow}, Flags); in getNode()
9091 case ISD::SMUL_LOHI: in getNode()
9092 case ISD::UMUL_LOHI: { in getNode()
9100 case ISD::STRICT_FP_EXTEND: in getNode()
9115 case ISD::STRICT_FP_ROUND: in getNode()
9136 case ISD::SRA_PARTS: in getNode()
9137 case ISD::SRL_PARTS: in getNode()
9138 case ISD::SHL_PARTS: in getNode()
9139 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && in getNode()
9142 else if (N3.getOpcode() == ISD::AND) in getNode()
9611 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break; in mutateStrictFPToFP()
9613 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break; in mutateStrictFPToFP()
9996 case ISD::ADD: in salvageDebugInfo()
10533 assert(AllNodes.front().getOpcode() == ISD::EntryToken && in AssignTopologicalOrder()
10572 SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other, in makeEquivalentMemoryOrdering()
10640 while (V.getOpcode() == ISD::BITCAST) in peekThroughBitcasts()
10646 while (V.getOpcode() == ISD::BITCAST && V.getOperand(0).hasOneUse()) in peekThroughOneUseBitcasts()
10652 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR) in peekThroughExtractSubvectors()
10658 if (V.getOpcode() != ISD::XOR) in isBitwiseNot()
10674 if (N->getOpcode() == ISD::SPLAT_VECTOR) { in isConstOrConstSplat()
10737 if (N.getOpcode() == ISD::SPLAT_VECTOR) in isConstOrConstSplatFP()
10796 : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)), in AddrSpaceCastSDNode()
10936 if (getOpcode() == ISD::TokenFactor) { in reachesChainWithoutSideEffects()
10977 SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, in matchBinOpReduction()
10978 ArrayRef<ISD::NodeType> CandidateBinOps, in matchBinOpReduction()
10981 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT || in matchBinOpReduction()
10987 if (llvm::none_of(CandidateBinOps, [Op](ISD::NodeType BinOp) { in matchBinOpReduction()
10998 case ISD::FADD: in matchBinOpReduction()
11017 BinOp = (ISD::NodeType)CandidateBinOp; in matchBinOpReduction()
11018 return getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Op), SubVT, Op, in matchBinOpReduction()
11073 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR || in matchBinOpReduction()
11074 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR || in matchBinOpReduction()
11089 BinOp = (ISD::NodeType)CandidateBinOp; in matchBinOpReduction()
11119 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, in UnrollVectorOp()
11133 case ISD::VSELECT: in UnrollVectorOp()
11134 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands)); in UnrollVectorOp()
11136 case ISD::SHL: in UnrollVectorOp()
11137 case ISD::SRA: in UnrollVectorOp()
11138 case ISD::SRL: in UnrollVectorOp()
11139 case ISD::ROTL: in UnrollVectorOp()
11140 case ISD::ROTR: in UnrollVectorOp()
11145 case ISD::SIGN_EXTEND_INREG: { in UnrollVectorOp()
11164 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO || in UnrollVectorOverflowOp()
11165 Opcode == ISD::USUBO || Opcode == ISD::SSUBO || in UnrollVectorOverflowOp()
11166 Opcode == ISD::UMULO || Opcode == ISD::SMULO) && in UnrollVectorOverflowOp()
11331 getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, getVectorIdxConstant(0, DL)); in SplitVector()
11336 Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N, in SplitVector()
11353 SDValue Lo = getNode(ISD::UMIN, DL, VT, N, HalfNumElts); in SplitEVL()
11354 SDValue Hi = getNode(ISD::USUBSAT, DL, VT, N, HalfNumElts); in SplitEVL()
11363 return getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, getUNDEF(WideVT), N, in WidenVector()
11378 Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Op, in ExtractVectorElements()
11687 if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP) in isConstant()
11746 if (ISD::isBuildVectorOfConstantSDNodes(N.getNode())) in isConstantIntBuildVectorOrConstantInt()
11751 if (GA->getOpcode() == ISD::GlobalAddress && in isConstantIntBuildVectorOrConstantInt()
11754 if ((N.getOpcode() == ISD::SPLAT_VECTOR) && in isConstantIntBuildVectorOrConstantInt()
11766 if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode())) in isConstantFPBuildVectorOrConstantFP()
11769 if ((N.getOpcode() == ISD::SPLAT_VECTOR) && in isConstantFPBuildVectorOrConstantFP()
11805 SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs); in getTokenFactor()
11809 return getNode(ISD::TokenFactor, DL, MVT::Other, Vals); in getTokenFactor()
11817 case ISD::ADD: in getNeutralElement()
11818 case ISD::OR: in getNeutralElement()
11819 case ISD::XOR: in getNeutralElement()
11820 case ISD::UMAX: in getNeutralElement()
11822 case ISD::MUL: in getNeutralElement()
11824 case ISD::AND: in getNeutralElement()
11825 case ISD::UMIN: in getNeutralElement()
11827 case ISD::SMAX: in getNeutralElement()
11829 case ISD::SMIN: in getNeutralElement()
11831 case ISD::FADD: in getNeutralElement()
11833 case ISD::FMUL: in getNeutralElement()
11835 case ISD::FMINNUM: in getNeutralElement()
11836 case ISD::FMAXNUM: { in getNeutralElement()
11842 if (Opcode == ISD::FMAXNUM) in getNeutralElement()