Lines Matching refs:ISD
273 assert(N->getOpcode() != ISD::DELETED_NODE && in AddToWorklist()
278 if (N->getOpcode() == ISD::HANDLENODE) in AddToWorklist()
405 ISD::CondCode CC);
588 SDValue N2, SDValue N3, ISD::CondCode CC,
592 ISD::CondCode CC);
595 SDValue N2, SDValue N3, ISD::CondCode CC);
604 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
713 case ISD::Constant: in getStoreSource()
714 case ISD::ConstantFP: in getStoreSource()
716 case ISD::BUILD_VECTOR: in getStoreSource()
717 if (ISD::isBuildVectorOfConstantSDNodes(StoreVal.getNode()) || in getStoreSource()
718 ISD::isBuildVectorOfConstantFPSDNodes(StoreVal.getNode())) in getStoreSource()
721 case ISD::EXTRACT_VECTOR_ELT: in getStoreSource()
722 case ISD::EXTRACT_SUBVECTOR: in getStoreSource()
724 case ISD::LOAD: in getStoreSource()
746 bool isLegalNarrowLdSt(LSBaseSDNode *LDSTN, ISD::LoadExtType ExtType,
866 ISD::NodeType ExtType);
928 if (auto RootMaskPos = ISD::getVPMaskIdx(Root->getOpcode())) in VPMatchContext()
932 ISD::getVPExplicitVectorLengthIdx(Root->getOpcode())) in VPMatchContext()
942 auto BaseOpc = ISD::getBaseOpcodeForVP(OpVal->getOpcode(), in match()
949 if (auto MaskPos = ISD::getVPMaskIdx(VPOpcode)) { in match()
952 !ISD::isConstantSplatVectorAllOnes(MaskOp.getNode())) in match()
957 if (auto VLenPos = ISD::getVPExplicitVectorLengthIdx(VPOpcode)) in match()
968 unsigned VPOpcode = ISD::getVPForBaseOpcode(Opcode); in getNode()
969 assert(ISD::getVPMaskIdx(VPOpcode) == 1 && in getNode()
970 ISD::getVPExplicitVectorLengthIdx(VPOpcode) == 2); in getNode()
977 unsigned VPOpcode = ISD::getVPForBaseOpcode(Opcode); in getNode()
978 assert(ISD::getVPMaskIdx(VPOpcode) == 2 && in getNode()
979 ISD::getVPExplicitVectorLengthIdx(VPOpcode) == 3); in getNode()
986 unsigned VPOpcode = ISD::getVPForBaseOpcode(Opcode); in getNode()
987 assert(ISD::getVPMaskIdx(VPOpcode) == 3 && in getNode()
988 ISD::getVPExplicitVectorLengthIdx(VPOpcode) == 4); in getNode()
995 unsigned VPOpcode = ISD::getVPForBaseOpcode(Opcode); in getNode()
996 assert(ISD::getVPMaskIdx(VPOpcode) == 1 && in getNode()
997 ISD::getVPExplicitVectorLengthIdx(VPOpcode) == 2); in getNode()
1004 unsigned VPOpcode = ISD::getVPForBaseOpcode(Opcode); in getNode()
1005 assert(ISD::getVPMaskIdx(VPOpcode) == 2 && in getNode()
1006 ISD::getVPExplicitVectorLengthIdx(VPOpcode) == 3); in getNode()
1013 unsigned VPOpcode = ISD::getVPForBaseOpcode(Opcode); in getNode()
1014 assert(ISD::getVPMaskIdx(VPOpcode) == 3 && in getNode()
1015 ISD::getVPExplicitVectorLengthIdx(VPOpcode) == 4); in getNode()
1022 unsigned VPOp = ISD::getVPForBaseOpcode(Op); in isOperationLegalOrCustom()
1097 if (N.getOpcode() == ISD::SETCC) { in isSetCCEquivalent()
1105 (N.getOpcode() == ISD::STRICT_FSETCC || in isSetCCEquivalent()
1106 N.getOpcode() == ISD::STRICT_FSETCCS)) { in isSetCCEquivalent()
1113 if (N.getOpcode() != ISD::SELECT_CC || !TLI.isConstTrueVal(N.getOperand(2)) || in isSetCCEquivalent()
1158 if (ISD::isConstantSplatVector(N, Val)) in isConstantSplatVectorMaskForType()
1170 if (N.getOpcode() != ISD::BUILD_VECTOR && N.getOpcode() != ISD::SPLAT_VECTOR) in isConstantOrConstantVector()
1187 if (V.getOpcode() != ISD::BUILD_VECTOR) in isAnyConstantBuildVector()
1190 ISD::isBuildVectorOfConstantFPSDNodes(V.getNode()); in isAnyConstantBuildVector()
1196 (LD->getOperand(2).getOpcode() != ISD::TargetConstant || in canSplitIdx()
1214 if (Opc != ISD::ADD || N0.getOpcode() != ISD::ADD) in reassociationCanBreakAddressingModePattern()
1257 if (GA->getOpcode() == ISD::GlobalAddress && TLI.isOffsetFoldingLegal(GA)) in reassociationCanBreakAddressingModePattern()
1306 if (N0.getOpcode() == ISD::ADD && N0->getFlags().hasNoUnsignedWrap() && in reassociateOpsCommutative()
1315 if (Opc == ISD::AND || Opc == ISD::OR) { in reassociateOpsCommutative()
1323 if (Opc == ISD::XOR) { in reassociateOpsCommutative()
1360 if (Opc == ISD::AND || Opc == ISD::OR) { in reassociateOpsCommutative()
1361 if (N1->getOpcode() == ISD::SETCC && N00->getOpcode() == ISD::SETCC && in reassociateOpsCommutative()
1362 N01->getOpcode() == ISD::SETCC) { in reassociateOpsCommutative()
1363 ISD::CondCode CC1 = cast<CondCodeSDNode>(N1.getOperand(2))->get(); in reassociateOpsCommutative()
1364 ISD::CondCode CC00 = cast<CondCodeSDNode>(N00.getOperand(2))->get(); in reassociateOpsCommutative()
1365 ISD::CondCode CC01 = cast<CondCodeSDNode>(N01.getOperand(2))->get(); in reassociateOpsCommutative()
1505 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0)); in ReplaceLoadWithPromotedLoad()
1520 if (ISD::isUNINDEXEDLoad(Op.getNode())) { in PromoteOperand()
1523 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD in PromoteOperand()
1534 case ISD::AssertSext: in PromoteOperand()
1536 return DAG.getNode(ISD::AssertSext, DL, PVT, Op0, Op.getOperand(1)); in PromoteOperand()
1538 case ISD::AssertZext: in PromoteOperand()
1540 return DAG.getNode(ISD::AssertZext, DL, PVT, Op0, Op.getOperand(1)); in PromoteOperand()
1542 case ISD::Constant: { in PromoteOperand()
1544 Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in PromoteOperand()
1549 if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT)) in PromoteOperand()
1551 return DAG.getNode(ISD::ANY_EXTEND, DL, PVT, Op); in PromoteOperand()
1555 if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT)) in SExtPromoteOperand()
1567 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, NewOp.getValueType(), NewOp, in SExtPromoteOperand()
1620 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1)); in PromoteIntBinOp()
1680 if (Opc == ISD::SRA) in PromoteIntShiftOp()
1682 else if (Opc == ISD::SRL) in PromoteIntShiftOp()
1693 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1)); in PromoteIntShiftOp()
1699 if (Op && Op.getOpcode() != ISD::DELETED_NODE) in PromoteIntShiftOp()
1737 if (!ISD::isUNINDEXEDLoad(Op.getNode())) in PromoteLoad()
1760 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD in PromoteLoad()
1765 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD); in PromoteLoad()
1885 assert(N->getOpcode() != ISD::DELETED_NODE && in Run()
1886 RV.getOpcode() != ISD::DELETED_NODE && in Run()
1904 if (RV.getOpcode() != ISD::EntryToken) in Run()
1923 case ISD::TokenFactor: return visitTokenFactor(N); in visit()
1924 case ISD::MERGE_VALUES: return visitMERGE_VALUES(N); in visit()
1925 case ISD::ADD: return visitADD(N); in visit()
1926 case ISD::SUB: return visitSUB(N); in visit()
1927 case ISD::SADDSAT: in visit()
1928 case ISD::UADDSAT: return visitADDSAT(N); in visit()
1929 case ISD::SSUBSAT: in visit()
1930 case ISD::USUBSAT: return visitSUBSAT(N); in visit()
1931 case ISD::ADDC: return visitADDC(N); in visit()
1932 case ISD::SADDO: in visit()
1933 case ISD::UADDO: return visitADDO(N); in visit()
1934 case ISD::SUBC: return visitSUBC(N); in visit()
1935 case ISD::SSUBO: in visit()
1936 case ISD::USUBO: return visitSUBO(N); in visit()
1937 case ISD::ADDE: return visitADDE(N); in visit()
1938 case ISD::UADDO_CARRY: return visitUADDO_CARRY(N); in visit()
1939 case ISD::SADDO_CARRY: return visitSADDO_CARRY(N); in visit()
1940 case ISD::SUBE: return visitSUBE(N); in visit()
1941 case ISD::USUBO_CARRY: return visitUSUBO_CARRY(N); in visit()
1942 case ISD::SSUBO_CARRY: return visitSSUBO_CARRY(N); in visit()
1943 case ISD::SMULFIX: in visit()
1944 case ISD::SMULFIXSAT: in visit()
1945 case ISD::UMULFIX: in visit()
1946 case ISD::UMULFIXSAT: return visitMULFIX(N); in visit()
1947 case ISD::MUL: return visitMUL(N); in visit()
1948 case ISD::SDIV: return visitSDIV(N); in visit()
1949 case ISD::UDIV: return visitUDIV(N); in visit()
1950 case ISD::SREM: in visit()
1951 case ISD::UREM: return visitREM(N); in visit()
1952 case ISD::MULHU: return visitMULHU(N); in visit()
1953 case ISD::MULHS: return visitMULHS(N); in visit()
1954 case ISD::AVGFLOORS: in visit()
1955 case ISD::AVGFLOORU: in visit()
1956 case ISD::AVGCEILS: in visit()
1957 case ISD::AVGCEILU: return visitAVG(N); in visit()
1958 case ISD::ABDS: in visit()
1959 case ISD::ABDU: return visitABD(N); in visit()
1960 case ISD::SMUL_LOHI: return visitSMUL_LOHI(N); in visit()
1961 case ISD::UMUL_LOHI: return visitUMUL_LOHI(N); in visit()
1962 case ISD::SMULO: in visit()
1963 case ISD::UMULO: return visitMULO(N); in visit()
1964 case ISD::SMIN: in visit()
1965 case ISD::SMAX: in visit()
1966 case ISD::UMIN: in visit()
1967 case ISD::UMAX: return visitIMINMAX(N); in visit()
1968 case ISD::AND: return visitAND(N); in visit()
1969 case ISD::OR: return visitOR(N); in visit()
1970 case ISD::XOR: return visitXOR(N); in visit()
1971 case ISD::SHL: return visitSHL(N); in visit()
1972 case ISD::SRA: return visitSRA(N); in visit()
1973 case ISD::SRL: return visitSRL(N); in visit()
1974 case ISD::ROTR: in visit()
1975 case ISD::ROTL: return visitRotate(N); in visit()
1976 case ISD::FSHL: in visit()
1977 case ISD::FSHR: return visitFunnelShift(N); in visit()
1978 case ISD::SSHLSAT: in visit()
1979 case ISD::USHLSAT: return visitSHLSAT(N); in visit()
1980 case ISD::ABS: return visitABS(N); in visit()
1981 case ISD::BSWAP: return visitBSWAP(N); in visit()
1982 case ISD::BITREVERSE: return visitBITREVERSE(N); in visit()
1983 case ISD::CTLZ: return visitCTLZ(N); in visit()
1984 case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N); in visit()
1985 case ISD::CTTZ: return visitCTTZ(N); in visit()
1986 case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N); in visit()
1987 case ISD::CTPOP: return visitCTPOP(N); in visit()
1988 case ISD::SELECT: return visitSELECT(N); in visit()
1989 case ISD::VSELECT: return visitVSELECT(N); in visit()
1990 case ISD::SELECT_CC: return visitSELECT_CC(N); in visit()
1991 case ISD::SETCC: return visitSETCC(N); in visit()
1992 case ISD::SETCCCARRY: return visitSETCCCARRY(N); in visit()
1993 case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N); in visit()
1994 case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N); in visit()
1995 case ISD::ANY_EXTEND: return visitANY_EXTEND(N); in visit()
1996 case ISD::AssertSext: in visit()
1997 case ISD::AssertZext: return visitAssertExt(N); in visit()
1998 case ISD::AssertAlign: return visitAssertAlign(N); in visit()
1999 case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N); in visit()
2000 case ISD::SIGN_EXTEND_VECTOR_INREG: in visit()
2001 case ISD::ZERO_EXTEND_VECTOR_INREG: in visit()
2002 case ISD::ANY_EXTEND_VECTOR_INREG: return visitEXTEND_VECTOR_INREG(N); in visit()
2003 case ISD::TRUNCATE: return visitTRUNCATE(N); in visit()
2004 case ISD::BITCAST: return visitBITCAST(N); in visit()
2005 case ISD::BUILD_PAIR: return visitBUILD_PAIR(N); in visit()
2006 case ISD::FADD: return visitFADD(N); in visit()
2007 case ISD::STRICT_FADD: return visitSTRICT_FADD(N); in visit()
2008 case ISD::FSUB: return visitFSUB(N); in visit()
2009 case ISD::FMUL: return visitFMUL(N); in visit()
2010 case ISD::FMA: return visitFMA<EmptyMatchContext>(N); in visit()
2011 case ISD::FMAD: return visitFMAD(N); in visit()
2012 case ISD::FDIV: return visitFDIV(N); in visit()
2013 case ISD::FREM: return visitFREM(N); in visit()
2014 case ISD::FSQRT: return visitFSQRT(N); in visit()
2015 case ISD::FCOPYSIGN: return visitFCOPYSIGN(N); in visit()
2016 case ISD::FPOW: return visitFPOW(N); in visit()
2017 case ISD::SINT_TO_FP: return visitSINT_TO_FP(N); in visit()
2018 case ISD::UINT_TO_FP: return visitUINT_TO_FP(N); in visit()
2019 case ISD::FP_TO_SINT: return visitFP_TO_SINT(N); in visit()
2020 case ISD::FP_TO_UINT: return visitFP_TO_UINT(N); in visit()
2021 case ISD::LRINT: in visit()
2022 case ISD::LLRINT: return visitXRINT(N); in visit()
2023 case ISD::FP_ROUND: return visitFP_ROUND(N); in visit()
2024 case ISD::FP_EXTEND: return visitFP_EXTEND(N); in visit()
2025 case ISD::FNEG: return visitFNEG(N); in visit()
2026 case ISD::FABS: return visitFABS(N); in visit()
2027 case ISD::FFLOOR: return visitFFLOOR(N); in visit()
2028 case ISD::FMINNUM: in visit()
2029 case ISD::FMAXNUM: in visit()
2030 case ISD::FMINIMUM: in visit()
2031 case ISD::FMAXIMUM: return visitFMinMax(N); in visit()
2032 case ISD::FCEIL: return visitFCEIL(N); in visit()
2033 case ISD::FTRUNC: return visitFTRUNC(N); in visit()
2034 case ISD::FFREXP: return visitFFREXP(N); in visit()
2035 case ISD::BRCOND: return visitBRCOND(N); in visit()
2036 case ISD::BR_CC: return visitBR_CC(N); in visit()
2037 case ISD::LOAD: return visitLOAD(N); in visit()
2038 case ISD::STORE: return visitSTORE(N); in visit()
2039 case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N); in visit()
2040 case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N); in visit()
2041 case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N); in visit()
2042 case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N); in visit()
2043 case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N); in visit()
2044 case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N); in visit()
2045 case ISD::SCALAR_TO_VECTOR: return visitSCALAR_TO_VECTOR(N); in visit()
2046 case ISD::INSERT_SUBVECTOR: return visitINSERT_SUBVECTOR(N); in visit()
2047 case ISD::MGATHER: return visitMGATHER(N); in visit()
2048 case ISD::MLOAD: return visitMLOAD(N); in visit()
2049 case ISD::MSCATTER: return visitMSCATTER(N); in visit()
2050 case ISD::MSTORE: return visitMSTORE(N); in visit()
2051 case ISD::LIFETIME_END: return visitLIFETIME_END(N); in visit()
2052 case ISD::FP_TO_FP16: return visitFP_TO_FP16(N); in visit()
2053 case ISD::FP16_TO_FP: return visitFP16_TO_FP(N); in visit()
2054 case ISD::FP_TO_BF16: return visitFP_TO_BF16(N); in visit()
2055 case ISD::BF16_TO_FP: return visitBF16_TO_FP(N); in visit()
2056 case ISD::FREEZE: return visitFREEZE(N); in visit()
2057 case ISD::GET_FPENV_MEM: return visitGET_FPENV_MEM(N); in visit()
2058 case ISD::SET_FPENV_MEM: return visitSET_FPENV_MEM(N); in visit()
2059 case ISD::VECREDUCE_FADD: in visit()
2060 case ISD::VECREDUCE_FMUL: in visit()
2061 case ISD::VECREDUCE_ADD: in visit()
2062 case ISD::VECREDUCE_MUL: in visit()
2063 case ISD::VECREDUCE_AND: in visit()
2064 case ISD::VECREDUCE_OR: in visit()
2065 case ISD::VECREDUCE_XOR: in visit()
2066 case ISD::VECREDUCE_SMAX: in visit()
2067 case ISD::VECREDUCE_SMIN: in visit()
2068 case ISD::VECREDUCE_UMAX: in visit()
2069 case ISD::VECREDUCE_UMIN: in visit()
2070 case ISD::VECREDUCE_FMAX: in visit()
2071 case ISD::VECREDUCE_FMIN: in visit()
2072 case ISD::VECREDUCE_FMAXIMUM: in visit()
2073 case ISD::VECREDUCE_FMINIMUM: return visitVECREDUCE(N); in visit()
2074 #define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) case ISD::SDOPC: in visit()
2092 assert(N->getOpcode() != ISD::DELETED_NODE && in combine()
2095 if (N->getOpcode() >= ISD::BUILTIN_OP_END || in combine()
2096 TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) { in combine()
2110 case ISD::ADD: in combine()
2111 case ISD::SUB: in combine()
2112 case ISD::MUL: in combine()
2113 case ISD::AND: in combine()
2114 case ISD::OR: in combine()
2115 case ISD::XOR: in combine()
2118 case ISD::SHL: in combine()
2119 case ISD::SRA: in combine()
2120 case ISD::SRL: in combine()
2123 case ISD::SIGN_EXTEND: in combine()
2124 case ISD::ZERO_EXTEND: in combine()
2125 case ISD::ANY_EXTEND: in combine()
2128 case ISD::LOAD: in combine()
2190 if (N->hasOneUse() && N->use_begin()->getOpcode() == ISD::TokenFactor) in visitTokenFactor()
2220 case ISD::EntryToken: in visitTokenFactor()
2226 case ISD::TokenFactor: in visitTokenFactor()
2306 case ISD::EntryToken: in visitTokenFactor()
2313 case ISD::TokenFactor: in visitTokenFactor()
2317 case ISD::LIFETIME_START: in visitTokenFactor()
2318 case ISD::LIFETIME_END: in visitTokenFactor()
2319 case ISD::CopyFromReg: in visitTokenFactor()
2320 case ISD::CopyToReg: in visitTokenFactor()
2390 if (N->getOpcode() == ISD::TRUNCATE) { in isTruncateOf()
2396 if (N.getOpcode() != ISD::SETCC || in isTruncateOf()
2398 cast<CondCodeSDNode>(N.getOperand(2))->get() != ISD::SETNE) in isTruncateOf()
2449 if (N->getOpcode() == ISD::ADD) { in canFoldInAddressingMode()
2458 } else if (N->getOpcode() == ISD::SUB) { in canFoldInAddressingMode()
2489 if (N1.getOpcode() != ISD::VSELECT || !N1.hasOneUse()) in foldSelectWithIdentityConstant()
2542 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) { in foldBinOpIntoSelect()
2547 if ((BinOpcode == ISD::SHL || BinOpcode == ISD::SRA || in foldBinOpIntoSelect()
2548 BinOpcode == ISD::SRL) && Sel.hasOneUse()) { in foldBinOpIntoSelect()
2558 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) in foldBinOpIntoSelect()
2577 (BinOpcode == ISD::AND || BinOpcode == ISD::OR) && in foldBinOpIntoSelect()
2592 if ((BinOpcode == ISD::AND && isNullOrNullSplat(CT)) || in foldBinOpIntoSelect()
2593 (BinOpcode == ISD::OR && isAllOnesOrAllOnesSplat(CT))) in foldBinOpIntoSelect()
2598 if ((BinOpcode == ISD::AND && isNullOrNullSplat(CF)) || in foldBinOpIntoSelect()
2599 (BinOpcode == ISD::OR && isAllOnesOrAllOnesSplat(CF))) in foldBinOpIntoSelect()
2625 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && in foldAddSubBoolOfMaskedVal()
2631 bool IsAdd = N->getOpcode() == ISD::ADD; in foldAddSubBoolOfMaskedVal()
2635 if (!CN || Z.getOpcode() != ISD::ZERO_EXTEND) in foldAddSubBoolOfMaskedVal()
2639 if (Z.getOperand(0).getOpcode() != ISD::SETCC || in foldAddSubBoolOfMaskedVal()
2645 ISD::CondCode CC = cast<CondCodeSDNode>(SetCC->getOperand(2))->get(); in foldAddSubBoolOfMaskedVal()
2646 if (CC != ISD::SETEQ || !isNullConstant(SetCC.getOperand(1)) || in foldAddSubBoolOfMaskedVal()
2647 SetCC.getOperand(0).getOpcode() != ISD::AND || in foldAddSubBoolOfMaskedVal()
2660 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit); in foldAddSubBoolOfMaskedVal()
2666 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && in foldAddSubOfSignBit()
2671 bool IsAdd = N->getOpcode() == ISD::ADD; in foldAddSubOfSignBit()
2675 ShiftOp.getOpcode() != ISD::SRL) in foldAddSubOfSignBit()
2695 IsAdd ? ISD::ADD : ISD::SUB, DL, VT, in foldAddSubOfSignBit()
2697 SDValue NewShift = DAG.getNode(IsAdd ? ISD::SRA : ISD::SRL, DL, VT, in foldAddSubOfSignBit()
2699 return DAG.getNode(ISD::ADD, DL, VT, NewShift, NewC); in foldAddSubOfSignBit()
2727 if (SDValue C = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N0, N1})) in visitADDLike()
2733 return DAG.getNode(ISD::ADD, DL, VT, N1, N0); in visitADDLike()
2745 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitADDLike()
2753 if (N0.getOpcode() == ISD::SUB) { in visitADDLike()
2758 if (SDValue Sub = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N1, N01})) in visitADDLike()
2759 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Sub); in visitADDLike()
2762 if (SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N1, N00})) in visitADDLike()
2763 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1)); in visitADDLike()
2770 if (N0.getOpcode() == ISD::SIGN_EXTEND && N0.hasOneUse() && in visitADDLike()
2774 (TLI.isOperationLegal(ISD::XOR, X.getValueType()) && in visitADDLike()
2775 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) && in visitADDLike()
2778 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not); in visitADDLike()
2788 if (SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N1, N01})) in visitADDLike()
2789 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add); in visitADDLike()
2796 if (!reassociationCanBreakAddressingModePattern(ISD::ADD, DL, N, N0, N1)) { in visitADDLike()
2797 if (SDValue RADD = reassociateOps(ISD::ADD, DL, N0, N1, N->getFlags())) in visitADDLike()
2817 ISD::ADD, DL, VT, in visitADDLike()
2818 DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)), in visitADDLike()
2830 reassociateReduction(ISD::VECREDUCE_ADD, ISD::ADD, DL, VT, N0, N1)) in visitADDLike()
2834 if (N0.getOpcode() == ISD::SUB && isNullOrNullSplat(N0.getOperand(0))) in visitADDLike()
2835 return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1)); in visitADDLike()
2838 if (N1.getOpcode() == ISD::SUB && isNullOrNullSplat(N1.getOperand(0))) in visitADDLike()
2839 return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1)); in visitADDLike()
2842 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1)) in visitADDLike()
2846 if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1)) in visitADDLike()
2850 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB && in visitADDLike()
2852 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), in visitADDLike()
2856 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB && in visitADDLike()
2858 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), in visitADDLike()
2862 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD && in visitADDLike()
2864 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), in visitADDLike()
2868 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD && in visitADDLike()
2870 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), in visitADDLike()
2874 if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) && in visitADDLike()
2875 N1.getOperand(0).getOpcode() == ISD::SUB && in visitADDLike()
2881 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB && in visitADDLike()
2889 return DAG.getNode(ISD::SUB, DL, VT, in visitADDLike()
2890 DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10), in visitADDLike()
2891 DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11)); in visitADDLike()
2895 if (N0.getOpcode() == ISD::UMAX && hasOperation(ISD::USUBSAT, VT)) { in visitADDLike()
2900 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchUSUBSAT, in visitADDLike()
2902 return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0), in visitADDLike()
2912 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in visitADDLike()
2916 if (N0.getOpcode() == ISD::ADD) { in visitADDLike()
2928 return DAG.getNode(ISD::SUB, DL, VT, A, Xor.getOperand(0)); in visitADDLike()
2935 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD && in visitADDLike()
2940 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0), in visitADDLike()
2942 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(1), Not); in visitADDLike()
2947 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() && in visitADDLike()
2949 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), N1); in visitADDLike()
2950 return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0)); in visitADDLike()
2978 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) && in visitADD()
2980 return DAG.getNode(ISD::OR, DL, VT, N0, N1); in visitADD()
2983 if (N0.getOpcode() == ISD::VSCALE && N1.getOpcode() == ISD::VSCALE) { in visitADD()
2990 if (N0.getOpcode() == ISD::ADD && in visitADD()
2991 N0.getOperand(1).getOpcode() == ISD::VSCALE && in visitADD()
2992 N1.getOpcode() == ISD::VSCALE) { in visitADD()
2996 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), VS); in visitADD()
3000 if (N0.getOpcode() == ISD::STEP_VECTOR && in visitADD()
3001 N1.getOpcode() == ISD::STEP_VECTOR) { in visitADD()
3009 if (N0.getOpcode() == ISD::ADD && in visitADD()
3010 N0.getOperand(1).getOpcode() == ISD::STEP_VECTOR && in visitADD()
3011 N1.getOpcode() == ISD::STEP_VECTOR) { in visitADD()
3016 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), SV); in visitADD()
3027 bool IsSigned = Opcode == ISD::SADDSAT; in visitADDSAT()
3049 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitADDSAT()
3059 return DAG.getNode(ISD::ADD, DL, VT, N0, N1); in visitADDSAT()
3070 if (V.getOpcode() == ISD::TRUNCATE || V.getOpcode() == ISD::ZERO_EXTEND) { in getAsCarry()
3075 if (V.getOpcode() == ISD::AND && isOneConstant(V.getOperand(1))) { in getAsCarry()
3094 if (V.getOpcode() != ISD::UADDO_CARRY && V.getOpcode() != ISD::USUBO_CARRY && in getAsCarry()
3095 V.getOpcode() != ISD::UADDO && V.getOpcode() != ISD::USUBO) in getAsCarry()
3118 if (N1.getOpcode() == ISD::ZERO_EXTEND) in foldAddSubMasked1()
3121 if (N1.getOpcode() != ISD::AND || !isOneOrOneSplat(N1->getOperand(1))) in foldAddSubMasked1()
3126 if (N10.getValueType() != VT && N10.getOpcode() == ISD::TRUNCATE) in foldAddSubMasked1()
3137 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, N0, N10); in foldAddSubMasked1()
3147 if (N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::SUB && in visitADDLikeCommutative()
3149 return DAG.getNode(ISD::SUB, DL, VT, N0, in visitADDLikeCommutative()
3150 DAG.getNode(ISD::SHL, DL, VT, in visitADDLikeCommutative()
3161 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD && in visitADDLikeCommutative()
3166 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0), in visitADDLikeCommutative()
3168 return DAG.getNode(ISD::SUB, DL, VT, N1, Not); in visitADDLikeCommutative()
3171 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse()) { in visitADDLikeCommutative()
3176 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), N1); in visitADDLikeCommutative()
3177 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1)); in visitADDLikeCommutative()
3182 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1)); in visitADDLikeCommutative()
3183 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(0)); in visitADDLikeCommutative()
3188 if (N0.getOpcode() == ISD::MUL && N0.getOperand(0) == N1 && in visitADDLikeCommutative()
3191 SDValue NewC = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), in visitADDLikeCommutative()
3193 return DAG.getNode(ISD::MUL, DL, VT, N0.getOperand(0), NewC); in visitADDLikeCommutative()
3199 if (N0.getOpcode() == ISD::SIGN_EXTEND && in visitADDLikeCommutative()
3202 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitADDLikeCommutative()
3203 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt); in visitADDLikeCommutative()
3207 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) { in visitADDLikeCommutative()
3210 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0), in visitADDLikeCommutative()
3212 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt); in visitADDLikeCommutative()
3217 if (N1.getOpcode() == ISD::UADDO_CARRY && isNullConstant(N1.getOperand(1)) && in visitADDLikeCommutative()
3219 return DAG.getNode(ISD::UADDO_CARRY, DL, N1->getVTList(), in visitADDLikeCommutative()
3223 if (TLI.isOperationLegalOrCustom(ISD::UADDO_CARRY, VT)) in visitADDLikeCommutative()
3225 return DAG.getNode(ISD::UADDO_CARRY, DL, in visitADDLikeCommutative()
3240 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDC()
3241 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue)); in visitADDC()
3247 return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0); in visitADDC()
3251 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, in visitADDC()
3256 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDC()
3257 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue)); in visitADDC()
3275 if (V.getOpcode() != ISD::XOR) in extractBooleanFlip()
3308 bool IsSigned = (ISD::SADDO == N->getOpcode()); in visitADDO()
3315 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDO()
3329 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDO()
3335 return DAG.getNode(ISD::SSUBO, DL, N->getVTList(), in visitADDO()
3340 SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(), in visitADDO()
3363 if (N1.getOpcode() == ISD::UADDO_CARRY && isNullConstant(N1.getOperand(1))) { in visitUADDOLike()
3367 return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(), N0, Y, in visitUADDOLike()
3372 if (TLI.isOperationLegalOrCustom(ISD::UADDO_CARRY, VT)) in visitUADDOLike()
3374 return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(), N0, in visitUADDOLike()
3389 return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(), in visitADDE()
3393 if (CarryIn.getOpcode() == ISD::CARRY_FALSE) in visitADDE()
3394 return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1); in visitADDE()
3409 return DAG.getNode(ISD::UADDO_CARRY, DL, N->getVTList(), N1, N0, CarryIn); in visitUADDO_CARRY()
3414 TLI.isOperationLegalOrCustom(ISD::UADDO, N->getValueType(0))) in visitUADDO_CARRY()
3415 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1); in visitUADDO_CARRY()
3424 return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt, in visitUADDO_CARRY()
3442 DAG.getNodeIfExists(ISD::UADDO_CARRY, N->getVTList(), Ops, N->getFlags()); in visitUADDO_CARRY()
3477 if (Carry1.getOpcode() != ISD::UADDO) in combineUADDO_CARRYDiamond()
3486 if (Carry0.getOpcode() == ISD::UADDO_CARRY && in combineUADDO_CARRYDiamond()
3489 } else if (Carry0.getOpcode() == ISD::UADDO && in combineUADDO_CARRYDiamond()
3502 DAG.getNode(ISD::UADDO_CARRY, DL, Carry0->getVTList(), A, B, Z); in combineUADDO_CARRYDiamond()
3504 return DAG.getNode(ISD::UADDO_CARRY, DL, N->getVTList(), X, in combineUADDO_CARRYDiamond()
3576 if (Opcode != ISD::UADDO && Opcode != ISD::USUBO) in combineCarryDiamond()
3597 if (Opcode == ISD::USUBO && CarryInOperandNum != 1) in combineCarryDiamond()
3601 unsigned NewOp = Opcode == ISD::UADDO ? ISD::UADDO_CARRY : ISD::USUBO_CARRY; in combineCarryDiamond()
3629 if (N->getOpcode() == ISD::AND) in combineCarryDiamond()
3641 SDValue Sub = DAG.getNode(ISD::USUBO_CARRY, DL, N->getVTList(), N1, in visitUADDO_CARRYLike()
3651 if ((N0.getOpcode() == ISD::ADD || in visitUADDO_CARRYLike()
3652 (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0 && in visitUADDO_CARRYLike()
3655 return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(), in visitUADDO_CARRYLike()
3679 return DAG.getNode(ISD::SSUBO_CARRY, SDLoc(N), N->getVTList(), N1, in visitSADDO_CARRYLike()
3696 return DAG.getNode(ISD::SADDO_CARRY, DL, N->getVTList(), N1, N0, CarryIn); in visitSADDO_CARRY()
3701 TLI.isOperationLegalOrCustom(ISD::SADDO, N->getValueType(0))) in visitSADDO_CARRY()
3702 return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0, N1); in visitSADDO_CARRY()
3723 return DAG.getNode(ISD::USUBSAT, DL, DstVT, LHS, RHS); in getTruncatedUSUBSAT()
3736 RHS = DAG.getNode(ISD::UMIN, DL, SrcVT, RHS, SatLimit); in getTruncatedUSUBSAT()
3737 RHS = DAG.getNode(ISD::TRUNCATE, DL, DstVT, RHS); in getTruncatedUSUBSAT()
3738 LHS = DAG.getNode(ISD::TRUNCATE, DL, DstVT, LHS); in getTruncatedUSUBSAT()
3739 return DAG.getNode(ISD::USUBSAT, DL, DstVT, LHS, RHS); in getTruncatedUSUBSAT()
3745 if (N->getOpcode() != ISD::SUB || in foldSubToUSubSat()
3746 !(!LegalOperations || hasOperation(ISD::USUBSAT, DstVT))) in foldSubToUSubSat()
3755 if (Op0.getOpcode() == ISD::UMAX && Op0.hasOneUse()) { in foldSubToUSubSat()
3764 if (Op1.getOpcode() == ISD::UMIN && Op1.hasOneUse()) { in foldSubToUSubSat()
3774 if (Op1.getOpcode() == ISD::TRUNCATE && in foldSubToUSubSat()
3775 Op1.getOperand(0).getOpcode() == ISD::UMIN && in foldSubToUSubSat()
3779 if (MinLHS.getOpcode() == ISD::ZERO_EXTEND && MinLHS.getOperand(0) == Op0) in foldSubToUSubSat()
3782 if (MinRHS.getOpcode() == ISD::ZERO_EXTEND && MinRHS.getOperand(0) == Op0) in foldSubToUSubSat()
3796 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) in tryFoldToZero()
3808 if (N->getOpcode() == ISD::FREEZE && N.hasOneUse()) in visitSUB()
3819 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N1})) in visitSUB()
3828 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitSUB()
3839 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
3849 if (N1->getOpcode() == ISD::SRA || N1->getOpcode() == ISD::SRL) { in visitSUB()
3852 auto NewSh = N1->getOpcode() == ISD::SRA ? ISD::SRL : ISD::SRA; in visitSUB()
3873 if (N1.getOpcode() == ISD::ABS && N1.hasOneUse() && in visitSUB()
3874 !TLI.isOperationLegalOrCustom(ISD::ABS, VT)) in visitSUB()
3881 if (N1S && N1S.getOpcode() == ISD::SUB && in visitSUB()
3889 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitSUB()
3892 if (N1.getOpcode() == ISD::SUB && isNullOrNullSplat(N1.getOperand(0))) in visitSUB()
3893 return DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(1)); in visitSUB()
3896 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0)) in visitSUB()
3900 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1) in visitSUB()
3904 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1) in visitSUB()
3908 if (N0.getOpcode() == ISD::ADD) { in visitSUB()
3910 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N01, N1})) in visitSUB()
3911 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), NewC); in visitSUB()
3915 if (N1.getOpcode() == ISD::ADD) { in visitSUB()
3917 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N11})) in visitSUB()
3918 return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0)); in visitSUB()
3922 if (N0.getOpcode() == ISD::SUB) { in visitSUB()
3924 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N01, N1})) in visitSUB()
3925 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), NewC); in visitSUB()
3929 if (N0.getOpcode() == ISD::SUB) { in visitSUB()
3931 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N00, N1})) in visitSUB()
3932 return DAG.getNode(ISD::SUB, DL, VT, NewC, N0.getOperand(1)); in visitSUB()
3936 if (N0.getOpcode() == ISD::ADD && in visitSUB()
3937 (N0.getOperand(1).getOpcode() == ISD::SUB || in visitSUB()
3938 N0.getOperand(1).getOpcode() == ISD::ADD) && in visitSUB()
3944 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1).getOpcode() == ISD::ADD && in visitSUB()
3946 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), in visitSUB()
3950 if (N0.getOpcode() == ISD::SUB && N0.getOperand(1).getOpcode() == ISD::SUB && in visitSUB()
3952 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), in visitSUB()
3956 if (N1.getOpcode() == ISD::SUB && N1.hasOneUse()) in visitSUB()
3957 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
3958 DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(1), in visitSUB()
3962 if (N1.getOpcode() == ISD::AND) { in visitSUB()
3970 DAG.getNode(ISD::XOR, DL, VT, B, DAG.getAllOnesConstant(DL, VT)); in visitSUB()
3971 return DAG.getNode(ISD::AND, DL, VT, A, InvB); in visitSUB()
3976 if (N1.getOpcode() == ISD::MUL && N1.hasOneUse()) { in visitSUB()
3977 if (N1.getOperand(0).getOpcode() == ISD::SUB && in visitSUB()
3979 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, in visitSUB()
3982 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul); in visitSUB()
3984 if (N1.getOperand(1).getOpcode() == ISD::SUB && in visitSUB()
3986 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, in visitSUB()
3989 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul); in visitSUB()
4012 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() && isOneOrOneSplat(N1)) { in visitSUB()
4013 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), in visitSUB()
4015 return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0)); in visitSUB()
4023 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(0)); in visitSUB()
4024 return DAG.getNode(ISD::ADD, DL, VT, Add, DAG.getConstant(1, DL, VT)); in visitSUB()
4029 if (N0.getOpcode() == ISD::ADD && N0.hasOneUse() && in visitSUB()
4031 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1); in visitSUB()
4032 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(1)); in visitSUB()
4035 if (N1.getOpcode() == ISD::ADD && N1.hasOneUse() && in visitSUB()
4037 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(0)); in visitSUB()
4038 return DAG.getNode(ISD::SUB, DL, VT, Sub, N1.getOperand(1)); in visitSUB()
4042 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() && in visitSUB()
4044 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1); in visitSUB()
4045 return DAG.getNode(ISD::SUB, DL, VT, Sub, N0.getOperand(1)); in visitSUB()
4048 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() && in visitSUB()
4050 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), N1); in visitSUB()
4051 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), Add); in visitSUB()
4057 if (N1.getOpcode() == ISD::ZERO_EXTEND && in visitSUB()
4061 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N1.getOperand(0)); in visitSUB()
4062 return DAG.getNode(ISD::ADD, DL, VT, N0, SExt); in visitSUB()
4066 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) { in visitSUB()
4067 if (N0.getOpcode() == ISD::XOR && N1.getOpcode() == ISD::SRA) { in visitSUB()
4073 return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0); in visitSUB()
4088 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) { in visitSUB()
4091 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0), in visitSUB()
4093 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt); in visitSUB()
4098 if (N1.getOpcode() == ISD::VSCALE && N1.hasOneUse()) { in visitSUB()
4100 return DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getVScale(DL, VT, -IntVal)); in visitSUB()
4104 if (N1.getOpcode() == ISD::STEP_VECTOR && N1.hasOneUse()) { in visitSUB()
4106 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
4112 if (!LegalOperations && N1.getOpcode() == ISD::SRL && N1.hasOneUse()) { in visitSUB()
4117 SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt); in visitSUB()
4118 return DAG.getNode(ISD::ADD, DL, VT, N0, SRA); in visitSUB()
4125 if (N1.getOpcode() == ISD::SHL) { in visitSUB()
4128 return DAG.getNode(ISD::ADD, DL, VT, N1, N0); in visitSUB()
4132 if (N0.getOpcode() == ISD::USUBO_CARRY && isNullConstant(N0.getOperand(1)) && in visitSUB()
4134 return DAG.getNode(ISD::USUBO_CARRY, DL, N0->getVTList(), in visitSUB()
4137 if (TLI.isOperationLegalOrCustom(ISD::UADDO_CARRY, VT)) { in visitSUB()
4142 SDValue NegX = DAG.getNode(ISD::SUB, DL, VT, Zero, X); in visitSUB()
4143 return DAG.getNode(ISD::UADDO_CARRY, DL, in visitSUB()
4156 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitSUB()
4173 if (SDValue R = MatchSubMaxMin(ISD::SMAX, ISD::SMIN, ISD::ABDS)) in visitSUB()
4175 if (SDValue R = MatchSubMaxMin(ISD::UMAX, ISD::UMIN, ISD::ABDU)) in visitSUB()
4186 bool IsSigned = Opcode == ISD::SSUBSAT; in visitSUBSAT()
4207 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitSUBSAT()
4217 return DAG.getNode(ISD::SUB, DL, VT, N0, N1); in visitSUBSAT()
4230 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitSUBC()
4231 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue)); in visitSUBC()
4236 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue)); in visitSUBC()
4240 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue)); in visitSUBC()
4244 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0), in visitSUBC()
4245 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue)); in visitSUBC()
4254 bool IsSigned = (ISD::SSUBO == N->getOpcode()); in visitSUBO()
4261 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitSUBO()
4273 return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0, in visitSUBO()
4283 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitSUBO()
4288 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0), in visitSUBO()
4300 if (CarryIn.getOpcode() == ISD::CARRY_FALSE) in visitSUBE()
4301 return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1); in visitSUBE()
4314 TLI.isOperationLegalOrCustom(ISD::USUBO, N->getValueType(0))) in visitUSUBO_CARRY()
4315 return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1); in visitUSUBO_CARRY()
4329 TLI.isOperationLegalOrCustom(ISD::SSUBO, N->getValueType(0))) in visitSSUBO_CARRY()
4330 return DAG.getNode(ISD::SSUBO, SDLoc(N), N->getVTList(), N0, N1); in visitSSUBO_CARRY()
4371 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MUL, DL, VT, {N0, N1})) in visitMUL()
4377 return DAG.getNode(ISD::MUL, DL, VT, N1, N0); in visitMUL()
4388 N1IsConst = ISD::isConstantSplatVector(N1.getNode(), ConstValue1); in visitMUL()
4421 return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc); in visitMUL()
4432 return DAG.getNode(ISD::SUB, DL, VT, in visitMUL()
4434 DAG.getNode(ISD::SHL, DL, VT, N0, in visitMUL()
4440 for (unsigned LoHiOpc : {ISD::UMUL_LOHI, ISD::SMUL_LOHI}) { in visitMUL()
4472 unsigned MathOp = ISD::DELETED_NODE; in visitMUL()
4478 MathOp = ISD::ADD; in visitMUL()
4480 MathOp = ISD::SUB; in visitMUL()
4482 if (MathOp != ISD::DELETED_NODE) { in visitMUL()
4484 MathOp == ISD::ADD ? (MulC - 1).logBase2() : (MulC + 1).logBase2(); in visitMUL()
4489 DAG.getNode(ISD::SHL, DL, VT, N0, DAG.getConstant(ShAmt, DL, VT)); in visitMUL()
4492 DAG.getNode(ISD::SHL, DL, VT, N0, in visitMUL()
4502 if (N0.getOpcode() == ISD::SHL) { in visitMUL()
4504 if (SDValue C3 = DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {N1, N01})) in visitMUL()
4505 return DAG.getNode(ISD::MUL, DL, VT, N0.getOperand(0), C3); in visitMUL()
4514 if (N0.getOpcode() == ISD::SHL && in visitMUL()
4517 } else if (N1.getOpcode() == ISD::SHL && in visitMUL()
4524 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, Sh.getOperand(0), Y); in visitMUL()
4525 return DAG.getNode(ISD::SHL, DL, VT, Mul, Sh.getOperand(1)); in visitMUL()
4530 if (N0.getOpcode() == ISD::ADD && in visitMUL()
4535 ISD::ADD, DL, VT, in visitMUL()
4536 DAG.getNode(ISD::MUL, SDLoc(N0), VT, N0.getOperand(0), N1), in visitMUL()
4537 DAG.getNode(ISD::MUL, SDLoc(N1), VT, N0.getOperand(1), N1)); in visitMUL()
4541 if (N0.getOpcode() == ISD::VSCALE && NC1) { in visitMUL()
4549 if (N0.getOpcode() == ISD::STEP_VECTOR && in visitMUL()
4550 ISD::isConstantSplatVector(N1.getNode(), MulVal)) { in visitMUL()
4572 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::AND, VT)) && in visitMUL()
4573 ISD::matchUnaryPredicate(N1, IsClearMask, /*AllowUndefs*/ true)) { in visitMUL()
4574 assert(N1.getOpcode() == ISD::BUILD_VECTOR && "Unknown constant vector"); in visitMUL()
4582 return DAG.getNode(ISD::AND, DL, VT, N0, DAG.getBuildVector(VT, DL, Mask)); in visitMUL()
4587 if (SDValue RMUL = reassociateOps(ISD::MUL, DL, N0, N1, N->getFlags())) in visitMUL()
4592 reassociateReduction(ISD::VECREDUCE_MUL, ISD::MUL, DL, VT, N0, N1)) in visitMUL()
4627 bool isSigned = (Opcode == ISD::SDIV) || (Opcode == ISD::SREM); in useDivRem()
4628 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; in useDivRem()
4646 if ((Opcode == ISD::SDIV) || (Opcode == ISD::UDIV)) { in useDivRem()
4647 OtherOpcode = isSigned ? ISD::SREM : ISD::UREM; in useDivRem()
4651 OtherOpcode = isSigned ? ISD::SDIV : ISD::UDIV; in useDivRem()
4660 if (User == Node || User->getOpcode() == ISD::DELETED_NODE || in useDivRem()
4681 if (UserOpc == ISD::SDIV || UserOpc == ISD::UDIV) in useDivRem()
4683 else if (UserOpc == ISD::SREM || UserOpc == ISD::UREM) in useDivRem()
4697 bool IsDiv = (ISD::SDIV == Opc) || (ISD::UDIV == Opc); in simplifyDivRem()
4744 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, {N0, N1})) in visitSDIV()
4759 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ), in visitSDIV()
4772 return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1); in visitSDIV()
4777 if (SDNode *RemNode = DAG.getNodeIfExists(ISD::SREM, N->getVTList(), in visitSDIV()
4779 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1); in visitSDIV()
4780 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitSDIV()
4812 return ISD::matchUnaryPredicate(Divisor, IsPowerOfTwo); in isDivisorPowerOfTwo()
4833 SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1); in visitSDIVLike()
4835 SDValue Inexact = DAG.getNode(ISD::SUB, DL, ShiftAmtTy, Bits, C1); in visitSDIVLike()
4840 SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0, in visitSDIVLike()
4845 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact); in visitSDIVLike()
4847 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl); in visitSDIVLike()
4849 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1); in visitSDIVLike()
4856 SDValue IsOne = DAG.getSetCC(DL, CCVT, N1, One, ISD::SETEQ); in visitSDIVLike()
4857 SDValue IsAllOnes = DAG.getSetCC(DL, CCVT, N1, AllOnes, ISD::SETEQ); in visitSDIVLike()
4858 SDValue IsOneOrAllOnes = DAG.getNode(ISD::OR, DL, CCVT, IsOne, IsAllOnes); in visitSDIVLike()
4864 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, Zero, Sra); in visitSDIVLike()
4867 SDValue IsNeg = DAG.getSetCC(DL, CCVT, N1, Zero, ISD::SETLT); in visitSDIVLike()
4892 if (SDValue C = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT, {N0, N1})) in visitUDIV()
4903 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ), in visitUDIV()
4917 if (SDNode *RemNode = DAG.getNodeIfExists(ISD::UREM, N->getVTList(), in visitUDIV()
4919 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1); in visitUDIV()
4920 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitUDIV()
4951 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc); in visitUDIVLike()
4956 if (N1.getOpcode() == ISD::SHL) { in visitUDIVLike()
4965 SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc); in visitUDIVLike()
4967 return DAG.getNode(ISD::SRL, DL, VT, N0, Add); in visitUDIVLike()
4984 !DAG.doesNodeExist(ISD::SDIV, N->getVTList(), {N0, N1})) { in buildOptimizedSREM()
5000 bool isSigned = (Opcode == ISD::SREM); in visitREM()
5012 SDValue EqualsNeg1 = DAG.getSetCC(DL, CCVT, F0, N1, ISD::SETEQ); in visitREM()
5026 return DAG.getNode(ISD::UREM, DL, VT, N0, N1); in visitREM()
5031 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne); in visitREM()
5033 return DAG.getNode(ISD::AND, DL, VT, N0, Add); in visitREM()
5039 if ((N1.getOpcode() == ISD::SHL || N1.getOpcode() == ISD::SRL) && in visitREM()
5042 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne); in visitREM()
5044 return DAG.getNode(ISD::AND, DL, VT, N0, Add); in visitREM()
5068 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV; in visitREM()
5072 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1); in visitREM()
5073 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitREM()
5094 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHS, DL, VT, {N0, N1})) in visitMULHS()
5100 return DAG.getNode(ISD::MULHS, DL, N->getVTList(), N1, N0); in visitMULHS()
5108 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitMULHS()
5118 return DAG.getNode(ISD::SRA, DL, N0.getValueType(), N0, in visitMULHS()
5128 if (!TLI.isOperationLegalOrCustom(ISD::MULHS, VT) && VT.isSimple() && in visitMULHS()
5133 if (TLI.isOperationLegal(ISD::MUL, NewVT)) { in visitMULHS()
5134 N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0); in visitMULHS()
5135 N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1); in visitMULHS()
5136 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1); in visitMULHS()
5137 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1, in visitMULHS()
5140 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in visitMULHS()
5154 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHU, DL, VT, {N0, N1})) in visitMULHU()
5160 return DAG.getNode(ISD::MULHU, DL, N->getVTList(), N1, N0); in visitMULHU()
5168 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitMULHU()
5186 hasOperation(ISD::SRL, VT)) { in visitMULHU()
5190 ISD::SUB, DL, VT, DAG.getConstant(NumEltBits, DL, VT), LogBase2); in visitMULHU()
5193 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc); in visitMULHU()
5199 if (!TLI.isOperationLegalOrCustom(ISD::MULHU, VT) && VT.isSimple() && in visitMULHU()
5204 if (TLI.isOperationLegal(ISD::MUL, NewVT)) { in visitMULHU()
5205 N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0); in visitMULHU()
5206 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1); in visitMULHU()
5207 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1); in visitMULHU()
5208 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1, in visitMULHU()
5211 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in visitMULHU()
5245 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) { in visitAVG()
5246 if (Opcode == ISD::AVGFLOORS) in visitAVG()
5247 return DAG.getNode(ISD::SRA, DL, VT, N0, DAG.getConstant(1, DL, VT)); in visitAVG()
5248 if (Opcode == ISD::AVGFLOORU) in visitAVG()
5249 return DAG.getNode(ISD::SRL, DL, VT, N0, DAG.getConstant(1, DL, VT)); in visitAVG()
5290 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) { in visitABD()
5291 if (Opcode == ISD::ABDS) in visitABD()
5292 return DAG.getNode(ISD::ABS, DL, VT, N0); in visitABD()
5293 if (Opcode == ISD::ABDU) in visitABD()
5303 if (Opcode == ISD::ABDS && hasOperation(ISD::ABDU, VT) && in visitABD()
5305 return DAG.getNode(ISD::ABDU, DL, VT, N1, N0); in visitABD()
5360 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS)) in visitSMUL_LOHI()
5370 return DAG.getNode(ISD::SMUL_LOHI, DL, N->getVTList(), N0, N1); in visitSMUL_LOHI()
5375 return DAG.getNode(ISD::SMUL_LOHI, DL, N->getVTList(), N1, N0); in visitSMUL_LOHI()
5383 if (TLI.isOperationLegal(ISD::MUL, NewVT)) { in visitSMUL_LOHI()
5384 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0); in visitSMUL_LOHI()
5385 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1); in visitSMUL_LOHI()
5386 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi); in visitSMUL_LOHI()
5388 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo, in visitSMUL_LOHI()
5391 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi); in visitSMUL_LOHI()
5393 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo); in visitSMUL_LOHI()
5402 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU)) in visitUMUL_LOHI()
5412 return DAG.getNode(ISD::UMUL_LOHI, DL, N->getVTList(), N0, N1); in visitUMUL_LOHI()
5417 return DAG.getNode(ISD::UMUL_LOHI, DL, N->getVTList(), N1, N0); in visitUMUL_LOHI()
5437 if (TLI.isOperationLegal(ISD::MUL, NewVT)) { in visitUMUL_LOHI()
5438 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0); in visitUMUL_LOHI()
5439 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1); in visitUMUL_LOHI()
5440 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi); in visitUMUL_LOHI()
5442 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo, in visitUMUL_LOHI()
5445 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi); in visitUMUL_LOHI()
5447 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo); in visitUMUL_LOHI()
5459 bool IsSigned = (ISD::SMULO == N->getOpcode()); in visitMULO()
5493 return DAG.getNode(IsSigned ? ISD::SADDO : ISD::UADDO, DL, in visitMULO()
5498 SDValue And = DAG.getNode(ISD::AND, DL, VT, N0, N1); in visitMULO()
5500 DAG.getConstant(0, DL, VT), ISD::SETNE); in visitMULO()
5506 return CombineTo(N, DAG.getNode(ISD::MUL, DL, VT, N0, N1), in visitMULO()
5518 SDValue N3, ISD::CondCode CC, unsigned &BW, in isSaturatingMinMax()
5521 ISD::CondCode CC) { in isSaturatingMinMax()
5524 if (N0 != N2 && (N2.getOpcode() != ISD::TRUNCATE || N0 != N2.getOperand(0))) in isSaturatingMinMax()
5535 return CC == ISD::SETLT ? ISD::SMIN : (CC == ISD::SETGT ? ISD::SMAX : 0); in isSaturatingMinMax()
5545 if (N0.getOpcode() == ISD::FP_TO_SINT && Opcode0 == ISD::SMAX) { in isSaturatingMinMax()
5564 ISD::CondCode N0CC; in isSaturatingMinMax()
5566 case ISD::SMIN: in isSaturatingMinMax()
5567 case ISD::SMAX: in isSaturatingMinMax()
5570 N0CC = N0.getOpcode() == ISD::SMIN ? ISD::SETLT : ISD::SETGT; in isSaturatingMinMax()
5572 case ISD::SELECT_CC: in isSaturatingMinMax()
5579 case ISD::SELECT: in isSaturatingMinMax()
5580 case ISD::VSELECT: in isSaturatingMinMax()
5581 if (N0.getOperand(0).getOpcode() != ISD::SETCC) in isSaturatingMinMax()
5597 ConstantSDNode *MinCOp = isConstOrConstSplat(Opcode0 == ISD::SMIN ? N1 : N01); in isSaturatingMinMax()
5598 ConstantSDNode *MaxCOp = isConstOrConstSplat(Opcode0 == ISD::SMIN ? N01 : N1); in isSaturatingMinMax()
5621 SDValue N3, ISD::CondCode CC, in PerformMinMaxFpToSatCombine()
5626 if (!Fp || Fp.getOpcode() != ISD::FP_TO_SINT) in PerformMinMaxFpToSatCombine()
5633 unsigned NewOpc = Unsigned ? ISD::FP_TO_UINT_SAT : ISD::FP_TO_SINT_SAT; in PerformMinMaxFpToSatCombine()
5643 SDValue N3, ISD::CondCode CC, in PerformUMinFpToSatCombine()
5649 (N2.getOpcode() != ISD::TRUNCATE || N0 != N2.getOperand(0))) || in PerformUMinFpToSatCombine()
5650 N0.getOpcode() != ISD::FP_TO_UINT || CC != ISD::SETULT) in PerformUMinFpToSatCombine()
5668 if (!DAG.getTargetLoweringInfo().shouldConvertFpToSat(ISD::FP_TO_UINT_SAT, in PerformUMinFpToSatCombine()
5673 DAG.getNode(ISD::FP_TO_UINT_SAT, SDLoc(N0), NewVT, N0.getOperand(0), in PerformUMinFpToSatCombine()
5710 case ISD::SMIN: AltOpcode = ISD::UMIN; break; in visitIMINMAX()
5711 case ISD::SMAX: AltOpcode = ISD::UMAX; break; in visitIMINMAX()
5712 case ISD::UMIN: AltOpcode = ISD::SMIN; break; in visitIMINMAX()
5713 case ISD::UMAX: AltOpcode = ISD::SMAX; break; in visitIMINMAX()
5720 if (Opcode == ISD::SMIN || Opcode == ISD::SMAX) in visitIMINMAX()
5722 N0, N1, N0, N1, Opcode == ISD::SMIN ? ISD::SETLT : ISD::SETGT, DAG)) in visitIMINMAX()
5724 if (Opcode == ISD::UMIN) in visitIMINMAX()
5725 if (SDValue S = PerformUMinFpToSatCombine(N0, N1, N0, N1, ISD::SETULT, DAG)) in visitIMINMAX()
5731 case ISD::SMIN: in visitIMINMAX()
5732 return ISD::VECREDUCE_SMIN; in visitIMINMAX()
5733 case ISD::SMAX: in visitIMINMAX()
5734 return ISD::VECREDUCE_SMAX; in visitIMINMAX()
5735 case ISD::UMIN: in visitIMINMAX()
5736 return ISD::VECREDUCE_UMIN; in visitIMINMAX()
5737 case ISD::UMAX: in visitIMINMAX()
5738 return ISD::VECREDUCE_UMAX; in visitIMINMAX()
5761 assert(ISD::isBitwiseLogicOp(LogicOpcode) && "Expected logic opcode"); in hoistLogicOpWithSameOpcodeHands()
5776 if (ISD::isExtOpcode(HandOpcode) || ISD::isExtVecInRegOpcode(HandOpcode) || in hoistLogicOpWithSameOpcodeHands()
5777 (HandOpcode == ISD::SIGN_EXTEND_INREG && in hoistLogicOpWithSameOpcodeHands()
5793 if ((HandOpcode == ISD::ANY_EXTEND || in hoistLogicOpWithSameOpcodeHands()
5794 HandOpcode == ISD::ANY_EXTEND_VECTOR_INREG) && in hoistLogicOpWithSameOpcodeHands()
5799 if (HandOpcode == ISD::SIGN_EXTEND_INREG) in hoistLogicOpWithSameOpcodeHands()
5805 if (HandOpcode == ISD::TRUNCATE) { in hoistLogicOpWithSameOpcodeHands()
5828 if ((HandOpcode == ISD::SHL || HandOpcode == ISD::SRL || in hoistLogicOpWithSameOpcodeHands()
5829 HandOpcode == ISD::SRA || HandOpcode == ISD::AND) && in hoistLogicOpWithSameOpcodeHands()
5839 if (HandOpcode == ISD::BSWAP) { in hoistLogicOpWithSameOpcodeHands()
5850 if ((HandOpcode == ISD::FSHL || HandOpcode == ISD::FSHR) && in hoistLogicOpWithSameOpcodeHands()
5869 if ((HandOpcode == ISD::BITCAST || HandOpcode == ISD::SCALAR_TO_VECTOR) && in hoistLogicOpWithSameOpcodeHands()
5892 if (HandOpcode == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) { in hoistLogicOpWithSameOpcodeHands()
5909 if (LogicOpcode == ISD::XOR && !ShOp.isUndef()) in hoistLogicOpWithSameOpcodeHands()
5922 if (LogicOpcode == ISD::XOR && !ShOp.isUndef()) in hoistLogicOpWithSameOpcodeHands()
5961 ISD::CondCode CC0 = cast<CondCodeSDNode>(N0CC)->get(); in foldLogicOfSetCCs()
5962 ISD::CondCode CC1 = cast<CondCodeSDNode>(N1CC)->get(); in foldLogicOfSetCCs()
5969 bool AndEqZero = IsAnd && CC1 == ISD::SETEQ && IsZero; in foldLogicOfSetCCs()
5971 bool AndGtNeg1 = IsAnd && CC1 == ISD::SETGT && IsNeg1; in foldLogicOfSetCCs()
5973 bool OrNeZero = !IsAnd && CC1 == ISD::SETNE && IsZero; in foldLogicOfSetCCs()
5975 bool OrLtZero = !IsAnd && CC1 == ISD::SETLT && IsZero; in foldLogicOfSetCCs()
5982 SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL); in foldLogicOfSetCCs()
5988 bool AndEqNeg1 = IsAnd && CC1 == ISD::SETEQ && IsNeg1; in foldLogicOfSetCCs()
5990 bool AndLtZero = IsAnd && CC1 == ISD::SETLT && IsZero; in foldLogicOfSetCCs()
5992 bool OrNeNeg1 = !IsAnd && CC1 == ISD::SETNE && IsNeg1; in foldLogicOfSetCCs()
5994 bool OrGtNeg1 = !IsAnd && CC1 == ISD::SETGT && IsNeg1; in foldLogicOfSetCCs()
6001 SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL); in foldLogicOfSetCCs()
6010 IsInteger && CC0 == ISD::SETNE && in foldLogicOfSetCCs()
6015 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One); in foldLogicOfSetCCs()
6017 return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE); in foldLogicOfSetCCs()
6026 if ((IsAnd && CC1 == ISD::SETEQ) || (!IsAnd && CC1 == ISD::SETNE)) { in foldLogicOfSetCCs()
6027 SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR); in foldLogicOfSetCCs()
6028 SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR); in foldLogicOfSetCCs()
6029 SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR); in foldLogicOfSetCCs()
6035 if ((IsAnd && CC1 == ISD::SETNE) || (!IsAnd && CC1 == ISD::SETEQ)) { in foldLogicOfSetCCs()
6045 if (LL == RL && ISD::matchBinaryPredicate(LR, RR, MatchDiffPow2)) { in foldLogicOfSetCCs()
6048 SDValue Max = DAG.getNode(ISD::UMAX, DL, OpVT, LR, RR); in foldLogicOfSetCCs()
6049 SDValue Min = DAG.getNode(ISD::UMIN, DL, OpVT, LR, RR); in foldLogicOfSetCCs()
6050 SDValue Offset = DAG.getNode(ISD::SUB, DL, OpVT, LL, Min); in foldLogicOfSetCCs()
6051 SDValue Diff = DAG.getNode(ISD::SUB, DL, OpVT, Max, Min); in foldLogicOfSetCCs()
6053 SDValue And = DAG.getNode(ISD::AND, DL, OpVT, Offset, Mask); in foldLogicOfSetCCs()
6062 CC1 = ISD::getSetCCSwappedOperands(CC1); in foldLogicOfSetCCs()
6069 ISD::CondCode NewCC = IsAnd ? ISD::getSetCCAndOperation(CC0, CC1, OpVT) in foldLogicOfSetCCs()
6070 : ISD::getSetCCOrOperation(CC0, CC1, OpVT); in foldLogicOfSetCCs()
6071 if (NewCC != ISD::SETCC_INVALID && in foldLogicOfSetCCs()
6074 TLI.isOperationLegal(ISD::SETCC, OpVT)))) in foldLogicOfSetCCs()
6092 ISD::CondCode CC, unsigned OrAndOpcode, in getMinMaxOpcodeForFP()
6103 if (((CC == ISD::SETLT || CC == ISD::SETLE) && (OrAndOpcode == ISD::OR)) || in getMinMaxOpcodeForFP()
6104 ((CC == ISD::SETGT || CC == ISD::SETGE) && (OrAndOpcode == ISD::AND))) in getMinMaxOpcodeForFP()
6107 ? ISD::FMINNUM_IEEE in getMinMaxOpcodeForFP()
6108 : ISD::DELETED_NODE; in getMinMaxOpcodeForFP()
6109 else if (((CC == ISD::SETGT || CC == ISD::SETGE) && in getMinMaxOpcodeForFP()
6110 (OrAndOpcode == ISD::OR)) || in getMinMaxOpcodeForFP()
6111 ((CC == ISD::SETLT || CC == ISD::SETLE) && in getMinMaxOpcodeForFP()
6112 (OrAndOpcode == ISD::AND))) in getMinMaxOpcodeForFP()
6115 ? ISD::FMAXNUM_IEEE in getMinMaxOpcodeForFP()
6116 : ISD::DELETED_NODE; in getMinMaxOpcodeForFP()
6126 else if (((CC == ISD::SETOLT || CC == ISD::SETOLE) && in getMinMaxOpcodeForFP()
6127 (OrAndOpcode == ISD::OR)) || in getMinMaxOpcodeForFP()
6128 ((CC == ISD::SETUGT || CC == ISD::SETUGE) && in getMinMaxOpcodeForFP()
6129 (OrAndOpcode == ISD::AND))) in getMinMaxOpcodeForFP()
6130 return isFMAXNUMFMINNUM ? ISD::FMINNUM in getMinMaxOpcodeForFP()
6133 ? ISD::FMINNUM_IEEE in getMinMaxOpcodeForFP()
6134 : ISD::DELETED_NODE; in getMinMaxOpcodeForFP()
6135 else if (((CC == ISD::SETOGT || CC == ISD::SETOGE) && in getMinMaxOpcodeForFP()
6136 (OrAndOpcode == ISD::OR)) || in getMinMaxOpcodeForFP()
6137 ((CC == ISD::SETULT || CC == ISD::SETULE) && in getMinMaxOpcodeForFP()
6138 (OrAndOpcode == ISD::AND))) in getMinMaxOpcodeForFP()
6139 return isFMAXNUMFMINNUM ? ISD::FMAXNUM in getMinMaxOpcodeForFP()
6142 ? ISD::FMAXNUM_IEEE in getMinMaxOpcodeForFP()
6143 : ISD::DELETED_NODE; in getMinMaxOpcodeForFP()
6144 return ISD::DELETED_NODE; in getMinMaxOpcodeForFP()
6150 (LogicOp->getOpcode() == ISD::AND || LogicOp->getOpcode() == ISD::OR) && in foldAndOrOfSETCC()
6156 if (LHS->getOpcode() != ISD::SETCC || RHS->getOpcode() != ISD::SETCC || in foldAndOrOfSETCC()
6172 ISD::CondCode CCL = cast<CondCodeSDNode>(LHS.getOperand(2))->get(); in foldAndOrOfSETCC()
6173 ISD::CondCode CCR = cast<CondCodeSDNode>(RHS.getOperand(2))->get(); in foldAndOrOfSETCC()
6188 bool isFMAXNUMFMINNUM_IEEE = TLI.isOperationLegal(ISD::FMAXNUM_IEEE, OpVT) && in foldAndOrOfSETCC()
6189 TLI.isOperationLegal(ISD::FMINNUM_IEEE, OpVT); in foldAndOrOfSETCC()
6190 bool isFMAXNUMFMINNUM = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, OpVT) && in foldAndOrOfSETCC()
6191 TLI.isOperationLegalOrCustom(ISD::FMINNUM, OpVT); in foldAndOrOfSETCC()
6192 if (((OpVT.isInteger() && TLI.isOperationLegal(ISD::UMAX, OpVT) && in foldAndOrOfSETCC()
6193 TLI.isOperationLegal(ISD::SMAX, OpVT) && in foldAndOrOfSETCC()
6194 TLI.isOperationLegal(ISD::UMIN, OpVT) && in foldAndOrOfSETCC()
6195 TLI.isOperationLegal(ISD::SMIN, OpVT)) || in foldAndOrOfSETCC()
6198 !ISD::isIntEqualitySetCC(CCL) && !ISD::isFPEqualitySetCC(CCL) && in foldAndOrOfSETCC()
6199 CCL != ISD::SETFALSE && CCL != ISD::SETO && CCL != ISD::SETUO && in foldAndOrOfSETCC()
6200 CCL != ISD::SETTRUE && in foldAndOrOfSETCC()
6201 (CCL == CCR || CCL == ISD::getSetCCSwappedOperands(CCR))) { in foldAndOrOfSETCC()
6204 ISD::CondCode CC = ISD::SETCC_INVALID; in foldAndOrOfSETCC()
6210 CC = ISD::getSetCCSwappedOperands(CCL); in foldAndOrOfSETCC()
6218 assert(CCL == ISD::getSetCCSwappedOperands(CCR) && "Unexpected CC"); in foldAndOrOfSETCC()
6234 if (CC == ISD::SETLT && isNullOrNullSplat(CommonValue)) in foldAndOrOfSETCC()
6235 CC = ISD::SETCC_INVALID; in foldAndOrOfSETCC()
6236 else if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(CommonValue)) in foldAndOrOfSETCC()
6237 CC = ISD::SETCC_INVALID; in foldAndOrOfSETCC()
6239 if (CC != ISD::SETCC_INVALID) { in foldAndOrOfSETCC()
6240 unsigned NewOpcode = ISD::DELETED_NODE; in foldAndOrOfSETCC()
6243 bool IsLess = (CC == ISD::SETLE || CC == ISD::SETULE || in foldAndOrOfSETCC()
6244 CC == ISD::SETLT || CC == ISD::SETULT); in foldAndOrOfSETCC()
6245 bool IsOr = (LogicOp->getOpcode() == ISD::OR); in foldAndOrOfSETCC()
6247 NewOpcode = IsSigned ? ISD::SMIN : ISD::UMIN; in foldAndOrOfSETCC()
6249 NewOpcode = IsSigned ? ISD::SMAX : ISD::UMAX; in foldAndOrOfSETCC()
6255 if (NewOpcode != ISD::DELETED_NODE) { in foldAndOrOfSETCC()
6267 CCL == (LogicOp->getOpcode() == ISD::AND ? ISD::SETNE : ISD::SETEQ) && in foldAndOrOfSETCC()
6276 DAG.doesNodeExist(ISD::ABS, DAG.getVTList(OpVT), {LHS0}))) { in foldAndOrOfSETCC()
6282 SDValue AbsOp = DAG.getNode(ISD::ABS, DL, OpVT, LHS0); in foldAndOrOfSETCC()
6283 return DAG.getNode(ISD::SETCC, DL, VT, AbsOp, in foldAndOrOfSETCC()
6311 SDValue AndOp = DAG.getNode(ISD::AND, DL, OpVT, NotOp, in foldAndOrOfSETCC()
6313 return DAG.getNode(ISD::SETCC, DL, VT, AndOp, in foldAndOrOfSETCC()
6317 SDValue AddOp = DAG.getNode(ISD::ADD, DL, OpVT, LHS0, in foldAndOrOfSETCC()
6319 SDValue AndOp = DAG.getNode(ISD::AND, DL, OpVT, AddOp, in foldAndOrOfSETCC()
6321 return DAG.getNode(ISD::SETCC, DL, VT, AndOp, in foldAndOrOfSETCC()
6345 if (T.getOpcode() != ISD::AND) in combineSelectAsExtAnd()
6355 return DAG.getNode(ISD::AND, DL, OpVT, CondMask, T.getOperand(0)); in combineSelectAsExtAnd()
6375 if (N1.getOpcode() == ISD::ADD) in visitANDLike()
6379 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL && in visitANDLike()
6398 DAG.getNode(ISD::ADD, DL0, VT, in visitANDLike()
6425 TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))) { in isAndLoadExtLoad()
6441 !TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT)) in isAndLoadExtLoad()
6444 if (!TLI.shouldReduceLoadWidth(LoadN, ISD::ZEXTLOAD, ExtVT)) in isAndLoadExtLoad()
6451 ISD::LoadExtType ExtType, EVT &MemVT, in isLegalNarrowLdSt()
6518 if (Load->getExtensionType() != ISD::NON_EXTLOAD && in isLegalNarrowLdSt()
6551 if ((N->getOpcode() == ISD::OR || N->getOpcode() == ISD::XOR) && in SearchForAndLoads()
6561 case ISD::LOAD: { in SearchForAndLoads()
6565 isLegalNarrowLdSt(Load, ISD::ZEXTLOAD, ExtVT)) { in SearchForAndLoads()
6568 if (Load->getExtensionType() == ISD::ZEXTLOAD && in SearchForAndLoads()
6580 case ISD::ZERO_EXTEND: in SearchForAndLoads()
6581 case ISD::AssertZext: { in SearchForAndLoads()
6584 EVT VT = Op.getOpcode() == ISD::AssertZext ? in SearchForAndLoads()
6594 case ISD::OR: in SearchForAndLoads()
6595 case ISD::XOR: in SearchForAndLoads()
6596 case ISD::AND: in SearchForAndLoads()
6653 SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode), in BackwardsPropagateMask()
6657 if (And.getOpcode() == ISD ::AND) in BackwardsPropagateMask()
6668 DAG.getNode(ISD::AND, SDLoc(Op0), Op0.getValueType(), Op0, MaskOp); in BackwardsPropagateMask()
6672 DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(), Op1, MaskOp); in BackwardsPropagateMask()
6683 SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0), in BackwardsPropagateMask()
6686 if (And.getOpcode() == ISD ::AND) in BackwardsPropagateMask()
6706 assert(N->getOpcode() == ISD::AND); in unfoldExtremeBitClearingToShifts()
6723 if (OuterShift == ISD::SHL) in unfoldExtremeBitClearingToShifts()
6724 InnerShift = ISD::SRL; in unfoldExtremeBitClearingToShifts()
6725 else if (OuterShift == ISD::SRL) in unfoldExtremeBitClearingToShifts()
6726 InnerShift = ISD::SHL; in unfoldExtremeBitClearingToShifts()
6758 assert(And->getOpcode() == ISD::AND && "Expected an 'and' op"); in combineShiftAnd1ToBitTest()
6762 if (And0.getOpcode() == ISD::ANY_EXTEND && And0.hasOneUse()) in combineShiftAnd1ToBitTest()
6779 if (Src.getOpcode() == ISD::TRUNCATE && Src.hasOneUse()) in combineShiftAnd1ToBitTest()
6784 if (Src.getOpcode() != ISD::SRL || !Src.hasOneUse()) in combineShiftAnd1ToBitTest()
6823 SDValue NewAnd = DAG.getNode(ISD::AND, DL, SrcVT, X, Mask); in combineShiftAnd1ToBitTest()
6825 SDValue Setcc = DAG.getSetCC(DL, CCVT, NewAnd, Zero, ISD::SETEQ); in combineShiftAnd1ToBitTest()
6837 if (N0.getOpcode() == ISD::SRA) in foldAndToUsubsat()
6841 if (N0.getOpcode() != ISD::XOR && N0.getOpcode() != ISD::ADD) in foldAndToUsubsat()
6844 if (N1.getOpcode() != ISD::SRA || !N0.hasOneUse() || !N1.hasOneUse() || in foldAndToUsubsat()
6859 return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0), SignMask); in foldAndToUsubsat()
6869 assert(ISD::isBitwiseLogicOp(LogicOpcode) && in foldLogicOfShifts()
6878 !(ShiftOpcode == ISD::SHL || ShiftOpcode == ISD::SRL || in foldLogicOfShifts()
6879 ShiftOpcode == ISD::SRA)) in foldLogicOfShifts()
6917 assert(ISD::isBitwiseLogicOp(LogicOpcode) && in foldLogicTreeOfShifts()
6955 if (SDValue C = DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, {N0, N1})) in visitAND()
6961 return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0); in visitAND()
6973 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitAND()
6979 if (ISD::isConstantSplatVectorAllOnes(N1.getNode())) in visitAND()
6985 if (MLoad && MLoad->getExtensionType() == ISD::EXTLOAD && Splat && in visitAND()
6989 if (TLI.isLoadExtLegal(ISD::ZEXTLOAD, ExtVT, LoadVT)) { in visitAND()
7000 ISD::ZEXTLOAD, MLoad->isExpandingLoad()); in visitAND()
7028 if (SDValue RAND = reassociateOps(ISD::AND, SDLoc(N), N0, N1, N->getFlags())) in visitAND()
7032 if (SDValue SD = reassociateReduction(ISD::VECREDUCE_AND, ISD::AND, SDLoc(N), in visitAND()
7040 if (N0.getOpcode() == ISD::OR && in visitAND()
7041 ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchSubset)) in visitAND()
7044 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) { in visitAND()
7053 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, N0Op0); in visitAND()
7058 TLI.isTypeDesirableForOp(ISD::AND, SrcVT) && in visitAND()
7061 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, in visitAND()
7062 DAG.getNode(ISD::AND, DL, SrcVT, N0Op0, in visitAND()
7068 if (ISD::isExtOpcode(N0.getOpcode())) { in visitAND()
7071 if (N0Op0.getOpcode() == ISD::AND && in visitAND()
7072 (ExtOpc != ISD::ZERO_EXTEND || !TLI.isZExtFree(N0Op0, VT)) && in visitAND()
7078 DAG.getNode(ISD::AND, DL, VT, N1, in visitAND()
7080 return DAG.getNode(ISD::AND, DL, VT, in visitAND()
7092 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitAND()
7094 N0.getOperand(0).getOpcode() == ISD::LOAD && in visitAND()
7096 (N0.getOpcode() == ISD::LOAD && N0.getResNo() == 0)) { in visitAND()
7097 LoadSDNode *Load = cast<LoadSDNode>( (N0.getOpcode() == ISD::LOAD) ? in visitAND()
7142 bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD, in visitAND()
7154 case ISD::EXTLOAD: B = CanZextLoadProfitably; break; in visitAND()
7155 case ISD::ZEXTLOAD: in visitAND()
7156 case ISD::NON_EXTLOAD: B = true; break; in visitAND()
7167 if (Load->getExtensionType() == ISD::EXTLOAD) { in visitAND()
7168 NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD, in visitAND()
7196 if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR && N0.hasOneUse() && N1C && in visitAND()
7197 ISD::isExtOpcode(N0.getOperand(0).getOpcode())) { in visitAND()
7204 (!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, ExtVT))) { in visitAND()
7208 DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), ExtVT, Extendee); in visitAND()
7210 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT, ZeroExtExtendee, in visitAND()
7228 GN0->getMemOperand(), GN0->getIndexType(), ISD::ZEXTLOAD); in visitAND()
7239 if (N1C && N0.getOpcode() == ISD::LOAD && !VT.isVector()) in visitAND()
7272 if (N1C && N1C->isOne() && N0.getOpcode() == ISD::SUB) { in visitAND()
7275 if (SubRHS.getOpcode() == ISD::ZERO_EXTEND && in visitAND()
7278 if (SubRHS.getOpcode() == ISD::SIGN_EXTEND && in visitAND()
7280 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0)); in visitAND()
7291 if (ISD::isUNINDEXEDLoad(N0.getNode()) && in visitAND()
7292 (ISD::isEXTLoad(N0.getNode()) || in visitAND()
7293 (ISD::isSEXTLoad(N0.getNode()) && N0.hasOneUse()))) { in visitAND()
7303 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) { in visitAND()
7305 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, LN0->getChain(), in visitAND()
7314 if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) { in visitAND()
7333 if (LHS->getOpcode() != ISD::SIGN_EXTEND) in visitAND()
7349 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, N0.getOperand(0)); in visitAND()
7351 if (hasOperation(ISD::USUBSAT, VT)) in visitAND()
7373 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT)) in MatchBSwapHWordLow()
7379 if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL) in MatchBSwapHWordLow()
7381 if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL) in MatchBSwapHWordLow()
7383 if (N0.getOpcode() == ISD::AND) { in MatchBSwapHWordLow()
7396 if (N1.getOpcode() == ISD::AND) { in MatchBSwapHWordLow()
7406 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL) in MatchBSwapHWordLow()
7408 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL) in MatchBSwapHWordLow()
7422 if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) { in MatchBSwapHWordLow()
7433 if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) { in MatchBSwapHWordLow()
7471 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00); in MatchBSwapHWordLow()
7474 Res = DAG.getNode(ISD::SRL, DL, VT, Res, in MatchBSwapHWordLow()
7492 if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL) in isBSwapHWordElement()
7497 if (Opc0 != ISD::AND && Opc0 != ISD::SHL && Opc0 != ISD::SRL) in isBSwapHWordElement()
7502 if (Opc == ISD::AND) in isBSwapHWordElement()
7504 else if (Opc0 == ISD::AND) in isBSwapHWordElement()
7518 if (Opc == ISD::SRL || (Opc == ISD::AND && Opc0 == ISD::SHL)) { in isBSwapHWordElement()
7528 if (Opc == ISD::AND) { in isBSwapHWordElement()
7532 if (Opc0 != ISD::SRL) in isBSwapHWordElement()
7540 if (Opc0 != ISD::SHL) in isBSwapHWordElement()
7546 } else if (Opc == ISD::SHL) { in isBSwapHWordElement()
7573 if (N.getOpcode() == ISD::OR) in isBSwapHWordPair()
7577 if (N.getOpcode() == ISD::SRL && N.getOperand(0).getOpcode() == ISD::BSWAP) { in isBSwapHWordPair()
7595 assert(N->getOpcode() == ISD::OR && VT == MVT::i32 && in matchBSwapHWordOrAndAnd()
7597 if (!TLI.isOperationLegalOrCustom(ISD::ROTR, VT)) in matchBSwapHWordOrAndAnd()
7599 if (N0.getOpcode() != ISD::AND || N1.getOpcode() != ISD::AND) in matchBSwapHWordOrAndAnd()
7613 if (Shift0.getOpcode() != ISD::SHL || Shift1.getOpcode() != ISD::SRL) in matchBSwapHWordOrAndAnd()
7625 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Shift0.getOperand(0)); in matchBSwapHWordOrAndAnd()
7627 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt); in matchBSwapHWordOrAndAnd()
7643 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT)) in MatchBSwapHWord()
7666 } else if (N0.getOpcode() == ISD::OR) { in MatchBSwapHWord()
7684 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, in MatchBSwapHWord()
7690 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT)) in MatchBSwapHWord()
7691 return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt); in MatchBSwapHWord()
7692 if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT)) in MatchBSwapHWord()
7693 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt); in MatchBSwapHWord()
7694 return DAG.getNode(ISD::OR, DL, VT, in MatchBSwapHWord()
7695 DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt), in MatchBSwapHWord()
7696 DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt)); in MatchBSwapHWord()
7713 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND && in visitORLike()
7729 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, in visitORLike()
7731 return DAG.getNode(ISD::AND, DL, VT, X, in visitORLike()
7739 if (N0.getOpcode() == ISD::AND && in visitORLike()
7740 N1.getOpcode() == ISD::AND && in visitORLike()
7744 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, in visitORLike()
7746 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X); in visitORLike()
7758 if (V->getOpcode() == ISD::ZERO_EXTEND || V->getOpcode() == ISD::TRUNCATE) in visitORCommutative()
7764 if (N0Resized.getOpcode() == ISD::AND) { in visitORCommutative()
7778 return DAG.getNode(ISD::OR, SDLoc(N), VT, in visitORCommutative()
7786 return DAG.getNode(ISD::OR, SDLoc(N), VT, in visitORCommutative()
7791 if (N0.getOpcode() == ISD::XOR) { in visitORCommutative()
7797 return DAG.getNode(ISD::OR, SDLoc(N), VT, N01, N1); in visitORCommutative()
7799 return DAG.getNode(ISD::OR, SDLoc(N), VT, N00, N1); in visitORCommutative()
7801 if (N1.getOpcode() == ISD::AND || N1.getOpcode() == ISD::OR) { in visitORCommutative()
7805 return DAG.getNode(ISD::OR, SDLoc(N), VT, N00, N01); in visitORCommutative()
7813 if (V->getOpcode() == ISD::ZERO_EXTEND) in visitORCommutative()
7819 if (N0.getOpcode() == ISD::FSHL && N1.getOpcode() == ISD::SHL && in visitORCommutative()
7825 if (N0.getOpcode() == ISD::FSHR && N1.getOpcode() == ISD::SRL && in visitORCommutative()
7843 if (SDValue C = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, {N0, N1})) in visitOR()
7849 return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0); in visitOR()
7857 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitOR()
7861 if (ISD::isConstantSplatVectorAllOnes(N1.getNode())) in visitOR()
7870 bool ZeroN00 = ISD::isBuildVectorAllZeros(N0.getOperand(0).getNode()); in visitOR()
7871 bool ZeroN01 = ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode()); in visitOR()
7872 bool ZeroN10 = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode()); in visitOR()
7873 bool ZeroN11 = ISD::isBuildVectorAllZeros(N1.getOperand(1).getNode()); in visitOR()
7956 if (SDValue ROR = reassociateOps(ISD::OR, SDLoc(N), N0, N1, N->getFlags())) in visitOR()
7960 if (SDValue SD = reassociateReduction(ISD::VECREDUCE_OR, ISD::OR, SDLoc(N), in visitOR()
7969 if (N0.getOpcode() == ISD::AND && N0->hasOneUse() && in visitOR()
7970 ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchIntersect, true)) { in visitOR()
7971 if (SDValue COR = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N1), VT, in visitOR()
7973 SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1); in visitOR()
7975 return DAG.getNode(ISD::AND, SDLoc(N), VT, COR, IOR); in visitOR()
8001 if ((!LegalOperations || TLI.isOperationLegal(ISD::ADD, VT)) && in visitOR()
8017 if (Op.getOpcode() == ISD::AND && in stripConstantMask()
8029 if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) { in matchRotateHalf()
8064 if (OppShift.getOpcode() != ISD::SHL && OppShift.getOpcode() != ISD::SRL) in extractShiftForRotate()
8078 if (OppShift.getOpcode() == ISD::SRL && OppShiftCst && in extractShiftForRotate()
8079 ExtractFrom.getOpcode() == ISD::ADD && in extractShiftForRotate()
8083 return DAG.getNode(ISD::SHL, DL, ShiftedVT, OppShiftLHS, in extractShiftForRotate()
8090 unsigned Opcode = ISD::DELETED_NODE; in extractShiftForRotate()
8103 if ((OppShift.getOpcode() != ISD::SRL || !SelectOpcode(ISD::SHL, ISD::MUL)) && in extractShiftForRotate()
8104 (OppShift.getOpcode() != ISD::SHL || !SelectOpcode(ISD::SRL, ISD::UDIV))) in extractShiftForRotate()
8232 if (Neg.getOpcode() != ISD::SUB) in matchRotateSub()
8267 (NegOp1.getOpcode() == ISD::TRUNCATE && Pos == NegOp1.getOperand(0))) in matchRotateSub()
8279 else if (Pos.getOpcode() == ISD::ADD && Pos.getOperand(0) == NegOp1) { in matchRotateSub()
8350 if (PosOpcode == ISD::FSHL && isPowerOf2_32(EltBits)) { in MatchFunnelPosNeg()
8360 if (IsBinOpImm(N1, ISD::SRL, 1) && in MatchFunnelPosNeg()
8361 IsBinOpImm(InnerNeg, ISD::XOR, EltBits - 1) && in MatchFunnelPosNeg()
8363 TLI.isOperationLegalOrCustom(ISD::FSHL, VT)) { in MatchFunnelPosNeg()
8364 return DAG.getNode(ISD::FSHL, DL, VT, N0, N1.getOperand(0), Pos); in MatchFunnelPosNeg()
8369 if (IsBinOpImm(N0, ISD::SHL, 1) && in MatchFunnelPosNeg()
8370 IsBinOpImm(InnerPos, ISD::XOR, EltBits - 1) && in MatchFunnelPosNeg()
8372 TLI.isOperationLegalOrCustom(ISD::FSHR, VT)) { in MatchFunnelPosNeg()
8373 return DAG.getNode(ISD::FSHR, DL, VT, N0.getOperand(0), N1, Neg); in MatchFunnelPosNeg()
8379 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N0.getOperand(1) && in MatchFunnelPosNeg()
8380 IsBinOpImm(InnerPos, ISD::XOR, EltBits - 1) && in MatchFunnelPosNeg()
8382 TLI.isOperationLegalOrCustom(ISD::FSHR, VT)) { in MatchFunnelPosNeg()
8383 return DAG.getNode(ISD::FSHR, DL, VT, N0.getOperand(0), N1, Neg); in MatchFunnelPosNeg()
8400 bool HasROTL = hasOperation(ISD::ROTL, VT); in MatchRotate()
8401 bool HasROTR = hasOperation(ISD::ROTR, VT); in MatchRotate()
8402 bool HasFSHL = hasOperation(ISD::FSHL, VT); in MatchRotate()
8403 bool HasFSHR = hasOperation(ISD::FSHR, VT); in MatchRotate()
8410 HasROTL |= TLI.getOperationAction(ISD::ROTL, VT) == TargetLowering::Custom; in MatchRotate()
8411 HasROTR |= TLI.getOperationAction(ISD::ROTR, VT) == TargetLowering::Custom; in MatchRotate()
8418 if (LHS.getOpcode() == ISD::TRUNCATE && RHS.getOpcode() == ISD::TRUNCATE && in MatchRotate()
8422 return DAG.getNode(ISD::TRUNCATE, SDLoc(LHS), LHS.getValueType(), Rot); in MatchRotate()
8468 if (RHSShift.getOpcode() == ISD::SHL) { in MatchRotate()
8475 if (LHSShift.getOpcode() != ISD::SHL || RHSShift.getOpcode() != ISD::SRL) in MatchRotate()
8496 SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt); in MatchRotate()
8497 Mask = DAG.getNode(ISD::AND, DL, VT, Mask, in MatchRotate()
8498 DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits)); in MatchRotate()
8501 SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt); in MatchRotate()
8502 Mask = DAG.getNode(ISD::AND, DL, VT, Mask, in MatchRotate()
8503 DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits)); in MatchRotate()
8506 Res = DAG.getNode(ISD::AND, DL, VT, Res, Mask); in MatchRotate()
8516 ISD::matchBinaryPredicate(LHSShiftAmt, RHSShiftAmt, MatchRotateSum)) { in MatchRotate()
8521 if (!Or.hasOneUse() || Or.getOpcode() != ISD::OR) in MatchRotate()
8539 SDValue RotX = DAG.getNode(ISD::ROTL, DL, VT, X, LHSShiftAmt); in MatchRotate()
8540 SDValue ShlY = DAG.getNode(ISD::SHL, DL, VT, Y, LHSShiftAmt); in MatchRotate()
8541 Res = DAG.getNode(ISD::OR, DL, VT, RotX, ShlY); in MatchRotate()
8544 SDValue RotX = DAG.getNode(ISD::ROTL, DL, VT, X, LHSShiftAmt); in MatchRotate()
8545 SDValue SrlY = DAG.getNode(ISD::SRL, DL, VT, Y, RHSShiftAmt); in MatchRotate()
8546 Res = DAG.getNode(ISD::OR, DL, VT, RotX, SrlY); in MatchRotate()
8562 if (ISD::matchBinaryPredicate(LHSShiftAmt, RHSShiftAmt, MatchRotateSum)) { in MatchRotate()
8566 Res = DAG.getNode(UseROTL ? ISD::ROTL : ISD::ROTR, DL, VT, LHSShiftArg, in MatchRotate()
8570 Res = DAG.getNode(UseFSHL ? ISD::FSHL : ISD::FSHR, DL, VT, LHSShiftArg, in MatchRotate()
8590 if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND || in MatchRotate()
8591 LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND || in MatchRotate()
8592 LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND || in MatchRotate()
8593 LHSShiftAmt.getOpcode() == ISD::TRUNCATE) && in MatchRotate()
8594 (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND || in MatchRotate()
8595 RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND || in MatchRotate()
8596 RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND || in MatchRotate()
8597 RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) { in MatchRotate()
8605 RExtOp0, HasROTL, ISD::ROTL, ISD::ROTR, DL); in MatchRotate()
8611 LExtOp0, HasROTR, ISD::ROTR, ISD::ROTL, DL); in MatchRotate()
8618 LExtOp0, RExtOp0, HasFSHL, ISD::FSHL, ISD::FSHR, DL); in MatchRotate()
8624 RExtOp0, LExtOp0, HasFSHR, ISD::FSHR, ISD::FSHL, DL); in MatchRotate()
8686 (Op.getOpcode() != ISD::LOAD || !Op.getValueType().isVector())) in calculateByteProvider()
8691 if (Op.getOpcode() != ISD::LOAD && VectorIndex.has_value()) in calculateByteProvider()
8702 case ISD::OR: { in calculateByteProvider()
8718 case ISD::SHL: { in calculateByteProvider()
8737 case ISD::ANY_EXTEND: in calculateByteProvider()
8738 case ISD::SIGN_EXTEND: in calculateByteProvider()
8739 case ISD::ZERO_EXTEND: { in calculateByteProvider()
8747 return Op.getOpcode() == ISD::ZERO_EXTEND in calculateByteProvider()
8754 case ISD::BSWAP: in calculateByteProvider()
8757 case ISD::EXTRACT_VECTOR_ELT: { in calculateByteProvider()
8784 case ISD::LOAD: { in calculateByteProvider()
8798 return L->getExtensionType() == ISD::ZEXTLOAD in calculateByteProvider()
8845 case ISD::TRUNCATE: in stripTruncAndExt()
8846 case ISD::ZERO_EXTEND: in stripTruncAndExt()
8847 case ISD::SIGN_EXTEND: in stripTruncAndExt()
8848 case ISD::ANY_EXTEND: in stripTruncAndExt()
8937 if (Trunc.getOpcode() != ISD::TRUNCATE) in mergeTruncStores()
8943 if ((WideVal.getOpcode() == ISD::SRL || WideVal.getOpcode() == ISD::SRA) && in mergeTruncStores()
9038 SourceValue = DAG.getNode(ISD::TRUNCATE, DL, WideVT, SourceValue); in mergeTruncStores()
9045 SourceValue = DAG.getNode(ISD::BSWAP, DL, WideVT, SourceValue); in mergeTruncStores()
9049 SourceValue = DAG.getNode(ISD::ROTR, DL, WideVT, SourceValue, RotAmt); in mergeTruncStores()
9092 assert(N->getOpcode() == ISD::OR && in MatchLoadCombine()
9205 !TLI.isOperationLegal(NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, in MatchLoadCombine()
9237 !TLI.isOperationLegal(ISD::BSWAP, VT)) in MatchLoadCombine()
9243 !TLI.isOperationLegal(ISD::SHL, VT)) in MatchLoadCombine()
9255 DAG.getExtLoad(NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, SDLoc(N), VT, in MatchLoadCombine()
9268 ? DAG.getNode(ISD::SHL, SDLoc(N), VT, NewLoad, in MatchLoadCombine()
9272 return DAG.getNode(ISD::BSWAP, SDLoc(N), VT, ShiftedLoad); in MatchLoadCombine()
9292 assert(N->getOpcode() == ISD::XOR); in unfoldMaskedMerge()
9304 if (And.getOpcode() != ISD::AND || !And.hasOneUse()) in unfoldMaskedMerge()
9307 if (Xor.getOpcode() != ISD::XOR || !Xor.hasOneUse()) in unfoldMaskedMerge()
9348 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, NotX, M); in unfoldMaskedMerge()
9350 SDValue RHS = DAG.getNode(ISD::OR, DL, VT, M, Y); in unfoldMaskedMerge()
9351 return DAG.getNode(ISD::AND, DL, VT, NotLHS, RHS); in unfoldMaskedMerge()
9360 SDValue LHS = DAG.getNode(ISD::OR, DL, VT, X, NotM); in unfoldMaskedMerge()
9362 SDValue RHS = DAG.getNode(ISD::AND, DL, VT, NotM, NotY); in unfoldMaskedMerge()
9364 return DAG.getNode(ISD::AND, DL, VT, LHS, NotRHS); in unfoldMaskedMerge()
9367 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, X, M); in unfoldMaskedMerge()
9369 SDValue RHS = DAG.getNode(ISD::AND, DL, VT, Y, NotM); in unfoldMaskedMerge()
9371 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS); in unfoldMaskedMerge()
9391 if (SDValue C = DAG.FoldConstantArithmetic(ISD::XOR, DL, VT, {N0, N1})) in visitXOR()
9397 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitXOR()
9405 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitXOR()
9417 if (SDValue RXOR = reassociateOps(ISD::XOR, DL, N0, N1, N->getFlags())) in visitXOR()
9422 reassociateReduction(ISD::VECREDUCE_XOR, ISD::XOR, DL, VT, N0, N1)) in visitXOR()
9426 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) && in visitXOR()
9428 return DAG.getNode(ISD::OR, DL, VT, N0, N1); in visitXOR()
9432 if ((!LegalOperations || TLI.isOperationLegal(ISD::ADD, VT)) && in visitXOR()
9442 ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(), in visitXOR()
9449 case ISD::SETCC: in visitXOR()
9451 case ISD::SELECT_CC: in visitXOR()
9454 case ISD::STRICT_FSETCC: in visitXOR()
9455 case ISD::STRICT_FSETCCS: { in visitXOR()
9461 N0.getOperand(0), N0Opcode == ISD::STRICT_FSETCCS); in visitXOR()
9474 if (isOneConstant(N1) && N0Opcode == ISD::ZERO_EXTEND && N0.hasOneUse() && in visitXOR()
9478 V = DAG.getNode(ISD::XOR, DL0, V.getValueType(), V, in visitXOR()
9481 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, V); in visitXOR()
9486 (N0Opcode == ISD::OR || N0Opcode == ISD::AND)) { in visitXOR()
9489 unsigned NewOpcode = N0Opcode == ISD::AND ? ISD::OR : ISD::AND; in visitXOR()
9490 N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00 in visitXOR()
9491 N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01 in visitXOR()
9498 (N0Opcode == ISD::OR || N0Opcode == ISD::AND)) { in visitXOR()
9501 unsigned NewOpcode = N0Opcode == ISD::AND ? ISD::OR : ISD::AND; in visitXOR()
9502 N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00 in visitXOR()
9503 N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01 in visitXOR()
9512 if (isAllOnesConstant(N1) && N0.getOpcode() == ISD::SUB && in visitXOR()
9514 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), in visitXOR()
9519 if (isAllOnesConstant(N1) && N0.getOpcode() == ISD::ADD && in visitXOR()
9525 if (N0Opcode == ISD::AND && N0.hasOneUse() && N0->getOperand(1) == N1) { in visitXOR()
9529 return DAG.getNode(ISD::AND, DL, VT, NotX, N1); in visitXOR()
9533 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) { in visitXOR()
9534 SDValue A = N0Opcode == ISD::ADD ? N0 : N1; in visitXOR()
9535 SDValue S = N0Opcode == ISD::SRA ? N0 : N1; in visitXOR()
9536 if (A.getOpcode() == ISD::ADD && S.getOpcode() == ISD::SRA) { in visitXOR()
9542 return DAG.getNode(ISD::ABS, DL, VT, S0); in visitXOR()
9568 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0Opcode == ISD::SHL && in visitXOR()
9570 return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT), in visitXOR()
9611 if (LogicOpcode != ISD::AND && LogicOpcode != ISD::OR && in combineShiftOfShiftedLogic()
9612 LogicOpcode != ISD::XOR) in combineShiftOfShiftedLogic()
9702 case ISD::OR: in visitShiftByConstant()
9703 case ISD::XOR: in visitShiftByConstant()
9704 case ISD::AND: in visitShiftByConstant()
9706 case ISD::ADD: in visitShiftByConstant()
9707 if (N->getOpcode() != ISD::SHL) in visitShiftByConstant()
9716 bool IsShiftByConstant = (BinOpLHSVal.getOpcode() == ISD::SHL || in visitShiftByConstant()
9717 BinOpLHSVal.getOpcode() == ISD::SRA || in visitShiftByConstant()
9718 BinOpLHSVal.getOpcode() == ISD::SRL) && in visitShiftByConstant()
9720 bool IsCopyOrSelect = BinOpLHSVal.getOpcode() == ISD::CopyFromReg || in visitShiftByConstant()
9721 BinOpLHSVal.getOpcode() == ISD::SELECT; in visitShiftByConstant()
9743 assert(N->getOpcode() == ISD::TRUNCATE); in distributeTruncateThroughAnd()
9744 assert(N->getOperand(0).getOpcode() == ISD::AND); in distributeTruncateThroughAnd()
9749 TLI.isTypeDesirableForOp(ISD::AND, TruncVT)) { in distributeTruncateThroughAnd()
9754 SDValue Trunc00 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00); in distributeTruncateThroughAnd()
9755 SDValue Trunc01 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N01); in distributeTruncateThroughAnd()
9758 return DAG.getNode(ISD::AND, DL, TruncVT, Trunc00, Trunc01); in distributeTruncateThroughAnd()
9789 if (ISD::matchUnaryPredicate(N1, MatchOutOfRange) && OutOfRange) { in visitRotate()
9793 DAG.FoldConstantArithmetic(ISD::UREM, dl, AmtVT, {N1, Bits})) in visitRotate()
9800 VT.getScalarSizeInBits() == 16 && hasOperation(ISD::BSWAP, VT)) in visitRotate()
9801 return DAG.getNode(ISD::BSWAP, dl, VT, N0); in visitRotate()
9808 if (N1.getOpcode() == ISD::TRUNCATE && in visitRotate()
9809 N1.getOperand(0).getOpcode() == ISD::AND) { in visitRotate()
9818 if (NextOp == ISD::ROTL || NextOp == ISD::ROTR) { in visitRotate()
9824 unsigned CombineOp = SameSide ? ISD::ADD : ISD::SUB; in visitRotate()
9826 SDValue Norm1 = DAG.FoldConstantArithmetic(ISD::UREM, dl, ShiftVT, in visitRotate()
9828 SDValue Norm2 = DAG.FoldConstantArithmetic(ISD::UREM, dl, ShiftVT, in visitRotate()
9833 CombinedShift = DAG.FoldConstantArithmetic(ISD::ADD, dl, ShiftVT, in visitRotate()
9836 ISD::UREM, dl, ShiftVT, {CombinedShift, BitsizeC}); in visitRotate()
9856 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, {N0, N1})) in visitSHL()
9868 if (N0.getOpcode() == ISD::AND) { in visitSHL()
9873 if (N01CV && N01CV->isConstant() && N00.getOpcode() == ISD::SETCC && in visitSHL()
9877 DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, {N01, N1})) in visitSHL()
9878 return DAG.getNode(ISD::AND, SDLoc(N), VT, N00, C); in visitSHL()
9892 if (N1.getOpcode() == ISD::TRUNCATE && in visitSHL()
9893 N1.getOperand(0).getOpcode() == ISD::AND) { in visitSHL()
9895 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, NewOp1); in visitSHL()
9899 if (N0.getOpcode() == ISD::SHL) { in visitSHL()
9907 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange)) in visitSHL()
9917 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) { in visitSHL()
9919 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1)); in visitSHL()
9920 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum); in visitSHL()
9929 if ((N0.getOpcode() == ISD::ZERO_EXTEND || in visitSHL()
9930 N0.getOpcode() == ISD::ANY_EXTEND || in visitSHL()
9931 N0.getOpcode() == ISD::SIGN_EXTEND) && in visitSHL()
9932 N0.getOperand(0).getOpcode() == ISD::SHL) { in visitSHL()
9946 if (ISD::matchBinaryPredicate(InnerShiftAmt, N1, MatchOutOfRange, in visitSHL()
9959 if (ISD::matchBinaryPredicate(InnerShiftAmt, N1, MatchInRange, in visitSHL()
9965 Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, Sum, N1); in visitSHL()
9966 return DAG.getNode(ISD::SHL, DL, VT, Ext, Sum); in visitSHL()
9973 if (N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() && in visitSHL()
9974 N0.getOperand(0).getOpcode() == ISD::SRL) { in visitSHL()
9984 if (ISD::matchBinaryPredicate(InnerShiftAmt, N1, MatchEqual, in visitSHL()
9990 NewSHL = DAG.getNode(ISD::SHL, DL, N0Op0.getValueType(), N0Op0, NewSHL); in visitSHL()
9992 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL); in visitSHL()
9996 if (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA) { in visitSHL()
10010 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchShiftAmount, in visitSHL()
10014 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N1, N01); in visitSHL()
10015 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff); in visitSHL()
10017 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchShiftAmount, in visitSHL()
10021 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N01, N1); in visitSHL()
10030 if (N0.getOpcode() == ISD::SRL && in visitSHL()
10033 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchShiftAmount, in visitSHL()
10037 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N01, N1); in visitSHL()
10039 Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, N01); in visitSHL()
10040 Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, Diff); in visitSHL()
10041 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Diff); in visitSHL()
10042 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSHL()
10044 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchShiftAmount, in visitSHL()
10048 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N1, N01); in visitSHL()
10050 Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, N1); in visitSHL()
10051 SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff); in visitSHL()
10052 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSHL()
10058 if (N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1) && in visitSHL()
10062 SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1); in visitSHL()
10063 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), HiBitsMask); in visitSHL()
10070 if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR) && in visitSHL()
10074 DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N1), VT, {N01, N1})) { in visitSHL()
10075 SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1); in visitSHL()
10079 if (N0.getOpcode() == ISD::OR && N0->getFlags().hasDisjoint()) in visitSHL()
10088 if (N0.getOpcode() == ISD::SIGN_EXTEND && in visitSHL()
10089 N0.getOperand(0).getOpcode() == ISD::ADD && in visitSHL()
10098 DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {ExtC, N1})) { in visitSHL()
10100 SDValue ShlX = DAG.getNode(ISD::SHL, DL, VT, ExtX, N1); in visitSHL()
10101 return DAG.getNode(ISD::ADD, DL, VT, ShlX, ShlC); in visitSHL()
10107 if (N0.getOpcode() == ISD::MUL && N0->hasOneUse()) { in visitSHL()
10110 DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N1), VT, {N01, N1})) in visitSHL()
10111 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), Shl); in visitSHL()
10123 if (N0.getOpcode() == ISD::VSCALE && N1C) { in visitSHL()
10131 if (N0.getOpcode() == ISD::STEP_VECTOR && in visitSHL()
10132 ISD::isConstantSplatVector(N1.getNode(), ShlVal)) { in visitSHL()
10149 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) && in combineShiftToMULH()
10162 if (ShiftOperand.getOpcode() != ISD::MUL) in combineShiftToMULH()
10169 bool IsSignExt = LeftOp.getOpcode() == ISD::SIGN_EXTEND; in combineShiftToMULH()
10170 bool IsZeroExt = LeftOp.getOpcode() == ISD::ZERO_EXTEND; in combineShiftToMULH()
10180 if (U->getOpcode() != ISD::SRL && U->getOpcode() != ISD::SRA) { in combineShiftToMULH()
10193 unsigned MulLoHiOp = IsSignExt ? ISD::SMUL_LOHI : ISD::UMUL_LOHI; in combineShiftToMULH()
10238 unsigned MulhOpcode = IsSignExt ? ISD::MULHS : ISD::MULHU; in combineShiftToMULH()
10255 bool IsSigned = N->getOpcode() == ISD::SRA; in combineShiftToMULH()
10263 if (Opcode != ISD::BSWAP && Opcode != ISD::BITREVERSE) in foldBitOrderCrossLogicOp()
10269 if (ISD::isBitwiseLogicOp(N0.getOpcode()) && N0.hasOneUse()) { in foldBitOrderCrossLogicOp()
10305 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, {N0, N1})) in visitSRA()
10326 if (N0.getOpcode() == ISD::SRA) { in visitSRA()
10342 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), SumOfShifts)) { in visitSRA()
10344 if (N1.getOpcode() == ISD::BUILD_VECTOR) in visitSRA()
10346 else if (N1.getOpcode() == ISD::SPLAT_VECTOR) { in visitSRA()
10353 return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0), ShiftValue); in visitSRA()
10362 if (N0.getOpcode() == ISD::SHL && N1C) { in visitSRA()
10381 TLI.isOperationLegalOrCustom(ISD::SIGN_EXTEND, TruncVT) && in visitSRA()
10382 TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) && in visitSRA()
10387 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, in visitSRA()
10389 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, in visitSRA()
10391 return DAG.getNode(ISD::SIGN_EXTEND, DL, in visitSRA()
10402 if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB) && N1C && in visitSRA()
10404 bool IsAdd = N0.getOpcode() == ISD::ADD; in visitSRA()
10406 if (Shl.getOpcode() == ISD::SHL && Shl.getOperand(1) == N1 && in visitSRA()
10433 Add = DAG.getNode(ISD::ADD, DL, TruncVT, Trunc, ShiftC); in visitSRA()
10435 Add = DAG.getNode(ISD::SUB, DL, TruncVT, ShiftC, Trunc); in visitSRA()
10443 if (N1.getOpcode() == ISD::TRUNCATE && in visitSRA()
10444 N1.getOperand(0).getOpcode() == ISD::AND) { in visitSRA()
10446 return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0, NewOp1); in visitSRA()
10453 if (N0.getOpcode() == ISD::TRUNCATE && in visitSRA()
10454 (N0.getOperand(0).getOpcode() == ISD::SRL || in visitSRA()
10455 N0.getOperand(0).getOpcode() == ISD::SRA) && in visitSRA()
10466 Amt = DAG.getNode(ISD::ADD, DL, LargeShiftVT, Amt, in visitSRA()
10469 DAG.getNode(ISD::SRA, DL, LargeVT, N0Op0.getOperand(0), Amt); in visitSRA()
10470 return DAG.getNode(ISD::TRUNCATE, DL, VT, SRA); in visitSRA()
10481 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, N1); in visitSRA()
10510 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, {N0, N1})) in visitSRL()
10528 if (N0.getOpcode() == ISD::SRL) { in visitSRL()
10536 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange)) in visitSRL()
10546 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) { in visitSRL()
10548 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1)); in visitSRL()
10549 return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Sum); in visitSRL()
10553 if (N1C && N0.getOpcode() == ISD::TRUNCATE && in visitSRL()
10554 N0.getOperand(0).getOpcode() == ISD::SRL) { in visitSRL()
10570 SDValue NewShift = DAG.getNode(ISD::SRL, DL, InnerShiftVT, in visitSRL()
10572 return DAG.getNode(ISD::TRUNCATE, DL, VT, NewShift); in visitSRL()
10580 SDValue NewShift = DAG.getNode(ISD::SRL, DL, InnerShiftVT, in visitSRL()
10585 SDValue And = DAG.getNode(ISD::AND, DL, InnerShiftVT, NewShift, Mask); in visitSRL()
10586 return DAG.getNode(ISD::TRUNCATE, DL, VT, And); in visitSRL()
10593 if (N0.getOpcode() == ISD::SHL && in visitSRL()
10603 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchShiftAmount, in visitSRL()
10608 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N01, N1); in visitSRL()
10610 Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, N01); in visitSRL()
10611 Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, Diff); in visitSRL()
10612 SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff); in visitSRL()
10613 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSRL()
10615 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchShiftAmount, in visitSRL()
10620 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N1, N01); in visitSRL()
10622 Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, N1); in visitSRL()
10623 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Diff); in visitSRL()
10624 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSRL()
10630 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) { in visitSRL()
10637 if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, SmallVT)) { in visitSRL()
10640 SDValue SmallShift = DAG.getNode(ISD::SRL, DL0, SmallVT, in visitSRL()
10647 return DAG.getNode(ISD::AND, DL, VT, in visitSRL()
10648 DAG.getNode(ISD::ANY_EXTEND, DL, VT, SmallShift), in visitSRL()
10656 if (N0.getOpcode() == ISD::SRA) in visitSRL()
10657 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), N1); in visitSRL()
10662 if (N1C && N0.getOpcode() == ISD::CTLZ && in visitSRL()
10687 Op = DAG.getNode(ISD::SRL, DL, VT, Op, in visitSRL()
10694 return DAG.getNode(ISD::XOR, DL, VT, in visitSRL()
10700 if (N1.getOpcode() == ISD::TRUNCATE && in visitSRL()
10701 N1.getOperand(0).getOpcode() == ISD::AND) { in visitSRL()
10703 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, NewOp1); in visitSRL()
10747 if (Use->getOpcode() == ISD::TRUNCATE && Use->hasOneUse()) in visitSRL()
10750 if (Use->getOpcode() == ISD::BRCOND || Use->getOpcode() == ISD::AND || in visitSRL()
10751 Use->getOpcode() == ISD::OR || Use->getOpcode() == ISD::XOR) in visitSRL()
10768 bool IsFSHL = N->getOpcode() == ISD::FSHL; in visitFunnelShift()
10802 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N1, in visitFunnelShift()
10806 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, in visitFunnelShift()
10821 (LHS->hasOneUse() || RHS->hasOneUse()) && ISD::isNON_EXTLoad(RHS) && in visitFunnelShift()
10822 ISD::isNON_EXTLoad(LHS)) { in visitFunnelShift()
10857 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N1, N2); in visitFunnelShift()
10859 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, N2); in visitFunnelShift()
10866 unsigned RotOpc = IsFSHL ? ISD::ROTL : ISD::ROTR; in visitFunnelShift()
10892 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::SHL, VT)) { in visitSHLSAT()
10894 if (N->getOpcode() == ISD::SSHLSAT && N1C && in visitSHLSAT()
10896 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, N1); in visitSHLSAT()
10899 if (N->getOpcode() == ISD::USHLSAT && N1C && in visitSHLSAT()
10902 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, N1); in visitSHLSAT()
10915 if (N->getOpcode() == ISD::TRUNCATE) in foldABSToABD()
10918 if (N->getOpcode() != ISD::ABS) in foldABSToABD()
10926 if (AbsOp1.getOpcode() != ISD::SUB) in foldABSToABD()
10937 (Opc0 != ISD::ZERO_EXTEND && Opc0 != ISD::SIGN_EXTEND && in foldABSToABD()
10938 Opc0 != ISD::SIGN_EXTEND_INREG)) { in foldABSToABD()
10940 if (AbsOp1->getFlags().hasNoSignedWrap() && hasOperation(ISD::ABDS, VT) && in foldABSToABD()
10942 SDValue ABD = DAG.getNode(ISD::ABDS, DL, VT, Op0, Op1); in foldABSToABD()
10949 if (Opc0 == ISD::SIGN_EXTEND_INREG) { in foldABSToABD()
10956 unsigned ABDOpcode = (Opc0 == ISD::ZERO_EXTEND) ? ISD::ABDU : ISD::ABDS; in foldABSToABD()
10964 DAG.getNode(ISD::TRUNCATE, DL, MaxVT, Op0), in foldABSToABD()
10965 DAG.getNode(ISD::TRUNCATE, DL, MaxVT, Op1)); in foldABSToABD()
10966 ABD = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, ABD); in foldABSToABD()
10985 if (SDValue C = DAG.FoldConstantArithmetic(ISD::ABS, SDLoc(N), VT, {N0})) in visitABS()
10988 if (N0.getOpcode() == ISD::ABS) in visitABS()
10999 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) { in visitABS()
11002 TLI.isTypeDesirableForOp(ISD::ABS, ExtVT) && in visitABS()
11003 hasOperation(ISD::ABS, ExtVT)) { in visitABS()
11006 ISD::ZERO_EXTEND, DL, VT, in visitABS()
11007 DAG.getNode(ISD::ABS, DL, ExtVT, in visitABS()
11008 DAG.getNode(ISD::TRUNCATE, DL, ExtVT, N0.getOperand(0)))); in visitABS()
11021 if (SDValue C = DAG.FoldConstantArithmetic(ISD::BSWAP, DL, VT, {N0})) in visitBSWAP()
11024 if (N0.getOpcode() == ISD::BSWAP) in visitBSWAP()
11031 if (N0.getOpcode() == ISD::BITREVERSE && N0.hasOneUse()) { in visitBSWAP()
11032 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, N0.getOperand(0)); in visitBSWAP()
11033 return DAG.getNode(ISD::BITREVERSE, DL, VT, BSwap); in visitBSWAP()
11039 if (BW >= 32 && N0.getOpcode() == ISD::SHL && N0.hasOneUse()) { in visitBSWAP()
11046 (!LegalOperations || hasOperation(ISD::BSWAP, HalfVT))) { in visitBSWAP()
11049 Res = DAG.getNode(ISD::SHL, DL, VT, Res, in visitBSWAP()
11052 Res = DAG.getNode(ISD::BSWAP, DL, HalfVT, Res); in visitBSWAP()
11061 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) && in visitBSWAP()
11066 SDValue NewSwap = DAG.getNode(ISD::BSWAP, DL, VT, N0.getOperand(0)); in visitBSWAP()
11067 unsigned InverseShift = N0.getOpcode() == ISD::SHL ? ISD::SRL : ISD::SHL; in visitBSWAP()
11084 if (SDValue C = DAG.FoldConstantArithmetic(ISD::BITREVERSE, DL, VT, {N0})) in visitBITREVERSE()
11087 if (N0.getOpcode() == ISD::BITREVERSE) in visitBITREVERSE()
11098 if (SDValue C = DAG.FoldConstantArithmetic(ISD::CTLZ, DL, VT, {N0})) in visitCTLZ()
11102 if (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ_ZERO_UNDEF, VT)) in visitCTLZ()
11104 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, DL, VT, N0); in visitCTLZ()
11116 DAG.FoldConstantArithmetic(ISD::CTLZ_ZERO_UNDEF, DL, VT, {N0})) in visitCTLZ_ZERO_UNDEF()
11127 if (SDValue C = DAG.FoldConstantArithmetic(ISD::CTTZ, DL, VT, {N0})) in visitCTTZ()
11131 if (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ_ZERO_UNDEF, VT)) in visitCTTZ()
11133 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, DL, VT, N0); in visitCTTZ()
11145 DAG.FoldConstantArithmetic(ISD::CTTZ_ZERO_UNDEF, DL, VT, {N0})) in visitCTTZ_ZERO_UNDEF()
11156 if (SDValue C = DAG.FoldConstantArithmetic(ISD::CTPOP, DL, VT, {N0})) in visitCTPOP()
11176 ISD::CondCode CC, in combineMinNumMaxNumImpl()
11181 case ISD::SETOLT: in combineMinNumMaxNumImpl()
11182 case ISD::SETOLE: in combineMinNumMaxNumImpl()
11183 case ISD::SETLT: in combineMinNumMaxNumImpl()
11184 case ISD::SETLE: in combineMinNumMaxNumImpl()
11185 case ISD::SETULT: in combineMinNumMaxNumImpl()
11186 case ISD::SETULE: { in combineMinNumMaxNumImpl()
11190 unsigned IEEEOpcode = (LHS == True) ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE; in combineMinNumMaxNumImpl()
11194 unsigned Opcode = (LHS == True) ? ISD::FMINNUM : ISD::FMAXNUM; in combineMinNumMaxNumImpl()
11199 case ISD::SETOGT: in combineMinNumMaxNumImpl()
11200 case ISD::SETOGE: in combineMinNumMaxNumImpl()
11201 case ISD::SETGT: in combineMinNumMaxNumImpl()
11202 case ISD::SETGE: in combineMinNumMaxNumImpl()
11203 case ISD::SETUGT: in combineMinNumMaxNumImpl()
11204 case ISD::SETUGE: { in combineMinNumMaxNumImpl()
11205 unsigned IEEEOpcode = (LHS == True) ? ISD::FMAXNUM_IEEE : ISD::FMINNUM_IEEE; in combineMinNumMaxNumImpl()
11209 unsigned Opcode = (LHS == True) ? ISD::FMAXNUM : ISD::FMINNUM; in combineMinNumMaxNumImpl()
11222 SDValue False, ISD::CondCode CC) { in combineMinNumMaxNum()
11252 return DAG.getNode(ISD::FNEG, DL, VT, Combined); in combineMinNumMaxNum()
11270 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse() || in foldSelectOfConstantsUsingSra()
11278 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in foldSelectOfConstantsUsingSra()
11279 if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(CondC) && in foldSelectOfConstantsUsingSra()
11284 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC); in foldSelectOfConstantsUsingSra()
11285 return DAG.getNode(ISD::OR, DL, VT, Sra, C1); in foldSelectOfConstantsUsingSra()
11287 if (CC == ISD::SETLT && isNullOrNullSplat(CondC) && isNullOrNullSplat(C2)) { in foldSelectOfConstantsUsingSra()
11291 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC); in foldSelectOfConstantsUsingSra()
11292 return DAG.getNode(ISD::AND, DL, VT, Sra, C1); in foldSelectOfConstantsUsingSra()
11302 if (Cond.getOpcode() != ISD::SETCC || !Cond->hasOneUse()) in shouldConvertSelectOfConstantsToMath()
11304 if (!TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT)) in shouldConvertSelectOfConstantsToMath()
11307 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in shouldConvertSelectOfConstantsToMath()
11308 if (CC == ISD::SETLT && isNullOrNullSplat(Cond.getOperand(1))) in shouldConvertSelectOfConstantsToMath()
11310 if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(Cond.getOperand(1))) in shouldConvertSelectOfConstantsToMath()
11349 DAG.getNode(ISD::XOR, DL, CondVT, Cond, DAG.getConstant(1, DL, CondVT)); in foldSelectOfConstants()
11399 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2); in foldSelectOfConstants()
11405 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2); in foldSelectOfConstants()
11413 return DAG.getNode(ISD::SHL, DL, VT, Cond, ShAmtC); in foldSelectOfConstants()
11419 return DAG.getNode(ISD::OR, DL, VT, Cond, N2); in foldSelectOfConstants()
11426 return DAG.getNode(ISD::OR, DL, VT, NotCond, N1); in foldSelectOfConstants()
11436 assert((N->getOpcode() == ISD::SELECT || N->getOpcode() == ISD::VSELECT) && in foldBoolSelectToLogic()
11447 return DAG.getNode(ISD::OR, SDLoc(N), VT, Cond, F); in foldBoolSelectToLogic()
11452 return DAG.getNode(ISD::AND, SDLoc(N), VT, Cond, T); in foldBoolSelectToLogic()
11457 return DAG.getNode(ISD::OR, SDLoc(N), VT, NotCond, T); in foldBoolSelectToLogic()
11463 return DAG.getNode(ISD::AND, SDLoc(N), VT, NotCond, F); in foldBoolSelectToLogic()
11474 if (N0.getOpcode() != ISD::SETCC || !N0.hasOneUse()) in foldVSelectToSignBitSplatMask()
11479 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in foldVSelectToSignBitSplatMask()
11485 if (CC == ISD::SETLT && isNullOrNullSplat(Cond1)) in foldVSelectToSignBitSplatMask()
11487 else if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(Cond1)) in foldVSelectToSignBitSplatMask()
11496 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt); in foldVSelectToSignBitSplatMask()
11497 return DAG.getNode(ISD::AND, DL, VT, Sra, N1); in foldVSelectToSignBitSplatMask()
11504 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt); in foldVSelectToSignBitSplatMask()
11505 return DAG.getNode(ISD::OR, DL, VT, Sra, N2); in foldVSelectToSignBitSplatMask()
11515 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt); in foldVSelectToSignBitSplatMask()
11517 return DAG.getNode(ISD::AND, DL, VT, Not, N2); in foldVSelectToSignBitSplatMask()
11569 if (N0->getOpcode() == ISD::AND && N0->hasOneUse()) { in visitSELECT()
11573 DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2, Flags); in visitSELECT()
11575 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, in visitSELECT()
11582 if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) { in visitSELECT()
11585 SDValue InnerSelect = DAG.getNode(ISD::SELECT, DL, N1.getValueType(), in visitSELECT()
11588 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, N1, in visitSELECT()
11596 if (N1->getOpcode() == ISD::SELECT && N1->hasOneUse()) { in visitSELECT()
11603 SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0); in visitSELECT()
11604 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), And, N1_1, in visitSELECT()
11609 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1_1, in visitSELECT()
11615 if (N2->getOpcode() == ISD::SELECT && N2->hasOneUse()) { in visitSELECT()
11622 SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0); in visitSELECT()
11623 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Or, N1, in visitSELECT()
11628 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1, in visitSELECT()
11635 if (N0.getOpcode() == ISD::SETCC) { in visitSELECT()
11637 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in visitSELECT()
11653 if (!LegalOperations && TLI.isOperationLegalOrCustom(ISD::UADDO, VT) && in visitSELECT()
11654 CC == ISD::SETUGT && N0.hasOneUse() && isAllOnesConstant(N1) && in visitSELECT()
11655 N2.getOpcode() == ISD::ADD && Cond0 == N2.getOperand(0)) { in visitSELECT()
11672 SDValue UAO = DAG.getNode(ISD::UADDO, DL, VTs, Cond0, N2.getOperand(1)); in visitSELECT()
11677 if (TLI.isOperationLegal(ISD::SELECT_CC, VT) || in visitSELECT()
11679 TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT))) { in visitSELECT()
11683 SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1, in visitSELECT()
11712 assert(LHS.getOpcode() == ISD::CONCAT_VECTORS && in ConvertSelectToConcatVector()
11713 RHS.getOpcode() == ISD::CONCAT_VECTORS && in ConvertSelectToConcatVector()
11714 Cond.getOpcode() == ISD::BUILD_VECTOR); in ConvertSelectToConcatVector()
11753 ISD::CONCAT_VECTORS, DL, VT, in ConvertSelectToConcatVector()
11773 BasePtr = DAG.getNode(ISD::ADD, DL, VT, BasePtr, SplatVal); in refineUniformBase()
11778 if (Index.getOpcode() != ISD::ADD) in refineUniformBase()
11783 BasePtr = DAG.getNode(ISD::ADD, DL, VT, BasePtr, SplatVal); in refineUniformBase()
11789 BasePtr = DAG.getNode(ISD::ADD, DL, VT, BasePtr, SplatVal); in refineUniformBase()
11797 bool refineIndexType(SDValue &Index, ISD::MemIndexType &IndexType, EVT DataVT, in refineIndexType()
11802 if (Index.getOpcode() == ISD::ZERO_EXTEND) { in refineIndexType()
11804 IndexType = ISD::UNSIGNED_SCALED; in refineIndexType()
11808 if (ISD::isIndexTypeSigned(IndexType)) { in refineIndexType()
11809 IndexType = ISD::UNSIGNED_SCALED; in refineIndexType()
11815 if (Index.getOpcode() == ISD::SIGN_EXTEND && in refineIndexType()
11816 ISD::isIndexTypeSigned(IndexType) && in refineIndexType()
11834 ISD::MemIndexType IndexType = MSC->getIndexType(); in visitVPSCATTER()
11838 if (ISD::isConstantSplatVectorAllZeros(Mask.getNode())) in visitVPSCATTER()
11864 ISD::MemIndexType IndexType = MSC->getIndexType(); in visitMSCATTER()
11868 if (ISD::isConstantSplatVectorAllZeros(Mask.getNode())) in visitMSCATTER()
11897 if (ISD::isConstantSplatVectorAllZeros(Mask.getNode())) in visitMSTORE()
11907 ISD::isConstantSplatVectorAllOnes(Mask.getNode())) && in visitMSTORE()
11911 if (N->getOpcode() != ISD::DELETED_NODE) in visitMSTORE()
11919 if (ISD::isConstantSplatVectorAllOnes(Mask.getNode()) && MST->isUnindexed() && in visitMSTORE()
11945 if (N->getOpcode() != ISD::DELETED_NODE) in visitMSTORE()
11955 if ((Value.getOpcode() == ISD::TRUNCATE) && Value->hasOneUse() && in visitMSTORE()
11993 ISD::MemIndexType IndexType = MGT->getIndexType(); in visitVPGATHER()
12021 ISD::MemIndexType IndexType = MGT->getIndexType(); in visitMGATHER()
12025 if (ISD::isConstantSplatVectorAllZeros(Mask.getNode())) in visitMGATHER()
12051 if (ISD::isConstantSplatVectorAllZeros(Mask.getNode())) in visitMLOAD()
12056 if (ISD::isConstantSplatVectorAllOnes(Mask.getNode()) && MLD->isUnindexed() && in visitMLOAD()
12057 !MLD->isExpandingLoad() && MLD->getExtensionType() == ISD::NON_EXTLOAD) { in visitMLOAD()
12097 !ISD::isBuildVectorOfConstantSDNodes(N1.getNode()) || in foldVSelectOfConstants()
12098 !ISD::isBuildVectorOfConstantSDNodes(N2.getNode())) in foldVSelectOfConstants()
12129 auto ExtendOpcode = AllAddOne ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND; in foldVSelectOfConstants()
12131 return DAG.getNode(ISD::ADD, DL, VT, ExtendedCond, N2); in foldVSelectOfConstants()
12136 if (ISD::isConstantSplatVector(N1.getNode(), Pow2C) && Pow2C.isPowerOf2() && in foldVSelectOfConstants()
12140 return DAG.getNode(ISD::SHL, DL, VT, ZextCond, ShAmtC); in foldVSelectOfConstants()
12175 if (N0.getOpcode() == ISD::SETCC) { in visitVSELECT()
12177 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in visitVSELECT()
12179 bool RHSIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode()); in visitVSELECT()
12181 if (((RHSIsAllZeros && (CC == ISD::SETGT || CC == ISD::SETGE)) || in visitVSELECT()
12182 (ISD::isBuildVectorAllOnes(RHS.getNode()) && CC == ISD::SETGT)) && in visitVSELECT()
12183 N1 == LHS && N2.getOpcode() == ISD::SUB && N1 == N2.getOperand(1)) in visitVSELECT()
12184 isAbs = ISD::isBuildVectorAllZeros(N2.getOperand(0).getNode()); in visitVSELECT()
12185 else if ((RHSIsAllZeros && (CC == ISD::SETLT || CC == ISD::SETLE)) && in visitVSELECT()
12186 N2 == LHS && N1.getOpcode() == ISD::SUB && N2 == N1.getOperand(1)) in visitVSELECT()
12187 isAbs = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode()); in visitVSELECT()
12190 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) in visitVSELECT()
12191 return DAG.getNode(ISD::ABS, DL, VT, LHS); in visitVSELECT()
12193 SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, LHS, in visitVSELECT()
12196 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift); in visitVSELECT()
12199 return DAG.getNode(ISD::XOR, DL, VT, Add, Shift); in visitVSELECT()
12230 auto LoadExtOpcode = IsSigned ? ISD::SEXTLOAD : ISD::ZEXTLOAD; in visitVSELECT()
12231 if (LHS.getOpcode() == ISD::LOAD && LHS.hasOneUse() && in visitVSELECT()
12234 TLI.isOperationLegalOrCustom(ISD::SETCC, WideVT)) { in visitVSELECT()
12239 auto ExtOpcode = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in visitVSELECT()
12253 if (N1.getOpcode() == ISD::SUB && N2.getOpcode() == ISD::SUB && in visitVSELECT()
12257 unsigned ABDOpc = IsSigned ? ISD::ABDS : ISD::ABDU; in visitVSELECT()
12260 case ISD::SETGT: in visitVSELECT()
12261 case ISD::SETGE: in visitVSELECT()
12262 case ISD::SETUGT: in visitVSELECT()
12263 case ISD::SETUGE: in visitVSELECT()
12267 case ISD::SETLT: in visitVSELECT()
12268 case ISD::SETLE: in visitVSELECT()
12269 case ISD::SETULT: in visitVSELECT()
12270 case ISD::SETULE: in visitVSELECT()
12281 if (hasOperation(ISD::UADDSAT, VT)) { in visitVSELECT()
12285 ISD::CondCode SatCC = CC; in visitVSELECT()
12286 if (ISD::isConstantSplatVectorAllOnes(N1.getNode())) { in visitVSELECT()
12288 SatCC = ISD::getSetCCInverse(SatCC, VT.getScalarType()); in visitVSELECT()
12289 } else if (ISD::isConstantSplatVectorAllOnes(N2.getNode())) { in visitVSELECT()
12293 if (Other && Other.getOpcode() == ISD::ADD) { in visitVSELECT()
12298 if (SatCC == ISD::SETUGE) { in visitVSELECT()
12300 SatCC = ISD::SETULE; in visitVSELECT()
12306 if (SatCC == ISD::SETULE && Other == CondRHS && in visitVSELECT()
12308 return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
12311 (OpRHS.getOpcode() == ISD::BUILD_VECTOR || in visitVSELECT()
12312 OpRHS.getOpcode() == ISD::SPLAT_VECTOR) && in visitVSELECT()
12320 if (SatCC == ISD::SETULE && in visitVSELECT()
12321 ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUADDSAT)) in visitVSELECT()
12322 return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
12328 if (hasOperation(ISD::USUBSAT, VT)) { in visitVSELECT()
12332 ISD::CondCode SatCC = CC; in visitVSELECT()
12333 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) { in visitVSELECT()
12335 SatCC = ISD::getSetCCInverse(SatCC, VT.getScalarType()); in visitVSELECT()
12336 } else if (ISD::isConstantSplatVectorAllZeros(N2.getNode())) { in visitVSELECT()
12344 if (Other && Other.getOpcode() == ISD::TRUNCATE && in visitVSELECT()
12345 Other.getOperand(0).getOpcode() == ISD::SUB && in visitVSELECT()
12346 (SatCC == ISD::SETUGE || SatCC == ISD::SETUGT)) { in visitVSELECT()
12349 if (LHS == OpLHS && RHS == OpRHS && LHS.getOpcode() == ISD::ZERO_EXTEND) in visitVSELECT()
12363 if ((SatCC == ISD::SETUGE || SatCC == ISD::SETUGT) && in visitVSELECT()
12364 Other.getOpcode() == ISD::SUB && OpRHS == CondRHS) in visitVSELECT()
12365 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
12367 if (OpRHS.getOpcode() == ISD::BUILD_VECTOR || in visitVSELECT()
12368 OpRHS.getOpcode() == ISD::SPLAT_VECTOR) { in visitVSELECT()
12369 if (CondRHS.getOpcode() == ISD::BUILD_VECTOR || in visitVSELECT()
12370 CondRHS.getOpcode() == ISD::SPLAT_VECTOR) { in visitVSELECT()
12379 if (SatCC == ISD::SETUGT && Other.getOpcode() == ISD::ADD && in visitVSELECT()
12380 ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUSUBSAT, in visitVSELECT()
12383 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
12392 if (SatCC == ISD::SETLT && Other.getOpcode() == ISD::XOR && in visitVSELECT()
12393 ISD::isConstantSplatVector(OpRHS.getNode(), SplatValue) && in visitVSELECT()
12394 ISD::isConstantSplatVectorAllZeros(CondRHS.getNode()) && in visitVSELECT()
12399 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
12412 if (ISD::isConstantSplatVectorAllOnes(N0.getNode())) in visitVSELECT()
12415 if (ISD::isConstantSplatVectorAllZeros(N0.getNode())) in visitVSELECT()
12421 if (N1.getOpcode() == ISD::CONCAT_VECTORS && in visitVSELECT()
12422 N2.getOpcode() == ISD::CONCAT_VECTORS && in visitVSELECT()
12423 ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) { in visitVSELECT()
12431 if (hasOperation(ISD::SRA, VT)) in visitVSELECT()
12447 ISD::CondCode CC = cast<CondCodeSDNode>(N4)->get(); in visitSELECT_CC()
12454 if (CC == ISD::SETEQ && !LegalTypes && N0.getValueType() == MVT::i1 && in visitSELECT_CC()
12474 if (SCC.getOpcode() == ISD::SETCC) { in visitSELECT_CC()
12476 ISD::SELECT_CC, SDLoc(N), N2.getValueType(), SCC.getOperand(0), in visitSELECT_CC()
12496 N->hasOneUse() && N->use_begin()->getOpcode() == ISD::BRCOND; in visitSETCC()
12498 ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(2))->get(); in visitSETCC()
12507 if (PreferSetCC && Combined.getOpcode() != ISD::SETCC) { in visitSETCC()
12532 if (Cond == ISD::SETNE || Cond == ISD::SETEQ) { in visitSETCC()
12534 return A.getOpcode() == ISD::AND && in visitSETCC()
12535 (B.getOpcode() == ISD::SRL || B.getOpcode() == ISD::SHL) && in visitSETCC()
12539 return (B.getOpcode() == ISD::ROTL || B.getOpcode() == ISD::ROTR) && in visitSETCC()
12592 ShiftOpc == ISD::SHL ? (~*AndCMask).isMask() : AndCMask->isMask(); in visitSETCC()
12606 if (NewShiftOpc == ISD::SHL || NewShiftOpc == ISD::SRL) { in visitSETCC()
12608 NewShiftOpc == ISD::SHL in visitSETCC()
12614 DAG.getNode(ISD::AND, DL, OpVT, ShiftOrRotate.getOperand(0), in visitSETCC()
12636 return DAG.getNode(ISD::SETCC, SDLoc(N), N->getVTList(), LHS, RHS, Cond); in visitSETCCCARRY()
12656 ISD::LoadExtType LoadExt = Load->getExtensionType(); in isCompatibleLoad()
12657 if (LoadExt == ISD::NON_EXTLOAD || LoadExt == ISD::EXTLOAD) in isCompatibleLoad()
12662 if ((LoadExt == ISD::SEXTLOAD && ExtOpcode != ISD::SIGN_EXTEND) || in isCompatibleLoad()
12663 (LoadExt == ISD::ZEXTLOAD && ExtOpcode != ISD::ZERO_EXTEND)) in isCompatibleLoad()
12683 assert((Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND || in tryToFoldExtendSelectLoad()
12684 Opcode == ISD::ANY_EXTEND) && in tryToFoldExtendSelectLoad()
12687 if (!(N0->getOpcode() == ISD::SELECT || N0->getOpcode() == ISD::VSELECT) || in tryToFoldExtendSelectLoad()
12696 auto ExtLoadOpcode = ISD::EXTLOAD; in tryToFoldExtendSelectLoad()
12697 if (Opcode == ISD::SIGN_EXTEND) in tryToFoldExtendSelectLoad()
12698 ExtLoadOpcode = ISD::SEXTLOAD; in tryToFoldExtendSelectLoad()
12699 else if (Opcode == ISD::ZERO_EXTEND) in tryToFoldExtendSelectLoad()
12700 ExtLoadOpcode = ISD::ZEXTLOAD; in tryToFoldExtendSelectLoad()
12708 (N0->getOpcode() == ISD::VSELECT && Level >= AfterLegalizeTypes && in tryToFoldExtendSelectLoad()
12709 TLI.getOperationAction(ISD::VSELECT, VT) != TargetLowering::Legal)) in tryToFoldExtendSelectLoad()
12730 assert((ISD::isExtOpcode(Opcode) || ISD::isExtVecInRegOpcode(Opcode)) && in tryToFoldExtendOfConstant()
12742 if (N0->getOpcode() == ISD::SELECT) { in tryToFoldExtendOfConstant()
12746 (Opcode != ISD::ZERO_EXTEND || !TLI.isZExtFree(N0.getValueType(), VT))) { in tryToFoldExtendOfConstant()
12757 if (FoldOpc == ISD::ANY_EXTEND) in tryToFoldExtendOfConstant()
12758 FoldOpc = ISD::SIGN_EXTEND; in tryToFoldExtendOfConstant()
12770 ISD::isBuildVectorOfConstantSDNodes(N0.getNode()))) in tryToFoldExtendOfConstant()
12782 if (Opcode == ISD::ANY_EXTEND || Opcode == ISD::ANY_EXTEND_VECTOR_INREG) in tryToFoldExtendOfConstant()
12793 if (Opcode == ISD::SIGN_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG) in tryToFoldExtendOfConstant()
12820 if (ExtOpc != ISD::ANY_EXTEND && User->getOpcode() == ISD::SETCC) { in ExtendUsesToFormExtLoad()
12821 ISD::CondCode CC = cast<CondCodeSDNode>(User->getOperand(2))->get(); in ExtendUsesToFormExtLoad()
12822 if (ExtOpc == ISD::ZERO_EXTEND && ISD::isSignedIntSetCC(CC)) in ExtendUsesToFormExtLoad()
12843 if (User->getOpcode() == ISD::CopyToReg) in ExtendUsesToFormExtLoad()
12852 if (Use.getResNo() == 0 && Use.getUser()->getOpcode() == ISD::CopyToReg) { in ExtendUsesToFormExtLoad()
12867 ISD::NodeType ExtType) { in ExtendSetCCUses()
12882 CombineTo(SetCC, DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops)); in ExtendSetCCUses()
12892 assert((N->getOpcode() == ISD::SIGN_EXTEND || in CombineExtLoad()
12893 N->getOpcode() == ISD::ZERO_EXTEND) && in CombineExtLoad()
12913 if (N0->getOpcode() != ISD::LOAD) in CombineExtLoad()
12918 if (!ISD::isNON_EXTLoad(LN0) || !ISD::isUNINDEXEDLoad(LN0) || in CombineExtLoad()
12928 ISD::LoadExtType ExtType = in CombineExtLoad()
12929 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD; in CombineExtLoad()
12968 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); in CombineExtLoad()
12969 SDValue NewValue = DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Loads); in CombineExtLoad()
12979 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), NewValue); in CombineExtLoad()
12980 ExtendSetCCUses(SetCCs, N0, NewValue, (ISD::NodeType)N->getOpcode()); in CombineExtLoad()
12988 assert(N->getOpcode() == ISD::ZERO_EXTEND); in CombineZExtLogicopShiftLoad()
12996 if (!ISD::isBitwiseLogicOp(N0.getOpcode()) || in CombineZExtLogicopShiftLoad()
12997 N0.getOperand(1).getOpcode() != ISD::Constant || in CombineZExtLogicopShiftLoad()
13003 if (!(N1.getOpcode() == ISD::SHL || N1.getOpcode() == ISD::SRL) || in CombineZExtLogicopShiftLoad()
13004 N1.getOperand(1).getOpcode() != ISD::Constant || in CombineZExtLogicopShiftLoad()
13013 if (!TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) || in CombineZExtLogicopShiftLoad()
13014 Load->getExtensionType() == ISD::SEXTLOAD || Load->isIndexed()) in CombineZExtLogicopShiftLoad()
13020 if (N1.getOpcode() == ISD::SHL && N0.getOpcode() != ISD::AND) in CombineZExtLogicopShiftLoad()
13028 ISD::ZERO_EXTEND, SetCCs, TLI)) in CombineZExtLogicopShiftLoad()
13032 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(Load), VT, in CombineZExtLogicopShiftLoad()
13045 ExtendSetCCUses(SetCCs, N1.getOperand(0), ExtLoad, ISD::ZERO_EXTEND); in CombineZExtLogicopShiftLoad()
13050 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(Load), in CombineZExtLogicopShiftLoad()
13067 assert((CastOpcode == ISD::SIGN_EXTEND || CastOpcode == ISD::ZERO_EXTEND || in matchVSelectOpSizesWithSetCC()
13068 CastOpcode == ISD::TRUNCATE || CastOpcode == ISD::FP_EXTEND || in matchVSelectOpSizesWithSetCC()
13069 CastOpcode == ISD::FP_ROUND) && in matchVSelectOpSizesWithSetCC()
13076 if (LegalOperations || !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT)) in matchVSelectOpSizesWithSetCC()
13080 if (VSel.getOpcode() != ISD::VSELECT || !VSel.hasOneUse() || in matchVSelectOpSizesWithSetCC()
13081 VSel.getOperand(0).getOpcode() != ISD::SETCC) in matchVSelectOpSizesWithSetCC()
13095 if (CastOpcode == ISD::FP_ROUND) { in matchVSelectOpSizesWithSetCC()
13103 return DAG.getNode(ISD::VSELECT, DL, VT, SetCC, CastA, CastB); in matchVSelectOpSizesWithSetCC()
13111 SDValue N0, ISD::LoadExtType ExtLoadType) { in tryToFoldExtOfExtload()
13113 bool isAExtLoad = (ExtLoadType == ISD::SEXTLOAD) ? ISD::isSEXTLoad(N0Node) in tryToFoldExtOfExtload()
13114 : ISD::isZEXTLoad(N0Node); in tryToFoldExtOfExtload()
13115 if ((!isAExtLoad && !ISD::isEXTLoad(N0Node)) || in tryToFoldExtOfExtload()
13116 !ISD::isUNINDEXEDLoad(N0Node) || !N0.hasOneUse()) in tryToFoldExtOfExtload()
13142 ISD::LoadExtType ExtLoadType, in tryToFoldExtOfLoad()
13143 ISD::NodeType ExtOpc) { in tryToFoldExtOfLoad()
13147 if (!ISD::isNON_EXTLoad(N0.getNode()) || in tryToFoldExtOfLoad()
13148 !ISD::isUNINDEXEDLoad(N0.getNode()) || in tryToFoldExtOfLoad()
13176 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad); in tryToFoldExtOfLoad()
13185 ISD::LoadExtType ExtLoadType, ISD::NodeType ExtOpc) { in tryToFoldExtOfMaskedLoad()
13190 if (!Ld || Ld->getExtensionType() != ISD::NON_EXTLOAD) in tryToFoldExtOfMaskedLoad()
13212 assert((N->getOpcode() == ISD::SIGN_EXTEND || in foldExtendedSignBitTest()
13213 N->getOpcode() == ISD::ZERO_EXTEND) && "Expected sext or zext"); in foldExtendedSignBitTest()
13216 if (LegalOperations || SetCC.getOpcode() != ISD::SETCC || in foldExtendedSignBitTest()
13222 ISD::CondCode CC = cast<CondCodeSDNode>(SetCC.getOperand(2))->get(); in foldExtendedSignBitTest()
13228 if (CC == ISD::SETGT && isAllOnesConstant(Ones) && VT == XVT) { in foldExtendedSignBitTest()
13239 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SRA : ISD::SRL; in foldExtendedSignBitTest()
13248 if (N0.getOpcode() != ISD::SETCC) in foldSextSetcc()
13253 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in foldSextSetcc()
13290 if (N0.hasOneUse() && TLI.isOperationLegalOrCustom(ISD::SETCC, VT) && in foldSextSetcc()
13291 !TLI.isOperationLegalOrCustom(ISD::SETCC, SVT)) { in foldSextSetcc()
13292 bool IsSignedCmp = ISD::isSignedIntSetCC(CC); in foldSextSetcc()
13293 unsigned LoadOpcode = IsSignedCmp ? ISD::SEXTLOAD : ISD::ZEXTLOAD; in foldSextSetcc()
13294 unsigned ExtOpcode = IsSignedCmp ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in foldSextSetcc()
13305 if (!(ISD::isNON_EXTLoad(V.getNode()) && in foldSextSetcc()
13306 ISD::isUNINDEXEDLoad(V.getNode()) && in foldSextSetcc()
13362 (!LegalOperations || TLI.isOperationLegal(ISD::SETCC, N00VT))) { in foldSextSetcc()
13389 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) in visitSIGN_EXTEND()
13390 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N0.getOperand(0)); in visitSIGN_EXTEND()
13394 if (N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG || in visitSIGN_EXTEND()
13395 N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG) in visitSIGN_EXTEND()
13396 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, SDLoc(N), VT, in visitSIGN_EXTEND()
13400 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) { in visitSIGN_EXTEND()
13403 if ((N00.getOpcode() == ISD::TRUNCATE || TLI.isTruncateFree(N00, ExtVT)) && in visitSIGN_EXTEND()
13405 SDValue T = DAG.getNode(ISD::TRUNCATE, DL, ExtVT, N00); in visitSIGN_EXTEND()
13406 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, T); in visitSIGN_EXTEND()
13410 if (N0.getOpcode() == ISD::TRUNCATE) { in visitSIGN_EXTEND()
13440 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op); in visitSIGN_EXTEND()
13445 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op); in visitSIGN_EXTEND()
13449 if (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, in visitSIGN_EXTEND()
13452 Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N0), VT, Op); in visitSIGN_EXTEND()
13454 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, Op); in visitSIGN_EXTEND()
13455 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Op, in visitSIGN_EXTEND()
13463 ISD::SEXTLOAD, ISD::SIGN_EXTEND)) in visitSIGN_EXTEND()
13468 ISD::SEXTLOAD, ISD::SIGN_EXTEND)) in visitSIGN_EXTEND()
13478 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::SEXTLOAD)) in visitSIGN_EXTEND()
13483 if (ISD::isBitwiseLogicOp(N0.getOpcode()) && in visitSIGN_EXTEND()
13485 N0.getOperand(1).getOpcode() == ISD::Constant && in visitSIGN_EXTEND()
13489 if (TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, MemVT) && in visitSIGN_EXTEND()
13490 LN00->getExtensionType() != ISD::ZEXTLOAD && LN00->isUnindexed()) { in visitSIGN_EXTEND()
13493 ISD::SIGN_EXTEND, SetCCs, TLI); in visitSIGN_EXTEND()
13495 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN00), VT, in visitSIGN_EXTEND()
13502 ExtendSetCCUses(SetCCs, N0.getOperand(0), ExtLoad, ISD::SIGN_EXTEND); in visitSIGN_EXTEND()
13509 DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And); in visitSIGN_EXTEND()
13515 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00), in visitSIGN_EXTEND()
13532 (!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) && in visitSIGN_EXTEND()
13536 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0, Flags); in visitSIGN_EXTEND()
13544 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() && in visitSIGN_EXTEND()
13546 N0.getOperand(1).getOpcode() == ISD::ZERO_EXTEND && in visitSIGN_EXTEND()
13547 TLI.isOperationLegalOrCustom(ISD::SUB, VT)) { in visitSIGN_EXTEND()
13553 if (N0.getOpcode() == ISD::ADD && N0.hasOneUse() && in visitSIGN_EXTEND()
13555 N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND && in visitSIGN_EXTEND()
13556 TLI.isOperationLegalOrCustom(ISD::ADD, VT)) { in visitSIGN_EXTEND()
13558 return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT)); in visitSIGN_EXTEND()
13564 (!LegalOperations || (TLI.isOperationLegal(ISD::ZERO_EXTEND, VT) && in visitSIGN_EXTEND()
13565 TLI.isOperationLegal(ISD::ADD, VT)))) { in visitSIGN_EXTEND()
13577 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NewXor); in visitSIGN_EXTEND()
13580 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitSIGN_EXTEND()
13581 return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT)); in visitSIGN_EXTEND()
13594 assert((Extend->getOpcode() == ISD::ZERO_EXTEND || in widenCtPop()
13595 Extend->getOpcode() == ISD::ANY_EXTEND) && "Expected extend op"); in widenCtPop()
13598 if (CtPop.getOpcode() != ISD::CTPOP || !CtPop.hasOneUse()) in widenCtPop()
13603 if (TLI.isOperationLegalOrCustom(ISD::CTPOP, CtPop.getValueType()) || in widenCtPop()
13604 !TLI.isOperationLegalOrCustom(ISD::CTPOP, VT)) in widenCtPop()
13610 return DAG.getNode(ISD::CTPOP, DL, VT, NewZext); in widenCtPop()
13616 assert(Extend->getOpcode() == ISD::ZERO_EXTEND && "Expected zero extend."); in widenAbs()
13623 if (Abs.getOpcode() != ISD::ABS || !Abs.hasOneUse()) in widenAbs()
13635 DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Abs), LegalVT, Abs.getOperand(0)); in widenAbs()
13636 SDValue NewAbs = DAG.getNode(ISD::ABS, SDLoc(Abs), LegalVT, SExt); in widenAbs()
13658 if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) in visitZERO_EXTEND()
13659 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitZERO_EXTEND()
13663 if (N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG || in visitZERO_EXTEND()
13664 N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) in visitZERO_EXTEND()
13665 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(N), VT, in visitZERO_EXTEND()
13690 if (N0.getOpcode() == ISD::TRUNCATE) { in visitZERO_EXTEND()
13709 if (!LegalOperations || (TLI.isOperationLegal(ISD::AND, SrcVT) && in visitZERO_EXTEND()
13710 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) { in visitZERO_EXTEND()
13721 if (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT)) { in visitZERO_EXTEND()
13734 if (N0.getOpcode() == ISD::AND && in visitZERO_EXTEND()
13735 N0.getOperand(0).getOpcode() == ISD::TRUNCATE && in visitZERO_EXTEND()
13736 N0.getOperand(1).getOpcode() == ISD::Constant && in visitZERO_EXTEND()
13742 return DAG.getNode(ISD::AND, DL, VT, in visitZERO_EXTEND()
13749 ISD::ZEXTLOAD, ISD::ZERO_EXTEND)) in visitZERO_EXTEND()
13754 ISD::ZEXTLOAD, ISD::ZERO_EXTEND)) in visitZERO_EXTEND()
13766 if (ISD::isBitwiseLogicOp(N0.getOpcode()) && !TLI.isZExtFree(N0, VT) && in visitZERO_EXTEND()
13768 N0.getOperand(1).getOpcode() == ISD::Constant && in visitZERO_EXTEND()
13772 if (TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) && in visitZERO_EXTEND()
13773 LN00->getExtensionType() != ISD::SEXTLOAD && LN00->isUnindexed()) { in visitZERO_EXTEND()
13777 if (N0.getOpcode() == ISD::AND) { in visitZERO_EXTEND()
13787 ISD::ZERO_EXTEND, SetCCs, TLI); in visitZERO_EXTEND()
13789 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN00), VT, in visitZERO_EXTEND()
13796 ExtendSetCCUses(SetCCs, N0.getOperand(0), ExtLoad, ISD::ZERO_EXTEND); in visitZERO_EXTEND()
13803 DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And); in visitZERO_EXTEND()
13809 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00), in visitZERO_EXTEND()
13825 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::ZEXTLOAD)) in visitZERO_EXTEND()
13831 if (N0.getOpcode() == ISD::SETCC) { in visitZERO_EXTEND()
13849 SDValue VSetCC = DAG.getNode(ISD::SETCC, DL, VT, N0.getOperand(0), in visitZERO_EXTEND()
13859 DAG.getNode(ISD::SETCC, DL, MatchingVectorType, N0.getOperand(0), in visitZERO_EXTEND()
13873 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, SCC); in visitZERO_EXTEND()
13877 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) && in visitZERO_EXTEND()
13882 if (ShVal.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse()) { in visitZERO_EXTEND()
13883 if (N0.getOpcode() == ISD::SHL) { in visitZERO_EXTEND()
13895 ShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShAmt); in visitZERO_EXTEND()
13898 DAG.getNode(ISD::ZERO_EXTEND, DL, VT, ShVal), ShAmt); in visitZERO_EXTEND()
13932 if (N0.getOpcode() == ISD::ANY_EXTEND || in visitANY_EXTEND()
13933 N0.getOpcode() == ISD::ZERO_EXTEND || in visitANY_EXTEND()
13934 N0.getOpcode() == ISD::SIGN_EXTEND) in visitANY_EXTEND()
13940 if (N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG || in visitANY_EXTEND()
13941 N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG || in visitANY_EXTEND()
13942 N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG) in visitANY_EXTEND()
13947 if (N0.getOpcode() == ISD::TRUNCATE) { in visitANY_EXTEND()
13960 if (N0.getOpcode() == ISD::TRUNCATE) in visitANY_EXTEND()
13965 if (N0.getOpcode() == ISD::AND && in visitANY_EXTEND()
13966 N0.getOperand(0).getOpcode() == ISD::TRUNCATE && in visitANY_EXTEND()
13967 N0.getOperand(1).getOpcode() == ISD::Constant && in visitANY_EXTEND()
13971 SDValue Y = DAG.getNode(ISD::ANY_EXTEND, DL, VT, N0.getOperand(1)); in visitANY_EXTEND()
13973 return DAG.getNode(ISD::AND, DL, VT, X, Y); in visitANY_EXTEND()
13983 ISD::ZEXTLOAD, ISD::ZERO_EXTEND)) in visitANY_EXTEND()
13985 } else if (ISD::isNON_EXTLoad(N0.getNode()) && in visitANY_EXTEND()
13986 ISD::isUNINDEXEDLoad(N0.getNode()) && in visitANY_EXTEND()
13987 TLI.isLoadExtLegal(ISD::EXTLOAD, VT, N0.getValueType())) { in visitANY_EXTEND()
13992 ExtendUsesToFormExtLoad(VT, N, N0, ISD::ANY_EXTEND, SetCCs, TLI); in visitANY_EXTEND()
13995 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, in visitANY_EXTEND()
13998 ExtendSetCCUses(SetCCs, N0, ExtLoad, ISD::ANY_EXTEND); in visitANY_EXTEND()
14007 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad); in visitANY_EXTEND()
14017 if (N0.getOpcode() == ISD::LOAD && !ISD::isNON_EXTLoad(N0.getNode()) && in visitANY_EXTEND()
14018 ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) { in visitANY_EXTEND()
14020 ISD::LoadExtType ExtType = LN0->getExtensionType(); in visitANY_EXTEND()
14033 if (N0.getOpcode() == ISD::SETCC) { in visitANY_EXTEND()
14097 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && in visitAssertExt()
14111 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewAssert); in visitAssertExt()
14117 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && in visitAssertExt()
14118 N0.getOperand(0).getOpcode() == ISD::AssertSext && in visitAssertExt()
14119 Opcode == ISD::AssertZext) { in visitAssertExt()
14126 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewAssert); in visitAssertExt()
14151 case ISD::ADD: in visitAssertAlign()
14152 case ISD::SUB: { in visitAssertAlign()
14178 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD; in reduceLoadWidth()
14198 if (Opc == ISD::SIGN_EXTEND_INREG) { in reduceLoadWidth()
14199 ExtType = ISD::SEXTLOAD; in reduceLoadWidth()
14201 } else if (Opc == ISD::SRL || Opc == ISD::SRA) { in reduceLoadWidth()
14219 ExtType = Opc == ISD::SRL ? ISD::ZEXTLOAD : ISD::SEXTLOAD; in reduceLoadWidth()
14225 if ((LN->getExtensionType() == ISD::SEXTLOAD || in reduceLoadWidth()
14226 LN->getExtensionType() == ISD::ZEXTLOAD) && in reduceLoadWidth()
14229 } else if (Opc == ISD::AND) { in reduceLoadWidth()
14245 ExtType = ISD::ZEXTLOAD; in reduceLoadWidth()
14254 if (Opc == ISD::SRL || N0.getOpcode() == ISD::SRL) { in reduceLoadWidth()
14255 SDValue SRL = Opc == ISD::SRL ? SDValue(N, 0) : N0; in reduceLoadWidth()
14282 if (LN->getExtensionType() == ISD::SEXTLOAD) in reduceLoadWidth()
14292 if (ExtType == ISD::SEXTLOAD) in reduceLoadWidth()
14295 ExtType = ISD::ZEXTLOAD; in reduceLoadWidth()
14302 if (SRL.hasOneUse() && Mask->getOpcode() == ISD::AND && in reduceLoadWidth()
14324 if (ShAmt == 0 && N0.getOpcode() == ISD::SHL && N0.hasOneUse() && in reduceLoadWidth()
14366 if (ExtType == ISD::NON_EXTLOAD) in reduceLoadWidth()
14393 Result = DAG.getNode(ISD::SHL, DL, VT, in reduceLoadWidth()
14403 Result = DAG.getNode(ISD::SHL, DL, VT, Result, ShiftC); in reduceLoadWidth()
14425 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0, N1); in visitSIGN_EXTEND_INREG()
14432 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG && in visitSIGN_EXTEND_INREG()
14434 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0.getOperand(0), in visitSIGN_EXTEND_INREG()
14441 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) { in visitSIGN_EXTEND_INREG()
14446 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT))) in visitSIGN_EXTEND_INREG()
14447 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00); in visitSIGN_EXTEND_INREG()
14453 if (ISD::isExtVecInRegOpcode(N0.getOpcode())) { in visitSIGN_EXTEND_INREG()
14458 bool IsZext = N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG; in visitSIGN_EXTEND_INREG()
14464 TLI.isOperationLegal(ISD::SIGN_EXTEND_VECTOR_INREG, VT))) in visitSIGN_EXTEND_INREG()
14465 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, SDLoc(N), VT, N00); in visitSIGN_EXTEND_INREG()
14470 if (N0.getOpcode() == ISD::ZERO_EXTEND) { in visitSIGN_EXTEND_INREG()
14473 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT))) in visitSIGN_EXTEND_INREG()
14474 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00); in visitSIGN_EXTEND_INREG()
14494 if (N0.getOpcode() == ISD::SRL) { in visitSIGN_EXTEND_INREG()
14501 return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0.getOperand(0), in visitSIGN_EXTEND_INREG()
14510 if (ISD::isEXTLoad(N0.getNode()) && in visitSIGN_EXTEND_INREG()
14511 ISD::isUNINDEXEDLoad(N0.getNode()) && in visitSIGN_EXTEND_INREG()
14515 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT))) { in visitSIGN_EXTEND_INREG()
14517 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
14528 if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && in visitSIGN_EXTEND_INREG()
14532 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT))) { in visitSIGN_EXTEND_INREG()
14534 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
14547 Ld->getExtensionType() != ISD::LoadExtType::NON_EXTLOAD && in visitSIGN_EXTEND_INREG()
14548 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT)) { in visitSIGN_EXTEND_INREG()
14552 Ld->getAddressingMode(), ISD::SEXTLOAD, Ld->isExpandingLoad()); in visitSIGN_EXTEND_INREG()
14569 GN0->getMemOperand(), GN0->getIndexType(), ISD::SEXTLOAD); in visitSIGN_EXTEND_INREG()
14579 if (ExtVTBits <= 16 && N0.getOpcode() == ISD::OR) { in visitSIGN_EXTEND_INREG()
14582 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, BSwap, N1); in visitSIGN_EXTEND_INREG()
14589 if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR && N0.hasOneUse() && in visitSIGN_EXTEND_INREG()
14590 ISD::isExtOpcode(N0.getOperand(0).getOpcode())) { in visitSIGN_EXTEND_INREG()
14597 TLI.isOperationLegal(ISD::SIGN_EXTEND, InnerExtVT))) { in visitSIGN_EXTEND_INREG()
14599 DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), InnerExtVT, Extendee); in visitSIGN_EXTEND_INREG()
14600 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT, SignExtExtendee, in visitSIGN_EXTEND_INREG()
14621 assert(ISD::isExtVecInRegOpcode(InregOpcode) && in foldExtendVectorInregToExtendOfSubvector()
14626 if (!Src.hasOneUse() || Src.getOpcode() != ISD::CONCAT_VECTORS) in foldExtendVectorInregToExtendOfSubvector()
14648 return N->getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG in visitEXTEND_VECTOR_INREG()
14677 if (N0.getOpcode() == ISD::TRUNCATE) in visitTRUNCATE()
14678 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0)); in visitTRUNCATE()
14681 if (SDValue C = DAG.FoldConstantArithmetic(ISD::TRUNCATE, SDLoc(N), VT, {N0})) in visitTRUNCATE()
14685 if (N0.getOpcode() == ISD::ZERO_EXTEND || in visitTRUNCATE()
14686 N0.getOpcode() == ISD::SIGN_EXTEND || in visitTRUNCATE()
14687 N0.getOpcode() == ISD::ANY_EXTEND) { in visitTRUNCATE()
14693 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0)); in visitTRUNCATE()
14701 if (!LegalTypes && N0.getOpcode() == ISD::SIGN_EXTEND_INREG && in visitTRUNCATE()
14707 SDValue TrX = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, X); in visitTRUNCATE()
14708 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, TrX, ExtVal); in visitTRUNCATE()
14713 if (N->hasOneUse() && (N->use_begin()->getOpcode() == ISD::ANY_EXTEND)) in visitTRUNCATE()
14726 if (N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitTRUNCATE()
14745 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TrTy, in visitTRUNCATE()
14752 if (N0.getOpcode() == ISD::SELECT && N0.hasOneUse()) { in visitTRUNCATE()
14753 if ((!LegalOperations || TLI.isOperationLegal(ISD::SELECT, SrcVT)) && in visitTRUNCATE()
14757 SDValue TruncOp0 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1)); in visitTRUNCATE()
14758 SDValue TruncOp1 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(2)); in visitTRUNCATE()
14759 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, Cond, TruncOp0, TruncOp1); in visitTRUNCATE()
14764 if (N0.getOpcode() == ISD::SHL && N0.hasOneUse() && in visitTRUNCATE()
14765 (!LegalOperations || TLI.isOperationLegal(ISD::SHL, VT)) && in visitTRUNCATE()
14766 TLI.isTypeDesirableForOp(ISD::SHL, VT)) { in visitTRUNCATE()
14774 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0)); in visitTRUNCATE()
14779 return DAG.getNode(ISD::SHL, SL, VT, Trunc, Amt); in visitTRUNCATE()
14790 if (N0.getOpcode() == ISD::BUILD_VECTOR && !LegalOperations && in visitTRUNCATE()
14799 SDValue TruncOp = DAG.getNode(ISD::TRUNCATE, DL, SVT, Op); in visitTRUNCATE()
14806 if (N0.getOpcode() == ISD::SPLAT_VECTOR && in visitTRUNCATE()
14808 (!LegalOperations || TLI.isOperationLegal(ISD::SPLAT_VECTOR, VT))) { in visitTRUNCATE()
14812 VT, DL, DAG.getNode(ISD::TRUNCATE, DL, SVT, N0->getOperand(0))); in visitTRUNCATE()
14820 N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() && in visitTRUNCATE()
14821 N0.getOperand(0).getOpcode() == ISD::BUILD_VECTOR && in visitTRUNCATE()
14853 if (N0.hasOneUse() && ISD::isUNINDEXEDLoad(N0.getNode())) { in visitTRUNCATE()
14867 if (N0.getOpcode() == ISD::CONCAT_VECTORS && !LegalTypes) { in visitTRUNCATE()
14900 SDValue NV = DAG.getNode(ISD::TRUNCATE, SDLoc(V), VTs[i], V); in visitTRUNCATE()
14904 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Opnds); in visitTRUNCATE()
14912 if (N0.getOpcode() == ISD::BITCAST && !VT.isVector()) { in visitTRUNCATE()
14917 TLI.isOperationLegal(ISD::EXTRACT_VECTOR_ELT, VecSrcVT))) { in visitTRUNCATE()
14921 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, VT, VecSrc, in visitTRUNCATE()
14934 if (!LegalTypes && N0.getOpcode() == ISD::EXTRACT_SUBVECTOR) { in visitTRUNCATE()
14936 if (N00.getOpcode() == ISD::SIGN_EXTEND || in visitTRUNCATE()
14937 N00.getOpcode() == ISD::ZERO_EXTEND || in visitTRUNCATE()
14938 N00.getOpcode() == ISD::ANY_EXTEND) { in visitTRUNCATE()
14941 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N0->getOperand(0)), VT, in visitTRUNCATE()
14954 case ISD::ADD: in visitTRUNCATE()
14955 case ISD::SUB: in visitTRUNCATE()
14956 case ISD::MUL: in visitTRUNCATE()
14957 case ISD::AND: in visitTRUNCATE()
14958 case ISD::OR: in visitTRUNCATE()
14959 case ISD::XOR: in visitTRUNCATE()
14968 SDValue NarrowL = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
14969 SDValue NarrowR = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1)); in visitTRUNCATE()
14974 case ISD::ADDE: in visitTRUNCATE()
14975 case ISD::UADDO_CARRY: in visitTRUNCATE()
14981 if (((!LegalOperations && N0.getOpcode() == ISD::UADDO_CARRY) || in visitTRUNCATE()
14985 SDValue X = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
14986 SDValue Y = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1)); in visitTRUNCATE()
14991 case ISD::USUBSAT: in visitTRUNCATE()
14996 N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND && in visitTRUNCATE()
15011 if (Elt.getOpcode() != ISD::MERGE_VALUES) in getBuildPairElt()
15019 assert(N->getOpcode() == ISD::BUILD_PAIR); in CombineConsecutiveLoads()
15030 if (!LD1 || !LD2 || !ISD::isNON_EXTLoad(LD1) || !ISD::isNON_EXTLoad(LD2) || in CombineConsecutiveLoads()
15038 if ((!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)) && in CombineConsecutiveLoads()
15073 case ISD::AND: in foldBitcastedFPLogic()
15074 FPOpcode = ISD::FABS; in foldBitcastedFPLogic()
15077 case ISD::XOR: in foldBitcastedFPLogic()
15078 FPOpcode = ISD::FNEG; in foldBitcastedFPLogic()
15081 case ISD::OR: in foldBitcastedFPLogic()
15082 FPOpcode = ISD::FABS; in foldBitcastedFPLogic()
15096 if (Op.getOpcode() == ISD::BITCAST && Op.getOperand(0).getValueType() == VT) in foldBitcastedFPLogic()
15099 return FPOpcode == ISD::FABS ? TLI.isFAbsFree(VT) : TLI.isFNegFree(VT); in foldBitcastedFPLogic()
15110 SDValue CastOp0 = DAG.getNode(ISD::BITCAST, SDLoc(N), VT, LogicOp0); in foldBitcastedFPLogic()
15113 if (N0.getOpcode() == ISD::OR) in foldBitcastedFPLogic()
15114 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, FPOp); in foldBitcastedFPLogic()
15138 N0.getOpcode() == ISD::BUILD_VECTOR && N0->hasOneUse() && in visitBITCAST()
15150 TLI.isOperationLegal(ISD::ConstantFP, VT)) || in visitBITCAST()
15152 TLI.isOperationLegal(ISD::Constant, VT))) { in visitBITCAST()
15160 if (N0.getOpcode() == ISD::BITCAST) in visitBITCAST()
15165 if (ISD::isBitwiseLogicOp(N0.getOpcode()) && VT.isInteger() && in visitBITCAST()
15168 return (V.getOpcode() == ISD::BITCAST && in visitBITCAST()
15170 (ISD::isBuildVectorOfConstantSDNodes(V.getNode()) && in visitBITCAST()
15181 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && in visitBITCAST()
15191 TLI.isOperationLegal(ISD::LOAD, VT))) { in visitBITCAST()
15219 if (((N0.getOpcode() == ISD::FNEG && !TLI.isFNegFree(N0.getValueType())) || in visitBITCAST()
15220 (N0.getOpcode() == ISD::FABS && !TLI.isFAbsFree(N0.getValueType()))) && in visitBITCAST()
15232 if (N0.getOpcode() == ISD::FNEG) { in visitBITCAST()
15236 assert(N0.getOpcode() == ISD::FABS); in visitBITCAST()
15238 DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(NewConv), MVT::i64, NewConv, in visitBITCAST()
15242 FlipBit = DAG.getNode(ISD::AND, SDLoc(N0), MVT::i64, Hi, SignBit); in visitBITCAST()
15246 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit); in visitBITCAST()
15248 return DAG.getNode(ISD::XOR, DL, VT, NewConv, FlipBits); in visitBITCAST()
15251 if (N0.getOpcode() == ISD::FNEG) in visitBITCAST()
15252 return DAG.getNode(ISD::XOR, DL, VT, in visitBITCAST()
15254 assert(N0.getOpcode() == ISD::FABS); in visitBITCAST()
15255 return DAG.getNode(ISD::AND, DL, VT, in visitBITCAST()
15270 if (N0.getOpcode() == ISD::FCOPYSIGN && N0->hasOneUse() && in visitBITCAST()
15282 X = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, X); in visitBITCAST()
15288 X = DAG.getNode(ISD::SRL, DL, in visitBITCAST()
15293 X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X); in visitBITCAST()
15303 SDValue XorResult = DAG.getNode(ISD::XOR, SDLoc(N0), VT, Cst, X); in visitBITCAST()
15306 ISD::EXTRACT_ELEMENT, SDLoc(XorResult), MVT::i64, XorResult, in visitBITCAST()
15311 DAG.getNode(ISD::AND, SDLoc(XorResult64), MVT::i64, XorResult64, in visitBITCAST()
15315 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit); in visitBITCAST()
15317 return DAG.getNode(ISD::XOR, SDLoc(N), VT, Cst, FlipBits); in visitBITCAST()
15320 X = DAG.getNode(ISD::AND, SDLoc(X), VT, in visitBITCAST()
15325 Cst = DAG.getNode(ISD::AND, SDLoc(Cst), VT, in visitBITCAST()
15329 return DAG.getNode(ISD::OR, SDLoc(N), VT, X, Cst); in visitBITCAST()
15334 if (N0.getOpcode() == ISD::BUILD_PAIR) in visitBITCAST()
15343 N0->getOpcode() == ISD::VECTOR_SHUFFLE && N0.hasOneUse() && in visitBITCAST()
15351 if (Op.getOpcode() == ISD::BITCAST && in visitBITCAST()
15406 bool AllowMultipleMaybePoisonOperands = N0.getOpcode() == ISD::BUILD_VECTOR || in visitFREEZE()
15407 N0.getOpcode() == ISD::BUILD_PAIR || in visitFREEZE()
15408 N0.getOpcode() == ISD::CONCAT_VECTORS; in visitFREEZE()
15430 if (MaybePoisonOperand.getOpcode() == ISD::UNDEF) in visitFREEZE()
15436 if (FrozenMaybePoisonOperand.getOpcode() == ISD::FREEZE && in visitFREEZE()
15446 if (N->getOpcode() == ISD::DELETED_NODE) in visitFREEZE()
15458 if (Op.getOpcode() == ISD::UNDEF) in visitFREEZE()
15488 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(BV), SrcEltVT, Op); in ConstantFoldBITCASTofBUILD_VECTOR()
15548 assert(N.getOpcode() == ISD::FMUL); in isContractableFMUL()
15579 (!LegalOperations || matcher.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFADDForFMACombine()
15604 unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA; in visitFADDForFMACombine()
15608 return matcher.match(N, ISD::FMA) || matcher.match(N, ISD::FMAD); in visitFADDForFMACombine()
15614 if (!matcher.match(N, ISD::FMUL)) in visitFADDForFMACombine()
15659 if (matcher.match(FMul, ISD::FMUL) && FMul.hasOneUse()) { in visitFADDForFMACombine()
15666 return FMA.getOpcode() == ISD::DELETED_NODE ? SDValue(N, 0) : FMA; in visitFADDForFMACombine()
15676 if (matcher.match(N0, ISD::FP_EXTEND)) { in visitFADDForFMACombine()
15683 matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)), in visitFADDForFMACombine()
15684 matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)), N1); in visitFADDForFMACombine()
15690 if (matcher.match(N1, ISD::FP_EXTEND)) { in visitFADDForFMACombine()
15697 matcher.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(0)), in visitFADDForFMACombine()
15698 matcher.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)), N0); in visitFADDForFMACombine()
15711 matcher.getNode(ISD::FP_EXTEND, SL, VT, U), in visitFADDForFMACombine()
15712 matcher.getNode(ISD::FP_EXTEND, SL, VT, V), Z)); in visitFADDForFMACombine()
15716 if (matcher.match(N02, ISD::FP_EXTEND)) { in visitFADDForFMACombine()
15737 matcher.getNode(ISD::FP_EXTEND, SL, VT, X), in visitFADDForFMACombine()
15738 matcher.getNode(ISD::FP_EXTEND, SL, VT, Y), in visitFADDForFMACombine()
15740 matcher.getNode(ISD::FP_EXTEND, SL, VT, U), in visitFADDForFMACombine()
15741 matcher.getNode(ISD::FP_EXTEND, SL, VT, V), Z)); in visitFADDForFMACombine()
15743 if (N0.getOpcode() == ISD::FP_EXTEND) { in visitFADDForFMACombine()
15761 if (N12.getOpcode() == ISD::FP_EXTEND) { in visitFADDForFMACombine()
15778 if (N1.getOpcode() == ISD::FP_EXTEND) { in visitFADDForFMACombine()
15816 (!LegalOperations || matcher.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFSUBForFMACombine()
15834 unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA; in visitFSUBForFMACombine()
15841 if (!matcher.match(N, ISD::FMUL)) in visitFSUBForFMACombine()
15851 matcher.getNode(ISD::FNEG, SL, VT, Z)); in visitFSUBForFMACombine()
15862 matcher.getNode(ISD::FNEG, SL, VT, YZ.getOperand(0)), in visitFSUBForFMACombine()
15888 if (matcher.match(N0, ISD::FNEG) && isContractableFMUL(N0.getOperand(0)) && in visitFSUBForFMACombine()
15893 matcher.getNode(ISD::FNEG, SL, VT, N00), N01, in visitFSUBForFMACombine()
15894 matcher.getNode(ISD::FNEG, SL, VT, N1)); in visitFSUBForFMACombine()
15901 if (matcher.match(N0, ISD::FP_EXTEND)) { in visitFSUBForFMACombine()
15908 matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)), in visitFSUBForFMACombine()
15909 matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)), in visitFSUBForFMACombine()
15910 matcher.getNode(ISD::FNEG, SL, VT, N1)); in visitFSUBForFMACombine()
15917 if (matcher.match(N1, ISD::FP_EXTEND)) { in visitFSUBForFMACombine()
15925 ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
15926 matcher.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(0))), in visitFSUBForFMACombine()
15927 matcher.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)), N0); in visitFSUBForFMACombine()
15937 if (matcher.match(N0, ISD::FP_EXTEND)) { in visitFSUBForFMACombine()
15939 if (matcher.match(N00, ISD::FNEG)) { in visitFSUBForFMACombine()
15945 ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
15948 matcher.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)), in visitFSUBForFMACombine()
15949 matcher.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)), in visitFSUBForFMACombine()
15961 if (matcher.match(N0, ISD::FNEG)) { in visitFSUBForFMACombine()
15963 if (matcher.match(N00, ISD::FP_EXTEND)) { in visitFSUBForFMACombine()
15969 ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
15972 matcher.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)), in visitFSUBForFMACombine()
15973 matcher.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)), in visitFSUBForFMACombine()
15989 return matcher.match(N, ISD::FMA) || matcher.match(N, ISD::FMAD); in visitFSUBForFMACombine()
16005 matcher.getNode(ISD::FNEG, SL, VT, N1))); in visitFSUBForFMACombine()
16017 matcher.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)), in visitFSUBForFMACombine()
16020 matcher.getNode(ISD::FNEG, SL, VT, N20), N21, N0)); in visitFSUBForFMACombine()
16027 if (matcher.match(N02, ISD::FP_EXTEND)) { in visitFSUBForFMACombine()
16036 matcher.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(0)), in visitFSUBForFMACombine()
16037 matcher.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(1)), in visitFSUBForFMACombine()
16038 matcher.getNode(ISD::FNEG, SL, VT, N1))); in visitFSUBForFMACombine()
16049 if (matcher.match(N0, ISD::FP_EXTEND)) { in visitFSUBForFMACombine()
16058 matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)), in visitFSUBForFMACombine()
16059 matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)), in visitFSUBForFMACombine()
16062 matcher.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(0)), in visitFSUBForFMACombine()
16063 matcher.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(1)), in visitFSUBForFMACombine()
16064 matcher.getNode(ISD::FNEG, SL, VT, N1))); in visitFSUBForFMACombine()
16071 if (isFusedOp(N1) && matcher.match(N1.getOperand(2), ISD::FP_EXTEND) && in visitFSUBForFMACombine()
16081 matcher.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)), in visitFSUBForFMACombine()
16085 matcher.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
16086 matcher.getNode(ISD::FP_EXTEND, SL, VT, N1200)), in visitFSUBForFMACombine()
16087 matcher.getNode(ISD::FP_EXTEND, SL, VT, N1201), N0)); in visitFSUBForFMACombine()
16097 if (matcher.match(N1, ISD::FP_EXTEND) && isFusedOp(N1.getOperand(0))) { in visitFSUBForFMACombine()
16109 matcher.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
16110 matcher.getNode(ISD::FP_EXTEND, SL, VT, N100)), in visitFSUBForFMACombine()
16111 matcher.getNode(ISD::FP_EXTEND, SL, VT, N101), in visitFSUBForFMACombine()
16114 matcher.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
16115 matcher.getNode(ISD::FP_EXTEND, SL, VT, N1020)), in visitFSUBForFMACombine()
16116 matcher.getNode(ISD::FP_EXTEND, SL, VT, N1021), N0)); in visitFSUBForFMACombine()
16133 assert(N->getOpcode() == ISD::FMUL && "Expected FMUL Operation"); in visitFMULForFMADistributiveCombine()
16139 SDValue FAdd = N0.getOpcode() == ISD::FADD ? N0 : N1; in visitFMULForFMADistributiveCombine()
16147 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFMULForFMADistributiveCombine()
16159 unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA; in visitFMULForFMADistributiveCombine()
16165 if (X.getOpcode() == ISD::FADD && (Aggressive || X->hasOneUse())) { in visitFMULForFMADistributiveCombine()
16172 DAG.getNode(ISD::FNEG, SL, VT, Y)); in visitFMULForFMADistributiveCombine()
16188 if (X.getOpcode() == ISD::FSUB && (Aggressive || X->hasOneUse())) { in visitFMULForFMADistributiveCombine()
16192 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y, in visitFMULForFMADistributiveCombine()
16196 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y, in visitFMULForFMADistributiveCombine()
16197 DAG.getNode(ISD::FNEG, SL, VT, Y)); in visitFMULForFMADistributiveCombine()
16202 DAG.getNode(ISD::FNEG, SL, VT, Y)); in visitFMULForFMADistributiveCombine()
16224 if (Fused.getOpcode() != ISD::DELETED_NODE) in visitVP_FADD()
16246 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FADD, DL, VT, {N0, N1})) in visitFADD()
16251 return DAG.getNode(ISD::FADD, DL, VT, N1, N0); in visitFADD()
16268 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) in visitFADD()
16271 return DAG.getNode(ISD::FSUB, DL, VT, N0, NegN1); in visitFADD()
16274 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) in visitFADD()
16277 return DAG.getNode(ISD::FSUB, DL, VT, N1, NegN0); in visitFADD()
16280 if (!FMul.hasOneUse() || FMul.getOpcode() != ISD::FMUL) in visitFADD()
16289 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B); in visitFADD()
16290 return DAG.getNode(ISD::FSUB, DL, VT, N1, Add); in visitFADD()
16295 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B); in visitFADD()
16296 return DAG.getNode(ISD::FSUB, DL, VT, N0, Add); in visitFADD()
16306 if (N0.getOpcode() == ISD::FNEG && N0.getOperand(0) == N1) in visitFADD()
16310 if (N1.getOpcode() == ISD::FNEG && N1.getOperand(0) == N0) in visitFADD()
16321 if (N1CFP && N0.getOpcode() == ISD::FADD && in visitFADD()
16323 SDValue NewC = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), N1); in visitFADD()
16324 return DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(0), NewC); in visitFADD()
16330 if (TLI.isOperationLegalOrCustom(ISD::FMUL, VT) && !N0CFP && !N1CFP) { in visitFADD()
16331 if (N0.getOpcode() == ISD::FMUL) { in visitFADD()
16339 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), in visitFADD()
16341 return DAG.getNode(ISD::FMUL, DL, VT, N1, NewCFP); in visitFADD()
16345 if (CFP01 && !CFP00 && N1.getOpcode() == ISD::FADD && in visitFADD()
16348 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), in visitFADD()
16350 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), NewCFP); in visitFADD()
16354 if (N1.getOpcode() == ISD::FMUL) { in visitFADD()
16362 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1), in visitFADD()
16364 return DAG.getNode(ISD::FMUL, DL, VT, N0, NewCFP); in visitFADD()
16368 if (CFP11 && !CFP10 && N0.getOpcode() == ISD::FADD && in visitFADD()
16371 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1), in visitFADD()
16373 return DAG.getNode(ISD::FMUL, DL, VT, N1.getOperand(0), NewCFP); in visitFADD()
16377 if (N0.getOpcode() == ISD::FADD) { in visitFADD()
16383 return DAG.getNode(ISD::FMUL, DL, VT, N1, in visitFADD()
16388 if (N1.getOpcode() == ISD::FADD) { in visitFADD()
16394 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFADD()
16400 if (N0.getOpcode() == ISD::FADD && N1.getOpcode() == ISD::FADD && in visitFADD()
16404 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), in visitFADD()
16410 if (SDValue SD = reassociateReduction(ISD::VECREDUCE_FADD, ISD::FADD, DL, in visitFADD()
16417 if (Fused.getOpcode() != ISD::DELETED_NODE) in visitFADD()
16434 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::STRICT_FSUB, VT)) in visitSTRICT_FADD()
16437 return DAG.getNode(ISD::STRICT_FSUB, DL, DAG.getVTList(VT, ChainVT), in visitSTRICT_FADD()
16442 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::STRICT_FSUB, VT)) in visitSTRICT_FADD()
16445 return DAG.getNode(ISD::STRICT_FSUB, DL, DAG.getVTList(VT, ChainVT), in visitSTRICT_FADD()
16466 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FSUB, DL, VT, {N0, N1})) in visitFSUB()
16504 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) in visitFSUB()
16505 return DAG.getNode(ISD::FNEG, DL, VT, N1); in visitFSUB()
16512 N1.getOpcode() == ISD::FADD) { in visitFSUB()
16515 return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(1)); in visitFSUB()
16518 return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(0)); in visitFSUB()
16524 return DAG.getNode(ISD::FADD, DL, VT, N0, NegN1); in visitFSUB()
16556 if (ConstOpIdx == 1 && N->getOpcode() == ISD::FDIV) in combineFMulOrFDivWithIntPow2()
16561 if (Pow2Op.getOpcode() != ISD::UINT_TO_FP && in combineFMulOrFDivWithIntPow2()
16562 (Pow2Op.getOpcode() != ISD::SINT_TO_FP || in combineFMulOrFDivWithIntPow2()
16587 N->getOpcode() == ISD::FMUL ? CurExp : (CurExp - MaxExpChange); in combineFMulOrFDivWithIntPow2()
16590 N->getOpcode() == ISD::FDIV ? CurExp : (CurExp + MaxExpChange); in combineFMulOrFDivWithIntPow2()
16604 return ISD::matchUnaryFpPredicate(ConstOp, IsFPConstValid); in combineFMulOrFDivWithIntPow2()
16633 SDValue Shift = DAG.getNode(ISD::SHL, DL, NewIntVT, Log2, MantissaShiftCnt); in combineFMulOrFDivWithIntPow2()
16635 DAG.getNode(N->getOpcode() == ISD::FMUL ? ISD::ADD : ISD::SUB, DL, in combineFMulOrFDivWithIntPow2()
16655 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FMUL, DL, VT, {N0, N1})) in visitFMUL()
16661 return DAG.getNode(ISD::FMUL, DL, VT, N1, N0); in visitFMUL()
16674 N0.getOpcode() == ISD::FMUL) { in visitFMUL()
16681 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, N01, N1); in visitFMUL()
16682 return DAG.getNode(ISD::FMUL, DL, VT, N00, MulConsts); in visitFMUL()
16688 if (N0.getOpcode() == ISD::FADD && N0.hasOneUse() && in visitFMUL()
16691 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, Two, N1); in visitFMUL()
16692 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), MulConsts); in visitFMUL()
16696 if (SDValue SD = reassociateReduction(ISD::VECREDUCE_FMUL, ISD::FMUL, DL, in visitFMUL()
16703 return DAG.getNode(ISD::FADD, DL, VT, N0, N0); in visitFMUL()
16707 if (!LegalOperations || TLI.isOperationLegal(ISD::FSUB, VT)) { in visitFMUL()
16708 return DAG.getNode(ISD::FSUB, DL, VT, in visitFMUL()
16726 return DAG.getNode(ISD::FMUL, DL, VT, NegN0, NegN1); in visitFMUL()
16732 (N0.getOpcode() == ISD::SELECT || N1.getOpcode() == ISD::SELECT) && in visitFMUL()
16733 TLI.isOperationLegal(ISD::FABS, VT)) { in visitFMUL()
16735 if (Select.getOpcode() != ISD::SELECT) in visitFMUL()
16743 Cond.getOpcode() == ISD::SETCC && Cond.getOperand(0) == X && in visitFMUL()
16746 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in visitFMUL()
16749 case ISD::SETOLT: in visitFMUL()
16750 case ISD::SETULT: in visitFMUL()
16751 case ISD::SETOLE: in visitFMUL()
16752 case ISD::SETULE: in visitFMUL()
16753 case ISD::SETLT: in visitFMUL()
16754 case ISD::SETLE: in visitFMUL()
16757 case ISD::SETOGT: in visitFMUL()
16758 case ISD::SETUGT: in visitFMUL()
16759 case ISD::SETOGE: in visitFMUL()
16760 case ISD::SETUGE: in visitFMUL()
16761 case ISD::SETGT: in visitFMUL()
16762 case ISD::SETGE: in visitFMUL()
16764 TLI.isOperationLegal(ISD::FNEG, VT)) in visitFMUL()
16765 return DAG.getNode(ISD::FNEG, DL, VT, in visitFMUL()
16766 DAG.getNode(ISD::FABS, DL, VT, X)); in visitFMUL()
16768 return DAG.getNode(ISD::FABS, DL, VT, X); in visitFMUL()
16809 return matcher.getNode(ISD::FMA, DL, VT, N0, N1, N2); in visitFMA()
16825 return matcher.getNode(ISD::FMA, DL, VT, NegN0, NegN1, N2); in visitFMA()
16838 return matcher.getNode(ISD::FADD, SDLoc(N), VT, N1, N2); in visitFMA()
16840 return matcher.getNode(ISD::FADD, SDLoc(N), VT, N0, N2); in visitFMA()
16845 return matcher.getNode(ISD::FMA, SDLoc(N), VT, N1, N0, N2); in visitFMA()
16849 if (matcher.match(N2, ISD::FMUL) && N0 == N2.getOperand(0) && in visitFMA()
16853 ISD::FMUL, DL, VT, N0, in visitFMA()
16854 matcher.getNode(ISD::FADD, DL, VT, N1, N2.getOperand(1))); in visitFMA()
16858 if (matcher.match(N0, ISD::FMUL) && in visitFMA()
16862 ISD::FMA, DL, VT, N0.getOperand(0), in visitFMA()
16863 matcher.getNode(ISD::FMUL, DL, VT, N1, N0.getOperand(1)), N2); in visitFMA()
16871 return matcher.getNode(ISD::FADD, DL, VT, N0, N2); in visitFMA()
16874 (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))) { in visitFMA()
16875 SDValue RHSNeg = matcher.getNode(ISD::FNEG, DL, VT, N0); in visitFMA()
16877 return matcher.getNode(ISD::FADD, DL, VT, N2, RHSNeg); in visitFMA()
16881 if (matcher.match(N0, ISD::FNEG) && in visitFMA()
16882 (TLI.isOperationLegal(ISD::ConstantFP, VT) || in visitFMA()
16885 return matcher.getNode(ISD::FMA, DL, VT, N0.getOperand(0), in visitFMA()
16886 matcher.getNode(ISD::FNEG, DL, VT, N1), N2); in visitFMA()
16894 return matcher.getNode(ISD::FMUL, DL, VT, N0, in visitFMA()
16895 matcher.getNode(ISD::FADD, DL, VT, N1, in visitFMA()
16900 if (N1CFP && matcher.match(N2, ISD::FNEG) && N2.getOperand(0) == N0) { in visitFMA()
16901 return matcher.getNode(ISD::FMUL, DL, VT, N0, in visitFMA()
16902 matcher.getNode(ISD::FADD, DL, VT, N1, in visitFMA()
16912 return matcher.getNode(ISD::FNEG, DL, VT, Neg); in visitFMA()
16926 return DAG.getNode(ISD::FMAD, DL, VT, N0, N1, N2); in visitFMAD()
16971 if (U->getOpcode() == ISD::FDIV && U->getOperand(1) == N1) { in combineRepeatedFPDivisors()
16973 if (U->getOperand(1).getOpcode() == ISD::FSQRT && in combineRepeatedFPDivisors()
16993 SDValue Reciprocal = DAG.getNode(ISD::FDIV, DL, VT, FPOne, N1, Flags); in combineRepeatedFPDivisors()
16999 SDValue NewNode = DAG.getNode(ISD::FMUL, SDLoc(U), VT, Dividend, in combineRepeatedFPDivisors()
17024 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FDIV, DL, VT, {N0, N1})) in visitFDIV()
17052 TLI.isOperationLegal(ISD::ConstantFP, VT) || in visitFDIV()
17054 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFDIV()
17060 if (N1.getOpcode() == ISD::FSQRT) { in visitFDIV()
17062 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
17063 } else if (N1.getOpcode() == ISD::FP_EXTEND && in visitFDIV()
17064 N1.getOperand(0).getOpcode() == ISD::FSQRT) { in visitFDIV()
17067 RV = DAG.getNode(ISD::FP_EXTEND, SDLoc(N1), VT, RV); in visitFDIV()
17069 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
17071 } else if (N1.getOpcode() == ISD::FP_ROUND && in visitFDIV()
17072 N1.getOperand(0).getOpcode() == ISD::FSQRT) { in visitFDIV()
17075 RV = DAG.getNode(ISD::FP_ROUND, SDLoc(N1), VT, RV, N1.getOperand(1)); in visitFDIV()
17077 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
17079 } else if (N1.getOpcode() == ISD::FMUL) { in visitFDIV()
17083 if (N1.getOperand(0).getOpcode() == ISD::FSQRT) { in visitFDIV()
17086 } else if (N1.getOperand(1).getOpcode() == ISD::FSQRT) { in visitFDIV()
17096 if (Y.getOpcode() == ISD::FABS && Y.hasOneUse()) in visitFDIV()
17103 SDValue AA = DAG.getNode(ISD::FMUL, DL, VT, A, A); in visitFDIV()
17105 DAG.getNode(ISD::FMUL, DL, VT, AA, Sqrt.getOperand(0)); in visitFDIV()
17107 return DAG.getNode(ISD::FMUL, DL, VT, N0, Rsqrt); in visitFDIV()
17117 SDValue Div = DAG.getNode(ISD::FDIV, SDLoc(N1), VT, Rsqrt, Y); in visitFDIV()
17119 return DAG.getNode(ISD::FMUL, DL, VT, N0, Div); in visitFDIV()
17133 if (N1.getOpcode() == ISD::FSQRT && N0 == N1.getOperand(0)) in visitFDIV()
17149 return DAG.getNode(ISD::FDIV, SDLoc(N), VT, NegN0, NegN1); in visitFDIV()
17169 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FREM, SDLoc(N), VT, {N0, N1})) in visitFREM()
17220 if (N1.getOpcode() != ISD::FP_EXTEND && in CanCombineFCOPYSIGN_EXTEND_ROUND()
17221 N1.getOpcode() != ISD::FP_ROUND) in CanCombineFCOPYSIGN_EXTEND_ROUND()
17235 DAG.FoldConstantArithmetic(ISD::FCOPYSIGN, SDLoc(N), VT, {N0, N1})) in visitFCOPYSIGN()
17243 if (!LegalOperations || TLI.isOperationLegal(ISD::FABS, VT)) in visitFCOPYSIGN()
17244 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFCOPYSIGN()
17246 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) in visitFCOPYSIGN()
17247 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, in visitFCOPYSIGN()
17248 DAG.getNode(ISD::FABS, SDLoc(N0), VT, N0)); in visitFCOPYSIGN()
17255 if (N0.getOpcode() == ISD::FABS || N0.getOpcode() == ISD::FNEG || in visitFCOPYSIGN()
17256 N0.getOpcode() == ISD::FCOPYSIGN) in visitFCOPYSIGN()
17257 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0.getOperand(0), N1); in visitFCOPYSIGN()
17260 if (N1.getOpcode() == ISD::FABS) in visitFCOPYSIGN()
17261 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFCOPYSIGN()
17264 if (N1.getOpcode() == ISD::FCOPYSIGN) in visitFCOPYSIGN()
17265 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1.getOperand(1)); in visitFCOPYSIGN()
17270 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1.getOperand(0)); in visitFCOPYSIGN()
17302 (!DAG.getTargetLoweringInfo().isOperationExpand(ISD::FPOW, VT) && in visitFPOW()
17303 DAG.getTargetLoweringInfo().isOperationExpand(ISD::FCBRT, VT))) in visitFPOW()
17306 return DAG.getNode(ISD::FCBRT, SDLoc(N), VT, N->getOperand(0)); in visitFPOW()
17331 if (!DAG.getTargetLoweringInfo().isOperationLegalOrCustom(ISD::FSQRT, VT)) in visitFPOW()
17341 SDValue Sqrt = DAG.getNode(ISD::FSQRT, DL, VT, N->getOperand(0)); in visitFPOW()
17342 SDValue SqrtSqrt = DAG.getNode(ISD::FSQRT, DL, VT, Sqrt); in visitFPOW()
17346 return DAG.getNode(ISD::FMUL, DL, VT, Sqrt, SqrtSqrt); in visitFPOW()
17361 if (!TLI.isOperationLegal(ISD::FTRUNC, VT) || in foldFPToIntToFP()
17368 if (N->getOpcode() == ISD::SINT_TO_FP && N0.getOpcode() == ISD::FP_TO_SINT && in foldFPToIntToFP()
17370 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0.getOperand(0)); in foldFPToIntToFP()
17372 if (N->getOpcode() == ISD::UINT_TO_FP && N0.getOpcode() == ISD::FP_TO_UINT && in foldFPToIntToFP()
17374 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0.getOperand(0)); in foldFPToIntToFP()
17392 TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) in visitSINT_TO_FP()
17393 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); in visitSINT_TO_FP()
17397 if (!hasOperation(ISD::SINT_TO_FP, OpVT) && in visitSINT_TO_FP()
17398 hasOperation(ISD::UINT_TO_FP, OpVT)) { in visitSINT_TO_FP()
17401 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); in visitSINT_TO_FP()
17406 if (N0.getOpcode() == ISD::SETCC && N0.getValueType() == MVT::i1 && in visitSINT_TO_FP()
17408 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitSINT_TO_FP()
17416 if (N0.getOpcode() == ISD::ZERO_EXTEND && in visitSINT_TO_FP()
17417 N0.getOperand(0).getOpcode() == ISD::SETCC && !VT.isVector() && in visitSINT_TO_FP()
17418 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitSINT_TO_FP()
17444 TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) in visitUINT_TO_FP()
17445 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); in visitUINT_TO_FP()
17449 if (!hasOperation(ISD::UINT_TO_FP, OpVT) && in visitUINT_TO_FP()
17450 hasOperation(ISD::SINT_TO_FP, OpVT)) { in visitUINT_TO_FP()
17453 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); in visitUINT_TO_FP()
17457 if (N0.getOpcode() == ISD::SETCC && !VT.isVector() && in visitUINT_TO_FP()
17458 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitUINT_TO_FP()
17475 if (N0.getOpcode() != ISD::UINT_TO_FP && N0.getOpcode() != ISD::SINT_TO_FP) in FoldIntToFPToInt()
17480 bool IsInputSigned = N0.getOpcode() == ISD::SINT_TO_FP; in FoldIntToFPToInt()
17481 bool IsOutputSigned = N->getOpcode() == ISD::FP_TO_SINT; in FoldIntToFPToInt()
17501 unsigned ExtOp = IsInputSigned && IsOutputSigned ? ISD::SIGN_EXTEND in FoldIntToFPToInt()
17502 : ISD::ZERO_EXTEND; in FoldIntToFPToInt()
17506 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Src); in FoldIntToFPToInt()
17522 return DAG.getNode(ISD::FP_TO_SINT, SDLoc(N), VT, N0); in visitFP_TO_SINT()
17537 return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), VT, N0); in visitFP_TO_UINT()
17564 DAG.FoldConstantArithmetic(ISD::FP_ROUND, SDLoc(N), VT, {N0, N1})) in visitFP_ROUND()
17568 if (N0.getOpcode() == ISD::FP_EXTEND && VT == N0.getOperand(0).getValueType()) in visitFP_ROUND()
17572 if (N0.getOpcode() == ISD::FP_ROUND) { in visitFP_ROUND()
17577 if (!hasOperation(ISD::FP_ROUND, VT)) in visitFP_ROUND()
17598 ISD::FP_ROUND, DL, VT, N0.getOperand(0), in visitFP_ROUND()
17608 if (N0.getOpcode() == ISD::FCOPYSIGN && N0->hasOneUse() && in visitFP_ROUND()
17611 SDValue Tmp = DAG.getNode(ISD::FP_ROUND, SDLoc(N0), VT, in visitFP_ROUND()
17614 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, in visitFP_ROUND()
17634 N->use_begin()->getOpcode() == ISD::FP_ROUND) in visitFP_EXTEND()
17639 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, N0); in visitFP_EXTEND()
17642 if (N0.getOpcode() == ISD::FP16_TO_FP && in visitFP_EXTEND()
17643 TLI.getOperationAction(ISD::FP16_TO_FP, VT) == TargetLowering::Legal) in visitFP_EXTEND()
17644 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), VT, N0.getOperand(0)); in visitFP_EXTEND()
17648 if (N0.getOpcode() == ISD::FP_ROUND in visitFP_EXTEND()
17653 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, in visitFP_EXTEND()
17655 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, In); in visitFP_EXTEND()
17659 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && in visitFP_EXTEND()
17660 TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, VT, N0.getValueType())) { in visitFP_EXTEND()
17662 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, in visitFP_EXTEND()
17669 DAG.getNode(ISD::FP_ROUND, SDLoc(N0), N0.getValueType(), ExtLoad, in visitFP_EXTEND()
17687 return DAG.getNode(ISD::FCEIL, SDLoc(N), VT, N0); in visitFCEIL()
17698 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0); in visitFTRUNC()
17705 case ISD::FRINT: in visitFTRUNC()
17706 case ISD::FTRUNC: in visitFTRUNC()
17707 case ISD::FNEARBYINT: in visitFTRUNC()
17708 case ISD::FROUNDEVEN: in visitFTRUNC()
17709 case ISD::FFLOOR: in visitFTRUNC()
17710 case ISD::FCEIL: in visitFTRUNC()
17722 return DAG.getNode(ISD::FFREXP, SDLoc(N), N->getVTList(), N0); in visitFFREXP()
17732 return DAG.getNode(ISD::FFLOOR, SDLoc(N), VT, N0); in visitFFLOOR()
17744 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0); in visitFNEG()
17754 if (N0.getOpcode() == ISD::FSUB && in visitFNEG()
17757 return DAG.getNode(ISD::FSUB, SDLoc(N), VT, N0.getOperand(1), in visitFNEG()
17773 bool PropagatesNaN = Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM; in visitFMinMax()
17774 bool IsMin = Opc == ISD::FMINNUM || Opc == ISD::FMINIMUM; in visitFMinMax()
17817 ? (IsMin ? ISD::VECREDUCE_FMINIMUM : ISD::VECREDUCE_FMAXIMUM) in visitFMinMax()
17818 : (IsMin ? ISD::VECREDUCE_FMIN : ISD::VECREDUCE_FMAX), in visitFMinMax()
17831 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFABS()
17834 if (N0.getOpcode() == ISD::FABS) in visitFABS()
17839 if (N0.getOpcode() == ISD::FNEG || N0.getOpcode() == ISD::FCOPYSIGN) in visitFABS()
17840 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0.getOperand(0)); in visitFABS()
17855 if (N1->getOpcode() == ISD::FREEZE && N1.hasOneUse()) { in visitBRCOND()
17856 return DAG.getNode(ISD::BRCOND, SDLoc(N), MVT::Other, Chain, in visitBRCOND()
17870 if (N1->getOpcode() == ISD::SETCC && N1.hasOneUse()) { in visitBRCOND()
17872 ISD::CondCode Cond = cast<CondCodeSDNode>(N1->getOperand(2))->get(); in visitBRCOND()
17878 auto IsAlwaysTrueOrFalse = [](ISD::CondCode Cond, ConstantSDNode *C) { in visitBRCOND()
17879 bool False = (Cond == ISD::SETULT && C->isZero()) || in visitBRCOND()
17880 (Cond == ISD::SETLT && C->isMinSignedValue()) || in visitBRCOND()
17881 (Cond == ISD::SETUGT && C->isAllOnes()) || in visitBRCOND()
17882 (Cond == ISD::SETGT && C->isMaxSignedValue()); in visitBRCOND()
17883 bool True = (Cond == ISD::SETULE && C->isAllOnes()) || in visitBRCOND()
17884 (Cond == ISD::SETLE && C->isMaxSignedValue()) || in visitBRCOND()
17885 (Cond == ISD::SETUGE && C->isZero()) || in visitBRCOND()
17886 (Cond == ISD::SETGE && C->isMinSignedValue()); in visitBRCOND()
17890 if (S0->getOpcode() == ISD::FREEZE && S0.hasOneUse() && S1C) { in visitBRCOND()
17896 if (S1->getOpcode() == ISD::FREEZE && S1.hasOneUse() && S0C) { in visitBRCOND()
17897 if (!IsAlwaysTrueOrFalse(ISD::getSetCCSwappedOperands(Cond), S0C)) { in visitBRCOND()
17905 ISD::BRCOND, SDLoc(N), MVT::Other, Chain, in visitBRCOND()
17917 if (N1.getOpcode() == ISD::SETCC && in visitBRCOND()
17918 TLI.isOperationLegalOrCustom(ISD::BR_CC, in visitBRCOND()
17920 return DAG.getNode(ISD::BR_CC, SDLoc(N), MVT::Other, in visitBRCOND()
17930 return DAG.getNode(ISD::BRCOND, SDLoc(N), MVT::Other, in visitBRCOND()
17938 if (N.getOpcode() == ISD::SRL || in rebuildSetCC()
17939 (N.getOpcode() == ISD::TRUNCATE && in rebuildSetCC()
17941 N.getOperand(0).getOpcode() == ISD::SRL))) { in rebuildSetCC()
17943 if (N.getOpcode() == ISD::TRUNCATE) in rebuildSetCC()
17966 if (Op0.getOpcode() == ISD::AND && Op1.getOpcode() == ISD::Constant) { in rebuildSetCC()
17969 if (AndOp1.getOpcode() == ISD::Constant) { in rebuildSetCC()
17977 ISD::SETNE); in rebuildSetCC()
17985 if (N.getOpcode() == ISD::XOR) { in rebuildSetCC()
17992 while (N.getOpcode() == ISD::XOR) { in rebuildSetCC()
18005 if (N.getOpcode() != ISD::XOR) in rebuildSetCC()
18011 if (Op0.getOpcode() != ISD::SETCC && Op1.getOpcode() != ISD::SETCC) { in rebuildSetCC()
18014 if (isBitwiseNot(N) && Op0.hasOneUse() && Op0.getOpcode() == ISD::XOR && in rebuildSetCC()
18027 Equal ? ISD::SETEQ : ISD::SETNE); in rebuildSetCC()
18053 if (Simp.getNode() && Simp.getOpcode() == ISD::SETCC) in visitBR_CC()
18054 return DAG.getNode(ISD::BR_CC, SDLoc(N), MVT::Other, in visitBR_CC()
18117 if (!getCombineLoadStoreParts(N, ISD::PRE_INC, ISD::PRE_DEC, IsLoad, IsMasked, in CombineToPreIndexedLoadStore()
18123 if ((Ptr.getOpcode() != ISD::ADD && Ptr.getOpcode() != ISD::SUB) || in CombineToPreIndexedLoadStore()
18130 ISD::MemIndexedMode AM = ISD::UNINDEXED; in CombineToPreIndexedLoadStore()
18198 if (Use.getUser()->getOpcode() != ISD::ADD && in CombineToPreIndexedLoadStore()
18199 Use.getUser()->getOpcode() != ISD::SUB) { in CombineToPreIndexedLoadStore()
18295 int X0 = (OtherUses[i]->getOpcode() == ISD::SUB && OffsetIdx == 1) ? -1 : 1; in CombineToPreIndexedLoadStore()
18296 int Y0 = (OtherUses[i]->getOpcode() == ISD::SUB && OffsetIdx == 0) ? -1 : 1; in CombineToPreIndexedLoadStore()
18297 int X1 = (AM == ISD::PRE_DEC && !Swapped) ? -1 : 1; in CombineToPreIndexedLoadStore()
18298 int Y1 = (AM == ISD::PRE_DEC && Swapped) ? -1 : 1; in CombineToPreIndexedLoadStore()
18300 unsigned Opcode = (Y0 * Y1 < 0) ? ISD::SUB : ISD::ADD; in CombineToPreIndexedLoadStore()
18330 ISD::MemIndexedMode &AM, in shouldCombineToPostInc()
18334 (PtrUse->getOpcode() != ISD::ADD && PtrUse->getOpcode() != ISD::SUB)) in shouldCombineToPostInc()
18357 if (getCombineLoadStoreParts(Use, ISD::POST_INC, ISD::POST_DEC, IsLoad, in shouldCombineToPostInc()
18368 if (Use->getOpcode() == ISD::ADD || Use->getOpcode() == ISD::SUB) { in shouldCombineToPostInc()
18380 ISD::MemIndexedMode &AM, in getPostIndexedLoadStoreOp()
18383 if (!getCombineLoadStoreParts(N, ISD::POST_INC, ISD::POST_DEC, IsLoad, in getPostIndexedLoadStoreOp()
18427 ISD::MemIndexedMode AM = ISD::UNINDEXED; in CombineToPostIndexedLoadStore()
18468 ISD::MemIndexedMode AM = LD->getAddressingMode(); in SplitIndexingFromLoad()
18469 assert(AM != ISD::UNINDEXED); in SplitIndexingFromLoad()
18476 assert((Inc.getOpcode() != ISD::TargetConstant || in SplitIndexingFromLoad()
18479 if (Inc.getOpcode() == ISD::TargetConstant) { in SplitIndexingFromLoad()
18486 (AM == ISD::PRE_INC || AM == ISD::POST_INC ? ISD::ADD : ISD::SUB); in SplitIndexingFromLoad()
18502 TLI.isOperationLegal(ISD::FTRUNC, STMemType)) { in getTruncatedStoreValue()
18503 Val = DAG.getNode(ISD::FTRUNC, SDLoc(ST), STMemType, Val); in getTruncatedStoreValue()
18508 Val = DAG.getNode(ISD::TRUNCATE, SDLoc(ST), STMemType, Val); in getTruncatedStoreValue()
18527 case ISD::NON_EXTLOAD: in extendLoadedValueToExtension()
18530 case ISD::EXTLOAD: in extendLoadedValueToExtension()
18531 Val = DAG.getNode(ISD::ANY_EXTEND, SDLoc(LD), LDType, Val); in extendLoadedValueToExtension()
18533 case ISD::SEXTLOAD: in extendLoadedValueToExtension()
18534 Val = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(LD), LDType, Val); in extendLoadedValueToExtension()
18536 case ISD::ZEXTLOAD: in extendLoadedValueToExtension()
18537 Val = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(LD), LDType, Val); in extendLoadedValueToExtension()
18549 if (Chain.getOpcode() == ISD::CALLSEQ_START) in getUniqueStoreFeeding()
18554 if (Chain.getOpcode() == ISD::TokenFactor) { in getUniqueStoreFeeding()
18663 !LDMemType.isVector() && LD->getExtensionType() != ISD::SEXTLOAD) { in ForwardStoreValueToDirectLoad()
18669 auto Val = DAG.getNode(ISD::AND, SDLoc(LD), LDType, ST->getValue(), Mask); in ForwardStoreValueToDirectLoad()
18680 TLI.isOperationLegal(ISD::SRL, STType)) { in ForwardStoreValueToDirectLoad()
18681 Val = DAG.getNode(ISD::SRL, SDLoc(LD), STType, Val, in ForwardStoreValueToDirectLoad()
18701 Val = DAG.getNode(ISD::TRUNCATE, SDLoc(LD), LDMemType, Val); in ForwardStoreValueToDirectLoad()
18810 if (LD->getExtensionType() == ISD::NON_EXTLOAD) { in visitLOAD()
18821 SDValue Token = DAG.getNode(ISD::TokenFactor, SDLoc(N), in visitLOAD()
19014 if (!TLI.isOperationLegal(ISD::LOAD, SliceType)) in isLegal()
19028 if (!TLI.isOperationLegal(ISD::ADD, PtrType)) in isLegal()
19034 !TLI.isOperationLegal(ISD::ZERO_EXTEND, TruncateType)) in isLegal()
19078 BaseAddr = DAG->getNode(ISD::ADD, DL, ArithType, BaseAddr, in loadSlice()
19095 DAG->getNode(ISD::ZERO_EXTEND, SDLoc(LastInst), FinalType, LastInst); in loadSlice()
19107 if (Use->getOpcode() != ISD::BITCAST) in canMergeExpensiveCrossRegisterBankCopy()
19117 if (ArgRC == ResRC || !TLI.isOperationLegal(ISD::LOAD, ResVT)) in canMergeExpensiveCrossRegisterBankCopy()
19138 if (!TLI.isOperationLegal(ISD::LOAD, ResVT)) in canMergeExpensiveCrossRegisterBankCopy()
19296 if (!LD->isSimple() || !ISD::isNormalLoad(LD) || in SliceUpLoad()
19325 if (User->getOpcode() == ISD::SRL && User->hasOneUse() && in SliceUpLoad()
19333 if (User->getOpcode() != ISD::TRUNCATE) in SliceUpLoad()
19376 if (SliceInst.getOpcode() != ISD::LOAD) in SliceUpLoad()
19378 assert(SliceInst->getOpcode() == ISD::LOAD && in SliceUpLoad()
19383 SDValue Chain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, in SliceUpLoad()
19398 if (V->getOpcode() != ISD::AND || in CheckForMaskedLoad()
19400 !ISD::isNormalLoad(V->getOperand(0).getNode())) in CheckForMaskedLoad()
19447 else if (Chain->getOpcode() == ISD::TokenFactor && in CheckForMaskedLoad()
19506 IVal = DAG.getNode(ISD::SRL, DL, IVal.getValueType(), IVal, in ShrinkLoadReplaceStoreWithStore()
19531 IVal = DAG.getNode(ISD::TRUNCATE, SDLoc(IVal), VT, IVal); in ShrinkLoadReplaceStoreWithStore()
19558 if ((Opc != ISD::OR && Opc != ISD::XOR && Opc != ISD::AND) || in ReduceLoadOpStoreWidth()
19567 if (Opc == ISD::OR && EnableShrinkLoadReplaceStoreWithStore) { in ReduceLoadOpStoreWidth()
19586 if (Value.getOperand(1).getOpcode() != ISD::Constant) in ReduceLoadOpStoreWidth()
19590 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && in ReduceLoadOpStoreWidth()
19602 if (Opc == ISD::AND) in ReduceLoadOpStoreWidth()
19630 if (Opc == ISD::AND) in ReduceLoadOpStoreWidth()
19678 if (ISD::isNormalStore(ST) && ISD::isNormalLoad(Value.getNode()) && in TransformFPLoadStorePair()
19699 if (!TLI.isOperationLegal(ISD::LOAD, IntVT) || in TransformFPLoadStorePair()
19700 !TLI.isOperationLegal(ISD::STORE, IntVT) || in TransformFPLoadStorePair()
19701 !TLI.isDesirableToTransformToIntegerOp(ISD::LOAD, VT) || in TransformFPLoadStorePair()
19702 !TLI.isDesirableToTransformToIntegerOp(ISD::STORE, VT) || in TransformFPLoadStorePair()
19757 if (Use->getOpcode() == ISD::MUL) { // We have another multiply use. in isMulAddWithConstProfitable()
19793 if (OtherOp->getOpcode() == ISD::ADD && in isMulAddWithConstProfitable()
19920 StoredVal = DAG.getNode(MemVT.isVector() ? ISD::CONCAT_VECTORS in mergeStoresOfConstantsOrVecElts()
19921 : ISD::BUILD_VECTOR, in mergeStoresOfConstantsOrVecElts()
19935 (Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT || in mergeStoresOfConstantsOrVecElts()
19936 Val.getOpcode() == ISD::EXTRACT_SUBVECTOR)) { in mergeStoresOfConstantsOrVecElts()
19942 Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { in mergeStoresOfConstantsOrVecElts()
19943 Val = DAG.getNode(ISD::BUILD_VECTOR, DL, MemVT, Val); in mergeStoresOfConstantsOrVecElts()
19945 unsigned OpC = MemVT.isVector() ? ISD::EXTRACT_SUBVECTOR in mergeStoresOfConstantsOrVecElts()
19946 : ISD::EXTRACT_VECTOR_ELT; in mergeStoresOfConstantsOrVecElts()
19956 StoredVal = DAG.getNode(MemVT.isVector() ? ISD::CONCAT_VECTORS in mergeStoresOfConstantsOrVecElts()
19957 : ISD::BUILD_VECTOR, in mergeStoresOfConstantsOrVecElts()
19989 } else if (ISD::isBuildVectorOfConstantSDNodes(Val.getNode()) || in mergeStoresOfConstantsOrVecElts()
19990 ISD::isBuildVectorOfConstantFPSDNodes(Val.getNode())) { in mergeStoresOfConstantsOrVecElts()
20132 if (OtherBC.getOpcode() != ISD::EXTRACT_VECTOR_ELT && in getStoreMergeCandidates()
20133 OtherBC.getOpcode() != ISD::EXTRACT_SUBVECTOR) in getStoreMergeCandidates()
20229 if (N->getOpcode() == ISD::TokenFactor) { in checkMergeStoreCandidatesForDependencies()
20345 else if (ISD::isBuildVectorAllZeros(StoredVal.getNode())) in tryStoreMergeOfConstants()
20572 (hasOperation(ISD::ROTL, PairVT) || in tryStoreMergeOfLoads()
20573 hasOperation(ISD::ROTR, PairVT))) { in tryStoreMergeOfLoads()
20656 TLI.isLoadExtLegal(ISD::ZEXTLOAD, LegalizedStoredValTy, StoreTy) && in tryStoreMergeOfLoads()
20657 TLI.isLoadExtLegal(ISD::SEXTLOAD, LegalizedStoredValTy, StoreTy) && in tryStoreMergeOfLoads()
20658 TLI.isLoadExtLegal(ISD::EXTLOAD, LegalizedStoredValTy, StoreTy) && in tryStoreMergeOfLoads()
20761 StoreOp = DAG.getNode(ISD::ROTL, LoadDL, JointMemOpVT, NewLoad, RotAmt); in tryStoreMergeOfLoads()
20771 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, LoadDL, ExtendedTy, in tryStoreMergeOfLoads()
20910 SDValue Token = DAG.getNode(ISD::TokenFactor, SL, in replaceStoreChain()
20922 if (Value.getOpcode() == ISD::TargetConstantFP) in replaceStoreOfFPConstant()
20925 if (!ISD::isNormalStore(ST)) in replaceStoreOfFPConstant()
20952 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) { in replaceStoreOfFPConstant()
20963 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i64)) { in replaceStoreOfFPConstant()
20970 if (ST->isSimple() && TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32) && in replaceStoreOfFPConstant()
20990 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in replaceStoreOfFPConstant()
21008 if (Value.getOpcode() != ISD::INSERT_VECTOR_ELT || !Value.hasOneUse()) in replaceStoreOfInsertLoad()
21024 !ISD::isNormalStore(ST) || in replaceStoreOfInsertLoad()
21062 if (Value.getOpcode() == ISD::BITCAST && !ST->isTruncatingStore() && in visitSTORE()
21072 TLI.isOperationLegal(ISD::STORE, SVT)) && in visitSTORE()
21127 if ((Value.getOpcode() == ISD::ZERO_EXTEND || in visitSTORE()
21128 Value.getOpcode() == ISD::SIGN_EXTEND || in visitSTORE()
21129 Value.getOpcode() == ISD::ANY_EXTEND) && in visitSTORE()
21131 TLI.isOperationLegalOrCustom(ISD::STORE, ST->getMemoryVT())) in visitSTORE()
21147 if (N->getOpcode() != ISD::DELETED_NODE) in visitSTORE()
21245 if ((Value.getOpcode() == ISD::FP_ROUND || in visitSTORE()
21246 Value.getOpcode() == ISD::TRUNCATE) && in visitSTORE()
21266 if (N->getOpcode() == ISD::DELETED_NODE || !isa<StoreSDNode>(N)) in visitSTORE()
21306 case ISD::TokenFactor: in visitLIFETIME_END()
21310 case ISD::LIFETIME_START: in visitLIFETIME_END()
21311 case ISD::LIFETIME_END: in visitLIFETIME_END()
21317 case ISD::STORE: { in visitLIFETIME_END()
21385 if (!Val.getValueType().isScalarInteger() || Val.getOpcode() != ISD::OR) in splitMergedValStore()
21392 if (Op1.getOpcode() != ISD::SHL) { in splitMergedValStore()
21394 if (Op1.getOpcode() != ISD::SHL) in splitMergedValStore()
21410 if (Lo.getOpcode() != ISD::ZERO_EXTEND || !Lo.hasOneUse() || in splitMergedValStore()
21413 Hi.getOpcode() != ISD::ZERO_EXTEND || !Hi.hasOneUse() || in splitMergedValStore()
21420 EVT LowTy = (Lo.getOperand(0).getOpcode() == ISD::BITCAST) in splitMergedValStore()
21423 EVT HighTy = (Hi.getOperand(0).getOpcode() == ISD::BITCAST) in splitMergedValStore()
21435 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Lo.getOperand(0)); in splitMergedValStore()
21436 Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Hi.getOperand(0)); in splitMergedValStore()
21460 if (Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in mergeEltWithShuffle()
21488 if (ArgVal.getOpcode() == ISD::CONCAT_VECTORS) { in mergeEltWithShuffle()
21525 assert(N->getOpcode() == ISD::INSERT_VECTOR_ELT && in mergeInsertEltWithShuffle()
21555 assert(N->getOpcode() == ISD::INSERT_VECTOR_ELT && in combineInsertEltToShuffle()
21559 if (InsertVal.getOpcode() != ISD::BITCAST || !InsertVal.hasOneUse() || in combineInsertEltToShuffle()
21599 SDValue PaddedSubV = DAG.getNode(ISD::CONCAT_VECTORS, DL, ShufVT, ConcatOps); in combineInsertEltToShuffle()
21636 if (Scalar.getOpcode() == ISD::ZERO_EXTEND || in combineInsertEltToLoad()
21637 Scalar.getOpcode() == ISD::SIGN_EXTEND || in combineInsertEltToLoad()
21638 Scalar.getOpcode() == ISD::ANY_EXTEND) { in combineInsertEltToLoad()
21659 !VecLoad->isSimple() || VecLoad->getExtensionType() != ISD::NON_EXTLOAD || in combineInsertEltToLoad()
21660 ScalarLoad->getExtensionType() != ISD::NON_EXTLOAD || in combineInsertEltToLoad()
21690 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), VecLoad->getBasePtr(), in combineInsertEltToLoad()
21719 if (InVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitINSERT_VECTOR_ELT()
21742 if (InVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitINSERT_VECTOR_ELT()
21755 if (InVec.getOpcode() == ISD::INSERT_VECTOR_ELT && InVec.hasOneUse() in visitINSERT_VECTOR_ELT()
21760 SDValue NewOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, in visitINSERT_VECTOR_ELT()
21763 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(InVec.getNode()), in visitINSERT_VECTOR_ELT()
21778 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) { in visitINSERT_VECTOR_ELT()
21819 if (CurVec.getOpcode() == ISD::BUILD_VECTOR && CurVec.hasOneUse()) { in visitINSERT_VECTOR_ELT()
21826 if (CurVec.getOpcode() == ISD::SCALAR_TO_VECTOR && CurVec.hasOneUse()) { in visitINSERT_VECTOR_ELT()
21832 if (CurVec.getOpcode() == ISD::INSERT_VECTOR_ELT && CurVec.hasOneUse()) in visitINSERT_VECTOR_ELT()
21849 if (CurVec.getOpcode() == ISD::VECTOR_SHUFFLE && CurVec.hasOneUse()) { in visitINSERT_VECTOR_ELT()
21882 return DAG.getNode(ISD::AND, DL, VT, CurVec, in visitINSERT_VECTOR_ELT()
21925 ISD::LoadExtType ExtTy = in scalarizeExtractedVectorLoad()
21926 ResultVT.bitsGT(VecEltVT) ? ISD::NON_EXTLOAD : ISD::EXTLOAD; in scalarizeExtractedVectorLoad()
21927 if (!TLI.isOperationLegalOrCustom(ISD::LOAD, VecEltVT) || in scalarizeExtractedVectorLoad()
21963 ISD::LoadExtType ExtType = in scalarizeExtractedVectorLoad()
21964 TLI.isLoadExtLegal(ISD::ZEXTLOAD, ResultVT, VecEltVT) ? ISD::ZEXTLOAD in scalarizeExtractedVectorLoad()
21965 : ISD::EXTLOAD; in scalarizeExtractedVectorLoad()
21978 Load = DAG.getNode(ISD::TRUNCATE, DL, ResultVT, Load); in scalarizeExtractedVectorLoad()
22009 ISD::isConstantSplatVector(Op0.getNode(), SplatVal) || in scalarizeExtractedBinop()
22011 ISD::isConstantSplatVector(Op1.getNode(), SplatVal)) { in scalarizeExtractedBinop()
22016 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Index); in scalarizeExtractedBinop()
22017 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op1, Index); in scalarizeExtractedBinop()
22106 case ISD::TRUNCATE: in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22113 case ISD::SRL: in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22130 if (User->getOpcode() != ISD::BUILD_VECTOR) in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22168 !(TLI.isOperationLegalOrCustom(ISD::BITCAST, NewVecVT) && in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22169 TLI.isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT, NewVecVT))) in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22178 SDValue V = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, NewScalarVT, NewVecOp, in refineExtractVectorEltIntoMultipleNarrowExtractVectorElts()
22199 if (VecOp.getOpcode() == ISD::INSERT_VECTOR_ELT && in visitEXTRACT_VECTOR_ELT()
22206 if (VecOp.getOpcode() == ISD::SCALAR_TO_VECTOR) { in visitEXTRACT_VECTOR_ELT()
22218 return DAG.getNode(ISD::TRUNCATE, DL, ScalarVT, InOp); in visitEXTRACT_VECTOR_ELT()
22219 return DAG.getNode(ISD::ANY_EXTEND, DL, ScalarVT, InOp); in visitEXTRACT_VECTOR_ELT()
22231 if (VecOp.hasOneUse() && VecOp.getOpcode() == ISD::FREEZE) { in visitEXTRACT_VECTOR_ELT()
22232 return DAG.getFreeze(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, in visitEXTRACT_VECTOR_ELT()
22237 if (((IndexC && VecOp.getOpcode() == ISD::BUILD_VECTOR) || in visitEXTRACT_VECTOR_ELT()
22238 VecOp.getOpcode() == ISD::SPLAT_VECTOR) && in visitEXTRACT_VECTOR_ELT()
22240 assert((VecOp.getOpcode() != ISD::BUILD_VECTOR || in visitEXTRACT_VECTOR_ELT()
22244 VecOp.getOpcode() == ISD::BUILD_VECTOR ? IndexC->getZExtValue() : 0; in visitEXTRACT_VECTOR_ELT()
22287 if (IndexC && VecOp.getOpcode() == ISD::BITCAST && VecVT.isInteger() && in visitEXTRACT_VECTOR_ELT()
22299 BCSrc.getOpcode() == ISD::SCALAR_TO_VECTOR) { in visitEXTRACT_VECTOR_ELT()
22326 if (IndexC && VecOp.getOpcode() == ISD::VECTOR_SHUFFLE) { in visitEXTRACT_VECTOR_ELT()
22344 if (SVInVec.getOpcode() == ISD::BUILD_VECTOR) { in visitEXTRACT_VECTOR_ELT()
22359 TLI.isOperationLegal(ISD::EXTRACT_VECTOR_ELT, VecVT) || in visitEXTRACT_VECTOR_ELT()
22360 TLI.isOperationExpand(ISD::VECTOR_SHUFFLE, VecVT)) { in visitEXTRACT_VECTOR_ELT()
22361 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, SVInVec, in visitEXTRACT_VECTOR_ELT()
22369 return Use->getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitEXTRACT_VECTOR_ELT()
22382 if (N->getOpcode() != ISD::DELETED_NODE) in visitEXTRACT_VECTOR_ELT()
22390 if (N->getOpcode() != ISD::DELETED_NODE) in visitEXTRACT_VECTOR_ELT()
22408 if (VecOp.getOpcode() == ISD::BITCAST) { in visitEXTRACT_VECTOR_ELT()
22424 ISD::isNormalLoad(VecOp.getNode()) && in visitEXTRACT_VECTOR_ELT()
22441 if (ISD::isNormalLoad(VecOp.getNode())) { in visitEXTRACT_VECTOR_ELT()
22443 } else if (VecOp.getOpcode() == ISD::SCALAR_TO_VECTOR && in visitEXTRACT_VECTOR_ELT()
22445 ISD::isNormalLoad(VecOp.getOperand(0).getNode())) { in visitEXTRACT_VECTOR_ELT()
22470 if (VecOp.getOpcode() == ISD::BITCAST) { in visitEXTRACT_VECTOR_ELT()
22477 if (ISD::isNormalLoad(VecOp.getNode())) { in visitEXTRACT_VECTOR_ELT()
22482 } else if (VecOp.getOpcode() == ISD::CONCAT_VECTORS && !BCNumEltsChanged && in visitEXTRACT_VECTOR_ELT()
22502 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, in visitEXTRACT_VECTOR_ELT()
22505 return DAG.getNode(ISD::BITCAST, SL, ScalarVT, Elt); in visitEXTRACT_VECTOR_ELT()
22548 bool AnyExt = In.getOpcode() == ISD::ANY_EXTEND; in reduceBuildVecExtToExtBuildVec()
22549 bool ZeroExt = In.getOpcode() == ISD::ZERO_EXTEND; in reduceBuildVecExtToExtBuildVec()
22605 assert((Cast.getOpcode() == ISD::ANY_EXTEND || in reduceBuildVecExtToExtBuildVec()
22606 Cast.getOpcode() == ISD::ZERO_EXTEND || in reduceBuildVecExtToExtBuildVec()
22626 (!TLI.isOperationLegal(ISD::BUILD_VECTOR, VecVT) && in reduceBuildVecExtToExtBuildVec()
22627 TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))) in reduceBuildVecExtToExtBuildVec()
22644 assert(N->getOpcode() == ISD::BUILD_VECTOR && "Expected build vector"); in reduceBuildVecTruncToBitCast()
22669 if (Op.getOpcode() == ISD::BITCAST) in reduceBuildVecTruncToBitCast()
22681 if (In.getOpcode() != ISD::TRUNCATE) in reduceBuildVecTruncToBitCast()
22686 if (In.getOpcode() != ISD::SRL) { in reduceBuildVecTruncToBitCast()
22758 VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in createBuildVecShuffle()
22767 VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1, in createBuildVecShuffle()
22769 VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1, ZeroIdx); in createBuildVecShuffle()
22782 !TLI.isOperationLegal(ISD::VECTOR_SHUFFLE, InVT1)) in createBuildVecShuffle()
22791 VecIn2 = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InVT1, in createBuildVecShuffle()
22799 VecIn2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in createBuildVecShuffle()
22815 if (LegalOperations && !TLI.isOperationLegal(ISD::VECTOR_SHUFFLE, InVT1)) in createBuildVecShuffle()
22819 VecIn2 = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InVT1, in createBuildVecShuffle()
22860 Shuffle = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuffle, ZeroIdx); in createBuildVecShuffle()
22866 assert(BV->getOpcode() == ISD::BUILD_VECTOR && "Expected build vector"); in reduceBuildVecToShuffleWithZero()
22893 if (Zext.getOpcode() != ISD::ZERO_EXTEND || !Zext.hasOneUse() || in reduceBuildVecToShuffleWithZero()
22894 Zext.getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT || in reduceBuildVecToShuffleWithZero()
22965 if (LegalOperations && !TLI.isOperationLegal(ISD::VECTOR_SHUFFLE, VT)) in reduceBuildVecToShuffle()
22998 if (Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in reduceBuildVecToShuffle()
23060 SDValue VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, Vec, in reduceBuildVecToShuffle()
23062 SDValue VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, Vec, in reduceBuildVecToShuffle()
23173 bool IsLeftShuffle = L.getOpcode() == ISD::VECTOR_SHUFFLE && in reduceBuildVecToShuffle()
23182 bool IsRightShuffle = R.getOpcode() == ISD::VECTOR_SHUFFLE && in reduceBuildVecToShuffle()
23223 FoundZeroExtend |= (Opc == ISD::ZERO_EXTEND); in convertBuildVecZextToZext()
23224 if ((Opc == ISD::ZERO_EXTEND || Opc == ISD::ANY_EXTEND) && in convertBuildVecZextToZext()
23225 Op.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT && in convertBuildVecZextToZext()
23256 In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InVT, In, in convertBuildVecZextToZext()
23258 return DAG.getNode(FoundZeroExtend ? ISD::ZERO_EXTEND : ISD::ANY_EXTEND, DL, in convertBuildVecZextToZext()
23285 (LegalOperations && !TLI.isOperationLegalOrCustom(ISD::BITCAST, OpIntVT))) in convertBuildVecZextToBuildVecWithZeros()
23314 if (Op.getOpcode() != ISD::ZERO_EXTEND) in convertBuildVecZextToBuildVecWithZeros()
23349 !(TLI.isOperationLegalOrCustom(ISD::TRUNCATE, NewScalarIntVT) && in convertBuildVecZextToBuildVecWithZeros()
23350 TLI.isOperationLegalOrCustom(ISD::BUILD_VECTOR, NewIntVT)))) in convertBuildVecZextToBuildVecWithZeros()
23373 Op = DAG.getNode(ISD::TRUNCATE, DL, NewScalarIntVT, Op); in convertBuildVecZextToBuildVecWithZeros()
23387 if (ISD::allOperandsUndef(N)) in visitBUILD_VECTOR()
23408 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), in visitBUILD_VECTOR()
23420 if ((Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT) && in visitBUILD_VECTOR()
23441 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), N->getValueType(0), in visitBUILD_VECTOR()
23462 if (TLI.getOperationAction(ISD::SPLAT_VECTOR, VT) != TargetLowering::Expand) in visitBUILD_VECTOR()
23465 return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), VT, V); in visitBUILD_VECTOR()
23484 SDValue ScalarUndef = DAG.getNode(ISD::UNDEF, DL, SVT); in combineConcatVectorOfScalars()
23490 if (ISD::BITCAST == Op.getOpcode() && in combineConcatVectorOfScalars()
23493 else if (ISD::UNDEF == Op.getOpcode()) in combineConcatVectorOfScalars()
23514 ScalarUndef = DAG.getNode(ISD::UNDEF, DL, SVT); in combineConcatVectorOfScalars()
23545 if (Op.getOpcode() != ISD::CONCAT_VECTORS) in combineConcatVectorOfConcatVectors()
23567 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, ConcatOps); in combineConcatVectorOfConcatVectors()
23597 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR) in combineConcatVectorOfExtracts()
23651 case ISD::SINT_TO_FP: in combineConcatVectorOfCasts()
23652 case ISD::UINT_TO_FP: in combineConcatVectorOfCasts()
23653 case ISD::FP_TO_SINT: in combineConcatVectorOfCasts()
23654 case ISD::FP_TO_UINT: in combineConcatVectorOfCasts()
23689 case ISD::SINT_TO_FP: in combineConcatVectorOfCasts()
23690 case ISD::UINT_TO_FP: in combineConcatVectorOfCasts()
23695 case ISD::FP_TO_SINT: in combineConcatVectorOfCasts()
23696 case ISD::FP_TO_UINT: in combineConcatVectorOfCasts()
23707 SDValue NewConcat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatSrcVT, SrcOps); in combineConcatVectorOfCasts()
23729 !TLI.isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, VT))) in combineConcatVectorOfShuffleAndItsOperands()
23807 NewShufOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, ShufOpParts); in combineConcatVectorOfShuffleAndItsOperands()
23821 if (ISD::allOperandsUndef(N)) in visitCONCAT_VECTORS()
23837 if (In.getOpcode() == ISD::CONCAT_VECTORS && In.hasOneUse() && in visitCONCAT_VECTORS()
23842 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in visitCONCAT_VECTORS()
23849 if (!LegalOperations && Scalar.getOpcode() == ISD::SCALAR_TO_VECTOR && in visitCONCAT_VECTORS()
23861 if (Scalar->getOpcode() == ISD::TRUNCATE && in visitCONCAT_VECTORS()
23883 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), NVT, Scalar); in visitCONCAT_VECTORS()
23893 return ISD::UNDEF == Op.getOpcode() || ISD::BUILD_VECTOR == Op.getOpcode(); in visitCONCAT_VECTORS()
23905 if (ISD::BUILD_VECTOR == Op.getOpcode()) { in visitCONCAT_VECTORS()
23917 if (ISD::UNDEF == Op.getOpcode()) in visitCONCAT_VECTORS()
23920 if (ISD::BUILD_VECTOR == Op.getOpcode()) { in visitCONCAT_VECTORS()
23927 DAG.getNode(ISD::TRUNCATE, SDLoc(N), MinVT, Op.getOperand(i))); in visitCONCAT_VECTORS()
23976 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR) in visitCONCAT_VECTORS()
24008 if (V.getOpcode() == ISD::INSERT_SUBVECTOR && in getSubVectorSrc()
24013 if (IndexC && V.getOpcode() == ISD::CONCAT_VECTORS && in getSubVectorSrc()
24083 if (BOpcode == ISD::FSUB) { in narrowExtractedVectorBinOp()
24133 SDValue X = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT, in narrowExtractedVectorBinOp()
24135 SDValue Y = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT, in narrowExtractedVectorBinOp()
24152 if (BOpcode != ISD::AND && BOpcode != ISD::OR && BOpcode != ISD::XOR) in narrowExtractedVectorBinOp()
24158 if (V.getOpcode() == ISD::CONCAT_VECTORS && V.getNumOperands() == 2) in narrowExtractedVectorBinOp()
24174 : DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT, in narrowExtractedVectorBinOp()
24178 : DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT, in narrowExtractedVectorBinOp()
24260 assert(N->getOpcode() == ISD::EXTRACT_SUBVECTOR && in foldExtractSubvectorFromShuffleVector()
24282 !TLI.isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, NarrowVT)) in foldExtractSubvectorFromShuffleVector()
24390 NewOps.emplace_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowVT, in foldExtractSubvectorFromShuffleVector()
24412 if (TLI.isOperationLegalOrCustomOrPromote(ISD::LOAD, NVT)) in visitEXTRACT_SUBVECTOR()
24418 if (ExtIdx == 0 && V.getOpcode() == ISD::EXTRACT_SUBVECTOR && V.hasOneUse()) { in visitEXTRACT_SUBVECTOR()
24421 TLI.isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, NVT)) { in visitEXTRACT_SUBVECTOR()
24422 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), NVT, V.getOperand(0), in visitEXTRACT_SUBVECTOR()
24428 if (V.getOpcode() == ISD::SPLAT_VECTOR) in visitEXTRACT_SUBVECTOR()
24430 if (!LegalOperations || TLI.isOperationLegal(ISD::SPLAT_VECTOR, NVT)) in visitEXTRACT_SUBVECTOR()
24435 if (V.getOpcode() == ISD::BITCAST && in visitEXTRACT_SUBVECTOR()
24437 (!LegalOperations || TLI.isOperationLegal(ISD::BITCAST, NVT))) { in visitEXTRACT_SUBVECTOR()
24447 if (TLI.isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, NewExtVT)) { in visitEXTRACT_SUBVECTOR()
24450 SDValue NewExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewExtVT, in visitEXTRACT_SUBVECTOR()
24466 if (TLI.isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, NewExtVT)) { in visitEXTRACT_SUBVECTOR()
24469 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewExtVT, in visitEXTRACT_SUBVECTOR()
24474 TLI.isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT, ScalarVT)) { in visitEXTRACT_SUBVECTOR()
24477 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, in visitEXTRACT_SUBVECTOR()
24486 if (V.getOpcode() == ISD::CONCAT_VECTORS) { in visitEXTRACT_SUBVECTOR()
24517 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NVT, in visitEXTRACT_SUBVECTOR()
24529 if (V.getOpcode() == ISD::BUILD_VECTOR) { in visitEXTRACT_SUBVECTOR()
24542 TLI.isOperationLegal(ISD::BUILD_VECTOR, ExtractVT))) && in visitEXTRACT_SUBVECTOR()
24549 Src = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Src); in visitEXTRACT_SUBVECTOR()
24561 if (V.getOpcode() == ISD::INSERT_SUBVECTOR) { in visitEXTRACT_SUBVECTOR()
24576 if (LegalOperations && !TLI.isOperationLegal(ISD::BITCAST, NVT)) in visitEXTRACT_SUBVECTOR()
24582 ISD::EXTRACT_SUBVECTOR, SDLoc(N), NVT, in visitEXTRACT_SUBVECTOR()
24603 if (N0.getOpcode() != ISD::CONCAT_VECTORS || N0.getNumOperands() != 2 || in foldShuffleOfConcatUndefs()
24604 N1.getOpcode() != ISD::CONCAT_VECTORS || N1.getNumOperands() != 2 || in foldShuffleOfConcatUndefs()
24643 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Shuf0, Shuf1); in foldShuffleOfConcatUndefs()
24674 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, N0, N1); in partitionShuffleOfConcats()
24708 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in partitionShuffleOfConcats()
24746 if (N0AnyConst && !N1AnyConst && !ISD::isBuildVectorAllZeros(N0.getNode())) in combineShuffleOfScalars()
24748 if (!N0AnyConst && N1AnyConst && !ISD::isBuildVectorAllZeros(N1.getNode())) in combineShuffleOfScalars()
24768 if (S.getOpcode() == ISD::BUILD_VECTOR) { in combineShuffleOfScalars()
24770 } else if (S.getOpcode() == ISD::SCALAR_TO_VECTOR) { in combineShuffleOfScalars()
24872 unsigned Opcode = ISD::ANY_EXTEND_VECTOR_INREG; in combineShuffleToAnyExtendVectorInreg()
24992 unsigned Opcode = ISD::ZERO_EXTEND_VECTOR_INREG; in combineShuffleToZeroExtendVectorInReg()
25024 if (!ISD::isExtVecInRegOpcode(Opcode)) in combineTruncationShuffle()
25180 if (Op0.getOpcode() != ISD::BITCAST) in combineShuffleOfBitcast()
25184 (!Op1.isUndef() && (Op1.getOpcode() != ISD::BITCAST || in combineShuffleOfBitcast()
25195 !TLI.isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, InVT))) in combineShuffleOfBitcast()
25324 if (Op0.getOpcode() != ISD::INSERT_VECTOR_ELT) in replaceShuffleOfInsert()
25342 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(Shuf), Op0.getValueType(), in replaceShuffleOfInsert()
25436 SDValue ExtL = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, L, Index); in visitVECTOR_SHUFFLE()
25437 SDValue ExtR = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, R, Index); in visitVECTOR_SHUFFLE()
25440 SDValue Insert = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, NewBO); in visitVECTOR_SHUFFLE()
25447 if ((!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) && in visitVECTOR_SHUFFLE()
25449 if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR && SplatIndex == 0) in visitVECTOR_SHUFFLE()
25452 if (N0.getOpcode() == ISD::INSERT_VECTOR_ELT) in visitVECTOR_SHUFFLE()
25459 if (N0.getOpcode() == ISD::BITCAST && N0.getOperand(0).hasOneUse() && in visitVECTOR_SHUFFLE()
25461 (N0.getOperand(0).getOpcode() == ISD::SCALAR_TO_VECTOR || in visitVECTOR_SHUFFLE()
25462 N0.getOperand(0).getOpcode() == ISD::BUILD_VECTOR)) { in visitVECTOR_SHUFFLE()
25479 if (V->getOpcode() == ISD::BITCAST) { in visitVECTOR_SHUFFLE()
25486 if (V->getOpcode() == ISD::BUILD_VECTOR) { in visitVECTOR_SHUFFLE()
25541 if (N0.getOpcode() == ISD::CONCAT_VECTORS && in visitVECTOR_SHUFFLE()
25544 (N1.getOpcode() == ISD::CONCAT_VECTORS && in visitVECTOR_SHUFFLE()
25553 if (N0.getOpcode() == ISD::CONCAT_VECTORS && N1.isUndef() && in visitVECTOR_SHUFFLE()
25568 SDValue NewCat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, in visitVECTOR_SHUFFLE()
25579 TLI.isOperationLegalOrCustom(ISD::INSERT_SUBVECTOR, VT)) { in visitVECTOR_SHUFFLE()
25582 assert(RHS.getOpcode() == ISD::CONCAT_VECTORS && "Can't find subvectors"); in visitVECTOR_SHUFFLE()
25619 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, LHS, in visitVECTOR_SHUFFLE()
25627 if (N1.getOpcode() == ISD::CONCAT_VECTORS) in visitVECTOR_SHUFFLE()
25630 if (N0.getOpcode() == ISD::CONCAT_VECTORS) { in visitVECTOR_SHUFFLE()
25684 if (TLI.isOperationLegalOrCustom(ISD::AND, IntVT)) in visitVECTOR_SHUFFLE()
25686 VT, DAG.getNode(ISD::AND, DL, IntVT, DAG.getBitcast(IntVT, N0), in visitVECTOR_SHUFFLE()
25701 if (N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() && in visitVECTOR_SHUFFLE()
25706 if (BC0.getOpcode() == ISD::VECTOR_SHUFFLE && BC0.hasOneUse()) { in visitVECTOR_SHUFFLE()
25880 if (N1.getOpcode() == ISD::VECTOR_SHUFFLE && in visitVECTOR_SHUFFLE()
25881 N0.getOpcode() != ISD::VECTOR_SHUFFLE) { in visitVECTOR_SHUFFLE()
25898 if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && in visitVECTOR_SHUFFLE()
25899 N1.getOpcode() == ISD::VECTOR_SHUFFLE && in visitVECTOR_SHUFFLE()
25913 if (N->getOperand(i).getOpcode() == ISD::VECTOR_SHUFFLE && in visitVECTOR_SHUFFLE()
25954 (Op00.getOpcode() == ISD::VECTOR_SHUFFLE || in visitVECTOR_SHUFFLE()
25955 Op10.getOpcode() == ISD::VECTOR_SHUFFLE || in visitVECTOR_SHUFFLE()
25956 Op01.getOpcode() == ISD::VECTOR_SHUFFLE || in visitVECTOR_SHUFFLE()
25957 Op11.getOpcode() == ISD::VECTOR_SHUFFLE)) { in visitVECTOR_SHUFFLE()
26056 if (C && EE.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitSCALAR_TO_VECTOR()
26076 if (Opcode != ISD::EXTRACT_VECTOR_ELT || in visitSCALAR_TO_VECTOR()
26083 SDValue Val = DAG.getNode(ISD::TRUNCATE, SDLoc(Scalar), VecEltVT, Scalar); in visitSCALAR_TO_VECTOR()
26084 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Val); in visitSCALAR_TO_VECTOR()
26113 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), SubVT, LegalShuffle, in visitSCALAR_TO_VECTOR()
26135 if (N0.isUndef() && N1.getOpcode() == ISD::EXTRACT_SUBVECTOR && in visitINSERT_SUBVECTOR()
26145 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), in visitINSERT_SUBVECTOR()
26148 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), in visitINSERT_SUBVECTOR()
26155 if (N0.isUndef() && N1.getOpcode() == ISD::SPLAT_VECTOR) in visitINSERT_SUBVECTOR()
26157 return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), VT, N1.getOperand(0)); in visitINSERT_SUBVECTOR()
26163 if (N0.isUndef() && N1.getOpcode() == ISD::BITCAST && in visitINSERT_SUBVECTOR()
26164 N1.getOperand(0).getOpcode() == ISD::EXTRACT_SUBVECTOR && in visitINSERT_SUBVECTOR()
26177 if (N0.getOpcode() == ISD::BITCAST && N1.getOpcode() == ISD::BITCAST) { in visitINSERT_SUBVECTOR()
26185 SDValue NewINSERT = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), in visitINSERT_SUBVECTOR()
26194 if (N0.getOpcode() == ISD::INSERT_SUBVECTOR && in visitINSERT_SUBVECTOR()
26197 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0.getOperand(0), in visitINSERT_SUBVECTOR()
26203 if (N0.isUndef() && N1.getOpcode() == ISD::INSERT_SUBVECTOR && in visitINSERT_SUBVECTOR()
26206 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0, in visitINSERT_SUBVECTOR()
26212 if ((N0.isUndef() || N0.getOpcode() == ISD::BITCAST) && in visitINSERT_SUBVECTOR()
26213 N1.getOpcode() == ISD::BITCAST) { in visitINSERT_SUBVECTOR()
26238 if (NewIdx && hasOperation(ISD::INSERT_SUBVECTOR, NewVT)) { in visitINSERT_SUBVECTOR()
26240 Res = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, NewVT, Res, N1Src, NewIdx); in visitINSERT_SUBVECTOR()
26250 if (N0.getOpcode() == ISD::INSERT_SUBVECTOR && N0.hasOneUse() && in visitINSERT_SUBVECTOR()
26255 SDValue NewOp = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, in visitINSERT_SUBVECTOR()
26258 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N0.getNode()), in visitINSERT_SUBVECTOR()
26265 if (N0.getOpcode() == ISD::CONCAT_VECTORS && N0.hasOneUse() && in visitINSERT_SUBVECTOR()
26272 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in visitINSERT_SUBVECTOR()
26286 if (N0->getOpcode() == ISD::FP16_TO_FP) in visitFP_TO_FP16()
26294 assert((Op == ISD::FP16_TO_FP || Op == ISD::BF16_TO_FP) && in visitFP16_TO_FP()
26300 if (!TLI.shouldKeepZExtForFP16Conv() && N0->getOpcode() == ISD::AND) { in visitFP16_TO_FP()
26314 if (N0->getOpcode() == ISD::BF16_TO_FP) in visitFP_TO_BF16()
26334 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getVectorElementType(), N0, in visitVECREDUCE()
26337 Res = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Res); in visitVECREDUCE()
26343 if (Opcode == ISD::VECREDUCE_AND || Opcode == ISD::VECREDUCE_OR) { in visitVECREDUCE()
26344 unsigned NewOpcode = Opcode == ISD::VECREDUCE_AND in visitVECREDUCE()
26345 ? ISD::VECREDUCE_UMIN : ISD::VECREDUCE_UMAX; in visitVECREDUCE()
26354 if (N0.getOpcode() == ISD::INSERT_SUBVECTOR && in visitVECREDUCE()
26358 if ((Opcode == ISD::VECREDUCE_OR && in visitVECREDUCE()
26360 (Opcode == ISD::VECREDUCE_AND && in visitVECREDUCE()
26381 if (N->getOpcode() == ISD::VP_GATHER) in visitVPOp()
26385 if (N->getOpcode() == ISD::VP_SCATTER) in visitVPOp()
26389 if (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD) in visitVPOp()
26393 if (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE) in visitVPOp()
26401 if (auto EVLIdx = ISD::getVPExplicitVectorLengthIdx(N->getOpcode())) in visitVPOp()
26403 if (auto MaskIdx = ISD::getVPMaskIdx(N->getOpcode())) in visitVPOp()
26405 ISD::isConstantSplatVectorAllZeros(N->getOperand(*MaskIdx).getNode()); in visitVPOp()
26410 case ISD::VP_FADD: in visitVPOp()
26412 case ISD::VP_FSUB: in visitVPOp()
26414 case ISD::VP_FMA: in visitVPOp()
26421 if (ISD::isVPBinaryOp(N->getOpcode())) in visitVPOp()
26433 if (ISD::isVPReduction(N->getOpcode())) in visitVPOp()
26535 assert(N->getOpcode() == ISD::AND && "Unexpected opcode!"); in XformToShuffleWithZero()
26547 if (RHS.getOpcode() != ISD::BUILD_VECTOR) in XformToShuffleWithZero()
26638 bool IsBothSplatVector = N0.getOpcode() == ISD::SPLAT_VECTOR && in scalarizeBinOpOfSplats()
26639 N1.getOpcode() == ISD::SPLAT_VECTOR; in scalarizeBinOpOfSplats()
26648 SDValue X = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src0, IndexC); in scalarizeBinOpOfSplats()
26649 SDValue Y = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src1, IndexC); in scalarizeBinOpOfSplats()
26654 if (N0.getOpcode() == ISD::BUILD_VECTOR && N0.getOpcode() == N1.getOpcode() && in scalarizeBinOpOfSplats()
26682 (N0.getOpcode() == ISD::SPLAT_VECTOR || in SimplifyVCastOp()
26690 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcEltVT, Src0, IndexC); in SimplifyVCastOp()
26738 Shuf0->getOperand(0).getOpcode() != ISD::INSERT_VECTOR_ELT) { in SimplifyVBinOp()
26747 Shuf1->getOperand(0).getOpcode() != ISD::INSERT_VECTOR_ELT) { in SimplifyVBinOp()
26760 if (LHS.getOpcode() == ISD::INSERT_SUBVECTOR && LHS.getOperand(0).isUndef() && in SimplifyVBinOp()
26761 RHS.getOpcode() == ISD::INSERT_SUBVECTOR && RHS.getOperand(0).isUndef() && in SimplifyVBinOp()
26775 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, VecC, NarrowBO, Z); in SimplifyVBinOp()
26781 return Concat.getOpcode() == ISD::CONCAT_VECTORS && in SimplifyVBinOp()
26784 ISD::isBuildVectorOfConstantSDNodes(Op.getNode()); in SimplifyVBinOp()
26806 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in SimplifyVBinOp()
26818 assert(N0.getOpcode() == ISD::SETCC && in SimplifySelect()
26830 if (SCC.getOpcode() == ISD::SELECT_CC) { in SimplifySelect()
26832 SDValue SETCC = DAG.getNode(ISD::SETCC, SDLoc(N0), in SimplifySelect()
26858 if (NaN->isNaN() && RHS.getOpcode() == ISD::FSQRT) { in SimplifySelectOps()
26861 ISD::CondCode CC; in SimplifySelectOps()
26865 if (TheSelect->getOpcode() == ISD::SELECT_CC) { in SimplifySelectOps()
26872 if (Cmp.getOpcode() == ISD::SETCC) { in SimplifySelectOps()
26879 Sqrt.getOperand(0) == CmpLHS && (CC == ISD::SETOLT || in SimplifySelectOps()
26880 CC == ISD::SETULT || CC == ISD::SETLT)) { in SimplifySelectOps()
26900 if (LHS.getOpcode() == ISD::LOAD) { in SimplifySelectOps()
26918 LLD->getExtensionType() != ISD::EXTLOAD && in SimplifySelectOps()
26919 RLD->getExtensionType() != ISD::EXTLOAD) || in SimplifySelectOps()
26929 LLD->getBasePtr().getOpcode() == ISD::TargetFrameIndex || in SimplifySelectOps()
26930 RLD->getBasePtr().getOpcode() == ISD::TargetFrameIndex || in SimplifySelectOps()
26958 if (TheSelect->getOpcode() == ISD::SELECT) { in SimplifySelectOps()
26995 Addr = DAG.getNode(ISD::SELECT_CC, SDLoc(TheSelect), in SimplifySelectOps()
27013 if (LLD->getExtensionType() == ISD::NON_EXTLOAD) { in SimplifySelectOps()
27021 LLD->getExtensionType() == ISD::EXTLOAD ? RLD->getExtensionType() in SimplifySelectOps()
27044 ISD::CondCode CC) { in foldSelectCCToShiftAnd()
27057 if (CC == ISD::SETGT && TLI.hasAndNot(N2)) { in foldSelectCCToShiftAnd()
27062 } else if (CC == ISD::SETLT) { in foldSelectCCToShiftAnd()
27079 SDValue Shift = DAG.getNode(ISD::SRL, DL, XType, N0, ShiftAmt); in foldSelectCCToShiftAnd()
27083 Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift); in foldSelectCCToShiftAnd()
27087 if (CC == ISD::SETGT) in foldSelectCCToShiftAnd()
27090 return DAG.getNode(ISD::AND, DL, AType, Shift, N2); in foldSelectCCToShiftAnd()
27099 SDValue Shift = DAG.getNode(ISD::SRA, DL, XType, N0, ShiftAmt); in foldSelectCCToShiftAnd()
27103 Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift); in foldSelectCCToShiftAnd()
27107 if (CC == ISD::SETGT) in foldSelectCCToShiftAnd()
27110 return DAG.getNode(ISD::AND, DL, AType, Shift, N2); in foldSelectCCToShiftAnd()
27172 bool IsFabs = N->getOpcode() == ISD::FABS; in foldSignChangeInBitcast()
27175 if (IsFree || N0.getOpcode() != ISD::BITCAST || !N0.hasOneUse()) in foldSignChangeInBitcast()
27202 Int = DAG.getNode(IsFabs ? ISD::AND : ISD::XOR, DL, IntVT, Int, in foldSignChangeInBitcast()
27214 ISD::CondCode CC) { in convertSelectOfFPConstantsToLoadOffset()
27227 if (TLI.getOperationAction(ISD::ConstantFP, VT) == TargetLowering::Legal || in convertSelectOfFPConstantsToLoadOffset()
27258 CPIdx = DAG.getNode(ISD::ADD, DL, CPIdx.getValueType(), CPIdx, CstOffset); in convertSelectOfFPConstantsToLoadOffset()
27268 SDValue N2, SDValue N3, ISD::CondCode CC, in SimplifySelectCC()
27303 if (CC == ISD::SETEQ && N0->getOpcode() == ISD::AND && in SimplifySelectCC()
27315 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N0), VT, AndLHS, ShlAmt); in SimplifySelectCC()
27322 SDValue Shr = DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl, ShrAmt); in SimplifySelectCC()
27324 return DAG.getNode(ISD::AND, DL, VT, Shr, N3); in SimplifySelectCC()
27336 (!LegalOperations || TLI.isOperationLegal(ISD::SETCC, CmpOpVT))) { in SimplifySelectCC()
27339 CC = ISD::getSetCCInverse(CC, CmpOpVT); in SimplifySelectCC()
27355 Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2), VT, SCC); in SimplifySelectCC()
27369 return DAG.getNode(ISD::SHL, DL, N2.getValueType(), Temp, in SimplifySelectCC()
27382 if (N1C && N1C->isZero() && (CC == ISD::SETEQ || CC == ISD::SETNE)) { in SimplifySelectCC()
27386 if (CC == ISD::SETNE) in SimplifySelectCC()
27393 if ((Count.getOpcode() == ISD::CTTZ || in SimplifySelectCC()
27394 Count.getOpcode() == ISD::CTTZ_ZERO_UNDEF) && in SimplifySelectCC()
27396 (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ, VT))) in SimplifySelectCC()
27397 return DAG.getNode(ISD::CTTZ, DL, VT, N0); in SimplifySelectCC()
27400 if ((Count.getOpcode() == ISD::CTLZ || in SimplifySelectCC()
27401 Count.getOpcode() == ISD::CTLZ_ZERO_UNDEF) && in SimplifySelectCC()
27403 (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ, VT))) in SimplifySelectCC()
27404 return DAG.getNode(ISD::CTLZ, DL, VT, N0); in SimplifySelectCC()
27413 ((N1C->isAllOnes() && CC == ISD::SETGT) || in SimplifySelectCC()
27414 (N1C->isZero() && CC == ISD::SETLT)) && in SimplifySelectCC()
27417 ISD::SRA, DL, CmpOpVT, N0, in SimplifySelectCC()
27419 return DAG.getNode(ISD::XOR, DL, VT, DAG.getSExtOrTrunc(ASR, DL, VT), in SimplifySelectCC()
27420 DAG.getSExtOrTrunc(CC == ISD::SETLT ? N3 : N2, DL, VT)); in SimplifySelectCC()
27433 ISD::CondCode Cond, const SDLoc &DL, in SimplifySetCC()
27540 case ISD::TRUNCATE: in takeInexpensiveLog2()
27541 case ISD::ZERO_EXTEND: in takeInexpensiveLog2()
27569 if (ISD::matchUnaryPredicate(Op, IsPowerOfTwo)) { in takeInexpensiveLog2()
27596 if (Op.getOpcode() == ISD::SHL) { in takeInexpensiveLog2()
27602 return DAG.getNode(ISD::ADD, DL, VT, LogX, in takeInexpensiveLog2()
27607 if ((Op.getOpcode() == ISD::SELECT || Op.getOpcode() == ISD::VSELECT) && in takeInexpensiveLog2()
27618 if ((Op.getOpcode() == ISD::UMIN || Op.getOpcode() == ISD::UMAX) && in takeInexpensiveLog2()
27645 SDValue Ctlz = DAG.getNode(ISD::CTLZ, DL, VT, V); in BuildLogBase2()
27647 SDValue LogBase2 = DAG.getNode(ISD::SUB, DL, VT, Base, Ctlz); in BuildLogBase2()
27692 MulEst = DAG.getNode(ISD::FMUL, DL, VT, N, Est, Flags); in BuildDivEstimate()
27696 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Op, MulEst, Flags); in BuildDivEstimate()
27699 NewEst = DAG.getNode(ISD::FSUB, DL, VT, in BuildDivEstimate()
27703 NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags); in BuildDivEstimate()
27706 Est = DAG.getNode(ISD::FADD, DL, VT, MulEst, NewEst, Flags); in BuildDivEstimate()
27711 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, N, Flags); in BuildDivEstimate()
27736 SDValue HalfArg = DAG.getNode(ISD::FMUL, DL, VT, ThreeHalves, Arg, Flags); in buildSqrtNROneConst()
27737 HalfArg = DAG.getNode(ISD::FSUB, DL, VT, HalfArg, Arg, Flags); in buildSqrtNROneConst()
27741 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, Est, Flags); in buildSqrtNROneConst()
27742 NewEst = DAG.getNode(ISD::FMUL, DL, VT, HalfArg, NewEst, Flags); in buildSqrtNROneConst()
27743 NewEst = DAG.getNode(ISD::FSUB, DL, VT, ThreeHalves, NewEst, Flags); in buildSqrtNROneConst()
27744 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags); in buildSqrtNROneConst()
27749 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, Arg, Flags); in buildSqrtNROneConst()
27774 SDValue AE = DAG.getNode(ISD::FMUL, DL, VT, Arg, Est, Flags); in buildSqrtNRTwoConst()
27775 SDValue AEE = DAG.getNode(ISD::FMUL, DL, VT, AE, Est, Flags); in buildSqrtNRTwoConst()
27776 SDValue RHS = DAG.getNode(ISD::FADD, DL, VT, AEE, MinusThree, Flags); in buildSqrtNRTwoConst()
27784 LHS = DAG.getNode(ISD::FMUL, DL, VT, Est, MinusHalf, Flags); in buildSqrtNRTwoConst()
27787 LHS = DAG.getNode(ISD::FMUL, DL, VT, AE, MinusHalf, Flags); in buildSqrtNRTwoConst()
27790 Est = DAG.getNode(ISD::FMUL, DL, VT, LHS, RHS, Flags); in buildSqrtNRTwoConst()
27839 Test.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT, DL, VT, in buildSqrtEstimateImpl()
27872 Offset = (LSN->getAddressingMode() == ISD::PRE_INC) in mayAlias()
27874 : (LSN->getAddressingMode() == ISD::PRE_DEC) in mayAlias()
28012 case ISD::EntryToken: in GatherAllAliases()
28016 case ISD::LOAD: in GatherAllAliases()
28017 case ISD::STORE: { in GatherAllAliases()
28031 case ISD::CopyFromReg: in GatherAllAliases()
28036 case ISD::LIFETIME_START: in GatherAllAliases()
28037 case ISD::LIFETIME_END: { in GatherAllAliases()
28074 if (Chain.getOpcode() == ISD::TokenFactor) { in GatherAllAliases()