Lines Matching refs:ISD
261 assert(N->getOpcode() != ISD::DELETED_NODE && in AddToWorklist()
266 if (N->getOpcode() == ISD::HANDLENODE) in AddToWorklist()
541 SDValue N2, SDValue N3, ISD::CondCode CC,
545 ISD::CondCode CC);
548 SDValue N2, SDValue N3, ISD::CondCode CC);
556 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
655 case ISD::Constant: in getStoreSource()
656 case ISD::ConstantFP: in getStoreSource()
658 case ISD::EXTRACT_VECTOR_ELT: in getStoreSource()
659 case ISD::EXTRACT_SUBVECTOR: in getStoreSource()
661 case ISD::LOAD: in getStoreSource()
683 bool isLegalNarrowLdSt(LSBaseSDNode *LDSTN, ISD::LoadExtType ExtType,
798 ISD::NodeType ExtType);
897 if (N.getOpcode() == ISD::SETCC) { in isSetCCEquivalent()
905 (N.getOpcode() == ISD::STRICT_FSETCC || in isSetCCEquivalent()
906 N.getOpcode() == ISD::STRICT_FSETCCS)) { in isSetCCEquivalent()
913 if (N.getOpcode() != ISD::SELECT_CC || !TLI.isConstTrueVal(N.getOperand(2)) || in isSetCCEquivalent()
958 if (ISD::isConstantSplatVector(N, Val)) in isConstantSplatVectorMaskForType()
970 if (N.getOpcode() != ISD::BUILD_VECTOR && N.getOpcode() != ISD::SPLAT_VECTOR) in isConstantOrConstantVector()
987 if (V.getOpcode() != ISD::BUILD_VECTOR) in isAnyConstantBuildVector()
990 ISD::isBuildVectorOfConstantFPSDNodes(V.getNode()); in isAnyConstantBuildVector()
996 (LD->getOperand(2).getOpcode() != ISD::TargetConstant || in canSplitIdx()
1014 if (Opc != ISD::ADD || N0.getOpcode() != ISD::ADD) in reassociationCanBreakAddressingModePattern()
1057 if (GA->getOpcode() == ISD::GlobalAddress && TLI.isOffsetFoldingLegal(GA)) in reassociationCanBreakAddressingModePattern()
1110 if (Opc == ISD::AND || Opc == ISD::OR) { in reassociateOpsCommutative()
1118 if (Opc == ISD::XOR) { in reassociateOpsCommutative()
1262 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0)); in ReplaceLoadWithPromotedLoad()
1276 if (ISD::isUNINDEXEDLoad(Op.getNode())) { in PromoteOperand()
1279 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD in PromoteOperand()
1290 case ISD::AssertSext: in PromoteOperand()
1292 return DAG.getNode(ISD::AssertSext, DL, PVT, Op0, Op.getOperand(1)); in PromoteOperand()
1294 case ISD::AssertZext: in PromoteOperand()
1296 return DAG.getNode(ISD::AssertZext, DL, PVT, Op0, Op.getOperand(1)); in PromoteOperand()
1298 case ISD::Constant: { in PromoteOperand()
1300 Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in PromoteOperand()
1305 if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT)) in PromoteOperand()
1307 return DAG.getNode(ISD::ANY_EXTEND, DL, PVT, Op); in PromoteOperand()
1311 if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT)) in SExtPromoteOperand()
1323 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, NewOp.getValueType(), NewOp, in SExtPromoteOperand()
1376 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1)); in PromoteIntBinOp()
1436 if (Opc == ISD::SRA) in PromoteIntShiftOp()
1438 else if (Opc == ISD::SRL) in PromoteIntShiftOp()
1449 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1)); in PromoteIntShiftOp()
1455 if (Op && Op.getOpcode() != ISD::DELETED_NODE) in PromoteIntShiftOp()
1493 if (!ISD::isUNINDEXEDLoad(Op.getNode())) in PromoteLoad()
1516 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD in PromoteLoad()
1521 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD); in PromoteLoad()
1634 assert(N->getOpcode() != ISD::DELETED_NODE && in Run()
1635 RV.getOpcode() != ISD::DELETED_NODE && in Run()
1653 if (RV.getOpcode() != ISD::EntryToken) { in Run()
1673 case ISD::TokenFactor: return visitTokenFactor(N); in visit()
1674 case ISD::MERGE_VALUES: return visitMERGE_VALUES(N); in visit()
1675 case ISD::ADD: return visitADD(N); in visit()
1676 case ISD::SUB: return visitSUB(N); in visit()
1677 case ISD::SADDSAT: in visit()
1678 case ISD::UADDSAT: return visitADDSAT(N); in visit()
1679 case ISD::SSUBSAT: in visit()
1680 case ISD::USUBSAT: return visitSUBSAT(N); in visit()
1681 case ISD::ADDC: return visitADDC(N); in visit()
1682 case ISD::SADDO: in visit()
1683 case ISD::UADDO: return visitADDO(N); in visit()
1684 case ISD::SUBC: return visitSUBC(N); in visit()
1685 case ISD::SSUBO: in visit()
1686 case ISD::USUBO: return visitSUBO(N); in visit()
1687 case ISD::ADDE: return visitADDE(N); in visit()
1688 case ISD::ADDCARRY: return visitADDCARRY(N); in visit()
1689 case ISD::SADDO_CARRY: return visitSADDO_CARRY(N); in visit()
1690 case ISD::SUBE: return visitSUBE(N); in visit()
1691 case ISD::SUBCARRY: return visitSUBCARRY(N); in visit()
1692 case ISD::SSUBO_CARRY: return visitSSUBO_CARRY(N); in visit()
1693 case ISD::SMULFIX: in visit()
1694 case ISD::SMULFIXSAT: in visit()
1695 case ISD::UMULFIX: in visit()
1696 case ISD::UMULFIXSAT: return visitMULFIX(N); in visit()
1697 case ISD::MUL: return visitMUL(N); in visit()
1698 case ISD::SDIV: return visitSDIV(N); in visit()
1699 case ISD::UDIV: return visitUDIV(N); in visit()
1700 case ISD::SREM: in visit()
1701 case ISD::UREM: return visitREM(N); in visit()
1702 case ISD::MULHU: return visitMULHU(N); in visit()
1703 case ISD::MULHS: return visitMULHS(N); in visit()
1704 case ISD::AVGFLOORS: in visit()
1705 case ISD::AVGFLOORU: in visit()
1706 case ISD::AVGCEILS: in visit()
1707 case ISD::AVGCEILU: return visitAVG(N); in visit()
1708 case ISD::SMUL_LOHI: return visitSMUL_LOHI(N); in visit()
1709 case ISD::UMUL_LOHI: return visitUMUL_LOHI(N); in visit()
1710 case ISD::SMULO: in visit()
1711 case ISD::UMULO: return visitMULO(N); in visit()
1712 case ISD::SMIN: in visit()
1713 case ISD::SMAX: in visit()
1714 case ISD::UMIN: in visit()
1715 case ISD::UMAX: return visitIMINMAX(N); in visit()
1716 case ISD::AND: return visitAND(N); in visit()
1717 case ISD::OR: return visitOR(N); in visit()
1718 case ISD::XOR: return visitXOR(N); in visit()
1719 case ISD::SHL: return visitSHL(N); in visit()
1720 case ISD::SRA: return visitSRA(N); in visit()
1721 case ISD::SRL: return visitSRL(N); in visit()
1722 case ISD::ROTR: in visit()
1723 case ISD::ROTL: return visitRotate(N); in visit()
1724 case ISD::FSHL: in visit()
1725 case ISD::FSHR: return visitFunnelShift(N); in visit()
1726 case ISD::SSHLSAT: in visit()
1727 case ISD::USHLSAT: return visitSHLSAT(N); in visit()
1728 case ISD::ABS: return visitABS(N); in visit()
1729 case ISD::BSWAP: return visitBSWAP(N); in visit()
1730 case ISD::BITREVERSE: return visitBITREVERSE(N); in visit()
1731 case ISD::CTLZ: return visitCTLZ(N); in visit()
1732 case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N); in visit()
1733 case ISD::CTTZ: return visitCTTZ(N); in visit()
1734 case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N); in visit()
1735 case ISD::CTPOP: return visitCTPOP(N); in visit()
1736 case ISD::SELECT: return visitSELECT(N); in visit()
1737 case ISD::VSELECT: return visitVSELECT(N); in visit()
1738 case ISD::SELECT_CC: return visitSELECT_CC(N); in visit()
1739 case ISD::SETCC: return visitSETCC(N); in visit()
1740 case ISD::SETCCCARRY: return visitSETCCCARRY(N); in visit()
1741 case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N); in visit()
1742 case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N); in visit()
1743 case ISD::ANY_EXTEND: return visitANY_EXTEND(N); in visit()
1744 case ISD::AssertSext: in visit()
1745 case ISD::AssertZext: return visitAssertExt(N); in visit()
1746 case ISD::AssertAlign: return visitAssertAlign(N); in visit()
1747 case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N); in visit()
1748 case ISD::SIGN_EXTEND_VECTOR_INREG: in visit()
1749 case ISD::ZERO_EXTEND_VECTOR_INREG: return visitEXTEND_VECTOR_INREG(N); in visit()
1750 case ISD::TRUNCATE: return visitTRUNCATE(N); in visit()
1751 case ISD::BITCAST: return visitBITCAST(N); in visit()
1752 case ISD::BUILD_PAIR: return visitBUILD_PAIR(N); in visit()
1753 case ISD::FADD: return visitFADD(N); in visit()
1754 case ISD::STRICT_FADD: return visitSTRICT_FADD(N); in visit()
1755 case ISD::FSUB: return visitFSUB(N); in visit()
1756 case ISD::FMUL: return visitFMUL(N); in visit()
1757 case ISD::FMA: return visitFMA(N); in visit()
1758 case ISD::FDIV: return visitFDIV(N); in visit()
1759 case ISD::FREM: return visitFREM(N); in visit()
1760 case ISD::FSQRT: return visitFSQRT(N); in visit()
1761 case ISD::FCOPYSIGN: return visitFCOPYSIGN(N); in visit()
1762 case ISD::FPOW: return visitFPOW(N); in visit()
1763 case ISD::SINT_TO_FP: return visitSINT_TO_FP(N); in visit()
1764 case ISD::UINT_TO_FP: return visitUINT_TO_FP(N); in visit()
1765 case ISD::FP_TO_SINT: return visitFP_TO_SINT(N); in visit()
1766 case ISD::FP_TO_UINT: return visitFP_TO_UINT(N); in visit()
1767 case ISD::FP_ROUND: return visitFP_ROUND(N); in visit()
1768 case ISD::FP_EXTEND: return visitFP_EXTEND(N); in visit()
1769 case ISD::FNEG: return visitFNEG(N); in visit()
1770 case ISD::FABS: return visitFABS(N); in visit()
1771 case ISD::FFLOOR: return visitFFLOOR(N); in visit()
1772 case ISD::FMINNUM: in visit()
1773 case ISD::FMAXNUM: in visit()
1774 case ISD::FMINIMUM: in visit()
1775 case ISD::FMAXIMUM: return visitFMinMax(N); in visit()
1776 case ISD::FCEIL: return visitFCEIL(N); in visit()
1777 case ISD::FTRUNC: return visitFTRUNC(N); in visit()
1778 case ISD::BRCOND: return visitBRCOND(N); in visit()
1779 case ISD::BR_CC: return visitBR_CC(N); in visit()
1780 case ISD::LOAD: return visitLOAD(N); in visit()
1781 case ISD::STORE: return visitSTORE(N); in visit()
1782 case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N); in visit()
1783 case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N); in visit()
1784 case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N); in visit()
1785 case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N); in visit()
1786 case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N); in visit()
1787 case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N); in visit()
1788 case ISD::SCALAR_TO_VECTOR: return visitSCALAR_TO_VECTOR(N); in visit()
1789 case ISD::INSERT_SUBVECTOR: return visitINSERT_SUBVECTOR(N); in visit()
1790 case ISD::MGATHER: return visitMGATHER(N); in visit()
1791 case ISD::MLOAD: return visitMLOAD(N); in visit()
1792 case ISD::MSCATTER: return visitMSCATTER(N); in visit()
1793 case ISD::MSTORE: return visitMSTORE(N); in visit()
1794 case ISD::LIFETIME_END: return visitLIFETIME_END(N); in visit()
1795 case ISD::FP_TO_FP16: return visitFP_TO_FP16(N); in visit()
1796 case ISD::FP16_TO_FP: return visitFP16_TO_FP(N); in visit()
1797 case ISD::FP_TO_BF16: return visitFP_TO_BF16(N); in visit()
1798 case ISD::FREEZE: return visitFREEZE(N); in visit()
1799 case ISD::VECREDUCE_FADD: in visit()
1800 case ISD::VECREDUCE_FMUL: in visit()
1801 case ISD::VECREDUCE_ADD: in visit()
1802 case ISD::VECREDUCE_MUL: in visit()
1803 case ISD::VECREDUCE_AND: in visit()
1804 case ISD::VECREDUCE_OR: in visit()
1805 case ISD::VECREDUCE_XOR: in visit()
1806 case ISD::VECREDUCE_SMAX: in visit()
1807 case ISD::VECREDUCE_SMIN: in visit()
1808 case ISD::VECREDUCE_UMAX: in visit()
1809 case ISD::VECREDUCE_UMIN: in visit()
1810 case ISD::VECREDUCE_FMAX: in visit()
1811 case ISD::VECREDUCE_FMIN: return visitVECREDUCE(N); in visit()
1812 #define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) case ISD::SDOPC: in visit()
1826 assert(N->getOpcode() != ISD::DELETED_NODE && in combine()
1829 if (N->getOpcode() >= ISD::BUILTIN_OP_END || in combine()
1830 TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) { in combine()
1844 case ISD::ADD: in combine()
1845 case ISD::SUB: in combine()
1846 case ISD::MUL: in combine()
1847 case ISD::AND: in combine()
1848 case ISD::OR: in combine()
1849 case ISD::XOR: in combine()
1852 case ISD::SHL: in combine()
1853 case ISD::SRA: in combine()
1854 case ISD::SRL: in combine()
1857 case ISD::SIGN_EXTEND: in combine()
1858 case ISD::ZERO_EXTEND: in combine()
1859 case ISD::ANY_EXTEND: in combine()
1862 case ISD::LOAD: in combine()
1924 if (N->hasOneUse() && N->use_begin()->getOpcode() == ISD::TokenFactor) in visitTokenFactor()
1954 case ISD::EntryToken: in visitTokenFactor()
1960 case ISD::TokenFactor: in visitTokenFactor()
2040 case ISD::EntryToken: in visitTokenFactor()
2047 case ISD::TokenFactor: in visitTokenFactor()
2051 case ISD::LIFETIME_START: in visitTokenFactor()
2052 case ISD::LIFETIME_END: in visitTokenFactor()
2053 case ISD::CopyFromReg: in visitTokenFactor()
2054 case ISD::CopyToReg: in visitTokenFactor()
2150 if (N->getOpcode() == ISD::ADD) { in canFoldInAddressingMode()
2159 } else if (N->getOpcode() == ISD::SUB) { in canFoldInAddressingMode()
2190 if (!N1.hasOneUse() || N1.getOpcode() != ISD::VSELECT) in foldSelectWithIdentityConstant()
2205 case ISD::FADD: // X + -0.0 --> X in foldSelectWithIdentityConstant()
2207 case ISD::FSUB: // X - 0.0 --> X in foldSelectWithIdentityConstant()
2209 case ISD::FMUL: // X * 1.0 --> X in foldSelectWithIdentityConstant()
2210 case ISD::FDIV: // X / 1.0 --> X in foldSelectWithIdentityConstant()
2216 case ISD::ADD: // X + 0 --> X in foldSelectWithIdentityConstant()
2217 case ISD::SUB: // X - 0 --> X in foldSelectWithIdentityConstant()
2218 case ISD::SHL: // X << 0 --> X in foldSelectWithIdentityConstant()
2219 case ISD::SRA: // X s>> 0 --> X in foldSelectWithIdentityConstant()
2220 case ISD::SRL: // X u>> 0 --> X in foldSelectWithIdentityConstant()
2222 case ISD::MUL: // X * 1 --> X in foldSelectWithIdentityConstant()
2267 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) { in foldBinOpIntoSelect()
2272 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) in foldBinOpIntoSelect()
2291 (BinOpcode == ISD::AND || BinOpcode == ISD::OR) && in foldBinOpIntoSelect()
2325 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && in foldAddSubBoolOfMaskedVal()
2331 bool IsAdd = N->getOpcode() == ISD::ADD; in foldAddSubBoolOfMaskedVal()
2335 if (!CN || Z.getOpcode() != ISD::ZERO_EXTEND) in foldAddSubBoolOfMaskedVal()
2339 if (Z.getOperand(0).getOpcode() != ISD::SETCC || in foldAddSubBoolOfMaskedVal()
2345 ISD::CondCode CC = cast<CondCodeSDNode>(SetCC->getOperand(2))->get(); in foldAddSubBoolOfMaskedVal()
2346 if (CC != ISD::SETEQ || !isNullConstant(SetCC.getOperand(1)) || in foldAddSubBoolOfMaskedVal()
2347 SetCC.getOperand(0).getOpcode() != ISD::AND || in foldAddSubBoolOfMaskedVal()
2360 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit); in foldAddSubBoolOfMaskedVal()
2366 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) && in foldAddSubOfSignBit()
2371 bool IsAdd = N->getOpcode() == ISD::ADD; in foldAddSubOfSignBit()
2375 ShiftOp.getOpcode() != ISD::SRL) in foldAddSubOfSignBit()
2395 IsAdd ? ISD::ADD : ISD::SUB, DL, VT, in foldAddSubOfSignBit()
2397 SDValue NewShift = DAG.getNode(IsAdd ? ISD::SRA : ISD::SRL, DL, VT, in foldAddSubOfSignBit()
2399 return DAG.getNode(ISD::ADD, DL, VT, NewShift, NewC); in foldAddSubOfSignBit()
2407 if (Opcode == ISD::OR) in isADDLike()
2409 if (Opcode == ISD::XOR) in isADDLike()
2430 if (SDValue C = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N0, N1})) in visitADDLike()
2436 return DAG.getNode(ISD::ADD, DL, VT, N1, N0); in visitADDLike()
2444 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitADDLike()
2452 if (N0.getOpcode() == ISD::SUB) { in visitADDLike()
2457 if (SDValue Sub = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N1, N01})) in visitADDLike()
2458 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Sub); in visitADDLike()
2461 if (SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N1, N00})) in visitADDLike()
2462 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1)); in visitADDLike()
2469 if (N0.getOpcode() == ISD::SIGN_EXTEND && N0.hasOneUse() && in visitADDLike()
2473 (TLI.isOperationLegal(ISD::XOR, X.getValueType()) && in visitADDLike()
2474 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) && in visitADDLike()
2477 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not); in visitADDLike()
2487 if (SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N1, N01})) in visitADDLike()
2488 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add); in visitADDLike()
2495 if (!reassociationCanBreakAddressingModePattern(ISD::ADD, DL, N, N0, N1)) { in visitADDLike()
2496 if (SDValue RADD = reassociateOps(ISD::ADD, DL, N0, N1, N->getFlags())) in visitADDLike()
2506 return DAG.getNode(ISD::ADD, DL, VT, in visitADDLike()
2507 DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)), in visitADDLike()
2518 if (N0.getOpcode() == ISD::SUB && isNullOrNullSplat(N0.getOperand(0))) in visitADDLike()
2519 return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1)); in visitADDLike()
2522 if (N1.getOpcode() == ISD::SUB && isNullOrNullSplat(N1.getOperand(0))) in visitADDLike()
2523 return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1)); in visitADDLike()
2526 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1)) in visitADDLike()
2530 if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1)) in visitADDLike()
2534 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB && in visitADDLike()
2536 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), in visitADDLike()
2540 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB && in visitADDLike()
2542 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), in visitADDLike()
2546 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD && in visitADDLike()
2548 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), in visitADDLike()
2552 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD && in visitADDLike()
2554 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), in visitADDLike()
2558 if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) && in visitADDLike()
2559 N1.getOperand(0).getOpcode() == ISD::SUB && in visitADDLike()
2565 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB && in visitADDLike()
2573 return DAG.getNode(ISD::SUB, DL, VT, in visitADDLike()
2574 DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10), in visitADDLike()
2575 DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11)); in visitADDLike()
2579 if (N0.getOpcode() == ISD::UMAX && hasOperation(ISD::USUBSAT, VT)) { in visitADDLike()
2584 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchUSUBSAT, in visitADDLike()
2586 return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0), in visitADDLike()
2596 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in visitADDLike()
2600 if (N0.getOpcode() == ISD::ADD) { in visitADDLike()
2612 return DAG.getNode(ISD::SUB, DL, VT, A, Xor.getOperand(0)); in visitADDLike()
2619 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD && in visitADDLike()
2621 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0), in visitADDLike()
2623 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(1), Not); in visitADDLike()
2628 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() && in visitADDLike()
2630 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), N1); in visitADDLike()
2631 return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0)); in visitADDLike()
2659 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) && in visitADD()
2661 return DAG.getNode(ISD::OR, DL, VT, N0, N1); in visitADD()
2664 if (N0.getOpcode() == ISD::VSCALE && N1.getOpcode() == ISD::VSCALE) { in visitADD()
2671 if ((N0.getOpcode() == ISD::ADD) && in visitADD()
2672 (N0.getOperand(1).getOpcode() == ISD::VSCALE) && in visitADD()
2673 (N1.getOpcode() == ISD::VSCALE)) { in visitADD()
2677 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), VS); in visitADD()
2681 if (N0.getOpcode() == ISD::STEP_VECTOR && in visitADD()
2682 N1.getOpcode() == ISD::STEP_VECTOR) { in visitADD()
2690 if ((N0.getOpcode() == ISD::ADD) && in visitADD()
2691 (N0.getOperand(1).getOpcode() == ISD::STEP_VECTOR) && in visitADD()
2692 (N1.getOpcode() == ISD::STEP_VECTOR)) { in visitADD()
2697 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), SV); in visitADD()
2729 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitADDSAT()
2738 if (Opcode == ISD::UADDSAT) in visitADDSAT()
2740 return DAG.getNode(ISD::ADD, DL, VT, N0, N1); in visitADDSAT()
2750 if (V.getOpcode() == ISD::TRUNCATE || V.getOpcode() == ISD::ZERO_EXTEND) { in getAsCarry()
2755 if (V.getOpcode() == ISD::AND && isOneConstant(V.getOperand(1))) { in getAsCarry()
2768 if (V.getOpcode() != ISD::ADDCARRY && V.getOpcode() != ISD::SUBCARRY && in getAsCarry()
2769 V.getOpcode() != ISD::UADDO && V.getOpcode() != ISD::USUBO) in getAsCarry()
2792 if (N1.getOpcode() != ISD::AND || !isOneOrOneSplat(N1->getOperand(1))) in foldAddSubMasked1()
2801 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, N0, N1.getOperand(0)); in foldAddSubMasked1()
2811 if (N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::SUB && in visitADDLikeCommutative()
2813 return DAG.getNode(ISD::SUB, DL, VT, N0, in visitADDLikeCommutative()
2814 DAG.getNode(ISD::SHL, DL, VT, in visitADDLikeCommutative()
2825 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD && in visitADDLikeCommutative()
2827 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0), in visitADDLikeCommutative()
2829 return DAG.getNode(ISD::SUB, DL, VT, N1, Not); in visitADDLikeCommutative()
2832 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse()) { in visitADDLikeCommutative()
2837 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), N1); in visitADDLikeCommutative()
2838 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1)); in visitADDLikeCommutative()
2843 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1)); in visitADDLikeCommutative()
2844 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(0)); in visitADDLikeCommutative()
2851 if (N0.getOpcode() == ISD::SIGN_EXTEND && in visitADDLikeCommutative()
2854 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitADDLikeCommutative()
2855 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt); in visitADDLikeCommutative()
2859 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) { in visitADDLikeCommutative()
2862 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0), in visitADDLikeCommutative()
2864 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt); in visitADDLikeCommutative()
2869 if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1)) && in visitADDLikeCommutative()
2871 return DAG.getNode(ISD::ADDCARRY, DL, N1->getVTList(), in visitADDLikeCommutative()
2875 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT)) in visitADDLikeCommutative()
2877 return DAG.getNode(ISD::ADDCARRY, DL, in visitADDLikeCommutative()
2892 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDC()
2893 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue)); in visitADDC()
2899 return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0); in visitADDC()
2903 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, in visitADDC()
2908 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDC()
2909 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue)); in visitADDC()
2927 if (V.getOpcode() != ISD::XOR) in extractBooleanFlip()
2960 bool IsSigned = (ISD::SADDO == N->getOpcode()); in visitADDO()
2967 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDO()
2982 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDO()
2987 SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(), in visitADDO()
3010 if (N1.getOpcode() == ISD::ADDCARRY && isNullConstant(N1.getOperand(1))) { in visitUADDOLike()
3014 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0, Y, in visitUADDOLike()
3019 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT)) in visitUADDOLike()
3021 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0, in visitUADDOLike()
3036 return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(), in visitADDE()
3040 if (CarryIn.getOpcode() == ISD::CARRY_FALSE) in visitADDE()
3041 return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1); in visitADDE()
3056 return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), N1, N0, CarryIn); in visitADDCARRY()
3061 TLI.isOperationLegalOrCustom(ISD::UADDO, N->getValueType(0))) in visitADDCARRY()
3062 return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1); in visitADDCARRY()
3071 return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt, in visitADDCARRY()
3095 return DAG.getNode(ISD::SADDO_CARRY, DL, N->getVTList(), N1, N0, CarryIn); in visitSADDO_CARRY()
3100 TLI.isOperationLegalOrCustom(ISD::SADDO, N->getValueType(0))) in visitSADDO_CARRY()
3101 return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0, N1); in visitSADDO_CARRY()
3134 if (Carry1.getOpcode() != ISD::UADDO) in combineADDCARRYDiamond()
3143 if (Carry0.getOpcode() == ISD::ADDCARRY && in combineADDCARRYDiamond()
3146 } else if (Carry0.getOpcode() == ISD::UADDO && in combineADDCARRYDiamond()
3158 SDValue NewY = DAG.getNode(ISD::ADDCARRY, DL, Carry0->getVTList(), A, B, Z); in combineADDCARRYDiamond()
3160 return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), X, in combineADDCARRYDiamond()
3232 if (Opcode != ISD::UADDO && Opcode != ISD::USUBO) in combineCarryDiamond()
3248 if (Opcode == ISD::USUBO && CarryInOperandNum != 1) in combineCarryDiamond()
3252 unsigned NewOp = Opcode == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY; in combineCarryDiamond()
3258 if (CarryIn.getOpcode() != ISD::ZERO_EXTEND) in combineCarryDiamond()
3283 if (N->getOpcode() == ISD::AND) in combineCarryDiamond()
3294 SDValue Sub = DAG.getNode(ISD::SUBCARRY, DL, N->getVTList(), N1, in visitADDCARRYLike()
3304 if ((N0.getOpcode() == ISD::ADD || in visitADDCARRYLike()
3305 (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0 && in visitADDCARRYLike()
3308 return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), in visitADDCARRYLike()
3336 return DAG.getNode(ISD::USUBSAT, DL, DstVT, LHS, RHS); in getTruncatedUSUBSAT()
3349 RHS = DAG.getNode(ISD::UMIN, DL, SrcVT, RHS, SatLimit); in getTruncatedUSUBSAT()
3350 RHS = DAG.getNode(ISD::TRUNCATE, DL, DstVT, RHS); in getTruncatedUSUBSAT()
3351 LHS = DAG.getNode(ISD::TRUNCATE, DL, DstVT, LHS); in getTruncatedUSUBSAT()
3352 return DAG.getNode(ISD::USUBSAT, DL, DstVT, LHS, RHS); in getTruncatedUSUBSAT()
3358 if (N->getOpcode() != ISD::SUB || in foldSubToUSubSat()
3359 !(!LegalOperations || hasOperation(ISD::USUBSAT, DstVT))) in foldSubToUSubSat()
3368 if (Op0.getOpcode() == ISD::UMAX && Op0.hasOneUse()) { in foldSubToUSubSat()
3377 if (Op1.getOpcode() == ISD::UMIN && Op1.hasOneUse()) { in foldSubToUSubSat()
3387 if (Op1.getOpcode() == ISD::TRUNCATE && in foldSubToUSubSat()
3388 Op1.getOperand(0).getOpcode() == ISD::UMIN && in foldSubToUSubSat()
3392 if (MinLHS.getOpcode() == ISD::ZERO_EXTEND && MinLHS.getOperand(0) == Op0) in foldSubToUSubSat()
3395 if (MinRHS.getOpcode() == ISD::ZERO_EXTEND && MinRHS.getOperand(0) == Op0) in foldSubToUSubSat()
3409 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) in tryFoldToZero()
3421 if (N->getOpcode() == ISD::FREEZE && N.hasOneUse()) in visitSUB()
3432 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N1})) in visitSUB()
3441 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitSUB()
3452 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
3462 if (N1->getOpcode() == ISD::SRA || N1->getOpcode() == ISD::SRL) { in visitSUB()
3465 auto NewSh = N1->getOpcode() == ISD::SRA ? ISD::SRL : ISD::SRA; in visitSUB()
3486 if (N1.getOpcode() == ISD::ABS && N1.hasOneUse() && in visitSUB()
3487 !TLI.isOperationLegalOrCustom(ISD::ABS, VT)) in visitSUB()
3494 if (N1S && N1S.getOpcode() == ISD::SUB && in visitSUB()
3505 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitSUB()
3508 if (N1.getOpcode() == ISD::SUB && isNullOrNullSplat(N1.getOperand(0))) in visitSUB()
3509 return DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(1)); in visitSUB()
3512 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0)) in visitSUB()
3516 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1) in visitSUB()
3520 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1) in visitSUB()
3524 if (N0.getOpcode() == ISD::ADD) { in visitSUB()
3526 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N01, N1})) in visitSUB()
3527 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), NewC); in visitSUB()
3531 if (N1.getOpcode() == ISD::ADD) { in visitSUB()
3533 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N11})) in visitSUB()
3534 return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0)); in visitSUB()
3538 if (N0.getOpcode() == ISD::SUB) { in visitSUB()
3540 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N01, N1})) in visitSUB()
3541 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), NewC); in visitSUB()
3545 if (N0.getOpcode() == ISD::SUB) { in visitSUB()
3547 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N00, N1})) in visitSUB()
3548 return DAG.getNode(ISD::SUB, DL, VT, NewC, N0.getOperand(1)); in visitSUB()
3552 if (N0.getOpcode() == ISD::ADD && in visitSUB()
3553 (N0.getOperand(1).getOpcode() == ISD::SUB || in visitSUB()
3554 N0.getOperand(1).getOpcode() == ISD::ADD) && in visitSUB()
3560 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1).getOpcode() == ISD::ADD && in visitSUB()
3562 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), in visitSUB()
3566 if (N0.getOpcode() == ISD::SUB && N0.getOperand(1).getOpcode() == ISD::SUB && in visitSUB()
3568 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), in visitSUB()
3572 if (N1.getOpcode() == ISD::SUB && N1.hasOneUse()) in visitSUB()
3573 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
3574 DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(1), in visitSUB()
3578 if (N1.getOpcode() == ISD::AND) { in visitSUB()
3586 DAG.getNode(ISD::XOR, DL, VT, B, DAG.getAllOnesConstant(DL, VT)); in visitSUB()
3587 return DAG.getNode(ISD::AND, DL, VT, A, InvB); in visitSUB()
3592 if (N1.getOpcode() == ISD::MUL && N1.hasOneUse()) { in visitSUB()
3593 if (N1.getOperand(0).getOpcode() == ISD::SUB && in visitSUB()
3595 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, in visitSUB()
3598 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul); in visitSUB()
3600 if (N1.getOperand(1).getOpcode() == ISD::SUB && in visitSUB()
3602 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, in visitSUB()
3605 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul); in visitSUB()
3628 if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB && isOneOrOneSplat(N1)) { in visitSUB()
3629 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), in visitSUB()
3631 return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0)); in visitSUB()
3639 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(0)); in visitSUB()
3640 return DAG.getNode(ISD::ADD, DL, VT, Add, DAG.getConstant(1, DL, VT)); in visitSUB()
3645 if (N0.hasOneUse() && N0.getOpcode() == ISD::ADD && in visitSUB()
3647 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1); in visitSUB()
3648 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(1)); in visitSUB()
3651 if (N1.hasOneUse() && N1.getOpcode() == ISD::ADD && in visitSUB()
3653 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(0)); in visitSUB()
3654 return DAG.getNode(ISD::SUB, DL, VT, Sub, N1.getOperand(1)); in visitSUB()
3658 if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB && in visitSUB()
3660 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1); in visitSUB()
3661 return DAG.getNode(ISD::SUB, DL, VT, Sub, N0.getOperand(1)); in visitSUB()
3664 if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB && in visitSUB()
3666 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), N1); in visitSUB()
3667 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), Add); in visitSUB()
3673 if (N1.getOpcode() == ISD::ZERO_EXTEND && in visitSUB()
3677 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N1.getOperand(0)); in visitSUB()
3678 return DAG.getNode(ISD::ADD, DL, VT, N0, SExt); in visitSUB()
3682 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) { in visitSUB()
3683 if (N0.getOpcode() == ISD::XOR && N1.getOpcode() == ISD::SRA) { in visitSUB()
3689 return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0); in visitSUB()
3697 if (N1C && GA->getOpcode() == ISD::GlobalAddress) in visitSUB()
3709 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) { in visitSUB()
3712 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0), in visitSUB()
3714 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt); in visitSUB()
3719 if (N1.getOpcode() == ISD::VSCALE) { in visitSUB()
3721 return DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getVScale(DL, VT, -IntVal)); in visitSUB()
3725 if (N1.getOpcode() == ISD::STEP_VECTOR && N1.hasOneUse()) { in visitSUB()
3727 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
3733 if (!LegalOperations && N1.getOpcode() == ISD::SRL && N1.hasOneUse()) { in visitSUB()
3738 SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt); in visitSUB()
3739 return DAG.getNode(ISD::ADD, DL, VT, N0, SRA); in visitSUB()
3746 if (N1.getOpcode() == ISD::SHL) { in visitSUB()
3749 return DAG.getNode(ISD::ADD, DL, VT, N1, N0); in visitSUB()
3752 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT)) { in visitSUB()
3757 SDValue NegX = DAG.getNode(ISD::SUB, DL, VT, Zero, X); in visitSUB()
3758 return DAG.getNode(ISD::ADDCARRY, DL, in visitSUB()
3771 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitSUB()
3802 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitSUBSAT()
3821 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitSUBC()
3822 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue)); in visitSUBC()
3827 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue)); in visitSUBC()
3831 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue)); in visitSUBC()
3835 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0), in visitSUBC()
3836 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue)); in visitSUBC()
3845 bool IsSigned = (ISD::SSUBO == N->getOpcode()); in visitSUBO()
3852 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitSUBO()
3864 return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0, in visitSUBO()
3874 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0), in visitSUBO()
3886 if (CarryIn.getOpcode() == ISD::CARRY_FALSE) in visitSUBE()
3887 return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1); in visitSUBE()
3900 TLI.isOperationLegalOrCustom(ISD::USUBO, N->getValueType(0))) in visitSUBCARRY()
3901 return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1); in visitSUBCARRY()
3915 TLI.isOperationLegalOrCustom(ISD::SSUBO, N->getValueType(0))) in visitSSUBO_CARRY()
3916 return DAG.getNode(ISD::SSUBO, SDLoc(N), N->getVTList(), N0, N1); in visitSSUBO_CARRY()
3957 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MUL, DL, VT, {N0, N1})) in visitMUL()
3963 return DAG.getNode(ISD::MUL, DL, VT, N1, N0); in visitMUL()
3974 N1IsConst = ISD::isConstantSplatVector(N1.getNode(), ConstValue1); in visitMUL()
3999 return DAG.getNode(ISD::SUB, DL, VT, in visitMUL()
4009 return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc); in visitMUL()
4017 return DAG.getNode(ISD::SUB, DL, VT, in visitMUL()
4019 DAG.getNode(ISD::SHL, DL, VT, N0, in visitMUL()
4043 unsigned MathOp = ISD::DELETED_NODE; in visitMUL()
4049 MathOp = ISD::ADD; in visitMUL()
4051 MathOp = ISD::SUB; in visitMUL()
4053 if (MathOp != ISD::DELETED_NODE) { in visitMUL()
4055 MathOp == ISD::ADD ? (MulC - 1).logBase2() : (MulC + 1).logBase2(); in visitMUL()
4060 DAG.getNode(ISD::SHL, DL, VT, N0, DAG.getConstant(ShAmt, DL, VT)); in visitMUL()
4063 DAG.getNode(ISD::SHL, DL, VT, N0, in visitMUL()
4067 R = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), R); in visitMUL()
4073 if (N0.getOpcode() == ISD::SHL) { in visitMUL()
4075 if (SDValue C3 = DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {N1, N01})) in visitMUL()
4076 return DAG.getNode(ISD::MUL, DL, VT, N0.getOperand(0), C3); in visitMUL()
4085 if (N0.getOpcode() == ISD::SHL && in visitMUL()
4088 } else if (N1.getOpcode() == ISD::SHL && in visitMUL()
4095 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, Sh.getOperand(0), Y); in visitMUL()
4096 return DAG.getNode(ISD::SHL, DL, VT, Mul, Sh.getOperand(1)); in visitMUL()
4102 N0.getOpcode() == ISD::ADD && in visitMUL()
4106 ISD::ADD, DL, VT, in visitMUL()
4107 DAG.getNode(ISD::MUL, SDLoc(N0), VT, N0.getOperand(0), N1), in visitMUL()
4108 DAG.getNode(ISD::MUL, SDLoc(N1), VT, N0.getOperand(1), N1)); in visitMUL()
4111 if (N0.getOpcode() == ISD::VSCALE) in visitMUL()
4120 if (N0.getOpcode() == ISD::STEP_VECTOR) in visitMUL()
4121 if (ISD::isConstantSplatVector(N1.getNode(), MulVal)) { in visitMUL()
4143 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::AND, VT)) && in visitMUL()
4144 ISD::matchUnaryPredicate(N1, IsClearMask, /*AllowUndefs*/ true)) { in visitMUL()
4145 assert(N1.getOpcode() == ISD::BUILD_VECTOR && "Unknown constant vector"); in visitMUL()
4153 return DAG.getNode(ISD::AND, DL, VT, N0, DAG.getBuildVector(VT, DL, Mask)); in visitMUL()
4158 if (SDValue RMUL = reassociateOps(ISD::MUL, DL, N0, N1, N->getFlags())) in visitMUL()
4193 bool isSigned = (Opcode == ISD::SDIV) || (Opcode == ISD::SREM); in useDivRem()
4194 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; in useDivRem()
4212 if ((Opcode == ISD::SDIV) || (Opcode == ISD::UDIV)) { in useDivRem()
4213 OtherOpcode = isSigned ? ISD::SREM : ISD::UREM; in useDivRem()
4217 OtherOpcode = isSigned ? ISD::SDIV : ISD::UDIV; in useDivRem()
4226 if (User == Node || User->getOpcode() == ISD::DELETED_NODE || in useDivRem()
4247 if (UserOpc == ISD::SDIV || UserOpc == ISD::UDIV) in useDivRem()
4249 else if (UserOpc == ISD::SREM || UserOpc == ISD::UREM) in useDivRem()
4263 bool IsDiv = (ISD::SDIV == Opc) || (ISD::UDIV == Opc); in simplifyDivRem()
4310 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, {N0, N1})) in visitSDIV()
4321 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0); in visitSDIV()
4325 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ), in visitSDIV()
4338 return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1); in visitSDIV()
4343 if (SDNode *RemNode = DAG.getNodeIfExists(ISD::SREM, N->getVTList(), in visitSDIV()
4345 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1); in visitSDIV()
4346 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitSDIV()
4378 return ISD::matchUnaryPredicate(Divisor, IsPowerOfTwo); in isDivisorPowerOfTwo()
4399 SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1); in visitSDIVLike()
4401 SDValue Inexact = DAG.getNode(ISD::SUB, DL, ShiftAmtTy, Bits, C1); in visitSDIVLike()
4406 SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0, in visitSDIVLike()
4411 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact); in visitSDIVLike()
4413 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl); in visitSDIVLike()
4415 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1); in visitSDIVLike()
4422 SDValue IsOne = DAG.getSetCC(DL, CCVT, N1, One, ISD::SETEQ); in visitSDIVLike()
4423 SDValue IsAllOnes = DAG.getSetCC(DL, CCVT, N1, AllOnes, ISD::SETEQ); in visitSDIVLike()
4424 SDValue IsOneOrAllOnes = DAG.getNode(ISD::OR, DL, CCVT, IsOne, IsAllOnes); in visitSDIVLike()
4430 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, Zero, Sra); in visitSDIVLike()
4433 SDValue IsNeg = DAG.getSetCC(DL, CCVT, N1, Zero, ISD::SETLT); in visitSDIVLike()
4458 if (SDValue C = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT, {N0, N1})) in visitUDIV()
4469 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ), in visitUDIV()
4482 if (SDNode *RemNode = DAG.getNodeIfExists(ISD::UREM, N->getVTList(), in visitUDIV()
4484 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1); in visitUDIV()
4485 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitUDIV()
4517 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc); in visitUDIVLike()
4521 if (N1.getOpcode() == ISD::SHL) { in visitUDIVLike()
4531 SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc); in visitUDIVLike()
4533 return DAG.getNode(ISD::SRL, DL, VT, N0, Add); in visitUDIVLike()
4549 !DAG.doesNodeExist(ISD::SDIV, N->getVTList(), {N0, N1})) { in buildOptimizedSREM()
4565 bool isSigned = (Opcode == ISD::SREM); in visitREM()
4576 SDValue EqualsNeg1 = DAG.getSetCC(DL, CCVT, F0, N1, ISD::SETEQ); in visitREM()
4590 return DAG.getNode(ISD::UREM, DL, VT, N0, N1); in visitREM()
4595 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne); in visitREM()
4597 return DAG.getNode(ISD::AND, DL, VT, N0, Add); in visitREM()
4603 if ((N1.getOpcode() == ISD::SHL || N1.getOpcode() == ISD::SRL) && in visitREM()
4606 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne); in visitREM()
4608 return DAG.getNode(ISD::AND, DL, VT, N0, Add); in visitREM()
4632 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV; in visitREM()
4636 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1); in visitREM()
4637 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitREM()
4658 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHS, DL, VT, {N0, N1})) in visitMULHS()
4664 return DAG.getNode(ISD::MULHS, DL, N->getVTList(), N1, N0); in visitMULHS()
4672 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitMULHS()
4682 return DAG.getNode(ISD::SRA, DL, N0.getValueType(), N0, in visitMULHS()
4692 if (!TLI.isOperationLegalOrCustom(ISD::MULHS, VT) && VT.isSimple() && in visitMULHS()
4697 if (TLI.isOperationLegal(ISD::MUL, NewVT)) { in visitMULHS()
4698 N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0); in visitMULHS()
4699 N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1); in visitMULHS()
4700 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1); in visitMULHS()
4701 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1, in visitMULHS()
4704 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in visitMULHS()
4718 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHU, DL, VT, {N0, N1})) in visitMULHU()
4724 return DAG.getNode(ISD::MULHU, DL, N->getVTList(), N1, N0); in visitMULHU()
4732 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitMULHU()
4750 DAG.isKnownToBeAPowerOfTwo(N1) && hasOperation(ISD::SRL, VT)) { in visitMULHU()
4754 ISD::SUB, DL, VT, DAG.getConstant(NumEltBits, DL, VT), LogBase2); in visitMULHU()
4757 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc); in visitMULHU()
4762 if (!TLI.isOperationLegalOrCustom(ISD::MULHU, VT) && VT.isSimple() && in visitMULHU()
4767 if (TLI.isOperationLegal(ISD::MUL, NewVT)) { in visitMULHU()
4768 N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0); in visitMULHU()
4769 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1); in visitMULHU()
4770 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1); in visitMULHU()
4771 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1, in visitMULHU()
4774 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in visitMULHU()
4808 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) { in visitAVG()
4809 if (Opcode == ISD::AVGFLOORS) in visitAVG()
4810 return DAG.getNode(ISD::SRA, DL, VT, N0, DAG.getConstant(1, DL, VT)); in visitAVG()
4811 if (Opcode == ISD::AVGFLOORU) in visitAVG()
4812 return DAG.getNode(ISD::SRL, DL, VT, N0, DAG.getConstant(1, DL, VT)); in visitAVG()
4877 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS)) in visitSMUL_LOHI()
4888 return DAG.getNode(ISD::SMUL_LOHI, DL, N->getVTList(), N1, N0); in visitSMUL_LOHI()
4896 if (TLI.isOperationLegal(ISD::MUL, NewVT)) { in visitSMUL_LOHI()
4897 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0); in visitSMUL_LOHI()
4898 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1); in visitSMUL_LOHI()
4899 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi); in visitSMUL_LOHI()
4901 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo, in visitSMUL_LOHI()
4904 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi); in visitSMUL_LOHI()
4906 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo); in visitSMUL_LOHI()
4915 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU)) in visitUMUL_LOHI()
4926 return DAG.getNode(ISD::UMUL_LOHI, DL, N->getVTList(), N1, N0); in visitUMUL_LOHI()
4946 if (TLI.isOperationLegal(ISD::MUL, NewVT)) { in visitUMUL_LOHI()
4947 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0); in visitUMUL_LOHI()
4948 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1); in visitUMUL_LOHI()
4949 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi); in visitUMUL_LOHI()
4951 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo, in visitUMUL_LOHI()
4954 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi); in visitUMUL_LOHI()
4956 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo); in visitUMUL_LOHI()
4968 bool IsSigned = (ISD::SMULO == N->getOpcode()); in visitMULO()
5002 return DAG.getNode(IsSigned ? ISD::SADDO : ISD::UADDO, DL, in visitMULO()
5008 SDValue And = DAG.getNode(ISD::AND, DL, VT, N0, N1); in visitMULO()
5011 DAG.getConstant(0, DL, VT), ISD::SETNE)); in visitMULO()
5021 return CombineTo(N, DAG.getNode(ISD::MUL, DL, VT, N0, N1), in visitMULO()
5029 return CombineTo(N, DAG.getNode(ISD::MUL, DL, VT, N0, N1), in visitMULO()
5043 SDValue N3, ISD::CondCode CC, unsigned &BW, in isSaturatingMinMax()
5046 ISD::CondCode CC) { in isSaturatingMinMax()
5049 if (N0 != N2 && (N2.getOpcode() != ISD::TRUNCATE || N0 != N2.getOperand(0))) in isSaturatingMinMax()
5060 return CC == ISD::SETLT ? ISD::SMIN : (CC == ISD::SETGT ? ISD::SMAX : 0); in isSaturatingMinMax()
5069 ISD::CondCode N0CC; in isSaturatingMinMax()
5071 case ISD::SMIN: in isSaturatingMinMax()
5072 case ISD::SMAX: in isSaturatingMinMax()
5075 N0CC = N0.getOpcode() == ISD::SMIN ? ISD::SETLT : ISD::SETGT; in isSaturatingMinMax()
5077 case ISD::SELECT_CC: in isSaturatingMinMax()
5084 case ISD::SELECT: in isSaturatingMinMax()
5085 case ISD::VSELECT: in isSaturatingMinMax()
5086 if (N0.getOperand(0).getOpcode() != ISD::SETCC) in isSaturatingMinMax()
5102 ConstantSDNode *MinCOp = isConstOrConstSplat(Opcode0 == ISD::SMIN ? N1 : N01); in isSaturatingMinMax()
5103 ConstantSDNode *MaxCOp = isConstOrConstSplat(Opcode0 == ISD::SMIN ? N01 : N1); in isSaturatingMinMax()
5126 SDValue N3, ISD::CondCode CC, in PerformMinMaxFpToSatCombine()
5131 if (!Fp || Fp.getOpcode() != ISD::FP_TO_SINT) in PerformMinMaxFpToSatCombine()
5138 unsigned NewOpc = Unsigned ? ISD::FP_TO_UINT_SAT : ISD::FP_TO_SINT_SAT; in PerformMinMaxFpToSatCombine()
5149 SDValue N3, ISD::CondCode CC, in PerformUMinFpToSatCombine()
5155 (N2.getOpcode() != ISD::TRUNCATE || N0 != N2.getOperand(0))) || in PerformUMinFpToSatCombine()
5156 N0.getOpcode() != ISD::FP_TO_UINT || CC != ISD::SETULT) in PerformUMinFpToSatCombine()
5174 if (!DAG.getTargetLoweringInfo().shouldConvertFpToSat(ISD::FP_TO_UINT_SAT, in PerformUMinFpToSatCombine()
5179 DAG.getNode(ISD::FP_TO_UINT_SAT, SDLoc(N0), NewVT, N0.getOperand(0), in PerformUMinFpToSatCombine()
5216 case ISD::SMIN: AltOpcode = ISD::UMIN; break; in visitIMINMAX()
5217 case ISD::SMAX: AltOpcode = ISD::UMAX; break; in visitIMINMAX()
5218 case ISD::UMIN: AltOpcode = ISD::SMIN; break; in visitIMINMAX()
5219 case ISD::UMAX: AltOpcode = ISD::SMAX; break; in visitIMINMAX()
5226 if (Opcode == ISD::SMIN || Opcode == ISD::SMAX) in visitIMINMAX()
5228 N0, N1, N0, N1, Opcode == ISD::SMIN ? ISD::SETLT : ISD::SETGT, DAG)) in visitIMINMAX()
5230 if (Opcode == ISD::UMIN) in visitIMINMAX()
5231 if (SDValue S = PerformUMinFpToSatCombine(N0, N1, N0, N1, ISD::SETULT, DAG)) in visitIMINMAX()
5248 assert((LogicOpcode == ISD::AND || LogicOpcode == ISD::OR || in hoistLogicOpWithSameOpcodeHands()
5249 LogicOpcode == ISD::XOR) && "Expected logic opcode"); in hoistLogicOpWithSameOpcodeHands()
5264 if (HandOpcode == ISD::ANY_EXTEND || HandOpcode == ISD::ZERO_EXTEND || in hoistLogicOpWithSameOpcodeHands()
5265 HandOpcode == ISD::SIGN_EXTEND) { in hoistLogicOpWithSameOpcodeHands()
5280 if (HandOpcode == ISD::ANY_EXTEND && LegalTypes && in hoistLogicOpWithSameOpcodeHands()
5289 if (HandOpcode == ISD::TRUNCATE) { in hoistLogicOpWithSameOpcodeHands()
5312 if ((HandOpcode == ISD::SHL || HandOpcode == ISD::SRL || in hoistLogicOpWithSameOpcodeHands()
5313 HandOpcode == ISD::SRA || HandOpcode == ISD::AND) && in hoistLogicOpWithSameOpcodeHands()
5323 if (HandOpcode == ISD::BSWAP) { in hoistLogicOpWithSameOpcodeHands()
5338 if ((HandOpcode == ISD::BITCAST || HandOpcode == ISD::SCALAR_TO_VECTOR) && in hoistLogicOpWithSameOpcodeHands()
5361 if (HandOpcode == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) { in hoistLogicOpWithSameOpcodeHands()
5378 if (LogicOpcode == ISD::XOR && !ShOp.isUndef()) in hoistLogicOpWithSameOpcodeHands()
5391 if (LogicOpcode == ISD::XOR && !ShOp.isUndef()) in hoistLogicOpWithSameOpcodeHands()
5430 ISD::CondCode CC0 = cast<CondCodeSDNode>(N0CC)->get(); in foldLogicOfSetCCs()
5431 ISD::CondCode CC1 = cast<CondCodeSDNode>(N1CC)->get(); in foldLogicOfSetCCs()
5438 bool AndEqZero = IsAnd && CC1 == ISD::SETEQ && IsZero; in foldLogicOfSetCCs()
5440 bool AndGtNeg1 = IsAnd && CC1 == ISD::SETGT && IsNeg1; in foldLogicOfSetCCs()
5442 bool OrNeZero = !IsAnd && CC1 == ISD::SETNE && IsZero; in foldLogicOfSetCCs()
5444 bool OrLtZero = !IsAnd && CC1 == ISD::SETLT && IsZero; in foldLogicOfSetCCs()
5451 SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL); in foldLogicOfSetCCs()
5457 bool AndEqNeg1 = IsAnd && CC1 == ISD::SETEQ && IsNeg1; in foldLogicOfSetCCs()
5459 bool AndLtZero = IsAnd && CC1 == ISD::SETLT && IsZero; in foldLogicOfSetCCs()
5461 bool OrNeNeg1 = !IsAnd && CC1 == ISD::SETNE && IsNeg1; in foldLogicOfSetCCs()
5463 bool OrGtNeg1 = !IsAnd && CC1 == ISD::SETGT && IsNeg1; in foldLogicOfSetCCs()
5470 SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL); in foldLogicOfSetCCs()
5479 IsInteger && CC0 == ISD::SETNE && in foldLogicOfSetCCs()
5484 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One); in foldLogicOfSetCCs()
5486 return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE); in foldLogicOfSetCCs()
5495 if ((IsAnd && CC1 == ISD::SETEQ) || (!IsAnd && CC1 == ISD::SETNE)) { in foldLogicOfSetCCs()
5496 SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR); in foldLogicOfSetCCs()
5497 SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR); in foldLogicOfSetCCs()
5498 SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR); in foldLogicOfSetCCs()
5504 if ((IsAnd && CC1 == ISD::SETNE) || (!IsAnd && CC1 == ISD::SETEQ)) { in foldLogicOfSetCCs()
5514 if (LL == RL && ISD::matchBinaryPredicate(LR, RR, MatchDiffPow2)) { in foldLogicOfSetCCs()
5517 SDValue Max = DAG.getNode(ISD::UMAX, DL, OpVT, LR, RR); in foldLogicOfSetCCs()
5518 SDValue Min = DAG.getNode(ISD::UMIN, DL, OpVT, LR, RR); in foldLogicOfSetCCs()
5519 SDValue Offset = DAG.getNode(ISD::SUB, DL, OpVT, LL, Min); in foldLogicOfSetCCs()
5520 SDValue Diff = DAG.getNode(ISD::SUB, DL, OpVT, Max, Min); in foldLogicOfSetCCs()
5522 SDValue And = DAG.getNode(ISD::AND, DL, OpVT, Offset, Mask); in foldLogicOfSetCCs()
5531 CC1 = ISD::getSetCCSwappedOperands(CC1); in foldLogicOfSetCCs()
5538 ISD::CondCode NewCC = IsAnd ? ISD::getSetCCAndOperation(CC0, CC1, OpVT) in foldLogicOfSetCCs()
5539 : ISD::getSetCCOrOperation(CC0, CC1, OpVT); in foldLogicOfSetCCs()
5540 if (NewCC != ISD::SETCC_INVALID && in foldLogicOfSetCCs()
5543 TLI.isOperationLegal(ISD::SETCC, OpVT)))) in foldLogicOfSetCCs()
5566 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL && in visitANDLike()
5586 DAG.getNode(ISD::ADD, DL0, VT, in visitANDLike()
5601 if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) { in visitANDLike()
5620 TLI.isTypeDesirableForOp(ISD::AND, HalfVT) && in visitANDLike()
5621 TLI.isTypeDesirableForOp(ISD::SRL, HalfVT) && in visitANDLike()
5634 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, HalfVT, in visitANDLike()
5639 SDValue Shift = DAG.getNode(ISD::SRL, SL, HalfVT, Trunc, ShiftK); in visitANDLike()
5640 SDValue And = DAG.getNode(ISD::AND, SL, HalfVT, Shift, NewMask); in visitANDLike()
5641 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, And); in visitANDLike()
5662 TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))) { in isAndLoadExtLoad()
5678 !TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT)) in isAndLoadExtLoad()
5681 if (!TLI.shouldReduceLoadWidth(LoadN, ISD::ZEXTLOAD, ExtVT)) in isAndLoadExtLoad()
5688 ISD::LoadExtType ExtType, EVT &MemVT, in isLegalNarrowLdSt()
5755 if (Load->getExtensionType() != ISD::NON_EXTLOAD && in isLegalNarrowLdSt()
5788 if ((N->getOpcode() == ISD::OR || N->getOpcode() == ISD::XOR) && in SearchForAndLoads()
5798 case ISD::LOAD: { in SearchForAndLoads()
5802 isLegalNarrowLdSt(Load, ISD::ZEXTLOAD, ExtVT)) { in SearchForAndLoads()
5805 if (Load->getExtensionType() == ISD::ZEXTLOAD && in SearchForAndLoads()
5817 case ISD::ZERO_EXTEND: in SearchForAndLoads()
5818 case ISD::AssertZext: { in SearchForAndLoads()
5821 EVT VT = Op.getOpcode() == ISD::AssertZext ? in SearchForAndLoads()
5831 case ISD::OR: in SearchForAndLoads()
5832 case ISD::XOR: in SearchForAndLoads()
5833 case ISD::AND: in SearchForAndLoads()
5890 SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode), in BackwardsPropagateMask()
5894 if (And.getOpcode() == ISD ::AND) in BackwardsPropagateMask()
5906 SDValue And = DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(), in BackwardsPropagateMask()
5915 SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0), in BackwardsPropagateMask()
5918 if (And.getOpcode() == ISD ::AND) in BackwardsPropagateMask()
5938 assert(N->getOpcode() == ISD::AND); in unfoldExtremeBitClearingToShifts()
5955 if (OuterShift == ISD::SHL) in unfoldExtremeBitClearingToShifts()
5956 InnerShift = ISD::SRL; in unfoldExtremeBitClearingToShifts()
5957 else if (OuterShift == ISD::SRL) in unfoldExtremeBitClearingToShifts()
5958 InnerShift = ISD::SHL; in unfoldExtremeBitClearingToShifts()
5990 assert(And->getOpcode() == ISD::AND && "Expected an 'and' op"); in combineShiftAnd1ToBitTest()
6000 if (And0.getOpcode() == ISD::ANY_EXTEND && And0.hasOneUse()) in combineShiftAnd1ToBitTest()
6017 if (Src.getOpcode() == ISD::TRUNCATE && Src.hasOneUse()) in combineShiftAnd1ToBitTest()
6022 if (Src.getOpcode() != ISD::SRL || !Src.hasOneUse()) in combineShiftAnd1ToBitTest()
6056 SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, X, Mask); in combineShiftAnd1ToBitTest()
6058 SDValue Setcc = DAG.getSetCC(DL, CCVT, NewAnd, Zero, ISD::SETEQ); in combineShiftAnd1ToBitTest()
6070 if (N0.getOpcode() == ISD::SRA) in foldAndToUsubsat()
6074 if (N0.getOpcode() != ISD::XOR && N0.getOpcode() != ISD::ADD) in foldAndToUsubsat()
6077 if (N1.getOpcode() != ISD::SRA || !N0.hasOneUse() || !N1.hasOneUse() || in foldAndToUsubsat()
6092 return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0), SignMask); in foldAndToUsubsat()
6102 assert((LogicOpcode == ISD::AND || LogicOpcode == ISD::OR || in foldLogicOfShifts()
6103 LogicOpcode == ISD::XOR) in foldLogicOfShifts()
6112 !(ShiftOpcode == ISD::SHL || ShiftOpcode == ISD::SRL || in foldLogicOfShifts()
6113 ShiftOpcode == ISD::SRA)) in foldLogicOfShifts()
6152 if (SDValue C = DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, {N0, N1})) in visitAND()
6158 return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0); in visitAND()
6166 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitAND()
6172 if (ISD::isConstantSplatVectorAllOnes(N1.getNode())) in visitAND()
6178 if (MLoad && MLoad->getExtensionType() == ISD::EXTLOAD && N0.hasOneUse() && in visitAND()
6182 if (TLI.isLoadExtLegal(ISD::ZEXTLOAD, ExtVT, LoadVT)) { in visitAND()
6193 ISD::ZEXTLOAD, MLoad->isExpandingLoad()); in visitAND()
6213 if (SDValue RAND = reassociateOps(ISD::AND, SDLoc(N), N0, N1, N->getFlags())) in visitAND()
6228 if (N0.getOpcode() == ISD::OR && in visitAND()
6229 ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchSubset)) in visitAND()
6232 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) { in visitAND()
6237 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), in visitAND()
6257 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitAND()
6259 N0.getOperand(0).getOpcode() == ISD::LOAD && in visitAND()
6261 (N0.getOpcode() == ISD::LOAD && N0.getResNo() == 0)) { in visitAND()
6262 LoadSDNode *Load = cast<LoadSDNode>( (N0.getOpcode() == ISD::LOAD) ? in visitAND()
6310 bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD, in visitAND()
6322 case ISD::EXTLOAD: B = CanZextLoadProfitably; break; in visitAND()
6323 case ISD::ZEXTLOAD: in visitAND()
6324 case ISD::NON_EXTLOAD: B = true; break; in visitAND()
6335 if (Load->getExtensionType() == ISD::EXTLOAD) { in visitAND()
6336 NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD, in visitAND()
6356 if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR && N0.hasOneUse() && N1C && in visitAND()
6357 ISD::isExtOpcode(N0.getOperand(0).getOpcode())) { in visitAND()
6364 (!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, ExtVT))) { in visitAND()
6368 DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), ExtVT, Extendee); in visitAND()
6370 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT, ZeroExtExtendee, in visitAND()
6388 GN0->getMemOperand(), GN0->getIndexType(), ISD::ZEXTLOAD); in visitAND()
6399 if (N1C && N0.getOpcode() == ISD::LOAD && !VT.isVector()) in visitAND()
6432 if (N1C && N1C->isOne() && N0.getOpcode() == ISD::SUB) { in visitAND()
6435 if (SubRHS.getOpcode() == ISD::ZERO_EXTEND && in visitAND()
6438 if (SubRHS.getOpcode() == ISD::SIGN_EXTEND && in visitAND()
6440 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0)); in visitAND()
6451 if (ISD::isUNINDEXEDLoad(N0.getNode()) && in visitAND()
6452 (ISD::isEXTLoad(N0.getNode()) || in visitAND()
6453 (ISD::isSEXTLoad(N0.getNode()) && N0.hasOneUse()))) { in visitAND()
6463 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) { in visitAND()
6465 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, LN0->getChain(), in visitAND()
6474 if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) { in visitAND()
6493 if (LHS->getOpcode() != ISD::SIGN_EXTEND) in visitAND()
6509 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, N0.getOperand(0)); in visitAND()
6511 if (hasOperation(ISD::USUBSAT, VT)) in visitAND()
6527 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT)) in MatchBSwapHWordLow()
6533 if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL) in MatchBSwapHWordLow()
6535 if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL) in MatchBSwapHWordLow()
6537 if (N0.getOpcode() == ISD::AND) { in MatchBSwapHWordLow()
6550 if (N1.getOpcode() == ISD::AND) { in MatchBSwapHWordLow()
6560 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL) in MatchBSwapHWordLow()
6562 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL) in MatchBSwapHWordLow()
6576 if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) { in MatchBSwapHWordLow()
6587 if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) { in MatchBSwapHWordLow()
6625 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00); in MatchBSwapHWordLow()
6628 Res = DAG.getNode(ISD::SRL, DL, VT, Res, in MatchBSwapHWordLow()
6646 if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL) in isBSwapHWordElement()
6651 if (Opc0 != ISD::AND && Opc0 != ISD::SHL && Opc0 != ISD::SRL) in isBSwapHWordElement()
6656 if (Opc == ISD::AND) in isBSwapHWordElement()
6658 else if (Opc0 == ISD::AND) in isBSwapHWordElement()
6672 if (Opc == ISD::SRL || (Opc == ISD::AND && Opc0 == ISD::SHL)) { in isBSwapHWordElement()
6682 if (Opc == ISD::AND) { in isBSwapHWordElement()
6686 if (Opc0 != ISD::SRL) in isBSwapHWordElement()
6694 if (Opc0 != ISD::SHL) in isBSwapHWordElement()
6700 } else if (Opc == ISD::SHL) { in isBSwapHWordElement()
6727 if (N.getOpcode() == ISD::OR) in isBSwapHWordPair()
6731 if (N.getOpcode() == ISD::SRL && N.getOperand(0).getOpcode() == ISD::BSWAP) { in isBSwapHWordPair()
6749 assert(N->getOpcode() == ISD::OR && VT == MVT::i32 && in matchBSwapHWordOrAndAnd()
6751 if (!TLI.isOperationLegalOrCustom(ISD::ROTR, VT)) in matchBSwapHWordOrAndAnd()
6753 if (N0.getOpcode() != ISD::AND || N1.getOpcode() != ISD::AND) in matchBSwapHWordOrAndAnd()
6767 if (Shift0.getOpcode() != ISD::SHL || Shift1.getOpcode() != ISD::SRL) in matchBSwapHWordOrAndAnd()
6779 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Shift0.getOperand(0)); in matchBSwapHWordOrAndAnd()
6781 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt); in matchBSwapHWordOrAndAnd()
6797 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT)) in MatchBSwapHWord()
6820 } else if (N0.getOpcode() == ISD::OR) { in MatchBSwapHWord()
6838 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, in MatchBSwapHWord()
6844 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT)) in MatchBSwapHWord()
6845 return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt); in MatchBSwapHWord()
6846 if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT)) in MatchBSwapHWord()
6847 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt); in MatchBSwapHWord()
6848 return DAG.getNode(ISD::OR, DL, VT, in MatchBSwapHWord()
6849 DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt), in MatchBSwapHWord()
6850 DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt)); in MatchBSwapHWord()
6867 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND && in visitORLike()
6883 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, in visitORLike()
6885 return DAG.getNode(ISD::AND, DL, VT, X, in visitORLike()
6893 if (N0.getOpcode() == ISD::AND && in visitORLike()
6894 N1.getOpcode() == ISD::AND && in visitORLike()
6898 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, in visitORLike()
6900 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X); in visitORLike()
6910 if (N0.getOpcode() == ISD::AND) { in visitORCommutative()
6918 return DAG.getNode(ISD::OR, SDLoc(N), VT, N00, N1); in visitORCommutative()
6923 return DAG.getNode(ISD::OR, SDLoc(N), VT, N01, N1); in visitORCommutative()
6930 if (V->getOpcode() == ISD::ZERO_EXTEND) in visitORCommutative()
6936 if (N0.getOpcode() == ISD::FSHL && N1.getOpcode() == ISD::SHL && in visitORCommutative()
6942 if (N0.getOpcode() == ISD::FSHR && N1.getOpcode() == ISD::SRL && in visitORCommutative()
6960 if (SDValue C = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, {N0, N1})) in visitOR()
6966 return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0); in visitOR()
6974 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitOR()
6978 if (ISD::isConstantSplatVectorAllOnes(N1.getNode())) in visitOR()
6987 bool ZeroN00 = ISD::isBuildVectorAllZeros(N0.getOperand(0).getNode()); in visitOR()
6988 bool ZeroN01 = ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode()); in visitOR()
6989 bool ZeroN10 = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode()); in visitOR()
6990 bool ZeroN11 = ISD::isBuildVectorAllZeros(N1.getOperand(1).getNode()); in visitOR()
7070 if (SDValue ROR = reassociateOps(ISD::OR, SDLoc(N), N0, N1, N->getFlags())) in visitOR()
7078 if (N0.getOpcode() == ISD::AND && N0->hasOneUse() && in visitOR()
7079 ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchIntersect, true)) { in visitOR()
7080 if (SDValue COR = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N1), VT, in visitOR()
7082 SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1); in visitOR()
7084 return DAG.getNode(ISD::AND, SDLoc(N), VT, COR, IOR); in visitOR()
7110 if ((!LegalOperations || TLI.isOperationLegal(ISD::ADD, VT)) && in visitOR()
7119 if (Op.getOpcode() == ISD::AND && in stripConstantMask()
7131 if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) { in matchRotateHalf()
7166 if (OppShift.getOpcode() != ISD::SHL && OppShift.getOpcode() != ISD::SRL) in extractShiftForRotate()
7180 if (OppShift.getOpcode() == ISD::SRL && OppShiftCst && in extractShiftForRotate()
7181 ExtractFrom.getOpcode() == ISD::ADD && in extractShiftForRotate()
7185 return DAG.getNode(ISD::SHL, DL, ShiftedVT, OppShiftLHS, in extractShiftForRotate()
7192 unsigned Opcode = ISD::DELETED_NODE; in extractShiftForRotate()
7205 if ((OppShift.getOpcode() != ISD::SRL || !SelectOpcode(ISD::SHL, ISD::MUL)) && in extractShiftForRotate()
7206 (OppShift.getOpcode() != ISD::SHL || !SelectOpcode(ISD::SRL, ISD::UDIV))) in extractShiftForRotate()
7334 if (Neg.getOpcode() != ISD::SUB) in matchRotateSub()
7369 (NegOp1.getOpcode() == ISD::TRUNCATE && Pos == NegOp1.getOperand(0))) in matchRotateSub()
7381 else if (Pos.getOpcode() == ISD::ADD && Pos.getOperand(0) == NegOp1) { in matchRotateSub()
7452 if (PosOpcode == ISD::FSHL && isPowerOf2_32(EltBits)) { in MatchFunnelPosNeg()
7462 if (IsBinOpImm(N1, ISD::SRL, 1) && in MatchFunnelPosNeg()
7463 IsBinOpImm(InnerNeg, ISD::XOR, EltBits - 1) && in MatchFunnelPosNeg()
7465 TLI.isOperationLegalOrCustom(ISD::FSHL, VT)) { in MatchFunnelPosNeg()
7466 return DAG.getNode(ISD::FSHL, DL, VT, N0, N1.getOperand(0), Pos); in MatchFunnelPosNeg()
7471 if (IsBinOpImm(N0, ISD::SHL, 1) && in MatchFunnelPosNeg()
7472 IsBinOpImm(InnerPos, ISD::XOR, EltBits - 1) && in MatchFunnelPosNeg()
7474 TLI.isOperationLegalOrCustom(ISD::FSHR, VT)) { in MatchFunnelPosNeg()
7475 return DAG.getNode(ISD::FSHR, DL, VT, N0.getOperand(0), N1, Neg); in MatchFunnelPosNeg()
7481 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N0.getOperand(1) && in MatchFunnelPosNeg()
7482 IsBinOpImm(InnerPos, ISD::XOR, EltBits - 1) && in MatchFunnelPosNeg()
7484 TLI.isOperationLegalOrCustom(ISD::FSHR, VT)) { in MatchFunnelPosNeg()
7485 return DAG.getNode(ISD::FSHR, DL, VT, N0.getOperand(0), N1, Neg); in MatchFunnelPosNeg()
7502 bool HasROTL = hasOperation(ISD::ROTL, VT); in MatchRotate()
7503 bool HasROTR = hasOperation(ISD::ROTR, VT); in MatchRotate()
7504 bool HasFSHL = hasOperation(ISD::FSHL, VT); in MatchRotate()
7505 bool HasFSHR = hasOperation(ISD::FSHR, VT); in MatchRotate()
7512 HasROTL |= TLI.getOperationAction(ISD::ROTL, VT) == TargetLowering::Custom; in MatchRotate()
7513 HasROTR |= TLI.getOperationAction(ISD::ROTR, VT) == TargetLowering::Custom; in MatchRotate()
7520 if (LHS.getOpcode() == ISD::TRUNCATE && RHS.getOpcode() == ISD::TRUNCATE && in MatchRotate()
7524 return DAG.getNode(ISD::TRUNCATE, SDLoc(LHS), LHS.getValueType(), Rot); in MatchRotate()
7570 if (RHSShift.getOpcode() == ISD::SHL) { in MatchRotate()
7577 if (LHSShift.getOpcode() != ISD::SHL || RHSShift.getOpcode() != ISD::SRL) in MatchRotate()
7598 SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt); in MatchRotate()
7599 Mask = DAG.getNode(ISD::AND, DL, VT, Mask, in MatchRotate()
7600 DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits)); in MatchRotate()
7603 SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt); in MatchRotate()
7604 Mask = DAG.getNode(ISD::AND, DL, VT, Mask, in MatchRotate()
7605 DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits)); in MatchRotate()
7608 Res = DAG.getNode(ISD::AND, DL, VT, Res, Mask); in MatchRotate()
7618 ISD::matchBinaryPredicate(LHSShiftAmt, RHSShiftAmt, MatchRotateSum)) { in MatchRotate()
7623 if (!Or.hasOneUse() || Or.getOpcode() != ISD::OR) in MatchRotate()
7641 SDValue RotX = DAG.getNode(ISD::ROTL, DL, VT, X, LHSShiftAmt); in MatchRotate()
7642 SDValue ShlY = DAG.getNode(ISD::SHL, DL, VT, Y, LHSShiftAmt); in MatchRotate()
7643 Res = DAG.getNode(ISD::OR, DL, VT, RotX, ShlY); in MatchRotate()
7646 SDValue RotX = DAG.getNode(ISD::ROTL, DL, VT, X, LHSShiftAmt); in MatchRotate()
7647 SDValue SrlY = DAG.getNode(ISD::SRL, DL, VT, Y, RHSShiftAmt); in MatchRotate()
7648 Res = DAG.getNode(ISD::OR, DL, VT, RotX, SrlY); in MatchRotate()
7664 if (ISD::matchBinaryPredicate(LHSShiftAmt, RHSShiftAmt, MatchRotateSum)) { in MatchRotate()
7668 Res = DAG.getNode(UseROTL ? ISD::ROTL : ISD::ROTR, DL, VT, LHSShiftArg, in MatchRotate()
7672 Res = DAG.getNode(UseFSHL ? ISD::FSHL : ISD::FSHR, DL, VT, LHSShiftArg, in MatchRotate()
7692 if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND || in MatchRotate()
7693 LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND || in MatchRotate()
7694 LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND || in MatchRotate()
7695 LHSShiftAmt.getOpcode() == ISD::TRUNCATE) && in MatchRotate()
7696 (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND || in MatchRotate()
7697 RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND || in MatchRotate()
7698 RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND || in MatchRotate()
7699 RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) { in MatchRotate()
7707 RExtOp0, HasROTL, ISD::ROTL, ISD::ROTR, DL); in MatchRotate()
7713 LExtOp0, HasROTR, ISD::ROTR, ISD::ROTL, DL); in MatchRotate()
7720 LExtOp0, RExtOp0, HasFSHL, ISD::FSHL, ISD::FSHR, DL); in MatchRotate()
7726 RExtOp0, LExtOp0, HasFSHR, ISD::FSHR, ISD::FSHL, DL); in MatchRotate()
7796 case ISD::OR: { in calculateByteProvider()
7810 case ISD::SHL: { in calculateByteProvider()
7825 case ISD::ANY_EXTEND: in calculateByteProvider()
7826 case ISD::SIGN_EXTEND: in calculateByteProvider()
7827 case ISD::ZERO_EXTEND: { in calculateByteProvider()
7835 return Op.getOpcode() == ISD::ZERO_EXTEND in calculateByteProvider()
7840 case ISD::BSWAP: in calculateByteProvider()
7843 case ISD::LOAD: { in calculateByteProvider()
7854 return L->getExtensionType() == ISD::ZEXTLOAD in calculateByteProvider()
7898 case ISD::TRUNCATE: in stripTruncAndExt()
7899 case ISD::ZERO_EXTEND: in stripTruncAndExt()
7900 case ISD::SIGN_EXTEND: in stripTruncAndExt()
7901 case ISD::ANY_EXTEND: in stripTruncAndExt()
7982 if (Trunc.getOpcode() != ISD::TRUNCATE) in mergeTruncStores()
7988 if ((WideVal.getOpcode() == ISD::SRL || WideVal.getOpcode() == ISD::SRA) && in mergeTruncStores()
8083 SourceValue = DAG.getNode(ISD::TRUNCATE, DL, WideVT, SourceValue); in mergeTruncStores()
8090 SourceValue = DAG.getNode(ISD::BSWAP, DL, WideVT, SourceValue); in mergeTruncStores()
8094 SourceValue = DAG.getNode(ISD::ROTR, DL, WideVT, SourceValue, RotAmt); in mergeTruncStores()
8137 assert(N->getOpcode() == ISD::OR && in MatchLoadCombine()
8233 !TLI.isOperationLegal(NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, in MatchLoadCombine()
8265 !TLI.isOperationLegal(ISD::BSWAP, VT)) in MatchLoadCombine()
8271 !TLI.isOperationLegal(ISD::SHL, VT)) in MatchLoadCombine()
8283 DAG.getExtLoad(NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, SDLoc(N), VT, in MatchLoadCombine()
8296 ? DAG.getNode(ISD::SHL, SDLoc(N), VT, NewLoad, in MatchLoadCombine()
8300 return DAG.getNode(ISD::BSWAP, SDLoc(N), VT, ShiftedLoad); in MatchLoadCombine()
8320 assert(N->getOpcode() == ISD::XOR); in unfoldMaskedMerge()
8332 if (And.getOpcode() != ISD::AND || !And.hasOneUse()) in unfoldMaskedMerge()
8335 if (Xor.getOpcode() != ISD::XOR || !Xor.hasOneUse()) in unfoldMaskedMerge()
8376 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, NotX, M); in unfoldMaskedMerge()
8378 SDValue RHS = DAG.getNode(ISD::OR, DL, VT, M, Y); in unfoldMaskedMerge()
8379 return DAG.getNode(ISD::AND, DL, VT, NotLHS, RHS); in unfoldMaskedMerge()
8388 SDValue LHS = DAG.getNode(ISD::OR, DL, VT, X, NotM); in unfoldMaskedMerge()
8390 SDValue RHS = DAG.getNode(ISD::AND, DL, VT, NotM, NotY); in unfoldMaskedMerge()
8392 return DAG.getNode(ISD::AND, DL, VT, LHS, NotRHS); in unfoldMaskedMerge()
8395 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, X, M); in unfoldMaskedMerge()
8397 SDValue RHS = DAG.getNode(ISD::AND, DL, VT, Y, NotM); in unfoldMaskedMerge()
8399 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS); in unfoldMaskedMerge()
8419 if (SDValue C = DAG.FoldConstantArithmetic(ISD::XOR, DL, VT, {N0, N1})) in visitXOR()
8425 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitXOR()
8433 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) in visitXOR()
8445 if (SDValue RXOR = reassociateOps(ISD::XOR, DL, N0, N1, N->getFlags())) in visitXOR()
8450 if ((!LegalOperations || TLI.isOperationLegal(ISD::ADD, VT)) && in visitXOR()
8460 ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(), in visitXOR()
8467 case ISD::SETCC: in visitXOR()
8469 case ISD::SELECT_CC: in visitXOR()
8472 case ISD::STRICT_FSETCC: in visitXOR()
8473 case ISD::STRICT_FSETCCS: { in visitXOR()
8479 N0.getOperand(0), N0Opcode == ISD::STRICT_FSETCCS); in visitXOR()
8492 if (isOneConstant(N1) && N0Opcode == ISD::ZERO_EXTEND && N0.hasOneUse() && in visitXOR()
8496 V = DAG.getNode(ISD::XOR, DL0, V.getValueType(), V, in visitXOR()
8499 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, V); in visitXOR()
8504 (N0Opcode == ISD::OR || N0Opcode == ISD::AND)) { in visitXOR()
8507 unsigned NewOpcode = N0Opcode == ISD::AND ? ISD::OR : ISD::AND; in visitXOR()
8508 N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00 in visitXOR()
8509 N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01 in visitXOR()
8516 (N0Opcode == ISD::OR || N0Opcode == ISD::AND)) { in visitXOR()
8519 unsigned NewOpcode = N0Opcode == ISD::AND ? ISD::OR : ISD::AND; in visitXOR()
8520 N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00 in visitXOR()
8521 N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01 in visitXOR()
8530 if (isAllOnesConstant(N1) && N0.getOpcode() == ISD::SUB && in visitXOR()
8532 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), in visitXOR()
8537 if (isAllOnesConstant(N1) && N0.getOpcode() == ISD::ADD && in visitXOR()
8539 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in visitXOR()
8544 if (N0Opcode == ISD::AND && N0.hasOneUse() && N0->getOperand(1) == N1) { in visitXOR()
8548 return DAG.getNode(ISD::AND, DL, VT, NotX, N1); in visitXOR()
8552 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) { in visitXOR()
8553 SDValue A = N0Opcode == ISD::ADD ? N0 : N1; in visitXOR()
8554 SDValue S = N0Opcode == ISD::SRA ? N0 : N1; in visitXOR()
8555 if (A.getOpcode() == ISD::ADD && S.getOpcode() == ISD::SRA) { in visitXOR()
8561 return DAG.getNode(ISD::ABS, DL, VT, S0); in visitXOR()
8587 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0Opcode == ISD::SHL && in visitXOR()
8589 return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT), in visitXOR()
8628 if (LogicOpcode != ISD::AND && LogicOpcode != ISD::OR && in combineShiftOfShiftedLogic()
8629 LogicOpcode != ISD::XOR) in combineShiftOfShiftedLogic()
8716 case ISD::OR: in visitShiftByConstant()
8717 case ISD::XOR: in visitShiftByConstant()
8718 case ISD::AND: in visitShiftByConstant()
8720 case ISD::ADD: in visitShiftByConstant()
8721 if (N->getOpcode() != ISD::SHL) in visitShiftByConstant()
8735 bool IsShiftByConstant = (BinOpLHSVal.getOpcode() == ISD::SHL || in visitShiftByConstant()
8736 BinOpLHSVal.getOpcode() == ISD::SRA || in visitShiftByConstant()
8737 BinOpLHSVal.getOpcode() == ISD::SRL) && in visitShiftByConstant()
8739 bool IsCopyOrSelect = BinOpLHSVal.getOpcode() == ISD::CopyFromReg || in visitShiftByConstant()
8740 BinOpLHSVal.getOpcode() == ISD::SELECT; in visitShiftByConstant()
8761 assert(N->getOpcode() == ISD::TRUNCATE); in distributeTruncateThroughAnd()
8762 assert(N->getOperand(0).getOpcode() == ISD::AND); in distributeTruncateThroughAnd()
8767 TLI.isTypeDesirableForOp(ISD::AND, TruncVT)) { in distributeTruncateThroughAnd()
8772 SDValue Trunc00 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00); in distributeTruncateThroughAnd()
8773 SDValue Trunc01 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N01); in distributeTruncateThroughAnd()
8776 return DAG.getNode(ISD::AND, DL, TruncVT, Trunc00, Trunc01); in distributeTruncateThroughAnd()
8807 if (ISD::matchUnaryPredicate(N1, MatchOutOfRange) && OutOfRange) { in visitRotate()
8811 DAG.FoldConstantArithmetic(ISD::UREM, dl, AmtVT, {N1, Bits})) in visitRotate()
8818 VT.getScalarSizeInBits() == 16 && hasOperation(ISD::BSWAP, VT)) in visitRotate()
8819 return DAG.getNode(ISD::BSWAP, dl, VT, N0); in visitRotate()
8826 if (N1.getOpcode() == ISD::TRUNCATE && in visitRotate()
8827 N1.getOperand(0).getOpcode() == ISD::AND) { in visitRotate()
8836 if (NextOp == ISD::ROTL || NextOp == ISD::ROTR) { in visitRotate()
8842 unsigned CombineOp = SameSide ? ISD::ADD : ISD::SUB; in visitRotate()
8844 SDValue Norm1 = DAG.FoldConstantArithmetic(ISD::UREM, dl, ShiftVT, in visitRotate()
8846 SDValue Norm2 = DAG.FoldConstantArithmetic(ISD::UREM, dl, ShiftVT, in visitRotate()
8852 ISD::UREM, dl, ShiftVT, {CombinedShift, BitsizeC}); in visitRotate()
8872 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, {N0, N1})) in visitSHL()
8884 if (N0.getOpcode() == ISD::AND) { in visitSHL()
8889 if (N01CV && N01CV->isConstant() && N00.getOpcode() == ISD::SETCC && in visitSHL()
8893 DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, {N01, N1})) in visitSHL()
8894 return DAG.getNode(ISD::AND, SDLoc(N), VT, N00, C); in visitSHL()
8908 if (N1.getOpcode() == ISD::TRUNCATE && in visitSHL()
8909 N1.getOperand(0).getOpcode() == ISD::AND) { in visitSHL()
8911 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, NewOp1); in visitSHL()
8918 if (N0.getOpcode() == ISD::SHL) { in visitSHL()
8926 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange)) in visitSHL()
8936 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) { in visitSHL()
8938 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1)); in visitSHL()
8939 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum); in visitSHL()
8948 if ((N0.getOpcode() == ISD::ZERO_EXTEND || in visitSHL()
8949 N0.getOpcode() == ISD::ANY_EXTEND || in visitSHL()
8950 N0.getOpcode() == ISD::SIGN_EXTEND) && in visitSHL()
8951 N0.getOperand(0).getOpcode() == ISD::SHL) { in visitSHL()
8965 if (ISD::matchBinaryPredicate(InnerShiftAmt, N1, MatchOutOfRange, in visitSHL()
8978 if (ISD::matchBinaryPredicate(InnerShiftAmt, N1, MatchInRange, in visitSHL()
8984 Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, Sum, N1); in visitSHL()
8985 return DAG.getNode(ISD::SHL, DL, VT, Ext, Sum); in visitSHL()
8992 if (N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() && in visitSHL()
8993 N0.getOperand(0).getOpcode() == ISD::SRL) { in visitSHL()
9003 if (ISD::matchBinaryPredicate(InnerShiftAmt, N1, MatchEqual, in visitSHL()
9009 NewSHL = DAG.getNode(ISD::SHL, DL, N0Op0.getValueType(), N0Op0, NewSHL); in visitSHL()
9011 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL); in visitSHL()
9015 if (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA) { in visitSHL()
9029 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchShiftAmount, in visitSHL()
9033 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N1, N01); in visitSHL()
9034 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff); in visitSHL()
9036 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchShiftAmount, in visitSHL()
9040 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N01, N1); in visitSHL()
9049 if (N0.getOpcode() == ISD::SRL && in visitSHL()
9052 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchShiftAmount, in visitSHL()
9056 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N01, N1); in visitSHL()
9058 Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, N01); in visitSHL()
9059 Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, Diff); in visitSHL()
9060 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Diff); in visitSHL()
9061 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSHL()
9063 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchShiftAmount, in visitSHL()
9067 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N1, N01); in visitSHL()
9069 Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, N1); in visitSHL()
9070 SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff); in visitSHL()
9071 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSHL()
9077 if (N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1) && in visitSHL()
9081 SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1); in visitSHL()
9082 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), HiBitsMask); in visitSHL()
9089 if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR) && in visitSHL()
9094 SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1); in visitSHL()
9095 SDValue Shl1 = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1); in visitSHL()
9102 if (N0.getOpcode() == ISD::MUL && N0->hasOneUse()) { in visitSHL()
9105 DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N1), VT, {N01, N1})) in visitSHL()
9106 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), Shl); in visitSHL()
9115 if (N0.getOpcode() == ISD::VSCALE) in visitSHL()
9124 if (N0.getOpcode() == ISD::STEP_VECTOR) in visitSHL()
9125 if (ISD::isConstantSplatVector(N1.getNode(), ShlVal)) { in visitSHL()
9142 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) && in combineShiftToMULH()
9155 if (ShiftOperand.getOpcode() != ISD::MUL) in combineShiftToMULH()
9162 bool IsSignExt = LeftOp.getOpcode() == ISD::SIGN_EXTEND; in combineShiftToMULH()
9163 bool IsZeroExt = LeftOp.getOpcode() == ISD::ZERO_EXTEND; in combineShiftToMULH()
9209 unsigned MulhOpcode = IsSignExt ? ISD::MULHS : ISD::MULHU; in combineShiftToMULH()
9217 return (N->getOpcode() == ISD::SRA ? DAG.getSExtOrTrunc(Result, DL, WideVT) in combineShiftToMULH()
9231 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, {N0, N1})) in visitSRA()
9251 if (N1C && N0.getOpcode() == ISD::SHL && N1 == N0.getOperand(1)) { in visitSRA()
9258 TLI.getOperationAction(ISD::SIGN_EXTEND_INREG, ExtVT) == in visitSRA()
9260 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, in visitSRA()
9270 if (N0.getOpcode() == ISD::SRA) { in visitSRA()
9286 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), SumOfShifts)) { in visitSRA()
9288 if (N1.getOpcode() == ISD::BUILD_VECTOR) in visitSRA()
9290 else if (N1.getOpcode() == ISD::SPLAT_VECTOR) { in visitSRA()
9297 return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0), ShiftValue); in visitSRA()
9306 if (N0.getOpcode() == ISD::SHL && N1C) { in visitSRA()
9325 TLI.isOperationLegalOrCustom(ISD::SIGN_EXTEND, TruncVT) && in visitSRA()
9326 TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) && in visitSRA()
9331 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, in visitSRA()
9333 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, in visitSRA()
9335 return DAG.getNode(ISD::SIGN_EXTEND, DL, in visitSRA()
9346 if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB) && N1C && in visitSRA()
9348 bool IsAdd = N0.getOpcode() == ISD::ADD; in visitSRA()
9350 if (Shl.getOpcode() == ISD::SHL && Shl.getOperand(1) == N1 && in visitSRA()
9377 Add = DAG.getNode(ISD::ADD, DL, TruncVT, Trunc, ShiftC); in visitSRA()
9379 Add = DAG.getNode(ISD::SUB, DL, TruncVT, ShiftC, Trunc); in visitSRA()
9387 if (N1.getOpcode() == ISD::TRUNCATE && in visitSRA()
9388 N1.getOperand(0).getOpcode() == ISD::AND) { in visitSRA()
9390 return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0, NewOp1); in visitSRA()
9397 if (N0.getOpcode() == ISD::TRUNCATE && in visitSRA()
9398 (N0.getOperand(0).getOpcode() == ISD::SRL || in visitSRA()
9399 N0.getOperand(0).getOpcode() == ISD::SRA) && in visitSRA()
9410 Amt = DAG.getNode(ISD::ADD, DL, LargeShiftVT, Amt, in visitSRA()
9413 DAG.getNode(ISD::SRA, DL, LargeVT, N0Op0.getOperand(0), Amt); in visitSRA()
9414 return DAG.getNode(ISD::TRUNCATE, DL, VT, SRA); in visitSRA()
9425 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, N1); in visitSRA()
9454 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, {N0, N1})) in visitSRL()
9472 if (N0.getOpcode() == ISD::SRL) { in visitSRL()
9480 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange)) in visitSRL()
9490 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) { in visitSRL()
9492 SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1)); in visitSRL()
9493 return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Sum); in visitSRL()
9497 if (N1C && N0.getOpcode() == ISD::TRUNCATE && in visitSRL()
9498 N0.getOperand(0).getOpcode() == ISD::SRL) { in visitSRL()
9514 SDValue NewShift = DAG.getNode(ISD::SRL, DL, InnerShiftVT, in visitSRL()
9516 return DAG.getNode(ISD::TRUNCATE, DL, VT, NewShift); in visitSRL()
9524 SDValue NewShift = DAG.getNode(ISD::SRL, DL, InnerShiftVT, in visitSRL()
9529 SDValue And = DAG.getNode(ISD::AND, DL, InnerShiftVT, NewShift, Mask); in visitSRL()
9530 return DAG.getNode(ISD::TRUNCATE, DL, VT, And); in visitSRL()
9537 if (N0.getOpcode() == ISD::SHL && in visitSRL()
9547 if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchShiftAmount, in visitSRL()
9552 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N01, N1); in visitSRL()
9554 Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, N01); in visitSRL()
9555 Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, Diff); in visitSRL()
9556 SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff); in visitSRL()
9557 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSRL()
9559 if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchShiftAmount, in visitSRL()
9564 SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N1, N01); in visitSRL()
9566 Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, N1); in visitSRL()
9567 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Diff); in visitSRL()
9568 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSRL()
9574 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) { in visitSRL()
9581 if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, SmallVT)) { in visitSRL()
9584 SDValue SmallShift = DAG.getNode(ISD::SRL, DL0, SmallVT, in visitSRL()
9591 return DAG.getNode(ISD::AND, DL, VT, in visitSRL()
9592 DAG.getNode(ISD::ANY_EXTEND, DL, VT, SmallShift), in visitSRL()
9600 if (N0.getOpcode() == ISD::SRA) in visitSRL()
9601 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), N1); in visitSRL()
9605 if (N1C && N0.getOpcode() == ISD::CTLZ && in visitSRL()
9629 Op = DAG.getNode(ISD::SRL, DL, VT, Op, in visitSRL()
9636 return DAG.getNode(ISD::XOR, DL, VT, in visitSRL()
9642 if (N1.getOpcode() == ISD::TRUNCATE && in visitSRL()
9643 N1.getOperand(0).getOpcode() == ISD::AND) { in visitSRL()
9645 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, NewOp1); in visitSRL()
9680 if (Use->getOpcode() == ISD::BRCOND) in visitSRL()
9682 else if (Use->getOpcode() == ISD::TRUNCATE && Use->hasOneUse()) { in visitSRL()
9685 if (Use->getOpcode() == ISD::BRCOND) in visitSRL()
9703 bool IsFSHL = N->getOpcode() == ISD::FSHL; in visitFunnelShift()
9737 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N1, in visitFunnelShift()
9741 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, in visitFunnelShift()
9756 (LHS->hasOneUse() || RHS->hasOneUse()) && ISD::isNON_EXTLoad(RHS) && in visitFunnelShift()
9757 ISD::isNON_EXTLoad(LHS)) { in visitFunnelShift()
9792 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N1, N2); in visitFunnelShift()
9794 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, N2); in visitFunnelShift()
9801 unsigned RotOpc = IsFSHL ? ISD::ROTL : ISD::ROTR; in visitFunnelShift()
9827 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::SHL, VT)) { in visitSHLSAT()
9829 if (N->getOpcode() == ISD::SSHLSAT && N1C && in visitSHLSAT()
9831 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, N1); in visitSHLSAT()
9834 if (N->getOpcode() == ISD::USHLSAT && N1C && in visitSHLSAT()
9837 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, N1); in visitSHLSAT()
9851 if (AbsOp1.getOpcode() != ISD::SUB) in combineABSToABD()
9860 (Opc0 != ISD::ZERO_EXTEND && Opc0 != ISD::SIGN_EXTEND)) in combineABSToABD()
9866 unsigned ABDOpcode = (Opc0 == ISD::SIGN_EXTEND) ? ISD::ABDS : ISD::ABDU; in combineABSToABD()
9875 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, ABD); in combineABSToABD()
9892 return DAG.getNode(ISD::ABS, SDLoc(N), VT, N0); in visitABS()
9894 if (N0.getOpcode() == ISD::ABS) in visitABS()
9913 return DAG.getNode(ISD::BSWAP, DL, VT, N0); in visitBSWAP()
9915 if (N0.getOpcode() == ISD::BSWAP) in visitBSWAP()
9922 if (N0.getOpcode() == ISD::BITREVERSE && N0.hasOneUse()) { in visitBSWAP()
9923 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, N0.getOperand(0)); in visitBSWAP()
9924 return DAG.getNode(ISD::BITREVERSE, DL, VT, BSwap); in visitBSWAP()
9930 if (BW >= 32 && N0.getOpcode() == ISD::SHL && N0.hasOneUse()) { in visitBSWAP()
9937 (!LegalOperations || hasOperation(ISD::BSWAP, HalfVT))) { in visitBSWAP()
9940 Res = DAG.getNode(ISD::SHL, DL, VT, Res, in visitBSWAP()
9943 Res = DAG.getNode(ISD::BSWAP, DL, HalfVT, Res); in visitBSWAP()
9952 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) && in visitBSWAP()
9957 SDValue NewSwap = DAG.getNode(ISD::BSWAP, DL, VT, N0.getOperand(0)); in visitBSWAP()
9958 unsigned InverseShift = N0.getOpcode() == ISD::SHL ? ISD::SRL : ISD::SHL; in visitBSWAP()
9972 return DAG.getNode(ISD::BITREVERSE, SDLoc(N), VT, N0); in visitBITREVERSE()
9974 if (N0.getOpcode() == ISD::BITREVERSE) in visitBITREVERSE()
9985 return DAG.getNode(ISD::CTLZ, SDLoc(N), VT, N0); in visitCTLZ()
9988 if (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ_ZERO_UNDEF, VT)) { in visitCTLZ()
9990 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0); in visitCTLZ()
10002 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0); in visitCTLZ_ZERO_UNDEF()
10012 return DAG.getNode(ISD::CTTZ, SDLoc(N), VT, N0); in visitCTTZ()
10015 if (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ_ZERO_UNDEF, VT)) { in visitCTTZ()
10017 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0); in visitCTTZ()
10029 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0); in visitCTTZ_ZERO_UNDEF()
10039 return DAG.getNode(ISD::CTPOP, SDLoc(N), VT, N0); in visitCTPOP()
10059 ISD::CondCode CC, const TargetLowering &TLI, in combineMinNumMaxNum()
10066 case ISD::SETOLT: in combineMinNumMaxNum()
10067 case ISD::SETOLE: in combineMinNumMaxNum()
10068 case ISD::SETLT: in combineMinNumMaxNum()
10069 case ISD::SETLE: in combineMinNumMaxNum()
10070 case ISD::SETULT: in combineMinNumMaxNum()
10071 case ISD::SETULE: { in combineMinNumMaxNum()
10075 unsigned IEEEOpcode = (LHS == True) ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE; in combineMinNumMaxNum()
10079 unsigned Opcode = (LHS == True) ? ISD::FMINNUM : ISD::FMAXNUM; in combineMinNumMaxNum()
10084 case ISD::SETOGT: in combineMinNumMaxNum()
10085 case ISD::SETOGE: in combineMinNumMaxNum()
10086 case ISD::SETGT: in combineMinNumMaxNum()
10087 case ISD::SETGE: in combineMinNumMaxNum()
10088 case ISD::SETUGT: in combineMinNumMaxNum()
10089 case ISD::SETUGE: { in combineMinNumMaxNum()
10090 unsigned IEEEOpcode = (LHS == True) ? ISD::FMAXNUM_IEEE : ISD::FMINNUM_IEEE; in combineMinNumMaxNum()
10094 unsigned Opcode = (LHS == True) ? ISD::FMAXNUM : ISD::FMINNUM; in combineMinNumMaxNum()
10114 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse() || in foldSelectOfConstantsUsingSra()
10122 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in foldSelectOfConstantsUsingSra()
10123 if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(CondC) && in foldSelectOfConstantsUsingSra()
10128 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC); in foldSelectOfConstantsUsingSra()
10129 return DAG.getNode(ISD::OR, DL, VT, Sra, C1); in foldSelectOfConstantsUsingSra()
10131 if (CC == ISD::SETLT && isNullOrNullSplat(CondC) && isNullOrNullSplat(C2)) { in foldSelectOfConstantsUsingSra()
10135 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC); in foldSelectOfConstantsUsingSra()
10136 return DAG.getNode(ISD::AND, DL, VT, Sra, C1); in foldSelectOfConstantsUsingSra()
10166 NotCond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, NotCond); in foldSelectOfConstants()
10173 NotCond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NotCond); in foldSelectOfConstants()
10179 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond); in foldSelectOfConstants()
10185 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond); in foldSelectOfConstants()
10199 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond); in foldSelectOfConstants()
10200 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2); in foldSelectOfConstants()
10205 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond); in foldSelectOfConstants()
10206 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2); in foldSelectOfConstants()
10212 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond); in foldSelectOfConstants()
10215 return DAG.getNode(ISD::SHL, DL, VT, Cond, ShAmtC); in foldSelectOfConstants()
10241 DAG.getNode(ISD::XOR, DL, CondVT, Cond, DAG.getConstant(1, DL, CondVT)); in foldSelectOfConstants()
10251 assert((N->getOpcode() == ISD::SELECT || N->getOpcode() == ISD::VSELECT) && in foldBoolSelectToLogic()
10262 return DAG.getNode(ISD::OR, SDLoc(N), VT, Cond, F); in foldBoolSelectToLogic()
10267 return DAG.getNode(ISD::AND, SDLoc(N), VT, Cond, T); in foldBoolSelectToLogic()
10272 return DAG.getNode(ISD::OR, SDLoc(N), VT, NotCond, T); in foldBoolSelectToLogic()
10278 return DAG.getNode(ISD::AND, SDLoc(N), VT, NotCond, F); in foldBoolSelectToLogic()
10289 if (N0.getOpcode() != ISD::SETCC || !N0.hasOneUse()) in foldVSelectToSignBitSplatMask()
10294 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in foldVSelectToSignBitSplatMask()
10300 if (CC == ISD::SETLT && isNullOrNullSplat(Cond1)) in foldVSelectToSignBitSplatMask()
10302 else if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(Cond1)) in foldVSelectToSignBitSplatMask()
10311 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt); in foldVSelectToSignBitSplatMask()
10312 return DAG.getNode(ISD::AND, DL, VT, Sra, N1); in foldVSelectToSignBitSplatMask()
10319 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt); in foldVSelectToSignBitSplatMask()
10320 return DAG.getNode(ISD::OR, DL, VT, Sra, N2); in foldVSelectToSignBitSplatMask()
10330 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt); in foldVSelectToSignBitSplatMask()
10332 return DAG.getNode(ISD::AND, DL, VT, Not, N2); in foldVSelectToSignBitSplatMask()
10377 if (N0->getOpcode() == ISD::AND && N0->hasOneUse()) { in visitSELECT()
10381 DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2, Flags); in visitSELECT()
10383 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, in visitSELECT()
10390 if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) { in visitSELECT()
10393 SDValue InnerSelect = DAG.getNode(ISD::SELECT, DL, N1.getValueType(), in visitSELECT()
10396 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, N1, in visitSELECT()
10404 if (N1->getOpcode() == ISD::SELECT && N1->hasOneUse()) { in visitSELECT()
10411 SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0); in visitSELECT()
10412 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), And, N1_1, in visitSELECT()
10417 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1_1, in visitSELECT()
10423 if (N2->getOpcode() == ISD::SELECT && N2->hasOneUse()) { in visitSELECT()
10430 SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0); in visitSELECT()
10431 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Or, N1, in visitSELECT()
10436 return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1, in visitSELECT()
10450 if (N0.getOpcode() == ISD::SETCC) { in visitSELECT()
10452 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in visitSELECT()
10468 if (!LegalOperations && TLI.isOperationLegalOrCustom(ISD::UADDO, VT) && in visitSELECT()
10469 CC == ISD::SETUGT && N0.hasOneUse() && isAllOnesConstant(N1) && in visitSELECT()
10470 N2.getOpcode() == ISD::ADD && Cond0 == N2.getOperand(0)) { in visitSELECT()
10487 SDValue UAO = DAG.getNode(ISD::UADDO, DL, VTs, Cond0, N2.getOperand(1)); in visitSELECT()
10492 if (TLI.isOperationLegal(ISD::SELECT_CC, VT) || in visitSELECT()
10494 TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT))) { in visitSELECT()
10498 SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1, in visitSELECT()
10524 assert(LHS.getOpcode() == ISD::CONCAT_VECTORS && in ConvertSelectToConcatVector()
10525 RHS.getOpcode() == ISD::CONCAT_VECTORS && in ConvertSelectToConcatVector()
10526 Cond.getOpcode() == ISD::BUILD_VECTOR); in ConvertSelectToConcatVector()
10565 ISD::CONCAT_VECTORS, DL, VT, in ConvertSelectToConcatVector()
10572 if (!isNullConstant(BasePtr) || Index.getOpcode() != ISD::ADD) in refineUniformBase()
10591 bool refineIndexType(SDValue &Index, ISD::MemIndexType &IndexType, EVT DataVT, in refineIndexType()
10596 if (Index.getOpcode() == ISD::ZERO_EXTEND) { in refineIndexType()
10599 IndexType = ISD::UNSIGNED_SCALED; in refineIndexType()
10603 if (ISD::isIndexTypeSigned(IndexType)) { in refineIndexType()
10604 IndexType = ISD::UNSIGNED_SCALED; in refineIndexType()
10610 if (Index.getOpcode() == ISD::SIGN_EXTEND && in refineIndexType()
10611 ISD::isIndexTypeSigned(IndexType)) { in refineIndexType()
10630 ISD::MemIndexType IndexType = MSC->getIndexType(); in visitMSCATTER()
10634 if (ISD::isConstantSplatVectorAllZeros(Mask.getNode())) in visitMSCATTER()
10663 if (ISD::isConstantSplatVectorAllZeros(Mask.getNode())) in visitMSTORE()
10668 if (ISD::isConstantSplatVectorAllOnes(Mask.getNode()) && MST->isUnindexed() && in visitMSTORE()
10694 if (N->getOpcode() != ISD::DELETED_NODE) in visitMSTORE()
10703 if ((Value.getOpcode() == ISD::TRUNCATE) && Value->hasOneUse() && in visitMSTORE()
10726 ISD::MemIndexType IndexType = MGT->getIndexType(); in visitMGATHER()
10730 if (ISD::isConstantSplatVectorAllZeros(Mask.getNode())) in visitMGATHER()
10756 if (ISD::isConstantSplatVectorAllZeros(Mask.getNode())) in visitMLOAD()
10761 if (ISD::isConstantSplatVectorAllOnes(Mask.getNode()) && MLD->isUnindexed() && in visitMLOAD()
10762 !MLD->isExpandingLoad() && MLD->getExtensionType() == ISD::NON_EXTLOAD) { in visitMLOAD()
10786 !ISD::isBuildVectorOfConstantSDNodes(N1.getNode()) || in foldVSelectOfConstants()
10787 !ISD::isBuildVectorOfConstantSDNodes(N2.getNode())) in foldVSelectOfConstants()
10818 auto ExtendOpcode = AllAddOne ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND; in foldVSelectOfConstants()
10820 return DAG.getNode(ISD::ADD, DL, VT, ExtendedCond, N2); in foldVSelectOfConstants()
10825 if (ISD::isConstantSplatVector(N1.getNode(), Pow2C) && Pow2C.isPowerOf2() && in foldVSelectOfConstants()
10829 return DAG.getNode(ISD::SHL, DL, VT, ZextCond, ShAmtC); in foldVSelectOfConstants()
10864 if (N0.getOpcode() == ISD::SETCC) { in visitVSELECT()
10866 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in visitVSELECT()
10868 bool RHSIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode()); in visitVSELECT()
10870 if (((RHSIsAllZeros && (CC == ISD::SETGT || CC == ISD::SETGE)) || in visitVSELECT()
10871 (ISD::isBuildVectorAllOnes(RHS.getNode()) && CC == ISD::SETGT)) && in visitVSELECT()
10872 N1 == LHS && N2.getOpcode() == ISD::SUB && N1 == N2.getOperand(1)) in visitVSELECT()
10873 isAbs = ISD::isBuildVectorAllZeros(N2.getOperand(0).getNode()); in visitVSELECT()
10874 else if ((RHSIsAllZeros && (CC == ISD::SETLT || CC == ISD::SETLE)) && in visitVSELECT()
10875 N2 == LHS && N1.getOpcode() == ISD::SUB && N2 == N1.getOperand(1)) in visitVSELECT()
10876 isAbs = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode()); in visitVSELECT()
10879 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) in visitVSELECT()
10880 return DAG.getNode(ISD::ABS, DL, VT, LHS); in visitVSELECT()
10882 SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, LHS, in visitVSELECT()
10885 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift); in visitVSELECT()
10888 return DAG.getNode(ISD::XOR, DL, VT, Add, Shift); in visitVSELECT()
10920 auto LoadExtOpcode = IsSigned ? ISD::SEXTLOAD : ISD::ZEXTLOAD; in visitVSELECT()
10921 if (LHS.getOpcode() == ISD::LOAD && LHS.hasOneUse() && in visitVSELECT()
10924 TLI.isOperationLegalOrCustom(ISD::SETCC, WideVT)) { in visitVSELECT()
10929 auto ExtOpcode = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in visitVSELECT()
10939 if (hasOperation(ISD::UADDSAT, VT)) { in visitVSELECT()
10943 ISD::CondCode SatCC = CC; in visitVSELECT()
10944 if (ISD::isConstantSplatVectorAllOnes(N1.getNode())) { in visitVSELECT()
10946 SatCC = ISD::getSetCCInverse(SatCC, VT.getScalarType()); in visitVSELECT()
10947 } else if (ISD::isConstantSplatVectorAllOnes(N2.getNode())) { in visitVSELECT()
10951 if (Other && Other.getOpcode() == ISD::ADD) { in visitVSELECT()
10956 if (SatCC == ISD::SETUGE) { in visitVSELECT()
10958 SatCC = ISD::SETULE; in visitVSELECT()
10964 if (SatCC == ISD::SETULE && Other == CondRHS && in visitVSELECT()
10966 return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
10969 (OpRHS.getOpcode() == ISD::BUILD_VECTOR || in visitVSELECT()
10970 OpRHS.getOpcode() == ISD::SPLAT_VECTOR) && in visitVSELECT()
10978 if (SatCC == ISD::SETULE && in visitVSELECT()
10979 ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUADDSAT)) in visitVSELECT()
10980 return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
10986 if (hasOperation(ISD::USUBSAT, VT)) { in visitVSELECT()
10990 ISD::CondCode SatCC = CC; in visitVSELECT()
10991 if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) { in visitVSELECT()
10993 SatCC = ISD::getSetCCInverse(SatCC, VT.getScalarType()); in visitVSELECT()
10994 } else if (ISD::isConstantSplatVectorAllZeros(N2.getNode())) { in visitVSELECT()
11002 if (Other && Other.getOpcode() == ISD::TRUNCATE && in visitVSELECT()
11003 Other.getOperand(0).getOpcode() == ISD::SUB && in visitVSELECT()
11004 (SatCC == ISD::SETUGE || SatCC == ISD::SETUGT)) { in visitVSELECT()
11007 if (LHS == OpLHS && RHS == OpRHS && LHS.getOpcode() == ISD::ZERO_EXTEND) in visitVSELECT()
11021 if ((SatCC == ISD::SETUGE || SatCC == ISD::SETUGT) && in visitVSELECT()
11022 Other.getOpcode() == ISD::SUB && OpRHS == CondRHS) in visitVSELECT()
11023 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
11025 if (OpRHS.getOpcode() == ISD::BUILD_VECTOR || in visitVSELECT()
11026 OpRHS.getOpcode() == ISD::SPLAT_VECTOR) { in visitVSELECT()
11027 if (CondRHS.getOpcode() == ISD::BUILD_VECTOR || in visitVSELECT()
11028 CondRHS.getOpcode() == ISD::SPLAT_VECTOR) { in visitVSELECT()
11037 if (SatCC == ISD::SETUGT && Other.getOpcode() == ISD::ADD && in visitVSELECT()
11038 ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUSUBSAT, in visitVSELECT()
11040 OpRHS = DAG.getNode(ISD::SUB, DL, VT, in visitVSELECT()
11042 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
11051 if (SatCC == ISD::SETLT && Other.getOpcode() == ISD::XOR && in visitVSELECT()
11052 ISD::isConstantSplatVector(OpRHS.getNode(), SplatValue) && in visitVSELECT()
11053 ISD::isConstantSplatVectorAllZeros(CondRHS.getNode()) && in visitVSELECT()
11058 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
11071 if (ISD::isConstantSplatVectorAllOnes(N0.getNode())) in visitVSELECT()
11074 if (ISD::isConstantSplatVectorAllZeros(N0.getNode())) in visitVSELECT()
11080 if (N1.getOpcode() == ISD::CONCAT_VECTORS && in visitVSELECT()
11081 N2.getOpcode() == ISD::CONCAT_VECTORS && in visitVSELECT()
11082 ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) { in visitVSELECT()
11090 if (hasOperation(ISD::SRA, VT)) in visitVSELECT()
11106 ISD::CondCode CC = cast<CondCodeSDNode>(N4)->get(); in visitSELECT_CC()
11128 if (SCC.getOpcode() == ISD::SETCC) { in visitSELECT_CC()
11130 ISD::SELECT_CC, SDLoc(N), N2.getValueType(), SCC.getOperand(0), in visitSELECT_CC()
11150 N->hasOneUse() && N->use_begin()->getOpcode() == ISD::BRCOND; in visitSETCC()
11152 ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(2))->get(); in visitSETCC()
11176 auto IsAlwaysTrueOrFalse = [](ISD::CondCode Cond, ConstantSDNode *C) { in visitSETCC()
11177 bool False = (Cond == ISD::SETULT && C->isZero()) || in visitSETCC()
11178 (Cond == ISD::SETLT && C->isMinSignedValue()) || in visitSETCC()
11179 (Cond == ISD::SETUGT && C->isAllOnes()) || in visitSETCC()
11180 (Cond == ISD::SETGT && C->isMaxSignedValue()); in visitSETCC()
11181 bool True = (Cond == ISD::SETULE && C->isAllOnes()) || in visitSETCC()
11182 (Cond == ISD::SETLE && C->isMaxSignedValue()) || in visitSETCC()
11183 (Cond == ISD::SETUGE && C->isZero()) || in visitSETCC()
11184 (Cond == ISD::SETGE && C->isMinSignedValue()); in visitSETCC()
11188 if (N0->getOpcode() == ISD::FREEZE && N0.hasOneUse() && N1C) { in visitSETCC()
11194 if (N1->getOpcode() == ISD::FREEZE && N1.hasOneUse() && N0C) { in visitSETCC()
11195 if (!IsAlwaysTrueOrFalse(ISD::getSetCCSwappedOperands(Cond), in visitSETCC()
11214 if (PreferSetCC && Combined.getOpcode() != ISD::SETCC) { in visitSETCC()
11236 return DAG.getNode(ISD::SETCC, SDLoc(N), N->getVTList(), LHS, RHS, Cond); in visitSETCCCARRY()
11256 ISD::LoadExtType LoadExt = Load->getExtensionType(); in isCompatibleLoad()
11257 if (LoadExt == ISD::NON_EXTLOAD || LoadExt == ISD::EXTLOAD) in isCompatibleLoad()
11262 if ((LoadExt == ISD::SEXTLOAD && ExtOpcode != ISD::SIGN_EXTEND) || in isCompatibleLoad()
11263 (LoadExt == ISD::ZEXTLOAD && ExtOpcode != ISD::ZERO_EXTEND)) in isCompatibleLoad()
11282 assert((Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND || in tryToFoldExtendSelectLoad()
11283 Opcode == ISD::ANY_EXTEND) && in tryToFoldExtendSelectLoad()
11286 if (!(N0->getOpcode() == ISD::SELECT || N0->getOpcode() == ISD::VSELECT) || in tryToFoldExtendSelectLoad()
11295 auto ExtLoadOpcode = ISD::EXTLOAD; in tryToFoldExtendSelectLoad()
11296 if (Opcode == ISD::SIGN_EXTEND) in tryToFoldExtendSelectLoad()
11297 ExtLoadOpcode = ISD::SEXTLOAD; in tryToFoldExtendSelectLoad()
11298 else if (Opcode == ISD::ZERO_EXTEND) in tryToFoldExtendSelectLoad()
11299 ExtLoadOpcode = ISD::ZEXTLOAD; in tryToFoldExtendSelectLoad()
11325 assert((Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND || in tryToFoldExtendOfConstant()
11326 Opcode == ISD::ANY_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG || in tryToFoldExtendOfConstant()
11327 Opcode == ISD::ZERO_EXTEND_VECTOR_INREG) in tryToFoldExtendOfConstant()
11339 if (N0->getOpcode() == ISD::SELECT) { in tryToFoldExtendOfConstant()
11343 (Opcode != ISD::ZERO_EXTEND || !TLI.isZExtFree(N0.getValueType(), VT))) { in tryToFoldExtendOfConstant()
11354 if (FoldOpc == ISD::ANY_EXTEND) in tryToFoldExtendOfConstant()
11355 FoldOpc = ISD::SIGN_EXTEND; in tryToFoldExtendOfConstant()
11367 ISD::isBuildVectorOfConstantSDNodes(N0.getNode()))) in tryToFoldExtendOfConstant()
11379 Opcode == ISD::ZERO_EXTEND || Opcode == ISD::ZERO_EXTEND_VECTOR_INREG; in tryToFoldExtendOfConstant()
11392 if (Opcode == ISD::SIGN_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG) in tryToFoldExtendOfConstant()
11419 if (ExtOpc != ISD::ANY_EXTEND && User->getOpcode() == ISD::SETCC) { in ExtendUsesToFormExtLoad()
11420 ISD::CondCode CC = cast<CondCodeSDNode>(User->getOperand(2))->get(); in ExtendUsesToFormExtLoad()
11421 if (ExtOpc == ISD::ZERO_EXTEND && ISD::isSignedIntSetCC(CC)) in ExtendUsesToFormExtLoad()
11442 if (User->getOpcode() == ISD::CopyToReg) in ExtendUsesToFormExtLoad()
11451 if (Use.getResNo() == 0 && Use.getUser()->getOpcode() == ISD::CopyToReg) { in ExtendUsesToFormExtLoad()
11466 ISD::NodeType ExtType) { in ExtendSetCCUses()
11481 CombineTo(SetCC, DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops)); in ExtendSetCCUses()
11491 assert((N->getOpcode() == ISD::SIGN_EXTEND || in CombineExtLoad()
11492 N->getOpcode() == ISD::ZERO_EXTEND) && in CombineExtLoad()
11512 if (N0->getOpcode() != ISD::LOAD) in CombineExtLoad()
11517 if (!ISD::isNON_EXTLoad(LN0) || !ISD::isUNINDEXEDLoad(LN0) || in CombineExtLoad()
11527 ISD::LoadExtType ExtType = in CombineExtLoad()
11528 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD; in CombineExtLoad()
11567 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); in CombineExtLoad()
11568 SDValue NewValue = DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Loads); in CombineExtLoad()
11578 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), NewValue); in CombineExtLoad()
11579 ExtendSetCCUses(SetCCs, N0, NewValue, (ISD::NodeType)N->getOpcode()); in CombineExtLoad()
11587 assert(N->getOpcode() == ISD::ZERO_EXTEND); in CombineZExtLogicopShiftLoad()
11595 if (!(N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR || in CombineZExtLogicopShiftLoad()
11596 N0.getOpcode() == ISD::XOR) || in CombineZExtLogicopShiftLoad()
11597 N0.getOperand(1).getOpcode() != ISD::Constant || in CombineZExtLogicopShiftLoad()
11603 if (!(N1.getOpcode() == ISD::SHL || N1.getOpcode() == ISD::SRL) || in CombineZExtLogicopShiftLoad()
11604 N1.getOperand(1).getOpcode() != ISD::Constant || in CombineZExtLogicopShiftLoad()
11613 if (!TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) || in CombineZExtLogicopShiftLoad()
11614 Load->getExtensionType() == ISD::SEXTLOAD || Load->isIndexed()) in CombineZExtLogicopShiftLoad()
11620 if (N1.getOpcode() == ISD::SHL && N0.getOpcode() != ISD::AND) in CombineZExtLogicopShiftLoad()
11628 ISD::ZERO_EXTEND, SetCCs, TLI)) in CombineZExtLogicopShiftLoad()
11632 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(Load), VT, in CombineZExtLogicopShiftLoad()
11645 ExtendSetCCUses(SetCCs, N1.getOperand(0), ExtLoad, ISD::ZERO_EXTEND); in CombineZExtLogicopShiftLoad()
11650 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(Load), in CombineZExtLogicopShiftLoad()
11667 assert((CastOpcode == ISD::SIGN_EXTEND || CastOpcode == ISD::ZERO_EXTEND || in matchVSelectOpSizesWithSetCC()
11668 CastOpcode == ISD::TRUNCATE || CastOpcode == ISD::FP_EXTEND || in matchVSelectOpSizesWithSetCC()
11669 CastOpcode == ISD::FP_ROUND) && in matchVSelectOpSizesWithSetCC()
11676 if (LegalOperations || !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT)) in matchVSelectOpSizesWithSetCC()
11680 if (VSel.getOpcode() != ISD::VSELECT || !VSel.hasOneUse() || in matchVSelectOpSizesWithSetCC()
11681 VSel.getOperand(0).getOpcode() != ISD::SETCC) in matchVSelectOpSizesWithSetCC()
11695 if (CastOpcode == ISD::FP_ROUND) { in matchVSelectOpSizesWithSetCC()
11703 return DAG.getNode(ISD::VSELECT, DL, VT, SetCC, CastA, CastB); in matchVSelectOpSizesWithSetCC()
11711 SDValue N0, ISD::LoadExtType ExtLoadType) { in tryToFoldExtOfExtload()
11713 bool isAExtLoad = (ExtLoadType == ISD::SEXTLOAD) ? ISD::isSEXTLoad(N0Node) in tryToFoldExtOfExtload()
11714 : ISD::isZEXTLoad(N0Node); in tryToFoldExtOfExtload()
11715 if ((!isAExtLoad && !ISD::isEXTLoad(N0Node)) || in tryToFoldExtOfExtload()
11716 !ISD::isUNINDEXEDLoad(N0Node) || !N0.hasOneUse()) in tryToFoldExtOfExtload()
11742 ISD::LoadExtType ExtLoadType, in tryToFoldExtOfLoad()
11743 ISD::NodeType ExtOpc) { in tryToFoldExtOfLoad()
11747 if (!ISD::isNON_EXTLoad(N0.getNode()) || in tryToFoldExtOfLoad()
11748 !ISD::isUNINDEXEDLoad(N0.getNode()) || in tryToFoldExtOfLoad()
11776 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad); in tryToFoldExtOfLoad()
11785 ISD::LoadExtType ExtLoadType, in tryToFoldExtOfMaskedLoad()
11786 ISD::NodeType ExtOpc) { in tryToFoldExtOfMaskedLoad()
11791 if (!Ld || Ld->getExtensionType() != ISD::NON_EXTLOAD) in tryToFoldExtOfMaskedLoad()
11812 assert((N->getOpcode() == ISD::SIGN_EXTEND || in foldExtendedSignBitTest()
11813 N->getOpcode() == ISD::ZERO_EXTEND) && "Expected sext or zext"); in foldExtendedSignBitTest()
11816 if (LegalOperations || SetCC.getOpcode() != ISD::SETCC || in foldExtendedSignBitTest()
11822 ISD::CondCode CC = cast<CondCodeSDNode>(SetCC.getOperand(2))->get(); in foldExtendedSignBitTest()
11828 if (CC == ISD::SETGT && isAllOnesConstant(Ones) && VT == XVT) { in foldExtendedSignBitTest()
11839 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SRA : ISD::SRL; in foldExtendedSignBitTest()
11848 if (N0.getOpcode() != ISD::SETCC) in foldSextSetcc()
11853 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in foldSextSetcc()
11890 if (N0.hasOneUse() && TLI.isOperationLegalOrCustom(ISD::SETCC, VT) && in foldSextSetcc()
11891 !TLI.isOperationLegalOrCustom(ISD::SETCC, SVT)) { in foldSextSetcc()
11892 bool IsSignedCmp = ISD::isSignedIntSetCC(CC); in foldSextSetcc()
11893 unsigned LoadOpcode = IsSignedCmp ? ISD::SEXTLOAD : ISD::ZEXTLOAD; in foldSextSetcc()
11894 unsigned ExtOpcode = IsSignedCmp ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in foldSextSetcc()
11905 if (!(ISD::isNON_EXTLoad(V.getNode()) && in foldSextSetcc()
11906 ISD::isUNINDEXEDLoad(V.getNode()) && in foldSextSetcc()
11962 (!LegalOperations || TLI.isOperationLegal(ISD::SETCC, N00VT))) { in foldSextSetcc()
11985 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) in visitSIGN_EXTEND()
11986 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N0.getOperand(0)); in visitSIGN_EXTEND()
11988 if (N0.getOpcode() == ISD::TRUNCATE) { in visitSIGN_EXTEND()
12018 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op); in visitSIGN_EXTEND()
12023 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op); in visitSIGN_EXTEND()
12027 if (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, in visitSIGN_EXTEND()
12030 Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N0), VT, Op); in visitSIGN_EXTEND()
12032 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, Op); in visitSIGN_EXTEND()
12033 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Op, in visitSIGN_EXTEND()
12041 ISD::SEXTLOAD, ISD::SIGN_EXTEND)) in visitSIGN_EXTEND()
12045 tryToFoldExtOfMaskedLoad(DAG, TLI, VT, N, N0, ISD::SEXTLOAD, in visitSIGN_EXTEND()
12046 ISD::SIGN_EXTEND)) in visitSIGN_EXTEND()
12056 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::SEXTLOAD)) in visitSIGN_EXTEND()
12061 if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR || in visitSIGN_EXTEND()
12062 N0.getOpcode() == ISD::XOR) && in visitSIGN_EXTEND()
12064 N0.getOperand(1).getOpcode() == ISD::Constant && in visitSIGN_EXTEND()
12068 if (TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, MemVT) && in visitSIGN_EXTEND()
12069 LN00->getExtensionType() != ISD::ZEXTLOAD && LN00->isUnindexed()) { in visitSIGN_EXTEND()
12072 ISD::SIGN_EXTEND, SetCCs, TLI); in visitSIGN_EXTEND()
12074 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN00), VT, in visitSIGN_EXTEND()
12081 ExtendSetCCUses(SetCCs, N0.getOperand(0), ExtLoad, ISD::SIGN_EXTEND); in visitSIGN_EXTEND()
12088 DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And); in visitSIGN_EXTEND()
12094 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00), in visitSIGN_EXTEND()
12110 if ((!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) && in visitSIGN_EXTEND()
12112 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0); in visitSIGN_EXTEND()
12119 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() && in visitSIGN_EXTEND()
12121 N0.getOperand(1).getOpcode() == ISD::ZERO_EXTEND && in visitSIGN_EXTEND()
12122 TLI.isOperationLegalOrCustom(ISD::SUB, VT)) { in visitSIGN_EXTEND()
12124 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Zext); in visitSIGN_EXTEND()
12128 if (N0.getOpcode() == ISD::ADD && N0.hasOneUse() && in visitSIGN_EXTEND()
12130 N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND && in visitSIGN_EXTEND()
12131 TLI.isOperationLegalOrCustom(ISD::ADD, VT)) { in visitSIGN_EXTEND()
12133 return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT)); in visitSIGN_EXTEND()
12139 (!LegalOperations || (TLI.isOperationLegal(ISD::ZERO_EXTEND, VT) && in visitSIGN_EXTEND()
12140 TLI.isOperationLegal(ISD::ADD, VT)))) { in visitSIGN_EXTEND()
12152 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NewXor); in visitSIGN_EXTEND()
12155 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitSIGN_EXTEND()
12156 return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT)); in visitSIGN_EXTEND()
12171 if (N->getOpcode() == ISD::TRUNCATE) { in isTruncateOf()
12177 if (N.getOpcode() != ISD::SETCC || in isTruncateOf()
12179 cast<CondCodeSDNode>(N.getOperand(2))->get() != ISD::SETNE) in isTruncateOf()
12202 assert((Extend->getOpcode() == ISD::ZERO_EXTEND || in widenCtPop()
12203 Extend->getOpcode() == ISD::ANY_EXTEND) && "Expected extend op"); in widenCtPop()
12206 if (CtPop.getOpcode() != ISD::CTPOP || !CtPop.hasOneUse()) in widenCtPop()
12211 if (TLI.isOperationLegalOrCustom(ISD::CTPOP, CtPop.getValueType()) || in widenCtPop()
12212 !TLI.isOperationLegalOrCustom(ISD::CTPOP, VT)) in widenCtPop()
12218 return DAG.getNode(ISD::CTPOP, DL, VT, NewZext); in widenCtPop()
12234 if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) in visitZERO_EXTEND()
12235 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, in visitZERO_EXTEND()
12256 if (N0.getOpcode() == ISD::TRUNCATE) { in visitZERO_EXTEND()
12275 if (!LegalOperations || (TLI.isOperationLegal(ISD::AND, SrcVT) && in visitZERO_EXTEND()
12276 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) { in visitZERO_EXTEND()
12287 if (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT)) { in visitZERO_EXTEND()
12300 if (N0.getOpcode() == ISD::AND && in visitZERO_EXTEND()
12301 N0.getOperand(0).getOpcode() == ISD::TRUNCATE && in visitZERO_EXTEND()
12302 N0.getOperand(1).getOpcode() == ISD::Constant && in visitZERO_EXTEND()
12310 return DAG.getNode(ISD::AND, DL, VT, in visitZERO_EXTEND()
12317 ISD::ZEXTLOAD, ISD::ZERO_EXTEND)) in visitZERO_EXTEND()
12321 tryToFoldExtOfMaskedLoad(DAG, TLI, VT, N, N0, ISD::ZEXTLOAD, in visitZERO_EXTEND()
12322 ISD::ZERO_EXTEND)) in visitZERO_EXTEND()
12334 if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR || in visitZERO_EXTEND()
12335 N0.getOpcode() == ISD::XOR) && in visitZERO_EXTEND()
12337 N0.getOperand(1).getOpcode() == ISD::Constant && in visitZERO_EXTEND()
12341 if (TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) && in visitZERO_EXTEND()
12342 LN00->getExtensionType() != ISD::SEXTLOAD && LN00->isUnindexed()) { in visitZERO_EXTEND()
12346 if (N0.getOpcode() == ISD::AND) { in visitZERO_EXTEND()
12356 ISD::ZERO_EXTEND, SetCCs, TLI); in visitZERO_EXTEND()
12358 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN00), VT, in visitZERO_EXTEND()
12366 ExtendSetCCUses(SetCCs, N0.getOperand(0), ExtLoad, ISD::ZERO_EXTEND); in visitZERO_EXTEND()
12373 DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And); in visitZERO_EXTEND()
12379 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00), in visitZERO_EXTEND()
12395 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::ZEXTLOAD)) in visitZERO_EXTEND()
12401 if (N0.getOpcode() == ISD::SETCC) { in visitZERO_EXTEND()
12420 SDValue VSetCC = DAG.getNode(ISD::SETCC, DL, VT, N0.getOperand(0), in visitZERO_EXTEND()
12430 DAG.getNode(ISD::SETCC, DL, MatchingVectorType, N0.getOperand(0), in visitZERO_EXTEND()
12445 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, SCC); in visitZERO_EXTEND()
12449 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) && in visitZERO_EXTEND()
12451 N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND && in visitZERO_EXTEND()
12454 if (N0.getOpcode() == ISD::SHL) { in visitZERO_EXTEND()
12468 ShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShAmt); in visitZERO_EXTEND()
12471 DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)), in visitZERO_EXTEND()
12501 if (N0.getOpcode() == ISD::ANY_EXTEND || in visitANY_EXTEND()
12502 N0.getOpcode() == ISD::ZERO_EXTEND || in visitANY_EXTEND()
12503 N0.getOpcode() == ISD::SIGN_EXTEND) in visitANY_EXTEND()
12508 if (N0.getOpcode() == ISD::TRUNCATE) { in visitANY_EXTEND()
12521 if (N0.getOpcode() == ISD::TRUNCATE) in visitANY_EXTEND()
12526 if (N0.getOpcode() == ISD::AND && in visitANY_EXTEND()
12527 N0.getOperand(0).getOpcode() == ISD::TRUNCATE && in visitANY_EXTEND()
12528 N0.getOperand(1).getOpcode() == ISD::Constant && in visitANY_EXTEND()
12533 SDValue Y = DAG.getNode(ISD::ANY_EXTEND, DL, VT, N0.getOperand(1)); in visitANY_EXTEND()
12535 return DAG.getNode(ISD::AND, DL, VT, X, Y); in visitANY_EXTEND()
12545 ISD::ZEXTLOAD, ISD::ZERO_EXTEND)) in visitANY_EXTEND()
12547 } else if (ISD::isNON_EXTLoad(N0.getNode()) && in visitANY_EXTEND()
12548 ISD::isUNINDEXEDLoad(N0.getNode()) && in visitANY_EXTEND()
12549 TLI.isLoadExtLegal(ISD::EXTLOAD, VT, N0.getValueType())) { in visitANY_EXTEND()
12554 ExtendUsesToFormExtLoad(VT, N, N0, ISD::ANY_EXTEND, SetCCs, TLI); in visitANY_EXTEND()
12557 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, in visitANY_EXTEND()
12560 ExtendSetCCUses(SetCCs, N0, ExtLoad, ISD::ANY_EXTEND); in visitANY_EXTEND()
12569 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad); in visitANY_EXTEND()
12579 if (N0.getOpcode() == ISD::LOAD && !ISD::isNON_EXTLoad(N0.getNode()) && in visitANY_EXTEND()
12580 ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) { in visitANY_EXTEND()
12582 ISD::LoadExtType ExtType = LN0->getExtensionType(); in visitANY_EXTEND()
12595 if (N0.getOpcode() == ISD::SETCC) { in visitANY_EXTEND()
12659 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && in visitAssertExt()
12673 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewAssert); in visitAssertExt()
12679 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && in visitAssertExt()
12680 N0.getOperand(0).getOpcode() == ISD::AssertSext && in visitAssertExt()
12681 Opcode == ISD::AssertZext) { in visitAssertExt()
12688 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewAssert); in visitAssertExt()
12713 case ISD::ADD: in visitAssertAlign()
12714 case ISD::SUB: { in visitAssertAlign()
12740 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD; in reduceLoadWidth()
12760 if (Opc == ISD::SIGN_EXTEND_INREG) { in reduceLoadWidth()
12761 ExtType = ISD::SEXTLOAD; in reduceLoadWidth()
12763 } else if (Opc == ISD::SRL || Opc == ISD::SRA) { in reduceLoadWidth()
12781 ExtType = Opc == ISD::SRL ? ISD::ZEXTLOAD : ISD::SEXTLOAD; in reduceLoadWidth()
12787 if ((LN->getExtensionType() == ISD::SEXTLOAD || in reduceLoadWidth()
12788 LN->getExtensionType() == ISD::ZEXTLOAD) && in reduceLoadWidth()
12791 } else if (Opc == ISD::AND) { in reduceLoadWidth()
12807 ExtType = ISD::ZEXTLOAD; in reduceLoadWidth()
12816 if (Opc == ISD::SRL || N0.getOpcode() == ISD::SRL) { in reduceLoadWidth()
12817 SDValue SRL = Opc == ISD::SRL ? SDValue(N, 0) : N0; in reduceLoadWidth()
12844 if (LN->getExtensionType() == ISD::SEXTLOAD) in reduceLoadWidth()
12854 if (ExtType == ISD::SEXTLOAD) in reduceLoadWidth()
12857 ExtType = ISD::ZEXTLOAD; in reduceLoadWidth()
12864 if (SRL.hasOneUse() && Mask->getOpcode() == ISD::AND && in reduceLoadWidth()
12886 if (ShAmt == 0 && N0.getOpcode() == ISD::SHL && N0.hasOneUse() && in reduceLoadWidth()
12928 if (ExtType == ISD::NON_EXTLOAD) in reduceLoadWidth()
12955 Result = DAG.getNode(ISD::SHL, DL, VT, in reduceLoadWidth()
12965 Result = DAG.getNode(ISD::SHL, DL, VT, Result, ShiftC); in reduceLoadWidth()
12987 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0, N1); in visitSIGN_EXTEND_INREG()
12994 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG && in visitSIGN_EXTEND_INREG()
12996 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0.getOperand(0), in visitSIGN_EXTEND_INREG()
13003 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) { in visitSIGN_EXTEND_INREG()
13008 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT))) in visitSIGN_EXTEND_INREG()
13009 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00); in visitSIGN_EXTEND_INREG()
13015 if (N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG || in visitSIGN_EXTEND_INREG()
13016 N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG || in visitSIGN_EXTEND_INREG()
13017 N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) { in visitSIGN_EXTEND_INREG()
13022 bool IsZext = N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG; in visitSIGN_EXTEND_INREG()
13028 TLI.isOperationLegal(ISD::SIGN_EXTEND_VECTOR_INREG, VT))) in visitSIGN_EXTEND_INREG()
13029 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, SDLoc(N), VT, N00); in visitSIGN_EXTEND_INREG()
13034 if (N0.getOpcode() == ISD::ZERO_EXTEND) { in visitSIGN_EXTEND_INREG()
13037 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT))) in visitSIGN_EXTEND_INREG()
13038 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00, N1); in visitSIGN_EXTEND_INREG()
13058 if (N0.getOpcode() == ISD::SRL) { in visitSIGN_EXTEND_INREG()
13065 return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0.getOperand(0), in visitSIGN_EXTEND_INREG()
13074 if (ISD::isEXTLoad(N0.getNode()) && in visitSIGN_EXTEND_INREG()
13075 ISD::isUNINDEXEDLoad(N0.getNode()) && in visitSIGN_EXTEND_INREG()
13079 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT))) { in visitSIGN_EXTEND_INREG()
13081 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
13092 if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && in visitSIGN_EXTEND_INREG()
13096 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT))) { in visitSIGN_EXTEND_INREG()
13098 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
13111 Ld->getExtensionType() != ISD::LoadExtType::NON_EXTLOAD && in visitSIGN_EXTEND_INREG()
13112 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT)) { in visitSIGN_EXTEND_INREG()
13116 Ld->getAddressingMode(), ISD::SEXTLOAD, Ld->isExpandingLoad()); in visitSIGN_EXTEND_INREG()
13133 GN0->getMemOperand(), GN0->getIndexType(), ISD::SEXTLOAD); in visitSIGN_EXTEND_INREG()
13143 if (ExtVTBits <= 16 && N0.getOpcode() == ISD::OR) { in visitSIGN_EXTEND_INREG()
13146 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, BSwap, N1); in visitSIGN_EXTEND_INREG()
13180 if (N0.getOpcode() == ISD::TRUNCATE) in visitTRUNCATE()
13181 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0)); in visitTRUNCATE()
13185 SDValue C = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0); in visitTRUNCATE()
13191 if (N0.getOpcode() == ISD::ZERO_EXTEND || in visitTRUNCATE()
13192 N0.getOpcode() == ISD::SIGN_EXTEND || in visitTRUNCATE()
13193 N0.getOpcode() == ISD::ANY_EXTEND) { in visitTRUNCATE()
13199 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0)); in visitTRUNCATE()
13207 if (!LegalTypes && N0.getOpcode() == ISD::SIGN_EXTEND_INREG && in visitTRUNCATE()
13213 SDValue TrX = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, X); in visitTRUNCATE()
13214 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, TrX, ExtVal); in visitTRUNCATE()
13219 if (N->hasOneUse() && (N->use_begin()->getOpcode() == ISD::ANY_EXTEND)) in visitTRUNCATE()
13232 if (N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitTRUNCATE()
13251 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TrTy, in visitTRUNCATE()
13258 if (N0.getOpcode() == ISD::SELECT && N0.hasOneUse()) { in visitTRUNCATE()
13259 if ((!LegalOperations || TLI.isOperationLegal(ISD::SELECT, SrcVT)) && in visitTRUNCATE()
13263 SDValue TruncOp0 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1)); in visitTRUNCATE()
13264 SDValue TruncOp1 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(2)); in visitTRUNCATE()
13265 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, Cond, TruncOp0, TruncOp1); in visitTRUNCATE()
13270 if (N0.getOpcode() == ISD::SHL && N0.hasOneUse() && in visitTRUNCATE()
13271 (!LegalOperations || TLI.isOperationLegal(ISD::SHL, VT)) && in visitTRUNCATE()
13272 TLI.isTypeDesirableForOp(ISD::SHL, VT)) { in visitTRUNCATE()
13280 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0)); in visitTRUNCATE()
13285 return DAG.getNode(ISD::SHL, SL, VT, Trunc, Amt); in visitTRUNCATE()
13293 if (N0.getOpcode() == ISD::BUILD_VECTOR && !LegalOperations && in visitTRUNCATE()
13301 SDValue TruncOp = DAG.getNode(ISD::TRUNCATE, DL, SVT, Op); in visitTRUNCATE()
13312 N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() && in visitTRUNCATE()
13313 N0.getOperand(0).getOpcode() == ISD::BUILD_VECTOR && in visitTRUNCATE()
13345 if (N0.hasOneUse() && ISD::isUNINDEXEDLoad(N0.getNode())) { in visitTRUNCATE()
13360 if (N0.getOpcode() == ISD::CONCAT_VECTORS && !LegalTypes) { in visitTRUNCATE()
13393 SDValue NV = DAG.getNode(ISD::TRUNCATE, SDLoc(V), VTs[i], V); in visitTRUNCATE()
13397 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Opnds); in visitTRUNCATE()
13405 if (N0.getOpcode() == ISD::BITCAST && !VT.isVector()) { in visitTRUNCATE()
13410 TLI.isOperationLegal(ISD::EXTRACT_VECTOR_ELT, VecSrcVT))) { in visitTRUNCATE()
13414 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, VT, VecSrc, in visitTRUNCATE()
13432 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Shorter); in visitTRUNCATE()
13439 if (!LegalTypes && N0.getOpcode() == ISD::EXTRACT_SUBVECTOR) { in visitTRUNCATE()
13441 if (N00.getOpcode() == ISD::SIGN_EXTEND || in visitTRUNCATE()
13442 N00.getOpcode() == ISD::ZERO_EXTEND || in visitTRUNCATE()
13443 N00.getOpcode() == ISD::ANY_EXTEND) { in visitTRUNCATE()
13446 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N0->getOperand(0)), VT, in visitTRUNCATE()
13459 case ISD::ADD: in visitTRUNCATE()
13460 case ISD::SUB: in visitTRUNCATE()
13461 case ISD::MUL: in visitTRUNCATE()
13462 case ISD::AND: in visitTRUNCATE()
13463 case ISD::OR: in visitTRUNCATE()
13464 case ISD::XOR: in visitTRUNCATE()
13473 SDValue NarrowL = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
13474 SDValue NarrowR = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1)); in visitTRUNCATE()
13479 case ISD::ADDE: in visitTRUNCATE()
13480 case ISD::ADDCARRY: in visitTRUNCATE()
13485 if (((!LegalOperations && N0.getOpcode() == ISD::ADDCARRY) || in visitTRUNCATE()
13489 SDValue X = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
13490 SDValue Y = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1)); in visitTRUNCATE()
13495 case ISD::USUBSAT: in visitTRUNCATE()
13500 N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND && in visitTRUNCATE()
13515 if (Elt.getOpcode() != ISD::MERGE_VALUES) in getBuildPairElt()
13523 assert(N->getOpcode() == ISD::BUILD_PAIR); in CombineConsecutiveLoads()
13534 if (!LD1 || !LD2 || !ISD::isNON_EXTLoad(LD1) || !ISD::isNON_EXTLoad(LD2) || in CombineConsecutiveLoads()
13542 if ((!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)) && in CombineConsecutiveLoads()
13576 case ISD::AND: in foldBitcastedFPLogic()
13577 FPOpcode = ISD::FABS; in foldBitcastedFPLogic()
13580 case ISD::XOR: in foldBitcastedFPLogic()
13581 FPOpcode = ISD::FNEG; in foldBitcastedFPLogic()
13584 case ISD::OR: in foldBitcastedFPLogic()
13585 FPOpcode = ISD::FABS; in foldBitcastedFPLogic()
13599 LogicOp0.getOpcode() == ISD::BITCAST && in foldBitcastedFPLogic()
13603 if (N0.getOpcode() == ISD::OR) in foldBitcastedFPLogic()
13604 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, FPOp); in foldBitcastedFPLogic()
13628 N0.getOpcode() == ISD::BUILD_VECTOR && N0->hasOneUse() && in visitBITCAST()
13640 TLI.isOperationLegal(ISD::ConstantFP, VT)) || in visitBITCAST()
13642 TLI.isOperationLegal(ISD::Constant, VT))) { in visitBITCAST()
13650 if (N0.getOpcode() == ISD::BITCAST) in visitBITCAST()
13655 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && in visitBITCAST()
13665 TLI.isOperationLegal(ISD::LOAD, VT))) { in visitBITCAST()
13694 if (((N0.getOpcode() == ISD::FNEG && !TLI.isFNegFree(N0.getValueType())) || in visitBITCAST()
13695 (N0.getOpcode() == ISD::FABS && !TLI.isFAbsFree(N0.getValueType()))) && in visitBITCAST()
13707 if (N0.getOpcode() == ISD::FNEG) { in visitBITCAST()
13711 assert(N0.getOpcode() == ISD::FABS); in visitBITCAST()
13713 DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(NewConv), MVT::i64, NewConv, in visitBITCAST()
13717 FlipBit = DAG.getNode(ISD::AND, SDLoc(N0), MVT::i64, Hi, SignBit); in visitBITCAST()
13721 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit); in visitBITCAST()
13723 return DAG.getNode(ISD::XOR, DL, VT, NewConv, FlipBits); in visitBITCAST()
13726 if (N0.getOpcode() == ISD::FNEG) in visitBITCAST()
13727 return DAG.getNode(ISD::XOR, DL, VT, in visitBITCAST()
13729 assert(N0.getOpcode() == ISD::FABS); in visitBITCAST()
13730 return DAG.getNode(ISD::AND, DL, VT, in visitBITCAST()
13745 if (N0.getOpcode() == ISD::FCOPYSIGN && N0->hasOneUse() && in visitBITCAST()
13757 X = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, X); in visitBITCAST()
13763 X = DAG.getNode(ISD::SRL, DL, in visitBITCAST()
13768 X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X); in visitBITCAST()
13778 SDValue XorResult = DAG.getNode(ISD::XOR, SDLoc(N0), VT, Cst, X); in visitBITCAST()
13781 ISD::EXTRACT_ELEMENT, SDLoc(XorResult), MVT::i64, XorResult, in visitBITCAST()
13786 DAG.getNode(ISD::AND, SDLoc(XorResult64), MVT::i64, XorResult64, in visitBITCAST()
13790 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit); in visitBITCAST()
13792 return DAG.getNode(ISD::XOR, SDLoc(N), VT, Cst, FlipBits); in visitBITCAST()
13795 X = DAG.getNode(ISD::AND, SDLoc(X), VT, in visitBITCAST()
13800 Cst = DAG.getNode(ISD::AND, SDLoc(Cst), VT, in visitBITCAST()
13804 return DAG.getNode(ISD::OR, SDLoc(N), VT, X, Cst); in visitBITCAST()
13809 if (N0.getOpcode() == ISD::BUILD_PAIR) in visitBITCAST()
13818 N0->getOpcode() == ISD::VECTOR_SHUFFLE && N0.hasOneUse() && in visitBITCAST()
13826 if (Op.getOpcode() == ISD::BITCAST && in visitBITCAST()
13871 if (N0.getOpcode() == ISD::BITCAST) in visitFREEZE()
13873 DAG.getNode(ISD::FREEZE, SDLoc(N0), in visitFREEZE()
13900 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(BV), SrcEltVT, Op); in ConstantFoldBITCASTofBUILD_VECTOR()
13960 assert(N.getOpcode() == ISD::FMUL); in isContractableFMUL()
13986 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFADDForFMACombine()
14004 unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA; in visitFADDForFMACombine()
14009 return Opcode == ISD::FMA || Opcode == ISD::FMAD; in visitFADDForFMACombine()
14015 if (N.getOpcode() != ISD::FMUL) in visitFADDForFMACombine()
14044 N0.getOperand(2).getOpcode() == ISD::FMUL && N0.hasOneUse() && in visitFADDForFMACombine()
14049 N1.getOperand(2).getOpcode() == ISD::FMUL && N1.hasOneUse() && in visitFADDForFMACombine()
14066 if (N0.getOpcode() == ISD::FP_EXTEND) { in visitFADDForFMACombine()
14072 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)), in visitFADDForFMACombine()
14073 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)), in visitFADDForFMACombine()
14080 if (N1.getOpcode() == ISD::FP_EXTEND) { in visitFADDForFMACombine()
14086 DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(0)), in visitFADDForFMACombine()
14087 DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)), in visitFADDForFMACombine()
14100 DAG.getNode(ISD::FP_EXTEND, SL, VT, U), in visitFADDForFMACombine()
14101 DAG.getNode(ISD::FP_EXTEND, SL, VT, V), in visitFADDForFMACombine()
14106 if (N02.getOpcode() == ISD::FP_EXTEND) { in visitFADDForFMACombine()
14126 PreferredFusedOpcode, SL, VT, DAG.getNode(ISD::FP_EXTEND, SL, VT, X), in visitFADDForFMACombine()
14127 DAG.getNode(ISD::FP_EXTEND, SL, VT, Y), in visitFADDForFMACombine()
14129 DAG.getNode(ISD::FP_EXTEND, SL, VT, U), in visitFADDForFMACombine()
14130 DAG.getNode(ISD::FP_EXTEND, SL, VT, V), Z)); in visitFADDForFMACombine()
14132 if (N0.getOpcode() == ISD::FP_EXTEND) { in visitFADDForFMACombine()
14150 if (N12.getOpcode() == ISD::FP_EXTEND) { in visitFADDForFMACombine()
14167 if (N1.getOpcode() == ISD::FP_EXTEND) { in visitFADDForFMACombine()
14199 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFSUBForFMACombine()
14217 unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA; in visitFSUBForFMACombine()
14224 if (N.getOpcode() != ISD::FMUL) in visitFSUBForFMACombine()
14233 XY.getOperand(1), DAG.getNode(ISD::FNEG, SL, VT, Z)); in visitFSUBForFMACombine()
14243 DAG.getNode(ISD::FNEG, SL, VT, YZ.getOperand(0)), in visitFSUBForFMACombine()
14269 if (N0.getOpcode() == ISD::FNEG && isContractableFMUL(N0.getOperand(0)) && in visitFSUBForFMACombine()
14274 DAG.getNode(ISD::FNEG, SL, VT, N00), N01, in visitFSUBForFMACombine()
14275 DAG.getNode(ISD::FNEG, SL, VT, N1)); in visitFSUBForFMACombine()
14282 if (N0.getOpcode() == ISD::FP_EXTEND) { in visitFSUBForFMACombine()
14288 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)), in visitFSUBForFMACombine()
14289 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)), in visitFSUBForFMACombine()
14290 DAG.getNode(ISD::FNEG, SL, VT, N1)); in visitFSUBForFMACombine()
14297 if (N1.getOpcode() == ISD::FP_EXTEND) { in visitFSUBForFMACombine()
14304 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
14305 DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(0))), in visitFSUBForFMACombine()
14306 DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)), N0); in visitFSUBForFMACombine()
14316 if (N0.getOpcode() == ISD::FP_EXTEND) { in visitFSUBForFMACombine()
14318 if (N00.getOpcode() == ISD::FNEG) { in visitFSUBForFMACombine()
14324 ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
14326 DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)), in visitFSUBForFMACombine()
14327 DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)), in visitFSUBForFMACombine()
14339 if (N0.getOpcode() == ISD::FNEG) { in visitFSUBForFMACombine()
14341 if (N00.getOpcode() == ISD::FP_EXTEND) { in visitFSUBForFMACombine()
14347 ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
14349 DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)), in visitFSUBForFMACombine()
14350 DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)), in visitFSUBForFMACombine()
14367 return Opcode == ISD::FMA || Opcode == ISD::FMAD; in visitFSUBForFMACombine()
14383 DAG.getNode(ISD::FNEG, SL, VT, N1))); in visitFSUBForFMACombine()
14395 DAG.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)), N1.getOperand(1), in visitFSUBForFMACombine()
14397 DAG.getNode(ISD::FNEG, SL, VT, N20), N21, N0)); in visitFSUBForFMACombine()
14404 if (N02.getOpcode() == ISD::FP_EXTEND) { in visitFSUBForFMACombine()
14413 DAG.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(0)), in visitFSUBForFMACombine()
14414 DAG.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(1)), in visitFSUBForFMACombine()
14415 DAG.getNode(ISD::FNEG, SL, VT, N1))); in visitFSUBForFMACombine()
14426 if (N0.getOpcode() == ISD::FP_EXTEND) { in visitFSUBForFMACombine()
14435 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)), in visitFSUBForFMACombine()
14436 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)), in visitFSUBForFMACombine()
14439 DAG.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(0)), in visitFSUBForFMACombine()
14440 DAG.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(1)), in visitFSUBForFMACombine()
14441 DAG.getNode(ISD::FNEG, SL, VT, N1))); in visitFSUBForFMACombine()
14448 if (isFusedOp(N1) && N1.getOperand(2).getOpcode() == ISD::FP_EXTEND && in visitFSUBForFMACombine()
14458 DAG.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)), N1.getOperand(1), in visitFSUBForFMACombine()
14460 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
14461 DAG.getNode(ISD::FP_EXTEND, SL, VT, N1200)), in visitFSUBForFMACombine()
14462 DAG.getNode(ISD::FP_EXTEND, SL, VT, N1201), N0)); in visitFSUBForFMACombine()
14472 if (N1.getOpcode() == ISD::FP_EXTEND && isFusedOp(N1.getOperand(0))) { in visitFSUBForFMACombine()
14484 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
14485 DAG.getNode(ISD::FP_EXTEND, SL, VT, N100)), in visitFSUBForFMACombine()
14486 DAG.getNode(ISD::FP_EXTEND, SL, VT, N101), in visitFSUBForFMACombine()
14488 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
14489 DAG.getNode(ISD::FP_EXTEND, SL, VT, N1020)), in visitFSUBForFMACombine()
14490 DAG.getNode(ISD::FP_EXTEND, SL, VT, N1021), N0)); in visitFSUBForFMACombine()
14507 assert(N->getOpcode() == ISD::FMUL && "Expected FMUL Operation"); in visitFMULForFMADistributiveCombine()
14513 SDValue FAdd = N0.getOpcode() == ISD::FADD ? N0 : N1; in visitFMULForFMADistributiveCombine()
14521 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFMULForFMADistributiveCombine()
14533 unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA; in visitFMULForFMADistributiveCombine()
14539 if (X.getOpcode() == ISD::FADD && (Aggressive || X->hasOneUse())) { in visitFMULForFMADistributiveCombine()
14546 DAG.getNode(ISD::FNEG, SL, VT, Y)); in visitFMULForFMADistributiveCombine()
14562 if (X.getOpcode() == ISD::FSUB && (Aggressive || X->hasOneUse())) { in visitFMULForFMADistributiveCombine()
14566 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y, in visitFMULForFMADistributiveCombine()
14570 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y, in visitFMULForFMADistributiveCombine()
14571 DAG.getNode(ISD::FNEG, SL, VT, Y)); in visitFMULForFMADistributiveCombine()
14576 DAG.getNode(ISD::FNEG, SL, VT, Y)); in visitFMULForFMADistributiveCombine()
14608 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FADD, DL, VT, {N0, N1})) in visitFADD()
14613 return DAG.getNode(ISD::FADD, DL, VT, N1, N0); in visitFADD()
14630 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) in visitFADD()
14633 return DAG.getNode(ISD::FSUB, DL, VT, N0, NegN1); in visitFADD()
14636 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) in visitFADD()
14639 return DAG.getNode(ISD::FSUB, DL, VT, N1, NegN0); in visitFADD()
14642 if (!FMul.hasOneUse() || FMul.getOpcode() != ISD::FMUL) in visitFADD()
14651 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B); in visitFADD()
14652 return DAG.getNode(ISD::FSUB, DL, VT, N1, Add); in visitFADD()
14657 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B); in visitFADD()
14658 return DAG.getNode(ISD::FSUB, DL, VT, N0, Add); in visitFADD()
14668 if (N0.getOpcode() == ISD::FNEG && N0.getOperand(0) == N1) in visitFADD()
14672 if (N1.getOpcode() == ISD::FNEG && N1.getOperand(0) == N0) in visitFADD()
14683 if (N1CFP && N0.getOpcode() == ISD::FADD && in visitFADD()
14685 SDValue NewC = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), N1); in visitFADD()
14686 return DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(0), NewC); in visitFADD()
14692 if (TLI.isOperationLegalOrCustom(ISD::FMUL, VT) && !N0CFP && !N1CFP) { in visitFADD()
14693 if (N0.getOpcode() == ISD::FMUL) { in visitFADD()
14699 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), in visitFADD()
14701 return DAG.getNode(ISD::FMUL, DL, VT, N1, NewCFP); in visitFADD()
14705 if (CFP01 && !CFP00 && N1.getOpcode() == ISD::FADD && in visitFADD()
14708 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), in visitFADD()
14710 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), NewCFP); in visitFADD()
14714 if (N1.getOpcode() == ISD::FMUL) { in visitFADD()
14720 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1), in visitFADD()
14722 return DAG.getNode(ISD::FMUL, DL, VT, N0, NewCFP); in visitFADD()
14726 if (CFP11 && !CFP10 && N0.getOpcode() == ISD::FADD && in visitFADD()
14729 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1), in visitFADD()
14731 return DAG.getNode(ISD::FMUL, DL, VT, N1.getOperand(0), NewCFP); in visitFADD()
14735 if (N0.getOpcode() == ISD::FADD) { in visitFADD()
14740 return DAG.getNode(ISD::FMUL, DL, VT, N1, in visitFADD()
14745 if (N1.getOpcode() == ISD::FADD) { in visitFADD()
14750 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFADD()
14756 if (N0.getOpcode() == ISD::FADD && N1.getOpcode() == ISD::FADD && in visitFADD()
14760 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), in visitFADD()
14784 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::STRICT_FSUB, VT)) in visitSTRICT_FADD()
14787 return DAG.getNode(ISD::STRICT_FSUB, DL, DAG.getVTList(VT, ChainVT), in visitSTRICT_FADD()
14792 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::STRICT_FSUB, VT)) in visitSTRICT_FADD()
14795 return DAG.getNode(ISD::STRICT_FSUB, DL, DAG.getVTList(VT, ChainVT), in visitSTRICT_FADD()
14816 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FSUB, DL, VT, {N0, N1})) in visitFSUB()
14854 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) in visitFSUB()
14855 return DAG.getNode(ISD::FNEG, DL, VT, N1); in visitFSUB()
14862 N1.getOpcode() == ISD::FADD) { in visitFSUB()
14865 return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(1)); in visitFSUB()
14868 return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(0)); in visitFSUB()
14874 return DAG.getNode(ISD::FADD, DL, VT, N0, NegN1); in visitFSUB()
14899 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FMUL, DL, VT, {N0, N1})) in visitFMUL()
14905 return DAG.getNode(ISD::FMUL, DL, VT, N1, N0); in visitFMUL()
14918 N0.getOpcode() == ISD::FMUL) { in visitFMUL()
14925 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, N01, N1); in visitFMUL()
14926 return DAG.getNode(ISD::FMUL, DL, VT, N00, MulConsts); in visitFMUL()
14932 if (N0.getOpcode() == ISD::FADD && N0.hasOneUse() && in visitFMUL()
14935 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, Two, N1); in visitFMUL()
14936 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), MulConsts); in visitFMUL()
14942 return DAG.getNode(ISD::FADD, DL, VT, N0, N0); in visitFMUL()
14946 if (!LegalOperations || TLI.isOperationLegal(ISD::FSUB, VT)) { in visitFMUL()
14947 return DAG.getNode(ISD::FSUB, DL, VT, in visitFMUL()
14964 return DAG.getNode(ISD::FMUL, DL, VT, NegN0, NegN1); in visitFMUL()
14969 (N0.getOpcode() == ISD::SELECT || N1.getOpcode() == ISD::SELECT) && in visitFMUL()
14970 TLI.isOperationLegal(ISD::FABS, VT)) { in visitFMUL()
14972 if (Select.getOpcode() != ISD::SELECT) in visitFMUL()
14980 Cond.getOpcode() == ISD::SETCC && Cond.getOperand(0) == X && in visitFMUL()
14983 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in visitFMUL()
14986 case ISD::SETOLT: in visitFMUL()
14987 case ISD::SETULT: in visitFMUL()
14988 case ISD::SETOLE: in visitFMUL()
14989 case ISD::SETULE: in visitFMUL()
14990 case ISD::SETLT: in visitFMUL()
14991 case ISD::SETLE: in visitFMUL()
14994 case ISD::SETOGT: in visitFMUL()
14995 case ISD::SETUGT: in visitFMUL()
14996 case ISD::SETOGE: in visitFMUL()
14997 case ISD::SETUGE: in visitFMUL()
14998 case ISD::SETGT: in visitFMUL()
14999 case ISD::SETGE: in visitFMUL()
15001 TLI.isOperationLegal(ISD::FNEG, VT)) in visitFMUL()
15002 return DAG.getNode(ISD::FNEG, DL, VT, in visitFMUL()
15003 DAG.getNode(ISD::FABS, DL, VT, X)); in visitFMUL()
15005 return DAG.getNode(ISD::FABS, DL, VT, X); in visitFMUL()
15040 return DAG.getNode(ISD::FMA, DL, VT, N0, N1, N2); in visitFMA()
15055 return DAG.getNode(ISD::FMA, DL, VT, NegN0, NegN1, N2); in visitFMA()
15066 return DAG.getNode(ISD::FADD, SDLoc(N), VT, N1, N2); in visitFMA()
15068 return DAG.getNode(ISD::FADD, SDLoc(N), VT, N0, N2); in visitFMA()
15073 return DAG.getNode(ISD::FMA, SDLoc(N), VT, N1, N0, N2); in visitFMA()
15077 if (N2.getOpcode() == ISD::FMUL && N0 == N2.getOperand(0) && in visitFMA()
15080 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFMA()
15081 DAG.getNode(ISD::FADD, DL, VT, N1, N2.getOperand(1))); in visitFMA()
15085 if (N0.getOpcode() == ISD::FMUL && in visitFMA()
15088 return DAG.getNode(ISD::FMA, DL, VT, N0.getOperand(0), in visitFMA()
15089 DAG.getNode(ISD::FMUL, DL, VT, N1, N0.getOperand(1)), in visitFMA()
15097 return DAG.getNode(ISD::FADD, DL, VT, N0, N2); in visitFMA()
15100 (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))) { in visitFMA()
15101 SDValue RHSNeg = DAG.getNode(ISD::FNEG, DL, VT, N0); in visitFMA()
15103 return DAG.getNode(ISD::FADD, DL, VT, N2, RHSNeg); in visitFMA()
15107 if (N0.getOpcode() == ISD::FNEG && in visitFMA()
15108 (TLI.isOperationLegal(ISD::ConstantFP, VT) || in visitFMA()
15111 return DAG.getNode(ISD::FMA, DL, VT, N0.getOperand(0), in visitFMA()
15112 DAG.getNode(ISD::FNEG, DL, VT, N1), N2); in visitFMA()
15120 ISD::FMUL, DL, VT, N0, in visitFMA()
15121 DAG.getNode(ISD::FADD, DL, VT, N1, DAG.getConstantFP(1.0, DL, VT))); in visitFMA()
15125 if (N1CFP && N2.getOpcode() == ISD::FNEG && N2.getOperand(0) == N0) { in visitFMA()
15127 ISD::FMUL, DL, VT, N0, in visitFMA()
15128 DAG.getNode(ISD::FADD, DL, VT, N1, DAG.getConstantFP(-1.0, DL, VT))); in visitFMA()
15137 return DAG.getNode(ISD::FNEG, DL, VT, Neg); in visitFMA()
15181 if (U->getOpcode() == ISD::FDIV && U->getOperand(1) == N1) { in combineRepeatedFPDivisors()
15183 if (U->getOperand(1).getOpcode() == ISD::FSQRT && in combineRepeatedFPDivisors()
15203 SDValue Reciprocal = DAG.getNode(ISD::FDIV, DL, VT, FPOne, N1, Flags); in combineRepeatedFPDivisors()
15209 SDValue NewNode = DAG.getNode(ISD::FMUL, SDLoc(U), VT, Dividend, in combineRepeatedFPDivisors()
15234 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FDIV, DL, VT, {N0, N1})) in visitFDIV()
15262 TLI.isOperationLegal(ISD::ConstantFP, VT) || in visitFDIV()
15264 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFDIV()
15270 if (N1.getOpcode() == ISD::FSQRT) { in visitFDIV()
15272 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
15273 } else if (N1.getOpcode() == ISD::FP_EXTEND && in visitFDIV()
15274 N1.getOperand(0).getOpcode() == ISD::FSQRT) { in visitFDIV()
15277 RV = DAG.getNode(ISD::FP_EXTEND, SDLoc(N1), VT, RV); in visitFDIV()
15279 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
15281 } else if (N1.getOpcode() == ISD::FP_ROUND && in visitFDIV()
15282 N1.getOperand(0).getOpcode() == ISD::FSQRT) { in visitFDIV()
15285 RV = DAG.getNode(ISD::FP_ROUND, SDLoc(N1), VT, RV, N1.getOperand(1)); in visitFDIV()
15287 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
15289 } else if (N1.getOpcode() == ISD::FMUL) { in visitFDIV()
15293 if (N1.getOperand(0).getOpcode() == ISD::FSQRT) { in visitFDIV()
15296 } else if (N1.getOperand(1).getOpcode() == ISD::FSQRT) { in visitFDIV()
15306 if (Y.getOpcode() == ISD::FABS && Y.hasOneUse()) in visitFDIV()
15313 SDValue AA = DAG.getNode(ISD::FMUL, DL, VT, A, A); in visitFDIV()
15315 DAG.getNode(ISD::FMUL, DL, VT, AA, Sqrt.getOperand(0)); in visitFDIV()
15317 return DAG.getNode(ISD::FMUL, DL, VT, N0, Rsqrt); in visitFDIV()
15327 SDValue Div = DAG.getNode(ISD::FDIV, SDLoc(N1), VT, Rsqrt, Y); in visitFDIV()
15329 return DAG.getNode(ISD::FMUL, DL, VT, N0, Div); in visitFDIV()
15343 if (N1.getOpcode() == ISD::FSQRT && N0 == N1.getOperand(0)) in visitFDIV()
15358 return DAG.getNode(ISD::FDIV, SDLoc(N), VT, NegN0, NegN1); in visitFDIV()
15374 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FREM, SDLoc(N), VT, {N0, N1})) in visitFREM()
15409 if ((N1.getOpcode() == ISD::FP_EXTEND || in CanCombineFCOPYSIGN_EXTEND_ROUND()
15410 N1.getOpcode() == ISD::FP_ROUND)) { in CanCombineFCOPYSIGN_EXTEND_ROUND()
15441 DAG.FoldConstantArithmetic(ISD::FCOPYSIGN, SDLoc(N), VT, {N0, N1})) in visitFCOPYSIGN()
15449 if (!LegalOperations || TLI.isOperationLegal(ISD::FABS, VT)) in visitFCOPYSIGN()
15450 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFCOPYSIGN()
15452 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) in visitFCOPYSIGN()
15453 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, in visitFCOPYSIGN()
15454 DAG.getNode(ISD::FABS, SDLoc(N0), VT, N0)); in visitFCOPYSIGN()
15461 if (N0.getOpcode() == ISD::FABS || N0.getOpcode() == ISD::FNEG || in visitFCOPYSIGN()
15462 N0.getOpcode() == ISD::FCOPYSIGN) in visitFCOPYSIGN()
15463 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0.getOperand(0), N1); in visitFCOPYSIGN()
15466 if (N1.getOpcode() == ISD::FABS) in visitFCOPYSIGN()
15467 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFCOPYSIGN()
15470 if (N1.getOpcode() == ISD::FCOPYSIGN) in visitFCOPYSIGN()
15471 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1.getOperand(1)); in visitFCOPYSIGN()
15476 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1.getOperand(0)); in visitFCOPYSIGN()
15508 (!DAG.getTargetLoweringInfo().isOperationExpand(ISD::FPOW, VT) && in visitFPOW()
15509 DAG.getTargetLoweringInfo().isOperationExpand(ISD::FCBRT, VT))) in visitFPOW()
15512 return DAG.getNode(ISD::FCBRT, SDLoc(N), VT, N->getOperand(0)); in visitFPOW()
15537 if (!DAG.getTargetLoweringInfo().isOperationLegalOrCustom(ISD::FSQRT, VT)) in visitFPOW()
15547 SDValue Sqrt = DAG.getNode(ISD::FSQRT, DL, VT, N->getOperand(0)); in visitFPOW()
15548 SDValue SqrtSqrt = DAG.getNode(ISD::FSQRT, DL, VT, Sqrt); in visitFPOW()
15552 return DAG.getNode(ISD::FMUL, DL, VT, Sqrt, SqrtSqrt); in visitFPOW()
15567 if (!TLI.isOperationLegal(ISD::FTRUNC, VT) || in foldFPToIntToFP()
15574 if (N->getOpcode() == ISD::SINT_TO_FP && N0.getOpcode() == ISD::FP_TO_SINT && in foldFPToIntToFP()
15576 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0.getOperand(0)); in foldFPToIntToFP()
15578 if (N->getOpcode() == ISD::UINT_TO_FP && N0.getOpcode() == ISD::FP_TO_UINT && in foldFPToIntToFP()
15580 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0.getOperand(0)); in foldFPToIntToFP()
15598 TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) in visitSINT_TO_FP()
15599 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); in visitSINT_TO_FP()
15603 if (!hasOperation(ISD::SINT_TO_FP, OpVT) && in visitSINT_TO_FP()
15604 hasOperation(ISD::UINT_TO_FP, OpVT)) { in visitSINT_TO_FP()
15607 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); in visitSINT_TO_FP()
15612 if (N0.getOpcode() == ISD::SETCC && N0.getValueType() == MVT::i1 && in visitSINT_TO_FP()
15614 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitSINT_TO_FP()
15622 if (N0.getOpcode() == ISD::ZERO_EXTEND && in visitSINT_TO_FP()
15623 N0.getOperand(0).getOpcode() == ISD::SETCC && !VT.isVector() && in visitSINT_TO_FP()
15624 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitSINT_TO_FP()
15650 TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) in visitUINT_TO_FP()
15651 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); in visitUINT_TO_FP()
15655 if (!hasOperation(ISD::UINT_TO_FP, OpVT) && in visitUINT_TO_FP()
15656 hasOperation(ISD::SINT_TO_FP, OpVT)) { in visitUINT_TO_FP()
15659 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); in visitUINT_TO_FP()
15663 if (N0.getOpcode() == ISD::SETCC && !VT.isVector() && in visitUINT_TO_FP()
15664 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitUINT_TO_FP()
15681 if (N0.getOpcode() != ISD::UINT_TO_FP && N0.getOpcode() != ISD::SINT_TO_FP) in FoldIntToFPToInt()
15686 bool IsInputSigned = N0.getOpcode() == ISD::SINT_TO_FP; in FoldIntToFPToInt()
15687 bool IsOutputSigned = N->getOpcode() == ISD::FP_TO_SINT; in FoldIntToFPToInt()
15707 unsigned ExtOp = IsInputSigned && IsOutputSigned ? ISD::SIGN_EXTEND in FoldIntToFPToInt()
15708 : ISD::ZERO_EXTEND; in FoldIntToFPToInt()
15712 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Src); in FoldIntToFPToInt()
15728 return DAG.getNode(ISD::FP_TO_SINT, SDLoc(N), VT, N0); in visitFP_TO_SINT()
15743 return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), VT, N0); in visitFP_TO_UINT()
15756 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, N0, N1); in visitFP_ROUND()
15759 if (N0.getOpcode() == ISD::FP_EXTEND && VT == N0.getOperand(0).getValueType()) in visitFP_ROUND()
15763 if (N0.getOpcode() == ISD::FP_ROUND) { in visitFP_ROUND()
15784 return DAG.getNode(ISD::FP_ROUND, DL, VT, N0.getOperand(0), in visitFP_ROUND()
15790 if (N0.getOpcode() == ISD::FCOPYSIGN && N0->hasOneUse()) { in visitFP_ROUND()
15791 SDValue Tmp = DAG.getNode(ISD::FP_ROUND, SDLoc(N0), VT, in visitFP_ROUND()
15794 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, in visitFP_ROUND()
15810 N->use_begin()->getOpcode() == ISD::FP_ROUND) in visitFP_EXTEND()
15815 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, N0); in visitFP_EXTEND()
15818 if (N0.getOpcode() == ISD::FP16_TO_FP && in visitFP_EXTEND()
15819 TLI.getOperationAction(ISD::FP16_TO_FP, VT) == TargetLowering::Legal) in visitFP_EXTEND()
15820 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), VT, N0.getOperand(0)); in visitFP_EXTEND()
15824 if (N0.getOpcode() == ISD::FP_ROUND in visitFP_EXTEND()
15829 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, in visitFP_EXTEND()
15831 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, In); in visitFP_EXTEND()
15835 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && in visitFP_EXTEND()
15836 TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, VT, N0.getValueType())) { in visitFP_EXTEND()
15838 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, in visitFP_EXTEND()
15844 DAG.getNode(ISD::FP_ROUND, SDLoc(N0), in visitFP_EXTEND()
15863 return DAG.getNode(ISD::FCEIL, SDLoc(N), VT, N0); in visitFCEIL()
15874 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0); in visitFTRUNC()
15881 case ISD::FRINT: in visitFTRUNC()
15882 case ISD::FTRUNC: in visitFTRUNC()
15883 case ISD::FNEARBYINT: in visitFTRUNC()
15884 case ISD::FFLOOR: in visitFTRUNC()
15885 case ISD::FCEIL: in visitFTRUNC()
15898 return DAG.getNode(ISD::FFLOOR, SDLoc(N), VT, N0); in visitFFLOOR()
15910 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0); in visitFNEG()
15920 if (N0.getOpcode() == ISD::FSUB && in visitFNEG()
15923 return DAG.getNode(ISD::FSUB, SDLoc(N), VT, N0.getOperand(1), in visitFNEG()
15939 bool PropagatesNaN = Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM; in visitFMinMax()
15940 bool IsMin = Opc == ISD::FMINNUM || Opc == ISD::FMINIMUM; in visitFMinMax()
15990 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFABS()
15993 if (N0.getOpcode() == ISD::FABS) in visitFABS()
15998 if (N0.getOpcode() == ISD::FNEG || N0.getOpcode() == ISD::FCOPYSIGN) in visitFABS()
15999 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0.getOperand(0)); in visitFABS()
16014 if (N1->getOpcode() == ISD::FREEZE && N1.hasOneUse()) { in visitBRCOND()
16015 return DAG.getNode(ISD::BRCOND, SDLoc(N), MVT::Other, Chain, in visitBRCOND()
16027 if (N1.getOpcode() == ISD::SETCC && in visitBRCOND()
16028 TLI.isOperationLegalOrCustom(ISD::BR_CC, in visitBRCOND()
16030 return DAG.getNode(ISD::BR_CC, SDLoc(N), MVT::Other, in visitBRCOND()
16040 return DAG.getNode(ISD::BRCOND, SDLoc(N), MVT::Other, in visitBRCOND()
16048 if (N.getOpcode() == ISD::SRL || in rebuildSetCC()
16049 (N.getOpcode() == ISD::TRUNCATE && in rebuildSetCC()
16051 N.getOperand(0).getOpcode() == ISD::SRL))) { in rebuildSetCC()
16053 if (N.getOpcode() == ISD::TRUNCATE) in rebuildSetCC()
16076 if (Op0.getOpcode() == ISD::AND && Op1.getOpcode() == ISD::Constant) { in rebuildSetCC()
16079 if (AndOp1.getOpcode() == ISD::Constant) { in rebuildSetCC()
16087 ISD::SETNE); in rebuildSetCC()
16095 if (N.getOpcode() == ISD::XOR) { in rebuildSetCC()
16102 while (N.getOpcode() == ISD::XOR) { in rebuildSetCC()
16115 if (N.getOpcode() != ISD::XOR) in rebuildSetCC()
16121 if (Op0.getOpcode() != ISD::SETCC && Op1.getOpcode() != ISD::SETCC) { in rebuildSetCC()
16124 if (isBitwiseNot(N) && Op0.hasOneUse() && Op0.getOpcode() == ISD::XOR && in rebuildSetCC()
16137 Equal ? ISD::SETEQ : ISD::SETNE); in rebuildSetCC()
16163 if (Simp.getNode() && Simp.getOpcode() == ISD::SETCC) in visitBR_CC()
16164 return DAG.getNode(ISD::BR_CC, SDLoc(N), MVT::Other, in visitBR_CC()
16227 if (!getCombineLoadStoreParts(N, ISD::PRE_INC, ISD::PRE_DEC, IsLoad, IsMasked, in CombineToPreIndexedLoadStore()
16233 if ((Ptr.getOpcode() != ISD::ADD && Ptr.getOpcode() != ISD::SUB) || in CombineToPreIndexedLoadStore()
16240 ISD::MemIndexedMode AM = ISD::UNINDEXED; in CombineToPreIndexedLoadStore()
16306 if (Use.getUser()->getOpcode() != ISD::ADD && in CombineToPreIndexedLoadStore()
16307 Use.getUser()->getOpcode() != ISD::SUB) { in CombineToPreIndexedLoadStore()
16403 int X0 = (OtherUses[i]->getOpcode() == ISD::SUB && OffsetIdx == 1) ? -1 : 1; in CombineToPreIndexedLoadStore()
16404 int Y0 = (OtherUses[i]->getOpcode() == ISD::SUB && OffsetIdx == 0) ? -1 : 1; in CombineToPreIndexedLoadStore()
16405 int X1 = (AM == ISD::PRE_DEC && !Swapped) ? -1 : 1; in CombineToPreIndexedLoadStore()
16406 int Y1 = (AM == ISD::PRE_DEC && Swapped) ? -1 : 1; in CombineToPreIndexedLoadStore()
16408 unsigned Opcode = (Y0 * Y1 < 0) ? ISD::SUB : ISD::ADD; in CombineToPreIndexedLoadStore()
16438 ISD::MemIndexedMode &AM, in shouldCombineToPostInc()
16442 (PtrUse->getOpcode() != ISD::ADD && PtrUse->getOpcode() != ISD::SUB)) in shouldCombineToPostInc()
16465 if (getCombineLoadStoreParts(Use, ISD::POST_INC, ISD::POST_DEC, IsLoad, in shouldCombineToPostInc()
16476 if (Use->getOpcode() == ISD::ADD || Use->getOpcode() == ISD::SUB) { in shouldCombineToPostInc()
16488 ISD::MemIndexedMode &AM, in getPostIndexedLoadStoreOp()
16491 if (!getCombineLoadStoreParts(N, ISD::POST_INC, ISD::POST_DEC, IsLoad, in getPostIndexedLoadStoreOp()
16534 ISD::MemIndexedMode AM = ISD::UNINDEXED; in CombineToPostIndexedLoadStore()
16575 ISD::MemIndexedMode AM = LD->getAddressingMode(); in SplitIndexingFromLoad()
16576 assert(AM != ISD::UNINDEXED); in SplitIndexingFromLoad()
16583 assert((Inc.getOpcode() != ISD::TargetConstant || in SplitIndexingFromLoad()
16586 if (Inc.getOpcode() == ISD::TargetConstant) { in SplitIndexingFromLoad()
16593 (AM == ISD::PRE_INC || AM == ISD::POST_INC ? ISD::ADD : ISD::SUB); in SplitIndexingFromLoad()
16610 TLI.isOperationLegal(ISD::FTRUNC, STMemType)) { in getTruncatedStoreValue()
16611 Val = DAG.getNode(ISD::FTRUNC, SDLoc(ST), STMemType, Val); in getTruncatedStoreValue()
16616 Val = DAG.getNode(ISD::TRUNCATE, SDLoc(ST), STMemType, Val); in getTruncatedStoreValue()
16635 case ISD::NON_EXTLOAD: in extendLoadedValueToExtension()
16638 case ISD::EXTLOAD: in extendLoadedValueToExtension()
16639 Val = DAG.getNode(ISD::ANY_EXTEND, SDLoc(LD), LDType, Val); in extendLoadedValueToExtension()
16641 case ISD::SEXTLOAD: in extendLoadedValueToExtension()
16642 Val = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(LD), LDType, Val); in extendLoadedValueToExtension()
16644 case ISD::ZEXTLOAD: in extendLoadedValueToExtension()
16645 Val = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(LD), LDType, Val); in extendLoadedValueToExtension()
16734 !LDMemType.isVector() && LD->getExtensionType() != ISD::SEXTLOAD) { in ForwardStoreValueToDirectLoad()
16740 auto Val = DAG.getNode(ISD::AND, SDLoc(LD), LDType, ST->getValue(), Mask); in ForwardStoreValueToDirectLoad()
16760 Val = DAG.getNode(ISD::TRUNCATE, SDLoc(LD), LDMemType, Val); in ForwardStoreValueToDirectLoad()
16868 if (LD->getExtensionType() == ISD::NON_EXTLOAD) { in visitLOAD()
16879 SDValue Token = DAG.getNode(ISD::TokenFactor, SDLoc(N), in visitLOAD()
17073 if (!TLI.isOperationLegal(ISD::LOAD, SliceType)) in isLegal()
17087 if (!TLI.isOperationLegal(ISD::ADD, PtrType)) in isLegal()
17093 !TLI.isOperationLegal(ISD::ZERO_EXTEND, TruncateType)) in isLegal()
17137 BaseAddr = DAG->getNode(ISD::ADD, DL, ArithType, BaseAddr, in loadSlice()
17154 DAG->getNode(ISD::ZERO_EXTEND, SDLoc(LastInst), FinalType, LastInst); in loadSlice()
17166 if (Use->getOpcode() != ISD::BITCAST) in canMergeExpensiveCrossRegisterBankCopy()
17176 if (ArgRC == ResRC || !TLI.isOperationLegal(ISD::LOAD, ResVT)) in canMergeExpensiveCrossRegisterBankCopy()
17197 if (!TLI.isOperationLegal(ISD::LOAD, ResVT)) in canMergeExpensiveCrossRegisterBankCopy()
17355 if (!LD->isSimple() || !ISD::isNormalLoad(LD) || in SliceUpLoad()
17384 if (User->getOpcode() == ISD::SRL && User->hasOneUse() && in SliceUpLoad()
17392 if (User->getOpcode() != ISD::TRUNCATE) in SliceUpLoad()
17435 if (SliceInst.getOpcode() != ISD::LOAD) in SliceUpLoad()
17437 assert(SliceInst->getOpcode() == ISD::LOAD && in SliceUpLoad()
17442 SDValue Chain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, in SliceUpLoad()
17457 if (V->getOpcode() != ISD::AND || in CheckForMaskedLoad()
17459 !ISD::isNormalLoad(V->getOperand(0).getNode())) in CheckForMaskedLoad()
17506 else if (Chain->getOpcode() == ISD::TokenFactor && in CheckForMaskedLoad()
17560 IVal = DAG.getNode(ISD::SRL, DL, IVal.getValueType(), IVal, in ShrinkLoadReplaceStoreWithStore()
17585 IVal = DAG.getNode(ISD::TRUNCATE, SDLoc(IVal), VT, IVal); in ShrinkLoadReplaceStoreWithStore()
17612 if ((Opc != ISD::OR && Opc != ISD::XOR && Opc != ISD::AND) || in ReduceLoadOpStoreWidth()
17621 if (Opc == ISD::OR && EnableShrinkLoadReplaceStoreWithStore) { in ReduceLoadOpStoreWidth()
17640 if (Value.getOperand(1).getOpcode() != ISD::Constant) in ReduceLoadOpStoreWidth()
17644 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && in ReduceLoadOpStoreWidth()
17656 if (Opc == ISD::AND) in ReduceLoadOpStoreWidth()
17684 if (Opc == ISD::AND) in ReduceLoadOpStoreWidth()
17732 if (ISD::isNormalStore(ST) && ISD::isNormalLoad(Value.getNode()) && in TransformFPLoadStorePair()
17753 if (!TLI.isOperationLegal(ISD::LOAD, IntVT) || in TransformFPLoadStorePair()
17754 !TLI.isOperationLegal(ISD::STORE, IntVT) || in TransformFPLoadStorePair()
17755 !TLI.isDesirableToTransformToIntegerOp(ISD::LOAD, VT) || in TransformFPLoadStorePair()
17756 !TLI.isDesirableToTransformToIntegerOp(ISD::STORE, VT) || in TransformFPLoadStorePair()
17811 if (Use->getOpcode() == ISD::MUL) { // We have another multiply use. in isMulAddWithConstProfitable()
17847 if (OtherOp->getOpcode() == ISD::ADD && in isMulAddWithConstProfitable()
17949 StoredVal = DAG.getNode(MemVT.isVector() ? ISD::CONCAT_VECTORS in mergeStoresOfConstantsOrVecElts()
17950 : ISD::BUILD_VECTOR, in mergeStoresOfConstantsOrVecElts()
17964 (Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT || in mergeStoresOfConstantsOrVecElts()
17965 Val.getOpcode() == ISD::EXTRACT_SUBVECTOR)) { in mergeStoresOfConstantsOrVecElts()
17971 unsigned OpC = MemVT.isVector() ? ISD::EXTRACT_SUBVECTOR in mergeStoresOfConstantsOrVecElts()
17972 : ISD::EXTRACT_VECTOR_ELT; in mergeStoresOfConstantsOrVecElts()
17982 StoredVal = DAG.getNode(MemVT.isVector() ? ISD::CONCAT_VECTORS in mergeStoresOfConstantsOrVecElts()
17983 : ISD::BUILD_VECTOR, in mergeStoresOfConstantsOrVecElts()
18138 if (OtherBC.getOpcode() != ISD::EXTRACT_VECTOR_ELT && in getStoreMergeCandidates()
18139 OtherBC.getOpcode() != ISD::EXTRACT_SUBVECTOR) in getStoreMergeCandidates()
18235 if (N->getOpcode() == ISD::TokenFactor) { in checkMergeStoreCandidatesForDependencies()
18578 (hasOperation(ISD::ROTL, PairVT) || in tryStoreMergeOfLoads()
18579 hasOperation(ISD::ROTR, PairVT))) { in tryStoreMergeOfLoads()
18662 TLI.isLoadExtLegal(ISD::ZEXTLOAD, LegalizedStoredValTy, StoreTy) && in tryStoreMergeOfLoads()
18663 TLI.isLoadExtLegal(ISD::SEXTLOAD, LegalizedStoredValTy, StoreTy) && in tryStoreMergeOfLoads()
18664 TLI.isLoadExtLegal(ISD::EXTLOAD, LegalizedStoredValTy, StoreTy) && in tryStoreMergeOfLoads()
18762 StoreOp = DAG.getNode(ISD::ROTL, LoadDL, JointMemOpVT, NewLoad, RotAmt); in tryStoreMergeOfLoads()
18770 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, LoadDL, ExtendedTy, in tryStoreMergeOfLoads()
18907 SDValue Token = DAG.getNode(ISD::TokenFactor, SL, in replaceStoreChain()
18919 if (Value.getOpcode() == ISD::TargetConstantFP) in replaceStoreOfFPConstant()
18922 if (!ISD::isNormalStore(ST)) in replaceStoreOfFPConstant()
18949 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) { in replaceStoreOfFPConstant()
18960 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i64)) { in replaceStoreOfFPConstant()
18968 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) { in replaceStoreOfFPConstant()
18987 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in replaceStoreOfFPConstant()
19003 if (Value.getOpcode() == ISD::BITCAST && !ST->isTruncatingStore() && in visitSTORE()
19013 TLI.isOperationLegal(ISD::STORE, SVT)) && in visitSTORE()
19067 if ((Value.getOpcode() == ISD::ZERO_EXTEND || in visitSTORE()
19068 Value.getOpcode() == ISD::SIGN_EXTEND || in visitSTORE()
19069 Value.getOpcode() == ISD::ANY_EXTEND) && in visitSTORE()
19071 TLI.isOperationLegalOrCustom(ISD::STORE, ST->getMemoryVT())) in visitSTORE()
19094 if (N->getOpcode() != ISD::DELETED_NODE) in visitSTORE()
19152 if ((Value.getOpcode() == ISD::FP_ROUND || in visitSTORE()
19153 Value.getOpcode() == ISD::TRUNCATE) && in visitSTORE()
19173 if (N->getOpcode() == ISD::DELETED_NODE || !isa<StoreSDNode>(N)) in visitSTORE()
19213 case ISD::TokenFactor: in visitLIFETIME_END()
19217 case ISD::LIFETIME_START: in visitLIFETIME_END()
19218 case ISD::LIFETIME_END: in visitLIFETIME_END()
19224 case ISD::STORE: { in visitLIFETIME_END()
19292 if (!Val.getValueType().isScalarInteger() || Val.getOpcode() != ISD::OR) in splitMergedValStore()
19299 if (Op1.getOpcode() != ISD::SHL) { in splitMergedValStore()
19301 if (Op1.getOpcode() != ISD::SHL) in splitMergedValStore()
19317 if (Lo.getOpcode() != ISD::ZERO_EXTEND || !Lo.hasOneUse() || in splitMergedValStore()
19320 Hi.getOpcode() != ISD::ZERO_EXTEND || !Hi.hasOneUse() || in splitMergedValStore()
19327 EVT LowTy = (Lo.getOperand(0).getOpcode() == ISD::BITCAST) in splitMergedValStore()
19330 EVT HighTy = (Hi.getOperand(0).getOpcode() == ISD::BITCAST) in splitMergedValStore()
19342 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Lo.getOperand(0)); in splitMergedValStore()
19343 Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Hi.getOperand(0)); in splitMergedValStore()
19360 assert(N->getOpcode() == ISD::INSERT_VECTOR_ELT && in combineInsertEltToShuffle()
19369 if (Vec.getOpcode() == ISD::VECTOR_SHUFFLE && Vec.hasOneUse() && in combineInsertEltToShuffle()
19370 InsertVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in combineInsertEltToShuffle()
19402 if (ArgVal.getOpcode() == ISD::CONCAT_VECTORS) { in combineInsertEltToShuffle()
19446 if (InsertVal.getOpcode() != ISD::BITCAST || !InsertVal.hasOneUse() || in combineInsertEltToShuffle()
19486 SDValue PaddedSubV = DAG.getNode(ISD::CONCAT_VECTORS, DL, ShufVT, ConcatOps); in combineInsertEltToShuffle()
19513 if (InVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitINSERT_VECTOR_ELT()
19544 if (InVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitINSERT_VECTOR_ELT()
19557 if (InVec.getOpcode() == ISD::INSERT_VECTOR_ELT && InVec.hasOneUse() in visitINSERT_VECTOR_ELT()
19562 SDValue NewOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, in visitINSERT_VECTOR_ELT()
19565 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(InVec.getNode()), in visitINSERT_VECTOR_ELT()
19571 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) { in visitINSERT_VECTOR_ELT()
19612 if (CurVec.getOpcode() == ISD::BUILD_VECTOR && CurVec.hasOneUse()) { in visitINSERT_VECTOR_ELT()
19619 if (CurVec.getOpcode() == ISD::SCALAR_TO_VECTOR && CurVec.hasOneUse()) { in visitINSERT_VECTOR_ELT()
19625 if (CurVec.getOpcode() == ISD::INSERT_VECTOR_ELT && CurVec.hasOneUse()) in visitINSERT_VECTOR_ELT()
19661 ISD::LoadExtType ExtTy = in scalarizeExtractedVectorLoad()
19662 ResultVT.bitsGT(VecEltVT) ? ISD::NON_EXTLOAD : ISD::EXTLOAD; in scalarizeExtractedVectorLoad()
19663 if (!TLI.isOperationLegalOrCustom(ISD::LOAD, VecEltVT) || in scalarizeExtractedVectorLoad()
19699 ISD::LoadExtType ExtType = in scalarizeExtractedVectorLoad()
19700 TLI.isLoadExtLegal(ISD::ZEXTLOAD, ResultVT, VecEltVT) ? ISD::ZEXTLOAD in scalarizeExtractedVectorLoad()
19701 : ISD::EXTLOAD; in scalarizeExtractedVectorLoad()
19714 Load = DAG.getNode(ISD::TRUNCATE, DL, ResultVT, Load); in scalarizeExtractedVectorLoad()
19745 ISD::isConstantSplatVector(Op0.getNode(), SplatVal) || in scalarizeExtractedBinop()
19747 ISD::isConstantSplatVector(Op1.getNode(), SplatVal)) { in scalarizeExtractedBinop()
19752 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Index); in scalarizeExtractedBinop()
19753 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op1, Index); in scalarizeExtractedBinop()
19773 if (VecOp.getOpcode() == ISD::INSERT_VECTOR_ELT && in visitEXTRACT_VECTOR_ELT()
19780 if (VecOp.getOpcode() == ISD::SCALAR_TO_VECTOR) { in visitEXTRACT_VECTOR_ELT()
19792 return DAG.getNode(ISD::TRUNCATE, DL, ScalarVT, InOp); in visitEXTRACT_VECTOR_ELT()
19804 if (((IndexC && VecOp.getOpcode() == ISD::BUILD_VECTOR) || in visitEXTRACT_VECTOR_ELT()
19805 VecOp.getOpcode() == ISD::SPLAT_VECTOR) && in visitEXTRACT_VECTOR_ELT()
19808 assert((VecOp.getOpcode() != ISD::BUILD_VECTOR || in visitEXTRACT_VECTOR_ELT()
19812 VecOp.getOpcode() == ISD::BUILD_VECTOR ? IndexC->getZExtValue() : 0; in visitEXTRACT_VECTOR_ELT()
19839 if (IndexC && VecOp.getOpcode() == ISD::BITCAST && VecVT.isInteger() && in visitEXTRACT_VECTOR_ELT()
19848 return DAG.getNode(ISD::TRUNCATE, DL, ScalarVT, BCSrc); in visitEXTRACT_VECTOR_ELT()
19851 BCSrc.getOpcode() == ISD::SCALAR_TO_VECTOR) { in visitEXTRACT_VECTOR_ELT()
19878 if (IndexC && VecOp.getOpcode() == ISD::VECTOR_SHUFFLE) { in visitEXTRACT_VECTOR_ELT()
19896 if (SVInVec.getOpcode() == ISD::BUILD_VECTOR) { in visitEXTRACT_VECTOR_ELT()
19911 TLI.isOperationLegal(ISD::EXTRACT_VECTOR_ELT, VecVT) || in visitEXTRACT_VECTOR_ELT()
19912 TLI.isOperationExpand(ISD::VECTOR_SHUFFLE, VecVT)) { in visitEXTRACT_VECTOR_ELT()
19913 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, SVInVec, in visitEXTRACT_VECTOR_ELT()
19921 return Use->getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitEXTRACT_VECTOR_ELT()
19934 if (N->getOpcode() != ISD::DELETED_NODE) in visitEXTRACT_VECTOR_ELT()
19942 if (N->getOpcode() != ISD::DELETED_NODE) in visitEXTRACT_VECTOR_ELT()
19957 if (VecOp.getOpcode() == ISD::BITCAST) { in visitEXTRACT_VECTOR_ELT()
19973 ISD::isNormalLoad(VecOp.getNode()) && in visitEXTRACT_VECTOR_ELT()
19990 if (ISD::isNormalLoad(VecOp.getNode())) { in visitEXTRACT_VECTOR_ELT()
19992 } else if (VecOp.getOpcode() == ISD::SCALAR_TO_VECTOR && in visitEXTRACT_VECTOR_ELT()
19994 ISD::isNormalLoad(VecOp.getOperand(0).getNode())) { in visitEXTRACT_VECTOR_ELT()
20019 if (VecOp.getOpcode() == ISD::BITCAST) { in visitEXTRACT_VECTOR_ELT()
20026 if (ISD::isNormalLoad(VecOp.getNode())) { in visitEXTRACT_VECTOR_ELT()
20031 } else if (VecOp.getOpcode() == ISD::CONCAT_VECTORS && !BCNumEltsChanged && in visitEXTRACT_VECTOR_ELT()
20051 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, in visitEXTRACT_VECTOR_ELT()
20054 return DAG.getNode(ISD::BITCAST, SL, ScalarVT, Elt); in visitEXTRACT_VECTOR_ELT()
20097 bool AnyExt = In.getOpcode() == ISD::ANY_EXTEND; in reduceBuildVecExtToExtBuildVec()
20098 bool ZeroExt = In.getOpcode() == ISD::ZERO_EXTEND; in reduceBuildVecExtToExtBuildVec()
20153 assert((Cast.getOpcode() == ISD::ANY_EXTEND || in reduceBuildVecExtToExtBuildVec()
20154 Cast.getOpcode() == ISD::ZERO_EXTEND || in reduceBuildVecExtToExtBuildVec()
20174 (!TLI.isOperationLegal(ISD::BUILD_VECTOR, VecVT) && in reduceBuildVecExtToExtBuildVec()
20175 TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))) in reduceBuildVecExtToExtBuildVec()
20192 assert(N->getOpcode() == ISD::BUILD_VECTOR && "Expected build vector"); in reduceBuildVecTruncToBitCast()
20211 if (Op.getOpcode() == ISD::BITCAST) in reduceBuildVecTruncToBitCast()
20223 if (In.getOpcode() != ISD::TRUNCATE) in reduceBuildVecTruncToBitCast()
20228 if (In.getOpcode() != ISD::SRL) { in reduceBuildVecTruncToBitCast()
20300 VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in createBuildVecShuffle()
20309 VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1, in createBuildVecShuffle()
20311 VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1, ZeroIdx); in createBuildVecShuffle()
20324 !TLI.isOperationLegal(ISD::VECTOR_SHUFFLE, InVT1)) in createBuildVecShuffle()
20333 VecIn2 = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InVT1, in createBuildVecShuffle()
20341 VecIn2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in createBuildVecShuffle()
20379 Shuffle = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuffle, ZeroIdx); in createBuildVecShuffle()
20385 assert(BV->getOpcode() == ISD::BUILD_VECTOR && "Expected build vector"); in reduceBuildVecToShuffleWithZero()
20412 if (Zext.getOpcode() != ISD::ZERO_EXTEND || !Zext.hasOneUse() || in reduceBuildVecToShuffleWithZero()
20413 Zext.getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT || in reduceBuildVecToShuffleWithZero()
20485 if (LegalOperations && !TLI.isOperationLegal(ISD::VECTOR_SHUFFLE, VT)) in reduceBuildVecToShuffle()
20518 if (Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in reduceBuildVecToShuffle()
20580 SDValue VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, Vec, in reduceBuildVecToShuffle()
20582 SDValue VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, Vec, in reduceBuildVecToShuffle()
20693 bool IsLeftShuffle = L.getOpcode() == ISD::VECTOR_SHUFFLE && in reduceBuildVecToShuffle()
20702 bool IsRightShuffle = R.getOpcode() == ISD::VECTOR_SHUFFLE && in reduceBuildVecToShuffle()
20743 FoundZeroExtend |= (Opc == ISD::ZERO_EXTEND); in convertBuildVecZextToZext()
20744 if ((Opc == ISD::ZERO_EXTEND || Opc == ISD::ANY_EXTEND) && in convertBuildVecZextToZext()
20745 Op.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT && in convertBuildVecZextToZext()
20776 In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InVT, In, in convertBuildVecZextToZext()
20778 return DAG.getNode(FoundZeroExtend ? ISD::ZERO_EXTEND : ISD::ANY_EXTEND, DL, in convertBuildVecZextToZext()
20786 if (ISD::allOperandsUndef(N)) in visitBUILD_VECTOR()
20807 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), in visitBUILD_VECTOR()
20819 if ((Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT) && in visitBUILD_VECTOR()
20840 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), N->getValueType(0), in visitBUILD_VECTOR()
20858 if (TLI.getOperationAction(ISD::SPLAT_VECTOR, VT) != TargetLowering::Expand) in visitBUILD_VECTOR()
20861 return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), VT, V); in visitBUILD_VECTOR()
20880 SDValue ScalarUndef = DAG.getNode(ISD::UNDEF, DL, SVT); in combineConcatVectorOfScalars()
20886 if (ISD::BITCAST == Op.getOpcode() && in combineConcatVectorOfScalars()
20889 else if (ISD::UNDEF == Op.getOpcode()) in combineConcatVectorOfScalars()
20910 ScalarUndef = DAG.getNode(ISD::UNDEF, DL, SVT); in combineConcatVectorOfScalars()
20941 if (Op.getOpcode() != ISD::CONCAT_VECTORS) in combineConcatVectorOfConcatVectors()
20963 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, ConcatOps); in combineConcatVectorOfConcatVectors()
20993 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR) in combineConcatVectorOfExtracts()
21047 case ISD::SINT_TO_FP: in combineConcatVectorOfCasts()
21048 case ISD::UINT_TO_FP: in combineConcatVectorOfCasts()
21049 case ISD::FP_TO_SINT: in combineConcatVectorOfCasts()
21050 case ISD::FP_TO_UINT: in combineConcatVectorOfCasts()
21085 case ISD::SINT_TO_FP: in combineConcatVectorOfCasts()
21086 case ISD::UINT_TO_FP: in combineConcatVectorOfCasts()
21091 case ISD::FP_TO_SINT: in combineConcatVectorOfCasts()
21092 case ISD::FP_TO_UINT: in combineConcatVectorOfCasts()
21103 SDValue NewConcat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatSrcVT, SrcOps); in combineConcatVectorOfCasts()
21114 if (ISD::allOperandsUndef(N)) in visitCONCAT_VECTORS()
21125 if (In.getOpcode() == ISD::CONCAT_VECTORS && In.hasOneUse()) { in visitCONCAT_VECTORS()
21129 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in visitCONCAT_VECTORS()
21136 if (!LegalOperations && Scalar.getOpcode() == ISD::SCALAR_TO_VECTOR && in visitCONCAT_VECTORS()
21148 if (Scalar->getOpcode() == ISD::TRUNCATE && in visitCONCAT_VECTORS()
21170 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), NVT, Scalar); in visitCONCAT_VECTORS()
21180 return ISD::UNDEF == Op.getOpcode() || ISD::BUILD_VECTOR == Op.getOpcode(); in visitCONCAT_VECTORS()
21192 if (ISD::BUILD_VECTOR == Op.getOpcode()) { in visitCONCAT_VECTORS()
21204 if (ISD::UNDEF == Op.getOpcode()) in visitCONCAT_VECTORS()
21207 if (ISD::BUILD_VECTOR == Op.getOpcode()) { in visitCONCAT_VECTORS()
21214 DAG.getNode(ISD::TRUNCATE, SDLoc(N), MinVT, Op.getOperand(i))); in visitCONCAT_VECTORS()
21259 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR) in visitCONCAT_VECTORS()
21291 if (V.getOpcode() == ISD::INSERT_SUBVECTOR && in getSubVectorSrc()
21296 if (IndexC && V.getOpcode() == ISD::CONCAT_VECTORS && in getSubVectorSrc()
21366 if (BOpcode == ISD::FSUB) { in narrowExtractedVectorBinOp()
21415 SDValue X = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT, in narrowExtractedVectorBinOp()
21417 SDValue Y = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT, in narrowExtractedVectorBinOp()
21434 if (BOpcode != ISD::AND && BOpcode != ISD::OR && BOpcode != ISD::XOR) in narrowExtractedVectorBinOp()
21440 if (V.getOpcode() == ISD::CONCAT_VECTORS && V.getNumOperands() == 2) in narrowExtractedVectorBinOp()
21456 : DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT, in narrowExtractedVectorBinOp()
21460 : DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT, in narrowExtractedVectorBinOp()
21538 assert(N->getOpcode() == ISD::EXTRACT_SUBVECTOR && in foldExtractSubvectorFromShuffleVector()
21560 !TLI.isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, NarrowVT)) in foldExtractSubvectorFromShuffleVector()
21664 NewOps.emplace_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowVT, in foldExtractSubvectorFromShuffleVector()
21686 if (TLI.isOperationLegalOrCustomOrPromote(ISD::LOAD, NVT)) in visitEXTRACT_SUBVECTOR()
21692 if (ExtIdx == 0 && V.getOpcode() == ISD::EXTRACT_SUBVECTOR && V.hasOneUse()) { in visitEXTRACT_SUBVECTOR()
21695 TLI.isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, NVT)) { in visitEXTRACT_SUBVECTOR()
21696 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), NVT, V.getOperand(0), in visitEXTRACT_SUBVECTOR()
21702 if (V.getOpcode() == ISD::SPLAT_VECTOR) in visitEXTRACT_SUBVECTOR()
21704 if (!LegalOperations || TLI.isOperationLegal(ISD::SPLAT_VECTOR, NVT)) in visitEXTRACT_SUBVECTOR()
21709 if (V.getOpcode() == ISD::BITCAST && in visitEXTRACT_SUBVECTOR()
21711 (!LegalOperations || TLI.isOperationLegal(ISD::BITCAST, NVT))) { in visitEXTRACT_SUBVECTOR()
21721 if (TLI.isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, NewExtVT)) { in visitEXTRACT_SUBVECTOR()
21724 SDValue NewExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewExtVT, in visitEXTRACT_SUBVECTOR()
21740 if (TLI.isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, NewExtVT)) { in visitEXTRACT_SUBVECTOR()
21743 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewExtVT, in visitEXTRACT_SUBVECTOR()
21748 TLI.isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT, ScalarVT)) { in visitEXTRACT_SUBVECTOR()
21751 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, in visitEXTRACT_SUBVECTOR()
21760 if (V.getOpcode() == ISD::CONCAT_VECTORS) { in visitEXTRACT_SUBVECTOR()
21791 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NVT, in visitEXTRACT_SUBVECTOR()
21803 if (V.getOpcode() == ISD::BUILD_VECTOR) { in visitEXTRACT_SUBVECTOR()
21816 TLI.isOperationLegal(ISD::BUILD_VECTOR, ExtractVT))) && in visitEXTRACT_SUBVECTOR()
21823 Src = DAG.getNode(ISD::TRUNCATE, SDLoc(N), InVT, Src); in visitEXTRACT_SUBVECTOR()
21835 if (V.getOpcode() == ISD::INSERT_SUBVECTOR) { in visitEXTRACT_SUBVECTOR()
21850 if (LegalOperations && !TLI.isOperationLegal(ISD::BITCAST, NVT)) in visitEXTRACT_SUBVECTOR()
21856 ISD::EXTRACT_SUBVECTOR, SDLoc(N), NVT, in visitEXTRACT_SUBVECTOR()
21877 if (N0.getOpcode() != ISD::CONCAT_VECTORS || N0.getNumOperands() != 2 || in foldShuffleOfConcatUndefs()
21878 N1.getOpcode() != ISD::CONCAT_VECTORS || N1.getNumOperands() != 2 || in foldShuffleOfConcatUndefs()
21917 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Shuf0, Shuf1); in foldShuffleOfConcatUndefs()
21948 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, N0, N1); in partitionShuffleOfConcats()
21982 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in partitionShuffleOfConcats()
22020 if (N0AnyConst && !N1AnyConst && !ISD::isBuildVectorAllZeros(N0.getNode())) in combineShuffleOfScalars()
22022 if (!N0AnyConst && N1AnyConst && !ISD::isBuildVectorAllZeros(N1.getNode())) in combineShuffleOfScalars()
22042 if (S.getOpcode() == ISD::BUILD_VECTOR) { in combineShuffleOfScalars()
22044 } else if (S.getOpcode() == ISD::SCALAR_TO_VECTOR) { in combineShuffleOfScalars()
22126 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND_VECTOR_INREG, OutVT)) in combineShuffleToVectorExtend()
22128 DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, in combineShuffleToVectorExtend()
22152 if (Opcode != ISD::ANY_EXTEND_VECTOR_INREG && in combineTruncationShuffle()
22153 Opcode != ISD::SIGN_EXTEND_VECTOR_INREG && in combineTruncationShuffle()
22154 Opcode != ISD::ZERO_EXTEND_VECTOR_INREG) in combineTruncationShuffle()
22267 if (Op0.getOpcode() != ISD::BITCAST) in combineShuffleOfBitcast()
22271 (!Op1.isUndef() && (Op1.getOpcode() != ISD::BITCAST || in combineShuffleOfBitcast()
22282 !TLI.isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, InVT))) in combineShuffleOfBitcast()
22411 if (Op0.getOpcode() != ISD::INSERT_VECTOR_ELT) in replaceShuffleOfInsert()
22429 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(Shuf), Op0.getValueType(), in replaceShuffleOfInsert()
22523 SDValue ExtL = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, L, Index); in visitVECTOR_SHUFFLE()
22524 SDValue ExtR = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, R, Index); in visitVECTOR_SHUFFLE()
22527 SDValue Insert = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, NewBO); in visitVECTOR_SHUFFLE()
22534 if ((!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) && in visitVECTOR_SHUFFLE()
22536 if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR && SplatIndex == 0) in visitVECTOR_SHUFFLE()
22539 if (N0.getOpcode() == ISD::INSERT_VECTOR_ELT) in visitVECTOR_SHUFFLE()
22549 if (V->getOpcode() == ISD::BITCAST) { in visitVECTOR_SHUFFLE()
22556 if (V->getOpcode() == ISD::BUILD_VECTOR) { in visitVECTOR_SHUFFLE()
22610 if (N0.getOpcode() == ISD::CONCAT_VECTORS && in visitVECTOR_SHUFFLE()
22613 (N1.getOpcode() == ISD::CONCAT_VECTORS && in visitVECTOR_SHUFFLE()
22622 if (N0.getOpcode() == ISD::CONCAT_VECTORS && N1.isUndef() && in visitVECTOR_SHUFFLE()
22637 SDValue NewCat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, in visitVECTOR_SHUFFLE()
22648 TLI.isOperationLegalOrCustom(ISD::INSERT_SUBVECTOR, VT)) { in visitVECTOR_SHUFFLE()
22651 assert(RHS.getOpcode() == ISD::CONCAT_VECTORS && "Can't find subvectors"); in visitVECTOR_SHUFFLE()
22688 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, LHS, in visitVECTOR_SHUFFLE()
22696 if (N1.getOpcode() == ISD::CONCAT_VECTORS) in visitVECTOR_SHUFFLE()
22699 if (N0.getOpcode() == ISD::CONCAT_VECTORS) { in visitVECTOR_SHUFFLE()
22756 if (TLI.isOperationLegalOrCustom(ISD::AND, IntVT)) in visitVECTOR_SHUFFLE()
22758 VT, DAG.getNode(ISD::AND, DL, IntVT, DAG.getBitcast(IntVT, N0), in visitVECTOR_SHUFFLE()
22773 if (N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() && in visitVECTOR_SHUFFLE()
22778 if (BC0.getOpcode() == ISD::VECTOR_SHUFFLE && BC0.hasOneUse()) { in visitVECTOR_SHUFFLE()
22952 if (N1.getOpcode() == ISD::VECTOR_SHUFFLE && in visitVECTOR_SHUFFLE()
22953 N0.getOpcode() != ISD::VECTOR_SHUFFLE) { in visitVECTOR_SHUFFLE()
22970 if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && in visitVECTOR_SHUFFLE()
22971 N1.getOpcode() == ISD::VECTOR_SHUFFLE && in visitVECTOR_SHUFFLE()
22985 if (N->getOperand(i).getOpcode() == ISD::VECTOR_SHUFFLE && in visitVECTOR_SHUFFLE()
23026 (Op00.getOpcode() == ISD::VECTOR_SHUFFLE || in visitVECTOR_SHUFFLE()
23027 Op10.getOpcode() == ISD::VECTOR_SHUFFLE || in visitVECTOR_SHUFFLE()
23028 Op01.getOpcode() == ISD::VECTOR_SHUFFLE || in visitVECTOR_SHUFFLE()
23029 Op11.getOpcode() == ISD::VECTOR_SHUFFLE)) { in visitVECTOR_SHUFFLE()
23098 if (InVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitSCALAR_TO_VECTOR()
23114 DAG.getNode(ISD::TRUNCATE, SDLoc(InVal), VT.getScalarType(), InVal); in visitSCALAR_TO_VECTOR()
23115 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Val); in visitSCALAR_TO_VECTOR()
23133 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), SubVT, in visitSCALAR_TO_VECTOR()
23157 if (N0.isUndef() && N1.getOpcode() == ISD::EXTRACT_SUBVECTOR && in visitINSERT_SUBVECTOR()
23163 if (N0.isUndef() && N1.getOpcode() == ISD::SPLAT_VECTOR) in visitINSERT_SUBVECTOR()
23164 return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), VT, N1.getOperand(0)); in visitINSERT_SUBVECTOR()
23170 if (N0.isUndef() && N1.getOpcode() == ISD::BITCAST && in visitINSERT_SUBVECTOR()
23171 N1.getOperand(0).getOpcode() == ISD::EXTRACT_SUBVECTOR && in visitINSERT_SUBVECTOR()
23184 if (N0.getOpcode() == ISD::BITCAST && N1.getOpcode() == ISD::BITCAST) { in visitINSERT_SUBVECTOR()
23192 SDValue NewINSERT = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), in visitINSERT_SUBVECTOR()
23201 if (N0.getOpcode() == ISD::INSERT_SUBVECTOR && in visitINSERT_SUBVECTOR()
23204 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0.getOperand(0), in visitINSERT_SUBVECTOR()
23210 if (N0.isUndef() && N1.getOpcode() == ISD::INSERT_SUBVECTOR && in visitINSERT_SUBVECTOR()
23212 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0, in visitINSERT_SUBVECTOR()
23218 if ((N0.isUndef() || N0.getOpcode() == ISD::BITCAST) && in visitINSERT_SUBVECTOR()
23219 N1.getOpcode() == ISD::BITCAST) { in visitINSERT_SUBVECTOR()
23244 if (NewIdx && hasOperation(ISD::INSERT_SUBVECTOR, NewVT)) { in visitINSERT_SUBVECTOR()
23246 Res = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, NewVT, Res, N1Src, NewIdx); in visitINSERT_SUBVECTOR()
23256 if (N0.getOpcode() == ISD::INSERT_SUBVECTOR && N0.hasOneUse() && in visitINSERT_SUBVECTOR()
23261 SDValue NewOp = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, in visitINSERT_SUBVECTOR()
23264 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N0.getNode()), in visitINSERT_SUBVECTOR()
23271 if (N0.getOpcode() == ISD::CONCAT_VECTORS && N0.hasOneUse() && in visitINSERT_SUBVECTOR()
23278 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in visitINSERT_SUBVECTOR()
23292 if (N0->getOpcode() == ISD::FP16_TO_FP) in visitFP_TO_FP16()
23302 if (!TLI.shouldKeepZExtForFP16Conv() && N0->getOpcode() == ISD::AND) { in visitFP16_TO_FP()
23305 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), N->getValueType(0), in visitFP16_TO_FP()
23317 if (N0->getOpcode() == ISD::BF16_TO_FP) in visitFP_TO_BF16()
23332 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getVectorElementType(), N0, in visitVECREDUCE()
23335 Res = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Res); in visitVECREDUCE()
23341 if (Opcode == ISD::VECREDUCE_AND || Opcode == ISD::VECREDUCE_OR) { in visitVECREDUCE()
23342 unsigned NewOpcode = Opcode == ISD::VECREDUCE_AND in visitVECREDUCE()
23343 ? ISD::VECREDUCE_UMIN : ISD::VECREDUCE_UMAX; in visitVECREDUCE()
23352 if (N0.getOpcode() == ISD::INSERT_SUBVECTOR && in visitVECREDUCE()
23356 if ((Opcode == ISD::VECREDUCE_OR && in visitVECREDUCE()
23358 (Opcode == ISD::VECREDUCE_AND && in visitVECREDUCE()
23371 if (auto EVLIdx = ISD::getVPExplicitVectorLengthIdx(N->getOpcode())) in visitVPOp()
23373 if (auto MaskIdx = ISD::getVPMaskIdx(N->getOpcode())) in visitVPOp()
23375 ISD::isConstantSplatVectorAllZeros(N->getOperand(*MaskIdx).getNode()); in visitVPOp()
23382 if (ISD::isVPBinaryOp(N->getOpcode())) in visitVPOp()
23394 if (ISD::isVPReduction(N->getOpcode())) in visitVPOp()
23405 assert(N->getOpcode() == ISD::AND && "Unexpected opcode!"); in XformToShuffleWithZero()
23417 if (RHS.getOpcode() != ISD::BUILD_VECTOR) in XformToShuffleWithZero()
23508 bool IsBothSplatVector = N0.getOpcode() == ISD::SPLAT_VECTOR && in scalarizeBinOpOfSplats()
23509 N1.getOpcode() == ISD::SPLAT_VECTOR; in scalarizeBinOpOfSplats()
23518 SDValue X = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src0, IndexC); in scalarizeBinOpOfSplats()
23519 SDValue Y = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src1, IndexC); in scalarizeBinOpOfSplats()
23524 if (N0.getOpcode() == ISD::BUILD_VECTOR && N0.getOpcode() == N1.getOpcode() && in scalarizeBinOpOfSplats()
23558 if (Opcode != ISD::UDIV && Opcode != ISD::SDIV && in SimplifyVBinOp()
23559 Opcode != ISD::UREM && Opcode != ISD::SREM && in SimplifyVBinOp()
23560 Opcode != ISD::UDIVREM && Opcode != ISD::SDIVREM) { in SimplifyVBinOp()
23580 Shuf0->getOperand(0).getOpcode() != ISD::INSERT_VECTOR_ELT) { in SimplifyVBinOp()
23589 Shuf1->getOperand(0).getOpcode() != ISD::INSERT_VECTOR_ELT) { in SimplifyVBinOp()
23602 if (LHS.getOpcode() == ISD::INSERT_SUBVECTOR && LHS.getOperand(0).isUndef() && in SimplifyVBinOp()
23603 RHS.getOpcode() == ISD::INSERT_SUBVECTOR && RHS.getOperand(0).isUndef() && in SimplifyVBinOp()
23617 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, VecC, NarrowBO, Z); in SimplifyVBinOp()
23623 return Concat.getOpcode() == ISD::CONCAT_VECTORS && in SimplifyVBinOp()
23626 ISD::isBuildVectorOfConstantSDNodes(Op.getNode()); in SimplifyVBinOp()
23648 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in SimplifyVBinOp()
23660 assert(N0.getOpcode() ==ISD::SETCC && "First argument must be a SetCC node!"); in SimplifySelect()
23671 if (SCC.getOpcode() == ISD::SELECT_CC) { in SimplifySelect()
23673 SDValue SETCC = DAG.getNode(ISD::SETCC, SDLoc(N0), in SimplifySelect()
23699 if (NaN->isNaN() && RHS.getOpcode() == ISD::FSQRT) { in SimplifySelectOps()
23702 ISD::CondCode CC; in SimplifySelectOps()
23706 if (TheSelect->getOpcode() == ISD::SELECT_CC) { in SimplifySelectOps()
23713 if (Cmp.getOpcode() == ISD::SETCC) { in SimplifySelectOps()
23720 Sqrt.getOperand(0) == CmpLHS && (CC == ISD::SETOLT || in SimplifySelectOps()
23721 CC == ISD::SETULT || CC == ISD::SETLT)) { in SimplifySelectOps()
23741 if (LHS.getOpcode() == ISD::LOAD) { in SimplifySelectOps()
23759 LLD->getExtensionType() != ISD::EXTLOAD && in SimplifySelectOps()
23760 RLD->getExtensionType() != ISD::EXTLOAD) || in SimplifySelectOps()
23770 LLD->getBasePtr().getOpcode() == ISD::TargetFrameIndex || in SimplifySelectOps()
23771 RLD->getBasePtr().getOpcode() == ISD::TargetFrameIndex || in SimplifySelectOps()
23799 if (TheSelect->getOpcode() == ISD::SELECT) { in SimplifySelectOps()
23836 Addr = DAG.getNode(ISD::SELECT_CC, SDLoc(TheSelect), in SimplifySelectOps()
23854 if (LLD->getExtensionType() == ISD::NON_EXTLOAD) { in SimplifySelectOps()
23862 LLD->getExtensionType() == ISD::EXTLOAD ? RLD->getExtensionType() in SimplifySelectOps()
23885 ISD::CondCode CC) { in foldSelectCCToShiftAnd()
23898 if (CC == ISD::SETGT && TLI.hasAndNot(N2)) { in foldSelectCCToShiftAnd()
23903 } else if (CC == ISD::SETLT) { in foldSelectCCToShiftAnd()
23920 SDValue Shift = DAG.getNode(ISD::SRL, DL, XType, N0, ShiftAmt); in foldSelectCCToShiftAnd()
23924 Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift); in foldSelectCCToShiftAnd()
23928 if (CC == ISD::SETGT) in foldSelectCCToShiftAnd()
23931 return DAG.getNode(ISD::AND, DL, AType, Shift, N2); in foldSelectCCToShiftAnd()
23940 SDValue Shift = DAG.getNode(ISD::SRA, DL, XType, N0, ShiftAmt); in foldSelectCCToShiftAnd()
23944 Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift); in foldSelectCCToShiftAnd()
23948 if (CC == ISD::SETGT) in foldSelectCCToShiftAnd()
23951 return DAG.getNode(ISD::AND, DL, AType, Shift, N2); in foldSelectCCToShiftAnd()
24010 bool IsFabs = N->getOpcode() == ISD::FABS; in foldSignChangeInBitcast()
24013 if (IsFree || N0.getOpcode() != ISD::BITCAST || !N0.hasOneUse()) in foldSignChangeInBitcast()
24040 Int = DAG.getNode(IsFabs ? ISD::AND : ISD::XOR, DL, IntVT, Int, in foldSignChangeInBitcast()
24052 ISD::CondCode CC) { in convertSelectOfFPConstantsToLoadOffset()
24065 if (TLI.getOperationAction(ISD::ConstantFP, VT) == TargetLowering::Legal || in convertSelectOfFPConstantsToLoadOffset()
24096 CPIdx = DAG.getNode(ISD::ADD, DL, CPIdx.getValueType(), CPIdx, CstOffset); in convertSelectOfFPConstantsToLoadOffset()
24106 SDValue N2, SDValue N3, ISD::CondCode CC, in SimplifySelectCC()
24141 if (CC == ISD::SETEQ && N0->getOpcode() == ISD::AND && in SimplifySelectCC()
24153 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N0), VT, AndLHS, ShlAmt); in SimplifySelectCC()
24160 SDValue Shr = DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl, ShrAmt); in SimplifySelectCC()
24162 return DAG.getNode(ISD::AND, DL, VT, Shr, N3); in SimplifySelectCC()
24174 (!LegalOperations || TLI.isOperationLegal(ISD::SETCC, CmpOpVT))) { in SimplifySelectCC()
24177 CC = ISD::getSetCCInverse(CC, CmpOpVT); in SimplifySelectCC()
24193 Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2), VT, SCC); in SimplifySelectCC()
24196 Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2), VT, SCC); in SimplifySelectCC()
24210 return DAG.getNode(ISD::SHL, DL, N2.getValueType(), Temp, in SimplifySelectCC()
24223 if (N1C && N1C->isZero() && (CC == ISD::SETEQ || CC == ISD::SETNE)) { in SimplifySelectCC()
24227 if (CC == ISD::SETNE) in SimplifySelectCC()
24234 if ((Count.getOpcode() == ISD::CTTZ || in SimplifySelectCC()
24235 Count.getOpcode() == ISD::CTTZ_ZERO_UNDEF) && in SimplifySelectCC()
24237 (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ, VT))) in SimplifySelectCC()
24238 return DAG.getNode(ISD::CTTZ, DL, VT, N0); in SimplifySelectCC()
24241 if ((Count.getOpcode() == ISD::CTLZ || in SimplifySelectCC()
24242 Count.getOpcode() == ISD::CTLZ_ZERO_UNDEF) && in SimplifySelectCC()
24244 (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ, VT))) in SimplifySelectCC()
24245 return DAG.getNode(ISD::CTLZ, DL, VT, N0); in SimplifySelectCC()
24254 ((N1C->isAllOnes() && CC == ISD::SETGT) || in SimplifySelectCC()
24255 (N1C->isZero() && CC == ISD::SETLT)) && in SimplifySelectCC()
24258 ISD::SRA, DL, CmpOpVT, N0, in SimplifySelectCC()
24260 return DAG.getNode(ISD::XOR, DL, VT, DAG.getSExtOrTrunc(ASR, DL, VT), in SimplifySelectCC()
24261 DAG.getSExtOrTrunc(CC == ISD::SETLT ? N3 : N2, DL, VT)); in SimplifySelectCC()
24274 ISD::CondCode Cond, const SDLoc &DL, in SimplifySetCC()
24367 SDValue Ctlz = DAG.getNode(ISD::CTLZ, DL, VT, V); in BuildLogBase2()
24369 SDValue LogBase2 = DAG.getNode(ISD::SUB, DL, VT, Base, Ctlz); in BuildLogBase2()
24414 MulEst = DAG.getNode(ISD::FMUL, DL, VT, N, Est, Flags); in BuildDivEstimate()
24418 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Op, MulEst, Flags); in BuildDivEstimate()
24421 NewEst = DAG.getNode(ISD::FSUB, DL, VT, in BuildDivEstimate()
24425 NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags); in BuildDivEstimate()
24428 Est = DAG.getNode(ISD::FADD, DL, VT, MulEst, NewEst, Flags); in BuildDivEstimate()
24433 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, N, Flags); in BuildDivEstimate()
24458 SDValue HalfArg = DAG.getNode(ISD::FMUL, DL, VT, ThreeHalves, Arg, Flags); in buildSqrtNROneConst()
24459 HalfArg = DAG.getNode(ISD::FSUB, DL, VT, HalfArg, Arg, Flags); in buildSqrtNROneConst()
24463 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, Est, Flags); in buildSqrtNROneConst()
24464 NewEst = DAG.getNode(ISD::FMUL, DL, VT, HalfArg, NewEst, Flags); in buildSqrtNROneConst()
24465 NewEst = DAG.getNode(ISD::FSUB, DL, VT, ThreeHalves, NewEst, Flags); in buildSqrtNROneConst()
24466 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags); in buildSqrtNROneConst()
24471 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, Arg, Flags); in buildSqrtNROneConst()
24496 SDValue AE = DAG.getNode(ISD::FMUL, DL, VT, Arg, Est, Flags); in buildSqrtNRTwoConst()
24497 SDValue AEE = DAG.getNode(ISD::FMUL, DL, VT, AE, Est, Flags); in buildSqrtNRTwoConst()
24498 SDValue RHS = DAG.getNode(ISD::FADD, DL, VT, AEE, MinusThree, Flags); in buildSqrtNRTwoConst()
24506 LHS = DAG.getNode(ISD::FMUL, DL, VT, Est, MinusHalf, Flags); in buildSqrtNRTwoConst()
24509 LHS = DAG.getNode(ISD::FMUL, DL, VT, AE, MinusHalf, Flags); in buildSqrtNRTwoConst()
24512 Est = DAG.getNode(ISD::FMUL, DL, VT, LHS, RHS, Flags); in buildSqrtNRTwoConst()
24561 Test.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT, DL, VT, in buildSqrtEstimateImpl()
24594 Offset = (LSN->getAddressingMode() == ISD::PRE_INC) in mayAlias()
24596 : (LSN->getAddressingMode() == ISD::PRE_DEC) in mayAlias()
24729 case ISD::EntryToken: in GatherAllAliases()
24733 case ISD::LOAD: in GatherAllAliases()
24734 case ISD::STORE: { in GatherAllAliases()
24748 case ISD::CopyFromReg: in GatherAllAliases()
24753 case ISD::LIFETIME_START: in GatherAllAliases()
24754 case ISD::LIFETIME_END: { in GatherAllAliases()
24791 if (Chain.getOpcode() == ISD::TokenFactor) { in GatherAllAliases()