Lines Matching refs:VT
258 for (MVT VT : MVT::all_valuetypes()) in DAGCombiner() local
259 if (EVT(VT).isSimple() && VT != MVT::Other && in DAGCombiner()
260 TLI.isTypeLegal(EVT(VT)) && in DAGCombiner()
261 VT.getSizeInBits().getKnownMinValue() >= MaximumLegalStoreInBits) in DAGCombiner()
262 MaximumLegalStoreInBits = VT.getSizeInBits().getKnownMinValue(); in DAGCombiner()
403 SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS,
576 EVT VT, SDValue N0, SDValue N1,
604 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
614 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
835 bool hasOperation(unsigned Opcode, EVT VT) { in hasOperation() argument
836 return TLI.isOperationLegalOrCustom(Opcode, VT, LegalOperations); in hasOperation()
854 bool isTypeLegal(const EVT &VT) { in isTypeLegal() argument
856 return TLI.isTypeLegal(VT); in isTypeLegal()
860 EVT getSetCCResultType(EVT VT) const { in getSetCCResultType()
861 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in getSetCCResultType()
912 bool isOperationLegalOrCustom(unsigned Op, EVT VT, in isOperationLegalOrCustom() argument
914 return TLI.isOperationLegalOrCustom(Op, VT, LegalOnly); in isOperationLegalOrCustom()
967 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand) { in getNode() argument
971 return DAG.getNode(VPOpcode, DL, VT, in getNode()
975 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, in getNode() argument
980 return DAG.getNode(VPOpcode, DL, VT, in getNode()
984 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, in getNode() argument
989 return DAG.getNode(VPOpcode, DL, VT, in getNode()
993 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand, in getNode() argument
998 return DAG.getNode(VPOpcode, DL, VT, {Operand, RootMaskOp, RootVectorLenOp}, in getNode()
1002 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, in getNode() argument
1007 return DAG.getNode(VPOpcode, DL, VT, {N1, N2, RootMaskOp, RootVectorLenOp}, in getNode()
1011 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, in getNode() argument
1016 return DAG.getNode(VPOpcode, DL, VT, in getNode()
1020 bool isOperationLegalOrCustom(unsigned Op, EVT VT, in isOperationLegalOrCustom() argument
1023 return TLI.isOperationLegalOrCustom(VPOp, VT, LegalOnly); in isOperationLegalOrCustom()
1243 EVT VT = LoadStore->getMemoryVT(); in reassociationCanBreakAddressingModePattern() local
1245 Type *AccessTy = VT.getTypeForEVT(*DAG.getContext()); in reassociationCanBreakAddressingModePattern()
1270 EVT VT = LoadStore->getMemoryVT(); in reassociationCanBreakAddressingModePattern() local
1272 Type *AccessTy = VT.getTypeForEVT(*DAG.getContext()); in reassociationCanBreakAddressingModePattern()
1287 EVT VT = N0.getValueType(); in reassociateOpsCommutative() local
1298 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, {N01, N1})) in reassociateOpsCommutative()
1299 return DAG.getNode(Opc, DL, VT, N00, OpNode); in reassociateOpsCommutative()
1309 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N00, N1, NewFlags); in reassociateOpsCommutative()
1310 return DAG.getNode(Opc, DL, VT, OpNode, N01, NewFlags); in reassociateOpsCommutative()
1335 if (SDNode *NE = DAG.getNodeIfExists(Opc, DAG.getVTList(VT), {N00, N1})) { in reassociateOpsCommutative()
1338 if (!DAG.doesNodeExist(Opc, DAG.getVTList(VT), {SDValue(NE, 0), N01})) in reassociateOpsCommutative()
1339 return DAG.getNode(Opc, DL, VT, SDValue(NE, 0), N01); in reassociateOpsCommutative()
1345 if (SDNode *NE = DAG.getNodeIfExists(Opc, DAG.getVTList(VT), {N01, N1})) { in reassociateOpsCommutative()
1348 if (!DAG.doesNodeExist(Opc, DAG.getVTList(VT), {SDValue(NE, 0), N00})) in reassociateOpsCommutative()
1349 return DAG.getNode(Opc, DL, VT, SDValue(NE, 0), N00); in reassociateOpsCommutative()
1367 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N00, N1, Flags); in reassociateOpsCommutative()
1368 return DAG.getNode(Opc, DL, VT, OpNode, N01, Flags); in reassociateOpsCommutative()
1371 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N01, N1, Flags); in reassociateOpsCommutative()
1372 return DAG.getNode(Opc, DL, VT, OpNode, N00, Flags); in reassociateOpsCommutative()
1403 const SDLoc &DL, EVT VT, SDValue N0, in reassociateReduction() argument
1411 return DAG.getNode(RedOpc, DL, VT, in reassociateReduction()
1504 EVT VT = Load->getValueType(0); in ReplaceLoadWithPromotedLoad() local
1505 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0)); in ReplaceLoadWithPromotedLoad()
1592 EVT VT = Op.getValueType(); in PromoteIntBinOp() local
1593 if (VT.isVector() || !VT.isInteger()) in PromoteIntBinOp()
1599 if (TLI.isTypeDesirableForOp(Opc, VT)) in PromoteIntBinOp()
1602 EVT PVT = VT; in PromoteIntBinOp()
1606 assert(PVT != VT && "Don't know what type to promote to!"); in PromoteIntBinOp()
1620 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1)); in PromoteIntBinOp()
1660 EVT VT = Op.getValueType(); in PromoteIntShiftOp() local
1661 if (VT.isVector() || !VT.isInteger()) in PromoteIntShiftOp()
1667 if (TLI.isTypeDesirableForOp(Opc, VT)) in PromoteIntShiftOp()
1670 EVT PVT = VT; in PromoteIntShiftOp()
1674 assert(PVT != VT && "Don't know what type to promote to!"); in PromoteIntShiftOp()
1693 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1)); in PromoteIntShiftOp()
1709 EVT VT = Op.getValueType(); in PromoteExtend() local
1710 if (VT.isVector() || !VT.isInteger()) in PromoteExtend()
1716 if (TLI.isTypeDesirableForOp(Opc, VT)) in PromoteExtend()
1719 EVT PVT = VT; in PromoteExtend()
1723 assert(PVT != VT && "Don't know what type to promote to!"); in PromoteExtend()
1728 return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0)); in PromoteExtend()
1740 EVT VT = Op.getValueType(); in PromoteLoad() local
1741 if (VT.isVector() || !VT.isInteger()) in PromoteLoad()
1747 if (TLI.isTypeDesirableForOp(Opc, VT)) in PromoteLoad()
1750 EVT PVT = VT; in PromoteLoad()
1754 assert(PVT != VT && "Don't know what type to promote to!"); in PromoteLoad()
1765 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD); in PromoteLoad()
2421 EVT VT; in canFoldInAddressingMode() local
2427 VT = LD->getMemoryVT(); in canFoldInAddressingMode()
2432 VT = ST->getMemoryVT(); in canFoldInAddressingMode()
2437 VT = LD->getMemoryVT(); in canFoldInAddressingMode()
2442 VT = ST->getMemoryVT(); in canFoldInAddressingMode()
2472 VT.getTypeForEVT(*DAG.getContext()), AS); in canFoldInAddressingMode()
2498 EVT VT = N->getValueType(0); in foldSelectWithIdentityConstant() local
2508 SDValue NewBO = DAG.getNode(Opcode, SDLoc(N), VT, F0, FVal, N->getFlags()); in foldSelectWithIdentityConstant()
2509 return DAG.getSelect(SDLoc(N), VT, Cond, F0, NewBO); in foldSelectWithIdentityConstant()
2514 SDValue NewBO = DAG.getNode(Opcode, SDLoc(N), VT, F0, TVal, N->getFlags()); in foldSelectWithIdentityConstant()
2515 return DAG.getSelect(SDLoc(N), VT, Cond, NewBO, F0); in foldSelectWithIdentityConstant()
2527 EVT VT = BO->getValueType(0); in foldBinOpIntoSelect() local
2528 if (TLI.shouldFoldSelectWithIdentityConstant(BinOpcode, VT)) { in foldBinOpIntoSelect()
2608 NewCT = SelOpNo ? DAG.FoldConstantArithmetic(BinOpcode, DL, VT, {CBO, CT}) in foldBinOpIntoSelect()
2609 : DAG.FoldConstantArithmetic(BinOpcode, DL, VT, {CT, CBO}); in foldBinOpIntoSelect()
2613 NewCF = SelOpNo ? DAG.FoldConstantArithmetic(BinOpcode, DL, VT, {CBO, CF}) in foldBinOpIntoSelect()
2614 : DAG.FoldConstantArithmetic(BinOpcode, DL, VT, {CF, CBO}); in foldBinOpIntoSelect()
2619 SDValue SelectOp = DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF); in foldBinOpIntoSelect()
2655 EVT VT = C.getValueType(); in foldAddSubBoolOfMaskedVal() local
2657 SDValue LowBit = DAG.getZExtOrTrunc(SetCC.getOperand(0), DL, VT); in foldAddSubBoolOfMaskedVal()
2658 SDValue C1 = IsAdd ? DAG.getConstant(CN->getAPIntValue() + 1, DL, VT) : in foldAddSubBoolOfMaskedVal()
2659 DAG.getConstant(CN->getAPIntValue() - 1, DL, VT); in foldAddSubBoolOfMaskedVal()
2660 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit); in foldAddSubBoolOfMaskedVal()
2684 EVT VT = ShiftOp.getValueType(); in foldAddSubOfSignBit() local
2687 if (!ShAmtC || ShAmtC->getAPIntValue() != (VT.getScalarSizeInBits() - 1)) in foldAddSubOfSignBit()
2695 IsAdd ? ISD::ADD : ISD::SUB, DL, VT, in foldAddSubOfSignBit()
2696 {ConstantOp, DAG.getConstant(1, DL, VT)})) { in foldAddSubOfSignBit()
2697 SDValue NewShift = DAG.getNode(IsAdd ? ISD::SRA : ISD::SRL, DL, VT, in foldAddSubOfSignBit()
2699 return DAG.getNode(ISD::ADD, DL, VT, NewShift, NewC); in foldAddSubOfSignBit()
2717 EVT VT = N0.getValueType(); in visitADDLike() local
2727 if (SDValue C = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N0, N1})) in visitADDLike()
2733 return DAG.getNode(ISD::ADD, DL, VT, N1, N0); in visitADDLike()
2736 return DAG.getConstant(APInt::getAllOnes(VT.getScalarSizeInBits()), in visitADDLike()
2737 SDLoc(N), VT); in visitADDLike()
2740 if (VT.isVector()) { in visitADDLike()
2758 if (SDValue Sub = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N1, N01})) in visitADDLike()
2759 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Sub); in visitADDLike()
2762 if (SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N1, N00})) in visitADDLike()
2763 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1)); in visitADDLike()
2775 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) && in visitADDLike()
2778 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not); in visitADDLike()
2788 if (SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N1, N01})) in visitADDLike()
2789 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add); in visitADDLike()
2817 ISD::ADD, DL, VT, in visitADDLike()
2818 DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)), in visitADDLike()
2830 reassociateReduction(ISD::VECREDUCE_ADD, ISD::ADD, DL, VT, N0, N1)) in visitADDLike()
2835 return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1)); in visitADDLike()
2839 return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1)); in visitADDLike()
2852 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), in visitADDLike()
2858 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), in visitADDLike()
2864 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), in visitADDLike()
2870 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), in visitADDLike()
2877 return DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0).getOperand(0), in visitADDLike()
2889 return DAG.getNode(ISD::SUB, DL, VT, in visitADDLike()
2890 DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10), in visitADDLike()
2891 DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11)); in visitADDLike()
2895 if (N0.getOpcode() == ISD::UMAX && hasOperation(ISD::USUBSAT, VT)) { in visitADDLike()
2902 return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0), in visitADDLike()
2912 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in visitADDLike()
2928 return DAG.getNode(ISD::SUB, DL, VT, A, Xor.getOperand(0)); in visitADDLike()
2935 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD && in visitADDLike()
2940 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0), in visitADDLike()
2941 DAG.getAllOnesConstant(DL, VT)); in visitADDLike()
2942 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(1), Not); in visitADDLike()
2949 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), N1); in visitADDLike()
2950 return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0)); in visitADDLike()
2965 EVT VT = N0.getValueType(); in visitADD() local
2978 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) && in visitADD()
2980 return DAG.getNode(ISD::OR, DL, VT, N0, N1); in visitADD()
2986 return DAG.getVScale(DL, VT, C0 + C1); in visitADD()
2995 SDValue VS = DAG.getVScale(DL, VT, VS0 + VS1); in visitADD()
2996 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), VS); in visitADD()
3005 return DAG.getStepVector(DL, VT, NewStep); in visitADD()
3015 SDValue SV = DAG.getStepVector(DL, VT, NewStep); in visitADD()
3016 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), SV); in visitADD()
3026 EVT VT = N0.getValueType(); in visitADDSAT() local
3032 return DAG.getAllOnesConstant(DL, VT); in visitADDSAT()
3035 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1})) in visitADDSAT()
3041 return DAG.getNode(Opcode, DL, VT, N1, N0); in visitADDSAT()
3044 if (VT.isVector()) { in visitADDSAT()
3059 return DAG.getNode(ISD::ADD, DL, VT, N0, N1); in visitADDSAT()
3098 EVT VT = V->getValueType(0); in getAsCarry() local
3099 if (!TLI.isOperationLegalOrCustom(V.getOpcode(), VT)) in getAsCarry()
3124 EVT VT = N0.getValueType(); in foldAddSubMasked1() local
3126 if (N10.getValueType() != VT && N10.getOpcode() == ISD::TRUNCATE) in foldAddSubMasked1()
3129 if (N10.getValueType() != VT) in foldAddSubMasked1()
3132 if (DAG.ComputeNumSignBits(N10) != VT.getScalarSizeInBits()) in foldAddSubMasked1()
3137 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, N0, N10); in foldAddSubMasked1()
3143 EVT VT = N0.getValueType(); in visitADDLikeCommutative() local
3149 return DAG.getNode(ISD::SUB, DL, VT, N0, in visitADDLikeCommutative()
3150 DAG.getNode(ISD::SHL, DL, VT, in visitADDLikeCommutative()
3161 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD && in visitADDLikeCommutative()
3166 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0), in visitADDLikeCommutative()
3167 DAG.getAllOnesConstant(DL, VT)); in visitADDLikeCommutative()
3168 return DAG.getNode(ISD::SUB, DL, VT, N1, Not); in visitADDLikeCommutative()
3176 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), N1); in visitADDLikeCommutative()
3177 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1)); in visitADDLikeCommutative()
3182 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1)); in visitADDLikeCommutative()
3183 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(0)); in visitADDLikeCommutative()
3191 SDValue NewC = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), in visitADDLikeCommutative()
3192 DAG.getConstant(1, DL, VT)); in visitADDLikeCommutative()
3193 return DAG.getNode(ISD::MUL, DL, VT, N0.getOperand(0), NewC); in visitADDLikeCommutative()
3201 TLI.getBooleanContents(VT) == TargetLowering::ZeroOrOneBooleanContent) { in visitADDLikeCommutative()
3202 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitADDLikeCommutative()
3203 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt); in visitADDLikeCommutative()
3210 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0), in visitADDLikeCommutative()
3211 DAG.getConstant(1, DL, VT)); in visitADDLikeCommutative()
3212 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt); in visitADDLikeCommutative()
3223 if (TLI.isOperationLegalOrCustom(ISD::UADDO_CARRY, VT)) in visitADDLikeCommutative()
3226 DAG.getVTList(VT, Carry.getValueType()), N0, in visitADDLikeCommutative()
3227 DAG.getConstant(0, DL, VT), Carry); in visitADDLikeCommutative()
3235 EVT VT = N0.getValueType(); in visitADDC() local
3240 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDC()
3256 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDC()
3282 EVT VT = V.getValueType(); in extractBooleanFlip() local
3285 switch(TLI.getBooleanContents(VT)) { in extractBooleanFlip()
3307 EVT VT = N0.getValueType(); in visitADDO() local
3315 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDO()
3329 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDO()
3336 DAG.getConstant(0, DL, VT), N0.getOperand(0)); in visitADDO()
3341 DAG.getConstant(0, DL, VT), N0.getOperand(0)); in visitADDO()
3357 EVT VT = N0.getValueType(); in visitUADDOLike() local
3358 if (VT.isVector()) in visitUADDOLike()
3372 if (TLI.isOperationLegalOrCustom(ISD::UADDO_CARRY, VT)) in visitUADDOLike()
3375 DAG.getConstant(0, SDLoc(N), VT), Carry); in visitUADDOLike()
3420 EVT VT = N0.getValueType(); in visitUADDO_CARRY() local
3422 SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT); in visitUADDO_CARRY()
3424 return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt, in visitUADDO_CARRY()
3425 DAG.getConstant(1, DL, VT)), in visitUADDO_CARRY()
3491 EVT VT = Combiner.getSetCCResultType(Carry0.getValueType()); in combineUADDO_CARRYDiamond() local
3492 Z = DAG.getConstant(1, SDLoc(Carry0.getOperand(1)), VT); in combineUADDO_CARRYDiamond()
3792 static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT, in tryFoldToZero() argument
3794 if (!VT.isVector()) in tryFoldToZero()
3795 return DAG.getConstant(0, DL, VT); in tryFoldToZero()
3796 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) in tryFoldToZero()
3797 return DAG.getConstant(0, DL, VT); in tryFoldToZero()
3804 EVT VT = N0.getValueType(); in visitSUB() local
3816 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations); in visitSUB()
3819 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N1})) in visitSUB()
3823 if (VT.isVector()) { in visitSUB()
3839 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
3840 DAG.getConstant(-N1C->getAPIntValue(), DL, VT)); in visitSUB()
3844 unsigned BitWidth = VT.getScalarSizeInBits(); in visitSUB()
3853 if (!LegalOperations || TLI.isOperationLegal(NewSh, VT)) in visitSUB()
3854 return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1)); in visitSUB()
3874 !TLI.isOperationLegalOrCustom(ISD::ABS, VT)) in visitSUB()
3879 if (VT.isVector()) { in visitSUB()
3883 return DAG.getSplat(VT, DL, N1S.getOperand(1)); in visitSUB()
3889 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitSUB()
3893 return DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(1)); in visitSUB()
3910 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N01, N1})) in visitSUB()
3911 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), NewC); in visitSUB()
3917 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N11})) in visitSUB()
3918 return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0)); in visitSUB()
3924 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N01, N1})) in visitSUB()
3925 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), NewC); in visitSUB()
3931 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N00, N1})) in visitSUB()
3932 return DAG.getNode(ISD::SUB, DL, VT, NewC, N0.getOperand(1)); in visitSUB()
3940 return DAG.getNode(N0.getOperand(1).getOpcode(), DL, VT, N0.getOperand(0), in visitSUB()
3946 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), in visitSUB()
3952 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), in visitSUB()
3957 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
3958 DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(1), in visitSUB()
3970 DAG.getNode(ISD::XOR, DL, VT, B, DAG.getAllOnesConstant(DL, VT)); in visitSUB()
3971 return DAG.getNode(ISD::AND, DL, VT, A, InvB); in visitSUB()
3979 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, in visitSUB()
3982 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul); in visitSUB()
3986 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, in visitSUB()
3989 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul); in visitSUB()
4008 if (SDValue V = foldSubToUSubSat(VT, N)) in visitSUB()
4013 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), in visitSUB()
4014 DAG.getAllOnesConstant(DL, VT)); in visitSUB()
4015 return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0)); in visitSUB()
4022 if (TLI.preferIncOfAddToSubOfNot(VT) && N1.hasOneUse() && isBitwiseNot(N1)) { in visitSUB()
4023 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(0)); in visitSUB()
4024 return DAG.getNode(ISD::ADD, DL, VT, Add, DAG.getConstant(1, DL, VT)); in visitSUB()
4031 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1); in visitSUB()
4032 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(1)); in visitSUB()
4037 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(0)); in visitSUB()
4038 return DAG.getNode(ISD::SUB, DL, VT, Sub, N1.getOperand(1)); in visitSUB()
4044 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1); in visitSUB()
4045 return DAG.getNode(ISD::SUB, DL, VT, Sub, N0.getOperand(1)); in visitSUB()
4050 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), N1); in visitSUB()
4051 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), Add); in visitSUB()
4059 TLI.getBooleanContents(VT) == in visitSUB()
4061 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N1.getOperand(0)); in visitSUB()
4062 return DAG.getNode(ISD::ADD, DL, VT, N0, SExt); in visitSUB()
4066 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) { in visitSUB()
4072 if (C->getAPIntValue() == (VT.getScalarSizeInBits() - 1)) in visitSUB()
4073 return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0); in visitSUB()
4084 DL, VT); in visitSUB()
4091 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0), in visitSUB()
4092 DAG.getConstant(1, DL, VT)); in visitSUB()
4093 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt); in visitSUB()
4100 return DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getVScale(DL, VT, -IntVal)); in visitSUB()
4106 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
4107 DAG.getStepVector(DL, VT, NewStep)); in visitSUB()
4117 SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt); in visitSUB()
4118 return DAG.getNode(ISD::ADD, DL, VT, N0, SRA); in visitSUB()
4127 if (ShlC && ShlC->getAPIntValue() == VT.getScalarSizeInBits() - 1) in visitSUB()
4128 return DAG.getNode(ISD::ADD, DL, VT, N1, N0); in visitSUB()
4137 if (TLI.isOperationLegalOrCustom(ISD::UADDO_CARRY, VT)) { in visitSUB()
4141 SDValue Zero = DAG.getConstant(0, DL, VT); in visitSUB()
4142 SDValue NegX = DAG.getNode(ISD::SUB, DL, VT, Zero, X); in visitSUB()
4144 DAG.getVTList(VT, Carry.getValueType()), NegX, Zero, in visitSUB()
4156 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitSUB()
4169 if (!hasOperation(Abd, VT)) in visitSUB()
4171 return DAG.getNode(Abd, DL, VT, N0.getOperand(0), N0.getOperand(1)); in visitSUB()
4185 EVT VT = N0.getValueType(); in visitSUBSAT() local
4191 return DAG.getConstant(0, DL, VT); in visitSUBSAT()
4195 return DAG.getConstant(0, DL, VT); in visitSUBSAT()
4198 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1})) in visitSUBSAT()
4202 if (VT.isVector()) { in visitSUBSAT()
4217 return DAG.getNode(ISD::SUB, DL, VT, N0, N1); in visitSUBSAT()
4225 EVT VT = N0.getValueType(); in visitSUBC() local
4230 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitSUBC()
4235 return CombineTo(N, DAG.getConstant(0, DL, VT), in visitSUBC()
4244 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0), in visitSUBC()
4253 EVT VT = N0.getValueType(); in visitSUBO() local
4261 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitSUBO()
4266 return CombineTo(N, DAG.getConstant(0, DL, VT), in visitSUBO()
4274 DAG.getConstant(-N1C->getAPIntValue(), DL, VT)); in visitSUBO()
4283 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitSUBO()
4288 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0), in visitSUBO()
4342 EVT VT = N0.getValueType(); in visitMULFIX() local
4346 return DAG.getConstant(0, SDLoc(N), VT); in visitMULFIX()
4351 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0, Scale); in visitMULFIX()
4355 return DAG.getConstant(0, SDLoc(N), VT); in visitMULFIX()
4363 EVT VT = N0.getValueType(); in visitMUL() local
4368 return DAG.getConstant(0, DL, VT); in visitMUL()
4371 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MUL, DL, VT, {N0, N1})) in visitMUL()
4377 return DAG.getNode(ISD::MUL, DL, VT, N1, N0); in visitMUL()
4384 if (VT.isVector()) { in visitMUL()
4390 ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && in visitMUL()
4413 return DAG.getNegative(N0, DL, VT); in visitMUL()
4417 (!VT.isVector() || Level <= AfterLegalizeVectorOps)) { in visitMUL()
4421 return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc); in visitMUL()
4432 return DAG.getNode(ISD::SUB, DL, VT, in visitMUL()
4433 DAG.getConstant(0, DL, VT), in visitMUL()
4434 DAG.getNode(ISD::SHL, DL, VT, N0, in visitMUL()
4441 if (!LegalOperations || TLI.isOperationLegalOrCustom(LoHiOpc, VT)) { in visitMUL()
4442 SDVTList LoHiVT = DAG.getVTList(VT, VT); in visitMUL()
4468 if (N1IsConst && TLI.decomposeMulByConstant(*DAG.getContext(), VT, N1)) { in visitMUL()
4486 assert(ShAmt < VT.getScalarSizeInBits() && in visitMUL()
4489 DAG.getNode(ISD::SHL, DL, VT, N0, DAG.getConstant(ShAmt, DL, VT)); in visitMUL()
4491 TZeros ? DAG.getNode(MathOp, DL, VT, Shl, in visitMUL()
4492 DAG.getNode(ISD::SHL, DL, VT, N0, in visitMUL()
4493 DAG.getConstant(TZeros, DL, VT))) in visitMUL()
4494 : DAG.getNode(MathOp, DL, VT, Shl, N0); in visitMUL()
4496 R = DAG.getNegative(R, DL, VT); in visitMUL()
4504 if (SDValue C3 = DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {N1, N01})) in visitMUL()
4505 return DAG.getNode(ISD::MUL, DL, VT, N0.getOperand(0), C3); in visitMUL()
4524 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, Sh.getOperand(0), Y); in visitMUL()
4525 return DAG.getNode(ISD::SHL, DL, VT, Mul, Sh.getOperand(1)); in visitMUL()
4535 ISD::ADD, DL, VT, in visitMUL()
4536 DAG.getNode(ISD::MUL, SDLoc(N0), VT, N0.getOperand(0), N1), in visitMUL()
4537 DAG.getNode(ISD::MUL, SDLoc(N1), VT, N0.getOperand(1), N1)); in visitMUL()
4544 return DAG.getVScale(DL, VT, C0 * C1); in visitMUL()
4553 return DAG.getStepVector(DL, VT, NewStep); in visitMUL()
4560 if (VT.isFixedLengthVector()) { in visitMUL()
4561 unsigned NumElts = VT.getVectorNumElements(); in visitMUL()
4572 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::AND, VT)) && in visitMUL()
4582 return DAG.getNode(ISD::AND, DL, VT, N0, DAG.getBuildVector(VT, DL, Mask)); in visitMUL()
4592 reassociateReduction(ISD::VECREDUCE_MUL, ISD::MUL, DL, VT, N0, N1)) in visitMUL()
4631 EVT VT = Node->getValueType(0); in useDivRem() local
4632 if (VT.isVector() || !VT.isInteger()) in useDivRem()
4635 if (!TLI.isTypeLegal(VT) && !TLI.isOperationCustom(DivRemOpc, VT)) in useDivRem()
4640 if (!TLI.isOperationLegalOrCustom(DivRemOpc, VT) && in useDivRem()
4648 if (TLI.isOperationLegalOrCustom(Opcode, VT)) in useDivRem()
4652 if (TLI.isOperationLegalOrCustom(OtherOpcode, VT)) in useDivRem()
4672 SDVTList VTs = DAG.getVTList(VT, VT); in useDivRem()
4693 EVT VT = N->getValueType(0); in simplifyDivRem() local
4706 return DAG.getUNDEF(VT); in simplifyDivRem()
4711 return DAG.getConstant(0, DL, VT); in simplifyDivRem()
4722 return DAG.getConstant(IsDiv ? 1 : 0, DL, VT); in simplifyDivRem()
4730 if ((N1C && N1C->isOne()) || (VT.getScalarType() == MVT::i1)) in simplifyDivRem()
4731 return IsDiv ? N0 : DAG.getConstant(0, DL, VT); in simplifyDivRem()
4739 EVT VT = N->getValueType(0); in visitSDIV() local
4740 EVT CCVT = getSetCCResultType(VT); in visitSDIV()
4744 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, {N0, N1})) in visitSDIV()
4748 if (VT.isVector()) in visitSDIV()
4755 return DAG.getNegative(N0, DL, VT); in visitSDIV()
4759 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ), in visitSDIV()
4760 DAG.getConstant(1, DL, VT), in visitSDIV()
4761 DAG.getConstant(0, DL, VT)); in visitSDIV()
4779 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1); in visitSDIV()
4780 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitSDIV()
4817 EVT VT = N->getValueType(0); in visitSDIVLike() local
4818 EVT CCVT = getSetCCResultType(VT); in visitSDIVLike()
4819 unsigned BitWidth = VT.getScalarSizeInBits(); in visitSDIVLike()
4833 SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1); in visitSDIVLike()
4840 SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0, in visitSDIVLike()
4845 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact); in visitSDIVLike()
4847 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl); in visitSDIVLike()
4849 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1); in visitSDIVLike()
4854 SDValue One = DAG.getConstant(1, DL, VT); in visitSDIVLike()
4855 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT); in visitSDIVLike()
4859 Sra = DAG.getSelect(DL, VT, IsOneOrAllOnes, N0, Sra); in visitSDIVLike()
4863 SDValue Zero = DAG.getConstant(0, DL, VT); in visitSDIVLike()
4864 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, Zero, Sra); in visitSDIVLike()
4868 SDValue Res = DAG.getSelect(DL, VT, IsNeg, Sub, Sra); in visitSDIVLike()
4887 EVT VT = N->getValueType(0); in visitUDIV() local
4888 EVT CCVT = getSetCCResultType(VT); in visitUDIV()
4892 if (SDValue C = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT, {N0, N1})) in visitUDIV()
4896 if (VT.isVector()) in visitUDIV()
4902 if (N1C && N1C->isAllOnes() && CCVT.isVector() == VT.isVector()) { in visitUDIV()
4903 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ), in visitUDIV()
4904 DAG.getConstant(1, DL, VT), in visitUDIV()
4905 DAG.getConstant(0, DL, VT)); in visitUDIV()
4919 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1); in visitUDIV()
4920 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitUDIV()
4941 EVT VT = N->getValueType(0); in visitUDIVLike() local
4951 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc); in visitUDIVLike()
4967 return DAG.getNode(ISD::SRL, DL, VT, N0, Add); in visitUDIVLike()
4997 EVT VT = N->getValueType(0); in visitREM() local
4998 EVT CCVT = getSetCCResultType(VT); in visitREM()
5004 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1})) in visitREM()
5010 CCVT.isVector() == VT.isVector()) { in visitREM()
5013 return DAG.getSelect(DL, VT, EqualsNeg1, DAG.getConstant(0, DL, VT), F0); in visitREM()
5026 return DAG.getNode(ISD::UREM, DL, VT, N0, N1); in visitREM()
5030 SDValue NegOne = DAG.getAllOnesConstant(DL, VT); in visitREM()
5031 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne); in visitREM()
5033 return DAG.getNode(ISD::AND, DL, VT, N0, Add); in visitREM()
5041 SDValue NegOne = DAG.getAllOnesConstant(DL, VT); in visitREM()
5042 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne); in visitREM()
5044 return DAG.getNode(ISD::AND, DL, VT, N0, Add); in visitREM()
5057 if (DAG.isKnownNeverZero(N1) && !TLI.isIntDivCheap(VT, Attr)) { in visitREM()
5072 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1); in visitREM()
5073 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitREM()
5090 EVT VT = N->getValueType(0); in visitMULHS() local
5094 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHS, DL, VT, {N0, N1})) in visitMULHS()
5102 if (VT.isVector()) { in visitMULHS()
5109 return DAG.getConstant(0, DL, VT); in visitMULHS()
5124 return DAG.getConstant(0, DL, VT); in visitMULHS()
5128 if (!TLI.isOperationLegalOrCustom(ISD::MULHS, VT) && VT.isSimple() && in visitMULHS()
5129 !VT.isVector()) { in visitMULHS()
5130 MVT Simple = VT.getSimpleVT(); in visitMULHS()
5140 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in visitMULHS()
5150 EVT VT = N->getValueType(0); in visitMULHU() local
5154 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHU, DL, VT, {N0, N1})) in visitMULHU()
5162 if (VT.isVector()) { in visitMULHU()
5169 return DAG.getConstant(0, DL, VT); in visitMULHU()
5182 return DAG.getConstant(0, DL, VT); in visitMULHU()
5186 hasOperation(ISD::SRL, VT)) { in visitMULHU()
5188 unsigned NumEltBits = VT.getScalarSizeInBits(); in visitMULHU()
5190 ISD::SUB, DL, VT, DAG.getConstant(NumEltBits, DL, VT), LogBase2); in visitMULHU()
5193 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc); in visitMULHU()
5199 if (!TLI.isOperationLegalOrCustom(ISD::MULHU, VT) && VT.isSimple() && in visitMULHU()
5200 !VT.isVector()) { in visitMULHU()
5201 MVT Simple = VT.getSimpleVT(); in visitMULHU()
5211 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in visitMULHU()
5228 EVT VT = N->getValueType(0); in visitAVG() local
5232 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1})) in visitAVG()
5240 if (VT.isVector()) { in visitAVG()
5247 return DAG.getNode(ISD::SRA, DL, VT, N0, DAG.getConstant(1, DL, VT)); in visitAVG()
5249 return DAG.getNode(ISD::SRL, DL, VT, N0, DAG.getConstant(1, DL, VT)); in visitAVG()
5272 EVT VT = N->getValueType(0); in visitABD() local
5276 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1})) in visitABD()
5284 if (VT.isVector()) { in visitABD()
5292 return DAG.getNode(ISD::ABS, DL, VT, N0); in visitABD()
5300 return DAG.getConstant(0, DL, VT); in visitABD()
5303 if (Opcode == ISD::ABDS && hasOperation(ISD::ABDU, VT) && in visitABD()
5305 return DAG.getNode(ISD::ABDU, DL, VT, N1, N0); in visitABD()
5365 EVT VT = N->getValueType(0); in visitSMUL_LOHI() local
5379 if (VT.isSimple() && !VT.isVector()) { in visitSMUL_LOHI()
5380 MVT Simple = VT.getSimpleVT(); in visitSMUL_LOHI()
5391 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi); in visitSMUL_LOHI()
5393 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo); in visitSMUL_LOHI()
5407 EVT VT = N->getValueType(0); in visitUMUL_LOHI() local
5421 SDValue Zero = DAG.getConstant(0, DL, VT); in visitUMUL_LOHI()
5427 SDValue Zero = DAG.getConstant(0, DL, VT); in visitUMUL_LOHI()
5433 if (VT.isSimple() && !VT.isVector()) { in visitUMUL_LOHI()
5434 MVT Simple = VT.getSimpleVT(); in visitUMUL_LOHI()
5445 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi); in visitUMUL_LOHI()
5447 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo); in visitUMUL_LOHI()
5458 EVT VT = N0.getValueType(); in visitMULO() local
5475 return CombineTo(N, DAG.getConstant(Result, DL, VT), in visitMULO()
5486 return CombineTo(N, DAG.getConstant(0, DL, VT), in visitMULO()
5492 (!IsSigned || VT.getScalarSizeInBits() > 2)) in visitMULO()
5497 if (IsSigned && VT.getScalarSizeInBits() == 1) { in visitMULO()
5498 SDValue And = DAG.getNode(ISD::AND, DL, VT, N0, N1); in visitMULO()
5500 DAG.getConstant(0, DL, VT), ISD::SETNE); in visitMULO()
5506 return CombineTo(N, DAG.getNode(ISD::MUL, DL, VT, N0, N1), in visitMULO()
5681 EVT VT = N0.getValueType(); in visitIMINMAX() local
5686 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1})) in visitIMINMAX()
5696 return DAG.getNode(Opcode, DL, VT, N1, N0); in visitIMINMAX()
5699 if (VT.isVector()) in visitIMINMAX()
5705 if (!TLI.isOperationLegal(Opcode, VT) && in visitIMINMAX()
5716 if (TLI.isOperationLegal(AltOpcode, VT)) in visitIMINMAX()
5717 return DAG.getNode(AltOpcode, DL, VT, N0, N1); in visitIMINMAX()
5744 SDLoc(N), VT, N0, N1)) in visitIMINMAX()
5758 EVT VT = N0.getValueType(); in hoistLogicOpWithSameOpcodeHands() local
5788 if ((VT.isVector() || LegalOperations) && in hoistLogicOpWithSameOpcodeHands()
5800 return DAG.getNode(HandOpcode, DL, VT, Logic, N0.getOperand(1)); in hoistLogicOpWithSameOpcodeHands()
5801 return DAG.getNode(HandOpcode, DL, VT, Logic); in hoistLogicOpWithSameOpcodeHands()
5818 if (TLI.isZExtFree(VT, XVT) && TLI.isTruncateFree(XVT, VT)) in hoistLogicOpWithSameOpcodeHands()
5823 return DAG.getNode(HandOpcode, DL, VT, Logic); in hoistLogicOpWithSameOpcodeHands()
5835 return DAG.getNode(HandOpcode, DL, VT, Logic, N0.getOperand(1)); in hoistLogicOpWithSameOpcodeHands()
5844 return DAG.getNode(HandOpcode, DL, VT, Logic); in hoistLogicOpWithSameOpcodeHands()
5857 SDValue Logic0 = DAG.getNode(LogicOpcode, DL, VT, X, Y); in hoistLogicOpWithSameOpcodeHands()
5858 SDValue Logic1 = DAG.getNode(LogicOpcode, DL, VT, X1, Y1); in hoistLogicOpWithSameOpcodeHands()
5859 return DAG.getNode(HandOpcode, DL, VT, Logic0, Logic1, S); in hoistLogicOpWithSameOpcodeHands()
5873 !(VT.isVector() && TLI.isTypeLegal(VT) && in hoistLogicOpWithSameOpcodeHands()
5876 return DAG.getNode(HandOpcode, DL, VT, Logic); in hoistLogicOpWithSameOpcodeHands()
5910 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations); in hoistLogicOpWithSameOpcodeHands()
5914 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT, in hoistLogicOpWithSameOpcodeHands()
5916 return DAG.getVectorShuffle(VT, DL, Logic, ShOp, SVN0->getMask()); in hoistLogicOpWithSameOpcodeHands()
5923 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations); in hoistLogicOpWithSameOpcodeHands()
5927 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT, N0.getOperand(1), in hoistLogicOpWithSameOpcodeHands()
5929 return DAG.getVectorShuffle(VT, DL, ShOp, Logic, SVN0->getMask()); in hoistLogicOpWithSameOpcodeHands()
5953 EVT VT = N0.getValueType(); in foldLogicOfSetCCs() local
5955 if (LegalOperations || VT.getScalarType() != MVT::i1) in foldLogicOfSetCCs()
5956 if (VT != getSetCCResultType(OpVT)) in foldLogicOfSetCCs()
5984 return DAG.getSetCC(DL, VT, Or, LR, CC1); in foldLogicOfSetCCs()
6003 return DAG.getSetCC(DL, VT, And, LR, CC1); in foldLogicOfSetCCs()
6017 return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE); in foldLogicOfSetCCs()
6031 return DAG.getSetCC(DL, VT, Or, Zero, CC1); in foldLogicOfSetCCs()
6055 return DAG.getSetCC(DL, VT, And, Zero, CC0); in foldLogicOfSetCCs()
6075 return DAG.getSetCC(DL, VT, LL, LR, NewCC); in foldLogicOfSetCCs()
6174 EVT VT = LogicOp->getValueType(0); in foldAndOrOfSETCC() local
6258 return DAG.getSetCC(DL, VT, MinMaxValue, CommonValue, CC); in foldAndOrOfSETCC()
6283 return DAG.getNode(ISD::SETCC, DL, VT, AbsOp, in foldAndOrOfSETCC()
6313 return DAG.getNode(ISD::SETCC, DL, VT, AndOp, in foldAndOrOfSETCC()
6321 return DAG.getNode(ISD::SETCC, DL, VT, AndOp, in foldAndOrOfSETCC()
6363 EVT VT = N1.getValueType(); in visitANDLike() local
6368 return DAG.getConstant(0, DL, VT); in visitANDLike()
6380 VT.getSizeInBits() <= 64 && N0->hasOneUse()) { in visitANDLike()
6389 if (ADDC.getSignificantBits() <= 64 && SRLC.ult(VT.getSizeInBits()) && in visitANDLike()
6391 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(), in visitANDLike()
6398 DAG.getNode(ISD::ADD, DL0, VT, in visitANDLike()
6399 N0.getOperand(0), DAG.getConstant(ADDC, DL, VT)); in visitANDLike()
6584 EVT VT = Op.getOpcode() == ISD::AssertZext ? in SearchForAndLoads() local
6590 if (ExtVT.bitsGE(VT)) in SearchForAndLoads()
6612 MVT VT = SDValue(NodeToMask, i).getSimpleValueType(); in SearchForAndLoads() local
6613 if (VT != MVT::Glue && VT != MVT::Other) { in SearchForAndLoads()
6744 EVT VT = N->getValueType(0); in unfoldExtremeBitClearingToShifts() local
6747 SDValue T0 = DAG.getNode(InnerShift, DL, VT, X, Y); in unfoldExtremeBitClearingToShifts()
6749 SDValue T1 = DAG.getNode(OuterShift, DL, VT, T0, Y); in unfoldExtremeBitClearingToShifts()
6834 EVT VT = N1.getValueType(); in foldAndToUsubsat() local
6848 unsigned BitWidth = VT.getScalarSizeInBits(); in foldAndToUsubsat()
6858 SDValue SignMask = DAG.getConstant(XorC->getAPIntValue(), DL, VT); in foldAndToUsubsat()
6859 return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0), SignMask); in foldAndToUsubsat()
6901 EVT VT = N->getValueType(0); in foldLogicOfShifts() local
6903 SDValue LogicX = DAG.getNode(LogicOpcode, DL, VT, X0, X1); in foldLogicOfShifts()
6904 SDValue NewShift = DAG.getNode(ShiftOpcode, DL, VT, LogicX, Y); in foldLogicOfShifts()
6905 return DAG.getNode(LogicOpcode, DL, VT, NewShift, Z); in foldLogicOfShifts()
6940 EVT VT = N->getValueType(0); in foldLogicTreeOfShifts() local
6942 return DAG.getNode(LogicOpcode, DL, VT, CombinedShifts, W); in foldLogicTreeOfShifts()
6948 EVT VT = N1.getValueType(); in visitAND() local
6955 if (SDValue C = DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, {N0, N1})) in visitAND()
6961 return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0); in visitAND()
6964 return DAG.getConstant(APInt::getZero(VT.getScalarSizeInBits()), SDLoc(N), in visitAND()
6965 VT); in visitAND()
6968 if (VT.isVector()) { in visitAND()
6988 EVT ExtVT = VT; in visitAND()
7016 unsigned BitWidth = VT.getScalarSizeInBits(); in visitAND()
7019 return DAG.getConstant(0, SDLoc(N), VT); in visitAND()
7033 VT, N0, N1)) in visitAND()
7053 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, N0Op0); in visitAND()
7057 TLI.isTruncateFree(VT, SrcVT) && TLI.isZExtFree(SrcVT, VT) && in visitAND()
7059 TLI.isNarrowingProfitable(VT, SrcVT)) { in visitAND()
7061 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, in visitAND()
7072 (ExtOpc != ISD::ZERO_EXTEND || !TLI.isZExtFree(N0Op0, VT)) && in visitAND()
7078 DAG.getNode(ISD::AND, DL, VT, N1, in visitAND()
7079 DAG.getNode(ExtOpc, DL, VT, N0Op0.getOperand(1))); in visitAND()
7080 return DAG.getNode(ISD::AND, DL, VT, in visitAND()
7081 DAG.getNode(ExtOpc, DL, VT, N0Op0.getOperand(0)), in visitAND()
7189 if (VT.isVector()) in visitAND()
7210 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT, ZeroExtExtendee, in visitAND()
7227 DAG.getVTList(VT, MVT::Other), MemVT, SDLoc(N), Ops, in visitAND()
7239 if (N1C && N0.getOpcode() == ISD::LOAD && !VT.isVector()) in visitAND()
7280 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0)); in visitAND()
7303 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) { in visitAND()
7305 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, LN0->getChain(), in visitAND()
7349 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, N0.getOperand(0)); in visitAND()
7351 if (hasOperation(ISD::USUBSAT, VT)) in visitAND()
7357 if (LegalOperations || VT.isVector()) in visitAND()
7370 EVT VT = N->getValueType(0); in MatchBSwapHWordLow() local
7371 if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16) in MatchBSwapHWordLow()
7373 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT)) in MatchBSwapHWordLow()
7451 unsigned OpSizeInBits = VT.getSizeInBits(); in MatchBSwapHWordLow()
7471 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00); in MatchBSwapHWordLow()
7474 Res = DAG.getNode(ISD::SRL, DL, VT, Res, in MatchBSwapHWordLow()
7476 getShiftAmountTy(VT))); in MatchBSwapHWordLow()
7594 SDValue N1, EVT VT, EVT ShiftAmountTy) { in matchBSwapHWordOrAndAnd() argument
7595 assert(N->getOpcode() == ISD::OR && VT == MVT::i32 && in matchBSwapHWordOrAndAnd()
7597 if (!TLI.isOperationLegalOrCustom(ISD::ROTR, VT)) in matchBSwapHWordOrAndAnd()
7625 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Shift0.getOperand(0)); in matchBSwapHWordOrAndAnd()
7627 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt); in matchBSwapHWordOrAndAnd()
7640 EVT VT = N->getValueType(0); in MatchBSwapHWord() local
7641 if (VT != MVT::i32) in MatchBSwapHWord()
7643 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT)) in MatchBSwapHWord()
7646 if (SDValue BSwap = matchBSwapHWordOrAndAnd(TLI, DAG, N, N0, N1, VT, in MatchBSwapHWord()
7647 getShiftAmountTy(VT))) in MatchBSwapHWord()
7651 if (SDValue BSwap = matchBSwapHWordOrAndAnd(TLI, DAG, N, N1, N0, VT, in MatchBSwapHWord()
7652 getShiftAmountTy(VT))) in MatchBSwapHWord()
7684 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, in MatchBSwapHWord()
7689 SDValue ShAmt = DAG.getConstant(16, DL, getShiftAmountTy(VT)); in MatchBSwapHWord()
7690 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT)) in MatchBSwapHWord()
7691 return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt); in MatchBSwapHWord()
7692 if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT)) in MatchBSwapHWord()
7693 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt); in MatchBSwapHWord()
7694 return DAG.getNode(ISD::OR, DL, VT, in MatchBSwapHWord()
7695 DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt), in MatchBSwapHWord()
7696 DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt)); in MatchBSwapHWord()
7702 EVT VT = N1.getValueType(); in visitORLike() local
7707 return DAG.getAllOnesConstant(DL, VT); in visitORLike()
7729 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, in visitORLike()
7731 return DAG.getNode(ISD::AND, DL, VT, X, in visitORLike()
7732 DAG.getConstant(LHSMask | RHSMask, DL, VT)); in visitORLike()
7744 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, in visitORLike()
7746 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X); in visitORLike()
7755 EVT VT = N0.getValueType(); in visitORCommutative() local
7778 return DAG.getNode(ISD::OR, SDLoc(N), VT, in visitORCommutative()
7779 DAG.getZExtOrTrunc(N00, SDLoc(N), VT), N1); in visitORCommutative()
7786 return DAG.getNode(ISD::OR, SDLoc(N), VT, in visitORCommutative()
7787 DAG.getZExtOrTrunc(N01, SDLoc(N), VT), N1); in visitORCommutative()
7797 return DAG.getNode(ISD::OR, SDLoc(N), VT, N01, N1); in visitORCommutative()
7799 return DAG.getNode(ISD::OR, SDLoc(N), VT, N00, N1); in visitORCommutative()
7805 return DAG.getNode(ISD::OR, SDLoc(N), VT, N00, N01); in visitORCommutative()
7836 EVT VT = N1.getValueType(); in visitOR() local
7843 if (SDValue C = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, {N0, N1})) in visitOR()
7849 return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0); in visitOR()
7852 if (VT.isVector()) { in visitOR()
7869 if (SV0 && SV1 && TLI.isTypeLegal(VT)) { in visitOR()
7879 int NumElts = VT.getVectorNumElements(); in visitOR()
7915 TLI.buildLegalVectorShuffle(VT, SDLoc(N), NewLHS, NewRHS, in visitOR()
7961 VT, N0, N1)) in visitOR()
7971 if (SDValue COR = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N1), VT, in visitOR()
7973 SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1); in visitOR()
7975 return DAG.getNode(ISD::AND, SDLoc(N), VT, COR, IOR); in visitOR()
8001 if ((!LegalOperations || TLI.isOperationLegal(ISD::ADD, VT)) && in visitOR()
8008 if (LegalOperations || VT.isVector()) in visitOR()
8311 EVT VT = Shifted.getValueType(); in MatchRotatePosNeg() local
8312 if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits(), DAG, in MatchRotatePosNeg()
8314 return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted, in MatchRotatePosNeg()
8332 EVT VT = N0.getValueType(); in MatchFunnelPosNeg() local
8333 unsigned EltBits = VT.getScalarSizeInBits(); in MatchFunnelPosNeg()
8343 return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, N0, N1, in MatchFunnelPosNeg()
8363 TLI.isOperationLegalOrCustom(ISD::FSHL, VT)) { in MatchFunnelPosNeg()
8364 return DAG.getNode(ISD::FSHL, DL, VT, N0, N1.getOperand(0), Pos); in MatchFunnelPosNeg()
8372 TLI.isOperationLegalOrCustom(ISD::FSHR, VT)) { in MatchFunnelPosNeg()
8373 return DAG.getNode(ISD::FSHR, DL, VT, N0.getOperand(0), N1, Neg); in MatchFunnelPosNeg()
8382 TLI.isOperationLegalOrCustom(ISD::FSHR, VT)) { in MatchFunnelPosNeg()
8383 return DAG.getNode(ISD::FSHR, DL, VT, N0.getOperand(0), N1, Neg); in MatchFunnelPosNeg()
8395 EVT VT = LHS.getValueType(); in MatchRotate() local
8400 bool HasROTL = hasOperation(ISD::ROTL, VT); in MatchRotate()
8401 bool HasROTR = hasOperation(ISD::ROTR, VT); in MatchRotate()
8402 bool HasFSHL = hasOperation(ISD::FSHL, VT); in MatchRotate()
8403 bool HasFSHR = hasOperation(ISD::FSHR, VT); in MatchRotate()
8408 if (VT.isScalarInteger() && TLI.getTypeAction(*DAG.getContext(), VT) == in MatchRotate()
8410 HasROTL |= TLI.getOperationAction(ISD::ROTL, VT) == TargetLowering::Custom; in MatchRotate()
8411 HasROTR |= TLI.getOperationAction(ISD::ROTR, VT) == TargetLowering::Custom; in MatchRotate()
8478 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in MatchRotate()
8492 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT); in MatchRotate()
8496 SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt); in MatchRotate()
8497 Mask = DAG.getNode(ISD::AND, DL, VT, Mask, in MatchRotate()
8498 DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits)); in MatchRotate()
8501 SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt); in MatchRotate()
8502 Mask = DAG.getNode(ISD::AND, DL, VT, Mask, in MatchRotate()
8503 DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits)); in MatchRotate()
8506 Res = DAG.getNode(ISD::AND, DL, VT, Res, Mask); in MatchRotate()
8515 if (TLI.isTypeLegal(VT) && LHS.hasOneUse() && RHS.hasOneUse() && in MatchRotate()
8539 SDValue RotX = DAG.getNode(ISD::ROTL, DL, VT, X, LHSShiftAmt); in MatchRotate()
8540 SDValue ShlY = DAG.getNode(ISD::SHL, DL, VT, Y, LHSShiftAmt); in MatchRotate()
8541 Res = DAG.getNode(ISD::OR, DL, VT, RotX, ShlY); in MatchRotate()
8544 SDValue RotX = DAG.getNode(ISD::ROTL, DL, VT, X, LHSShiftAmt); in MatchRotate()
8545 SDValue SrlY = DAG.getNode(ISD::SRL, DL, VT, Y, RHSShiftAmt); in MatchRotate()
8546 Res = DAG.getNode(ISD::OR, DL, VT, RotX, SrlY); in MatchRotate()
8566 Res = DAG.getNode(UseROTL ? ISD::ROTL : ISD::ROTR, DL, VT, LHSShiftArg, in MatchRotate()
8570 Res = DAG.getNode(UseFSHL ? ISD::FSHL : ISD::FSHR, DL, VT, LHSShiftArg, in MatchRotate()
9096 EVT VT = N->getValueType(0); in MatchLoadCombine() local
9097 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64) in MatchLoadCombine()
9099 unsigned ByteWidth = VT.getSizeInBits() / 8; in MatchLoadCombine()
9237 !TLI.isOperationLegal(ISD::BSWAP, VT)) in MatchLoadCombine()
9243 !TLI.isOperationLegal(ISD::SHL, VT)) in MatchLoadCombine()
9255 DAG.getExtLoad(NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, SDLoc(N), VT, in MatchLoadCombine()
9268 ? DAG.getNode(ISD::SHL, SDLoc(N), VT, NewLoad, in MatchLoadCombine()
9269 DAG.getShiftAmountConstant(ZeroExtendedBytes * 8, VT, in MatchLoadCombine()
9272 return DAG.getNode(ISD::BSWAP, SDLoc(N), VT, ShiftedLoad); in MatchLoadCombine()
9298 EVT VT = N->getValueType(0); in unfoldMaskedMerge() local
9347 SDValue NotX = DAG.getNOT(DL, X, VT); in unfoldMaskedMerge()
9348 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, NotX, M); in unfoldMaskedMerge()
9349 SDValue NotLHS = DAG.getNOT(DL, LHS, VT); in unfoldMaskedMerge()
9350 SDValue RHS = DAG.getNode(ISD::OR, DL, VT, M, Y); in unfoldMaskedMerge()
9351 return DAG.getNode(ISD::AND, DL, VT, NotLHS, RHS); in unfoldMaskedMerge()
9360 SDValue LHS = DAG.getNode(ISD::OR, DL, VT, X, NotM); in unfoldMaskedMerge()
9361 SDValue NotY = DAG.getNOT(DL, Y, VT); in unfoldMaskedMerge()
9362 SDValue RHS = DAG.getNode(ISD::AND, DL, VT, NotM, NotY); in unfoldMaskedMerge()
9363 SDValue NotRHS = DAG.getNOT(DL, RHS, VT); in unfoldMaskedMerge()
9364 return DAG.getNode(ISD::AND, DL, VT, LHS, NotRHS); in unfoldMaskedMerge()
9367 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, X, M); in unfoldMaskedMerge()
9368 SDValue NotM = DAG.getNOT(DL, M, VT); in unfoldMaskedMerge()
9369 SDValue RHS = DAG.getNode(ISD::AND, DL, VT, Y, NotM); in unfoldMaskedMerge()
9371 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS); in unfoldMaskedMerge()
9377 EVT VT = N0.getValueType(); in visitXOR() local
9382 return DAG.getConstant(0, DL, VT); in visitXOR()
9391 if (SDValue C = DAG.FoldConstantArithmetic(ISD::XOR, DL, VT, {N0, N1})) in visitXOR()
9397 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitXOR()
9400 if (VT.isVector()) { in visitXOR()
9422 reassociateReduction(ISD::VECREDUCE_XOR, ISD::XOR, DL, VT, N0, N1)) in visitXOR()
9426 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) && in visitXOR()
9428 return DAG.getNode(ISD::OR, DL, VT, N0, N1); in visitXOR()
9432 if ((!LegalOperations || TLI.isOperationLegal(ISD::ADD, VT)) && in visitXOR()
9450 return DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC); in visitXOR()
9460 DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC, in visitXOR()
9481 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, V); in visitXOR()
9485 if (isOneConstant(N1) && VT == MVT::i1 && N0.hasOneUse() && in visitXOR()
9490 N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00 in visitXOR()
9491 N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01 in visitXOR()
9493 return DAG.getNode(NewOpcode, DL, VT, N00, N01); in visitXOR()
9502 N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00 in visitXOR()
9503 N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01 in visitXOR()
9505 return DAG.getNode(NewOpcode, DL, VT, N00, N01); in visitXOR()
9514 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), in visitXOR()
9515 DAG.getAllOnesConstant(DL, VT)); in visitXOR()
9521 return DAG.getNegative(N0.getOperand(0), DL, VT); in visitXOR()
9527 SDValue NotX = DAG.getNOT(SDLoc(X), X, VT); in visitXOR()
9529 return DAG.getNode(ISD::AND, DL, VT, NotX, N1); in visitXOR()
9533 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) { in visitXOR()
9541 if (C->getAPIntValue() == (VT.getScalarSizeInBits() - 1)) in visitXOR()
9542 return DAG.getNode(ISD::ABS, DL, VT, S0); in visitXOR()
9548 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations); in visitXOR()
9568 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0Opcode == ISD::SHL && in visitXOR()
9570 return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT), in visitXOR()
9665 EVT VT = Shift->getValueType(0); in combineShiftOfShiftedLogic() local
9668 SDValue NewShift1 = DAG.getNode(ShiftOpcode, DL, VT, X, ShiftSumC); in combineShiftOfShiftedLogic()
9669 SDValue NewShift2 = DAG.getNode(ShiftOpcode, DL, VT, Y, C1); in combineShiftOfShiftedLogic()
9670 return DAG.getNode(LogicOpcode, DL, VT, NewShift1, NewShift2); in combineShiftOfShiftedLogic()
9731 EVT VT = N->getValueType(0); in visitShiftByConstant() local
9733 N->getOpcode(), DL, VT, {LHS.getOperand(1), N->getOperand(1)})) { in visitShiftByConstant()
9734 SDValue NewShift = DAG.getNode(N->getOpcode(), DL, VT, LHS.getOperand(0), in visitShiftByConstant()
9736 return DAG.getNode(LHS.getOpcode(), DL, VT, NewShift, NewRHS); in visitShiftByConstant()
9769 EVT VT = N->getValueType(0); in visitRotate() local
9770 unsigned Bitsize = VT.getScalarSizeInBits(); in visitRotate()
9794 return DAG.getNode(N->getOpcode(), dl, VT, N0, Amt); in visitRotate()
9800 VT.getScalarSizeInBits() == 16 && hasOperation(ISD::BSWAP, VT)) in visitRotate()
9801 return DAG.getNode(ISD::BSWAP, dl, VT, N0); in visitRotate()
9811 return DAG.getNode(N->getOpcode(), dl, VT, N0, NewOp1); in visitRotate()
9837 return DAG.getNode(N->getOpcode(), dl, VT, N0->getOperand(0), in visitRotate()
9851 EVT VT = N0.getValueType(); in visitSHL() local
9853 unsigned OpSizeInBits = VT.getScalarSizeInBits(); in visitSHL()
9856 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, {N0, N1})) in visitSHL()
9860 if (VT.isVector()) { in visitSHL()
9877 DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, {N01, N1})) in visitSHL()
9878 return DAG.getNode(ISD::AND, SDLoc(N), VT, N00, C); in visitSHL()
9889 return DAG.getConstant(0, SDLoc(N), VT); in visitSHL()
9895 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, NewOp1); in visitSHL()
9908 return DAG.getConstant(0, SDLoc(N), VT); in visitSHL()
9920 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum); in visitSHL()
9949 return DAG.getConstant(0, SDLoc(N), VT); in visitSHL()
9963 SDValue Ext = DAG.getNode(N0.getOpcode(), DL, VT, N0Op0.getOperand(0)); in visitSHL()
9966 return DAG.getNode(ISD::SHL, DL, VT, Ext, Sum); in visitSHL()
9978 auto MatchEqual = [VT](ConstantSDNode *LHS, ConstantSDNode *RHS) { in visitSHL()
9982 return c1.ult(VT.getScalarSizeInBits()) && (c1 == c2); in visitSHL()
9992 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL); in visitSHL()
10015 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff); in visitSHL()
10022 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0), Diff); in visitSHL()
10038 SDValue Mask = DAG.getAllOnesConstant(DL, VT); in visitSHL()
10039 Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, N01); in visitSHL()
10040 Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, Diff); in visitSHL()
10041 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Diff); in visitSHL()
10042 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSHL()
10049 SDValue Mask = DAG.getAllOnesConstant(DL, VT); in visitSHL()
10050 Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, N1); in visitSHL()
10051 SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff); in visitSHL()
10052 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSHL()
10061 SDValue AllBits = DAG.getAllOnesConstant(DL, VT); in visitSHL()
10062 SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1); in visitSHL()
10063 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), HiBitsMask); in visitSHL()
10074 DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N1), VT, {N01, N1})) { in visitSHL()
10075 SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1); in visitSHL()
10081 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, Shl0, Shl1, Flags); in visitSHL()
10095 if (SDValue ExtC = DAG.FoldConstantArithmetic(N0.getOpcode(), DL, VT, in visitSHL()
10098 DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {ExtC, N1})) { in visitSHL()
10099 SDValue ExtX = DAG.getNode(N0.getOpcode(), DL, VT, Add.getOperand(0)); in visitSHL()
10100 SDValue ShlX = DAG.getNode(ISD::SHL, DL, VT, ExtX, N1); in visitSHL()
10101 return DAG.getNode(ISD::ADD, DL, VT, ShlX, ShlC); in visitSHL()
10110 DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N1), VT, {N01, N1})) in visitSHL()
10111 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), Shl); in visitSHL()
10126 return DAG.getVScale(SDLoc(N), VT, C0 << C1); in visitSHL()
10136 return DAG.getStepVector(SDLoc(N), VT, NewStep); in visitSHL()
10267 EVT VT = N->getValueType(0); in foldBitOrderCrossLogicOp() local
10276 return DAG.getNode(N0.getOpcode(), DL, VT, OldLHS.getOperand(0), in foldBitOrderCrossLogicOp()
10281 SDValue NewBitReorder = DAG.getNode(Opcode, DL, VT, OldRHS); in foldBitOrderCrossLogicOp()
10282 return DAG.getNode(N0.getOpcode(), DL, VT, OldLHS.getOperand(0), in foldBitOrderCrossLogicOp()
10287 SDValue NewBitReorder = DAG.getNode(Opcode, DL, VT, OldLHS); in foldBitOrderCrossLogicOp()
10288 return DAG.getNode(N0.getOpcode(), DL, VT, NewBitReorder, in foldBitOrderCrossLogicOp()
10301 EVT VT = N0.getValueType(); in visitSRA() local
10302 unsigned OpSizeInBits = VT.getScalarSizeInBits(); in visitSRA()
10305 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, {N0, N1})) in visitSRA()
10315 if (VT.isVector()) in visitSRA()
10353 return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0), ShiftValue); in visitSRA()
10370 if (VT.isVector()) in visitSRA()
10371 TruncVT = EVT::getVectorVT(Ctx, TruncVT, VT.getVectorElementCount()); in visitSRA()
10382 TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) && in visitSRA()
10383 TLI.isTruncateFree(VT, TruncVT)) { in visitSRA()
10387 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, in visitSRA()
10416 if (VT.isVector()) in visitSRA()
10417 TruncVT = EVT::getVectorVT(Ctx, TruncVT, VT.getVectorElementCount()); in visitSRA()
10424 TLI.isTruncateFree(VT, TruncVT)) { in visitSRA()
10436 return DAG.getSExtOrTrunc(Add, DL, VT); in visitSRA()
10446 return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0, NewOp1); in visitSRA()
10470 return DAG.getNode(ISD::TRUNCATE, DL, VT, SRA); in visitSRA()
10481 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, N1); in visitSRA()
10505 EVT VT = N0.getValueType(); in visitSRL() local
10507 unsigned OpSizeInBits = VT.getScalarSizeInBits(); in visitSRL()
10510 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, {N0, N1})) in visitSRL()
10514 if (VT.isVector()) in visitSRL()
10525 return DAG.getConstant(0, SDLoc(N), VT); in visitSRL()
10537 return DAG.getConstant(0, SDLoc(N), VT); in visitSRL()
10549 return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Sum); in visitSRL()
10568 return DAG.getConstant(0, DL, VT); in visitSRL()
10572 return DAG.getNode(ISD::TRUNCATE, DL, VT, NewShift); in visitSRL()
10586 return DAG.getNode(ISD::TRUNCATE, DL, VT, And); in visitSRL()
10609 SDValue Mask = DAG.getAllOnesConstant(DL, VT); in visitSRL()
10610 Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, N01); in visitSRL()
10611 Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, Diff); in visitSRL()
10612 SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff); in visitSRL()
10613 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSRL()
10621 SDValue Mask = DAG.getAllOnesConstant(DL, VT); in visitSRL()
10622 Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, N1); in visitSRL()
10623 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Diff); in visitSRL()
10624 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSRL()
10635 return DAG.getUNDEF(VT); in visitSRL()
10647 return DAG.getNode(ISD::AND, DL, VT, in visitSRL()
10648 DAG.getNode(ISD::ANY_EXTEND, DL, VT, SmallShift), in visitSRL()
10649 DAG.getConstant(Mask, DL, VT)); in visitSRL()
10657 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), N1); in visitSRL()
10669 if (Known.One.getBoolValue()) return DAG.getConstant(0, SDLoc(N0), VT); in visitSRL()
10674 if (UnknownBits == 0) return DAG.getConstant(1, SDLoc(N0), VT); in visitSRL()
10687 Op = DAG.getNode(ISD::SRL, DL, VT, Op, in visitSRL()
10694 return DAG.getNode(ISD::XOR, DL, VT, in visitSRL()
10695 Op, DAG.getConstant(1, DL, VT)); in visitSRL()
10703 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, NewOp1); in visitSRL()
10764 EVT VT = N->getValueType(0); in visitFunnelShift() local
10769 unsigned BitWidth = VT.getScalarSizeInBits(); in visitFunnelShift()
10789 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N0, N1, in visitFunnelShift()
10802 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N1, in visitFunnelShift()
10806 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, in visitFunnelShift()
10815 if ((BitWidth % 8) == 0 && (ShAmt % 8) == 0 && !VT.isVector() && in visitFunnelShift()
10829 if (TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, in visitFunnelShift()
10837 VT, DL, RHS->getChain(), NewPtr, in visitFunnelShift()
10857 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N1, N2); in visitFunnelShift()
10859 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, N2); in visitFunnelShift()
10867 if (N0 == N1 && hasOperation(RotOpc, VT)) in visitFunnelShift()
10868 return DAG.getNode(RotOpc, SDLoc(N), VT, N0, N2); in visitFunnelShift()
10883 EVT VT = N0.getValueType(); in visitSHLSAT() local
10887 DAG.FoldConstantArithmetic(N->getOpcode(), SDLoc(N), VT, {N0, N1})) in visitSHLSAT()
10892 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::SHL, VT)) { in visitSHLSAT()
10896 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, N1); in visitSHLSAT()
10902 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, N1); in visitSHLSAT()
10921 EVT VT = N->getValueType(0); in foldABSToABD() local
10940 if (AbsOp1->getFlags().hasNoSignedWrap() && hasOperation(ISD::ABDS, VT) && in foldABSToABD()
10941 TLI.preferABDSToABSWithNSW(VT)) { in foldABSToABD()
10942 SDValue ABD = DAG.getNode(ISD::ABDS, DL, VT, Op0, Op1); in foldABSToABD()
10966 ABD = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, ABD); in foldABSToABD()
10972 if (hasOperation(ABDOpcode, VT)) { in foldABSToABD()
10973 SDValue ABD = DAG.getNode(ABDOpcode, DL, VT, Op0, Op1); in foldABSToABD()
10982 EVT VT = N->getValueType(0); in visitABS() local
10985 if (SDValue C = DAG.FoldConstantArithmetic(ISD::ABS, SDLoc(N), VT, {N0})) in visitABS()
11001 if (TLI.isTruncateFree(VT, ExtVT) && TLI.isZExtFree(ExtVT, VT) && in visitABS()
11006 ISD::ZERO_EXTEND, DL, VT, in visitABS()
11017 EVT VT = N->getValueType(0); in visitBSWAP() local
11021 if (SDValue C = DAG.FoldConstantArithmetic(ISD::BSWAP, DL, VT, {N0})) in visitBSWAP()
11032 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, N0.getOperand(0)); in visitBSWAP()
11033 return DAG.getNode(ISD::BITREVERSE, DL, VT, BSwap); in visitBSWAP()
11038 unsigned BW = VT.getScalarSizeInBits(); in visitBSWAP()
11045 TLI.isTruncateFree(VT, HalfVT) && in visitBSWAP()
11049 Res = DAG.getNode(ISD::SHL, DL, VT, Res, in visitBSWAP()
11050 DAG.getConstant(NewShAmt, DL, getShiftAmountTy(VT))); in visitBSWAP()
11053 return DAG.getZExtOrTrunc(Res, DL, VT); in visitBSWAP()
11066 SDValue NewSwap = DAG.getNode(ISD::BSWAP, DL, VT, N0.getOperand(0)); in visitBSWAP()
11068 return DAG.getNode(InverseShift, DL, VT, NewSwap, N0.getOperand(1)); in visitBSWAP()
11080 EVT VT = N->getValueType(0); in visitBITREVERSE() local
11084 if (SDValue C = DAG.FoldConstantArithmetic(ISD::BITREVERSE, DL, VT, {N0})) in visitBITREVERSE()
11094 EVT VT = N->getValueType(0); in visitCTLZ() local
11098 if (SDValue C = DAG.FoldConstantArithmetic(ISD::CTLZ, DL, VT, {N0})) in visitCTLZ()
11102 if (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ_ZERO_UNDEF, VT)) in visitCTLZ()
11104 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, DL, VT, N0); in visitCTLZ()
11111 EVT VT = N->getValueType(0); in visitCTLZ_ZERO_UNDEF() local
11116 DAG.FoldConstantArithmetic(ISD::CTLZ_ZERO_UNDEF, DL, VT, {N0})) in visitCTLZ_ZERO_UNDEF()
11123 EVT VT = N->getValueType(0); in visitCTTZ() local
11127 if (SDValue C = DAG.FoldConstantArithmetic(ISD::CTTZ, DL, VT, {N0})) in visitCTTZ()
11131 if (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ_ZERO_UNDEF, VT)) in visitCTTZ()
11133 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, DL, VT, N0); in visitCTTZ()
11140 EVT VT = N->getValueType(0); in visitCTTZ_ZERO_UNDEF() local
11145 DAG.FoldConstantArithmetic(ISD::CTTZ_ZERO_UNDEF, DL, VT, {N0})) in visitCTTZ_ZERO_UNDEF()
11152 EVT VT = N->getValueType(0); in visitCTPOP() local
11156 if (SDValue C = DAG.FoldConstantArithmetic(ISD::CTPOP, DL, VT, {N0})) in visitCTPOP()
11167 EVT VT = LHS.getValueType(); in isLegalToCombineMinNumMaxNum() local
11169 return Options.NoSignedZerosFPMath && VT.isFloatingPoint() && in isLegalToCombineMinNumMaxNum()
11170 TLI.isProfitableToCombineMinNumMaxNum(VT) && in isLegalToCombineMinNumMaxNum()
11174 static SDValue combineMinNumMaxNumImpl(const SDLoc &DL, EVT VT, SDValue LHS, in combineMinNumMaxNumImpl() argument
11179 EVT TransformVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); in combineMinNumMaxNumImpl()
11191 if (TLI.isOperationLegalOrCustom(IEEEOpcode, VT)) in combineMinNumMaxNumImpl()
11192 return DAG.getNode(IEEEOpcode, DL, VT, LHS, RHS); in combineMinNumMaxNumImpl()
11196 return DAG.getNode(Opcode, DL, VT, LHS, RHS); in combineMinNumMaxNumImpl()
11206 if (TLI.isOperationLegalOrCustom(IEEEOpcode, VT)) in combineMinNumMaxNumImpl()
11207 return DAG.getNode(IEEEOpcode, DL, VT, LHS, RHS); in combineMinNumMaxNumImpl()
11211 return DAG.getNode(Opcode, DL, VT, LHS, RHS); in combineMinNumMaxNumImpl()
11220 SDValue DAGCombiner::combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS, in combineMinNumMaxNum() argument
11224 return combineMinNumMaxNumImpl(DL, VT, LHS, RHS, True, False, CC, TLI, DAG); in combineMinNumMaxNum()
11249 SDValue Combined = combineMinNumMaxNumImpl(DL, VT, LHS, RHS, NegTrue, in combineMinNumMaxNum()
11252 return DAG.getNode(ISD::FNEG, DL, VT, Combined); in combineMinNumMaxNum()
11269 EVT VT = N->getValueType(0); in foldSelectOfConstantsUsingSra() local
11271 VT != Cond.getOperand(0).getValueType()) in foldSelectOfConstantsUsingSra()
11283 SDValue ShAmtC = DAG.getConstant(X.getScalarValueSizeInBits() - 1, DL, VT); in foldSelectOfConstantsUsingSra()
11284 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC); in foldSelectOfConstantsUsingSra()
11285 return DAG.getNode(ISD::OR, DL, VT, Sra, C1); in foldSelectOfConstantsUsingSra()
11290 SDValue ShAmtC = DAG.getConstant(X.getScalarValueSizeInBits() - 1, DL, VT); in foldSelectOfConstantsUsingSra()
11291 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC); in foldSelectOfConstantsUsingSra()
11292 return DAG.getNode(ISD::AND, DL, VT, Sra, C1); in foldSelectOfConstantsUsingSra()
11297 static bool shouldConvertSelectOfConstantsToMath(const SDValue &Cond, EVT VT, in shouldConvertSelectOfConstantsToMath() argument
11299 if (!TLI.convertSelectOfConstantsToMath(VT)) in shouldConvertSelectOfConstantsToMath()
11304 if (!TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT)) in shouldConvertSelectOfConstantsToMath()
11320 EVT VT = N->getValueType(0); in foldSelectOfConstants() local
11324 if (!VT.isInteger()) in foldSelectOfConstants()
11350 if (VT.bitsEq(CondVT)) in foldSelectOfConstants()
11352 return DAG.getZExtOrTrunc(NotCond, DL, VT); in foldSelectOfConstants()
11366 return DAG.getZExtOrTrunc(Cond, DL, VT); in foldSelectOfConstants()
11370 return DAG.getSExtOrTrunc(Cond, DL, VT); in foldSelectOfConstants()
11375 NotCond = DAG.getZExtOrTrunc(NotCond, DL, VT); in foldSelectOfConstants()
11382 NotCond = DAG.getSExtOrTrunc(NotCond, DL, VT); in foldSelectOfConstants()
11388 if (!shouldConvertSelectOfConstantsToMath(Cond, VT, TLI)) in foldSelectOfConstants()
11398 Cond = DAG.getZExtOrTrunc(Cond, DL, VT); in foldSelectOfConstants()
11399 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2); in foldSelectOfConstants()
11404 Cond = DAG.getSExtOrTrunc(Cond, DL, VT); in foldSelectOfConstants()
11405 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2); in foldSelectOfConstants()
11410 Cond = DAG.getZExtOrTrunc(Cond, DL, VT); in foldSelectOfConstants()
11412 DAG.getShiftAmountConstant(C1Val.exactLogBase2(), VT, DL); in foldSelectOfConstants()
11413 return DAG.getNode(ISD::SHL, DL, VT, Cond, ShAmtC); in foldSelectOfConstants()
11418 Cond = DAG.getSExtOrTrunc(Cond, DL, VT); in foldSelectOfConstants()
11419 return DAG.getNode(ISD::OR, DL, VT, Cond, N2); in foldSelectOfConstants()
11425 NotCond = DAG.getSExtOrTrunc(NotCond, DL, VT); in foldSelectOfConstants()
11426 return DAG.getNode(ISD::OR, DL, VT, NotCond, N1); in foldSelectOfConstants()
11440 EVT VT = N->getValueType(0); in foldBoolSelectToLogic() local
11441 if (VT != Cond.getValueType() || VT.getScalarSizeInBits() != 1) in foldBoolSelectToLogic()
11447 return DAG.getNode(ISD::OR, SDLoc(N), VT, Cond, F); in foldBoolSelectToLogic()
11452 return DAG.getNode(ISD::AND, SDLoc(N), VT, Cond, T); in foldBoolSelectToLogic()
11456 SDValue NotCond = DAG.getNOT(SDLoc(N), Cond, VT); in foldBoolSelectToLogic()
11457 return DAG.getNode(ISD::OR, SDLoc(N), VT, NotCond, T); in foldBoolSelectToLogic()
11462 SDValue NotCond = DAG.getNOT(SDLoc(N), Cond, VT); in foldBoolSelectToLogic()
11463 return DAG.getNode(ISD::AND, SDLoc(N), VT, NotCond, F); in foldBoolSelectToLogic()
11473 EVT VT = N->getValueType(0); in foldVSelectToSignBitSplatMask() local
11480 if (VT != Cond0.getValueType()) in foldVSelectToSignBitSplatMask()
11495 SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT); in foldVSelectToSignBitSplatMask()
11496 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt); in foldVSelectToSignBitSplatMask()
11497 return DAG.getNode(ISD::AND, DL, VT, Sra, N1); in foldVSelectToSignBitSplatMask()
11503 SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT); in foldVSelectToSignBitSplatMask()
11504 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt); in foldVSelectToSignBitSplatMask()
11505 return DAG.getNode(ISD::OR, DL, VT, Sra, N2); in foldVSelectToSignBitSplatMask()
11514 SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT); in foldVSelectToSignBitSplatMask()
11515 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt); in foldVSelectToSignBitSplatMask()
11516 SDValue Not = DAG.getNOT(DL, Sra, VT); in foldVSelectToSignBitSplatMask()
11517 return DAG.getNode(ISD::AND, DL, VT, Not, N2); in foldVSelectToSignBitSplatMask()
11531 EVT VT = N->getValueType(0); in visitSELECT() local
11544 SDValue SelectOp = DAG.getSelect(DL, VT, F, N2, N1); in visitSELECT()
11566 TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT); in visitSELECT()
11645 combineMinNumMaxNum(DL, VT, Cond0, Cond1, N1, N2, CC)) in visitSELECT()
11653 if (!LegalOperations && TLI.isOperationLegalOrCustom(ISD::UADDO, VT) && in visitSELECT()
11671 SDVTList VTs = DAG.getVTList(VT, VT0); in visitSELECT()
11673 return DAG.getSelect(DL, VT, UAO.getValue(1), N1, UAO.getValue(0)); in visitSELECT()
11677 if (TLI.isOperationLegal(ISD::SELECT_CC, VT) || in visitSELECT()
11679 TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT))) { in visitSELECT()
11683 SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1, in visitSELECT()
11693 if (!VT.isVector()) in visitSELECT()
11710 EVT VT = N->getValueType(0); in ConvertSelectToConcatVector() local
11711 int NumElems = VT.getVectorNumElements(); in ConvertSelectToConcatVector()
11753 ISD::CONCAT_VECTORS, DL, VT, in ConvertSelectToConcatVector()
11768 EVT VT = BasePtr.getValueType(); in refineUniformBase() local
11772 SplatVal.getValueType() == VT) { in refineUniformBase()
11773 BasePtr = DAG.getNode(ISD::ADD, DL, VT, BasePtr, SplatVal); in refineUniformBase()
11774 Index = DAG.getSplat(Index.getValueType(), DL, DAG.getConstant(0, DL, VT)); in refineUniformBase()
11782 SplatVal && SplatVal.getValueType() == VT) { in refineUniformBase()
11783 BasePtr = DAG.getNode(ISD::ADD, DL, VT, BasePtr, SplatVal); in refineUniformBase()
11788 SplatVal && SplatVal.getValueType() == VT) { in refineUniformBase()
11789 BasePtr = DAG.getNode(ISD::ADD, DL, VT, BasePtr, SplatVal); in refineUniformBase()
12094 EVT VT = N->getValueType(0); in foldVSelectOfConstants() local
12096 !shouldConvertSelectOfConstantsToMath(Cond, VT, TLI) || in foldVSelectOfConstants()
12106 unsigned Elts = VT.getVectorNumElements(); in foldVSelectOfConstants()
12130 SDValue ExtendedCond = DAG.getNode(ExtendOpcode, DL, VT, Cond); in foldVSelectOfConstants()
12131 return DAG.getNode(ISD::ADD, DL, VT, ExtendedCond, N2); in foldVSelectOfConstants()
12138 SDValue ZextCond = DAG.getZExtOrTrunc(Cond, DL, VT); in foldVSelectOfConstants()
12139 SDValue ShAmtC = DAG.getConstant(Pow2C.exactLogBase2(), DL, VT); in foldVSelectOfConstants()
12140 return DAG.getNode(ISD::SHL, DL, VT, ZextCond, ShAmtC); in foldVSelectOfConstants()
12157 EVT VT = N->getValueType(0); in visitVSELECT() local
12168 return DAG.getSelect(DL, VT, F, N2, N1); in visitVSELECT()
12190 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) in visitVSELECT()
12191 return DAG.getNode(ISD::ABS, DL, VT, LHS); in visitVSELECT()
12193 SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, LHS, in visitVSELECT()
12194 DAG.getConstant(VT.getScalarSizeInBits() - 1, in visitVSELECT()
12195 DL, getShiftAmountTy(VT))); in visitVSELECT()
12196 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift); in visitVSELECT()
12199 return DAG.getNode(ISD::XOR, DL, VT, Add, Shift); in visitVSELECT()
12209 if (SDValue FMinMax = combineMinNumMaxNum(DL, VT, LHS, RHS, N1, N2, CC)) in visitVSELECT()
12258 if (hasOperation(ABDOpc, VT)) { in visitVSELECT()
12265 return DAG.getNode(ABDOpc, DL, VT, LHS, RHS); in visitVSELECT()
12272 return DAG.getNode(ABDOpc, DL, VT, LHS, RHS); in visitVSELECT()
12281 if (hasOperation(ISD::UADDSAT, VT)) { in visitVSELECT()
12288 SatCC = ISD::getSetCCInverse(SatCC, VT.getScalarType()); in visitVSELECT()
12308 return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
12322 return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
12328 if (hasOperation(ISD::USUBSAT, VT)) { in visitVSELECT()
12335 SatCC = ISD::getSetCCInverse(SatCC, VT.getScalarType()); in visitVSELECT()
12350 if (SDValue R = getTruncatedUSUBSAT(VT, LHS.getValueType(), LHS, RHS, in visitVSELECT()
12365 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
12382 OpRHS = DAG.getNegative(OpRHS, DL, VT); in visitVSELECT()
12383 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
12398 OpRHS = DAG.getConstant(SplatValue, DL, VT); in visitVSELECT()
12399 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
12431 if (hasOperation(ISD::SRA, VT)) in visitVSELECT()
12499 EVT VT = N->getValueType(0); in visitSETCC() local
12502 SDValue Combined = SimplifySetCC(VT, N0, N1, Cond, SDLoc(N), !PreferSetCC); in visitSETCC()
12620 return DAG.getSetCC(DL, VT, NewAndOrOp, NewShiftOrRotate, Cond); in visitSETCC()
12680 EVT VT = N->getValueType(0); in tryToFoldExtendSelectLoad() local
12706 if (!TLI.isLoadExtLegal(ExtLoadOpcode, VT, Load1->getMemoryVT()) || in tryToFoldExtendSelectLoad()
12707 !TLI.isLoadExtLegal(ExtLoadOpcode, VT, Load2->getMemoryVT()) || in tryToFoldExtendSelectLoad()
12709 TLI.getOperationAction(ISD::VSELECT, VT) != TargetLowering::Legal)) in tryToFoldExtendSelectLoad()
12712 SDValue Ext1 = DAG.getNode(Opcode, DL, VT, Op1); in tryToFoldExtendSelectLoad()
12713 SDValue Ext2 = DAG.getNode(Opcode, DL, VT, Op2); in tryToFoldExtendSelectLoad()
12714 return DAG.getSelect(DL, VT, N0->getOperand(0), Ext1, Ext2); in tryToFoldExtendSelectLoad()
12727 EVT VT = N->getValueType(0); in tryToFoldExtendOfConstant() local
12737 return DAG.getNode(Opcode, DL, VT, N0); in tryToFoldExtendOfConstant()
12746 (Opcode != ISD::ZERO_EXTEND || !TLI.isZExtFree(N0.getValueType(), VT))) { in tryToFoldExtendOfConstant()
12759 return DAG.getSelect(DL, VT, N0->getOperand(0), in tryToFoldExtendOfConstant()
12760 DAG.getNode(FoldOpc, DL, VT, Op1), in tryToFoldExtendOfConstant()
12761 DAG.getNode(FoldOpc, DL, VT, Op2)); in tryToFoldExtendOfConstant()
12768 EVT SVT = VT.getScalarType(); in tryToFoldExtendOfConstant()
12769 if (!(VT.isVector() && (!LegalTypes || TLI.isTypeLegal(SVT)) && in tryToFoldExtendOfConstant()
12777 unsigned NumElts = VT.getVectorNumElements(); in tryToFoldExtendOfConstant()
12799 return DAG.getBuildVector(VT, DL, Elts); in tryToFoldExtendOfConstant()
12806 static bool ExtendUsesToFormExtLoad(EVT VT, SDNode *N, SDValue N0, in ExtendUsesToFormExtLoad() argument
12811 bool isTruncFree = TLI.isTruncateFree(VT, N0.getValueType()); in ExtendUsesToFormExtLoad()
12989 EVT VT = N->getValueType(0); in CombineZExtLogicopShiftLoad() local
12991 if (TLI.isZExtFree(OrigVT, VT)) in CombineZExtLogicopShiftLoad()
12998 (LegalOperations && !TLI.isOperationLegal(N0.getOpcode(), VT))) in CombineZExtLogicopShiftLoad()
13005 (LegalOperations && !TLI.isOperationLegal(N1.getOpcode(), VT))) in CombineZExtLogicopShiftLoad()
13013 if (!TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) || in CombineZExtLogicopShiftLoad()
13027 if (!ExtendUsesToFormExtLoad(VT, N1.getNode(), N1.getOperand(0), in CombineZExtLogicopShiftLoad()
13032 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(Load), VT, in CombineZExtLogicopShiftLoad()
13037 SDValue Shift = DAG.getNode(N1.getOpcode(), DL1, VT, ExtLoad, in CombineZExtLogicopShiftLoad()
13040 APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits()); in CombineZExtLogicopShiftLoad()
13042 SDValue And = DAG.getNode(N0.getOpcode(), DL0, VT, Shift, in CombineZExtLogicopShiftLoad()
13043 DAG.getConstant(Mask, DL0, VT)); in CombineZExtLogicopShiftLoad()
13075 EVT VT = Cast->getValueType(0); in matchVSelectOpSizesWithSetCC() local
13076 if (LegalOperations || !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT)) in matchVSelectOpSizesWithSetCC()
13087 if (SetCCVT.getSizeInBits() != VT.getSizeInBits()) in matchVSelectOpSizesWithSetCC()
13097 CastA = DAG.getNode(CastOpcode, DL, VT, A, Cast->getOperand(1)); in matchVSelectOpSizesWithSetCC()
13098 CastB = DAG.getNode(CastOpcode, DL, VT, B, Cast->getOperand(1)); in matchVSelectOpSizesWithSetCC()
13100 CastA = DAG.getNode(CastOpcode, DL, VT, A); in matchVSelectOpSizesWithSetCC()
13101 CastB = DAG.getNode(CastOpcode, DL, VT, B); in matchVSelectOpSizesWithSetCC()
13103 return DAG.getNode(ISD::VSELECT, DL, VT, SetCC, CastA, CastB); in matchVSelectOpSizesWithSetCC()
13109 const TargetLowering &TLI, EVT VT, in tryToFoldExtOfExtload() argument
13122 VT.isVector()) && in tryToFoldExtOfExtload()
13123 !TLI.isLoadExtLegal(ExtLoadType, VT, MemVT)) in tryToFoldExtOfExtload()
13127 DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(), in tryToFoldExtOfExtload()
13140 const TargetLowering &TLI, EVT VT, in tryToFoldExtOfLoad() argument
13149 ((LegalOperations || VT.isFixedLengthVector() || in tryToFoldExtOfLoad()
13151 !TLI.isLoadExtLegal(ExtLoadType, VT, N0.getValueType()))) in tryToFoldExtOfLoad()
13157 DoXform = ExtendUsesToFormExtLoad(VT, N, N0, ExtOpc, SetCCs, TLI); in tryToFoldExtOfLoad()
13158 if (VT.isVector()) in tryToFoldExtOfLoad()
13164 SDValue ExtLoad = DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(), in tryToFoldExtOfLoad()
13183 tryToFoldExtOfMaskedLoad(SelectionDAG &DAG, const TargetLowering &TLI, EVT VT, in tryToFoldExtOfMaskedLoad() argument
13194 !TLI.isLoadExtLegalOrCustom(ExtLoadType, VT, Ld->getValueType(0))) in tryToFoldExtOfMaskedLoad()
13201 SDValue PassThru = DAG.getNode(ExtOpc, dl, VT, Ld->getPassThru()); in tryToFoldExtOfMaskedLoad()
13203 VT, dl, Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(), Ld->getMask(), in tryToFoldExtOfMaskedLoad()
13223 EVT VT = N->getValueType(0); in foldExtendedSignBitTest() local
13228 if (CC == ISD::SETGT && isAllOnesConstant(Ones) && VT == XVT) { in foldExtendedSignBitTest()
13233 unsigned ShCt = VT.getSizeInBits() - 1; in foldExtendedSignBitTest()
13235 if (!TLI.shouldAvoidTransformToShift(VT, ShCt)) { in foldExtendedSignBitTest()
13236 SDValue NotX = DAG.getNOT(DL, X, VT); in foldExtendedSignBitTest()
13237 SDValue ShiftAmount = DAG.getConstant(ShCt, DL, VT); in foldExtendedSignBitTest()
13240 return DAG.getNode(ShiftOpcode, DL, VT, NotX, ShiftAmount); in foldExtendedSignBitTest()
13254 EVT VT = N->getValueType(0); in foldSextSetcc() local
13264 if (VT.isVector() && !LegalOperations && in foldSextSetcc()
13276 if (VT.getSizeInBits() == SVT.getSizeInBits()) in foldSextSetcc()
13277 return DAG.getSetCC(DL, VT, N00, N01, CC); in foldSextSetcc()
13285 return DAG.getSExtOrTrunc(VsetCC, DL, VT); in foldSextSetcc()
13290 if (N0.hasOneUse() && TLI.isOperationLegalOrCustom(ISD::SETCC, VT) && in foldSextSetcc()
13308 TLI.isLoadExtLegal(LoadOpcode, VT, V.getValueType()))) in foldSextSetcc()
13322 if (User->getOpcode() != ExtOpcode || User->getValueType(0) != VT) in foldSextSetcc()
13329 SDValue Ext0 = DAG.getNode(ExtOpcode, DL, VT, N00); in foldSextSetcc()
13330 SDValue Ext1 = DAG.getNode(ExtOpcode, DL, VT, N01); in foldSextSetcc()
13331 return DAG.getSetCC(DL, VT, Ext0, Ext1, CC); in foldSextSetcc()
13349 ? DAG.getAllOnesConstant(DL, VT) in foldSextSetcc()
13350 : DAG.getBoolConstant(true, DL, VT, N00VT); in foldSextSetcc()
13351 SDValue Zero = DAG.getConstant(0, DL, VT); in foldSextSetcc()
13355 if (!VT.isVector() && !shouldConvertSelectOfConstantsToMath(N0, VT, TLI)) { in foldSextSetcc()
13364 return DAG.getSelect(DL, VT, SetCC, ExtTrueVal, Zero); in foldSextSetcc()
13373 EVT VT = N->getValueType(0); in visitSIGN_EXTEND() local
13376 if (VT.isVector()) in visitSIGN_EXTEND()
13382 return DAG.getConstant(0, DL, VT); in visitSIGN_EXTEND()
13390 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N0.getOperand(0)); in visitSIGN_EXTEND()
13396 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, SDLoc(N), VT, in visitSIGN_EXTEND()
13406 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, T); in visitSIGN_EXTEND()
13428 unsigned DestBits = VT.getScalarSizeInBits(); in visitSIGN_EXTEND()
13440 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op); in visitSIGN_EXTEND()
13445 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op); in visitSIGN_EXTEND()
13452 Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N0), VT, Op); in visitSIGN_EXTEND()
13454 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, Op); in visitSIGN_EXTEND()
13455 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Op, in visitSIGN_EXTEND()
13462 tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0, in visitSIGN_EXTEND()
13467 tryToFoldExtOfMaskedLoad(DAG, TLI, VT, LegalOperations, N, N0, in visitSIGN_EXTEND()
13478 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::SEXTLOAD)) in visitSIGN_EXTEND()
13486 (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) { in visitSIGN_EXTEND()
13489 if (TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, MemVT) && in visitSIGN_EXTEND()
13492 bool DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0), in visitSIGN_EXTEND()
13495 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN00), VT, in visitSIGN_EXTEND()
13499 APInt Mask = N0.getConstantOperandAPInt(1).sext(VT.getSizeInBits()); in visitSIGN_EXTEND()
13500 SDValue And = DAG.getNode(N0.getOpcode(), DL, VT, in visitSIGN_EXTEND()
13501 ExtLoad, DAG.getConstant(Mask, DL, VT)); in visitSIGN_EXTEND()
13531 if (!TLI.isSExtCheaperThanZExt(N0.getValueType(), VT) && in visitSIGN_EXTEND()
13532 (!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) && in visitSIGN_EXTEND()
13536 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0, Flags); in visitSIGN_EXTEND()
13547 TLI.isOperationLegalOrCustom(ISD::SUB, VT)) { in visitSIGN_EXTEND()
13548 SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(1).getOperand(0), DL, VT); in visitSIGN_EXTEND()
13549 return DAG.getNegative(Zext, DL, VT); in visitSIGN_EXTEND()
13556 TLI.isOperationLegalOrCustom(ISD::ADD, VT)) { in visitSIGN_EXTEND()
13557 SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(0).getOperand(0), DL, VT); in visitSIGN_EXTEND()
13558 return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT)); in visitSIGN_EXTEND()
13564 (!LegalOperations || (TLI.isOperationLegal(ISD::ZERO_EXTEND, VT) && in visitSIGN_EXTEND()
13565 TLI.isOperationLegal(ISD::ADD, VT)))) { in visitSIGN_EXTEND()
13577 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NewXor); in visitSIGN_EXTEND()
13580 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitSIGN_EXTEND()
13581 return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT)); in visitSIGN_EXTEND()
13601 EVT VT = Extend->getValueType(0); in widenCtPop() local
13604 !TLI.isOperationLegalOrCustom(ISD::CTPOP, VT)) in widenCtPop()
13609 SDValue NewZext = DAG.getZExtOrTrunc(CtPop.getOperand(0), DL, VT); in widenCtPop()
13610 return DAG.getNode(ISD::CTPOP, DL, VT, NewZext); in widenCtPop()
13618 EVT VT = Extend->getValueType(0); in widenAbs() local
13619 if (VT.isVector()) in widenAbs()
13637 return DAG.getZExtOrTrunc(NewAbs, SDLoc(Extend), VT); in widenAbs()
13642 EVT VT = N->getValueType(0); in visitZERO_EXTEND() local
13645 if (VT.isVector()) in visitZERO_EXTEND()
13651 return DAG.getConstant(0, DL, VT); in visitZERO_EXTEND()
13659 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitZERO_EXTEND()
13665 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(N), VT, in visitZERO_EXTEND()
13680 VT.getScalarSizeInBits())); in visitZERO_EXTEND()
13682 SDValue ZExtOrTrunc = DAG.getZExtOrTrunc(Op, DL, VT); in visitZERO_EXTEND()
13708 if (SrcVT.bitsLT(VT) && VT.isVector()) { in visitZERO_EXTEND()
13710 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) { in visitZERO_EXTEND()
13714 SDValue ZExtOrTrunc = DAG.getZExtOrTrunc(Op, DL, VT); in visitZERO_EXTEND()
13721 if (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT)) { in visitZERO_EXTEND()
13722 SDValue Op = DAG.getAnyExtOrTrunc(N0.getOperand(0), DL, VT); in visitZERO_EXTEND()
13738 !TLI.isZExtFree(N0.getValueType(), VT))) { in visitZERO_EXTEND()
13740 X = DAG.getAnyExtOrTrunc(X, SDLoc(X), VT); in visitZERO_EXTEND()
13741 APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits()); in visitZERO_EXTEND()
13742 return DAG.getNode(ISD::AND, DL, VT, in visitZERO_EXTEND()
13743 X, DAG.getConstant(Mask, DL, VT)); in visitZERO_EXTEND()
13748 tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0, in visitZERO_EXTEND()
13753 tryToFoldExtOfMaskedLoad(DAG, TLI, VT, LegalOperations, N, N0, in visitZERO_EXTEND()
13766 if (ISD::isBitwiseLogicOp(N0.getOpcode()) && !TLI.isZExtFree(N0, VT) && in visitZERO_EXTEND()
13769 (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) { in visitZERO_EXTEND()
13772 if (TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) && in visitZERO_EXTEND()
13786 DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0), in visitZERO_EXTEND()
13789 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN00), VT, in visitZERO_EXTEND()
13793 APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits()); in visitZERO_EXTEND()
13794 SDValue And = DAG.getNode(N0.getOpcode(), DL, VT, in visitZERO_EXTEND()
13795 ExtLoad, DAG.getConstant(Mask, DL, VT)); in visitZERO_EXTEND()
13825 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::ZEXTLOAD)) in visitZERO_EXTEND()
13836 if (!LegalOperations && VT.isVector() && in visitZERO_EXTEND()
13847 if (VT.getSizeInBits() == N00VT.getSizeInBits()) { in visitZERO_EXTEND()
13849 SDValue VSetCC = DAG.getNode(ISD::SETCC, DL, VT, N0.getOperand(0), in visitZERO_EXTEND()
13861 return DAG.getZeroExtendInReg(DAG.getAnyExtOrTrunc(VsetCC, DL, VT), DL, in visitZERO_EXTEND()
13873 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, SCC); in visitZERO_EXTEND()
13878 !TLI.isZExtFree(N0, VT)) { in visitZERO_EXTEND()
13894 if (Log2_32_Ceil(VT.getSizeInBits()) > ShAmt.getValueSizeInBits()) in visitZERO_EXTEND()
13897 return DAG.getNode(N0.getOpcode(), DL, VT, in visitZERO_EXTEND()
13898 DAG.getNode(ISD::ZERO_EXTEND, DL, VT, ShVal), ShAmt); in visitZERO_EXTEND()
13920 EVT VT = N->getValueType(0); in visitANY_EXTEND() local
13924 return DAG.getUNDEF(VT); in visitANY_EXTEND()
13935 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0)); in visitANY_EXTEND()
13943 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0)); in visitANY_EXTEND()
13961 return DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT); in visitANY_EXTEND()
13970 SDValue X = DAG.getAnyExtOrTrunc(N0.getOperand(0).getOperand(0), DL, VT); in visitANY_EXTEND()
13971 SDValue Y = DAG.getNode(ISD::ANY_EXTEND, DL, VT, N0.getOperand(1)); in visitANY_EXTEND()
13973 return DAG.getNode(ISD::AND, DL, VT, X, Y); in visitANY_EXTEND()
13979 if (VT.isVector()) { in visitANY_EXTEND()
13982 tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0, in visitANY_EXTEND()
13987 TLI.isLoadExtLegal(ISD::EXTLOAD, VT, N0.getValueType())) { in visitANY_EXTEND()
13992 ExtendUsesToFormExtLoad(VT, N, N0, ISD::ANY_EXTEND, SetCCs, TLI); in visitANY_EXTEND()
13995 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, in visitANY_EXTEND()
14022 if (!LegalOperations || TLI.isLoadExtLegal(ExtType, VT, MemVT)) { in visitANY_EXTEND()
14024 VT, LN0->getChain(), LN0->getBasePtr(), in visitANY_EXTEND()
14042 if (VT.isVector() && !LegalOperations) { in visitANY_EXTEND()
14052 if (VT.getSizeInBits() == N00VT.getSizeInBits()) in visitANY_EXTEND()
14053 return DAG.getSetCC(SDLoc(N), VT, N0.getOperand(0), in visitANY_EXTEND()
14065 return DAG.getAnyExtOrTrunc(VsetCC, SDLoc(N), VT); in visitANY_EXTEND()
14071 DL, N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, DL, VT), in visitANY_EXTEND()
14072 DAG.getConstant(0, DL, VT), in visitANY_EXTEND()
14180 EVT VT = N->getValueType(0); in reduceLoadWidth() local
14181 EVT ExtVT = VT; in reduceLoadWidth()
14184 if (VT.isVector()) in reduceLoadWidth()
14325 ExtVT == VT && TLI.isNarrowingProfitable(N0.getValueType(), VT)) { in reduceLoadWidth()
14367 Load = DAG.getLoad(VT, DL, LN0->getChain(), NewPtr, in reduceLoadWidth()
14371 Load = DAG.getExtLoad(ExtType, DL, VT, LN0->getChain(), NewPtr, in reduceLoadWidth()
14385 ShImmTy = VT; in reduceLoadWidth()
14390 if (ShLeftAmt >= VT.getScalarSizeInBits()) in reduceLoadWidth()
14391 Result = DAG.getConstant(0, DL, VT); in reduceLoadWidth()
14393 Result = DAG.getNode(ISD::SHL, DL, VT, in reduceLoadWidth()
14402 SDValue ShiftC = DAG.getConstant(ShAmt, DL, VT); in reduceLoadWidth()
14403 Result = DAG.getNode(ISD::SHL, DL, VT, Result, ShiftC); in reduceLoadWidth()
14414 EVT VT = N->getValueType(0); in visitSIGN_EXTEND_INREG() local
14416 unsigned VTBits = VT.getScalarSizeInBits(); in visitSIGN_EXTEND_INREG()
14421 return DAG.getConstant(0, SDLoc(N), VT); in visitSIGN_EXTEND_INREG()
14425 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0, N1); in visitSIGN_EXTEND_INREG()
14434 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0.getOperand(0), in visitSIGN_EXTEND_INREG()
14446 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT))) in visitSIGN_EXTEND_INREG()
14447 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00); in visitSIGN_EXTEND_INREG()
14464 TLI.isOperationLegal(ISD::SIGN_EXTEND_VECTOR_INREG, VT))) in visitSIGN_EXTEND_INREG()
14465 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, SDLoc(N), VT, N00); in visitSIGN_EXTEND_INREG()
14473 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT))) in visitSIGN_EXTEND_INREG()
14474 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00); in visitSIGN_EXTEND_INREG()
14501 return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0.getOperand(0), in visitSIGN_EXTEND_INREG()
14515 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT))) { in visitSIGN_EXTEND_INREG()
14517 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
14532 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT))) { in visitSIGN_EXTEND_INREG()
14534 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
14548 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT)) { in visitSIGN_EXTEND_INREG()
14550 VT, SDLoc(N), Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(), in visitSIGN_EXTEND_INREG()
14568 DAG.getVTList(VT, MVT::Other), ExtVT, SDLoc(N), Ops, in visitSIGN_EXTEND_INREG()
14582 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, BSwap, N1); in visitSIGN_EXTEND_INREG()
14600 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT, SignExtExtendee, in visitSIGN_EXTEND_INREG()
14616 EVT VT = N->getValueType(0); in foldExtendVectorInregToExtendOfSubvector() local
14619 VT.getVectorElementCount()); in foldExtendVectorInregToExtendOfSubvector()
14635 if (LegalOperations && !TLI.isOperationLegal(Opcode, VT)) in foldExtendVectorInregToExtendOfSubvector()
14638 return DAG.getNode(Opcode, SDLoc(N), VT, Src); in foldExtendVectorInregToExtendOfSubvector()
14643 EVT VT = N->getValueType(0); in visitEXTEND_VECTOR_INREG() local
14649 ? DAG.getUNDEF(VT) in visitEXTEND_VECTOR_INREG()
14650 : DAG.getConstant(0, SDLoc(N), VT); in visitEXTEND_VECTOR_INREG()
14668 EVT VT = N->getValueType(0); in visitTRUNCATE() local
14674 return DAG.getUNDEF(VT); in visitTRUNCATE()
14678 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0)); in visitTRUNCATE()
14681 if (SDValue C = DAG.FoldConstantArithmetic(ISD::TRUNCATE, SDLoc(N), VT, {N0})) in visitTRUNCATE()
14689 if (N0.getOperand(0).getValueType().bitsLT(VT)) in visitTRUNCATE()
14690 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0)); in visitTRUNCATE()
14692 if (N0.getOperand(0).getValueType().bitsGT(VT)) in visitTRUNCATE()
14693 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0)); in visitTRUNCATE()
14706 if (ExtVT.bitsLT(VT) && TLI.preferSextInRegOfTruncate(VT, SrcVT, ExtVT)) { in visitTRUNCATE()
14707 SDValue TrX = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, X); in visitTRUNCATE()
14708 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, TrX, ExtVal); in visitTRUNCATE()
14727 LegalTypes && !LegalOperations && N0->hasOneUse() && VT != MVT::i1) { in visitTRUNCATE()
14754 TLI.isTruncateFree(SrcVT, VT)) { in visitTRUNCATE()
14757 SDValue TruncOp0 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1)); in visitTRUNCATE()
14758 SDValue TruncOp1 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(2)); in visitTRUNCATE()
14759 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, Cond, TruncOp0, TruncOp1); in visitTRUNCATE()
14765 (!LegalOperations || TLI.isOperationLegal(ISD::SHL, VT)) && in visitTRUNCATE()
14766 TLI.isTypeDesirableForOp(ISD::SHL, VT)) { in visitTRUNCATE()
14769 unsigned Size = VT.getScalarSizeInBits(); in visitTRUNCATE()
14772 EVT AmtVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); in visitTRUNCATE()
14774 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0)); in visitTRUNCATE()
14779 return DAG.getNode(ISD::SHL, SL, VT, Trunc, Amt); in visitTRUNCATE()
14783 if (SDValue V = foldSubToUSubSat(VT, N0.getNode())) in visitTRUNCATE()
14792 TLI.isTruncateFree(SrcVT.getScalarType(), VT.getScalarType()) && in visitTRUNCATE()
14794 (!LegalTypes || TLI.isTypeLegal(VT.getScalarType()))) { in visitTRUNCATE()
14796 EVT SVT = VT.getScalarType(); in visitTRUNCATE()
14802 return DAG.getBuildVector(VT, DL, TruncOps); in visitTRUNCATE()
14807 (!LegalTypes || TLI.isTypeLegal(VT.getScalarType())) && in visitTRUNCATE()
14808 (!LegalOperations || TLI.isOperationLegal(ISD::SPLAT_VECTOR, VT))) { in visitTRUNCATE()
14810 EVT SVT = VT.getScalarType(); in visitTRUNCATE()
14812 VT, DL, DAG.getNode(ISD::TRUNCATE, DL, SVT, N0->getOperand(0))); in visitTRUNCATE()
14819 if (Level == AfterLegalizeVectorOps && VT.isVector() && in visitTRUNCATE()
14825 EVT TruncVecEltTy = VT.getVectorElementType(); in visitTRUNCATE()
14831 unsigned TruncVecNumElts = VT.getVectorNumElements(); in visitTRUNCATE()
14841 return DAG.getBuildVector(VT, SDLoc(N), Opnds); in visitTRUNCATE()
14847 if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getOpcode(), VT)) { in visitTRUNCATE()
14855 if (LN0->isSimple() && LN0->getMemoryVT().bitsLE(VT)) { in visitTRUNCATE()
14857 LN0->getExtensionType(), SDLoc(LN0), VT, LN0->getChain(), in visitTRUNCATE()
14885 VT.getVectorElementType(), in visitTRUNCATE()
14890 return DAG.getUNDEF(VT); in visitTRUNCATE()
14904 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Opnds); in visitTRUNCATE()
14912 if (N0.getOpcode() == ISD::BITCAST && !VT.isVector()) { in visitTRUNCATE()
14915 if (VecSrcVT.isVector() && VecSrcVT.getScalarType() == VT && in visitTRUNCATE()
14921 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, VT, VecSrc, in visitTRUNCATE()
14940 VT.getVectorElementType()) in visitTRUNCATE()
14941 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N0->getOperand(0)), VT, in visitTRUNCATE()
14966 if (VT.isScalarInteger() || TLI.isOperationLegal(N0.getOpcode(), VT)) { in visitTRUNCATE()
14968 SDValue NarrowL = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
14969 SDValue NarrowR = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1)); in visitTRUNCATE()
14970 return DAG.getNode(N0.getOpcode(), DL, VT, NarrowL, NarrowR); in visitTRUNCATE()
14982 TLI.isOperationLegal(N0.getOpcode(), VT)) && in visitTRUNCATE()
14985 SDValue X = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
14986 SDValue Y = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1)); in visitTRUNCATE()
14987 SDVTList VTs = DAG.getVTList(VT, N0->getValueType(1)); in visitTRUNCATE()
14998 VT.getScalarSizeInBits() && in visitTRUNCATE()
14999 hasOperation(N0.getOpcode(), VT)) { in visitTRUNCATE()
15000 return getTruncatedUSUBSAT(VT, SrcVT, N0.getOperand(0), N0.getOperand(1), in visitTRUNCATE()
15018 SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) { in CombineConsecutiveLoads() argument
15038 if ((!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)) && in CombineConsecutiveLoads()
15040 TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, in CombineConsecutiveLoads()
15042 return DAG.getLoad(VT, SDLoc(N), LD1->getChain(), LD1->getBasePtr(), in CombineConsecutiveLoads()
15058 EVT VT = N->getValueType(0); in foldBitcastedFPLogic() local
15062 if (!VT.isFloatingPoint()) in foldBitcastedFPLogic()
15067 if (VT.getScalarSizeInBits() != SourceVT.getScalarSizeInBits()) in foldBitcastedFPLogic()
15089 if (LegalOperations && !TLI.isOperationLegal(FPOpcode, VT)) in foldBitcastedFPLogic()
15095 auto IsBitCastOrFree = [&TLI, FPOpcode](SDValue Op, EVT VT) { in foldBitcastedFPLogic() argument
15096 if (Op.getOpcode() == ISD::BITCAST && Op.getOperand(0).getValueType() == VT) in foldBitcastedFPLogic()
15099 return FPOpcode == ISD::FABS ? TLI.isFAbsFree(VT) : TLI.isFNegFree(VT); in foldBitcastedFPLogic()
15109 IsBitCastOrFree(LogicOp0, VT)) { in foldBitcastedFPLogic()
15110 SDValue CastOp0 = DAG.getNode(ISD::BITCAST, SDLoc(N), VT, LogicOp0); in foldBitcastedFPLogic()
15111 SDValue FPOp = DAG.getNode(FPOpcode, SDLoc(N), VT, CastOp0); in foldBitcastedFPLogic()
15114 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, FPOp); in foldBitcastedFPLogic()
15123 EVT VT = N->getValueType(0); in visitBITCAST() local
15126 return DAG.getUNDEF(VT); in visitBITCAST()
15134 if (VT.isVector() && in visitBITCAST()
15136 (!LegalOperations && VT.isInteger() && N0.getValueType().isInteger() && in visitBITCAST()
15137 TLI.isTypeLegal(VT.getVectorElementType()))) && in visitBITCAST()
15141 VT.getVectorElementType()); in visitBITCAST()
15149 (isa<ConstantSDNode>(N0) && VT.isFloatingPoint() && !VT.isVector() && in visitBITCAST()
15150 TLI.isOperationLegal(ISD::ConstantFP, VT)) || in visitBITCAST()
15151 (isa<ConstantFPSDNode>(N0) && VT.isInteger() && !VT.isVector() && in visitBITCAST()
15152 TLI.isOperationLegal(ISD::Constant, VT))) { in visitBITCAST()
15153 SDValue C = DAG.getBitcast(VT, N0); in visitBITCAST()
15161 return DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
15165 if (ISD::isBitwiseLogicOp(N0.getOpcode()) && VT.isInteger() && in visitBITCAST()
15167 auto IsFreeBitcast = [VT](SDValue V) { in visitBITCAST()
15169 V.getOperand(0).getValueType() == VT) || in visitBITCAST()
15174 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, in visitBITCAST()
15175 DAG.getBitcast(VT, N0.getOperand(0)), in visitBITCAST()
15176 DAG.getBitcast(VT, N0.getOperand(1))); in visitBITCAST()
15184 TLI.hasBigEndianPartOrdering(VT, DAG.getDataLayout()) && in visitBITCAST()
15191 TLI.isOperationLegal(ISD::LOAD, VT))) { in visitBITCAST()
15194 if (TLI.isLoadBitCastBeneficial(N0.getValueType(), VT, DAG, in visitBITCAST()
15197 DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(), in visitBITCAST()
15221 N0->hasOneUse() && VT.isInteger() && !VT.isVector() && in visitBITCAST()
15223 SDValue NewConv = DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
15228 assert(VT.getSizeInBits() == 128); in visitBITCAST()
15230 APInt::getSignMask(VT.getSizeInBits() / 2), SDLoc(N0), MVT::i64); in visitBITCAST()
15246 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit); in visitBITCAST()
15248 return DAG.getNode(ISD::XOR, DL, VT, NewConv, FlipBits); in visitBITCAST()
15250 APInt SignBit = APInt::getSignMask(VT.getSizeInBits()); in visitBITCAST()
15252 return DAG.getNode(ISD::XOR, DL, VT, in visitBITCAST()
15253 NewConv, DAG.getConstant(SignBit, DL, VT)); in visitBITCAST()
15255 return DAG.getNode(ISD::AND, DL, VT, in visitBITCAST()
15256 NewConv, DAG.getConstant(~SignBit, DL, VT)); in visitBITCAST()
15271 isa<ConstantFPSDNode>(N0.getOperand(0)) && VT.isInteger() && in visitBITCAST()
15272 !VT.isVector()) { in visitBITCAST()
15280 unsigned VTWidth = VT.getSizeInBits(); in visitBITCAST()
15282 X = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, X); in visitBITCAST()
15293 X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X); in visitBITCAST()
15298 APInt SignBit = APInt::getSignMask(VT.getSizeInBits() / 2); in visitBITCAST()
15299 SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
15301 SDValue X = DAG.getBitcast(VT, N0.getOperand(1)); in visitBITCAST()
15303 SDValue XorResult = DAG.getNode(ISD::XOR, SDLoc(N0), VT, Cst, X); in visitBITCAST()
15315 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit); in visitBITCAST()
15317 return DAG.getNode(ISD::XOR, SDLoc(N), VT, Cst, FlipBits); in visitBITCAST()
15319 APInt SignBit = APInt::getSignMask(VT.getSizeInBits()); in visitBITCAST()
15320 X = DAG.getNode(ISD::AND, SDLoc(X), VT, in visitBITCAST()
15321 X, DAG.getConstant(SignBit, SDLoc(X), VT)); in visitBITCAST()
15324 SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
15325 Cst = DAG.getNode(ISD::AND, SDLoc(Cst), VT, in visitBITCAST()
15326 Cst, DAG.getConstant(~SignBit, SDLoc(Cst), VT)); in visitBITCAST()
15329 return DAG.getNode(ISD::OR, SDLoc(N), VT, X, Cst); in visitBITCAST()
15335 if (SDValue CombineLD = CombineConsecutiveLoads(N0.getNode(), VT)) in visitBITCAST()
15342 if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT) && VT.isVector() && in visitBITCAST()
15344 VT.getVectorNumElements() >= N0.getValueType().getVectorNumElements() && in visitBITCAST()
15345 !(VT.getVectorNumElements() % N0.getValueType().getVectorNumElements())) { in visitBITCAST()
15352 Op.getOperand(0).getValueType() == VT) in visitBITCAST()
15355 return DAG.getBitcast(VT, Op); in visitBITCAST()
15368 VT.getVectorNumElements() / N0.getValueType().getVectorNumElements(); in visitBITCAST()
15375 TLI.buildLegalVectorShuffle(VT, SDLoc(N), SV0, SV1, NewMask, DAG); in visitBITCAST()
15384 EVT VT = N->getValueType(0); in visitBUILD_PAIR() local
15385 return CombineConsecutiveLoads(N, VT); in visitBUILD_PAIR()
15492 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, in ConstantFoldBITCASTofBUILD_VECTOR() local
15494 return DAG.getBuildVector(VT, SDLoc(BV), Ops); in ConstantFoldBITCASTofBUILD_VECTOR()
15541 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, Ops.size()); in ConstantFoldBITCASTofBUILD_VECTOR() local
15542 return DAG.getBuildVector(VT, DL, Ops); in ConstantFoldBITCASTofBUILD_VECTOR()
15564 EVT VT = N->getValueType(0); in visitFADDForFMACombine() local
15578 TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT) && in visitFADDForFMACombine()
15579 (!LegalOperations || matcher.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFADDForFMACombine()
15600 if (TLI.generateFMAsInMachineCombiner(VT, OptLevel)) in visitFADDForFMACombine()
15605 bool Aggressive = TLI.enableAggressiveFMAFusion(VT); in visitFADDForFMACombine()
15627 return matcher.getNode(PreferredFusedOpcode, SL, VT, N0.getOperand(0), in visitFADDForFMACombine()
15634 return matcher.getNode(PreferredFusedOpcode, SL, VT, N1.getOperand(0), in visitFADDForFMACombine()
15662 SDValue CDE = matcher.getNode(PreferredFusedOpcode, SL, VT, C, D, E); in visitFADDForFMACombine()
15679 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFADDForFMACombine()
15682 PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
15683 matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)), in visitFADDForFMACombine()
15684 matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)), N1); in visitFADDForFMACombine()
15693 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFADDForFMACombine()
15696 PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
15697 matcher.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(0)), in visitFADDForFMACombine()
15698 matcher.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)), N0); in visitFADDForFMACombine()
15709 PreferredFusedOpcode, SL, VT, X, Y, in visitFADDForFMACombine()
15710 matcher.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
15711 matcher.getNode(ISD::FP_EXTEND, SL, VT, U), in visitFADDForFMACombine()
15712 matcher.getNode(ISD::FP_EXTEND, SL, VT, V), Z)); in visitFADDForFMACombine()
15719 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFADDForFMACombine()
15736 PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
15737 matcher.getNode(ISD::FP_EXTEND, SL, VT, X), in visitFADDForFMACombine()
15738 matcher.getNode(ISD::FP_EXTEND, SL, VT, Y), in visitFADDForFMACombine()
15739 matcher.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
15740 matcher.getNode(ISD::FP_EXTEND, SL, VT, U), in visitFADDForFMACombine()
15741 matcher.getNode(ISD::FP_EXTEND, SL, VT, V), Z)); in visitFADDForFMACombine()
15748 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFADDForFMACombine()
15764 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFADDForFMACombine()
15783 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFADDForFMACombine()
15801 EVT VT = N->getValueType(0); in visitFSUBForFMACombine() local
15815 TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT) && in visitFSUBForFMACombine()
15816 (!LegalOperations || matcher.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFSUBForFMACombine()
15830 if (TLI.generateFMAsInMachineCombiner(VT, OptLevel)) in visitFSUBForFMACombine()
15835 bool Aggressive = TLI.enableAggressiveFMAFusion(VT); in visitFSUBForFMACombine()
15849 return matcher.getNode(PreferredFusedOpcode, SL, VT, XY.getOperand(0), in visitFSUBForFMACombine()
15851 matcher.getNode(ISD::FNEG, SL, VT, Z)); in visitFSUBForFMACombine()
15861 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
15862 matcher.getNode(ISD::FNEG, SL, VT, YZ.getOperand(0)), in visitFSUBForFMACombine()
15892 return matcher.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
15893 matcher.getNode(ISD::FNEG, SL, VT, N00), N01, in visitFSUBForFMACombine()
15894 matcher.getNode(ISD::FNEG, SL, VT, N1)); in visitFSUBForFMACombine()
15904 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
15907 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
15908 matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)), in visitFSUBForFMACombine()
15909 matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)), in visitFSUBForFMACombine()
15910 matcher.getNode(ISD::FNEG, SL, VT, N1)); in visitFSUBForFMACombine()
15920 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
15923 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
15925 ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
15926 matcher.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(0))), in visitFSUBForFMACombine()
15927 matcher.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)), N0); in visitFSUBForFMACombine()
15942 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
15945 ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
15947 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
15948 matcher.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)), in visitFSUBForFMACombine()
15949 matcher.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)), in visitFSUBForFMACombine()
15966 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
15969 ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
15971 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
15972 matcher.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)), in visitFSUBForFMACombine()
15973 matcher.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)), in visitFSUBForFMACombine()
16001 PreferredFusedOpcode, SL, VT, N0.getOperand(0), N0.getOperand(1), in visitFSUBForFMACombine()
16002 matcher.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
16005 matcher.getNode(ISD::FNEG, SL, VT, N1))); in visitFSUBForFMACombine()
16016 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
16017 matcher.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)), in visitFSUBForFMACombine()
16019 matcher.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
16020 matcher.getNode(ISD::FNEG, SL, VT, N20), N21, N0)); in visitFSUBForFMACombine()
16030 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
16033 PreferredFusedOpcode, SL, VT, N0.getOperand(0), N0.getOperand(1), in visitFSUBForFMACombine()
16035 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
16036 matcher.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(0)), in visitFSUBForFMACombine()
16037 matcher.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(1)), in visitFSUBForFMACombine()
16038 matcher.getNode(ISD::FNEG, SL, VT, N1))); in visitFSUBForFMACombine()
16054 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
16057 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
16058 matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)), in visitFSUBForFMACombine()
16059 matcher.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)), in visitFSUBForFMACombine()
16061 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
16062 matcher.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(0)), in visitFSUBForFMACombine()
16063 matcher.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(1)), in visitFSUBForFMACombine()
16064 matcher.getNode(ISD::FNEG, SL, VT, N1))); in visitFSUBForFMACombine()
16075 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
16080 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
16081 matcher.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)), in visitFSUBForFMACombine()
16084 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
16085 matcher.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
16086 matcher.getNode(ISD::FP_EXTEND, SL, VT, N1200)), in visitFSUBForFMACombine()
16087 matcher.getNode(ISD::FP_EXTEND, SL, VT, N1201), N0)); in visitFSUBForFMACombine()
16103 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
16108 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
16109 matcher.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
16110 matcher.getNode(ISD::FP_EXTEND, SL, VT, N100)), in visitFSUBForFMACombine()
16111 matcher.getNode(ISD::FP_EXTEND, SL, VT, N101), in visitFSUBForFMACombine()
16113 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
16114 matcher.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
16115 matcher.getNode(ISD::FP_EXTEND, SL, VT, N1020)), in visitFSUBForFMACombine()
16116 matcher.getNode(ISD::FP_EXTEND, SL, VT, N1021), N0)); in visitFSUBForFMACombine()
16130 EVT VT = N->getValueType(0); in visitFMULForFMADistributiveCombine() local
16146 TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT) && in visitFMULForFMADistributiveCombine()
16147 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFMULForFMADistributiveCombine()
16160 bool Aggressive = TLI.enableAggressiveFMAFusion(VT); in visitFMULForFMADistributiveCombine()
16168 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y, in visitFMULForFMADistributiveCombine()
16171 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y, in visitFMULForFMADistributiveCombine()
16172 DAG.getNode(ISD::FNEG, SL, VT, Y)); in visitFMULForFMADistributiveCombine()
16191 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFMULForFMADistributiveCombine()
16192 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y, in visitFMULForFMADistributiveCombine()
16195 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFMULForFMADistributiveCombine()
16196 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y, in visitFMULForFMADistributiveCombine()
16197 DAG.getNode(ISD::FNEG, SL, VT, Y)); in visitFMULForFMADistributiveCombine()
16201 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y, in visitFMULForFMADistributiveCombine()
16202 DAG.getNode(ISD::FNEG, SL, VT, Y)); in visitFMULForFMADistributiveCombine()
16204 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y, in visitFMULForFMADistributiveCombine()
16236 EVT VT = N->getValueType(0); in visitFADD() local
16246 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FADD, DL, VT, {N0, N1})) in visitFADD()
16251 return DAG.getNode(ISD::FADD, DL, VT, N1, N0); in visitFADD()
16254 if (VT.isVector()) in visitFADD()
16268 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) in visitFADD()
16271 return DAG.getNode(ISD::FSUB, DL, VT, N0, NegN1); in visitFADD()
16274 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) in visitFADD()
16277 return DAG.getNode(ISD::FSUB, DL, VT, N1, NegN0); in visitFADD()
16289 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B); in visitFADD()
16290 return DAG.getNode(ISD::FSUB, DL, VT, N1, Add); in visitFADD()
16295 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B); in visitFADD()
16296 return DAG.getNode(ISD::FSUB, DL, VT, N0, Add); in visitFADD()
16307 return DAG.getConstantFP(0.0, DL, VT); in visitFADD()
16311 return DAG.getConstantFP(0.0, DL, VT); in visitFADD()
16323 SDValue NewC = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), N1); in visitFADD()
16324 return DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(0), NewC); in visitFADD()
16330 if (TLI.isOperationLegalOrCustom(ISD::FMUL, VT) && !N0CFP && !N1CFP) { in visitFADD()
16339 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), in visitFADD()
16340 DAG.getConstantFP(1.0, DL, VT)); in visitFADD()
16341 return DAG.getNode(ISD::FMUL, DL, VT, N1, NewCFP); in visitFADD()
16348 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), in visitFADD()
16349 DAG.getConstantFP(2.0, DL, VT)); in visitFADD()
16350 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), NewCFP); in visitFADD()
16362 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1), in visitFADD()
16363 DAG.getConstantFP(1.0, DL, VT)); in visitFADD()
16364 return DAG.getNode(ISD::FMUL, DL, VT, N0, NewCFP); in visitFADD()
16371 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1), in visitFADD()
16372 DAG.getConstantFP(2.0, DL, VT)); in visitFADD()
16373 return DAG.getNode(ISD::FMUL, DL, VT, N1.getOperand(0), NewCFP); in visitFADD()
16383 return DAG.getNode(ISD::FMUL, DL, VT, N1, in visitFADD()
16384 DAG.getConstantFP(3.0, DL, VT)); in visitFADD()
16394 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFADD()
16395 DAG.getConstantFP(3.0, DL, VT)); in visitFADD()
16404 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), in visitFADD()
16405 DAG.getConstantFP(4.0, DL, VT)); in visitFADD()
16411 VT, N0, N1, Flags)) in visitFADD()
16428 EVT VT = N->getValueType(0); in visitSTRICT_FADD() local
16434 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::STRICT_FSUB, VT)) in visitSTRICT_FADD()
16437 return DAG.getNode(ISD::STRICT_FSUB, DL, DAG.getVTList(VT, ChainVT), in visitSTRICT_FADD()
16442 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::STRICT_FSUB, VT)) in visitSTRICT_FADD()
16445 return DAG.getNode(ISD::STRICT_FSUB, DL, DAG.getVTList(VT, ChainVT), in visitSTRICT_FADD()
16456 EVT VT = N->getValueType(0); in visitFSUB() local
16466 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FSUB, DL, VT, {N0, N1})) in visitFSUB()
16470 if (VT.isVector()) in visitFSUB()
16488 return DAG.getConstantFP(0.0f, DL, VT); in visitFSUB()
16499 DenormalMode DenormMode = DAG.getDenormalMode(VT); in visitFSUB()
16504 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) in visitFSUB()
16505 return DAG.getNode(ISD::FNEG, DL, VT, N1); in visitFSUB()
16515 return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(1)); in visitFSUB()
16518 return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(0)); in visitFSUB()
16524 return DAG.getNode(ISD::FADD, DL, VT, N0, NegN1); in visitFSUB()
16551 EVT VT = N->getValueType(0); in combineFMulOrFDivWithIntPow2() local
16617 EVT NewIntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits()); in combineFMulOrFDivWithIntPow2()
16618 if (VT.isVector()) in combineFMulOrFDivWithIntPow2()
16620 VT.getVectorElementCount()); in combineFMulOrFDivWithIntPow2()
16637 SDValue ResAsFP = DAG.getBitcast(VT, ResAsInt); in combineFMulOrFDivWithIntPow2()
16645 EVT VT = N->getValueType(0); in visitFMUL() local
16655 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FMUL, DL, VT, {N0, N1})) in visitFMUL()
16661 return DAG.getNode(ISD::FMUL, DL, VT, N1, N0); in visitFMUL()
16664 if (VT.isVector()) in visitFMUL()
16681 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, N01, N1); in visitFMUL()
16682 return DAG.getNode(ISD::FMUL, DL, VT, N00, MulConsts); in visitFMUL()
16690 const SDValue Two = DAG.getConstantFP(2.0, DL, VT); in visitFMUL()
16691 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, Two, N1); in visitFMUL()
16692 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), MulConsts); in visitFMUL()
16697 VT, N0, N1, Flags)) in visitFMUL()
16703 return DAG.getNode(ISD::FADD, DL, VT, N0, N0); in visitFMUL()
16707 if (!LegalOperations || TLI.isOperationLegal(ISD::FSUB, VT)) { in visitFMUL()
16708 return DAG.getNode(ISD::FSUB, DL, VT, in visitFMUL()
16709 DAG.getConstantFP(-0.0, DL, VT), N0, Flags); in visitFMUL()
16726 return DAG.getNode(ISD::FMUL, DL, VT, NegN0, NegN1); in visitFMUL()
16733 TLI.isOperationLegal(ISD::FABS, VT)) { in visitFMUL()
16764 TLI.isOperationLegal(ISD::FNEG, VT)) in visitFMUL()
16765 return DAG.getNode(ISD::FNEG, DL, VT, in visitFMUL()
16766 DAG.getNode(ISD::FABS, DL, VT, X)); in visitFMUL()
16768 return DAG.getNode(ISD::FABS, DL, VT, X); in visitFMUL()
16795 EVT VT = N->getValueType(0); in visitFMA() local
16809 return matcher.getNode(ISD::FMA, DL, VT, N0, N1, N2); in visitFMA()
16825 return matcher.getNode(ISD::FMA, DL, VT, NegN0, NegN1, N2); in visitFMA()
16838 return matcher.getNode(ISD::FADD, SDLoc(N), VT, N1, N2); in visitFMA()
16840 return matcher.getNode(ISD::FADD, SDLoc(N), VT, N0, N2); in visitFMA()
16845 return matcher.getNode(ISD::FMA, SDLoc(N), VT, N1, N0, N2); in visitFMA()
16853 ISD::FMUL, DL, VT, N0, in visitFMA()
16854 matcher.getNode(ISD::FADD, DL, VT, N1, N2.getOperand(1))); in visitFMA()
16862 ISD::FMA, DL, VT, N0.getOperand(0), in visitFMA()
16863 matcher.getNode(ISD::FMUL, DL, VT, N1, N0.getOperand(1)), N2); in visitFMA()
16871 return matcher.getNode(ISD::FADD, DL, VT, N0, N2); in visitFMA()
16874 (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))) { in visitFMA()
16875 SDValue RHSNeg = matcher.getNode(ISD::FNEG, DL, VT, N0); in visitFMA()
16877 return matcher.getNode(ISD::FADD, DL, VT, N2, RHSNeg); in visitFMA()
16882 (TLI.isOperationLegal(ISD::ConstantFP, VT) || in visitFMA()
16884 !TLI.isFPImmLegal(N1CFP->getValueAPF(), VT, ForCodeSize)))) { in visitFMA()
16885 return matcher.getNode(ISD::FMA, DL, VT, N0.getOperand(0), in visitFMA()
16886 matcher.getNode(ISD::FNEG, DL, VT, N1), N2); in visitFMA()
16894 return matcher.getNode(ISD::FMUL, DL, VT, N0, in visitFMA()
16895 matcher.getNode(ISD::FADD, DL, VT, N1, in visitFMA()
16896 DAG.getConstantFP(1.0, DL, VT))); in visitFMA()
16901 return matcher.getNode(ISD::FMUL, DL, VT, N0, in visitFMA()
16902 matcher.getNode(ISD::FADD, DL, VT, N1, in visitFMA()
16903 DAG.getConstantFP(-1.0, DL, VT))); in visitFMA()
16909 if (!TLI.isFNegFree(VT)) in visitFMA()
16912 return matcher.getNode(ISD::FNEG, DL, VT, Neg); in visitFMA()
16920 EVT VT = N->getValueType(0); in visitFMAD() local
16926 return DAG.getNode(ISD::FMAD, DL, VT, N0, N1, N2); in visitFMAD()
16960 EVT VT = N->getValueType(0); in combineRepeatedFPDivisors() local
16961 if (VT.isVector() && DAG.isSplatValue(N1)) in combineRepeatedFPDivisors()
16962 NumElts = VT.getVectorMinNumElements(); in combineRepeatedFPDivisors()
16992 SDValue FPOne = DAG.getConstantFP(1.0, DL, VT); in combineRepeatedFPDivisors()
16993 SDValue Reciprocal = DAG.getNode(ISD::FDIV, DL, VT, FPOne, N1, Flags); in combineRepeatedFPDivisors()
16999 SDValue NewNode = DAG.getNode(ISD::FMUL, SDLoc(U), VT, Dividend, in combineRepeatedFPDivisors()
17014 EVT VT = N->getValueType(0); in visitFDIV() local
17024 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FDIV, DL, VT, {N0, N1})) in visitFDIV()
17028 if (VT.isVector()) in visitFDIV()
17052 TLI.isOperationLegal(ISD::ConstantFP, VT) || in visitFDIV()
17053 TLI.isFPImmLegal(Recip, VT, ForCodeSize))) in visitFDIV()
17054 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFDIV()
17055 DAG.getConstantFP(Recip, DL, VT)); in visitFDIV()
17062 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
17067 RV = DAG.getNode(ISD::FP_EXTEND, SDLoc(N1), VT, RV); in visitFDIV()
17069 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
17075 RV = DAG.getNode(ISD::FP_ROUND, SDLoc(N1), VT, RV, N1.getOperand(1)); in visitFDIV()
17077 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
17103 SDValue AA = DAG.getNode(ISD::FMUL, DL, VT, A, A); in visitFDIV()
17105 DAG.getNode(ISD::FMUL, DL, VT, AA, Sqrt.getOperand(0)); in visitFDIV()
17107 return DAG.getNode(ISD::FMUL, DL, VT, N0, Rsqrt); in visitFDIV()
17117 SDValue Div = DAG.getNode(ISD::FDIV, SDLoc(N1), VT, Rsqrt, Y); in visitFDIV()
17119 return DAG.getNode(ISD::FMUL, DL, VT, N0, Div); in visitFDIV()
17149 return DAG.getNode(ISD::FDIV, SDLoc(N), VT, NegN0, NegN1); in visitFDIV()
17161 EVT VT = N->getValueType(0); in visitFREM() local
17169 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FREM, SDLoc(N), VT, {N0, N1})) in visitFREM()
17231 EVT VT = N->getValueType(0); in visitFCOPYSIGN() local
17235 DAG.FoldConstantArithmetic(ISD::FCOPYSIGN, SDLoc(N), VT, {N0, N1})) in visitFCOPYSIGN()
17243 if (!LegalOperations || TLI.isOperationLegal(ISD::FABS, VT)) in visitFCOPYSIGN()
17244 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFCOPYSIGN()
17246 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) in visitFCOPYSIGN()
17247 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, in visitFCOPYSIGN()
17248 DAG.getNode(ISD::FABS, SDLoc(N0), VT, N0)); in visitFCOPYSIGN()
17257 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0.getOperand(0), N1); in visitFCOPYSIGN()
17261 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFCOPYSIGN()
17265 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1.getOperand(1)); in visitFCOPYSIGN()
17270 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1.getOperand(0)); in visitFCOPYSIGN()
17285 EVT VT = N->getValueType(0); in visitFPOW() local
17286 if ((VT == MVT::f32 && ExponentC->getValueAPF().isExactlyValue(1.0f/3.0f)) || in visitFPOW()
17287 (VT == MVT::f64 && ExponentC->getValueAPF().isExactlyValue(1.0/3.0))) { in visitFPOW()
17302 (!DAG.getTargetLoweringInfo().isOperationExpand(ISD::FPOW, VT) && in visitFPOW()
17303 DAG.getTargetLoweringInfo().isOperationExpand(ISD::FCBRT, VT))) in visitFPOW()
17306 return DAG.getNode(ISD::FCBRT, SDLoc(N), VT, N->getOperand(0)); in visitFPOW()
17331 if (!DAG.getTargetLoweringInfo().isOperationLegalOrCustom(ISD::FSQRT, VT)) in visitFPOW()
17341 SDValue Sqrt = DAG.getNode(ISD::FSQRT, DL, VT, N->getOperand(0)); in visitFPOW()
17342 SDValue SqrtSqrt = DAG.getNode(ISD::FSQRT, DL, VT, Sqrt); in visitFPOW()
17346 return DAG.getNode(ISD::FMUL, DL, VT, Sqrt, SqrtSqrt); in visitFPOW()
17360 EVT VT = N->getValueType(0); in foldFPToIntToFP() local
17361 if (!TLI.isOperationLegal(ISD::FTRUNC, VT) || in foldFPToIntToFP()
17369 N0.getOperand(0).getValueType() == VT) in foldFPToIntToFP()
17370 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0.getOperand(0)); in foldFPToIntToFP()
17373 N0.getOperand(0).getValueType() == VT) in foldFPToIntToFP()
17374 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0.getOperand(0)); in foldFPToIntToFP()
17381 EVT VT = N->getValueType(0); in visitSINT_TO_FP() local
17386 return DAG.getConstantFP(0.0, SDLoc(N), VT); in visitSINT_TO_FP()
17392 TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) in visitSINT_TO_FP()
17393 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); in visitSINT_TO_FP()
17401 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); in visitSINT_TO_FP()
17407 !VT.isVector() && in visitSINT_TO_FP()
17408 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitSINT_TO_FP()
17410 return DAG.getSelect(DL, VT, N0, DAG.getConstantFP(-1.0, DL, VT), in visitSINT_TO_FP()
17411 DAG.getConstantFP(0.0, DL, VT)); in visitSINT_TO_FP()
17417 N0.getOperand(0).getOpcode() == ISD::SETCC && !VT.isVector() && in visitSINT_TO_FP()
17418 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitSINT_TO_FP()
17420 return DAG.getSelect(DL, VT, N0.getOperand(0), in visitSINT_TO_FP()
17421 DAG.getConstantFP(1.0, DL, VT), in visitSINT_TO_FP()
17422 DAG.getConstantFP(0.0, DL, VT)); in visitSINT_TO_FP()
17433 EVT VT = N->getValueType(0); in visitUINT_TO_FP() local
17438 return DAG.getConstantFP(0.0, SDLoc(N), VT); in visitUINT_TO_FP()
17444 TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) in visitUINT_TO_FP()
17445 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); in visitUINT_TO_FP()
17453 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); in visitUINT_TO_FP()
17457 if (N0.getOpcode() == ISD::SETCC && !VT.isVector() && in visitUINT_TO_FP()
17458 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitUINT_TO_FP()
17460 return DAG.getSelect(DL, VT, N0, DAG.getConstantFP(1.0, DL, VT), in visitUINT_TO_FP()
17461 DAG.getConstantFP(0.0, DL, VT)); in visitUINT_TO_FP()
17473 EVT VT = N->getValueType(0); in FoldIntToFPToInt() local
17493 unsigned OutputSize = (int)VT.getScalarSizeInBits(); in FoldIntToFPToInt()
17500 if (VT.getScalarSizeInBits() > SrcVT.getScalarSizeInBits()) { in FoldIntToFPToInt()
17503 return DAG.getNode(ExtOp, SDLoc(N), VT, Src); in FoldIntToFPToInt()
17505 if (VT.getScalarSizeInBits() < SrcVT.getScalarSizeInBits()) in FoldIntToFPToInt()
17506 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Src); in FoldIntToFPToInt()
17507 return DAG.getBitcast(VT, Src); in FoldIntToFPToInt()
17514 EVT VT = N->getValueType(0); in visitFP_TO_SINT() local
17518 return DAG.getUNDEF(VT); in visitFP_TO_SINT()
17522 return DAG.getNode(ISD::FP_TO_SINT, SDLoc(N), VT, N0); in visitFP_TO_SINT()
17529 EVT VT = N->getValueType(0); in visitFP_TO_UINT() local
17533 return DAG.getUNDEF(VT); in visitFP_TO_UINT()
17537 return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), VT, N0); in visitFP_TO_UINT()
17544 EVT VT = N->getValueType(0); in visitXRINT() local
17548 return DAG.getUNDEF(VT); in visitXRINT()
17552 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N0); in visitXRINT()
17560 EVT VT = N->getValueType(0); in visitFP_ROUND() local
17564 DAG.FoldConstantArithmetic(ISD::FP_ROUND, SDLoc(N), VT, {N0, N1})) in visitFP_ROUND()
17568 if (N0.getOpcode() == ISD::FP_EXTEND && VT == N0.getOperand(0).getValueType()) in visitFP_ROUND()
17577 if (!hasOperation(ISD::FP_ROUND, VT)) in visitFP_ROUND()
17587 if (N0.getOperand(0).getValueType() == MVT::f80 && VT == MVT::f16) in visitFP_ROUND()
17598 ISD::FP_ROUND, DL, VT, N0.getOperand(0), in visitFP_ROUND()
17609 CanCombineFCOPYSIGN_EXTEND_ROUND(VT, in visitFP_ROUND()
17611 SDValue Tmp = DAG.getNode(ISD::FP_ROUND, SDLoc(N0), VT, in visitFP_ROUND()
17614 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, in visitFP_ROUND()
17626 EVT VT = N->getValueType(0); in visitFP_EXTEND() local
17628 if (VT.isVector()) in visitFP_EXTEND()
17639 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, N0); in visitFP_EXTEND()
17643 TLI.getOperationAction(ISD::FP16_TO_FP, VT) == TargetLowering::Legal) in visitFP_EXTEND()
17644 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), VT, N0.getOperand(0)); in visitFP_EXTEND()
17651 if (In.getValueType() == VT) return In; in visitFP_EXTEND()
17652 if (VT.bitsLT(In.getValueType())) in visitFP_EXTEND()
17653 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, in visitFP_EXTEND()
17655 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, In); in visitFP_EXTEND()
17660 TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, VT, N0.getValueType())) { in visitFP_EXTEND()
17662 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, in visitFP_EXTEND()
17683 EVT VT = N->getValueType(0); in visitFCEIL() local
17687 return DAG.getNode(ISD::FCEIL, SDLoc(N), VT, N0); in visitFCEIL()
17694 EVT VT = N->getValueType(0); in visitFTRUNC() local
17698 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0); in visitFTRUNC()
17728 EVT VT = N->getValueType(0); in visitFFLOOR() local
17732 return DAG.getNode(ISD::FFLOOR, SDLoc(N), VT, N0); in visitFFLOOR()
17739 EVT VT = N->getValueType(0); in visitFNEG() local
17744 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0); in visitFNEG()
17757 return DAG.getNode(ISD::FSUB, SDLoc(N), VT, N0.getOperand(1), in visitFNEG()
17770 EVT VT = N->getValueType(0); in visitFMinMax() local
17778 if (SDValue C = DAG.FoldConstantArithmetic(Opc, SDLoc(N), VT, {N0, N1})) in visitFMinMax()
17784 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0); in visitFMinMax()
17819 Opc, SDLoc(N), VT, N0, N1, Flags)) in visitFMinMax()
17827 EVT VT = N->getValueType(0); in visitFABS() local
17831 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFABS()
17840 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0.getOperand(0)); in visitFABS()
18068 EVT VT = LD->getMemoryVT(); in getCombineLoadStoreParts() local
18069 if (!TLI.isIndexedLoadLegal(Inc, VT) && !TLI.isIndexedLoadLegal(Dec, VT)) in getCombineLoadStoreParts()
18075 EVT VT = ST->getMemoryVT(); in getCombineLoadStoreParts() local
18076 if (!TLI.isIndexedStoreLegal(Inc, VT) && !TLI.isIndexedStoreLegal(Dec, VT)) in getCombineLoadStoreParts()
18083 EVT VT = LD->getMemoryVT(); in getCombineLoadStoreParts() local
18084 if (!TLI.isIndexedMaskedLoadLegal(Inc, VT) && in getCombineLoadStoreParts()
18085 !TLI.isIndexedMaskedLoadLegal(Dec, VT)) in getCombineLoadStoreParts()
18092 EVT VT = ST->getMemoryVT(); in getCombineLoadStoreParts() local
18093 if (!TLI.isIndexedMaskedStoreLegal(Inc, VT) && in getCombineLoadStoreParts()
18094 !TLI.isIndexedMaskedStoreLegal(Dec, VT)) in getCombineLoadStoreParts()
19481 MVT VT = MVT::getIntegerVT(NumBytes * 8); in ShrinkLoadReplaceStoreWithStore() local
19484 if (DC->isTypeLegal(VT)) in ShrinkLoadReplaceStoreWithStore()
19487 TLI.isTruncStoreLegal(IVal.getValueType(), VT)) in ShrinkLoadReplaceStoreWithStore()
19498 !TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, in ShrinkLoadReplaceStoreWithStore()
19528 VT, St->getOriginalAlign()); in ShrinkLoadReplaceStoreWithStore()
19531 IVal = DAG.getNode(ISD::TRUNCATE, SDLoc(IVal), VT, IVal); in ShrinkLoadReplaceStoreWithStore()
19551 EVT VT = Value.getValueType(); in ReduceLoadOpStoreWidth() local
19553 if (ST->isTruncatingStore() || VT.isVector()) in ReduceLoadOpStoreWidth()
19615 !TLI.isNarrowingProfitable(VT, NewVT))) { in ReduceLoadOpStoreWidth()
19681 EVT VT = LD->getMemoryVT(); in TransformFPLoadStorePair() local
19682 if (!VT.isFloatingPoint() || in TransformFPLoadStorePair()
19683 VT != ST->getMemoryVT() || in TransformFPLoadStorePair()
19690 TypeSize VTSize = VT.getSizeInBits(); in TransformFPLoadStorePair()
19701 !TLI.isDesirableToTransformToIntegerOp(ISD::LOAD, VT) || in TransformFPLoadStorePair()
19702 !TLI.isDesirableToTransformToIntegerOp(ISD::STORE, VT) || in TransformFPLoadStorePair()
21434 EVT VT = EVT::getIntegerVT(*DAG.getContext(), HalfValBitSize); in splitMergedValStore() local
21435 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Lo.getOperand(0)); in splitMergedValStore()
21436 Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Hi.getOperand(0)); in splitMergedValStore()
21566 EVT VT = DestVec.getValueType(); in combineInsertEltToShuffle() local
21572 unsigned ExtendRatio = VT.getSizeInBits() / SubVecVT.getSizeInBits(); in combineInsertEltToShuffle()
21607 return DAG.getBitcast(VT, Shuf); in combineInsertEltToShuffle()
21614 EVT VT = N->getValueType(0); in combineInsertEltToLoad() local
21617 if (!VT.isFixedLengthVector() || in combineInsertEltToLoad()
21618 (InsIndex != 0 && InsIndex != VT.getVectorNumElements() - 1)) in combineInsertEltToLoad()
21628 (InsIndex == VT.getVectorNumElements() - 1 && in combineInsertEltToLoad()
21672 VecLoad, ScalarLoad, VT.getVectorNumElements() * EltSize / 8, -1)) in combineInsertEltToLoad()
21700 return Extend ? DAG.getNode(Extend, DL, VT, Load) : Load; in combineInsertEltToLoad()
21709 EVT VT = InVec.getValueType(); in visitINSERT_VECTOR_ELT() local
21713 if (IndexC && VT.isFixedLengthVector() && in visitINSERT_VECTOR_ELT()
21714 IndexC->getZExtValue() >= VT.getVectorNumElements()) in visitINSERT_VECTOR_ELT()
21715 return DAG.getUNDEF(VT); in visitINSERT_VECTOR_ELT()
21726 if (InVec.isUndef() && TLI.shouldSplatInsEltVarIndex(VT)) in visitINSERT_VECTOR_ELT()
21727 return DAG.getSplat(VT, DL, InVal); in visitINSERT_VECTOR_ELT()
21731 if (VT.isScalableVector()) in visitINSERT_VECTOR_ELT()
21734 unsigned NumElts = VT.getVectorNumElements(); in visitINSERT_VECTOR_ELT()
21743 InVal.getOperand(0).getValueType() == VT && in visitINSERT_VECTOR_ELT()
21760 SDValue NewOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, in visitINSERT_VECTOR_ELT()
21764 VT, NewOp, InVec.getOperand(1), InVec.getOperand(2)); in visitINSERT_VECTOR_ELT()
21778 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) { in visitINSERT_VECTOR_ELT()
21781 return DAG.getBuildVector(VT, DL, {InVal}); in visitINSERT_VECTOR_ELT()
21789 if (VT.isInteger()) { in visitINSERT_VECTOR_ELT()
21802 Op = VT.isInteger() ? DAG.getAnyExtOrTrunc(Op, DL, MaxEltVT) : Op; in visitINSERT_VECTOR_ELT()
21806 return DAG.getBuildVector(VT, DL, Ops); in visitINSERT_VECTOR_ELT()
21868 TLI.buildLegalVectorShuffle(VT, DL, LHS, RHS, Mask, DAG)) in visitINSERT_VECTOR_ELT()
21882 return DAG.getNode(ISD::AND, DL, VT, CurVec, in visitINSERT_VECTOR_ELT()
21883 DAG.getBuildVector(VT, DL, Mask)); in visitINSERT_VECTOR_ELT()
21898 SDValue Zero = VT.isInteger() ? DAG.getConstant(0, DL, MaxEltVT) in visitINSERT_VECTOR_ELT()
22015 EVT VT = ExtElt->getValueType(0); in scalarizeExtractedBinop() local
22016 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Index); in scalarizeExtractedBinop()
22017 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op1, Index); in scalarizeExtractedBinop()
22018 return DAG.getNode(Vec.getOpcode(), DL, VT, Ext0, Ext1); in scalarizeExtractedBinop()
22533 EVT VT = N->getValueType(0); in reduceBuildVecExtToExtBuildVec() local
22577 EVT OutScalarTy = VT.getScalarType(); in reduceBuildVecExtToExtBuildVec()
22599 unsigned NewBVElems = ElemRatio * VT.getVectorNumElements(); in reduceBuildVecExtToExtBuildVec()
22622 assert(VecVT.getSizeInBits() == VT.getSizeInBits() && in reduceBuildVecExtToExtBuildVec()
22627 TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))) in reduceBuildVecExtToExtBuildVec()
22636 return DAG.getBitcast(VT, BV); in reduceBuildVecExtToExtBuildVec()
22646 EVT VT = N->getValueType(0); in reduceBuildVecTruncToBitCast() local
22650 if (Level < AfterLegalizeTypes && TLI.isTypeLegal(VT)) in reduceBuildVecTruncToBitCast()
22658 EVT OutScalarTy = VT.getScalarType(); in reduceBuildVecTruncToBitCast()
22717 if (!Src || Src.getValueType().getSizeInBits() != VT.getSizeInBits()) in reduceBuildVecTruncToBitCast()
22720 return DAG.getBitcast(VT, Src); in reduceBuildVecTruncToBitCast()
22729 EVT VT = N->getValueType(0); in createBuildVecShuffle() local
22733 unsigned NumElems = VT.getVectorNumElements(); in createBuildVecShuffle()
22740 uint64_t VTSize = VT.getFixedSizeInBits(); in createBuildVecShuffle()
22749 if (InVT1 != VT || InVT2 != VT) { in createBuildVecShuffle()
22758 VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in createBuildVecShuffle()
22761 if (!TLI.isExtractSubvectorCheap(VT, InVT1, NumElems)) in createBuildVecShuffle()
22767 VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1, in createBuildVecShuffle()
22769 VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1, ZeroIdx); in createBuildVecShuffle()
22799 VecIn2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in createBuildVecShuffle()
22801 if (!TLI.isExtractSubvectorCheap(VT, InVT1, NumElems) || in createBuildVecShuffle()
22806 if (VT.getVectorNumElements() <= 2 || !VecIn2.getNode()) in createBuildVecShuffle()
22860 Shuffle = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuffle, ZeroIdx); in createBuildVecShuffle()
22891 EVT VT = BV->getValueType(0); in reduceBuildVecToShuffleWithZero() local
22896 Zext.getValueSizeInBits() != VT.getScalarSizeInBits()) in reduceBuildVecToShuffleWithZero()
22905 Extract.getOperand(0).getValueSizeInBits() != VT.getSizeInBits()) in reduceBuildVecToShuffleWithZero()
22938 return DAG.getBitcast(VT, Shuf); in reduceBuildVecToShuffleWithZero()
22955 EVT VT = N->getValueType(0); in reduceBuildVecToShuffle() local
22958 if (!isTypeLegal(VT)) in reduceBuildVecToShuffle()
22965 if (LegalOperations && !TLI.isOperationLegal(ISD::VECTOR_SHUFFLE, VT)) in reduceBuildVecToShuffle()
23011 if (VT.getVectorElementType() != in reduceBuildVecToShuffle()
23133 Shuffles.push_back(VT.isInteger() ? DAG.getConstant(0, DL, VT) in reduceBuildVecToShuffle()
23134 : DAG.getConstantFP(0.0, DL, VT)); in reduceBuildVecToShuffle()
23160 Shuffles.push_back(DAG.getUNDEF(VT)); in reduceBuildVecToShuffle()
23164 Shuffles[CurSize] = DAG.getUNDEF(VT); in reduceBuildVecToShuffle()
23203 Shuffles[In] = DAG.getVectorShuffle(VT, DL, L, R, Mask); in reduceBuildVecToShuffle()
23217 EVT VT = N->getValueType(0); in convertBuildVecZextToZext() local
23237 if (Offset < 0 || (Offset % VT.getVectorNumElements()) != 0) in convertBuildVecZextToZext()
23259 VT, In); in convertBuildVecZextToZext()
23278 EVT VT = N->getValueType(0); in convertBuildVecZextToBuildVecWithZeros() local
23280 assert(!VT.isScalableVector() && "Encountered scalable BUILD_VECTOR?"); in convertBuildVecZextToBuildVecWithZeros()
23288 unsigned EltBitwidth = VT.getScalarSizeInBits(); in convertBuildVecZextToBuildVecWithZeros()
23294 APInt KnownZeroOps(VT.getVectorNumElements(), 0); in convertBuildVecZextToBuildVecWithZeros()
23379 NewBV = DAG.getBitcast(VT, NewBV); in convertBuildVecZextToBuildVecWithZeros()
23384 EVT VT = N->getValueType(0); in visitBUILD_VECTOR() local
23388 return DAG.getUNDEF(VT); in visitBUILD_VECTOR()
23410 return DAG.getBitcast(VT, Concat); in visitBUILD_VECTOR()
23462 if (TLI.getOperationAction(ISD::SPLAT_VECTOR, VT) != TargetLowering::Expand) in visitBUILD_VECTOR()
23465 return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), VT, V); in visitBUILD_VECTOR()
23480 EVT VT = N->getValueType(0); in combineConcatVectorOfScalars() local
23528 VT.getSizeInBits() / SVT.getSizeInBits()); in combineConcatVectorOfScalars()
23529 return DAG.getBitcast(VT, DAG.getBuildVector(VecVT, DL, Ops)); in combineConcatVectorOfScalars()
23537 EVT VT = N->getValueType(0); in combineConcatVectorOfConcatVectors() local
23567 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, ConcatOps); in combineConcatVectorOfConcatVectors()
23575 EVT VT = N->getValueType(0); in combineConcatVectorOfExtracts() local
23579 if (VT.isScalableVector()) in combineConcatVectorOfExtracts()
23582 int NumElts = VT.getVectorNumElements(); in combineConcatVectorOfExtracts()
23585 SDValue SV0 = DAG.getUNDEF(VT), SV1 = DAG.getUNDEF(VT); in combineConcatVectorOfExtracts()
23617 if (ExtVT.getSizeInBits() != VT.getSizeInBits()) in combineConcatVectorOfExtracts()
23644 return TLI.buildLegalVectorShuffle(VT, SDLoc(N), DAG.getBitcast(VT, SV0), in combineConcatVectorOfExtracts()
23645 DAG.getBitcast(VT, SV1), Mask, DAG); in combineConcatVectorOfExtracts()
23683 EVT VT = N->getValueType(0); in combineConcatVectorOfCasts() local
23692 !TLI.isTypeLegal(VT)) in combineConcatVectorOfCasts()
23697 if (!TLI.isOperationLegalOrCustom(CastOpcode, VT) || in combineConcatVectorOfCasts()
23708 return DAG.getNode(CastOpcode, DL, VT, NewConcat); in combineConcatVectorOfCasts()
23717 EVT VT = N->getValueType(0); in combineConcatVectorOfShuffleAndItsOperands() local
23719 if (VT.isScalableVector()) in combineConcatVectorOfShuffleAndItsOperands()
23727 if ((LegalTypes && !TLI.isTypeLegal(VT)) || in combineConcatVectorOfShuffleAndItsOperands()
23729 !TLI.isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, VT))) in combineConcatVectorOfShuffleAndItsOperands()
23769 VT.getVectorNumElements()); in combineConcatVectorOfShuffleAndItsOperands()
23773 Mask.reserve(VT.getVectorNumElements()); in combineConcatVectorOfShuffleAndItsOperands()
23792 if (!TLI.isShuffleMaskLegal(Mask, VT)) in combineConcatVectorOfShuffleAndItsOperands()
23802 NewShufOp = DAG.getUNDEF(VT); in combineConcatVectorOfShuffleAndItsOperands()
23807 NewShufOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, ShufOpParts); in combineConcatVectorOfShuffleAndItsOperands()
23811 return DAG.getVectorShuffle(VT, dl, ShufOps[0], ShufOps[1], Mask); in combineConcatVectorOfShuffleAndItsOperands()
23820 EVT VT = N->getValueType(0); in visitCONCAT_VECTORS() local
23822 return DAG.getUNDEF(VT); in visitCONCAT_VECTORS()
23842 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in visitCONCAT_VECTORS()
23872 if (VT.getSizeInBits() % SclTy.getSizeInBits()) in visitCONCAT_VECTORS()
23875 unsigned VNTNumElms = VT.getSizeInBits() / SclTy.getSizeInBits(); in visitCONCAT_VECTORS()
23884 return DAG.getBitcast(VT, Res); in visitCONCAT_VECTORS()
23897 EVT SVT = VT.getScalarType(); in visitCONCAT_VECTORS()
23932 assert(VT.getVectorNumElements() == Opnds.size() && in visitCONCAT_VECTORS()
23934 return DAG.getBuildVector(VT, SDLoc(N), Opnds); in visitCONCAT_VECTORS()
23942 if (Level < AfterLegalizeVectorOps && TLI.isTypeLegal(VT)) { in visitCONCAT_VECTORS()
24097 EVT VT = Extract->getValueType(0); in narrowExtractedVectorBinOp() local
24099 assert(ExtractIndex % VT.getVectorNumElements() == 0 && in narrowExtractedVectorBinOp()
24104 unsigned NarrowWidth = VT.getSizeInBits(); in narrowExtractedVectorBinOp()
24126 unsigned ConcatOpNum = ExtractIndex / VT.getVectorNumElements(); in narrowExtractedVectorBinOp()
24139 return DAG.getBitcast(VT, NarrowBinOp); in narrowExtractedVectorBinOp()
24182 return DAG.getBitcast(VT, NarrowBinOp); in narrowExtractedVectorBinOp()
24201 EVT VT = Extract->getValueType(0); in narrowExtractedVectorLoad() local
24204 if (!VT.isByteSized()) in narrowExtractedVectorLoad()
24208 unsigned NumElts = VT.getVectorMinNumElements(); in narrowExtractedVectorLoad()
24220 TypeSize Offset = VT.getStoreSize() * (Index / NumElts); in narrowExtractedVectorLoad()
24223 if (!TLI.shouldReduceLoadWidth(Ld, Ld->getExtensionType(), VT)) in narrowExtractedVectorLoad()
24233 uint64_t StoreSize = MemoryLocation::getSizeOrUnknown(VT.getStoreSize()); in narrowExtractedVectorLoad()
24244 SDValue NewLd = DAG.getLoad(VT, DL, Ld->getChain(), NewAddr, MMO); in narrowExtractedVectorLoad()
24611 EVT VT = Shuf->getValueType(0); in foldShuffleOfConcatUndefs() local
24612 unsigned NumElts = VT.getVectorNumElements(); in foldShuffleOfConcatUndefs()
24631 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), in foldShuffleOfConcatUndefs()
24643 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Shuf0, Shuf1); in foldShuffleOfConcatUndefs()
24649 EVT VT = N->getValueType(0); in partitionShuffleOfConcats() local
24650 unsigned NumElts = VT.getVectorNumElements(); in partitionShuffleOfConcats()
24674 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, N0, N1); in partitionShuffleOfConcats()
24708 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in partitionShuffleOfConcats()
24730 EVT VT = SVN->getValueType(0); in combineShuffleOfScalars() local
24731 unsigned NumElts = VT.getVectorNumElements(); in combineShuffleOfScalars()
24764 SDValue Op = DAG.getUNDEF(VT.getScalarType()); in combineShuffleOfScalars()
24792 EVT SVT = VT.getScalarType(); in combineShuffleOfScalars()
24796 if (SVT != VT.getScalarType()) in combineShuffleOfScalars()
24802 return DAG.getBuildVector(VT, SDLoc(SVN), Ops); in combineShuffleOfScalars()
24811 unsigned Opcode, EVT VT, std::function<bool(unsigned)> Match, in canCombineShuffleToExtendVectorInreg() argument
24817 if (!VT.isInteger() || IsBigEndian) in canCombineShuffleToExtendVectorInreg()
24820 unsigned NumElts = VT.getVectorNumElements(); in canCombineShuffleToExtendVectorInreg()
24821 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in canCombineShuffleToExtendVectorInreg()
24852 EVT VT = SVN->getValueType(0); in combineShuffleToAnyExtendVectorInreg() local
24856 if (!VT.isInteger() || IsBigEndian) in combineShuffleToAnyExtendVectorInreg()
24860 auto isAnyExtend = [NumElts = VT.getVectorNumElements(), in combineShuffleToAnyExtendVectorInreg()
24877 Opcode, VT, isAnyExtend, DAG, TLI, /*LegalTypes=*/true, LegalOperations); in combineShuffleToAnyExtendVectorInreg()
24880 return DAG.getBitcast(VT, DAG.getNode(Opcode, SDLoc(SVN), *OutVT, N0)); in combineShuffleToAnyExtendVectorInreg()
24891 EVT VT = SVN->getValueType(0); in combineShuffleToZeroExtendVectorInReg() local
24892 assert(!VT.isScalableVector() && "Encountered scalable shuffle?"); in combineShuffleToZeroExtendVectorInReg()
24893 unsigned NumElts = VT.getVectorNumElements(); in combineShuffleToZeroExtendVectorInReg()
24894 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in combineShuffleToZeroExtendVectorInReg()
24898 if (!VT.isInteger() || IsBigEndian) in combineShuffleToZeroExtendVectorInReg()
24962 if (LegalTypes && !TLI.isTypeLegal(PrescaledVT) && TLI.isTypeLegal(VT)) in combineShuffleToZeroExtendVectorInReg()
25001 return DAG.getBitcast(VT, DAG.getNode(Opcode, SDLoc(SVN), *OutVT, in combineShuffleToZeroExtendVectorInReg()
25014 EVT VT = SVN->getValueType(0); in combineTruncationShuffle() local
25018 if (!VT.isInteger() || IsBigEndian) in combineTruncationShuffle()
25029 unsigned NumElts = VT.getVectorNumElements(); in combineTruncationShuffle()
25030 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in combineTruncationShuffle()
25061 return DAG.getBitcast(VT, N00); in combineTruncationShuffle()
25073 EVT VT = Shuf->getValueType(0); in combineShuffleOfSplatVal() local
25074 unsigned NumElts = VT.getVectorNumElements(); in combineShuffleOfSplatVal()
25102 return DAG.getUNDEF(VT); // All undef - result is undef. in combineShuffleOfSplatVal()
25114 return DAG.getVectorShuffle(VT, SDLoc(Shuf), Shuf->getOperand(0), in combineShuffleOfSplatVal()
25179 EVT VT = SVN->getValueType(0); in combineShuffleOfBitcast() local
25191 int VTLanes = VT.getVectorNumElements(); in combineShuffleOfBitcast()
25215 return DAG.getBitcast(VT, NewShuf); in combineShuffleOfBitcast()
25260 EVT VT = OuterShuf->getValueType(0); in formSplatFromShuffles() local
25261 assert(VT == InnerShuf->getValueType(0) && "Expected matching shuffle types"); in formSplatFromShuffles()
25262 if (!DAG.getTargetLoweringInfo().isShuffleMaskLegal(CombinedMask, VT)) in formSplatFromShuffles()
25265 return DAG.getVectorShuffle(VT, SDLoc(OuterShuf), InnerShuf->getOperand(0), in formSplatFromShuffles()
25375 EVT VT = N->getValueType(0); in visitVECTOR_SHUFFLE() local
25376 unsigned NumElts = VT.getVectorNumElements(); in visitVECTOR_SHUFFLE()
25381 assert(N0.getValueType() == VT && "Vector shuffle must be normalized in DAG"); in visitVECTOR_SHUFFLE()
25385 return DAG.getUNDEF(VT); in visitVECTOR_SHUFFLE()
25391 return DAG.getVectorShuffle(VT, SDLoc(N), N0, DAG.getUNDEF(VT), in visitVECTOR_SHUFFLE()
25411 return DAG.getVectorShuffle(VT, SDLoc(N), N0, N1, NewMask); in visitVECTOR_SHUFFLE()
25428 if (N0.hasOneUse() && TLI.isExtractVecEltCheap(VT, SplatIndex) && in visitVECTOR_SHUFFLE()
25434 EVT EltVT = VT.getScalarType(); in visitVECTOR_SHUFFLE()
25440 SDValue Insert = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, NewBO); in visitVECTOR_SHUFFLE()
25441 SmallVector<int, 16> ZeroMask(VT.getVectorNumElements(), 0); in visitVECTOR_SHUFFLE()
25442 return DAG.getVectorShuffle(VT, DL, Insert, DAG.getUNDEF(VT), ZeroMask); in visitVECTOR_SHUFFLE()
25447 if ((!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) && in visitVECTOR_SHUFFLE()
25450 return DAG.getSplatBuildVector(VT, SDLoc(N), N0.getOperand(0)); in visitVECTOR_SHUFFLE()
25455 return DAG.getSplatBuildVector(VT, SDLoc(N), N0.getOperand(1)); in visitVECTOR_SHUFFLE()
25464 if (VT.getScalarSizeInBits() <= N00VT.getScalarSizeInBits() && in visitVECTOR_SHUFFLE()
25465 VT.isInteger() && N00VT.isInteger()) { in visitVECTOR_SHUFFLE()
25467 TLI.getTypeToTransformTo(*DAG.getContext(), VT.getScalarType()); in visitVECTOR_SHUFFLE()
25470 return DAG.getSplatBuildVector(VT, SDLoc(N), Op); in visitVECTOR_SHUFFLE()
25517 if (V->getValueType(0) != VT) in visitVECTOR_SHUFFLE()
25518 NewBV = DAG.getBitcast(VT, NewBV); in visitVECTOR_SHUFFLE()
25566 if (TLI.isShuffleMaskLegal(NewMask, VT)) { in visitVECTOR_SHUFFLE()
25568 SDValue NewCat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, in visitVECTOR_SHUFFLE()
25570 return DAG.getVectorShuffle(VT, SDLoc(N), NewCat, N1, NewMask); in visitVECTOR_SHUFFLE()
25578 if (Level < AfterLegalizeVectorOps && TLI.isTypeLegal(VT) && in visitVECTOR_SHUFFLE()
25579 TLI.isOperationLegalOrCustom(ISD::INSERT_SUBVECTOR, VT)) { in visitVECTOR_SHUFFLE()
25619 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, LHS, in visitVECTOR_SHUFFLE()
25640 if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT)) { in visitVECTOR_SHUFFLE()
25662 EVT IntVT = VT.changeVectorElementTypeToInteger(); in visitVECTOR_SHUFFLE()
25663 EVT IntSVT = VT.getVectorElementType().changeTypeToInteger(); in visitVECTOR_SHUFFLE()
25681 VT, DAG.getVectorShuffle(IntVT, DL, DAG.getBitcast(IntVT, N0), in visitVECTOR_SHUFFLE()
25686 VT, DAG.getNode(ISD::AND, DL, IntVT, DAG.getBitcast(IntVT, N0), in visitVECTOR_SHUFFLE()
25694 if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT)) in visitVECTOR_SHUFFLE()
25703 TLI.isTypeLegal(VT)) { in visitVECTOR_SHUFFLE()
25707 EVT SVT = VT.getScalarType(); in visitVECTOR_SHUFFLE()
25712 EVT ScaleVT = SVT.bitsLT(InnerSVT) ? VT : InnerVT; in visitVECTOR_SHUFFLE()
25747 VT, DAG.getVectorShuffle(ScaleVT, SDLoc(N), SV0, SV1, NewMask)); in visitVECTOR_SHUFFLE()
25763 [NumElts, &VT](bool Commute, ShuffleVectorSDNode *SVN, in visitVECTOR_SHUFFLE()
25867 if (TLI.isShuffleMaskLegal(Mask, VT)) in visitVECTOR_SHUFFLE()
25872 return TLI.isShuffleMaskLegal(Mask, VT); in visitVECTOR_SHUFFLE()
25875 if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT)) { in visitVECTOR_SHUFFLE()
25884 assert(N1->getOperand(0).getValueType() == VT && in visitVECTOR_SHUFFLE()
25918 assert(OtherSV->getOperand(0).getValueType() == VT && in visitVECTOR_SHUFFLE()
25927 return DAG.getUNDEF(VT); in visitVECTOR_SHUFFLE()
25929 return DAG.getVectorShuffle(VT, SDLoc(N), in visitVECTOR_SHUFFLE()
25930 SV0 ? SV0 : DAG.getUNDEF(VT), in visitVECTOR_SHUFFLE()
25931 SV1 ? SV1 : DAG.getUNDEF(VT), Mask); in visitVECTOR_SHUFFLE()
25952 if (Op00.getValueType() == VT && Op10.getValueType() == VT && in visitVECTOR_SHUFFLE()
25953 Op01.getValueType() == VT && Op11.getValueType() == VT && in visitVECTOR_SHUFFLE()
26003 VT, DL, LeftSV0 ? LeftSV0 : DAG.getUNDEF(VT), in visitVECTOR_SHUFFLE()
26004 LeftSV1 ? LeftSV1 : DAG.getUNDEF(VT), LeftMask); in visitVECTOR_SHUFFLE()
26006 VT, DL, RightSV0 ? RightSV0 : DAG.getUNDEF(VT), in visitVECTOR_SHUFFLE()
26007 RightSV1 ? RightSV1 : DAG.getUNDEF(VT), RightMask); in visitVECTOR_SHUFFLE()
26008 return DAG.getNode(SrcOpcode, DL, VT, LHS, RHS); in visitVECTOR_SHUFFLE()
26029 EVT VT = N->getValueType(0); in visitSCALAR_TO_VECTOR() local
26030 if (!VT.isFixedLengthVector()) in visitSCALAR_TO_VECTOR()
26040 EVT VecEltVT = VT.getScalarType(); in visitSCALAR_TO_VECTOR()
26047 DAG.isSafeToSpeculativelyExecute(Opcode) && hasOperation(Opcode, VT)) { in visitSCALAR_TO_VECTOR()
26049 SmallVector<int, 8> ShufMask(VT.getVectorNumElements(), -1); in visitSCALAR_TO_VECTOR()
26057 EE.getOperand(0).getValueType() == VT && in visitSCALAR_TO_VECTOR()
26062 if (TLI.isShuffleMaskLegal(ShufMask, VT)) { in visitSCALAR_TO_VECTOR()
26065 DAG.getConstant(C->getAPIntValue(), DL, VT)}; in visitSCALAR_TO_VECTOR()
26066 SDValue VecBO = DAG.getNode(Opcode, DL, VT, V[i], V[1 - i]); in visitSCALAR_TO_VECTOR()
26067 return DAG.getVectorShuffle(VT, DL, VecBO, DAG.getUNDEF(VT), in visitSCALAR_TO_VECTOR()
26084 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Val); in visitSCALAR_TO_VECTOR()
26094 unsigned VTNumElts = VT.getVectorNumElements(); in visitSCALAR_TO_VECTOR()
26105 if (VT == SrcVT) in visitSCALAR_TO_VECTOR()
26122 EVT VT = N->getValueType(0); in visitINSERT_SUBVECTOR() local
26138 if (SrcVT == VT) in visitINSERT_SUBVECTOR()
26143 VT.isScalableVector() == SrcVT.isScalableVector()) { in visitINSERT_SUBVECTOR()
26144 if (VT.getVectorMinNumElements() >= SrcVT.getVectorMinNumElements()) in visitINSERT_SUBVECTOR()
26146 VT, N0, N1.getOperand(0), N2); in visitINSERT_SUBVECTOR()
26149 VT, N1.getOperand(0), N2); in visitINSERT_SUBVECTOR()
26157 return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), VT, N1.getOperand(0)); in visitINSERT_SUBVECTOR()
26167 VT.getVectorElementCount() && in visitINSERT_SUBVECTOR()
26169 VT.getSizeInBits()) { in visitINSERT_SUBVECTOR()
26170 return DAG.getBitcast(VT, N1.getOperand(0).getOperand(0)); in visitINSERT_SUBVECTOR()
26184 CN0VT.getVectorElementCount() == VT.getVectorElementCount()) { in visitINSERT_SUBVECTOR()
26187 return DAG.getBitcast(VT, NewINSERT); in visitINSERT_SUBVECTOR()
26197 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0.getOperand(0), in visitINSERT_SUBVECTOR()
26206 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0, in visitINSERT_SUBVECTOR()
26224 ElementCount NumElts = VT.getVectorElementCount(); in visitINSERT_SUBVECTOR()
26225 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in visitINSERT_SUBVECTOR()
26241 return DAG.getBitcast(VT, Res); in visitINSERT_SUBVECTOR()
26255 SDValue NewOp = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, in visitINSERT_SUBVECTOR()
26259 VT, NewOp, N0.getOperand(1), N0.getOperand(2)); in visitINSERT_SUBVECTOR()
26272 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in visitINSERT_SUBVECTOR()
26327 EVT VT = N0.getValueType(); in visitVECREDUCE() local
26331 if (VT.getVectorElementCount().isScalar()) { in visitVECREDUCE()
26334 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getVectorElementType(), N0, in visitVECREDUCE()
26346 if (!TLI.isOperationLegalOrCustom(Opcode, VT) && in visitVECREDUCE()
26347 TLI.isOperationLegalOrCustom(NewOpcode, VT) && in visitVECREDUCE()
26348 DAG.ComputeNumSignBits(N0) == VT.getScalarSizeInBits()) in visitVECREDUCE()
26537 EVT VT = N->getValueType(0); in XformToShuffleWithZero() local
26601 return DAG.getBitcast(VT, DAG.getVectorShuffle(ClearVT, DL, in XformToShuffleWithZero()
26626 EVT VT = N->getValueType(0); in scalarizeBinOpOfSplats() local
26627 EVT EltVT = VT.getVectorElementType(); in scalarizeBinOpOfSplats()
26643 !(IsBothSplatVector || TLI.isExtractVecEltCheap(VT, Index0)) || in scalarizeBinOpOfSplats()
26659 SmallVector<SDValue, 8> Ops(VT.getVectorNumElements(), DAG.getUNDEF(EltVT)); in scalarizeBinOpOfSplats()
26661 return DAG.getBuildVector(VT, DL, Ops); in scalarizeBinOpOfSplats()
26665 return DAG.getSplat(VT, DL, ScalarBO); in scalarizeBinOpOfSplats()
26670 EVT VT = N->getValueType(0); in SimplifyVCastOp() local
26671 assert(VT.isVector() && "SimplifyVCastOp only works on vectors!"); in SimplifyVCastOp()
26672 EVT EltVT = VT.getVectorElementType(); in SimplifyVCastOp()
26683 TLI.isExtractVecEltCheap(VT, Index0)) && in SimplifyVCastOp()
26692 if (VT.isScalableVector()) in SimplifyVCastOp()
26693 return DAG.getSplatVector(VT, DL, ScalarBO); in SimplifyVCastOp()
26694 SmallVector<SDValue, 8> Ops(VT.getVectorNumElements(), ScalarBO); in SimplifyVCastOp()
26695 return DAG.getBuildVector(VT, DL, Ops); in SimplifyVCastOp()
26703 EVT VT = N->getValueType(0); in SimplifyVBinOp() local
26704 assert(VT.isVector() && "SimplifyVBinOp only works on vectors!"); in SimplifyVBinOp()
26724 SDValue NewBinOp = DAG.getNode(Opcode, DL, VT, LHS.getOperand(0), in SimplifyVBinOp()
26727 return DAG.getVectorShuffle(VT, DL, NewBinOp, UndefV, Shuf0->getMask()); in SimplifyVBinOp()
26741 SDValue NewBinOp = DAG.getNode(Opcode, DL, VT, X, RHS, Flags); in SimplifyVBinOp()
26742 return DAG.getVectorShuffle(VT, DL, NewBinOp, DAG.getUNDEF(VT), in SimplifyVBinOp()
26750 SDValue NewBinOp = DAG.getNode(Opcode, DL, VT, LHS, X, Flags); in SimplifyVBinOp()
26751 return DAG.getVectorShuffle(VT, DL, NewBinOp, DAG.getUNDEF(VT), in SimplifyVBinOp()
26773 DAG.getNode(Opcode, DL, VT, DAG.getUNDEF(VT), DAG.getUNDEF(VT)); in SimplifyVBinOp()
26775 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, VecC, NarrowBO, Z); in SimplifyVBinOp()
26806 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in SimplifyVBinOp()
27171 EVT VT = N->getValueType(0); in foldSignChangeInBitcast() local
27173 bool IsFree = IsFabs ? TLI.isFAbsFree(VT) : TLI.isFNegFree(VT); in foldSignChangeInBitcast()
27205 return DAG.getBitcast(VT, Int); in foldSignChangeInBitcast()
27222 EVT VT = N2.getValueType(); in convertSelectOfFPConstantsToLoadOffset() local
27223 if (!TV || !FV || !TLI.isTypeLegal(VT)) in convertSelectOfFPConstantsToLoadOffset()
27227 if (TLI.getOperationAction(ISD::ConstantFP, VT) == TargetLowering::Legal || in convertSelectOfFPConstantsToLoadOffset()
27275 EVT VT = N2.getValueType(); in SimplifySelectCC() local
27304 N0->getValueType(0) == VT && isNullConstant(N1) && isNullConstant(N2)) { in SimplifySelectCC()
27310 if (TLI.shouldFoldSelectWithSingleBitTest(VT, AndMask)) { in SimplifySelectCC()
27315 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N0), VT, AndLHS, ShlAmt); in SimplifySelectCC()
27322 SDValue Shr = DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl, ShrAmt); in SimplifySelectCC()
27324 return DAG.getNode(ISD::AND, DL, VT, Shr, N3); in SimplifySelectCC()
27352 Temp = DAG.getZExtOrTrunc(SCC, SDLoc(N2), VT); in SimplifySelectCC()
27355 Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2), VT, SCC); in SimplifySelectCC()
27365 if (TLI.shouldAvoidTransformToShift(VT, ShCt)) in SimplifySelectCC()
27390 if (ValueOnZeroC->getAPIntValue() == VT.getSizeInBits()) { in SimplifySelectCC()
27396 (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ, VT))) in SimplifySelectCC()
27397 return DAG.getNode(ISD::CTTZ, DL, VT, N0); in SimplifySelectCC()
27403 (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ, VT))) in SimplifySelectCC()
27404 return DAG.getNode(ISD::CTLZ, DL, VT, N0); in SimplifySelectCC()
27415 !TLI.shouldAvoidTransformToShift(VT, CmpOpVT.getScalarSizeInBits() - 1)) { in SimplifySelectCC()
27419 return DAG.getNode(ISD::XOR, DL, VT, DAG.getSExtOrTrunc(ASR, DL, VT), in SimplifySelectCC()
27420 DAG.getSExtOrTrunc(CC == ISD::SETLT ? N3 : N2, DL, VT)); in SimplifySelectCC()
27432 SDValue DAGCombiner::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, in SimplifySetCC() argument
27437 return TLI.SimplifySetCC(VT, N0, N1, Cond, foldBooleans, DagCombineInfo, DL); in SimplifySetCC()
27532 static SDValue takeInexpensiveLog2(SelectionDAG &DAG, const SDLoc &DL, EVT VT, in takeInexpensiveLog2() argument
27535 assert(VT.isInteger() && "Only integer types are supported!"); in takeInexpensiveLog2()
27550 if (VT.isScalableVector()) in takeInexpensiveLog2()
27570 if (!VT.isVector()) in takeInexpensiveLog2()
27571 return DAG.getConstant(Pow2Constants.back().logBase2(), DL, VT); in takeInexpensiveLog2()
27576 DAG.getConstant(Pow2.logBase2(), DL, VT.getScalarType())); in takeInexpensiveLog2()
27577 return DAG.getBuildVector(VT, DL, Log2Ops); in takeInexpensiveLog2()
27600 if (SDValue LogX = takeInexpensiveLog2(DAG, DL, VT, Op.getOperand(0), in takeInexpensiveLog2()
27602 return DAG.getNode(ISD::ADD, DL, VT, LogX, in takeInexpensiveLog2()
27603 CastToVT(VT, Op.getOperand(1))); in takeInexpensiveLog2()
27609 if (SDValue LogX = takeInexpensiveLog2(DAG, DL, VT, Op.getOperand(1), in takeInexpensiveLog2()
27611 if (SDValue LogY = takeInexpensiveLog2(DAG, DL, VT, Op.getOperand(2), in takeInexpensiveLog2()
27613 return DAG.getSelect(DL, VT, Op.getOperand(0), LogX, LogY); in takeInexpensiveLog2()
27623 takeInexpensiveLog2(DAG, DL, VT, Op.getOperand(0), Depth + 1, in takeInexpensiveLog2()
27626 takeInexpensiveLog2(DAG, DL, VT, Op.getOperand(1), Depth + 1, in takeInexpensiveLog2()
27628 return DAG.getNode(Op.getOpcode(), DL, VT, LogX, LogY); in takeInexpensiveLog2()
27639 EVT VT = OutVT ? *OutVT : V.getValueType(); in BuildLogBase2() local
27641 takeInexpensiveLog2(DAG, DL, VT, V, /*Depth*/ 0, KnownNonZero); in BuildLogBase2()
27645 SDValue Ctlz = DAG.getNode(ISD::CTLZ, DL, VT, V); in BuildLogBase2()
27646 SDValue Base = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT); in BuildLogBase2()
27647 SDValue LogBase2 = DAG.getNode(ISD::SUB, DL, VT, Base, Ctlz); in BuildLogBase2()
27665 EVT VT = Op.getValueType(); in BuildDivEstimate() local
27666 if (VT.getScalarType() != MVT::f16 && VT.getScalarType() != MVT::f32 && in BuildDivEstimate()
27667 VT.getScalarType() != MVT::f64) in BuildDivEstimate()
27672 int Enabled = TLI.getRecipEstimateDivEnabled(VT, MF); in BuildDivEstimate()
27678 int Iterations = TLI.getDivRefinementSteps(VT, MF); in BuildDivEstimate()
27684 SDValue FPOne = DAG.getConstantFP(1.0, DL, VT); in BuildDivEstimate()
27692 MulEst = DAG.getNode(ISD::FMUL, DL, VT, N, Est, Flags); in BuildDivEstimate()
27696 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Op, MulEst, Flags); in BuildDivEstimate()
27699 NewEst = DAG.getNode(ISD::FSUB, DL, VT, in BuildDivEstimate()
27703 NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags); in BuildDivEstimate()
27706 Est = DAG.getNode(ISD::FADD, DL, VT, MulEst, NewEst, Flags); in BuildDivEstimate()
27711 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, N, Flags); in BuildDivEstimate()
27730 EVT VT = Arg.getValueType(); in buildSqrtNROneConst() local
27732 SDValue ThreeHalves = DAG.getConstantFP(1.5, DL, VT); in buildSqrtNROneConst()
27736 SDValue HalfArg = DAG.getNode(ISD::FMUL, DL, VT, ThreeHalves, Arg, Flags); in buildSqrtNROneConst()
27737 HalfArg = DAG.getNode(ISD::FSUB, DL, VT, HalfArg, Arg, Flags); in buildSqrtNROneConst()
27741 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, Est, Flags); in buildSqrtNROneConst()
27742 NewEst = DAG.getNode(ISD::FMUL, DL, VT, HalfArg, NewEst, Flags); in buildSqrtNROneConst()
27743 NewEst = DAG.getNode(ISD::FSUB, DL, VT, ThreeHalves, NewEst, Flags); in buildSqrtNROneConst()
27744 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags); in buildSqrtNROneConst()
27749 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, Arg, Flags); in buildSqrtNROneConst()
27762 EVT VT = Arg.getValueType(); in buildSqrtNRTwoConst() local
27764 SDValue MinusThree = DAG.getConstantFP(-3.0, DL, VT); in buildSqrtNRTwoConst()
27765 SDValue MinusHalf = DAG.getConstantFP(-0.5, DL, VT); in buildSqrtNRTwoConst()
27774 SDValue AE = DAG.getNode(ISD::FMUL, DL, VT, Arg, Est, Flags); in buildSqrtNRTwoConst()
27775 SDValue AEE = DAG.getNode(ISD::FMUL, DL, VT, AE, Est, Flags); in buildSqrtNRTwoConst()
27776 SDValue RHS = DAG.getNode(ISD::FADD, DL, VT, AEE, MinusThree, Flags); in buildSqrtNRTwoConst()
27784 LHS = DAG.getNode(ISD::FMUL, DL, VT, Est, MinusHalf, Flags); in buildSqrtNRTwoConst()
27787 LHS = DAG.getNode(ISD::FMUL, DL, VT, AE, MinusHalf, Flags); in buildSqrtNRTwoConst()
27790 Est = DAG.getNode(ISD::FMUL, DL, VT, LHS, RHS, Flags); in buildSqrtNRTwoConst()
27805 EVT VT = Op.getValueType(); in buildSqrtEstimateImpl() local
27806 if (VT.getScalarType() != MVT::f16 && VT.getScalarType() != MVT::f32 && in buildSqrtEstimateImpl()
27807 VT.getScalarType() != MVT::f64) in buildSqrtEstimateImpl()
27812 int Enabled = TLI.getRecipEstimateSqrtEnabled(VT, MF); in buildSqrtEstimateImpl()
27818 int Iterations = TLI.getSqrtRefinementSteps(VT, MF); in buildSqrtEstimateImpl()
27833 SDValue Test = TLI.getSqrtInputTest(Op, DAG, DAG.getDenormalMode(VT)); in buildSqrtEstimateImpl()
27839 Test.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT, DL, VT, in buildSqrtEstimateImpl()