Lines Matching refs:VT
246 for (MVT VT : MVT::all_valuetypes()) in DAGCombiner() local
247 if (EVT(VT).isSimple() && VT != MVT::Other && in DAGCombiner()
248 TLI.isTypeLegal(EVT(VT)) && in DAGCombiner()
249 VT.getSizeInBits().getKnownMinSize() >= MaximumLegalStoreInBits) in DAGCombiner()
250 MaximumLegalStoreInBits = VT.getSizeInBits().getKnownMinSize(); in DAGCombiner()
556 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
566 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
767 bool hasOperation(unsigned Opcode, EVT VT) { in hasOperation() argument
768 return TLI.isOperationLegalOrCustom(Opcode, VT, LegalOperations); in hasOperation()
786 bool isTypeLegal(const EVT &VT) { in isTypeLegal() argument
788 return TLI.isTypeLegal(VT); in isTypeLegal()
792 EVT getSetCCResultType(EVT VT) const { in getSetCCResultType()
793 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in getSetCCResultType()
1043 EVT VT = LoadStore->getMemoryVT(); in reassociationCanBreakAddressingModePattern() local
1045 Type *AccessTy = VT.getTypeForEVT(*DAG.getContext()); in reassociationCanBreakAddressingModePattern()
1070 EVT VT = LoadStore->getMemoryVT(); in reassociationCanBreakAddressingModePattern() local
1072 Type *AccessTy = VT.getTypeForEVT(*DAG.getContext()); in reassociationCanBreakAddressingModePattern()
1086 EVT VT = N0.getValueType(); in reassociateOpsCommutative() local
1097 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, {N01, N1})) in reassociateOpsCommutative()
1098 return DAG.getNode(Opc, DL, VT, N00, OpNode); in reassociateOpsCommutative()
1104 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N00, N1); in reassociateOpsCommutative()
1105 return DAG.getNode(Opc, DL, VT, OpNode, N01); in reassociateOpsCommutative()
1130 if (SDNode *NE = DAG.getNodeIfExists(Opc, DAG.getVTList(VT), {N00, N1})) { in reassociateOpsCommutative()
1133 if (!DAG.doesNodeExist(Opc, DAG.getVTList(VT), {SDValue(NE, 0), N01})) in reassociateOpsCommutative()
1134 return DAG.getNode(Opc, DL, VT, SDValue(NE, 0), N01); in reassociateOpsCommutative()
1140 if (SDNode *NE = DAG.getNodeIfExists(Opc, DAG.getVTList(VT), {N01, N1})) { in reassociateOpsCommutative()
1143 if (!DAG.doesNodeExist(Opc, DAG.getVTList(VT), {SDValue(NE, 0), N00})) in reassociateOpsCommutative()
1144 return DAG.getNode(Opc, DL, VT, SDValue(NE, 0), N00); in reassociateOpsCommutative()
1261 EVT VT = Load->getValueType(0); in ReplaceLoadWithPromotedLoad() local
1262 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0)); in ReplaceLoadWithPromotedLoad()
1348 EVT VT = Op.getValueType(); in PromoteIntBinOp() local
1349 if (VT.isVector() || !VT.isInteger()) in PromoteIntBinOp()
1355 if (TLI.isTypeDesirableForOp(Opc, VT)) in PromoteIntBinOp()
1358 EVT PVT = VT; in PromoteIntBinOp()
1362 assert(PVT != VT && "Don't know what type to promote to!"); in PromoteIntBinOp()
1376 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1)); in PromoteIntBinOp()
1416 EVT VT = Op.getValueType(); in PromoteIntShiftOp() local
1417 if (VT.isVector() || !VT.isInteger()) in PromoteIntShiftOp()
1423 if (TLI.isTypeDesirableForOp(Opc, VT)) in PromoteIntShiftOp()
1426 EVT PVT = VT; in PromoteIntShiftOp()
1430 assert(PVT != VT && "Don't know what type to promote to!"); in PromoteIntShiftOp()
1449 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1)); in PromoteIntShiftOp()
1465 EVT VT = Op.getValueType(); in PromoteExtend() local
1466 if (VT.isVector() || !VT.isInteger()) in PromoteExtend()
1472 if (TLI.isTypeDesirableForOp(Opc, VT)) in PromoteExtend()
1475 EVT PVT = VT; in PromoteExtend()
1479 assert(PVT != VT && "Don't know what type to promote to!"); in PromoteExtend()
1484 return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0)); in PromoteExtend()
1496 EVT VT = Op.getValueType(); in PromoteLoad() local
1497 if (VT.isVector() || !VT.isInteger()) in PromoteLoad()
1503 if (TLI.isTypeDesirableForOp(Opc, VT)) in PromoteLoad()
1506 EVT PVT = VT; in PromoteLoad()
1510 assert(PVT != VT && "Don't know what type to promote to!"); in PromoteLoad()
1521 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD); in PromoteLoad()
2122 EVT VT; in canFoldInAddressingMode() local
2128 VT = LD->getMemoryVT(); in canFoldInAddressingMode()
2133 VT = ST->getMemoryVT(); in canFoldInAddressingMode()
2138 VT = LD->getMemoryVT(); in canFoldInAddressingMode()
2143 VT = ST->getMemoryVT(); in canFoldInAddressingMode()
2173 VT.getTypeForEVT(*DAG.getContext()), AS); in canFoldInAddressingMode()
2194 EVT VT = N->getValueType(0); in foldSelectWithIdentityConstant() local
2233 SDValue NewBO = DAG.getNode(Opcode, SDLoc(N), VT, F0, FVal, N->getFlags()); in foldSelectWithIdentityConstant()
2234 return DAG.getSelect(SDLoc(N), VT, Cond, F0, NewBO); in foldSelectWithIdentityConstant()
2239 SDValue NewBO = DAG.getNode(Opcode, SDLoc(N), VT, F0, TVal, N->getFlags()); in foldSelectWithIdentityConstant()
2240 return DAG.getSelect(SDLoc(N), VT, Cond, NewBO, F0); in foldSelectWithIdentityConstant()
2252 EVT VT = BO->getValueType(0); in foldBinOpIntoSelect() local
2253 if (TLI.shouldFoldSelectWithIdentityConstant(BinOpcode, VT)) { in foldBinOpIntoSelect()
2305 SDValue NewCT = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CT) in foldBinOpIntoSelect()
2306 : DAG.getNode(BinOpcode, DL, VT, CT, CBO); in foldBinOpIntoSelect()
2312 SDValue NewCF = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CF) in foldBinOpIntoSelect()
2313 : DAG.getNode(BinOpcode, DL, VT, CF, CBO); in foldBinOpIntoSelect()
2319 SDValue SelectOp = DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF); in foldBinOpIntoSelect()
2355 EVT VT = C.getValueType(); in foldAddSubBoolOfMaskedVal() local
2357 SDValue LowBit = DAG.getZExtOrTrunc(SetCC.getOperand(0), DL, VT); in foldAddSubBoolOfMaskedVal()
2358 SDValue C1 = IsAdd ? DAG.getConstant(CN->getAPIntValue() + 1, DL, VT) : in foldAddSubBoolOfMaskedVal()
2359 DAG.getConstant(CN->getAPIntValue() - 1, DL, VT); in foldAddSubBoolOfMaskedVal()
2360 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit); in foldAddSubBoolOfMaskedVal()
2384 EVT VT = ShiftOp.getValueType(); in foldAddSubOfSignBit() local
2387 if (!ShAmtC || ShAmtC->getAPIntValue() != (VT.getScalarSizeInBits() - 1)) in foldAddSubOfSignBit()
2395 IsAdd ? ISD::ADD : ISD::SUB, DL, VT, in foldAddSubOfSignBit()
2396 {ConstantOp, DAG.getConstant(1, DL, VT)})) { in foldAddSubOfSignBit()
2397 SDValue NewShift = DAG.getNode(IsAdd ? ISD::SRA : ISD::SRL, DL, VT, in foldAddSubOfSignBit()
2399 return DAG.getNode(ISD::ADD, DL, VT, NewShift, NewC); in foldAddSubOfSignBit()
2420 EVT VT = N0.getValueType(); in visitADDLike() local
2430 if (SDValue C = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N0, N1})) in visitADDLike()
2436 return DAG.getNode(ISD::ADD, DL, VT, N1, N0); in visitADDLike()
2439 if (VT.isVector()) { in visitADDLike()
2457 if (SDValue Sub = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N1, N01})) in visitADDLike()
2458 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Sub); in visitADDLike()
2461 if (SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N1, N00})) in visitADDLike()
2462 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1)); in visitADDLike()
2474 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) && in visitADDLike()
2477 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not); in visitADDLike()
2487 if (SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N1, N01})) in visitADDLike()
2488 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add); in visitADDLike()
2506 return DAG.getNode(ISD::ADD, DL, VT, in visitADDLike()
2507 DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)), in visitADDLike()
2519 return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1)); in visitADDLike()
2523 return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1)); in visitADDLike()
2536 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), in visitADDLike()
2542 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), in visitADDLike()
2548 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), in visitADDLike()
2554 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), in visitADDLike()
2561 return DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0).getOperand(0), in visitADDLike()
2573 return DAG.getNode(ISD::SUB, DL, VT, in visitADDLike()
2574 DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10), in visitADDLike()
2575 DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11)); in visitADDLike()
2579 if (N0.getOpcode() == ISD::UMAX && hasOperation(ISD::USUBSAT, VT)) { in visitADDLike()
2586 return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0), in visitADDLike()
2596 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in visitADDLike()
2612 return DAG.getNode(ISD::SUB, DL, VT, A, Xor.getOperand(0)); in visitADDLike()
2619 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD && in visitADDLike()
2621 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0), in visitADDLike()
2622 DAG.getAllOnesConstant(DL, VT)); in visitADDLike()
2623 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(1), Not); in visitADDLike()
2630 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), N1); in visitADDLike()
2631 return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0)); in visitADDLike()
2646 EVT VT = N0.getValueType(); in visitADD() local
2659 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) && in visitADD()
2661 return DAG.getNode(ISD::OR, DL, VT, N0, N1); in visitADD()
2667 return DAG.getVScale(DL, VT, C0 + C1); in visitADD()
2676 SDValue VS = DAG.getVScale(DL, VT, VS0 + VS1); in visitADD()
2677 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), VS); in visitADD()
2686 return DAG.getStepVector(DL, VT, NewStep); in visitADD()
2696 SDValue SV = DAG.getStepVector(DL, VT, NewStep); in visitADD()
2697 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), SV); in visitADD()
2707 EVT VT = N0.getValueType(); in visitADDSAT() local
2712 return DAG.getAllOnesConstant(DL, VT); in visitADDSAT()
2715 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1})) in visitADDSAT()
2721 return DAG.getNode(Opcode, DL, VT, N1, N0); in visitADDSAT()
2724 if (VT.isVector()) { in visitADDSAT()
2740 return DAG.getNode(ISD::ADD, DL, VT, N0, N1); in visitADDSAT()
2772 EVT VT = V->getValueType(0); in getAsCarry() local
2773 if (!TLI.isOperationLegalOrCustom(V.getOpcode(), VT)) in getAsCarry()
2795 EVT VT = N0.getValueType(); in foldAddSubMasked1() local
2796 if (DAG.ComputeNumSignBits(N1.getOperand(0)) != VT.getScalarSizeInBits()) in foldAddSubMasked1()
2801 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, N0, N1.getOperand(0)); in foldAddSubMasked1()
2807 EVT VT = N0.getValueType(); in visitADDLikeCommutative() local
2813 return DAG.getNode(ISD::SUB, DL, VT, N0, in visitADDLikeCommutative()
2814 DAG.getNode(ISD::SHL, DL, VT, in visitADDLikeCommutative()
2825 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD && in visitADDLikeCommutative()
2827 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0), in visitADDLikeCommutative()
2828 DAG.getAllOnesConstant(DL, VT)); in visitADDLikeCommutative()
2829 return DAG.getNode(ISD::SUB, DL, VT, N1, Not); in visitADDLikeCommutative()
2837 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), N1); in visitADDLikeCommutative()
2838 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1)); in visitADDLikeCommutative()
2843 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1)); in visitADDLikeCommutative()
2844 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(0)); in visitADDLikeCommutative()
2853 TLI.getBooleanContents(VT) == TargetLowering::ZeroOrOneBooleanContent) { in visitADDLikeCommutative()
2854 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitADDLikeCommutative()
2855 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt); in visitADDLikeCommutative()
2862 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0), in visitADDLikeCommutative()
2863 DAG.getConstant(1, DL, VT)); in visitADDLikeCommutative()
2864 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt); in visitADDLikeCommutative()
2875 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT)) in visitADDLikeCommutative()
2878 DAG.getVTList(VT, Carry.getValueType()), N0, in visitADDLikeCommutative()
2879 DAG.getConstant(0, DL, VT), Carry); in visitADDLikeCommutative()
2887 EVT VT = N0.getValueType(); in visitADDC() local
2892 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDC()
2908 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDC()
2934 EVT VT = V.getValueType(); in extractBooleanFlip() local
2937 switch(TLI.getBooleanContents(VT)) { in extractBooleanFlip()
2959 EVT VT = N0.getValueType(); in visitADDO() local
2967 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDO()
2982 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDO()
2988 DAG.getConstant(0, DL, VT), N0.getOperand(0)); in visitADDO()
3004 EVT VT = N0.getValueType(); in visitUADDOLike() local
3005 if (VT.isVector()) in visitUADDOLike()
3019 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT)) in visitUADDOLike()
3022 DAG.getConstant(0, SDLoc(N), VT), Carry); in visitUADDOLike()
3067 EVT VT = N0.getValueType(); in visitADDCARRY() local
3069 SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT); in visitADDCARRY()
3071 return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt, in visitADDCARRY()
3072 DAG.getConstant(1, DL, VT)), in visitADDCARRY()
3148 EVT VT = Combiner.getSetCCResultType(Carry0.getValueType()); in combineADDCARRYDiamond() local
3149 Z = DAG.getConstant(1, SDLoc(Carry0.getOperand(1)), VT); in combineADDCARRYDiamond()
3405 static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT, in tryFoldToZero() argument
3407 if (!VT.isVector()) in tryFoldToZero()
3408 return DAG.getConstant(0, DL, VT); in tryFoldToZero()
3409 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) in tryFoldToZero()
3410 return DAG.getConstant(0, DL, VT); in tryFoldToZero()
3417 EVT VT = N0.getValueType(); in visitSUB() local
3429 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations); in visitSUB()
3432 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N1})) in visitSUB()
3436 if (VT.isVector()) { in visitSUB()
3452 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
3453 DAG.getConstant(-N1C->getAPIntValue(), DL, VT)); in visitSUB()
3457 unsigned BitWidth = VT.getScalarSizeInBits(); in visitSUB()
3466 if (!LegalOperations || TLI.isOperationLegal(NewSh, VT)) in visitSUB()
3467 return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1)); in visitSUB()
3487 !TLI.isOperationLegalOrCustom(ISD::ABS, VT)) in visitSUB()
3492 if (VT.isVector()) { in visitSUB()
3496 if (VT.isScalableVector()) in visitSUB()
3497 return DAG.getSplatVector(VT, DL, N1S.getOperand(1)); in visitSUB()
3498 return DAG.getSplatBuildVector(VT, DL, N1S.getOperand(1)); in visitSUB()
3505 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitSUB()
3509 return DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(1)); in visitSUB()
3526 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N01, N1})) in visitSUB()
3527 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), NewC); in visitSUB()
3533 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N11})) in visitSUB()
3534 return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0)); in visitSUB()
3540 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N01, N1})) in visitSUB()
3541 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), NewC); in visitSUB()
3547 if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N00, N1})) in visitSUB()
3548 return DAG.getNode(ISD::SUB, DL, VT, NewC, N0.getOperand(1)); in visitSUB()
3556 return DAG.getNode(N0.getOperand(1).getOpcode(), DL, VT, N0.getOperand(0), in visitSUB()
3562 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), in visitSUB()
3568 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), in visitSUB()
3573 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
3574 DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(1), in visitSUB()
3586 DAG.getNode(ISD::XOR, DL, VT, B, DAG.getAllOnesConstant(DL, VT)); in visitSUB()
3587 return DAG.getNode(ISD::AND, DL, VT, A, InvB); in visitSUB()
3595 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, in visitSUB()
3598 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul); in visitSUB()
3602 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, in visitSUB()
3605 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul); in visitSUB()
3624 if (SDValue V = foldSubToUSubSat(VT, N)) in visitSUB()
3629 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), in visitSUB()
3630 DAG.getAllOnesConstant(DL, VT)); in visitSUB()
3631 return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0)); in visitSUB()
3638 if (TLI.preferIncOfAddToSubOfNot(VT) && N1.hasOneUse() && isBitwiseNot(N1)) { in visitSUB()
3639 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(0)); in visitSUB()
3640 return DAG.getNode(ISD::ADD, DL, VT, Add, DAG.getConstant(1, DL, VT)); in visitSUB()
3647 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1); in visitSUB()
3648 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(1)); in visitSUB()
3653 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(0)); in visitSUB()
3654 return DAG.getNode(ISD::SUB, DL, VT, Sub, N1.getOperand(1)); in visitSUB()
3660 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1); in visitSUB()
3661 return DAG.getNode(ISD::SUB, DL, VT, Sub, N0.getOperand(1)); in visitSUB()
3666 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), N1); in visitSUB()
3667 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), Add); in visitSUB()
3675 TLI.getBooleanContents(VT) == in visitSUB()
3677 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N1.getOperand(0)); in visitSUB()
3678 return DAG.getNode(ISD::ADD, DL, VT, N0, SExt); in visitSUB()
3682 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) { in visitSUB()
3688 if (C->getAPIntValue() == (VT.getScalarSizeInBits() - 1)) in visitSUB()
3689 return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0); in visitSUB()
3698 return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT, in visitSUB()
3705 DL, VT); in visitSUB()
3712 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0), in visitSUB()
3713 DAG.getConstant(1, DL, VT)); in visitSUB()
3714 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt); in visitSUB()
3721 return DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getVScale(DL, VT, -IntVal)); in visitSUB()
3727 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
3728 DAG.getStepVector(DL, VT, NewStep)); in visitSUB()
3738 SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt); in visitSUB()
3739 return DAG.getNode(ISD::ADD, DL, VT, N0, SRA); in visitSUB()
3748 if (ShlC && ShlC->getAPIntValue() == VT.getScalarSizeInBits() - 1) in visitSUB()
3749 return DAG.getNode(ISD::ADD, DL, VT, N1, N0); in visitSUB()
3752 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT)) { in visitSUB()
3756 SDValue Zero = DAG.getConstant(0, DL, VT); in visitSUB()
3757 SDValue NegX = DAG.getNode(ISD::SUB, DL, VT, Zero, X); in visitSUB()
3759 DAG.getVTList(VT, Carry.getValueType()), NegX, Zero, in visitSUB()
3771 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitSUB()
3781 EVT VT = N0.getValueType(); in visitSUBSAT() local
3786 return DAG.getConstant(0, DL, VT); in visitSUBSAT()
3790 return DAG.getConstant(0, DL, VT); in visitSUBSAT()
3793 if (SDValue C = DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, {N0, N1})) in visitSUBSAT()
3797 if (VT.isVector()) { in visitSUBSAT()
3816 EVT VT = N0.getValueType(); in visitSUBC() local
3821 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitSUBC()
3826 return CombineTo(N, DAG.getConstant(0, DL, VT), in visitSUBC()
3835 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0), in visitSUBC()
3844 EVT VT = N0.getValueType(); in visitSUBO() local
3852 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitSUBO()
3857 return CombineTo(N, DAG.getConstant(0, DL, VT), in visitSUBO()
3865 DAG.getConstant(-N1C->getAPIntValue(), DL, VT)); in visitSUBO()
3874 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0), in visitSUBO()
3928 EVT VT = N0.getValueType(); in visitMULFIX() local
3932 return DAG.getConstant(0, SDLoc(N), VT); in visitMULFIX()
3937 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0, Scale); in visitMULFIX()
3941 return DAG.getConstant(0, SDLoc(N), VT); in visitMULFIX()
3949 EVT VT = N0.getValueType(); in visitMUL() local
3954 return DAG.getConstant(0, DL, VT); in visitMUL()
3957 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MUL, DL, VT, {N0, N1})) in visitMUL()
3963 return DAG.getNode(ISD::MUL, DL, VT, N1, N0); in visitMUL()
3970 if (VT.isVector()) { in visitMUL()
3976 ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && in visitMUL()
3999 return DAG.getNode(ISD::SUB, DL, VT, in visitMUL()
4000 DAG.getConstant(0, DL, VT), N0); in visitMUL()
4005 (!VT.isVector() || Level <= AfterLegalizeVectorOps)) { in visitMUL()
4009 return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc); in visitMUL()
4017 return DAG.getNode(ISD::SUB, DL, VT, in visitMUL()
4018 DAG.getConstant(0, DL, VT), in visitMUL()
4019 DAG.getNode(ISD::SHL, DL, VT, N0, in visitMUL()
4039 if (N1IsConst && TLI.decomposeMulByConstant(*DAG.getContext(), VT, N1)) { in visitMUL()
4057 assert(ShAmt < VT.getScalarSizeInBits() && in visitMUL()
4060 DAG.getNode(ISD::SHL, DL, VT, N0, DAG.getConstant(ShAmt, DL, VT)); in visitMUL()
4062 TZeros ? DAG.getNode(MathOp, DL, VT, Shl, in visitMUL()
4063 DAG.getNode(ISD::SHL, DL, VT, N0, in visitMUL()
4064 DAG.getConstant(TZeros, DL, VT))) in visitMUL()
4065 : DAG.getNode(MathOp, DL, VT, Shl, N0); in visitMUL()
4067 R = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), R); in visitMUL()
4075 if (SDValue C3 = DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {N1, N01})) in visitMUL()
4076 return DAG.getNode(ISD::MUL, DL, VT, N0.getOperand(0), C3); in visitMUL()
4095 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, Sh.getOperand(0), Y); in visitMUL()
4096 return DAG.getNode(ISD::SHL, DL, VT, Mul, Sh.getOperand(1)); in visitMUL()
4106 ISD::ADD, DL, VT, in visitMUL()
4107 DAG.getNode(ISD::MUL, SDLoc(N0), VT, N0.getOperand(0), N1), in visitMUL()
4108 DAG.getNode(ISD::MUL, SDLoc(N1), VT, N0.getOperand(1), N1)); in visitMUL()
4115 return DAG.getVScale(DL, VT, C0 * C1); in visitMUL()
4124 return DAG.getStepVector(DL, VT, NewStep); in visitMUL()
4131 if (VT.isFixedLengthVector()) { in visitMUL()
4132 unsigned NumElts = VT.getVectorNumElements(); in visitMUL()
4143 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::AND, VT)) && in visitMUL()
4153 return DAG.getNode(ISD::AND, DL, VT, N0, DAG.getBuildVector(VT, DL, Mask)); in visitMUL()
4197 EVT VT = Node->getValueType(0); in useDivRem() local
4198 if (VT.isVector() || !VT.isInteger()) in useDivRem()
4201 if (!TLI.isTypeLegal(VT) && !TLI.isOperationCustom(DivRemOpc, VT)) in useDivRem()
4206 if (!TLI.isOperationLegalOrCustom(DivRemOpc, VT) && in useDivRem()
4214 if (TLI.isOperationLegalOrCustom(Opcode, VT)) in useDivRem()
4218 if (TLI.isOperationLegalOrCustom(OtherOpcode, VT)) in useDivRem()
4238 SDVTList VTs = DAG.getVTList(VT, VT); in useDivRem()
4259 EVT VT = N->getValueType(0); in simplifyDivRem() local
4272 return DAG.getUNDEF(VT); in simplifyDivRem()
4277 return DAG.getConstant(0, DL, VT); in simplifyDivRem()
4288 return DAG.getConstant(IsDiv ? 1 : 0, DL, VT); in simplifyDivRem()
4296 if ((N1C && N1C->isOne()) || (VT.getScalarType() == MVT::i1)) in simplifyDivRem()
4297 return IsDiv ? N0 : DAG.getConstant(0, DL, VT); in simplifyDivRem()
4305 EVT VT = N->getValueType(0); in visitSDIV() local
4306 EVT CCVT = getSetCCResultType(VT); in visitSDIV()
4310 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, {N0, N1})) in visitSDIV()
4314 if (VT.isVector()) in visitSDIV()
4321 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0); in visitSDIV()
4325 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ), in visitSDIV()
4326 DAG.getConstant(1, DL, VT), in visitSDIV()
4327 DAG.getConstant(0, DL, VT)); in visitSDIV()
4345 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1); in visitSDIV()
4346 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitSDIV()
4383 EVT VT = N->getValueType(0); in visitSDIVLike() local
4384 EVT CCVT = getSetCCResultType(VT); in visitSDIVLike()
4385 unsigned BitWidth = VT.getScalarSizeInBits(); in visitSDIVLike()
4399 SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1); in visitSDIVLike()
4406 SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0, in visitSDIVLike()
4411 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact); in visitSDIVLike()
4413 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl); in visitSDIVLike()
4415 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1); in visitSDIVLike()
4420 SDValue One = DAG.getConstant(1, DL, VT); in visitSDIVLike()
4421 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT); in visitSDIVLike()
4425 Sra = DAG.getSelect(DL, VT, IsOneOrAllOnes, N0, Sra); in visitSDIVLike()
4429 SDValue Zero = DAG.getConstant(0, DL, VT); in visitSDIVLike()
4430 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, Zero, Sra); in visitSDIVLike()
4434 SDValue Res = DAG.getSelect(DL, VT, IsNeg, Sub, Sra); in visitSDIVLike()
4453 EVT VT = N->getValueType(0); in visitUDIV() local
4454 EVT CCVT = getSetCCResultType(VT); in visitUDIV()
4458 if (SDValue C = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT, {N0, N1})) in visitUDIV()
4462 if (VT.isVector()) in visitUDIV()
4469 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ), in visitUDIV()
4470 DAG.getConstant(1, DL, VT), in visitUDIV()
4471 DAG.getConstant(0, DL, VT)); in visitUDIV()
4484 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1); in visitUDIV()
4485 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitUDIV()
4506 EVT VT = N->getValueType(0); in visitUDIVLike() local
4517 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc); in visitUDIVLike()
4533 return DAG.getNode(ISD::SRL, DL, VT, N0, Add); in visitUDIVLike()
4562 EVT VT = N->getValueType(0); in visitREM() local
4563 EVT CCVT = getSetCCResultType(VT); in visitREM()
4569 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1})) in visitREM()
4577 return DAG.getSelect(DL, VT, EqualsNeg1, DAG.getConstant(0, DL, VT), F0); in visitREM()
4590 return DAG.getNode(ISD::UREM, DL, VT, N0, N1); in visitREM()
4594 SDValue NegOne = DAG.getAllOnesConstant(DL, VT); in visitREM()
4595 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne); in visitREM()
4597 return DAG.getNode(ISD::AND, DL, VT, N0, Add); in visitREM()
4605 SDValue NegOne = DAG.getAllOnesConstant(DL, VT); in visitREM()
4606 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne); in visitREM()
4608 return DAG.getNode(ISD::AND, DL, VT, N0, Add); in visitREM()
4621 if (DAG.isKnownNeverZero(N1) && !TLI.isIntDivCheap(VT, Attr)) { in visitREM()
4636 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1); in visitREM()
4637 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitREM()
4654 EVT VT = N->getValueType(0); in visitMULHS() local
4658 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHS, DL, VT, {N0, N1})) in visitMULHS()
4666 if (VT.isVector()) { in visitMULHS()
4673 return DAG.getConstant(0, DL, VT); in visitMULHS()
4688 return DAG.getConstant(0, DL, VT); in visitMULHS()
4692 if (!TLI.isOperationLegalOrCustom(ISD::MULHS, VT) && VT.isSimple() && in visitMULHS()
4693 !VT.isVector()) { in visitMULHS()
4694 MVT Simple = VT.getSimpleVT(); in visitMULHS()
4704 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in visitMULHS()
4714 EVT VT = N->getValueType(0); in visitMULHU() local
4718 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHU, DL, VT, {N0, N1})) in visitMULHU()
4726 if (VT.isVector()) { in visitMULHU()
4733 return DAG.getConstant(0, DL, VT); in visitMULHU()
4746 return DAG.getConstant(0, DL, VT); in visitMULHU()
4750 DAG.isKnownToBeAPowerOfTwo(N1) && hasOperation(ISD::SRL, VT)) { in visitMULHU()
4751 unsigned NumEltBits = VT.getScalarSizeInBits(); in visitMULHU()
4754 ISD::SUB, DL, VT, DAG.getConstant(NumEltBits, DL, VT), LogBase2); in visitMULHU()
4757 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc); in visitMULHU()
4762 if (!TLI.isOperationLegalOrCustom(ISD::MULHU, VT) && VT.isSimple() && in visitMULHU()
4763 !VT.isVector()) { in visitMULHU()
4764 MVT Simple = VT.getSimpleVT(); in visitMULHU()
4774 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in visitMULHU()
4791 EVT VT = N->getValueType(0); in visitAVG() local
4795 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1})) in visitAVG()
4803 if (VT.isVector()) { in visitAVG()
4810 return DAG.getNode(ISD::SRA, DL, VT, N0, DAG.getConstant(1, DL, VT)); in visitAVG()
4812 return DAG.getNode(ISD::SRL, DL, VT, N0, DAG.getConstant(1, DL, VT)); in visitAVG()
4882 EVT VT = N->getValueType(0); in visitSMUL_LOHI() local
4892 if (VT.isSimple() && !VT.isVector()) { in visitSMUL_LOHI()
4893 MVT Simple = VT.getSimpleVT(); in visitSMUL_LOHI()
4904 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi); in visitSMUL_LOHI()
4906 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo); in visitSMUL_LOHI()
4920 EVT VT = N->getValueType(0); in visitUMUL_LOHI() local
4930 SDValue Zero = DAG.getConstant(0, DL, VT); in visitUMUL_LOHI()
4936 SDValue Zero = DAG.getConstant(0, DL, VT); in visitUMUL_LOHI()
4942 if (VT.isSimple() && !VT.isVector()) { in visitUMUL_LOHI()
4943 MVT Simple = VT.getSimpleVT(); in visitUMUL_LOHI()
4954 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi); in visitUMUL_LOHI()
4956 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo); in visitUMUL_LOHI()
4967 EVT VT = N0.getValueType(); in visitMULO() local
4984 return CombineTo(N, DAG.getConstant(Result, DL, VT), in visitMULO()
4995 return CombineTo(N, DAG.getConstant(0, DL, VT), in visitMULO()
5001 (!IsSigned || VT.getScalarSizeInBits() > 2)) in visitMULO()
5007 if (VT.getScalarSizeInBits() == 1) { in visitMULO()
5008 SDValue And = DAG.getNode(ISD::AND, DL, VT, N0, N1); in visitMULO()
5011 DAG.getConstant(0, DL, VT), ISD::SETNE)); in visitMULO()
5020 if (SignBits > VT.getScalarSizeInBits() + 1) in visitMULO()
5021 return CombineTo(N, DAG.getNode(ISD::MUL, DL, VT, N0, N1), in visitMULO()
5029 return CombineTo(N, DAG.getNode(ISD::MUL, DL, VT, N0, N1), in visitMULO()
5187 EVT VT = N0.getValueType(); in visitIMINMAX() local
5192 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1})) in visitIMINMAX()
5202 return DAG.getNode(Opcode, DL, VT, N1, N0); in visitIMINMAX()
5205 if (VT.isVector()) in visitIMINMAX()
5211 if (!TLI.isOperationLegal(Opcode, VT) && in visitIMINMAX()
5222 if (TLI.isOperationLegal(AltOpcode, VT)) in visitIMINMAX()
5223 return DAG.getNode(AltOpcode, DL, VT, N0, N1); in visitIMINMAX()
5245 EVT VT = N0.getValueType(); in hoistLogicOpWithSameOpcodeHands() local
5275 if ((VT.isVector() || LegalOperations) && in hoistLogicOpWithSameOpcodeHands()
5285 return DAG.getNode(HandOpcode, DL, VT, Logic); in hoistLogicOpWithSameOpcodeHands()
5302 if (TLI.isZExtFree(VT, XVT) && TLI.isTruncateFree(XVT, VT)) in hoistLogicOpWithSameOpcodeHands()
5307 return DAG.getNode(HandOpcode, DL, VT, Logic); in hoistLogicOpWithSameOpcodeHands()
5319 return DAG.getNode(HandOpcode, DL, VT, Logic, N0.getOperand(1)); in hoistLogicOpWithSameOpcodeHands()
5328 return DAG.getNode(HandOpcode, DL, VT, Logic); in hoistLogicOpWithSameOpcodeHands()
5342 !(VT.isVector() && TLI.isTypeLegal(VT) && in hoistLogicOpWithSameOpcodeHands()
5345 return DAG.getNode(HandOpcode, DL, VT, Logic); in hoistLogicOpWithSameOpcodeHands()
5379 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations); in hoistLogicOpWithSameOpcodeHands()
5383 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT, in hoistLogicOpWithSameOpcodeHands()
5385 return DAG.getVectorShuffle(VT, DL, Logic, ShOp, SVN0->getMask()); in hoistLogicOpWithSameOpcodeHands()
5392 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations); in hoistLogicOpWithSameOpcodeHands()
5396 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT, N0.getOperand(1), in hoistLogicOpWithSameOpcodeHands()
5398 return DAG.getVectorShuffle(VT, DL, ShOp, Logic, SVN0->getMask()); in hoistLogicOpWithSameOpcodeHands()
5422 EVT VT = N0.getValueType(); in foldLogicOfSetCCs() local
5424 if (LegalOperations || VT.getScalarType() != MVT::i1) in foldLogicOfSetCCs()
5425 if (VT != getSetCCResultType(OpVT)) in foldLogicOfSetCCs()
5453 return DAG.getSetCC(DL, VT, Or, LR, CC1); in foldLogicOfSetCCs()
5472 return DAG.getSetCC(DL, VT, And, LR, CC1); in foldLogicOfSetCCs()
5486 return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE); in foldLogicOfSetCCs()
5500 return DAG.getSetCC(DL, VT, Or, Zero, CC1); in foldLogicOfSetCCs()
5524 return DAG.getSetCC(DL, VT, And, Zero, CC0); in foldLogicOfSetCCs()
5544 return DAG.getSetCC(DL, VT, LL, LR, NewCC); in foldLogicOfSetCCs()
5555 EVT VT = N1.getValueType(); in visitANDLike() local
5560 return DAG.getConstant(0, DL, VT); in visitANDLike()
5567 VT.getSizeInBits() <= 64 && N0->hasOneUse()) { in visitANDLike()
5577 SRLC.ult(VT.getSizeInBits()) && in visitANDLike()
5579 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(), in visitANDLike()
5586 DAG.getNode(ISD::ADD, DL0, VT, in visitANDLike()
5587 N0.getOperand(0), DAG.getConstant(ADDC, DL, VT)); in visitANDLike()
5604 unsigned Size = VT.getSizeInBits(); in visitANDLike()
5619 TLI.isNarrowingProfitable(VT, HalfVT) && in visitANDLike()
5622 TLI.isTruncateFree(VT, HalfVT) && in visitANDLike()
5623 TLI.isZExtFree(HalfVT, VT)) { in visitANDLike()
5641 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, And); in visitANDLike()
5821 EVT VT = Op.getOpcode() == ISD::AssertZext ? in SearchForAndLoads() local
5827 if (ExtVT.bitsGE(VT)) in SearchForAndLoads()
5849 MVT VT = SDValue(NodeToMask, i).getSimpleValueType(); in SearchForAndLoads() local
5850 if (VT != MVT::Glue && VT != MVT::Other) { in SearchForAndLoads()
5976 EVT VT = N->getValueType(0); in unfoldExtremeBitClearingToShifts() local
5979 SDValue T0 = DAG.getNode(InnerShift, DL, VT, X, Y); in unfoldExtremeBitClearingToShifts()
5981 SDValue T1 = DAG.getNode(OuterShift, DL, VT, T0, Y); in unfoldExtremeBitClearingToShifts()
5993 EVT VT = And->getValueType(0); in combineShiftAnd1ToBitTest() local
5995 if (!TLI.isTypeLegal(VT)) in combineShiftAnd1ToBitTest()
6028 unsigned VTBitWidth = VT.getScalarSizeInBits(); in combineShiftAnd1ToBitTest()
6052 SDValue X = DAG.getZExtOrTrunc(Src, DL, VT); in combineShiftAnd1ToBitTest()
6053 EVT CCVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in combineShiftAnd1ToBitTest()
6055 APInt::getOneBitSet(VTBitWidth, ShiftAmtC->getZExtValue()), DL, VT); in combineShiftAnd1ToBitTest()
6056 SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, X, Mask); in combineShiftAnd1ToBitTest()
6057 SDValue Zero = DAG.getConstant(0, DL, VT); in combineShiftAnd1ToBitTest()
6059 return DAG.getZExtOrTrunc(Setcc, DL, VT); in combineShiftAnd1ToBitTest()
6067 EVT VT = N1.getValueType(); in foldAndToUsubsat() local
6081 unsigned BitWidth = VT.getScalarSizeInBits(); in foldAndToUsubsat()
6091 SDValue SignMask = DAG.getConstant(XorC->getAPIntValue(), DL, VT); in foldAndToUsubsat()
6092 return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0), SignMask); in foldAndToUsubsat()
6135 EVT VT = N->getValueType(0); in foldLogicOfShifts() local
6137 SDValue LogicX = DAG.getNode(LogicOpcode, DL, VT, X0, X1); in foldLogicOfShifts()
6138 SDValue NewShift = DAG.getNode(ShiftOpcode, DL, VT, LogicX, Y); in foldLogicOfShifts()
6139 return DAG.getNode(LogicOpcode, DL, VT, NewShift, Z); in foldLogicOfShifts()
6145 EVT VT = N1.getValueType(); in visitAND() local
6152 if (SDValue C = DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, {N0, N1})) in visitAND()
6158 return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0); in visitAND()
6161 if (VT.isVector()) { in visitAND()
6181 EVT ExtVT = VT; in visitAND()
6204 unsigned BitWidth = VT.getScalarSizeInBits(); in visitAND()
6207 return DAG.getConstant(0, SDLoc(N), VT); in visitAND()
6217 if (VT.isVector()) in visitAND()
6370 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT, ZeroExtExtendee, in visitAND()
6387 DAG.getVTList(VT, MVT::Other), MemVT, SDLoc(N), Ops, in visitAND()
6399 if (N1C && N0.getOpcode() == ISD::LOAD && !VT.isVector()) in visitAND()
6440 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0)); in visitAND()
6463 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) { in visitAND()
6465 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, LN0->getChain(), in visitAND()
6509 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, N0.getOperand(0)); in visitAND()
6511 if (hasOperation(ISD::USUBSAT, VT)) in visitAND()
6524 EVT VT = N->getValueType(0); in MatchBSwapHWordLow() local
6525 if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16) in MatchBSwapHWordLow()
6527 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT)) in MatchBSwapHWordLow()
6605 unsigned OpSizeInBits = VT.getSizeInBits(); in MatchBSwapHWordLow()
6625 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00); in MatchBSwapHWordLow()
6628 Res = DAG.getNode(ISD::SRL, DL, VT, Res, in MatchBSwapHWordLow()
6630 getShiftAmountTy(VT))); in MatchBSwapHWordLow()
6748 SDValue N1, EVT VT, EVT ShiftAmountTy) { in matchBSwapHWordOrAndAnd() argument
6749 assert(N->getOpcode() == ISD::OR && VT == MVT::i32 && in matchBSwapHWordOrAndAnd()
6751 if (!TLI.isOperationLegalOrCustom(ISD::ROTR, VT)) in matchBSwapHWordOrAndAnd()
6779 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Shift0.getOperand(0)); in matchBSwapHWordOrAndAnd()
6781 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt); in matchBSwapHWordOrAndAnd()
6794 EVT VT = N->getValueType(0); in MatchBSwapHWord() local
6795 if (VT != MVT::i32) in MatchBSwapHWord()
6797 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT)) in MatchBSwapHWord()
6800 if (SDValue BSwap = matchBSwapHWordOrAndAnd(TLI, DAG, N, N0, N1, VT, in MatchBSwapHWord()
6801 getShiftAmountTy(VT))) in MatchBSwapHWord()
6805 if (SDValue BSwap = matchBSwapHWordOrAndAnd(TLI, DAG, N, N1, N0, VT, in MatchBSwapHWord()
6806 getShiftAmountTy(VT))) in MatchBSwapHWord()
6838 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, in MatchBSwapHWord()
6843 SDValue ShAmt = DAG.getConstant(16, DL, getShiftAmountTy(VT)); in MatchBSwapHWord()
6844 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT)) in MatchBSwapHWord()
6845 return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt); in MatchBSwapHWord()
6846 if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT)) in MatchBSwapHWord()
6847 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt); in MatchBSwapHWord()
6848 return DAG.getNode(ISD::OR, DL, VT, in MatchBSwapHWord()
6849 DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt), in MatchBSwapHWord()
6850 DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt)); in MatchBSwapHWord()
6856 EVT VT = N1.getValueType(); in visitORLike() local
6861 return DAG.getAllOnesConstant(DL, VT); in visitORLike()
6883 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, in visitORLike()
6885 return DAG.getNode(ISD::AND, DL, VT, X, in visitORLike()
6886 DAG.getConstant(LHSMask | RHSMask, DL, VT)); in visitORLike()
6898 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, in visitORLike()
6900 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X); in visitORLike()
6909 EVT VT = N0.getValueType(); in visitORCommutative() local
6918 return DAG.getNode(ISD::OR, SDLoc(N), VT, N00, N1); in visitORCommutative()
6923 return DAG.getNode(ISD::OR, SDLoc(N), VT, N01, N1); in visitORCommutative()
6953 EVT VT = N1.getValueType(); in visitOR() local
6960 if (SDValue C = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, {N0, N1})) in visitOR()
6966 return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0); in visitOR()
6969 if (VT.isVector()) { in visitOR()
6986 if (SV0 && SV1 && TLI.isTypeLegal(VT)) { in visitOR()
6996 int NumElts = VT.getVectorNumElements(); in visitOR()
7032 TLI.buildLegalVectorShuffle(VT, SDLoc(N), NewLHS, NewRHS, in visitOR()
7080 if (SDValue COR = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N1), VT, in visitOR()
7082 SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1); in visitOR()
7084 return DAG.getNode(ISD::AND, SDLoc(N), VT, COR, IOR); in visitOR()
7110 if ((!LegalOperations || TLI.isOperationLegal(ISD::ADD, VT)) && in visitOR()
7413 EVT VT = Shifted.getValueType(); in MatchRotatePosNeg() local
7414 if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits(), DAG, in MatchRotatePosNeg()
7416 return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted, in MatchRotatePosNeg()
7434 EVT VT = N0.getValueType(); in MatchFunnelPosNeg() local
7435 unsigned EltBits = VT.getScalarSizeInBits(); in MatchFunnelPosNeg()
7445 return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, N0, N1, in MatchFunnelPosNeg()
7465 TLI.isOperationLegalOrCustom(ISD::FSHL, VT)) { in MatchFunnelPosNeg()
7466 return DAG.getNode(ISD::FSHL, DL, VT, N0, N1.getOperand(0), Pos); in MatchFunnelPosNeg()
7474 TLI.isOperationLegalOrCustom(ISD::FSHR, VT)) { in MatchFunnelPosNeg()
7475 return DAG.getNode(ISD::FSHR, DL, VT, N0.getOperand(0), N1, Neg); in MatchFunnelPosNeg()
7484 TLI.isOperationLegalOrCustom(ISD::FSHR, VT)) { in MatchFunnelPosNeg()
7485 return DAG.getNode(ISD::FSHR, DL, VT, N0.getOperand(0), N1, Neg); in MatchFunnelPosNeg()
7497 EVT VT = LHS.getValueType(); in MatchRotate() local
7502 bool HasROTL = hasOperation(ISD::ROTL, VT); in MatchRotate()
7503 bool HasROTR = hasOperation(ISD::ROTR, VT); in MatchRotate()
7504 bool HasFSHL = hasOperation(ISD::FSHL, VT); in MatchRotate()
7505 bool HasFSHR = hasOperation(ISD::FSHR, VT); in MatchRotate()
7510 if (VT.isScalarInteger() && TLI.getTypeAction(*DAG.getContext(), VT) == in MatchRotate()
7512 HasROTL |= TLI.getOperationAction(ISD::ROTL, VT) == TargetLowering::Custom; in MatchRotate()
7513 HasROTR |= TLI.getOperationAction(ISD::ROTR, VT) == TargetLowering::Custom; in MatchRotate()
7580 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in MatchRotate()
7594 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT); in MatchRotate()
7598 SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt); in MatchRotate()
7599 Mask = DAG.getNode(ISD::AND, DL, VT, Mask, in MatchRotate()
7600 DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits)); in MatchRotate()
7603 SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt); in MatchRotate()
7604 Mask = DAG.getNode(ISD::AND, DL, VT, Mask, in MatchRotate()
7605 DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits)); in MatchRotate()
7608 Res = DAG.getNode(ISD::AND, DL, VT, Res, Mask); in MatchRotate()
7617 if (TLI.isTypeLegal(VT) && LHS.hasOneUse() && RHS.hasOneUse() && in MatchRotate()
7641 SDValue RotX = DAG.getNode(ISD::ROTL, DL, VT, X, LHSShiftAmt); in MatchRotate()
7642 SDValue ShlY = DAG.getNode(ISD::SHL, DL, VT, Y, LHSShiftAmt); in MatchRotate()
7643 Res = DAG.getNode(ISD::OR, DL, VT, RotX, ShlY); in MatchRotate()
7646 SDValue RotX = DAG.getNode(ISD::ROTL, DL, VT, X, LHSShiftAmt); in MatchRotate()
7647 SDValue SrlY = DAG.getNode(ISD::SRL, DL, VT, Y, RHSShiftAmt); in MatchRotate()
7648 Res = DAG.getNode(ISD::OR, DL, VT, RotX, SrlY); in MatchRotate()
7668 Res = DAG.getNode(UseROTL ? ISD::ROTL : ISD::ROTR, DL, VT, LHSShiftArg, in MatchRotate()
7672 Res = DAG.getNode(UseFSHL ? ISD::FSHL : ISD::FSHR, DL, VT, LHSShiftArg, in MatchRotate()
8141 EVT VT = N->getValueType(0); in MatchLoadCombine() local
8142 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64) in MatchLoadCombine()
8144 unsigned ByteWidth = VT.getSizeInBits() / 8; in MatchLoadCombine()
8265 !TLI.isOperationLegal(ISD::BSWAP, VT)) in MatchLoadCombine()
8271 !TLI.isOperationLegal(ISD::SHL, VT)) in MatchLoadCombine()
8283 DAG.getExtLoad(NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, SDLoc(N), VT, in MatchLoadCombine()
8296 ? DAG.getNode(ISD::SHL, SDLoc(N), VT, NewLoad, in MatchLoadCombine()
8297 DAG.getShiftAmountConstant(ZeroExtendedBytes * 8, VT, in MatchLoadCombine()
8300 return DAG.getNode(ISD::BSWAP, SDLoc(N), VT, ShiftedLoad); in MatchLoadCombine()
8326 EVT VT = N->getValueType(0); in unfoldMaskedMerge() local
8375 SDValue NotX = DAG.getNOT(DL, X, VT); in unfoldMaskedMerge()
8376 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, NotX, M); in unfoldMaskedMerge()
8377 SDValue NotLHS = DAG.getNOT(DL, LHS, VT); in unfoldMaskedMerge()
8378 SDValue RHS = DAG.getNode(ISD::OR, DL, VT, M, Y); in unfoldMaskedMerge()
8379 return DAG.getNode(ISD::AND, DL, VT, NotLHS, RHS); in unfoldMaskedMerge()
8388 SDValue LHS = DAG.getNode(ISD::OR, DL, VT, X, NotM); in unfoldMaskedMerge()
8389 SDValue NotY = DAG.getNOT(DL, Y, VT); in unfoldMaskedMerge()
8390 SDValue RHS = DAG.getNode(ISD::AND, DL, VT, NotM, NotY); in unfoldMaskedMerge()
8391 SDValue NotRHS = DAG.getNOT(DL, RHS, VT); in unfoldMaskedMerge()
8392 return DAG.getNode(ISD::AND, DL, VT, LHS, NotRHS); in unfoldMaskedMerge()
8395 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, X, M); in unfoldMaskedMerge()
8396 SDValue NotM = DAG.getNOT(DL, M, VT); in unfoldMaskedMerge()
8397 SDValue RHS = DAG.getNode(ISD::AND, DL, VT, Y, NotM); in unfoldMaskedMerge()
8399 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS); in unfoldMaskedMerge()
8405 EVT VT = N0.getValueType(); in visitXOR() local
8410 return DAG.getConstant(0, DL, VT); in visitXOR()
8419 if (SDValue C = DAG.FoldConstantArithmetic(ISD::XOR, DL, VT, {N0, N1})) in visitXOR()
8425 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitXOR()
8428 if (VT.isVector()) { in visitXOR()
8450 if ((!LegalOperations || TLI.isOperationLegal(ISD::ADD, VT)) && in visitXOR()
8468 return DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC); in visitXOR()
8478 DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC, in visitXOR()
8499 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, V); in visitXOR()
8503 if (isOneConstant(N1) && VT == MVT::i1 && N0.hasOneUse() && in visitXOR()
8508 N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00 in visitXOR()
8509 N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01 in visitXOR()
8511 return DAG.getNode(NewOpcode, DL, VT, N00, N01); in visitXOR()
8520 N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00 in visitXOR()
8521 N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01 in visitXOR()
8523 return DAG.getNode(NewOpcode, DL, VT, N00, N01); in visitXOR()
8532 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), in visitXOR()
8533 DAG.getAllOnesConstant(DL, VT)); in visitXOR()
8539 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in visitXOR()
8546 SDValue NotX = DAG.getNOT(SDLoc(X), X, VT); in visitXOR()
8548 return DAG.getNode(ISD::AND, DL, VT, NotX, N1); in visitXOR()
8552 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) { in visitXOR()
8560 if (C->getAPIntValue() == (VT.getScalarSizeInBits() - 1)) in visitXOR()
8561 return DAG.getNode(ISD::ABS, DL, VT, S0); in visitXOR()
8567 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations); in visitXOR()
8587 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0Opcode == ISD::SHL && in visitXOR()
8589 return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT), in visitXOR()
8675 EVT VT = Shift->getValueType(0); in combineShiftOfShiftedLogic() local
8678 SDValue NewShift1 = DAG.getNode(ShiftOpcode, DL, VT, X, ShiftSumC); in combineShiftOfShiftedLogic()
8679 SDValue NewShift2 = DAG.getNode(ShiftOpcode, DL, VT, Y, C1); in combineShiftOfShiftedLogic()
8680 return DAG.getNode(LogicOpcode, DL, VT, NewShift1, NewShift2); in combineShiftOfShiftedLogic()
8750 EVT VT = N->getValueType(0); in visitShiftByConstant() local
8751 SDValue NewRHS = DAG.getNode(N->getOpcode(), DL, VT, LHS.getOperand(1), in visitShiftByConstant()
8755 SDValue NewShift = DAG.getNode(N->getOpcode(), DL, VT, LHS.getOperand(0), in visitShiftByConstant()
8757 return DAG.getNode(LHS.getOpcode(), DL, VT, NewShift, NewRHS); in visitShiftByConstant()
8787 EVT VT = N->getValueType(0); in visitRotate() local
8788 unsigned Bitsize = VT.getScalarSizeInBits(); in visitRotate()
8812 return DAG.getNode(N->getOpcode(), dl, VT, N0, Amt); in visitRotate()
8818 VT.getScalarSizeInBits() == 16 && hasOperation(ISD::BSWAP, VT)) in visitRotate()
8819 return DAG.getNode(ISD::BSWAP, dl, VT, N0); in visitRotate()
8829 return DAG.getNode(N->getOpcode(), dl, VT, N0, NewOp1); in visitRotate()
8853 return DAG.getNode(N->getOpcode(), dl, VT, N0->getOperand(0), in visitRotate()
8867 EVT VT = N0.getValueType(); in visitSHL() local
8869 unsigned OpSizeInBits = VT.getScalarSizeInBits(); in visitSHL()
8872 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, {N0, N1})) in visitSHL()
8876 if (VT.isVector()) { in visitSHL()
8893 DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, {N01, N1})) in visitSHL()
8894 return DAG.getNode(ISD::AND, SDLoc(N), VT, N00, C); in visitSHL()
8905 return DAG.getConstant(0, SDLoc(N), VT); in visitSHL()
8911 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, NewOp1); in visitSHL()
8927 return DAG.getConstant(0, SDLoc(N), VT); in visitSHL()
8939 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum); in visitSHL()
8968 return DAG.getConstant(0, SDLoc(N), VT); in visitSHL()
8982 SDValue Ext = DAG.getNode(N0.getOpcode(), DL, VT, N0Op0.getOperand(0)); in visitSHL()
8985 return DAG.getNode(ISD::SHL, DL, VT, Ext, Sum); in visitSHL()
8997 auto MatchEqual = [VT](ConstantSDNode *LHS, ConstantSDNode *RHS) { in visitSHL()
9001 return c1.ult(VT.getScalarSizeInBits()) && (c1 == c2); in visitSHL()
9011 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL); in visitSHL()
9034 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff); in visitSHL()
9041 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0), Diff); in visitSHL()
9057 SDValue Mask = DAG.getAllOnesConstant(DL, VT); in visitSHL()
9058 Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, N01); in visitSHL()
9059 Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, Diff); in visitSHL()
9060 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Diff); in visitSHL()
9061 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSHL()
9068 SDValue Mask = DAG.getAllOnesConstant(DL, VT); in visitSHL()
9069 Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, N1); in visitSHL()
9070 SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff); in visitSHL()
9071 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSHL()
9080 SDValue AllBits = DAG.getAllOnesConstant(DL, VT); in visitSHL()
9081 SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1); in visitSHL()
9082 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), HiBitsMask); in visitSHL()
9094 SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1); in visitSHL()
9095 SDValue Shl1 = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1); in visitSHL()
9098 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, Shl0, Shl1); in visitSHL()
9105 DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N1), VT, {N01, N1})) in visitSHL()
9106 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), Shl); in visitSHL()
9119 return DAG.getVScale(SDLoc(N), VT, C0 << C1); in visitSHL()
9129 return DAG.getStepVector(SDLoc(N), VT, NewStep); in visitSHL()
9227 EVT VT = N0.getValueType(); in visitSRA() local
9228 unsigned OpSizeInBits = VT.getScalarSizeInBits(); in visitSRA()
9231 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, {N0, N1})) in visitSRA()
9241 if (VT.isVector()) in visitSRA()
9254 if (VT.isVector()) in visitSRA()
9256 VT.getVectorElementCount()); in visitSRA()
9260 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, in visitSRA()
9297 return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0), ShiftValue); in visitSRA()
9314 if (VT.isVector()) in visitSRA()
9315 TruncVT = EVT::getVectorVT(Ctx, TruncVT, VT.getVectorElementCount()); in visitSRA()
9326 TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) && in visitSRA()
9327 TLI.isTruncateFree(VT, TruncVT)) { in visitSRA()
9331 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, in visitSRA()
9360 if (VT.isVector()) in visitSRA()
9361 TruncVT = EVT::getVectorVT(Ctx, TruncVT, VT.getVectorElementCount()); in visitSRA()
9368 TLI.isTruncateFree(VT, TruncVT)) { in visitSRA()
9380 return DAG.getSExtOrTrunc(Add, DL, VT); in visitSRA()
9390 return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0, NewOp1); in visitSRA()
9414 return DAG.getNode(ISD::TRUNCATE, DL, VT, SRA); in visitSRA()
9425 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, N1); in visitSRA()
9449 EVT VT = N0.getValueType(); in visitSRL() local
9451 unsigned OpSizeInBits = VT.getScalarSizeInBits(); in visitSRL()
9454 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, {N0, N1})) in visitSRL()
9458 if (VT.isVector()) in visitSRL()
9469 return DAG.getConstant(0, SDLoc(N), VT); in visitSRL()
9481 return DAG.getConstant(0, SDLoc(N), VT); in visitSRL()
9493 return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Sum); in visitSRL()
9512 return DAG.getConstant(0, DL, VT); in visitSRL()
9516 return DAG.getNode(ISD::TRUNCATE, DL, VT, NewShift); in visitSRL()
9530 return DAG.getNode(ISD::TRUNCATE, DL, VT, And); in visitSRL()
9553 SDValue Mask = DAG.getAllOnesConstant(DL, VT); in visitSRL()
9554 Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, N01); in visitSRL()
9555 Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, Diff); in visitSRL()
9556 SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff); in visitSRL()
9557 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSRL()
9565 SDValue Mask = DAG.getAllOnesConstant(DL, VT); in visitSRL()
9566 Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, N1); in visitSRL()
9567 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Diff); in visitSRL()
9568 return DAG.getNode(ISD::AND, DL, VT, Shift, Mask); in visitSRL()
9579 return DAG.getUNDEF(VT); in visitSRL()
9591 return DAG.getNode(ISD::AND, DL, VT, in visitSRL()
9592 DAG.getNode(ISD::ANY_EXTEND, DL, VT, SmallShift), in visitSRL()
9593 DAG.getConstant(Mask, DL, VT)); in visitSRL()
9601 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), N1); in visitSRL()
9611 if (Known.One.getBoolValue()) return DAG.getConstant(0, SDLoc(N0), VT); in visitSRL()
9616 if (UnknownBits == 0) return DAG.getConstant(1, SDLoc(N0), VT); in visitSRL()
9629 Op = DAG.getNode(ISD::SRL, DL, VT, Op, in visitSRL()
9636 return DAG.getNode(ISD::XOR, DL, VT, in visitSRL()
9637 Op, DAG.getConstant(1, DL, VT)); in visitSRL()
9645 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, NewOp1); in visitSRL()
9699 EVT VT = N->getValueType(0); in visitFunnelShift() local
9704 unsigned BitWidth = VT.getScalarSizeInBits(); in visitFunnelShift()
9724 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N0, N1, in visitFunnelShift()
9737 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N1, in visitFunnelShift()
9741 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, in visitFunnelShift()
9750 if ((BitWidth % 8) == 0 && (ShAmt % 8) == 0 && !VT.isVector() && in visitFunnelShift()
9764 if (TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, in visitFunnelShift()
9772 VT, DL, RHS->getChain(), NewPtr, in visitFunnelShift()
9792 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N1, N2); in visitFunnelShift()
9794 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, N2); in visitFunnelShift()
9802 if (N0 == N1 && hasOperation(RotOpc, VT)) in visitFunnelShift()
9803 return DAG.getNode(RotOpc, SDLoc(N), VT, N0, N2); in visitFunnelShift()
9818 EVT VT = N0.getValueType(); in visitSHLSAT() local
9822 DAG.FoldConstantArithmetic(N->getOpcode(), SDLoc(N), VT, {N0, N1})) in visitSHLSAT()
9827 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::SHL, VT)) { in visitSHLSAT()
9831 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, N1); in visitSHLSAT()
9837 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, N1); in visitSHLSAT()
9863 EVT VT = N->getValueType(0); in combineABSToABD() local
9875 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, ABD); in combineABSToABD()
9880 if (TLI.isOperationLegalOrCustom(ABDOpcode, VT)) in combineABSToABD()
9881 return DAG.getNode(ABDOpcode, SDLoc(N), VT, Op0, Op1); in combineABSToABD()
9888 EVT VT = N->getValueType(0); in visitABS() local
9892 return DAG.getNode(ISD::ABS, SDLoc(N), VT, N0); in visitABS()
9908 EVT VT = N->getValueType(0); in visitBSWAP() local
9913 return DAG.getNode(ISD::BSWAP, DL, VT, N0); in visitBSWAP()
9923 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, N0.getOperand(0)); in visitBSWAP()
9924 return DAG.getNode(ISD::BITREVERSE, DL, VT, BSwap); in visitBSWAP()
9929 unsigned BW = VT.getScalarSizeInBits(); in visitBSWAP()
9936 TLI.isTruncateFree(VT, HalfVT) && in visitBSWAP()
9940 Res = DAG.getNode(ISD::SHL, DL, VT, Res, in visitBSWAP()
9941 DAG.getConstant(NewShAmt, DL, getShiftAmountTy(VT))); in visitBSWAP()
9944 return DAG.getZExtOrTrunc(Res, DL, VT); in visitBSWAP()
9957 SDValue NewSwap = DAG.getNode(ISD::BSWAP, DL, VT, N0.getOperand(0)); in visitBSWAP()
9959 return DAG.getNode(InverseShift, DL, VT, NewSwap, N0.getOperand(1)); in visitBSWAP()
9968 EVT VT = N->getValueType(0); in visitBITREVERSE() local
9972 return DAG.getNode(ISD::BITREVERSE, SDLoc(N), VT, N0); in visitBITREVERSE()
9981 EVT VT = N->getValueType(0); in visitCTLZ() local
9985 return DAG.getNode(ISD::CTLZ, SDLoc(N), VT, N0); in visitCTLZ()
9988 if (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ_ZERO_UNDEF, VT)) { in visitCTLZ()
9990 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0); in visitCTLZ()
9998 EVT VT = N->getValueType(0); in visitCTLZ_ZERO_UNDEF() local
10002 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0); in visitCTLZ_ZERO_UNDEF()
10008 EVT VT = N->getValueType(0); in visitCTTZ() local
10012 return DAG.getNode(ISD::CTTZ, SDLoc(N), VT, N0); in visitCTTZ()
10015 if (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ_ZERO_UNDEF, VT)) { in visitCTTZ()
10017 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0); in visitCTTZ()
10025 EVT VT = N->getValueType(0); in visitCTTZ_ZERO_UNDEF() local
10029 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0); in visitCTTZ_ZERO_UNDEF()
10035 EVT VT = N->getValueType(0); in visitCTPOP() local
10039 return DAG.getNode(ISD::CTPOP, SDLoc(N), VT, N0); in visitCTPOP()
10049 EVT VT = LHS.getValueType(); in isLegalToCombineMinNumMaxNum() local
10051 return Options.NoSignedZerosFPMath && VT.isFloatingPoint() && in isLegalToCombineMinNumMaxNum()
10052 TLI.isProfitableToCombineMinNumMaxNum(VT) && in isLegalToCombineMinNumMaxNum()
10057 static SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS, in combineMinNumMaxNum() argument
10064 EVT TransformVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); in combineMinNumMaxNum()
10076 if (TLI.isOperationLegalOrCustom(IEEEOpcode, VT)) in combineMinNumMaxNum()
10077 return DAG.getNode(IEEEOpcode, DL, VT, LHS, RHS); in combineMinNumMaxNum()
10081 return DAG.getNode(Opcode, DL, VT, LHS, RHS); in combineMinNumMaxNum()
10091 if (TLI.isOperationLegalOrCustom(IEEEOpcode, VT)) in combineMinNumMaxNum()
10092 return DAG.getNode(IEEEOpcode, DL, VT, LHS, RHS); in combineMinNumMaxNum()
10096 return DAG.getNode(Opcode, DL, VT, LHS, RHS); in combineMinNumMaxNum()
10113 EVT VT = N->getValueType(0); in foldSelectOfConstantsUsingSra() local
10115 VT != Cond.getOperand(0).getValueType()) in foldSelectOfConstantsUsingSra()
10127 SDValue ShAmtC = DAG.getConstant(X.getScalarValueSizeInBits() - 1, DL, VT); in foldSelectOfConstantsUsingSra()
10128 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC); in foldSelectOfConstantsUsingSra()
10129 return DAG.getNode(ISD::OR, DL, VT, Sra, C1); in foldSelectOfConstantsUsingSra()
10134 SDValue ShAmtC = DAG.getConstant(X.getScalarValueSizeInBits() - 1, DL, VT); in foldSelectOfConstantsUsingSra()
10135 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC); in foldSelectOfConstantsUsingSra()
10136 return DAG.getNode(ISD::AND, DL, VT, Sra, C1); in foldSelectOfConstantsUsingSra()
10145 EVT VT = N->getValueType(0); in foldSelectOfConstants() local
10149 if (!VT.isInteger()) in foldSelectOfConstants()
10165 if (VT != MVT::i1) in foldSelectOfConstants()
10166 NotCond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, NotCond); in foldSelectOfConstants()
10172 if (VT != MVT::i1) in foldSelectOfConstants()
10173 NotCond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NotCond); in foldSelectOfConstants()
10178 if (VT != MVT::i1) in foldSelectOfConstants()
10179 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond); in foldSelectOfConstants()
10184 if (VT != MVT::i1) in foldSelectOfConstants()
10185 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond); in foldSelectOfConstants()
10191 if (TLI.convertSelectOfConstantsToMath(VT)) { in foldSelectOfConstants()
10198 if (VT != MVT::i1) in foldSelectOfConstants()
10199 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond); in foldSelectOfConstants()
10200 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2); in foldSelectOfConstants()
10204 if (VT != MVT::i1) in foldSelectOfConstants()
10205 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond); in foldSelectOfConstants()
10206 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2); in foldSelectOfConstants()
10211 if (VT != MVT::i1) in foldSelectOfConstants()
10212 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond); in foldSelectOfConstants()
10214 DAG.getShiftAmountConstant(C1Val.exactLogBase2(), VT, DL); in foldSelectOfConstants()
10215 return DAG.getNode(ISD::SHL, DL, VT, Cond, ShAmtC); in foldSelectOfConstants()
10242 if (VT.bitsEq(CondVT)) in foldSelectOfConstants()
10244 return DAG.getZExtOrTrunc(NotCond, DL, VT); in foldSelectOfConstants()
10255 EVT VT = N->getValueType(0); in foldBoolSelectToLogic() local
10256 if (VT != Cond.getValueType() || VT.getScalarSizeInBits() != 1) in foldBoolSelectToLogic()
10262 return DAG.getNode(ISD::OR, SDLoc(N), VT, Cond, F); in foldBoolSelectToLogic()
10267 return DAG.getNode(ISD::AND, SDLoc(N), VT, Cond, T); in foldBoolSelectToLogic()
10271 SDValue NotCond = DAG.getNOT(SDLoc(N), Cond, VT); in foldBoolSelectToLogic()
10272 return DAG.getNode(ISD::OR, SDLoc(N), VT, NotCond, T); in foldBoolSelectToLogic()
10277 SDValue NotCond = DAG.getNOT(SDLoc(N), Cond, VT); in foldBoolSelectToLogic()
10278 return DAG.getNode(ISD::AND, SDLoc(N), VT, NotCond, F); in foldBoolSelectToLogic()
10288 EVT VT = N->getValueType(0); in foldVSelectToSignBitSplatMask() local
10295 if (VT != Cond0.getValueType()) in foldVSelectToSignBitSplatMask()
10310 SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT); in foldVSelectToSignBitSplatMask()
10311 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt); in foldVSelectToSignBitSplatMask()
10312 return DAG.getNode(ISD::AND, DL, VT, Sra, N1); in foldVSelectToSignBitSplatMask()
10318 SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT); in foldVSelectToSignBitSplatMask()
10319 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt); in foldVSelectToSignBitSplatMask()
10320 return DAG.getNode(ISD::OR, DL, VT, Sra, N2); in foldVSelectToSignBitSplatMask()
10329 SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT); in foldVSelectToSignBitSplatMask()
10330 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt); in foldVSelectToSignBitSplatMask()
10331 SDValue Not = DAG.getNOT(DL, Sra, VT); in foldVSelectToSignBitSplatMask()
10332 return DAG.getNode(ISD::AND, DL, VT, Not, N2); in foldVSelectToSignBitSplatMask()
10346 EVT VT = N->getValueType(0); in visitSELECT() local
10374 TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT); in visitSELECT()
10444 SDValue SelectOp = DAG.getSelect(DL, VT, F, N2, N1); in visitSELECT()
10459 if (SDValue FMinMax = combineMinNumMaxNum(DL, VT, Cond0, Cond1, N1, N2, in visitSELECT()
10468 if (!LegalOperations && TLI.isOperationLegalOrCustom(ISD::UADDO, VT) && in visitSELECT()
10486 SDVTList VTs = DAG.getVTList(VT, VT0); in visitSELECT()
10488 return DAG.getSelect(DL, VT, UAO.getValue(1), N1, UAO.getValue(0)); in visitSELECT()
10492 if (TLI.isOperationLegal(ISD::SELECT_CC, VT) || in visitSELECT()
10494 TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT))) { in visitSELECT()
10498 SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1, in visitSELECT()
10508 if (!VT.isVector()) in visitSELECT()
10522 EVT VT = N->getValueType(0); in ConvertSelectToConcatVector() local
10523 int NumElems = VT.getVectorNumElements(); in ConvertSelectToConcatVector()
10565 ISD::CONCAT_VECTORS, DL, VT, in ConvertSelectToConcatVector()
10783 EVT VT = N->getValueType(0); in foldVSelectOfConstants() local
10785 !TLI.convertSelectOfConstantsToMath(VT) || in foldVSelectOfConstants()
10795 unsigned Elts = VT.getVectorNumElements(); in foldVSelectOfConstants()
10819 SDValue ExtendedCond = DAG.getNode(ExtendOpcode, DL, VT, Cond); in foldVSelectOfConstants()
10820 return DAG.getNode(ISD::ADD, DL, VT, ExtendedCond, N2); in foldVSelectOfConstants()
10827 SDValue ZextCond = DAG.getZExtOrTrunc(Cond, DL, VT); in foldVSelectOfConstants()
10828 SDValue ShAmtC = DAG.getConstant(Pow2C.exactLogBase2(), DL, VT); in foldVSelectOfConstants()
10829 return DAG.getNode(ISD::SHL, DL, VT, ZextCond, ShAmtC); in foldVSelectOfConstants()
10846 EVT VT = N->getValueType(0); in visitVSELECT() local
10857 return DAG.getSelect(DL, VT, F, N2, N1); in visitVSELECT()
10879 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) in visitVSELECT()
10880 return DAG.getNode(ISD::ABS, DL, VT, LHS); in visitVSELECT()
10882 SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, LHS, in visitVSELECT()
10883 DAG.getConstant(VT.getScalarSizeInBits() - 1, in visitVSELECT()
10884 DL, getShiftAmountTy(VT))); in visitVSELECT()
10885 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift); in visitVSELECT()
10888 return DAG.getNode(ISD::XOR, DL, VT, Add, Shift); in visitVSELECT()
10899 combineMinNumMaxNum(DL, VT, LHS, RHS, N1, N2, CC, TLI, DAG)) in visitVSELECT()
10939 if (hasOperation(ISD::UADDSAT, VT)) { in visitVSELECT()
10946 SatCC = ISD::getSetCCInverse(SatCC, VT.getScalarType()); in visitVSELECT()
10966 return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
10980 return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
10986 if (hasOperation(ISD::USUBSAT, VT)) { in visitVSELECT()
10993 SatCC = ISD::getSetCCInverse(SatCC, VT.getScalarType()); in visitVSELECT()
11008 if (SDValue R = getTruncatedUSUBSAT(VT, LHS.getValueType(), LHS, RHS, in visitVSELECT()
11023 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
11040 OpRHS = DAG.getNode(ISD::SUB, DL, VT, in visitVSELECT()
11041 DAG.getConstant(0, DL, VT), OpRHS); in visitVSELECT()
11042 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
11057 OpRHS = DAG.getConstant(SplatValue, DL, VT); in visitVSELECT()
11058 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
11090 if (hasOperation(ISD::SRA, VT)) in visitVSELECT()
11153 EVT VT = N->getValueType(0); in visitSETCC() local
11203 return DAG.getFreeze(DAG.getSetCC(SDLoc(N), VT, N0, N1, Cond)); in visitSETCC()
11206 SDValue Combined = SimplifySetCC(VT, N->getOperand(0), N->getOperand(1), Cond, in visitSETCC()
11279 EVT VT = N->getValueType(0); in tryToFoldExtendSelectLoad() local
11303 if (!TLI.isLoadExtLegal(ExtLoadOpcode, VT, Load1->getMemoryVT()) || in tryToFoldExtendSelectLoad()
11304 !TLI.isLoadExtLegal(ExtLoadOpcode, VT, Load2->getMemoryVT())) in tryToFoldExtendSelectLoad()
11307 SDValue Ext1 = DAG.getNode(Opcode, DL, VT, Op1); in tryToFoldExtendSelectLoad()
11308 SDValue Ext2 = DAG.getNode(Opcode, DL, VT, Op2); in tryToFoldExtendSelectLoad()
11309 return DAG.getSelect(DL, VT, N0->getOperand(0), Ext1, Ext2); in tryToFoldExtendSelectLoad()
11322 EVT VT = N->getValueType(0); in tryToFoldExtendOfConstant() local
11334 return DAG.getNode(Opcode, DL, VT, N0); in tryToFoldExtendOfConstant()
11343 (Opcode != ISD::ZERO_EXTEND || !TLI.isZExtFree(N0.getValueType(), VT))) { in tryToFoldExtendOfConstant()
11356 return DAG.getSelect(DL, VT, N0->getOperand(0), in tryToFoldExtendOfConstant()
11357 DAG.getNode(FoldOpc, DL, VT, Op1), in tryToFoldExtendOfConstant()
11358 DAG.getNode(FoldOpc, DL, VT, Op2)); in tryToFoldExtendOfConstant()
11365 EVT SVT = VT.getScalarType(); in tryToFoldExtendOfConstant()
11366 if (!(VT.isVector() && (!LegalTypes || TLI.isTypeLegal(SVT)) && in tryToFoldExtendOfConstant()
11374 unsigned NumElts = VT.getVectorNumElements(); in tryToFoldExtendOfConstant()
11398 return DAG.getBuildVector(VT, DL, Elts); in tryToFoldExtendOfConstant()
11405 static bool ExtendUsesToFormExtLoad(EVT VT, SDNode *N, SDValue N0, in ExtendUsesToFormExtLoad() argument
11410 bool isTruncFree = TLI.isTruncateFree(VT, N0.getValueType()); in ExtendUsesToFormExtLoad()
11588 EVT VT = N->getValueType(0); in CombineZExtLogicopShiftLoad() local
11590 if (TLI.isZExtFree(OrigVT, VT)) in CombineZExtLogicopShiftLoad()
11598 (LegalOperations && !TLI.isOperationLegal(N0.getOpcode(), VT))) in CombineZExtLogicopShiftLoad()
11605 (LegalOperations && !TLI.isOperationLegal(N1.getOpcode(), VT))) in CombineZExtLogicopShiftLoad()
11613 if (!TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) || in CombineZExtLogicopShiftLoad()
11627 if (!ExtendUsesToFormExtLoad(VT, N1.getNode(), N1.getOperand(0), in CombineZExtLogicopShiftLoad()
11632 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(Load), VT, in CombineZExtLogicopShiftLoad()
11637 SDValue Shift = DAG.getNode(N1.getOpcode(), DL1, VT, ExtLoad, in CombineZExtLogicopShiftLoad()
11640 APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits()); in CombineZExtLogicopShiftLoad()
11642 SDValue And = DAG.getNode(N0.getOpcode(), DL0, VT, Shift, in CombineZExtLogicopShiftLoad()
11643 DAG.getConstant(Mask, DL0, VT)); in CombineZExtLogicopShiftLoad()
11675 EVT VT = Cast->getValueType(0); in matchVSelectOpSizesWithSetCC() local
11676 if (LegalOperations || !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT)) in matchVSelectOpSizesWithSetCC()
11687 if (SetCCVT.getSizeInBits() != VT.getSizeInBits()) in matchVSelectOpSizesWithSetCC()
11697 CastA = DAG.getNode(CastOpcode, DL, VT, A, Cast->getOperand(1)); in matchVSelectOpSizesWithSetCC()
11698 CastB = DAG.getNode(CastOpcode, DL, VT, B, Cast->getOperand(1)); in matchVSelectOpSizesWithSetCC()
11700 CastA = DAG.getNode(CastOpcode, DL, VT, A); in matchVSelectOpSizesWithSetCC()
11701 CastB = DAG.getNode(CastOpcode, DL, VT, B); in matchVSelectOpSizesWithSetCC()
11703 return DAG.getNode(ISD::VSELECT, DL, VT, SetCC, CastA, CastB); in matchVSelectOpSizesWithSetCC()
11709 const TargetLowering &TLI, EVT VT, in tryToFoldExtOfExtload() argument
11722 VT.isVector()) && in tryToFoldExtOfExtload()
11723 !TLI.isLoadExtLegal(ExtLoadType, VT, MemVT)) in tryToFoldExtOfExtload()
11727 DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(), in tryToFoldExtOfExtload()
11740 const TargetLowering &TLI, EVT VT, in tryToFoldExtOfLoad() argument
11749 ((LegalOperations || VT.isFixedLengthVector() || in tryToFoldExtOfLoad()
11751 !TLI.isLoadExtLegal(ExtLoadType, VT, N0.getValueType()))) in tryToFoldExtOfLoad()
11757 DoXform = ExtendUsesToFormExtLoad(VT, N, N0, ExtOpc, SetCCs, TLI); in tryToFoldExtOfLoad()
11758 if (VT.isVector()) in tryToFoldExtOfLoad()
11764 SDValue ExtLoad = DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(), in tryToFoldExtOfLoad()
11783 const TargetLowering &TLI, EVT VT, in tryToFoldExtOfMaskedLoad() argument
11794 if (!TLI.isLoadExtLegalOrCustom(ExtLoadType, VT, Ld->getValueType(0))) in tryToFoldExtOfMaskedLoad()
11801 SDValue PassThru = DAG.getNode(ExtOpc, dl, VT, Ld->getPassThru()); in tryToFoldExtOfMaskedLoad()
11803 VT, dl, Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(), Ld->getMask(), in tryToFoldExtOfMaskedLoad()
11823 EVT VT = N->getValueType(0); in foldExtendedSignBitTest() local
11828 if (CC == ISD::SETGT && isAllOnesConstant(Ones) && VT == XVT) { in foldExtendedSignBitTest()
11833 unsigned ShCt = VT.getSizeInBits() - 1; in foldExtendedSignBitTest()
11835 if (!TLI.shouldAvoidTransformToShift(VT, ShCt)) { in foldExtendedSignBitTest()
11836 SDValue NotX = DAG.getNOT(DL, X, VT); in foldExtendedSignBitTest()
11837 SDValue ShiftAmount = DAG.getConstant(ShCt, DL, VT); in foldExtendedSignBitTest()
11840 return DAG.getNode(ShiftOpcode, DL, VT, NotX, ShiftAmount); in foldExtendedSignBitTest()
11854 EVT VT = N->getValueType(0); in foldSextSetcc() local
11864 if (VT.isVector() && !LegalOperations && in foldSextSetcc()
11876 if (VT.getSizeInBits() == SVT.getSizeInBits()) in foldSextSetcc()
11877 return DAG.getSetCC(DL, VT, N00, N01, CC); in foldSextSetcc()
11885 return DAG.getSExtOrTrunc(VsetCC, DL, VT); in foldSextSetcc()
11890 if (N0.hasOneUse() && TLI.isOperationLegalOrCustom(ISD::SETCC, VT) && in foldSextSetcc()
11908 TLI.isLoadExtLegal(LoadOpcode, VT, V.getValueType()))) in foldSextSetcc()
11922 if (User->getOpcode() != ExtOpcode || User->getValueType(0) != VT) in foldSextSetcc()
11929 SDValue Ext0 = DAG.getNode(ExtOpcode, DL, VT, N00); in foldSextSetcc()
11930 SDValue Ext1 = DAG.getNode(ExtOpcode, DL, VT, N01); in foldSextSetcc()
11931 return DAG.getSetCC(DL, VT, Ext0, Ext1, CC); in foldSextSetcc()
11949 ? DAG.getAllOnesConstant(DL, VT) in foldSextSetcc()
11950 : DAG.getBoolConstant(true, DL, VT, N00VT); in foldSextSetcc()
11951 SDValue Zero = DAG.getConstant(0, DL, VT); in foldSextSetcc()
11955 if (!VT.isVector() && !TLI.convertSelectOfConstantsToMath(VT)) { in foldSextSetcc()
11964 return DAG.getSelect(DL, VT, SetCC, ExtTrueVal, Zero); in foldSextSetcc()
11973 EVT VT = N->getValueType(0); in visitSIGN_EXTEND() local
11978 return DAG.getConstant(0, DL, VT); in visitSIGN_EXTEND()
11986 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N0.getOperand(0)); in visitSIGN_EXTEND()
12006 unsigned DestBits = VT.getScalarSizeInBits(); in visitSIGN_EXTEND()
12018 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op); in visitSIGN_EXTEND()
12023 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op); in visitSIGN_EXTEND()
12030 Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N0), VT, Op); in visitSIGN_EXTEND()
12032 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, Op); in visitSIGN_EXTEND()
12033 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Op, in visitSIGN_EXTEND()
12040 tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0, in visitSIGN_EXTEND()
12045 tryToFoldExtOfMaskedLoad(DAG, TLI, VT, N, N0, ISD::SEXTLOAD, in visitSIGN_EXTEND()
12056 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::SEXTLOAD)) in visitSIGN_EXTEND()
12065 (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) { in visitSIGN_EXTEND()
12068 if (TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, MemVT) && in visitSIGN_EXTEND()
12071 bool DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0), in visitSIGN_EXTEND()
12074 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN00), VT, in visitSIGN_EXTEND()
12078 APInt Mask = N0.getConstantOperandAPInt(1).sext(VT.getSizeInBits()); in visitSIGN_EXTEND()
12079 SDValue And = DAG.getNode(N0.getOpcode(), DL, VT, in visitSIGN_EXTEND()
12080 ExtLoad, DAG.getConstant(Mask, DL, VT)); in visitSIGN_EXTEND()
12110 if ((!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) && in visitSIGN_EXTEND()
12112 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0); in visitSIGN_EXTEND()
12122 TLI.isOperationLegalOrCustom(ISD::SUB, VT)) { in visitSIGN_EXTEND()
12123 SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(1).getOperand(0), DL, VT); in visitSIGN_EXTEND()
12124 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Zext); in visitSIGN_EXTEND()
12131 TLI.isOperationLegalOrCustom(ISD::ADD, VT)) { in visitSIGN_EXTEND()
12132 SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(0).getOperand(0), DL, VT); in visitSIGN_EXTEND()
12133 return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT)); in visitSIGN_EXTEND()
12139 (!LegalOperations || (TLI.isOperationLegal(ISD::ZERO_EXTEND, VT) && in visitSIGN_EXTEND()
12140 TLI.isOperationLegal(ISD::ADD, VT)))) { in visitSIGN_EXTEND()
12152 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NewXor); in visitSIGN_EXTEND()
12155 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitSIGN_EXTEND()
12156 return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT)); in visitSIGN_EXTEND()
12209 EVT VT = Extend->getValueType(0); in widenCtPop() local
12212 !TLI.isOperationLegalOrCustom(ISD::CTPOP, VT)) in widenCtPop()
12217 SDValue NewZext = DAG.getZExtOrTrunc(CtPop.getOperand(0), DL, VT); in widenCtPop()
12218 return DAG.getNode(ISD::CTPOP, DL, VT, NewZext); in widenCtPop()
12223 EVT VT = N->getValueType(0); in visitZERO_EXTEND() local
12227 return DAG.getConstant(0, SDLoc(N), VT); in visitZERO_EXTEND()
12235 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, in visitZERO_EXTEND()
12250 VT.getScalarSizeInBits())); in visitZERO_EXTEND()
12252 return DAG.getZExtOrTrunc(Op, SDLoc(N), VT); in visitZERO_EXTEND()
12274 if (SrcVT.bitsLT(VT) && VT.isVector()) { in visitZERO_EXTEND()
12276 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) { in visitZERO_EXTEND()
12280 SDValue ZExtOrTrunc = DAG.getZExtOrTrunc(Op, SDLoc(N), VT); in visitZERO_EXTEND()
12287 if (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT)) { in visitZERO_EXTEND()
12288 SDValue Op = DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT); in visitZERO_EXTEND()
12305 !TLI.isZExtFree(N0.getValueType(), VT))) { in visitZERO_EXTEND()
12307 X = DAG.getAnyExtOrTrunc(X, SDLoc(X), VT); in visitZERO_EXTEND()
12308 APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits()); in visitZERO_EXTEND()
12310 return DAG.getNode(ISD::AND, DL, VT, in visitZERO_EXTEND()
12311 X, DAG.getConstant(Mask, DL, VT)); in visitZERO_EXTEND()
12316 tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0, in visitZERO_EXTEND()
12321 tryToFoldExtOfMaskedLoad(DAG, TLI, VT, N, N0, ISD::ZEXTLOAD, in visitZERO_EXTEND()
12338 (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) { in visitZERO_EXTEND()
12341 if (TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) && in visitZERO_EXTEND()
12355 DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0), in visitZERO_EXTEND()
12358 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN00), VT, in visitZERO_EXTEND()
12362 APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits()); in visitZERO_EXTEND()
12364 SDValue And = DAG.getNode(N0.getOpcode(), DL, VT, in visitZERO_EXTEND()
12365 ExtLoad, DAG.getConstant(Mask, DL, VT)); in visitZERO_EXTEND()
12395 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::ZEXTLOAD)) in visitZERO_EXTEND()
12406 if (!LegalOperations && VT.isVector() && in visitZERO_EXTEND()
12418 if (VT.getSizeInBits() == N00VT.getSizeInBits()) { in visitZERO_EXTEND()
12420 SDValue VSetCC = DAG.getNode(ISD::SETCC, DL, VT, N0.getOperand(0), in visitZERO_EXTEND()
12432 return DAG.getZeroExtendInReg(DAG.getAnyExtOrTrunc(VsetCC, DL, VT), DL, in visitZERO_EXTEND()
12445 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, SCC); in visitZERO_EXTEND()
12467 if (Log2_32_Ceil(VT.getSizeInBits()) > ShAmt.getValueSizeInBits()) in visitZERO_EXTEND()
12470 return DAG.getNode(N0.getOpcode(), DL, VT, in visitZERO_EXTEND()
12471 DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)), in visitZERO_EXTEND()
12489 EVT VT = N->getValueType(0); in visitANY_EXTEND() local
12493 return DAG.getUNDEF(VT); in visitANY_EXTEND()
12504 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0)); in visitANY_EXTEND()
12522 return DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT); in visitANY_EXTEND()
12532 SDValue X = DAG.getAnyExtOrTrunc(N0.getOperand(0).getOperand(0), DL, VT); in visitANY_EXTEND()
12533 SDValue Y = DAG.getNode(ISD::ANY_EXTEND, DL, VT, N0.getOperand(1)); in visitANY_EXTEND()
12535 return DAG.getNode(ISD::AND, DL, VT, X, Y); in visitANY_EXTEND()
12541 if (VT.isVector()) { in visitANY_EXTEND()
12544 tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0, in visitANY_EXTEND()
12549 TLI.isLoadExtLegal(ISD::EXTLOAD, VT, N0.getValueType())) { in visitANY_EXTEND()
12554 ExtendUsesToFormExtLoad(VT, N, N0, ISD::ANY_EXTEND, SetCCs, TLI); in visitANY_EXTEND()
12557 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, in visitANY_EXTEND()
12584 if (!LegalOperations || TLI.isLoadExtLegal(ExtType, VT, MemVT)) { in visitANY_EXTEND()
12586 VT, LN0->getChain(), LN0->getBasePtr(), in visitANY_EXTEND()
12604 if (VT.isVector() && !LegalOperations) { in visitANY_EXTEND()
12614 if (VT.getSizeInBits() == N00VT.getSizeInBits()) in visitANY_EXTEND()
12615 return DAG.getSetCC(SDLoc(N), VT, N0.getOperand(0), in visitANY_EXTEND()
12627 return DAG.getAnyExtOrTrunc(VsetCC, SDLoc(N), VT); in visitANY_EXTEND()
12633 DL, N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, DL, VT), in visitANY_EXTEND()
12634 DAG.getConstant(0, DL, VT), in visitANY_EXTEND()
12742 EVT VT = N->getValueType(0); in reduceLoadWidth() local
12743 EVT ExtVT = VT; in reduceLoadWidth()
12746 if (VT.isVector()) in reduceLoadWidth()
12887 ExtVT == VT && TLI.isNarrowingProfitable(N0.getValueType(), VT)) { in reduceLoadWidth()
12929 Load = DAG.getLoad(VT, DL, LN0->getChain(), NewPtr, in reduceLoadWidth()
12933 Load = DAG.getExtLoad(ExtType, DL, VT, LN0->getChain(), NewPtr, in reduceLoadWidth()
12947 ShImmTy = VT; in reduceLoadWidth()
12952 if (ShLeftAmt >= VT.getScalarSizeInBits()) in reduceLoadWidth()
12953 Result = DAG.getConstant(0, DL, VT); in reduceLoadWidth()
12955 Result = DAG.getNode(ISD::SHL, DL, VT, in reduceLoadWidth()
12964 SDValue ShiftC = DAG.getConstant(ShAmt, DL, VT); in reduceLoadWidth()
12965 Result = DAG.getNode(ISD::SHL, DL, VT, Result, ShiftC); in reduceLoadWidth()
12976 EVT VT = N->getValueType(0); in visitSIGN_EXTEND_INREG() local
12978 unsigned VTBits = VT.getScalarSizeInBits(); in visitSIGN_EXTEND_INREG()
12983 return DAG.getConstant(0, SDLoc(N), VT); in visitSIGN_EXTEND_INREG()
12987 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0, N1); in visitSIGN_EXTEND_INREG()
12996 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0.getOperand(0), in visitSIGN_EXTEND_INREG()
13008 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT))) in visitSIGN_EXTEND_INREG()
13009 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00); in visitSIGN_EXTEND_INREG()
13028 TLI.isOperationLegal(ISD::SIGN_EXTEND_VECTOR_INREG, VT))) in visitSIGN_EXTEND_INREG()
13029 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, SDLoc(N), VT, N00); in visitSIGN_EXTEND_INREG()
13037 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT))) in visitSIGN_EXTEND_INREG()
13038 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00, N1); in visitSIGN_EXTEND_INREG()
13065 return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0.getOperand(0), in visitSIGN_EXTEND_INREG()
13079 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT))) { in visitSIGN_EXTEND_INREG()
13081 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
13096 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT))) { in visitSIGN_EXTEND_INREG()
13098 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
13112 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT)) { in visitSIGN_EXTEND_INREG()
13114 VT, SDLoc(N), Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(), in visitSIGN_EXTEND_INREG()
13132 DAG.getVTList(VT, MVT::Other), ExtVT, SDLoc(N), Ops, in visitSIGN_EXTEND_INREG()
13146 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, BSwap, N1); in visitSIGN_EXTEND_INREG()
13154 EVT VT = N->getValueType(0); in visitEXTEND_VECTOR_INREG() local
13158 return DAG.getConstant(0, SDLoc(N), VT); in visitEXTEND_VECTOR_INREG()
13171 EVT VT = N->getValueType(0); in visitTRUNCATE() local
13176 if (SrcVT == VT) in visitTRUNCATE()
13181 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0)); in visitTRUNCATE()
13185 SDValue C = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0); in visitTRUNCATE()
13195 if (N0.getOperand(0).getValueType().bitsLT(VT)) in visitTRUNCATE()
13196 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0)); in visitTRUNCATE()
13198 if (N0.getOperand(0).getValueType().bitsGT(VT)) in visitTRUNCATE()
13199 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0)); in visitTRUNCATE()
13212 if (ExtVT.bitsLT(VT)) { in visitTRUNCATE()
13213 SDValue TrX = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, X); in visitTRUNCATE()
13214 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, TrX, ExtVal); in visitTRUNCATE()
13233 LegalTypes && !LegalOperations && N0->hasOneUse() && VT != MVT::i1) { in visitTRUNCATE()
13260 TLI.isTruncateFree(SrcVT, VT)) { in visitTRUNCATE()
13263 SDValue TruncOp0 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1)); in visitTRUNCATE()
13264 SDValue TruncOp1 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(2)); in visitTRUNCATE()
13265 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, Cond, TruncOp0, TruncOp1); in visitTRUNCATE()
13271 (!LegalOperations || TLI.isOperationLegal(ISD::SHL, VT)) && in visitTRUNCATE()
13272 TLI.isTypeDesirableForOp(ISD::SHL, VT)) { in visitTRUNCATE()
13275 unsigned Size = VT.getScalarSizeInBits(); in visitTRUNCATE()
13278 EVT AmtVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); in visitTRUNCATE()
13280 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0)); in visitTRUNCATE()
13285 return DAG.getNode(ISD::SHL, SL, VT, Trunc, Amt); in visitTRUNCATE()
13289 if (SDValue V = foldSubToUSubSat(VT, N0.getNode())) in visitTRUNCATE()
13294 TLI.isTruncateFree(SrcVT.getScalarType(), VT.getScalarType()) && in visitTRUNCATE()
13296 (!LegalTypes || TLI.isTypeLegal(VT.getScalarType()))) { in visitTRUNCATE()
13298 EVT SVT = VT.getScalarType(); in visitTRUNCATE()
13304 return DAG.getBuildVector(VT, DL, TruncOps); in visitTRUNCATE()
13311 if (Level == AfterLegalizeVectorOps && VT.isVector() && in visitTRUNCATE()
13317 EVT TruncVecEltTy = VT.getVectorElementType(); in visitTRUNCATE()
13323 unsigned TruncVecNumElts = VT.getVectorNumElements(); in visitTRUNCATE()
13333 return DAG.getBuildVector(VT, SDLoc(N), Opnds); in visitTRUNCATE()
13339 if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getOpcode(), VT)) { in visitTRUNCATE()
13347 if (LN0->isSimple() && LN0->getMemoryVT().bitsLT(VT)) { in visitTRUNCATE()
13349 VT, LN0->getChain(), LN0->getBasePtr(), in visitTRUNCATE()
13378 VT.getVectorElementType(), in visitTRUNCATE()
13383 return DAG.getUNDEF(VT); in visitTRUNCATE()
13397 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Opnds); in visitTRUNCATE()
13405 if (N0.getOpcode() == ISD::BITCAST && !VT.isVector()) { in visitTRUNCATE()
13408 if (VecSrcVT.isVector() && VecSrcVT.getScalarType() == VT && in visitTRUNCATE()
13414 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, VT, VecSrc, in visitTRUNCATE()
13428 if (!VT.isVector()) { in visitTRUNCATE()
13430 APInt::getLowBitsSet(N0.getValueSizeInBits(), VT.getSizeInBits()); in visitTRUNCATE()
13432 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Shorter); in visitTRUNCATE()
13445 VT.getVectorElementType()) in visitTRUNCATE()
13446 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N0->getOperand(0)), VT, in visitTRUNCATE()
13471 if (VT.isScalarInteger() || TLI.isOperationLegal(N0.getOpcode(), VT)) { in visitTRUNCATE()
13473 SDValue NarrowL = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
13474 SDValue NarrowR = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1)); in visitTRUNCATE()
13475 return DAG.getNode(N0.getOpcode(), DL, VT, NarrowL, NarrowR); in visitTRUNCATE()
13486 TLI.isOperationLegal(N0.getOpcode(), VT)) && in visitTRUNCATE()
13489 SDValue X = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
13490 SDValue Y = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1)); in visitTRUNCATE()
13491 SDVTList VTs = DAG.getVTList(VT, N0->getValueType(1)); in visitTRUNCATE()
13502 VT.getScalarSizeInBits() && in visitTRUNCATE()
13503 hasOperation(N0.getOpcode(), VT)) { in visitTRUNCATE()
13504 return getTruncatedUSUBSAT(VT, SrcVT, N0.getOperand(0), N0.getOperand(1), in visitTRUNCATE()
13522 SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) { in CombineConsecutiveLoads() argument
13542 if ((!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)) && in CombineConsecutiveLoads()
13544 TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, in CombineConsecutiveLoads()
13546 return DAG.getLoad(VT, SDLoc(N), LD1->getChain(), LD1->getBasePtr(), in CombineConsecutiveLoads()
13562 EVT VT = N->getValueType(0); in foldBitcastedFPLogic() local
13563 if (!VT.isFloatingPoint() || !TLI.hasBitPreservingFPLogic(VT)) in foldBitcastedFPLogic()
13570 if (VT.getScalarSizeInBits() != SourceVT.getScalarSizeInBits()) in foldBitcastedFPLogic()
13600 LogicOp0.getOperand(0).getValueType() == VT) { in foldBitcastedFPLogic()
13601 SDValue FPOp = DAG.getNode(FPOpcode, SDLoc(N), VT, LogicOp0.getOperand(0)); in foldBitcastedFPLogic()
13604 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, FPOp); in foldBitcastedFPLogic()
13613 EVT VT = N->getValueType(0); in visitBITCAST() local
13616 return DAG.getUNDEF(VT); in visitBITCAST()
13624 if (VT.isVector() && in visitBITCAST()
13626 (!LegalOperations && VT.isInteger() && N0.getValueType().isInteger() && in visitBITCAST()
13627 TLI.isTypeLegal(VT.getVectorElementType()))) && in visitBITCAST()
13631 VT.getVectorElementType()); in visitBITCAST()
13639 (isa<ConstantSDNode>(N0) && VT.isFloatingPoint() && !VT.isVector() && in visitBITCAST()
13640 TLI.isOperationLegal(ISD::ConstantFP, VT)) || in visitBITCAST()
13641 (isa<ConstantFPSDNode>(N0) && VT.isInteger() && !VT.isVector() && in visitBITCAST()
13642 TLI.isOperationLegal(ISD::Constant, VT))) { in visitBITCAST()
13643 SDValue C = DAG.getBitcast(VT, N0); in visitBITCAST()
13651 return DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
13658 TLI.hasBigEndianPartOrdering(VT, DAG.getDataLayout()) && in visitBITCAST()
13665 TLI.isOperationLegal(ISD::LOAD, VT))) { in visitBITCAST()
13668 if (TLI.isLoadBitCastBeneficial(N0.getValueType(), VT, DAG, in visitBITCAST()
13671 DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(), in visitBITCAST()
13696 N0->hasOneUse() && VT.isInteger() && !VT.isVector() && in visitBITCAST()
13698 SDValue NewConv = DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
13703 assert(VT.getSizeInBits() == 128); in visitBITCAST()
13705 APInt::getSignMask(VT.getSizeInBits() / 2), SDLoc(N0), MVT::i64); in visitBITCAST()
13721 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit); in visitBITCAST()
13723 return DAG.getNode(ISD::XOR, DL, VT, NewConv, FlipBits); in visitBITCAST()
13725 APInt SignBit = APInt::getSignMask(VT.getSizeInBits()); in visitBITCAST()
13727 return DAG.getNode(ISD::XOR, DL, VT, in visitBITCAST()
13728 NewConv, DAG.getConstant(SignBit, DL, VT)); in visitBITCAST()
13730 return DAG.getNode(ISD::AND, DL, VT, in visitBITCAST()
13731 NewConv, DAG.getConstant(~SignBit, DL, VT)); in visitBITCAST()
13746 isa<ConstantFPSDNode>(N0.getOperand(0)) && VT.isInteger() && in visitBITCAST()
13747 !VT.isVector()) { in visitBITCAST()
13755 unsigned VTWidth = VT.getSizeInBits(); in visitBITCAST()
13757 X = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, X); in visitBITCAST()
13768 X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X); in visitBITCAST()
13773 APInt SignBit = APInt::getSignMask(VT.getSizeInBits() / 2); in visitBITCAST()
13774 SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
13776 SDValue X = DAG.getBitcast(VT, N0.getOperand(1)); in visitBITCAST()
13778 SDValue XorResult = DAG.getNode(ISD::XOR, SDLoc(N0), VT, Cst, X); in visitBITCAST()
13790 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit); in visitBITCAST()
13792 return DAG.getNode(ISD::XOR, SDLoc(N), VT, Cst, FlipBits); in visitBITCAST()
13794 APInt SignBit = APInt::getSignMask(VT.getSizeInBits()); in visitBITCAST()
13795 X = DAG.getNode(ISD::AND, SDLoc(X), VT, in visitBITCAST()
13796 X, DAG.getConstant(SignBit, SDLoc(X), VT)); in visitBITCAST()
13799 SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
13800 Cst = DAG.getNode(ISD::AND, SDLoc(Cst), VT, in visitBITCAST()
13801 Cst, DAG.getConstant(~SignBit, SDLoc(Cst), VT)); in visitBITCAST()
13804 return DAG.getNode(ISD::OR, SDLoc(N), VT, X, Cst); in visitBITCAST()
13810 if (SDValue CombineLD = CombineConsecutiveLoads(N0.getNode(), VT)) in visitBITCAST()
13817 if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT) && VT.isVector() && in visitBITCAST()
13819 VT.getVectorNumElements() >= N0.getValueType().getVectorNumElements() && in visitBITCAST()
13820 !(VT.getVectorNumElements() % N0.getValueType().getVectorNumElements())) { in visitBITCAST()
13827 Op.getOperand(0).getValueType() == VT) in visitBITCAST()
13830 return DAG.getBitcast(VT, Op); in visitBITCAST()
13843 VT.getVectorNumElements() / N0.getValueType().getVectorNumElements(); in visitBITCAST()
13850 TLI.buildLegalVectorShuffle(VT, SDLoc(N), SV0, SV1, NewMask, DAG); in visitBITCAST()
13859 EVT VT = N->getValueType(0); in visitBUILD_PAIR() local
13860 return CombineConsecutiveLoads(N, VT); in visitBUILD_PAIR()
13904 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, in ConstantFoldBITCASTofBUILD_VECTOR() local
13906 return DAG.getBuildVector(VT, SDLoc(BV), Ops); in ConstantFoldBITCASTofBUILD_VECTOR()
13953 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, Ops.size()); in ConstantFoldBITCASTofBUILD_VECTOR() local
13954 return DAG.getBuildVector(VT, DL, Ops); in ConstantFoldBITCASTofBUILD_VECTOR()
13975 EVT VT = N->getValueType(0); in visitFADDForFMACombine() local
13985 TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT) && in visitFADDForFMACombine()
13986 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFADDForFMACombine()
14000 if (TLI.generateFMAsInMachineCombiner(VT, OptLevel)) in visitFADDForFMACombine()
14005 bool Aggressive = TLI.enableAggressiveFMAFusion(VT); in visitFADDForFMACombine()
14028 return DAG.getNode(PreferredFusedOpcode, SL, VT, N0.getOperand(0), in visitFADDForFMACombine()
14035 return DAG.getNode(PreferredFusedOpcode, SL, VT, N1.getOperand(0), in visitFADDForFMACombine()
14059 SDValue CDE = DAG.getNode(PreferredFusedOpcode, SL, VT, C, D, E); in visitFADDForFMACombine()
14060 return DAG.getNode(PreferredFusedOpcode, SL, VT, A, B, CDE); in visitFADDForFMACombine()
14069 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFADDForFMACombine()
14071 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
14072 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)), in visitFADDForFMACombine()
14073 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)), in visitFADDForFMACombine()
14083 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFADDForFMACombine()
14085 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
14086 DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(0)), in visitFADDForFMACombine()
14087 DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)), in visitFADDForFMACombine()
14098 return DAG.getNode(PreferredFusedOpcode, SL, VT, X, Y, in visitFADDForFMACombine()
14099 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
14100 DAG.getNode(ISD::FP_EXTEND, SL, VT, U), in visitFADDForFMACombine()
14101 DAG.getNode(ISD::FP_EXTEND, SL, VT, V), in visitFADDForFMACombine()
14109 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFADDForFMACombine()
14126 PreferredFusedOpcode, SL, VT, DAG.getNode(ISD::FP_EXTEND, SL, VT, X), in visitFADDForFMACombine()
14127 DAG.getNode(ISD::FP_EXTEND, SL, VT, Y), in visitFADDForFMACombine()
14128 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
14129 DAG.getNode(ISD::FP_EXTEND, SL, VT, U), in visitFADDForFMACombine()
14130 DAG.getNode(ISD::FP_EXTEND, SL, VT, V), Z)); in visitFADDForFMACombine()
14137 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFADDForFMACombine()
14153 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFADDForFMACombine()
14172 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFADDForFMACombine()
14189 EVT VT = N->getValueType(0); in visitFSUBForFMACombine() local
14198 TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT) && in visitFSUBForFMACombine()
14199 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFSUBForFMACombine()
14213 if (TLI.generateFMAsInMachineCombiner(VT, OptLevel)) in visitFSUBForFMACombine()
14218 bool Aggressive = TLI.enableAggressiveFMAFusion(VT); in visitFSUBForFMACombine()
14232 return DAG.getNode(PreferredFusedOpcode, SL, VT, XY.getOperand(0), in visitFSUBForFMACombine()
14233 XY.getOperand(1), DAG.getNode(ISD::FNEG, SL, VT, Z)); in visitFSUBForFMACombine()
14242 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
14243 DAG.getNode(ISD::FNEG, SL, VT, YZ.getOperand(0)), in visitFSUBForFMACombine()
14273 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
14274 DAG.getNode(ISD::FNEG, SL, VT, N00), N01, in visitFSUBForFMACombine()
14275 DAG.getNode(ISD::FNEG, SL, VT, N1)); in visitFSUBForFMACombine()
14285 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
14287 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
14288 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)), in visitFSUBForFMACombine()
14289 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)), in visitFSUBForFMACombine()
14290 DAG.getNode(ISD::FNEG, SL, VT, N1)); in visitFSUBForFMACombine()
14300 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
14303 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
14304 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
14305 DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(0))), in visitFSUBForFMACombine()
14306 DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)), N0); in visitFSUBForFMACombine()
14321 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
14324 ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
14325 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
14326 DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)), in visitFSUBForFMACombine()
14327 DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)), in visitFSUBForFMACombine()
14344 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
14347 ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
14348 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
14349 DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)), in visitFSUBForFMACombine()
14350 DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)), in visitFSUBForFMACombine()
14378 return DAG.getNode(PreferredFusedOpcode, SL, VT, N0.getOperand(0), in visitFSUBForFMACombine()
14380 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
14383 DAG.getNode(ISD::FNEG, SL, VT, N1))); in visitFSUBForFMACombine()
14394 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
14395 DAG.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)), N1.getOperand(1), in visitFSUBForFMACombine()
14396 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
14397 DAG.getNode(ISD::FNEG, SL, VT, N20), N21, N0)); in visitFSUBForFMACombine()
14407 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
14410 PreferredFusedOpcode, SL, VT, N0.getOperand(0), N0.getOperand(1), in visitFSUBForFMACombine()
14412 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
14413 DAG.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(0)), in visitFSUBForFMACombine()
14414 DAG.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(1)), in visitFSUBForFMACombine()
14415 DAG.getNode(ISD::FNEG, SL, VT, N1))); in visitFSUBForFMACombine()
14431 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
14434 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
14435 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)), in visitFSUBForFMACombine()
14436 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)), in visitFSUBForFMACombine()
14438 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
14439 DAG.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(0)), in visitFSUBForFMACombine()
14440 DAG.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(1)), in visitFSUBForFMACombine()
14441 DAG.getNode(ISD::FNEG, SL, VT, N1))); in visitFSUBForFMACombine()
14452 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
14457 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
14458 DAG.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)), N1.getOperand(1), in visitFSUBForFMACombine()
14459 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
14460 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
14461 DAG.getNode(ISD::FP_EXTEND, SL, VT, N1200)), in visitFSUBForFMACombine()
14462 DAG.getNode(ISD::FP_EXTEND, SL, VT, N1201), N0)); in visitFSUBForFMACombine()
14478 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
14483 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
14484 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
14485 DAG.getNode(ISD::FP_EXTEND, SL, VT, N100)), in visitFSUBForFMACombine()
14486 DAG.getNode(ISD::FP_EXTEND, SL, VT, N101), in visitFSUBForFMACombine()
14487 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
14488 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
14489 DAG.getNode(ISD::FP_EXTEND, SL, VT, N1020)), in visitFSUBForFMACombine()
14490 DAG.getNode(ISD::FP_EXTEND, SL, VT, N1021), N0)); in visitFSUBForFMACombine()
14504 EVT VT = N->getValueType(0); in visitFMULForFMADistributiveCombine() local
14520 TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT) && in visitFMULForFMADistributiveCombine()
14521 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFMULForFMADistributiveCombine()
14534 bool Aggressive = TLI.enableAggressiveFMAFusion(VT); in visitFMULForFMADistributiveCombine()
14542 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y, in visitFMULForFMADistributiveCombine()
14545 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y, in visitFMULForFMADistributiveCombine()
14546 DAG.getNode(ISD::FNEG, SL, VT, Y)); in visitFMULForFMADistributiveCombine()
14565 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFMULForFMADistributiveCombine()
14566 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y, in visitFMULForFMADistributiveCombine()
14569 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFMULForFMADistributiveCombine()
14570 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y, in visitFMULForFMADistributiveCombine()
14571 DAG.getNode(ISD::FNEG, SL, VT, Y)); in visitFMULForFMADistributiveCombine()
14575 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y, in visitFMULForFMADistributiveCombine()
14576 DAG.getNode(ISD::FNEG, SL, VT, Y)); in visitFMULForFMADistributiveCombine()
14578 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y, in visitFMULForFMADistributiveCombine()
14598 EVT VT = N->getValueType(0); in visitFADD() local
14608 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FADD, DL, VT, {N0, N1})) in visitFADD()
14613 return DAG.getNode(ISD::FADD, DL, VT, N1, N0); in visitFADD()
14616 if (VT.isVector()) in visitFADD()
14630 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) in visitFADD()
14633 return DAG.getNode(ISD::FSUB, DL, VT, N0, NegN1); in visitFADD()
14636 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) in visitFADD()
14639 return DAG.getNode(ISD::FSUB, DL, VT, N1, NegN0); in visitFADD()
14651 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B); in visitFADD()
14652 return DAG.getNode(ISD::FSUB, DL, VT, N1, Add); in visitFADD()
14657 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B); in visitFADD()
14658 return DAG.getNode(ISD::FSUB, DL, VT, N0, Add); in visitFADD()
14669 return DAG.getConstantFP(0.0, DL, VT); in visitFADD()
14673 return DAG.getConstantFP(0.0, DL, VT); in visitFADD()
14685 SDValue NewC = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), N1); in visitFADD()
14686 return DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(0), NewC); in visitFADD()
14692 if (TLI.isOperationLegalOrCustom(ISD::FMUL, VT) && !N0CFP && !N1CFP) { in visitFADD()
14699 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), in visitFADD()
14700 DAG.getConstantFP(1.0, DL, VT)); in visitFADD()
14701 return DAG.getNode(ISD::FMUL, DL, VT, N1, NewCFP); in visitFADD()
14708 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), in visitFADD()
14709 DAG.getConstantFP(2.0, DL, VT)); in visitFADD()
14710 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), NewCFP); in visitFADD()
14720 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1), in visitFADD()
14721 DAG.getConstantFP(1.0, DL, VT)); in visitFADD()
14722 return DAG.getNode(ISD::FMUL, DL, VT, N0, NewCFP); in visitFADD()
14729 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1), in visitFADD()
14730 DAG.getConstantFP(2.0, DL, VT)); in visitFADD()
14731 return DAG.getNode(ISD::FMUL, DL, VT, N1.getOperand(0), NewCFP); in visitFADD()
14740 return DAG.getNode(ISD::FMUL, DL, VT, N1, in visitFADD()
14741 DAG.getConstantFP(3.0, DL, VT)); in visitFADD()
14750 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFADD()
14751 DAG.getConstantFP(3.0, DL, VT)); in visitFADD()
14760 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), in visitFADD()
14761 DAG.getConstantFP(4.0, DL, VT)); in visitFADD()
14778 EVT VT = N->getValueType(0); in visitSTRICT_FADD() local
14784 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::STRICT_FSUB, VT)) in visitSTRICT_FADD()
14787 return DAG.getNode(ISD::STRICT_FSUB, DL, DAG.getVTList(VT, ChainVT), in visitSTRICT_FADD()
14792 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::STRICT_FSUB, VT)) in visitSTRICT_FADD()
14795 return DAG.getNode(ISD::STRICT_FSUB, DL, DAG.getVTList(VT, ChainVT), in visitSTRICT_FADD()
14806 EVT VT = N->getValueType(0); in visitFSUB() local
14816 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FSUB, DL, VT, {N0, N1})) in visitFSUB()
14820 if (VT.isVector()) in visitFSUB()
14838 return DAG.getConstantFP(0.0f, DL, VT); in visitFSUB()
14849 DenormalMode DenormMode = DAG.getDenormalMode(VT); in visitFSUB()
14854 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) in visitFSUB()
14855 return DAG.getNode(ISD::FNEG, DL, VT, N1); in visitFSUB()
14865 return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(1)); in visitFSUB()
14868 return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(0)); in visitFSUB()
14874 return DAG.getNode(ISD::FADD, DL, VT, N0, NegN1); in visitFSUB()
14889 EVT VT = N->getValueType(0); in visitFMUL() local
14899 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FMUL, DL, VT, {N0, N1})) in visitFMUL()
14905 return DAG.getNode(ISD::FMUL, DL, VT, N1, N0); in visitFMUL()
14908 if (VT.isVector()) in visitFMUL()
14925 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, N01, N1); in visitFMUL()
14926 return DAG.getNode(ISD::FMUL, DL, VT, N00, MulConsts); in visitFMUL()
14934 const SDValue Two = DAG.getConstantFP(2.0, DL, VT); in visitFMUL()
14935 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, Two, N1); in visitFMUL()
14936 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), MulConsts); in visitFMUL()
14942 return DAG.getNode(ISD::FADD, DL, VT, N0, N0); in visitFMUL()
14946 if (!LegalOperations || TLI.isOperationLegal(ISD::FSUB, VT)) { in visitFMUL()
14947 return DAG.getNode(ISD::FSUB, DL, VT, in visitFMUL()
14948 DAG.getConstantFP(-0.0, DL, VT), N0, Flags); in visitFMUL()
14964 return DAG.getNode(ISD::FMUL, DL, VT, NegN0, NegN1); in visitFMUL()
14970 TLI.isOperationLegal(ISD::FABS, VT)) { in visitFMUL()
15001 TLI.isOperationLegal(ISD::FNEG, VT)) in visitFMUL()
15002 return DAG.getNode(ISD::FNEG, DL, VT, in visitFMUL()
15003 DAG.getNode(ISD::FABS, DL, VT, X)); in visitFMUL()
15005 return DAG.getNode(ISD::FABS, DL, VT, X); in visitFMUL()
15027 EVT VT = N->getValueType(0); in visitFMA() local
15040 return DAG.getNode(ISD::FMA, DL, VT, N0, N1, N2); in visitFMA()
15055 return DAG.getNode(ISD::FMA, DL, VT, NegN0, NegN1, N2); in visitFMA()
15066 return DAG.getNode(ISD::FADD, SDLoc(N), VT, N1, N2); in visitFMA()
15068 return DAG.getNode(ISD::FADD, SDLoc(N), VT, N0, N2); in visitFMA()
15073 return DAG.getNode(ISD::FMA, SDLoc(N), VT, N1, N0, N2); in visitFMA()
15080 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFMA()
15081 DAG.getNode(ISD::FADD, DL, VT, N1, N2.getOperand(1))); in visitFMA()
15088 return DAG.getNode(ISD::FMA, DL, VT, N0.getOperand(0), in visitFMA()
15089 DAG.getNode(ISD::FMUL, DL, VT, N1, N0.getOperand(1)), in visitFMA()
15097 return DAG.getNode(ISD::FADD, DL, VT, N0, N2); in visitFMA()
15100 (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))) { in visitFMA()
15101 SDValue RHSNeg = DAG.getNode(ISD::FNEG, DL, VT, N0); in visitFMA()
15103 return DAG.getNode(ISD::FADD, DL, VT, N2, RHSNeg); in visitFMA()
15108 (TLI.isOperationLegal(ISD::ConstantFP, VT) || in visitFMA()
15109 (N1.hasOneUse() && !TLI.isFPImmLegal(N1CFP->getValueAPF(), VT, in visitFMA()
15111 return DAG.getNode(ISD::FMA, DL, VT, N0.getOperand(0), in visitFMA()
15112 DAG.getNode(ISD::FNEG, DL, VT, N1), N2); in visitFMA()
15120 ISD::FMUL, DL, VT, N0, in visitFMA()
15121 DAG.getNode(ISD::FADD, DL, VT, N1, DAG.getConstantFP(1.0, DL, VT))); in visitFMA()
15127 ISD::FMUL, DL, VT, N0, in visitFMA()
15128 DAG.getNode(ISD::FADD, DL, VT, N1, DAG.getConstantFP(-1.0, DL, VT))); in visitFMA()
15134 if (!TLI.isFNegFree(VT)) in visitFMA()
15137 return DAG.getNode(ISD::FNEG, DL, VT, Neg); in visitFMA()
15170 EVT VT = N->getValueType(0); in combineRepeatedFPDivisors() local
15171 if (VT.isVector() && DAG.isSplatValue(N1)) in combineRepeatedFPDivisors()
15172 NumElts = VT.getVectorMinNumElements(); in combineRepeatedFPDivisors()
15202 SDValue FPOne = DAG.getConstantFP(1.0, DL, VT); in combineRepeatedFPDivisors()
15203 SDValue Reciprocal = DAG.getNode(ISD::FDIV, DL, VT, FPOne, N1, Flags); in combineRepeatedFPDivisors()
15209 SDValue NewNode = DAG.getNode(ISD::FMUL, SDLoc(U), VT, Dividend, in combineRepeatedFPDivisors()
15224 EVT VT = N->getValueType(0); in visitFDIV() local
15234 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FDIV, DL, VT, {N0, N1})) in visitFDIV()
15238 if (VT.isVector()) in visitFDIV()
15262 TLI.isOperationLegal(ISD::ConstantFP, VT) || in visitFDIV()
15263 TLI.isFPImmLegal(Recip, VT, ForCodeSize))) in visitFDIV()
15264 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFDIV()
15265 DAG.getConstantFP(Recip, DL, VT)); in visitFDIV()
15272 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
15277 RV = DAG.getNode(ISD::FP_EXTEND, SDLoc(N1), VT, RV); in visitFDIV()
15279 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
15285 RV = DAG.getNode(ISD::FP_ROUND, SDLoc(N1), VT, RV, N1.getOperand(1)); in visitFDIV()
15287 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
15313 SDValue AA = DAG.getNode(ISD::FMUL, DL, VT, A, A); in visitFDIV()
15315 DAG.getNode(ISD::FMUL, DL, VT, AA, Sqrt.getOperand(0)); in visitFDIV()
15317 return DAG.getNode(ISD::FMUL, DL, VT, N0, Rsqrt); in visitFDIV()
15327 SDValue Div = DAG.getNode(ISD::FDIV, SDLoc(N1), VT, Rsqrt, Y); in visitFDIV()
15329 return DAG.getNode(ISD::FMUL, DL, VT, N0, Div); in visitFDIV()
15358 return DAG.getNode(ISD::FDIV, SDLoc(N), VT, NegN0, NegN1); in visitFDIV()
15366 EVT VT = N->getValueType(0); in visitFREM() local
15374 if (SDValue C = DAG.FoldConstantArithmetic(ISD::FREM, SDLoc(N), VT, {N0, N1})) in visitFREM()
15437 EVT VT = N->getValueType(0); in visitFCOPYSIGN() local
15441 DAG.FoldConstantArithmetic(ISD::FCOPYSIGN, SDLoc(N), VT, {N0, N1})) in visitFCOPYSIGN()
15449 if (!LegalOperations || TLI.isOperationLegal(ISD::FABS, VT)) in visitFCOPYSIGN()
15450 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFCOPYSIGN()
15452 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) in visitFCOPYSIGN()
15453 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, in visitFCOPYSIGN()
15454 DAG.getNode(ISD::FABS, SDLoc(N0), VT, N0)); in visitFCOPYSIGN()
15463 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0.getOperand(0), N1); in visitFCOPYSIGN()
15467 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFCOPYSIGN()
15471 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1.getOperand(1)); in visitFCOPYSIGN()
15476 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1.getOperand(0)); in visitFCOPYSIGN()
15491 EVT VT = N->getValueType(0); in visitFPOW() local
15492 if ((VT == MVT::f32 && ExponentC->getValueAPF().isExactlyValue(1.0f/3.0f)) || in visitFPOW()
15493 (VT == MVT::f64 && ExponentC->getValueAPF().isExactlyValue(1.0/3.0))) { in visitFPOW()
15508 (!DAG.getTargetLoweringInfo().isOperationExpand(ISD::FPOW, VT) && in visitFPOW()
15509 DAG.getTargetLoweringInfo().isOperationExpand(ISD::FCBRT, VT))) in visitFPOW()
15512 return DAG.getNode(ISD::FCBRT, SDLoc(N), VT, N->getOperand(0)); in visitFPOW()
15537 if (!DAG.getTargetLoweringInfo().isOperationLegalOrCustom(ISD::FSQRT, VT)) in visitFPOW()
15547 SDValue Sqrt = DAG.getNode(ISD::FSQRT, DL, VT, N->getOperand(0)); in visitFPOW()
15548 SDValue SqrtSqrt = DAG.getNode(ISD::FSQRT, DL, VT, Sqrt); in visitFPOW()
15552 return DAG.getNode(ISD::FMUL, DL, VT, Sqrt, SqrtSqrt); in visitFPOW()
15566 EVT VT = N->getValueType(0); in foldFPToIntToFP() local
15567 if (!TLI.isOperationLegal(ISD::FTRUNC, VT) || in foldFPToIntToFP()
15575 N0.getOperand(0).getValueType() == VT) in foldFPToIntToFP()
15576 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0.getOperand(0)); in foldFPToIntToFP()
15579 N0.getOperand(0).getValueType() == VT) in foldFPToIntToFP()
15580 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0.getOperand(0)); in foldFPToIntToFP()
15587 EVT VT = N->getValueType(0); in visitSINT_TO_FP() local
15592 return DAG.getConstantFP(0.0, SDLoc(N), VT); in visitSINT_TO_FP()
15598 TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) in visitSINT_TO_FP()
15599 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); in visitSINT_TO_FP()
15607 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); in visitSINT_TO_FP()
15613 !VT.isVector() && in visitSINT_TO_FP()
15614 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitSINT_TO_FP()
15616 return DAG.getSelect(DL, VT, N0, DAG.getConstantFP(-1.0, DL, VT), in visitSINT_TO_FP()
15617 DAG.getConstantFP(0.0, DL, VT)); in visitSINT_TO_FP()
15623 N0.getOperand(0).getOpcode() == ISD::SETCC && !VT.isVector() && in visitSINT_TO_FP()
15624 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitSINT_TO_FP()
15626 return DAG.getSelect(DL, VT, N0.getOperand(0), in visitSINT_TO_FP()
15627 DAG.getConstantFP(1.0, DL, VT), in visitSINT_TO_FP()
15628 DAG.getConstantFP(0.0, DL, VT)); in visitSINT_TO_FP()
15639 EVT VT = N->getValueType(0); in visitUINT_TO_FP() local
15644 return DAG.getConstantFP(0.0, SDLoc(N), VT); in visitUINT_TO_FP()
15650 TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) in visitUINT_TO_FP()
15651 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); in visitUINT_TO_FP()
15659 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); in visitUINT_TO_FP()
15663 if (N0.getOpcode() == ISD::SETCC && !VT.isVector() && in visitUINT_TO_FP()
15664 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitUINT_TO_FP()
15666 return DAG.getSelect(DL, VT, N0, DAG.getConstantFP(1.0, DL, VT), in visitUINT_TO_FP()
15667 DAG.getConstantFP(0.0, DL, VT)); in visitUINT_TO_FP()
15679 EVT VT = N->getValueType(0); in FoldIntToFPToInt() local
15699 unsigned OutputSize = (int)VT.getScalarSizeInBits(); in FoldIntToFPToInt()
15706 if (VT.getScalarSizeInBits() > SrcVT.getScalarSizeInBits()) { in FoldIntToFPToInt()
15709 return DAG.getNode(ExtOp, SDLoc(N), VT, Src); in FoldIntToFPToInt()
15711 if (VT.getScalarSizeInBits() < SrcVT.getScalarSizeInBits()) in FoldIntToFPToInt()
15712 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Src); in FoldIntToFPToInt()
15713 return DAG.getBitcast(VT, Src); in FoldIntToFPToInt()
15720 EVT VT = N->getValueType(0); in visitFP_TO_SINT() local
15724 return DAG.getUNDEF(VT); in visitFP_TO_SINT()
15728 return DAG.getNode(ISD::FP_TO_SINT, SDLoc(N), VT, N0); in visitFP_TO_SINT()
15735 EVT VT = N->getValueType(0); in visitFP_TO_UINT() local
15739 return DAG.getUNDEF(VT); in visitFP_TO_UINT()
15743 return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), VT, N0); in visitFP_TO_UINT()
15752 EVT VT = N->getValueType(0); in visitFP_ROUND() local
15756 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, N0, N1); in visitFP_ROUND()
15759 if (N0.getOpcode() == ISD::FP_EXTEND && VT == N0.getOperand(0).getValueType()) in visitFP_ROUND()
15774 if (N0.getOperand(0).getValueType() == MVT::f80 && VT == MVT::f16) in visitFP_ROUND()
15784 return DAG.getNode(ISD::FP_ROUND, DL, VT, N0.getOperand(0), in visitFP_ROUND()
15791 SDValue Tmp = DAG.getNode(ISD::FP_ROUND, SDLoc(N0), VT, in visitFP_ROUND()
15794 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, in visitFP_ROUND()
15806 EVT VT = N->getValueType(0); in visitFP_EXTEND() local
15815 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, N0); in visitFP_EXTEND()
15819 TLI.getOperationAction(ISD::FP16_TO_FP, VT) == TargetLowering::Legal) in visitFP_EXTEND()
15820 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), VT, N0.getOperand(0)); in visitFP_EXTEND()
15827 if (In.getValueType() == VT) return In; in visitFP_EXTEND()
15828 if (VT.bitsLT(In.getValueType())) in visitFP_EXTEND()
15829 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, in visitFP_EXTEND()
15831 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, In); in visitFP_EXTEND()
15836 TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, VT, N0.getValueType())) { in visitFP_EXTEND()
15838 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, in visitFP_EXTEND()
15859 EVT VT = N->getValueType(0); in visitFCEIL() local
15863 return DAG.getNode(ISD::FCEIL, SDLoc(N), VT, N0); in visitFCEIL()
15870 EVT VT = N->getValueType(0); in visitFTRUNC() local
15874 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0); in visitFTRUNC()
15894 EVT VT = N->getValueType(0); in visitFFLOOR() local
15898 return DAG.getNode(ISD::FFLOOR, SDLoc(N), VT, N0); in visitFFLOOR()
15905 EVT VT = N->getValueType(0); in visitFNEG() local
15910 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0); in visitFNEG()
15923 return DAG.getNode(ISD::FSUB, SDLoc(N), VT, N0.getOperand(1), in visitFNEG()
15936 EVT VT = N->getValueType(0); in visitFMinMax() local
15944 if (SDValue C = DAG.FoldConstantArithmetic(Opc, SDLoc(N), VT, {N0, N1})) in visitFMinMax()
15950 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0); in visitFMinMax()
15986 EVT VT = N->getValueType(0); in visitFABS() local
15990 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFABS()
15999 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0.getOperand(0)); in visitFABS()
16178 EVT VT = LD->getMemoryVT(); in getCombineLoadStoreParts() local
16179 if (!TLI.isIndexedLoadLegal(Inc, VT) && !TLI.isIndexedLoadLegal(Dec, VT)) in getCombineLoadStoreParts()
16185 EVT VT = ST->getMemoryVT(); in getCombineLoadStoreParts() local
16186 if (!TLI.isIndexedStoreLegal(Inc, VT) && !TLI.isIndexedStoreLegal(Dec, VT)) in getCombineLoadStoreParts()
16193 EVT VT = LD->getMemoryVT(); in getCombineLoadStoreParts() local
16194 if (!TLI.isIndexedMaskedLoadLegal(Inc, VT) && in getCombineLoadStoreParts()
16195 !TLI.isIndexedMaskedLoadLegal(Dec, VT)) in getCombineLoadStoreParts()
16202 EVT VT = ST->getMemoryVT(); in getCombineLoadStoreParts() local
16203 if (!TLI.isIndexedMaskedStoreLegal(Inc, VT) && in getCombineLoadStoreParts()
16204 !TLI.isIndexedMaskedStoreLegal(Dec, VT)) in getCombineLoadStoreParts()
17540 MVT VT = MVT::getIntegerVT(NumBytes * 8); in ShrinkLoadReplaceStoreWithStore() local
17543 if (DC->isTypeLegal(VT)) in ShrinkLoadReplaceStoreWithStore()
17546 TLI.isTruncStoreLegal(IVal.getValueType(), VT)) in ShrinkLoadReplaceStoreWithStore()
17552 !TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, in ShrinkLoadReplaceStoreWithStore()
17582 VT, St->getOriginalAlign()); in ShrinkLoadReplaceStoreWithStore()
17585 IVal = DAG.getNode(ISD::TRUNCATE, SDLoc(IVal), VT, IVal); in ShrinkLoadReplaceStoreWithStore()
17605 EVT VT = Value.getValueType(); in ReduceLoadOpStoreWidth() local
17607 if (ST->isTruncatingStore() || VT.isVector()) in ReduceLoadOpStoreWidth()
17669 !TLI.isNarrowingProfitable(VT, NewVT))) { in ReduceLoadOpStoreWidth()
17735 EVT VT = LD->getMemoryVT(); in TransformFPLoadStorePair() local
17736 if (!VT.isFloatingPoint() || in TransformFPLoadStorePair()
17737 VT != ST->getMemoryVT() || in TransformFPLoadStorePair()
17744 TypeSize VTSize = VT.getSizeInBits(); in TransformFPLoadStorePair()
17755 !TLI.isDesirableToTransformToIntegerOp(ISD::LOAD, VT) || in TransformFPLoadStorePair()
17756 !TLI.isDesirableToTransformToIntegerOp(ISD::STORE, VT) || in TransformFPLoadStorePair()
19341 EVT VT = EVT::getIntegerVT(*DAG.getContext(), HalfValBitSize); in splitMergedValStore() local
19342 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Lo.getOperand(0)); in splitMergedValStore()
19343 Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Hi.getOperand(0)); in splitMergedValStore()
19453 EVT VT = DestVec.getValueType(); in combineInsertEltToShuffle() local
19459 unsigned ExtendRatio = VT.getSizeInBits() / SubVecVT.getSizeInBits(); in combineInsertEltToShuffle()
19494 return DAG.getBitcast(VT, Shuf); in combineInsertEltToShuffle()
19503 EVT VT = InVec.getValueType(); in visitINSERT_VECTOR_ELT() local
19507 if (IndexC && VT.isFixedLengthVector() && in visitINSERT_VECTOR_ELT()
19508 IndexC->getZExtValue() >= VT.getVectorNumElements()) in visitINSERT_VECTOR_ELT()
19509 return DAG.getUNDEF(VT); in visitINSERT_VECTOR_ELT()
19520 if (InVec.isUndef() && TLI.shouldSplatInsEltVarIndex(VT)) { in visitINSERT_VECTOR_ELT()
19521 if (VT.isScalableVector()) in visitINSERT_VECTOR_ELT()
19522 return DAG.getSplatVector(VT, DL, InVal); in visitINSERT_VECTOR_ELT()
19524 SmallVector<SDValue, 8> Ops(VT.getVectorNumElements(), InVal); in visitINSERT_VECTOR_ELT()
19525 return DAG.getBuildVector(VT, DL, Ops); in visitINSERT_VECTOR_ELT()
19530 if (VT.isScalableVector()) in visitINSERT_VECTOR_ELT()
19533 unsigned NumElts = VT.getVectorNumElements(); in visitINSERT_VECTOR_ELT()
19545 InVal.getOperand(0).getValueType() == VT && in visitINSERT_VECTOR_ELT()
19562 SDValue NewOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, in visitINSERT_VECTOR_ELT()
19566 VT, NewOp, InVec.getOperand(1), InVec.getOperand(2)); in visitINSERT_VECTOR_ELT()
19571 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) { in visitINSERT_VECTOR_ELT()
19574 return DAG.getBuildVector(VT, DL, {InVal}); in visitINSERT_VECTOR_ELT()
19582 if (VT.isInteger()) { in visitINSERT_VECTOR_ELT()
19595 Op = VT.isInteger() ? DAG.getAnyExtOrTrunc(Op, DL, MaxEltVT) : Op; in visitINSERT_VECTOR_ELT()
19599 return DAG.getBuildVector(VT, DL, Ops); in visitINSERT_VECTOR_ELT()
19751 EVT VT = ExtElt->getValueType(0); in scalarizeExtractedBinop() local
19752 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Index); in scalarizeExtractedBinop()
19753 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op1, Index); in scalarizeExtractedBinop()
19754 return DAG.getNode(Vec.getOpcode(), DL, VT, Ext0, Ext1); in scalarizeExtractedBinop()
20082 EVT VT = N->getValueType(0); in reduceBuildVecExtToExtBuildVec() local
20126 EVT OutScalarTy = VT.getScalarType(); in reduceBuildVecExtToExtBuildVec()
20147 unsigned NewBVElems = ElemRatio * VT.getVectorNumElements(); in reduceBuildVecExtToExtBuildVec()
20170 assert(VecVT.getSizeInBits() == VT.getSizeInBits() && in reduceBuildVecExtToExtBuildVec()
20175 TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))) in reduceBuildVecExtToExtBuildVec()
20184 return DAG.getBitcast(VT, BV); in reduceBuildVecExtToExtBuildVec()
20199 EVT VT = N->getValueType(0); in reduceBuildVecTruncToBitCast() local
20200 EVT OutScalarTy = VT.getScalarType(); in reduceBuildVecTruncToBitCast()
20259 if (Src.getValueType().getSizeInBits() != VT.getSizeInBits()) in reduceBuildVecTruncToBitCast()
20262 return DAG.getBitcast(VT, Src); in reduceBuildVecTruncToBitCast()
20271 EVT VT = N->getValueType(0); in createBuildVecShuffle() local
20275 unsigned NumElems = VT.getVectorNumElements(); in createBuildVecShuffle()
20282 uint64_t VTSize = VT.getFixedSizeInBits(); in createBuildVecShuffle()
20291 if (InVT1 != VT || InVT2 != VT) { in createBuildVecShuffle()
20300 VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in createBuildVecShuffle()
20303 if (!TLI.isExtractSubvectorCheap(VT, InVT1, NumElems)) in createBuildVecShuffle()
20309 VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1, in createBuildVecShuffle()
20311 VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1, ZeroIdx); in createBuildVecShuffle()
20341 VecIn2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in createBuildVecShuffle()
20379 Shuffle = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuffle, ZeroIdx); in createBuildVecShuffle()
20410 EVT VT = BV->getValueType(0); in reduceBuildVecToShuffleWithZero() local
20415 Zext.getValueSizeInBits() != VT.getScalarSizeInBits()) in reduceBuildVecToShuffleWithZero()
20424 Extract.getOperand(0).getValueSizeInBits() != VT.getSizeInBits()) in reduceBuildVecToShuffleWithZero()
20458 return DAG.getBitcast(VT, Shuf); in reduceBuildVecToShuffleWithZero()
20475 EVT VT = N->getValueType(0); in reduceBuildVecToShuffle() local
20478 if (!isTypeLegal(VT)) in reduceBuildVecToShuffle()
20485 if (LegalOperations && !TLI.isOperationLegal(ISD::VECTOR_SHUFFLE, VT)) in reduceBuildVecToShuffle()
20531 if (VT.getVectorElementType() != in reduceBuildVecToShuffle()
20653 Shuffles.push_back(VT.isInteger() ? DAG.getConstant(0, DL, VT) in reduceBuildVecToShuffle()
20654 : DAG.getConstantFP(0.0, DL, VT)); in reduceBuildVecToShuffle()
20680 Shuffles.push_back(DAG.getUNDEF(VT)); in reduceBuildVecToShuffle()
20684 Shuffles[CurSize] = DAG.getUNDEF(VT); in reduceBuildVecToShuffle()
20723 Shuffles[In] = DAG.getVectorShuffle(VT, DL, L, R, Mask); in reduceBuildVecToShuffle()
20737 EVT VT = N->getValueType(0); in convertBuildVecZextToZext() local
20757 if (Offset < 0 || (Offset % VT.getVectorNumElements()) != 0) in convertBuildVecZextToZext()
20779 VT, In); in convertBuildVecZextToZext()
20783 EVT VT = N->getValueType(0); in visitBUILD_VECTOR() local
20787 return DAG.getUNDEF(VT); in visitBUILD_VECTOR()
20809 return DAG.getBitcast(VT, Concat); in visitBUILD_VECTOR()
20858 if (TLI.getOperationAction(ISD::SPLAT_VECTOR, VT) != TargetLowering::Expand) in visitBUILD_VECTOR()
20861 return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), VT, V); in visitBUILD_VECTOR()
20876 EVT VT = N->getValueType(0); in combineConcatVectorOfScalars() local
20924 VT.getSizeInBits() / SVT.getSizeInBits()); in combineConcatVectorOfScalars()
20925 return DAG.getBitcast(VT, DAG.getBuildVector(VecVT, DL, Ops)); in combineConcatVectorOfScalars()
20933 EVT VT = N->getValueType(0); in combineConcatVectorOfConcatVectors() local
20963 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, ConcatOps); in combineConcatVectorOfConcatVectors()
20971 EVT VT = N->getValueType(0); in combineConcatVectorOfExtracts() local
20975 if (VT.isScalableVector()) in combineConcatVectorOfExtracts()
20978 int NumElts = VT.getVectorNumElements(); in combineConcatVectorOfExtracts()
20981 SDValue SV0 = DAG.getUNDEF(VT), SV1 = DAG.getUNDEF(VT); in combineConcatVectorOfExtracts()
21013 if (ExtVT.getSizeInBits() != VT.getSizeInBits()) in combineConcatVectorOfExtracts()
21040 return TLI.buildLegalVectorShuffle(VT, SDLoc(N), DAG.getBitcast(VT, SV0), in combineConcatVectorOfExtracts()
21041 DAG.getBitcast(VT, SV1), Mask, DAG); in combineConcatVectorOfExtracts()
21079 EVT VT = N->getValueType(0); in combineConcatVectorOfCasts() local
21088 !TLI.isTypeLegal(VT)) in combineConcatVectorOfCasts()
21093 if (!TLI.isOperationLegalOrCustom(CastOpcode, VT) || in combineConcatVectorOfCasts()
21104 return DAG.getNode(CastOpcode, DL, VT, NewConcat); in combineConcatVectorOfCasts()
21113 EVT VT = N->getValueType(0); in visitCONCAT_VECTORS() local
21115 return DAG.getUNDEF(VT); in visitCONCAT_VECTORS()
21129 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in visitCONCAT_VECTORS()
21159 if (VT.getSizeInBits() % SclTy.getSizeInBits()) in visitCONCAT_VECTORS()
21162 unsigned VNTNumElms = VT.getSizeInBits() / SclTy.getSizeInBits(); in visitCONCAT_VECTORS()
21171 return DAG.getBitcast(VT, Res); in visitCONCAT_VECTORS()
21184 EVT SVT = VT.getScalarType(); in visitCONCAT_VECTORS()
21219 assert(VT.getVectorNumElements() == Opnds.size() && in visitCONCAT_VECTORS()
21221 return DAG.getBuildVector(VT, SDLoc(N), Opnds); in visitCONCAT_VECTORS()
21229 if (Level < AfterLegalizeVectorOps && TLI.isTypeLegal(VT)) { in visitCONCAT_VECTORS()
21380 EVT VT = Extract->getValueType(0); in narrowExtractedVectorBinOp() local
21382 assert(ExtractIndex % VT.getVectorNumElements() == 0 && in narrowExtractedVectorBinOp()
21387 unsigned NarrowWidth = VT.getSizeInBits(); in narrowExtractedVectorBinOp()
21408 unsigned ConcatOpNum = ExtractIndex / VT.getVectorNumElements(); in narrowExtractedVectorBinOp()
21421 return DAG.getBitcast(VT, NarrowBinOp); in narrowExtractedVectorBinOp()
21464 return DAG.getBitcast(VT, NarrowBinOp); in narrowExtractedVectorBinOp()
21483 EVT VT = Extract->getValueType(0); in narrowExtractedVectorLoad() local
21486 if (!VT.isByteSized()) in narrowExtractedVectorLoad()
21490 unsigned NumElts = VT.getVectorMinNumElements(); in narrowExtractedVectorLoad()
21498 TypeSize Offset = VT.getStoreSize() * (Index / NumElts); in narrowExtractedVectorLoad()
21501 if (!TLI.shouldReduceLoadWidth(Ld, Ld->getExtensionType(), VT)) in narrowExtractedVectorLoad()
21511 uint64_t StoreSize = MemoryLocation::getSizeOrUnknown(VT.getStoreSize()); in narrowExtractedVectorLoad()
21522 SDValue NewLd = DAG.getLoad(VT, DL, Ld->getChain(), NewAddr, MMO); in narrowExtractedVectorLoad()
21885 EVT VT = Shuf->getValueType(0); in foldShuffleOfConcatUndefs() local
21886 unsigned NumElts = VT.getVectorNumElements(); in foldShuffleOfConcatUndefs()
21905 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), in foldShuffleOfConcatUndefs()
21917 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Shuf0, Shuf1); in foldShuffleOfConcatUndefs()
21923 EVT VT = N->getValueType(0); in partitionShuffleOfConcats() local
21924 unsigned NumElts = VT.getVectorNumElements(); in partitionShuffleOfConcats()
21948 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, N0, N1); in partitionShuffleOfConcats()
21982 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in partitionShuffleOfConcats()
22004 EVT VT = SVN->getValueType(0); in combineShuffleOfScalars() local
22005 unsigned NumElts = VT.getVectorNumElements(); in combineShuffleOfScalars()
22038 SDValue Op = DAG.getUNDEF(VT.getScalarType()); in combineShuffleOfScalars()
22066 EVT SVT = VT.getScalarType(); in combineShuffleOfScalars()
22070 if (SVT != VT.getScalarType()) in combineShuffleOfScalars()
22076 return DAG.getBuildVector(VT, SDLoc(SVN), Ops); in combineShuffleOfScalars()
22087 EVT VT = SVN->getValueType(0); in combineShuffleToVectorExtend() local
22091 if (!VT.isInteger() || IsBigEndian) in combineShuffleToVectorExtend()
22094 unsigned NumElts = VT.getVectorNumElements(); in combineShuffleToVectorExtend()
22095 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in combineShuffleToVectorExtend()
22127 return DAG.getBitcast(VT, in combineShuffleToVectorExtend()
22142 EVT VT = SVN->getValueType(0); in combineTruncationShuffle() local
22146 if (!VT.isInteger() || IsBigEndian) in combineTruncationShuffle()
22159 unsigned NumElts = VT.getVectorNumElements(); in combineTruncationShuffle()
22160 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in combineTruncationShuffle()
22191 return DAG.getBitcast(VT, N00); in combineTruncationShuffle()
22266 EVT VT = SVN->getValueType(0); in combineShuffleOfBitcast() local
22278 int VTLanes = VT.getVectorNumElements(); in combineShuffleOfBitcast()
22302 return DAG.getBitcast(VT, NewShuf); in combineShuffleOfBitcast()
22347 EVT VT = OuterShuf->getValueType(0); in formSplatFromShuffles() local
22348 assert(VT == InnerShuf->getValueType(0) && "Expected matching shuffle types"); in formSplatFromShuffles()
22349 if (!DAG.getTargetLoweringInfo().isShuffleMaskLegal(CombinedMask, VT)) in formSplatFromShuffles()
22352 return DAG.getVectorShuffle(VT, SDLoc(OuterShuf), InnerShuf->getOperand(0), in formSplatFromShuffles()
22462 EVT VT = N->getValueType(0); in visitVECTOR_SHUFFLE() local
22463 unsigned NumElts = VT.getVectorNumElements(); in visitVECTOR_SHUFFLE()
22468 assert(N0.getValueType() == VT && "Vector shuffle must be normalized in DAG"); in visitVECTOR_SHUFFLE()
22472 return DAG.getUNDEF(VT); in visitVECTOR_SHUFFLE()
22478 return DAG.getVectorShuffle(VT, SDLoc(N), N0, DAG.getUNDEF(VT), in visitVECTOR_SHUFFLE()
22498 return DAG.getVectorShuffle(VT, SDLoc(N), N0, N1, NewMask); in visitVECTOR_SHUFFLE()
22515 if (N0.hasOneUse() && TLI.isExtractVecEltCheap(VT, SplatIndex) && in visitVECTOR_SHUFFLE()
22521 EVT EltVT = VT.getScalarType(); in visitVECTOR_SHUFFLE()
22527 SDValue Insert = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, NewBO); in visitVECTOR_SHUFFLE()
22528 SmallVector<int, 16> ZeroMask(VT.getVectorNumElements(), 0); in visitVECTOR_SHUFFLE()
22529 return DAG.getVectorShuffle(VT, DL, Insert, DAG.getUNDEF(VT), ZeroMask); in visitVECTOR_SHUFFLE()
22534 if ((!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) && in visitVECTOR_SHUFFLE()
22537 return DAG.getSplatBuildVector(VT, SDLoc(N), N0.getOperand(0)); in visitVECTOR_SHUFFLE()
22542 return DAG.getSplatBuildVector(VT, SDLoc(N), N0.getOperand(1)); in visitVECTOR_SHUFFLE()
22587 if (V->getValueType(0) != VT) in visitVECTOR_SHUFFLE()
22588 NewBV = DAG.getBitcast(VT, NewBV); in visitVECTOR_SHUFFLE()
22635 if (TLI.isShuffleMaskLegal(NewMask, VT)) { in visitVECTOR_SHUFFLE()
22637 SDValue NewCat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, in visitVECTOR_SHUFFLE()
22639 return DAG.getVectorShuffle(VT, SDLoc(N), NewCat, N1, NewMask); in visitVECTOR_SHUFFLE()
22647 if (Level < AfterLegalizeVectorOps && TLI.isTypeLegal(VT) && in visitVECTOR_SHUFFLE()
22648 TLI.isOperationLegalOrCustom(ISD::INSERT_SUBVECTOR, VT)) { in visitVECTOR_SHUFFLE()
22688 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, LHS, in visitVECTOR_SHUFFLE()
22709 if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT)) { in visitVECTOR_SHUFFLE()
22734 EVT IntVT = VT.changeVectorElementTypeToInteger(); in visitVECTOR_SHUFFLE()
22735 EVT IntSVT = VT.getVectorElementType().changeTypeToInteger(); in visitVECTOR_SHUFFLE()
22753 VT, DAG.getVectorShuffle(IntVT, DL, DAG.getBitcast(IntVT, N0), in visitVECTOR_SHUFFLE()
22758 VT, DAG.getNode(ISD::AND, DL, IntVT, DAG.getBitcast(IntVT, N0), in visitVECTOR_SHUFFLE()
22766 if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT)) in visitVECTOR_SHUFFLE()
22775 TLI.isTypeLegal(VT)) { in visitVECTOR_SHUFFLE()
22779 EVT SVT = VT.getScalarType(); in visitVECTOR_SHUFFLE()
22784 EVT ScaleVT = SVT.bitsLT(InnerSVT) ? VT : InnerVT; in visitVECTOR_SHUFFLE()
22819 VT, DAG.getVectorShuffle(ScaleVT, SDLoc(N), SV0, SV1, NewMask)); in visitVECTOR_SHUFFLE()
22835 [NumElts, &VT](bool Commute, ShuffleVectorSDNode *SVN, in visitVECTOR_SHUFFLE()
22939 if (TLI.isShuffleMaskLegal(Mask, VT)) in visitVECTOR_SHUFFLE()
22944 return TLI.isShuffleMaskLegal(Mask, VT); in visitVECTOR_SHUFFLE()
22947 if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT)) { in visitVECTOR_SHUFFLE()
22956 assert(N1->getOperand(0).getValueType() == VT && in visitVECTOR_SHUFFLE()
22990 assert(OtherSV->getOperand(0).getValueType() == VT && in visitVECTOR_SHUFFLE()
22999 return DAG.getUNDEF(VT); in visitVECTOR_SHUFFLE()
23001 return DAG.getVectorShuffle(VT, SDLoc(N), in visitVECTOR_SHUFFLE()
23002 SV0 ? SV0 : DAG.getUNDEF(VT), in visitVECTOR_SHUFFLE()
23003 SV1 ? SV1 : DAG.getUNDEF(VT), Mask); in visitVECTOR_SHUFFLE()
23024 if (Op00.getValueType() == VT && Op10.getValueType() == VT && in visitVECTOR_SHUFFLE()
23025 Op01.getValueType() == VT && Op11.getValueType() == VT && in visitVECTOR_SHUFFLE()
23075 VT, DL, LeftSV0 ? LeftSV0 : DAG.getUNDEF(VT), in visitVECTOR_SHUFFLE()
23076 LeftSV1 ? LeftSV1 : DAG.getUNDEF(VT), LeftMask); in visitVECTOR_SHUFFLE()
23078 VT, DL, RightSV0 ? RightSV0 : DAG.getUNDEF(VT), in visitVECTOR_SHUFFLE()
23079 RightSV1 ? RightSV1 : DAG.getUNDEF(VT), RightMask); in visitVECTOR_SHUFFLE()
23080 return DAG.getNode(SrcOpcode, DL, VT, LHS, RHS); in visitVECTOR_SHUFFLE()
23094 EVT VT = N->getValueType(0); in visitSCALAR_TO_VECTOR() local
23099 VT.isFixedLengthVector() && in visitSCALAR_TO_VECTOR()
23110 if (VT.getScalarType() != InVal.getValueType() && in visitSCALAR_TO_VECTOR()
23112 isTypeLegal(VT.getScalarType())) { in visitSCALAR_TO_VECTOR()
23114 DAG.getNode(ISD::TRUNCATE, SDLoc(InVal), VT.getScalarType(), InVal); in visitSCALAR_TO_VECTOR()
23115 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Val); in visitSCALAR_TO_VECTOR()
23117 if (VT.getScalarType() == InVecT.getScalarType() && in visitSCALAR_TO_VECTOR()
23118 VT.getVectorNumElements() <= InVecT.getVectorNumElements()) { in visitSCALAR_TO_VECTOR()
23125 if (VT == InVecT) in visitSCALAR_TO_VECTOR()
23128 if (VT.getVectorNumElements() != InVecT.getVectorNumElements()) { in visitSCALAR_TO_VECTOR()
23132 VT.getVectorNumElements()); in visitSCALAR_TO_VECTOR()
23145 EVT VT = N->getValueType(0); in visitINSERT_SUBVECTOR() local
23158 N1.getOperand(1) == N2 && N1.getOperand(0).getValueType() == VT) in visitINSERT_SUBVECTOR()
23164 return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), VT, N1.getOperand(0)); in visitINSERT_SUBVECTOR()
23174 VT.getVectorElementCount() && in visitINSERT_SUBVECTOR()
23176 VT.getSizeInBits()) { in visitINSERT_SUBVECTOR()
23177 return DAG.getBitcast(VT, N1.getOperand(0).getOperand(0)); in visitINSERT_SUBVECTOR()
23191 CN0VT.getVectorElementCount() == VT.getVectorElementCount()) { in visitINSERT_SUBVECTOR()
23194 return DAG.getBitcast(VT, NewINSERT); in visitINSERT_SUBVECTOR()
23204 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0.getOperand(0), in visitINSERT_SUBVECTOR()
23212 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0, in visitINSERT_SUBVECTOR()
23230 ElementCount NumElts = VT.getVectorElementCount(); in visitINSERT_SUBVECTOR()
23231 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in visitINSERT_SUBVECTOR()
23247 return DAG.getBitcast(VT, Res); in visitINSERT_SUBVECTOR()
23261 SDValue NewOp = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, in visitINSERT_SUBVECTOR()
23265 VT, NewOp, N0.getOperand(1), N0.getOperand(2)); in visitINSERT_SUBVECTOR()
23278 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in visitINSERT_SUBVECTOR()
23325 EVT VT = N0.getValueType(); in visitVECREDUCE() local
23329 if (VT.getVectorElementCount().isScalar()) { in visitVECREDUCE()
23332 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getVectorElementType(), N0, in visitVECREDUCE()
23344 if (!TLI.isOperationLegalOrCustom(Opcode, VT) && in visitVECREDUCE()
23345 TLI.isOperationLegalOrCustom(NewOpcode, VT) && in visitVECREDUCE()
23346 DAG.ComputeNumSignBits(N0) == VT.getScalarSizeInBits()) in visitVECREDUCE()
23407 EVT VT = N->getValueType(0); in XformToShuffleWithZero() local
23471 return DAG.getBitcast(VT, DAG.getVectorShuffle(ClearVT, DL, in XformToShuffleWithZero()
23496 EVT VT = N->getValueType(0); in scalarizeBinOpOfSplats() local
23497 EVT EltVT = VT.getVectorElementType(); in scalarizeBinOpOfSplats()
23513 !(IsBothSplatVector || TLI.isExtractVecEltCheap(VT, Index0)) || in scalarizeBinOpOfSplats()
23529 SmallVector<SDValue, 8> Ops(VT.getVectorNumElements(), DAG.getUNDEF(EltVT)); in scalarizeBinOpOfSplats()
23531 return DAG.getBuildVector(VT, DL, Ops); in scalarizeBinOpOfSplats()
23535 if (VT.isScalableVector()) in scalarizeBinOpOfSplats()
23536 return DAG.getSplatVector(VT, DL, ScalarBO); in scalarizeBinOpOfSplats()
23537 SmallVector<SDValue, 8> Ops(VT.getVectorNumElements(), ScalarBO); in scalarizeBinOpOfSplats()
23538 return DAG.getBuildVector(VT, DL, Ops); in scalarizeBinOpOfSplats()
23543 EVT VT = N->getValueType(0); in SimplifyVBinOp() local
23544 assert(VT.isVector() && "SimplifyVBinOp only works on vectors!"); in SimplifyVBinOp()
23566 SDValue NewBinOp = DAG.getNode(Opcode, DL, VT, LHS.getOperand(0), in SimplifyVBinOp()
23569 return DAG.getVectorShuffle(VT, DL, NewBinOp, UndefV, Shuf0->getMask()); in SimplifyVBinOp()
23583 SDValue NewBinOp = DAG.getNode(Opcode, DL, VT, X, RHS, Flags); in SimplifyVBinOp()
23584 return DAG.getVectorShuffle(VT, DL, NewBinOp, DAG.getUNDEF(VT), in SimplifyVBinOp()
23592 SDValue NewBinOp = DAG.getNode(Opcode, DL, VT, LHS, X, Flags); in SimplifyVBinOp()
23593 return DAG.getVectorShuffle(VT, DL, NewBinOp, DAG.getUNDEF(VT), in SimplifyVBinOp()
23615 DAG.getNode(Opcode, DL, VT, DAG.getUNDEF(VT), DAG.getUNDEF(VT)); in SimplifyVBinOp()
23617 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, VecC, NarrowBO, Z); in SimplifyVBinOp()
23648 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in SimplifyVBinOp()
23959 EVT VT = N->getValueType(0); in foldSelectOfBinops() local
23981 DAG.getSelect(DL, VT, N0, N1.getOperand(0), N2.getOperand(0)); in foldSelectOfBinops()
23992 VT == N1.getOperand(1).getValueType() && in foldSelectOfBinops()
23993 VT == N2.getOperand(1).getValueType()) { in foldSelectOfBinops()
23995 DAG.getSelect(DL, VT, N0, N1.getOperand(1), N2.getOperand(1)); in foldSelectOfBinops()
24009 EVT VT = N->getValueType(0); in foldSignChangeInBitcast() local
24011 bool IsFree = IsFabs ? TLI.isFAbsFree(VT) : TLI.isFNegFree(VT); in foldSignChangeInBitcast()
24043 return DAG.getBitcast(VT, Int); in foldSignChangeInBitcast()
24060 EVT VT = N2.getValueType(); in convertSelectOfFPConstantsToLoadOffset() local
24061 if (!TV || !FV || !TLI.isTypeLegal(VT)) in convertSelectOfFPConstantsToLoadOffset()
24065 if (TLI.getOperationAction(ISD::ConstantFP, VT) == TargetLowering::Legal || in convertSelectOfFPConstantsToLoadOffset()
24113 EVT VT = N2.getValueType(); in SimplifySelectCC() local
24142 N0->getValueType(0) == VT && isNullConstant(N1) && isNullConstant(N2)) { in SimplifySelectCC()
24149 if (!TLI.shouldAvoidTransformToShift(VT, ShCt)) { in SimplifySelectCC()
24153 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N0), VT, AndLHS, ShlAmt); in SimplifySelectCC()
24160 SDValue Shr = DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl, ShrAmt); in SimplifySelectCC()
24162 return DAG.getNode(ISD::AND, DL, VT, Shr, N3); in SimplifySelectCC()
24190 if (VT.bitsLT(SCC.getValueType())) in SimplifySelectCC()
24191 Temp = DAG.getZeroExtendInReg(SCC, SDLoc(N2), VT); in SimplifySelectCC()
24193 Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2), VT, SCC); in SimplifySelectCC()
24196 Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2), VT, SCC); in SimplifySelectCC()
24206 if (TLI.shouldAvoidTransformToShift(VT, ShCt)) in SimplifySelectCC()
24231 if (ValueOnZeroC->getAPIntValue() == VT.getSizeInBits()) { in SimplifySelectCC()
24237 (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ, VT))) in SimplifySelectCC()
24238 return DAG.getNode(ISD::CTTZ, DL, VT, N0); in SimplifySelectCC()
24244 (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ, VT))) in SimplifySelectCC()
24245 return DAG.getNode(ISD::CTLZ, DL, VT, N0); in SimplifySelectCC()
24256 !TLI.shouldAvoidTransformToShift(VT, CmpOpVT.getScalarSizeInBits() - 1)) { in SimplifySelectCC()
24260 return DAG.getNode(ISD::XOR, DL, VT, DAG.getSExtOrTrunc(ASR, DL, VT), in SimplifySelectCC()
24261 DAG.getSExtOrTrunc(CC == ISD::SETLT ? N3 : N2, DL, VT)); in SimplifySelectCC()
24273 SDValue DAGCombiner::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, in SimplifySetCC() argument
24278 return TLI.SimplifySetCC(VT, N0, N1, Cond, foldBooleans, DagCombineInfo, DL); in SimplifySetCC()
24366 EVT VT = V.getValueType(); in BuildLogBase2() local
24367 SDValue Ctlz = DAG.getNode(ISD::CTLZ, DL, VT, V); in BuildLogBase2()
24368 SDValue Base = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT); in BuildLogBase2()
24369 SDValue LogBase2 = DAG.getNode(ISD::SUB, DL, VT, Base, Ctlz); in BuildLogBase2()
24387 EVT VT = Op.getValueType(); in BuildDivEstimate() local
24388 if (VT.getScalarType() != MVT::f16 && VT.getScalarType() != MVT::f32 && in BuildDivEstimate()
24389 VT.getScalarType() != MVT::f64) in BuildDivEstimate()
24394 int Enabled = TLI.getRecipEstimateDivEnabled(VT, MF); in BuildDivEstimate()
24400 int Iterations = TLI.getDivRefinementSteps(VT, MF); in BuildDivEstimate()
24406 SDValue FPOne = DAG.getConstantFP(1.0, DL, VT); in BuildDivEstimate()
24414 MulEst = DAG.getNode(ISD::FMUL, DL, VT, N, Est, Flags); in BuildDivEstimate()
24418 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Op, MulEst, Flags); in BuildDivEstimate()
24421 NewEst = DAG.getNode(ISD::FSUB, DL, VT, in BuildDivEstimate()
24425 NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags); in BuildDivEstimate()
24428 Est = DAG.getNode(ISD::FADD, DL, VT, MulEst, NewEst, Flags); in BuildDivEstimate()
24433 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, N, Flags); in BuildDivEstimate()
24452 EVT VT = Arg.getValueType(); in buildSqrtNROneConst() local
24454 SDValue ThreeHalves = DAG.getConstantFP(1.5, DL, VT); in buildSqrtNROneConst()
24458 SDValue HalfArg = DAG.getNode(ISD::FMUL, DL, VT, ThreeHalves, Arg, Flags); in buildSqrtNROneConst()
24459 HalfArg = DAG.getNode(ISD::FSUB, DL, VT, HalfArg, Arg, Flags); in buildSqrtNROneConst()
24463 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, Est, Flags); in buildSqrtNROneConst()
24464 NewEst = DAG.getNode(ISD::FMUL, DL, VT, HalfArg, NewEst, Flags); in buildSqrtNROneConst()
24465 NewEst = DAG.getNode(ISD::FSUB, DL, VT, ThreeHalves, NewEst, Flags); in buildSqrtNROneConst()
24466 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags); in buildSqrtNROneConst()
24471 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, Arg, Flags); in buildSqrtNROneConst()
24484 EVT VT = Arg.getValueType(); in buildSqrtNRTwoConst() local
24486 SDValue MinusThree = DAG.getConstantFP(-3.0, DL, VT); in buildSqrtNRTwoConst()
24487 SDValue MinusHalf = DAG.getConstantFP(-0.5, DL, VT); in buildSqrtNRTwoConst()
24496 SDValue AE = DAG.getNode(ISD::FMUL, DL, VT, Arg, Est, Flags); in buildSqrtNRTwoConst()
24497 SDValue AEE = DAG.getNode(ISD::FMUL, DL, VT, AE, Est, Flags); in buildSqrtNRTwoConst()
24498 SDValue RHS = DAG.getNode(ISD::FADD, DL, VT, AEE, MinusThree, Flags); in buildSqrtNRTwoConst()
24506 LHS = DAG.getNode(ISD::FMUL, DL, VT, Est, MinusHalf, Flags); in buildSqrtNRTwoConst()
24509 LHS = DAG.getNode(ISD::FMUL, DL, VT, AE, MinusHalf, Flags); in buildSqrtNRTwoConst()
24512 Est = DAG.getNode(ISD::FMUL, DL, VT, LHS, RHS, Flags); in buildSqrtNRTwoConst()
24527 EVT VT = Op.getValueType(); in buildSqrtEstimateImpl() local
24528 if (VT.getScalarType() != MVT::f16 && VT.getScalarType() != MVT::f32 && in buildSqrtEstimateImpl()
24529 VT.getScalarType() != MVT::f64) in buildSqrtEstimateImpl()
24534 int Enabled = TLI.getRecipEstimateSqrtEnabled(VT, MF); in buildSqrtEstimateImpl()
24540 int Iterations = TLI.getSqrtRefinementSteps(VT, MF); in buildSqrtEstimateImpl()
24555 SDValue Test = TLI.getSqrtInputTest(Op, DAG, DAG.getDenormalMode(VT)); in buildSqrtEstimateImpl()
24561 Test.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT, DL, VT, in buildSqrtEstimateImpl()