Lines Matching refs:VT
249 for (MVT VT : MVT::all_valuetypes()) in DAGCombiner() local
250 if (EVT(VT).isSimple() && VT != MVT::Other && in DAGCombiner()
251 TLI.isTypeLegal(EVT(VT)) && in DAGCombiner()
252 VT.getSizeInBits().getKnownMinSize() >= MaximumLegalStoreInBits) in DAGCombiner()
253 MaximumLegalStoreInBits = VT.getSizeInBits().getKnownMinSize(); in DAGCombiner()
556 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
566 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
766 bool hasOperation(unsigned Opcode, EVT VT) { in hasOperation() argument
767 return TLI.isOperationLegalOrCustom(Opcode, VT, LegalOperations); in hasOperation()
785 bool isTypeLegal(const EVT &VT) { in isTypeLegal() argument
787 return TLI.isTypeLegal(VT); in isTypeLegal()
791 EVT getSetCCResultType(EVT VT) const { in getSetCCResultType()
792 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in getSetCCResultType()
1040 EVT VT = LoadStore->getMemoryVT(); in reassociationCanBreakAddressingModePattern() local
1042 Type *AccessTy = VT.getTypeForEVT(*DAG.getContext()); in reassociationCanBreakAddressingModePattern()
1060 EVT VT = N0.getValueType(); in reassociateOpsCommutative() local
1069 DAG.FoldConstantArithmetic(Opc, DL, VT, {N0.getOperand(1), N1})) in reassociateOpsCommutative()
1070 return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode); in reassociateOpsCommutative()
1076 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0.getOperand(0), N1); in reassociateOpsCommutative()
1079 return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1)); in reassociateOpsCommutative()
1197 EVT VT = Load->getValueType(0); in ReplaceLoadWithPromotedLoad() local
1198 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0)); in ReplaceLoadWithPromotedLoad()
1284 EVT VT = Op.getValueType(); in PromoteIntBinOp() local
1285 if (VT.isVector() || !VT.isInteger()) in PromoteIntBinOp()
1291 if (TLI.isTypeDesirableForOp(Opc, VT)) in PromoteIntBinOp()
1294 EVT PVT = VT; in PromoteIntBinOp()
1298 assert(PVT != VT && "Don't know what type to promote to!"); in PromoteIntBinOp()
1312 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1)); in PromoteIntBinOp()
1352 EVT VT = Op.getValueType(); in PromoteIntShiftOp() local
1353 if (VT.isVector() || !VT.isInteger()) in PromoteIntShiftOp()
1359 if (TLI.isTypeDesirableForOp(Opc, VT)) in PromoteIntShiftOp()
1362 EVT PVT = VT; in PromoteIntShiftOp()
1366 assert(PVT != VT && "Don't know what type to promote to!"); in PromoteIntShiftOp()
1385 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1)); in PromoteIntShiftOp()
1401 EVT VT = Op.getValueType(); in PromoteExtend() local
1402 if (VT.isVector() || !VT.isInteger()) in PromoteExtend()
1408 if (TLI.isTypeDesirableForOp(Opc, VT)) in PromoteExtend()
1411 EVT PVT = VT; in PromoteExtend()
1415 assert(PVT != VT && "Don't know what type to promote to!"); in PromoteExtend()
1420 return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0)); in PromoteExtend()
1432 EVT VT = Op.getValueType(); in PromoteLoad() local
1433 if (VT.isVector() || !VT.isInteger()) in PromoteLoad()
1439 if (TLI.isTypeDesirableForOp(Opc, VT)) in PromoteLoad()
1442 EVT PVT = VT; in PromoteLoad()
1446 assert(PVT != VT && "Don't know what type to promote to!"); in PromoteLoad()
1457 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD); in PromoteLoad()
2049 EVT VT; in canFoldInAddressingMode() local
2055 VT = LD->getMemoryVT(); in canFoldInAddressingMode()
2060 VT = ST->getMemoryVT(); in canFoldInAddressingMode()
2065 VT = LD->getMemoryVT(); in canFoldInAddressingMode()
2070 VT = ST->getMemoryVT(); in canFoldInAddressingMode()
2098 VT.getTypeForEVT(*DAG.getContext()), AS); in canFoldInAddressingMode()
2145 EVT VT = BO->getValueType(0); in foldBinOpIntoSelect() local
2151 SDValue NewCT = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CT) in foldBinOpIntoSelect()
2152 : DAG.getNode(BinOpcode, DL, VT, CT, CBO); in foldBinOpIntoSelect()
2158 SDValue NewCF = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CF) in foldBinOpIntoSelect()
2159 : DAG.getNode(BinOpcode, DL, VT, CF, CBO); in foldBinOpIntoSelect()
2165 SDValue SelectOp = DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF); in foldBinOpIntoSelect()
2201 EVT VT = C.getValueType(); in foldAddSubBoolOfMaskedVal() local
2203 SDValue LowBit = DAG.getZExtOrTrunc(SetCC.getOperand(0), DL, VT); in foldAddSubBoolOfMaskedVal()
2204 SDValue C1 = IsAdd ? DAG.getConstant(CN->getAPIntValue() + 1, DL, VT) : in foldAddSubBoolOfMaskedVal()
2205 DAG.getConstant(CN->getAPIntValue() - 1, DL, VT); in foldAddSubBoolOfMaskedVal()
2206 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit); in foldAddSubBoolOfMaskedVal()
2230 EVT VT = ShiftOp.getValueType(); in foldAddSubOfSignBit() local
2233 if (!ShAmtC || ShAmtC->getAPIntValue() != (VT.getScalarSizeInBits() - 1)) in foldAddSubOfSignBit()
2241 SDValue NewShift = DAG.getNode(ShOpcode, DL, VT, Not.getOperand(0), ShAmt); in foldAddSubOfSignBit()
2243 DAG.FoldConstantArithmetic(IsAdd ? ISD::ADD : ISD::SUB, DL, VT, in foldAddSubOfSignBit()
2244 {ConstantOp, DAG.getConstant(1, DL, VT)})) in foldAddSubOfSignBit()
2245 return DAG.getNode(ISD::ADD, DL, VT, NewShift, NewC); in foldAddSubOfSignBit()
2255 EVT VT = N0.getValueType(); in visitADDLike() local
2259 if (VT.isVector()) { in visitADDLike()
2280 return DAG.getNode(ISD::ADD, DL, VT, N1, N0); in visitADDLike()
2282 return DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N0, N1}); in visitADDLike()
2294 DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N1, N0.getOperand(1)}); in visitADDLike()
2296 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Sub); in visitADDLike()
2303 DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N1, N0.getOperand(0)}); in visitADDLike()
2305 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1)); in visitADDLike()
2317 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) && in visitADDLike()
2320 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not); in visitADDLike()
2329 if (SDValue Add0 = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, in visitADDLike()
2331 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add0); in visitADDLike()
2349 return DAG.getNode(ISD::ADD, DL, VT, in visitADDLike()
2350 DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)), in visitADDLike()
2362 return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1)); in visitADDLike()
2366 return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1)); in visitADDLike()
2379 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), in visitADDLike()
2385 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), in visitADDLike()
2391 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), in visitADDLike()
2397 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), in visitADDLike()
2404 return DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0).getOperand(0), in visitADDLike()
2415 return DAG.getNode(ISD::SUB, DL, VT, in visitADDLike()
2416 DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10), in visitADDLike()
2417 DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11)); in visitADDLike()
2421 if (N0.getOpcode() == ISD::UMAX && hasOperation(ISD::USUBSAT, VT)) { in visitADDLike()
2428 return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0), in visitADDLike()
2438 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in visitADDLike()
2454 return DAG.getNode(ISD::SUB, DL, VT, A, Xor.getOperand(0)); in visitADDLike()
2461 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.hasOneUse() && in visitADDLike()
2463 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0), in visitADDLike()
2464 DAG.getAllOnesConstant(DL, VT)); in visitADDLike()
2465 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(1), Not); in visitADDLike()
2472 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), N1); in visitADDLike()
2473 return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0)); in visitADDLike()
2488 EVT VT = N0.getValueType(); in visitADD() local
2501 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) && in visitADD()
2503 return DAG.getNode(ISD::OR, DL, VT, N0, N1); in visitADD()
2509 return DAG.getVScale(DL, VT, C0 + C1); in visitADD()
2518 SDValue VS = DAG.getVScale(DL, VT, VS0 + VS1); in visitADD()
2519 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), VS); in visitADD()
2528 return DAG.getStepVector(DL, VT, NewStep); in visitADD()
2538 SDValue SV = DAG.getStepVector(DL, VT, NewStep); in visitADD()
2539 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), SV); in visitADD()
2549 EVT VT = N0.getValueType(); in visitADDSAT() local
2553 if (VT.isVector()) { in visitADDSAT()
2565 return DAG.getAllOnesConstant(DL, VT); in visitADDSAT()
2570 return DAG.getNode(Opcode, DL, VT, N1, N0); in visitADDSAT()
2572 return DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}); in visitADDSAT()
2582 return DAG.getNode(ISD::ADD, DL, VT, N0, N1); in visitADDSAT()
2614 EVT VT = V.getNode()->getValueType(0); in getAsCarry() local
2615 if (!TLI.isOperationLegalOrCustom(V.getOpcode(), VT)) in getAsCarry()
2637 EVT VT = N0.getValueType(); in foldAddSubMasked1() local
2638 if (DAG.ComputeNumSignBits(N1.getOperand(0)) != VT.getScalarSizeInBits()) in foldAddSubMasked1()
2643 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, N0, N1.getOperand(0)); in foldAddSubMasked1()
2649 EVT VT = N0.getValueType(); in visitADDLikeCommutative() local
2655 return DAG.getNode(ISD::SUB, DL, VT, N0, in visitADDLikeCommutative()
2656 DAG.getNode(ISD::SHL, DL, VT, in visitADDLikeCommutative()
2667 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.hasOneUse() && in visitADDLikeCommutative()
2669 SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0), in visitADDLikeCommutative()
2670 DAG.getAllOnesConstant(DL, VT)); in visitADDLikeCommutative()
2671 return DAG.getNode(ISD::SUB, DL, VT, N1, Not); in visitADDLikeCommutative()
2679 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), N1); in visitADDLikeCommutative()
2680 return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1)); in visitADDLikeCommutative()
2686 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1)); in visitADDLikeCommutative()
2687 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(0)); in visitADDLikeCommutative()
2695 TLI.getBooleanContents(VT) == TargetLowering::ZeroOrOneBooleanContent) { in visitADDLikeCommutative()
2696 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitADDLikeCommutative()
2697 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt); in visitADDLikeCommutative()
2704 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0), in visitADDLikeCommutative()
2705 DAG.getConstant(1, DL, VT)); in visitADDLikeCommutative()
2706 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt); in visitADDLikeCommutative()
2717 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT)) in visitADDLikeCommutative()
2720 DAG.getVTList(VT, Carry.getValueType()), N0, in visitADDLikeCommutative()
2721 DAG.getConstant(0, DL, VT), Carry); in visitADDLikeCommutative()
2729 EVT VT = N0.getValueType(); in visitADDC() local
2734 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDC()
2750 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDC()
2776 EVT VT = V.getValueType(); in extractBooleanFlip() local
2779 switch(TLI.getBooleanContents(VT)) { in extractBooleanFlip()
2801 EVT VT = N0.getValueType(); in visitADDO() local
2809 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDO()
2824 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDO()
2830 DAG.getConstant(0, DL, VT), N0.getOperand(0)); in visitADDO()
2846 EVT VT = N0.getValueType(); in visitUADDOLike() local
2847 if (VT.isVector()) in visitUADDOLike()
2861 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT)) in visitUADDOLike()
2864 DAG.getConstant(0, SDLoc(N), VT), Carry); in visitUADDOLike()
2909 EVT VT = N0.getValueType(); in visitADDCARRY() local
2911 SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT); in visitADDCARRY()
2913 return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt, in visitADDCARRY()
2914 DAG.getConstant(1, DL, VT)), in visitADDCARRY()
2990 EVT VT = Combiner.getSetCCResultType(Carry0.getValueType()); in combineADDCARRYDiamond() local
2991 Z = DAG.getConstant(1, SDLoc(Carry0.getOperand(1)), VT); in combineADDCARRYDiamond()
3243 static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT, in tryFoldToZero() argument
3245 if (!VT.isVector()) in tryFoldToZero()
3246 return DAG.getConstant(0, DL, VT); in tryFoldToZero()
3247 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) in tryFoldToZero()
3248 return DAG.getConstant(0, DL, VT); in tryFoldToZero()
3255 EVT VT = N0.getValueType(); in visitSUB() local
3259 if (VT.isVector()) { in visitSUB()
3271 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations); in visitSUB()
3274 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N1})) in visitSUB()
3284 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
3285 DAG.getConstant(-N1C->getAPIntValue(), DL, VT)); in visitSUB()
3289 unsigned BitWidth = VT.getScalarSizeInBits(); in visitSUB()
3298 if (!LegalOperations || TLI.isOperationLegal(NewSh, VT)) in visitSUB()
3299 return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1)); in visitSUB()
3320 !TLI.isOperationLegalOrCustom(ISD::ABS, VT) && in visitSUB()
3325 if (VT.isVector()) { in visitSUB()
3329 if (VT.isScalableVector()) in visitSUB()
3330 return DAG.getSplatVector(VT, DL, N1S.getOperand(1)); in visitSUB()
3331 return DAG.getSplatBuildVector(VT, DL, N1S.getOperand(1)); in visitSUB()
3338 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitSUB()
3342 return DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(1)); in visitSUB()
3361 DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0.getOperand(1), N1}); in visitSUB()
3363 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), NewC); in visitSUB()
3371 SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N11}); in visitSUB()
3373 return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0)); in visitSUB()
3382 DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N0.getOperand(1), N1}); in visitSUB()
3384 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), NewC); in visitSUB()
3392 DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0.getOperand(0), N1}); in visitSUB()
3394 return DAG.getNode(ISD::SUB, DL, VT, NewC, N0.getOperand(1)); in visitSUB()
3402 return DAG.getNode(N0.getOperand(1).getOpcode(), DL, VT, N0.getOperand(0), in visitSUB()
3408 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), in visitSUB()
3414 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), in visitSUB()
3419 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
3420 DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(1), in visitSUB()
3432 DAG.getNode(ISD::XOR, DL, VT, B, DAG.getAllOnesConstant(DL, VT)); in visitSUB()
3433 return DAG.getNode(ISD::AND, DL, VT, A, InvB); in visitSUB()
3441 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, in visitSUB()
3444 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul); in visitSUB()
3448 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, in visitSUB()
3451 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul); in visitSUB()
3470 if (SDValue V = foldSubToUSubSat(VT, N)) in visitSUB()
3475 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), in visitSUB()
3476 DAG.getAllOnesConstant(DL, VT)); in visitSUB()
3477 return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0)); in visitSUB()
3484 if (TLI.preferIncOfAddToSubOfNot(VT) && N1.hasOneUse() && isBitwiseNot(N1)) { in visitSUB()
3485 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(0)); in visitSUB()
3486 return DAG.getNode(ISD::ADD, DL, VT, Add, DAG.getConstant(1, DL, VT)); in visitSUB()
3493 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1); in visitSUB()
3494 return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(1)); in visitSUB()
3499 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(0)); in visitSUB()
3500 return DAG.getNode(ISD::SUB, DL, VT, Sub, N1.getOperand(1)); in visitSUB()
3506 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1); in visitSUB()
3507 return DAG.getNode(ISD::SUB, DL, VT, Sub, N0.getOperand(1)); in visitSUB()
3512 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), N1); in visitSUB()
3513 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), Add); in visitSUB()
3521 TLI.getBooleanContents(VT) == in visitSUB()
3523 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N1.getOperand(0)); in visitSUB()
3524 return DAG.getNode(ISD::ADD, DL, VT, N0, SExt); in visitSUB()
3528 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) { in visitSUB()
3534 if (C->getAPIntValue() == (VT.getScalarSizeInBits() - 1)) in visitSUB()
3535 return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0); in visitSUB()
3544 return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT, in visitSUB()
3551 DL, VT); in visitSUB()
3558 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0), in visitSUB()
3559 DAG.getConstant(1, DL, VT)); in visitSUB()
3560 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt); in visitSUB()
3567 return DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getVScale(DL, VT, -IntVal)); in visitSUB()
3573 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
3574 DAG.getStepVector(DL, VT, NewStep)); in visitSUB()
3584 SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt); in visitSUB()
3585 return DAG.getNode(ISD::ADD, DL, VT, N0, SRA); in visitSUB()
3589 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT)) { in visitSUB()
3593 SDValue Zero = DAG.getConstant(0, DL, VT); in visitSUB()
3594 SDValue NegX = DAG.getNode(ISD::SUB, DL, VT, Zero, X); in visitSUB()
3596 DAG.getVTList(VT, Carry.getValueType()), NegX, Zero, in visitSUB()
3607 EVT VT = N0.getValueType(); in visitSUBSAT() local
3611 if (VT.isVector()) { in visitSUBSAT()
3621 return DAG.getConstant(0, DL, VT); in visitSUBSAT()
3625 return DAG.getConstant(0, DL, VT); in visitSUBSAT()
3628 if (SDValue C = DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, {N0, N1})) in visitSUBSAT()
3641 EVT VT = N0.getValueType(); in visitSUBC() local
3646 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitSUBC()
3651 return CombineTo(N, DAG.getConstant(0, DL, VT), in visitSUBC()
3660 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0), in visitSUBC()
3669 EVT VT = N0.getValueType(); in visitSUBO() local
3677 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitSUBO()
3682 return CombineTo(N, DAG.getConstant(0, DL, VT), in visitSUBO()
3690 DAG.getConstant(-N1C->getAPIntValue(), DL, VT)); in visitSUBO()
3699 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0), in visitSUBO()
3753 EVT VT = N0.getValueType(); in visitMULFIX() local
3757 return DAG.getConstant(0, SDLoc(N), VT); in visitMULFIX()
3762 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0, Scale); in visitMULFIX()
3766 return DAG.getConstant(0, SDLoc(N), VT); in visitMULFIX()
3774 EVT VT = N0.getValueType(); in visitMUL() local
3778 return DAG.getConstant(0, SDLoc(N), VT); in visitMUL()
3785 if (VT.isVector()) { in visitMUL()
3791 ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && in visitMUL()
3802 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT, {N0, N1})) in visitMUL()
3808 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0); in visitMUL()
3824 return DAG.getNode(ISD::SUB, DL, VT, in visitMUL()
3825 DAG.getConstant(0, DL, VT), N0); in visitMUL()
3831 (!VT.isVector() || Level <= AfterLegalizeVectorOps)) { in visitMUL()
3836 return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc); in visitMUL()
3845 return DAG.getNode(ISD::SUB, DL, VT, in visitMUL()
3846 DAG.getConstant(0, DL, VT), in visitMUL()
3847 DAG.getNode(ISD::SHL, DL, VT, N0, in visitMUL()
3867 if (N1IsConst && TLI.decomposeMulByConstant(*DAG.getContext(), VT, N1)) { in visitMUL()
3885 assert(ShAmt < VT.getScalarSizeInBits() && in visitMUL()
3889 DAG.getNode(ISD::SHL, DL, VT, N0, DAG.getConstant(ShAmt, DL, VT)); in visitMUL()
3891 TZeros ? DAG.getNode(MathOp, DL, VT, Shl, in visitMUL()
3892 DAG.getNode(ISD::SHL, DL, VT, N0, in visitMUL()
3893 DAG.getConstant(TZeros, DL, VT))) in visitMUL()
3894 : DAG.getNode(MathOp, DL, VT, Shl, N0); in visitMUL()
3896 R = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), R); in visitMUL()
3905 SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT, N1, N0.getOperand(1)); in visitMUL()
3907 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), C3); in visitMUL()
3927 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, Sh.getOperand(0), Y); in visitMUL()
3928 return DAG.getNode(ISD::SHL, SDLoc(N), VT, Mul, Sh.getOperand(1)); in visitMUL()
3937 return DAG.getNode(ISD::ADD, SDLoc(N), VT, in visitMUL()
3938 DAG.getNode(ISD::MUL, SDLoc(N0), VT, in visitMUL()
3940 DAG.getNode(ISD::MUL, SDLoc(N1), VT, in visitMUL()
3948 return DAG.getVScale(SDLoc(N), VT, C0 * C1); in visitMUL()
3957 return DAG.getStepVector(SDLoc(N), VT, NewStep); in visitMUL()
3964 if (VT.isFixedLengthVector()) { in visitMUL()
3965 unsigned NumElts = VT.getVectorNumElements(); in visitMUL()
3976 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::AND, VT)) && in visitMUL()
3987 return DAG.getNode(ISD::AND, DL, VT, N0, DAG.getBuildVector(VT, DL, Mask)); in visitMUL()
4027 EVT VT = Node->getValueType(0); in useDivRem() local
4028 if (VT.isVector() || !VT.isInteger()) in useDivRem()
4031 if (!TLI.isTypeLegal(VT) && !TLI.isOperationCustom(DivRemOpc, VT)) in useDivRem()
4036 if (!TLI.isOperationLegalOrCustom(DivRemOpc, VT) && in useDivRem()
4044 if (TLI.isOperationLegalOrCustom(Opcode, VT)) in useDivRem()
4048 if (TLI.isOperationLegalOrCustom(OtherOpcode, VT)) in useDivRem()
4070 SDVTList VTs = DAG.getVTList(VT, VT); in useDivRem()
4091 EVT VT = N->getValueType(0); in simplifyDivRem() local
4104 return DAG.getUNDEF(VT); in simplifyDivRem()
4109 return DAG.getConstant(0, DL, VT); in simplifyDivRem()
4120 return DAG.getConstant(IsDiv ? 1 : 0, DL, VT); in simplifyDivRem()
4128 if ((N1C && N1C->isOne()) || (VT.getScalarType() == MVT::i1)) in simplifyDivRem()
4129 return IsDiv ? N0 : DAG.getConstant(0, DL, VT); in simplifyDivRem()
4137 EVT VT = N->getValueType(0); in visitSDIV() local
4138 EVT CCVT = getSetCCResultType(VT); in visitSDIV()
4141 if (VT.isVector()) in visitSDIV()
4149 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, {N0, N1})) in visitSDIV()
4154 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0); in visitSDIV()
4158 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ), in visitSDIV()
4159 DAG.getConstant(1, DL, VT), in visitSDIV()
4160 DAG.getConstant(0, DL, VT)); in visitSDIV()
4178 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1); in visitSDIV()
4179 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitSDIV()
4200 EVT VT = N->getValueType(0); in visitSDIVLike() local
4201 EVT CCVT = getSetCCResultType(VT); in visitSDIVLike()
4202 unsigned BitWidth = VT.getScalarSizeInBits(); in visitSDIVLike()
4228 SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1); in visitSDIVLike()
4235 SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0, in visitSDIVLike()
4240 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact); in visitSDIVLike()
4242 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl); in visitSDIVLike()
4244 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1); in visitSDIVLike()
4249 SDValue One = DAG.getConstant(1, DL, VT); in visitSDIVLike()
4250 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT); in visitSDIVLike()
4254 Sra = DAG.getSelect(DL, VT, IsOneOrAllOnes, N0, Sra); in visitSDIVLike()
4258 SDValue Zero = DAG.getConstant(0, DL, VT); in visitSDIVLike()
4259 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, Zero, Sra); in visitSDIVLike()
4263 SDValue Res = DAG.getSelect(DL, VT, IsNeg, Sub, Sra); in visitSDIVLike()
4282 EVT VT = N->getValueType(0); in visitUDIV() local
4283 EVT CCVT = getSetCCResultType(VT); in visitUDIV()
4286 if (VT.isVector()) in visitUDIV()
4294 if (SDValue C = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT, {N0, N1})) in visitUDIV()
4299 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ), in visitUDIV()
4300 DAG.getConstant(1, DL, VT), in visitUDIV()
4301 DAG.getConstant(0, DL, VT)); in visitUDIV()
4314 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1); in visitUDIV()
4315 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitUDIV()
4336 EVT VT = N->getValueType(0); in visitUDIVLike() local
4347 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc); in visitUDIVLike()
4363 return DAG.getNode(ISD::SRL, DL, VT, N0, Add); in visitUDIVLike()
4382 EVT VT = N->getValueType(0); in visitREM() local
4383 EVT CCVT = getSetCCResultType(VT); in visitREM()
4390 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1})) in visitREM()
4395 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ), in visitREM()
4396 DAG.getConstant(0, DL, VT), N0); in visitREM()
4408 return DAG.getNode(ISD::UREM, DL, VT, N0, N1); in visitREM()
4412 SDValue NegOne = DAG.getAllOnesConstant(DL, VT); in visitREM()
4413 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne); in visitREM()
4415 return DAG.getNode(ISD::AND, DL, VT, N0, Add); in visitREM()
4420 SDValue NegOne = DAG.getAllOnesConstant(DL, VT); in visitREM()
4421 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne); in visitREM()
4423 return DAG.getNode(ISD::AND, DL, VT, N0, Add); in visitREM()
4436 if (DAG.isKnownNeverZero(N1) && !TLI.isIntDivCheap(VT, Attr)) { in visitREM()
4445 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1); in visitREM()
4446 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitREM()
4463 EVT VT = N->getValueType(0); in visitMULHS() local
4466 if (VT.isVector()) { in visitMULHS()
4471 return DAG.getConstant(0, DL, VT); in visitMULHS()
4475 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHS, DL, VT, {N0, N1})) in visitMULHS()
4489 return DAG.getConstant(0, DL, VT); in visitMULHS()
4493 if (!TLI.isOperationLegalOrCustom(ISD::MULHS, VT) && VT.isSimple() && in visitMULHS()
4494 !VT.isVector()) { in visitMULHS()
4495 MVT Simple = VT.getSimpleVT(); in visitMULHS()
4505 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in visitMULHS()
4515 EVT VT = N->getValueType(0); in visitMULHU() local
4518 if (VT.isVector()) { in visitMULHU()
4523 return DAG.getConstant(0, DL, VT); in visitMULHU()
4527 if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHU, DL, VT, {N0, N1})) in visitMULHU()
4538 return DAG.getConstant(0, DL, VT); in visitMULHU()
4542 DAG.isKnownToBeAPowerOfTwo(N1) && hasOperation(ISD::SRL, VT)) { in visitMULHU()
4543 unsigned NumEltBits = VT.getScalarSizeInBits(); in visitMULHU()
4546 ISD::SUB, DL, VT, DAG.getConstant(NumEltBits, DL, VT), LogBase2); in visitMULHU()
4549 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc); in visitMULHU()
4554 if (!TLI.isOperationLegalOrCustom(ISD::MULHU, VT) && VT.isSimple() && in visitMULHU()
4555 !VT.isVector()) { in visitMULHU()
4556 MVT Simple = VT.getSimpleVT(); in visitMULHU()
4566 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in visitMULHU()
4626 EVT VT = N->getValueType(0); in visitSMUL_LOHI() local
4631 if (VT.isSimple() && !VT.isVector()) { in visitSMUL_LOHI()
4632 MVT Simple = VT.getSimpleVT(); in visitSMUL_LOHI()
4643 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi); in visitSMUL_LOHI()
4645 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo); in visitSMUL_LOHI()
4657 EVT VT = N->getValueType(0); in visitUMUL_LOHI() local
4662 SDValue Zero = DAG.getConstant(0, DL, VT); in visitUMUL_LOHI()
4668 SDValue Zero = DAG.getConstant(0, DL, VT); in visitUMUL_LOHI()
4674 if (VT.isSimple() && !VT.isVector()) { in visitUMUL_LOHI()
4675 MVT Simple = VT.getSimpleVT(); in visitUMUL_LOHI()
4686 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi); in visitUMUL_LOHI()
4688 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo); in visitUMUL_LOHI()
4699 EVT VT = N0.getValueType(); in visitMULO() local
4716 return CombineTo(N, DAG.getConstant(Result, DL, VT), in visitMULO()
4727 return CombineTo(N, DAG.getConstant(0, DL, VT), in visitMULO()
4737 if (VT.getScalarSizeInBits() == 1) { in visitMULO()
4738 SDValue And = DAG.getNode(ISD::AND, DL, VT, N0, N1); in visitMULO()
4741 DAG.getConstant(0, DL, VT), ISD::SETNE)); in visitMULO()
4750 if (SignBits > VT.getScalarSizeInBits() + 1) in visitMULO()
4751 return CombineTo(N, DAG.getNode(ISD::MUL, DL, VT, N0, N1), in visitMULO()
4759 return CombineTo(N, DAG.getNode(ISD::MUL, DL, VT, N0, N1), in visitMULO()
4769 EVT VT = N0.getValueType(); in visitIMINMAX() local
4773 if (VT.isVector()) in visitIMINMAX()
4778 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, SDLoc(N), VT, {N0, N1})) in visitIMINMAX()
4784 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0); in visitIMINMAX()
4788 if (!TLI.isOperationLegal(Opcode, VT) && in visitIMINMAX()
4799 if (TLI.isOperationLegal(AltOpcode, VT)) in visitIMINMAX()
4800 return DAG.getNode(AltOpcode, SDLoc(N), VT, N0, N1); in visitIMINMAX()
4814 EVT VT = N0.getValueType(); in hoistLogicOpWithSameOpcodeHands() local
4844 if ((VT.isVector() || LegalOperations) && in hoistLogicOpWithSameOpcodeHands()
4854 return DAG.getNode(HandOpcode, DL, VT, Logic); in hoistLogicOpWithSameOpcodeHands()
4871 if (TLI.isZExtFree(VT, XVT) && TLI.isTruncateFree(XVT, VT)) in hoistLogicOpWithSameOpcodeHands()
4876 return DAG.getNode(HandOpcode, DL, VT, Logic); in hoistLogicOpWithSameOpcodeHands()
4888 return DAG.getNode(HandOpcode, DL, VT, Logic, N0.getOperand(1)); in hoistLogicOpWithSameOpcodeHands()
4897 return DAG.getNode(HandOpcode, DL, VT, Logic); in hoistLogicOpWithSameOpcodeHands()
4911 !(VT.isVector() && TLI.isTypeLegal(VT) && in hoistLogicOpWithSameOpcodeHands()
4914 return DAG.getNode(HandOpcode, DL, VT, Logic); in hoistLogicOpWithSameOpcodeHands()
4948 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations); in hoistLogicOpWithSameOpcodeHands()
4952 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT, in hoistLogicOpWithSameOpcodeHands()
4954 return DAG.getVectorShuffle(VT, DL, Logic, ShOp, SVN0->getMask()); in hoistLogicOpWithSameOpcodeHands()
4961 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations); in hoistLogicOpWithSameOpcodeHands()
4965 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT, N0.getOperand(1), in hoistLogicOpWithSameOpcodeHands()
4967 return DAG.getVectorShuffle(VT, DL, ShOp, Logic, SVN0->getMask()); in hoistLogicOpWithSameOpcodeHands()
4991 EVT VT = N0.getValueType(); in foldLogicOfSetCCs() local
4993 if (LegalOperations || VT.getScalarType() != MVT::i1) in foldLogicOfSetCCs()
4994 if (VT != getSetCCResultType(OpVT)) in foldLogicOfSetCCs()
5022 return DAG.getSetCC(DL, VT, Or, LR, CC1); in foldLogicOfSetCCs()
5041 return DAG.getSetCC(DL, VT, And, LR, CC1); in foldLogicOfSetCCs()
5055 return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE); in foldLogicOfSetCCs()
5069 return DAG.getSetCC(DL, VT, Or, Zero, CC1); in foldLogicOfSetCCs()
5094 return DAG.getSetCC(DL, VT, And, Zero, CC0); in foldLogicOfSetCCs()
5115 return DAG.getSetCC(DL, VT, LL, LR, NewCC); in foldLogicOfSetCCs()
5126 EVT VT = N1.getValueType(); in visitANDLike() local
5131 return DAG.getConstant(0, DL, VT); in visitANDLike()
5138 VT.getSizeInBits() <= 64 && N0->hasOneUse()) { in visitANDLike()
5148 SRLC.ult(VT.getSizeInBits()) && in visitANDLike()
5150 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(), in visitANDLike()
5157 DAG.getNode(ISD::ADD, DL0, VT, in visitANDLike()
5158 N0.getOperand(0), DAG.getConstant(ADDC, DL, VT)); in visitANDLike()
5175 unsigned Size = VT.getSizeInBits(); in visitANDLike()
5190 TLI.isNarrowingProfitable(VT, HalfVT) && in visitANDLike()
5193 TLI.isTruncateFree(VT, HalfVT) && in visitANDLike()
5194 TLI.isZExtFree(HalfVT, VT)) { in visitANDLike()
5212 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, And); in visitANDLike()
5392 EVT VT = Op.getOpcode() == ISD::AssertZext ? in SearchForAndLoads() local
5398 if (ExtVT.bitsGE(VT)) in SearchForAndLoads()
5420 MVT VT = SDValue(NodeToMask, i).getSimpleValueType(); in SearchForAndLoads() local
5421 if (VT != MVT::Glue && VT != MVT::Other) { in SearchForAndLoads()
5547 EVT VT = N->getValueType(0); in unfoldExtremeBitClearingToShifts() local
5550 SDValue T0 = DAG.getNode(InnerShift, DL, VT, X, Y); in unfoldExtremeBitClearingToShifts()
5552 SDValue T1 = DAG.getNode(OuterShift, DL, VT, T0, Y); in unfoldExtremeBitClearingToShifts()
5564 EVT VT = And->getValueType(0); in combineShiftAnd1ToBitTest() local
5566 if (!TLI.isTypeLegal(VT)) in combineShiftAnd1ToBitTest()
5593 unsigned VTBitWidth = VT.getSizeInBits(); in combineShiftAnd1ToBitTest()
5600 SDValue X = DAG.getZExtOrTrunc(Srl.getOperand(0), DL, VT); in combineShiftAnd1ToBitTest()
5601 EVT CCVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in combineShiftAnd1ToBitTest()
5603 APInt::getOneBitSet(VTBitWidth, ShiftAmt.getZExtValue()), DL, VT); in combineShiftAnd1ToBitTest()
5604 SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, X, Mask); in combineShiftAnd1ToBitTest()
5605 SDValue Zero = DAG.getConstant(0, DL, VT); in combineShiftAnd1ToBitTest()
5607 return DAG.getZExtOrTrunc(Setcc, DL, VT); in combineShiftAnd1ToBitTest()
5613 EVT VT = N1.getValueType(); in visitAND() local
5620 if (VT.isVector()) { in visitAND()
5646 EVT ExtVT = VT; in visitAND()
5668 if (SDValue C = DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, {N0, N1})) in visitAND()
5674 return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0); in visitAND()
5681 unsigned BitWidth = VT.getScalarSizeInBits(); in visitAND()
5684 return DAG.getConstant(0, SDLoc(N), VT); in visitAND()
5694 if (VT.isVector()) in visitAND()
5844 DAG.getVTList(VT, MVT::Other), MemVT, SDLoc(N), Ops, in visitAND()
5857 if (!VT.isVector() && N1C && (N0.getOpcode() == ISD::LOAD || in visitAND()
5901 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0)); in visitAND()
5924 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) { in visitAND()
5926 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, LN0->getChain(), in visitAND()
5971 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, N0.getOperand(0)); in visitAND()
5982 EVT VT = N->getValueType(0); in MatchBSwapHWordLow() local
5983 if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16) in MatchBSwapHWordLow()
5985 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT)) in MatchBSwapHWordLow()
6063 unsigned OpSizeInBits = VT.getSizeInBits(); in MatchBSwapHWordLow()
6079 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00); in MatchBSwapHWordLow()
6082 Res = DAG.getNode(ISD::SRL, DL, VT, Res, in MatchBSwapHWordLow()
6084 getShiftAmountTy(VT))); in MatchBSwapHWordLow()
6202 SDValue N1, EVT VT, EVT ShiftAmountTy) { in matchBSwapHWordOrAndAnd() argument
6203 assert(N->getOpcode() == ISD::OR && VT == MVT::i32 && in matchBSwapHWordOrAndAnd()
6205 if (!TLI.isOperationLegalOrCustom(ISD::ROTR, VT)) in matchBSwapHWordOrAndAnd()
6233 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Shift0.getOperand(0)); in matchBSwapHWordOrAndAnd()
6235 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt); in matchBSwapHWordOrAndAnd()
6248 EVT VT = N->getValueType(0); in MatchBSwapHWord() local
6249 if (VT != MVT::i32) in MatchBSwapHWord()
6251 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT)) in MatchBSwapHWord()
6254 if (SDValue BSwap = matchBSwapHWordOrAndAnd(TLI, DAG, N, N0, N1, VT, in MatchBSwapHWord()
6255 getShiftAmountTy(VT))) in MatchBSwapHWord()
6259 if (SDValue BSwap = matchBSwapHWordOrAndAnd(TLI, DAG, N, N1, N0, VT, in MatchBSwapHWord()
6260 getShiftAmountTy(VT))) in MatchBSwapHWord()
6291 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, in MatchBSwapHWord()
6296 SDValue ShAmt = DAG.getConstant(16, DL, getShiftAmountTy(VT)); in MatchBSwapHWord()
6297 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT)) in MatchBSwapHWord()
6298 return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt); in MatchBSwapHWord()
6299 if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT)) in MatchBSwapHWord()
6300 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt); in MatchBSwapHWord()
6301 return DAG.getNode(ISD::OR, DL, VT, in MatchBSwapHWord()
6302 DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt), in MatchBSwapHWord()
6303 DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt)); in MatchBSwapHWord()
6309 EVT VT = N1.getValueType(); in visitORLike() local
6314 return DAG.getAllOnesConstant(DL, VT); in visitORLike()
6336 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, in visitORLike()
6338 return DAG.getNode(ISD::AND, DL, VT, X, in visitORLike()
6339 DAG.getConstant(LHSMask | RHSMask, DL, VT)); in visitORLike()
6351 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, in visitORLike()
6353 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X); in visitORLike()
6362 EVT VT = N0.getValueType(); in visitORCommutative() local
6366 return DAG.getNode(ISD::OR, SDLoc(N), VT, N0.getOperand(0), N1); in visitORCommutative()
6370 return DAG.getNode(ISD::OR, SDLoc(N), VT, N0.getOperand(1), N1); in visitORCommutative()
6379 EVT VT = N1.getValueType(); in visitOR() local
6386 if (VT.isVector()) { in visitOR()
6409 TLI.isTypeLegal(VT)) { in visitOR()
6421 int NumElts = VT.getVectorNumElements(); in visitOR()
6459 TLI.buildLegalVectorShuffle(VT, SDLoc(N), NewLHS, NewRHS, in visitOR()
6470 if (SDValue C = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, {N0, N1})) in visitOR()
6476 return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0); in visitOR()
6516 if (SDValue COR = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N1), VT, in visitOR()
6518 SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1); in visitOR()
6520 return DAG.getNode(ISD::AND, SDLoc(N), VT, COR, IOR); in visitOR()
6546 if ((!LegalOperations || TLI.isOperationLegal(ISD::ADD, VT)) && in visitOR()
6843 EVT VT = Shifted.getValueType(); in MatchRotatePosNeg() local
6844 if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits(), DAG, in MatchRotatePosNeg()
6846 bool HasPos = TLI.isOperationLegalOrCustom(PosOpcode, VT); in MatchRotatePosNeg()
6847 return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted, in MatchRotatePosNeg()
6864 EVT VT = N0.getValueType(); in MatchFunnelPosNeg() local
6865 unsigned EltBits = VT.getScalarSizeInBits(); in MatchFunnelPosNeg()
6875 bool HasPos = TLI.isOperationLegalOrCustom(PosOpcode, VT); in MatchFunnelPosNeg()
6876 return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, N0, N1, in MatchFunnelPosNeg()
6896 TLI.isOperationLegalOrCustom(ISD::FSHL, VT)) { in MatchFunnelPosNeg()
6897 return DAG.getNode(ISD::FSHL, DL, VT, N0, N1.getOperand(0), Pos); in MatchFunnelPosNeg()
6905 TLI.isOperationLegalOrCustom(ISD::FSHR, VT)) { in MatchFunnelPosNeg()
6906 return DAG.getNode(ISD::FSHR, DL, VT, N0.getOperand(0), N1, Neg); in MatchFunnelPosNeg()
6915 TLI.isOperationLegalOrCustom(ISD::FSHR, VT)) { in MatchFunnelPosNeg()
6916 return DAG.getNode(ISD::FSHR, DL, VT, N0.getOperand(0), N1, Neg); in MatchFunnelPosNeg()
6929 EVT VT = LHS.getValueType(); in MatchRotate() local
6930 if (!TLI.isTypeLegal(VT)) in MatchRotate()
6934 bool HasROTL = hasOperation(ISD::ROTL, VT); in MatchRotate()
6935 bool HasROTR = hasOperation(ISD::ROTR, VT); in MatchRotate()
6936 bool HasFSHL = hasOperation(ISD::FSHL, VT); in MatchRotate()
6937 bool HasFSHR = hasOperation(ISD::FSHR, VT); in MatchRotate()
7002 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in MatchRotate()
7020 Res = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT, LHSShiftArg, in MatchRotate()
7023 Res = DAG.getNode(HasFSHL ? ISD::FSHL : ISD::FSHR, DL, VT, LHSShiftArg, in MatchRotate()
7028 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT); in MatchRotate()
7032 SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt); in MatchRotate()
7033 Mask = DAG.getNode(ISD::AND, DL, VT, Mask, in MatchRotate()
7034 DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits)); in MatchRotate()
7037 SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt); in MatchRotate()
7038 Mask = DAG.getNode(ISD::AND, DL, VT, Mask, in MatchRotate()
7039 DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits)); in MatchRotate()
7042 Res = DAG.getNode(ISD::AND, DL, VT, Res, Mask); in MatchRotate()
7505 EVT VT = N->getValueType(0); in MatchLoadCombine() local
7506 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64) in MatchLoadCombine()
7508 unsigned ByteWidth = VT.getSizeInBits() / 8; in MatchLoadCombine()
7629 !TLI.isOperationLegal(ISD::BSWAP, VT)) in MatchLoadCombine()
7635 !TLI.isOperationLegal(ISD::SHL, VT)) in MatchLoadCombine()
7647 DAG.getExtLoad(NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, SDLoc(N), VT, in MatchLoadCombine()
7660 ? DAG.getNode(ISD::SHL, SDLoc(N), VT, NewLoad, in MatchLoadCombine()
7661 DAG.getShiftAmountConstant(ZeroExtendedBytes * 8, VT, in MatchLoadCombine()
7664 return DAG.getNode(ISD::BSWAP, SDLoc(N), VT, ShiftedLoad); in MatchLoadCombine()
7687 EVT VT = N->getValueType(0); in unfoldMaskedMerge() local
7735 SDValue NotX = DAG.getNOT(DL, X, VT); in unfoldMaskedMerge()
7736 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, NotX, M); in unfoldMaskedMerge()
7737 SDValue NotLHS = DAG.getNOT(DL, LHS, VT); in unfoldMaskedMerge()
7738 SDValue RHS = DAG.getNode(ISD::OR, DL, VT, M, Y); in unfoldMaskedMerge()
7739 return DAG.getNode(ISD::AND, DL, VT, NotLHS, RHS); in unfoldMaskedMerge()
7742 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, X, M); in unfoldMaskedMerge()
7743 SDValue NotM = DAG.getNOT(DL, M, VT); in unfoldMaskedMerge()
7744 SDValue RHS = DAG.getNode(ISD::AND, DL, VT, Y, NotM); in unfoldMaskedMerge()
7746 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS); in unfoldMaskedMerge()
7752 EVT VT = N0.getValueType(); in visitXOR() local
7755 if (VT.isVector()) { in visitXOR()
7769 return DAG.getConstant(0, DL, VT); in visitXOR()
7778 if (SDValue C = DAG.FoldConstantArithmetic(ISD::XOR, DL, VT, {N0, N1})) in visitXOR()
7784 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitXOR()
7810 return DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC); in visitXOR()
7820 DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC, in visitXOR()
7841 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, V); in visitXOR()
7845 if (isOneConstant(N1) && VT == MVT::i1 && N0.hasOneUse() && in visitXOR()
7850 N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00 in visitXOR()
7851 N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01 in visitXOR()
7853 return DAG.getNode(NewOpcode, DL, VT, N00, N01); in visitXOR()
7862 N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00 in visitXOR()
7863 N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01 in visitXOR()
7865 return DAG.getNode(NewOpcode, DL, VT, N00, N01); in visitXOR()
7874 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), in visitXOR()
7875 DAG.getAllOnesConstant(DL, VT)); in visitXOR()
7881 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in visitXOR()
7888 SDValue NotX = DAG.getNOT(SDLoc(X), X, VT); in visitXOR()
7890 return DAG.getNode(ISD::AND, DL, VT, NotX, N1); in visitXOR()
7896 unsigned BitWidth = VT.getScalarSizeInBits(); in visitXOR()
7908 SDValue Not = DAG.getNOT(DL, N0.getOperand(0), VT); in visitXOR()
7909 return DAG.getNode(N0Opcode, DL, VT, Not, N0.getOperand(1)); in visitXOR()
7916 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) { in visitXOR()
7924 if (C->getAPIntValue() == (VT.getScalarSizeInBits() - 1)) in visitXOR()
7925 return DAG.getNode(ISD::ABS, DL, VT, S0); in visitXOR()
7931 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations); in visitXOR()
7951 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0Opcode == ISD::SHL && in visitXOR()
7953 return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT), in visitXOR()
8034 EVT VT = Shift->getValueType(0); in combineShiftOfShiftedLogic() local
8037 SDValue NewShift1 = DAG.getNode(ShiftOpcode, DL, VT, X, ShiftSumC); in combineShiftOfShiftedLogic()
8038 SDValue NewShift2 = DAG.getNode(ShiftOpcode, DL, VT, Y, C1); in combineShiftOfShiftedLogic()
8039 return DAG.getNode(LogicOpcode, DL, VT, NewShift1, NewShift2); in combineShiftOfShiftedLogic()
8109 EVT VT = N->getValueType(0); in visitShiftByConstant() local
8110 SDValue NewRHS = DAG.getNode(N->getOpcode(), DL, VT, LHS.getOperand(1), in visitShiftByConstant()
8114 SDValue NewShift = DAG.getNode(N->getOpcode(), DL, VT, LHS.getOperand(0), in visitShiftByConstant()
8116 return DAG.getNode(LHS.getOpcode(), DL, VT, NewShift, NewRHS); in visitShiftByConstant()
8146 EVT VT = N->getValueType(0); in visitRotate() local
8147 unsigned Bitsize = VT.getScalarSizeInBits(); in visitRotate()
8171 return DAG.getNode(N->getOpcode(), dl, VT, N0, Amt); in visitRotate()
8177 VT.getScalarSizeInBits() == 16 && hasOperation(ISD::BSWAP, VT)) in visitRotate()
8178 return DAG.getNode(ISD::BSWAP, dl, VT, N0); in visitRotate()
8188 return DAG.getNode(N->getOpcode(), dl, VT, N0, NewOp1); in visitRotate()
8205 return DAG.getNode(N->getOpcode(), dl, VT, N0->getOperand(0), in visitRotate()
8219 EVT VT = N0.getValueType(); in visitSHL() local
8221 unsigned OpSizeInBits = VT.getScalarSizeInBits(); in visitSHL()
8224 if (VT.isVector()) { in visitSHL()
8241 DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, {N01, N1})) in visitSHL()
8242 return DAG.getNode(ISD::AND, SDLoc(N), VT, N00, C); in visitSHL()
8251 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, {N0, N1})) in visitSHL()
8260 return DAG.getConstant(0, SDLoc(N), VT); in visitSHL()
8266 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, NewOp1); in visitSHL()
8282 return DAG.getConstant(0, SDLoc(N), VT); in visitSHL()
8294 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum); in visitSHL()
8323 return DAG.getConstant(0, SDLoc(N), VT); in visitSHL()
8337 SDValue Ext = DAG.getNode(N0.getOpcode(), DL, VT, N0Op0.getOperand(0)); in visitSHL()
8340 return DAG.getNode(ISD::SHL, DL, VT, Ext, Sum); in visitSHL()
8352 auto MatchEqual = [VT](ConstantSDNode *LHS, ConstantSDNode *RHS) { in visitSHL()
8356 return c1.ult(VT.getScalarSizeInBits()) && (c1 == c2); in visitSHL()
8366 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL); in visitSHL()
8380 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), in visitSHL()
8382 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0), in visitSHL()
8404 Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), in visitSHL()
8409 Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), in visitSHL()
8413 return DAG.getNode(ISD::AND, DL, VT, Shift, in visitSHL()
8414 DAG.getConstant(Mask, DL, VT)); in visitSHL()
8423 SDValue AllBits = DAG.getAllOnesConstant(DL, VT); in visitSHL()
8424 SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1); in visitSHL()
8425 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), HiBitsMask); in visitSHL()
8437 SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1); in visitSHL()
8438 SDValue Shl1 = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1); in visitSHL()
8441 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, Shl0, Shl1); in visitSHL()
8448 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1); in visitSHL()
8450 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), Shl); in visitSHL()
8462 return DAG.getVScale(SDLoc(N), VT, C0 << C1); in visitSHL()
8472 return DAG.getStepVector(SDLoc(N), VT, NewStep); in visitSHL()
8555 EVT VT = N0.getValueType(); in visitSRA() local
8556 unsigned OpSizeInBits = VT.getScalarSizeInBits(); in visitSRA()
8565 if (VT.isVector()) in visitSRA()
8572 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, {N0, N1})) in visitSRA()
8583 if (VT.isVector()) in visitSRA()
8585 VT.getVectorElementCount()); in visitSRA()
8589 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, in visitSRA()
8626 return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0), ShiftValue); in visitSRA()
8643 if (VT.isVector()) in visitSRA()
8644 TruncVT = EVT::getVectorVT(Ctx, TruncVT, VT.getVectorElementCount()); in visitSRA()
8655 TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) && in visitSRA()
8656 TLI.isTruncateFree(VT, TruncVT)) { in visitSRA()
8660 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, in visitSRA()
8683 if (VT.isVector()) in visitSRA()
8684 TruncVT = EVT::getVectorVT(Ctx, TruncVT, VT.getVectorElementCount()); in visitSRA()
8691 TLI.isTruncateFree(VT, TruncVT)) { in visitSRA()
8697 return DAG.getSExtOrTrunc(Add, DL, VT); in visitSRA()
8706 return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0, NewOp1); in visitSRA()
8728 return DAG.getNode(ISD::TRUNCATE, DL, VT, SRA); in visitSRA()
8739 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, N1); in visitSRA()
8759 EVT VT = N0.getValueType(); in visitSRL() local
8760 unsigned OpSizeInBits = VT.getScalarSizeInBits(); in visitSRL()
8763 if (VT.isVector()) in visitSRL()
8770 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, {N0, N1})) in visitSRL()
8779 return DAG.getConstant(0, SDLoc(N), VT); in visitSRL()
8791 return DAG.getConstant(0, SDLoc(N), VT); in visitSRL()
8804 return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Sum); in visitSRL()
8823 return DAG.getConstant(0, DL, VT); in visitSRL()
8827 return DAG.getNode(ISD::TRUNCATE, DL, VT, NewShift); in visitSRL()
8841 return DAG.getNode(ISD::TRUNCATE, DL, VT, And); in visitSRL()
8852 DAG.getNode(ISD::SRL, DL, VT, DAG.getAllOnesConstant(DL, VT), N1); in visitSRL()
8854 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), Mask); in visitSRL()
8864 return DAG.getUNDEF(VT); in visitSRL()
8876 return DAG.getNode(ISD::AND, DL, VT, in visitSRL()
8877 DAG.getNode(ISD::ANY_EXTEND, DL, VT, SmallShift), in visitSRL()
8878 DAG.getConstant(Mask, DL, VT)); in visitSRL()
8886 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), N1); in visitSRL()
8896 if (Known.One.getBoolValue()) return DAG.getConstant(0, SDLoc(N0), VT); in visitSRL()
8901 if (UnknownBits == 0) return DAG.getConstant(1, SDLoc(N0), VT); in visitSRL()
8914 Op = DAG.getNode(ISD::SRL, DL, VT, Op, in visitSRL()
8921 return DAG.getNode(ISD::XOR, DL, VT, in visitSRL()
8922 Op, DAG.getConstant(1, DL, VT)); in visitSRL()
8930 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, NewOp1); in visitSRL()
8984 EVT VT = N->getValueType(0); in visitFunnelShift() local
8989 unsigned BitWidth = VT.getScalarSizeInBits(); in visitFunnelShift()
9009 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N0, N1, in visitFunnelShift()
9022 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N1, in visitFunnelShift()
9026 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, in visitFunnelShift()
9035 if ((BitWidth % 8) == 0 && (ShAmt % 8) == 0 && !VT.isVector() && in visitFunnelShift()
9049 if (TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, in visitFunnelShift()
9057 VT, DL, RHS->getChain(), NewPtr, in visitFunnelShift()
9077 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N1, N2); in visitFunnelShift()
9079 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, N2); in visitFunnelShift()
9087 if (N0 == N1 && hasOperation(RotOpc, VT)) in visitFunnelShift()
9088 return DAG.getNode(RotOpc, SDLoc(N), VT, N0, N2); in visitFunnelShift()
9133 EVT VT = N->getValueType(0); in visitABS() local
9137 return DAG.getNode(ISD::ABS, SDLoc(N), VT, N0); in visitABS()
9153 EVT VT = N->getValueType(0); in visitBSWAP() local
9157 return DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N0); in visitBSWAP()
9166 EVT VT = N->getValueType(0); in visitBITREVERSE() local
9170 return DAG.getNode(ISD::BITREVERSE, SDLoc(N), VT, N0); in visitBITREVERSE()
9179 EVT VT = N->getValueType(0); in visitCTLZ() local
9183 return DAG.getNode(ISD::CTLZ, SDLoc(N), VT, N0); in visitCTLZ()
9186 if (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ_ZERO_UNDEF, VT)) { in visitCTLZ()
9188 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0); in visitCTLZ()
9196 EVT VT = N->getValueType(0); in visitCTLZ_ZERO_UNDEF() local
9200 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0); in visitCTLZ_ZERO_UNDEF()
9206 EVT VT = N->getValueType(0); in visitCTTZ() local
9210 return DAG.getNode(ISD::CTTZ, SDLoc(N), VT, N0); in visitCTTZ()
9213 if (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ_ZERO_UNDEF, VT)) { in visitCTTZ()
9215 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0); in visitCTTZ()
9223 EVT VT = N->getValueType(0); in visitCTTZ_ZERO_UNDEF() local
9227 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0); in visitCTTZ_ZERO_UNDEF()
9233 EVT VT = N->getValueType(0); in visitCTPOP() local
9237 return DAG.getNode(ISD::CTPOP, SDLoc(N), VT, N0); in visitCTPOP()
9247 EVT VT = LHS.getValueType(); in isLegalToCombineMinNumMaxNum() local
9249 return Options.NoSignedZerosFPMath && VT.isFloatingPoint() && in isLegalToCombineMinNumMaxNum()
9250 TLI.isProfitableToCombineMinNumMaxNum(VT) && in isLegalToCombineMinNumMaxNum()
9255 static SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS, in combineMinNumMaxNum() argument
9262 EVT TransformVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); in combineMinNumMaxNum()
9274 if (TLI.isOperationLegalOrCustom(IEEEOpcode, VT)) in combineMinNumMaxNum()
9275 return DAG.getNode(IEEEOpcode, DL, VT, LHS, RHS); in combineMinNumMaxNum()
9279 return DAG.getNode(Opcode, DL, VT, LHS, RHS); in combineMinNumMaxNum()
9289 if (TLI.isOperationLegalOrCustom(IEEEOpcode, VT)) in combineMinNumMaxNum()
9290 return DAG.getNode(IEEEOpcode, DL, VT, LHS, RHS); in combineMinNumMaxNum()
9294 return DAG.getNode(Opcode, DL, VT, LHS, RHS); in combineMinNumMaxNum()
9311 EVT VT = N->getValueType(0); in foldSelectOfConstantsUsingSra() local
9313 VT != Cond.getOperand(0).getValueType()) in foldSelectOfConstantsUsingSra()
9325 SDValue ShAmtC = DAG.getConstant(X.getScalarValueSizeInBits() - 1, DL, VT); in foldSelectOfConstantsUsingSra()
9326 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC); in foldSelectOfConstantsUsingSra()
9327 return DAG.getNode(ISD::OR, DL, VT, Sra, C1); in foldSelectOfConstantsUsingSra()
9332 SDValue ShAmtC = DAG.getConstant(X.getScalarValueSizeInBits() - 1, DL, VT); in foldSelectOfConstantsUsingSra()
9333 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC); in foldSelectOfConstantsUsingSra()
9334 return DAG.getNode(ISD::AND, DL, VT, Sra, C1); in foldSelectOfConstantsUsingSra()
9343 EVT VT = N->getValueType(0); in foldSelectOfConstants() local
9347 if (!VT.isInteger()) in foldSelectOfConstants()
9363 if (VT != MVT::i1) in foldSelectOfConstants()
9364 NotCond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, NotCond); in foldSelectOfConstants()
9370 if (VT != MVT::i1) in foldSelectOfConstants()
9371 NotCond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NotCond); in foldSelectOfConstants()
9376 if (VT != MVT::i1) in foldSelectOfConstants()
9377 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond); in foldSelectOfConstants()
9382 if (VT != MVT::i1) in foldSelectOfConstants()
9383 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond); in foldSelectOfConstants()
9389 if (TLI.convertSelectOfConstantsToMath(VT)) { in foldSelectOfConstants()
9396 if (VT != MVT::i1) in foldSelectOfConstants()
9397 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond); in foldSelectOfConstants()
9398 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2); in foldSelectOfConstants()
9402 if (VT != MVT::i1) in foldSelectOfConstants()
9403 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond); in foldSelectOfConstants()
9404 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2); in foldSelectOfConstants()
9409 if (VT != MVT::i1) in foldSelectOfConstants()
9410 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond); in foldSelectOfConstants()
9411 SDValue ShAmtC = DAG.getConstant(C1Val.exactLogBase2(), DL, VT); in foldSelectOfConstants()
9412 return DAG.getNode(ISD::SHL, DL, VT, Cond, ShAmtC); in foldSelectOfConstants()
9439 if (VT.bitsEq(CondVT)) in foldSelectOfConstants()
9441 return DAG.getZExtOrTrunc(NotCond, DL, VT); in foldSelectOfConstants()
9452 EVT VT = N->getValueType(0); in foldBoolSelectToLogic() local
9453 if (VT != Cond.getValueType() || VT.getScalarSizeInBits() != 1) in foldBoolSelectToLogic()
9459 return DAG.getNode(ISD::OR, SDLoc(N), VT, Cond, F); in foldBoolSelectToLogic()
9464 return DAG.getNode(ISD::AND, SDLoc(N), VT, Cond, T); in foldBoolSelectToLogic()
9468 SDValue NotCond = DAG.getNOT(SDLoc(N), Cond, VT); in foldBoolSelectToLogic()
9469 return DAG.getNode(ISD::OR, SDLoc(N), VT, NotCond, T); in foldBoolSelectToLogic()
9474 SDValue NotCond = DAG.getNOT(SDLoc(N), Cond, VT); in foldBoolSelectToLogic()
9475 return DAG.getNode(ISD::AND, SDLoc(N), VT, NotCond, F); in foldBoolSelectToLogic()
9485 EVT VT = N->getValueType(0); in visitSELECT() local
9513 TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT); in visitSELECT()
9583 SDValue SelectOp = DAG.getSelect(DL, VT, F, N2, N1); in visitSELECT()
9598 if (SDValue FMinMax = combineMinNumMaxNum(DL, VT, Cond0, Cond1, N1, N2, in visitSELECT()
9607 if (!LegalOperations && TLI.isOperationLegalOrCustom(ISD::UADDO, VT) && in visitSELECT()
9625 SDVTList VTs = DAG.getVTList(VT, VT0); in visitSELECT()
9627 return DAG.getSelect(DL, VT, UAO.getValue(1), N1, UAO.getValue(0)); in visitSELECT()
9631 if (TLI.isOperationLegal(ISD::SELECT_CC, VT) || in visitSELECT()
9633 TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT))) { in visitSELECT()
9637 SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1, in visitSELECT()
9647 if (!VT.isVector()) in visitSELECT()
9661 EVT VT = N->getValueType(0); in ConvertSelectToConcatVector() local
9662 int NumElems = VT.getVectorNumElements(); in ConvertSelectToConcatVector()
9704 ISD::CONCAT_VECTORS, DL, VT, in ConvertSelectToConcatVector()
9871 EVT VT = N->getValueType(0); in foldVSelectOfConstants() local
9873 !TLI.convertSelectOfConstantsToMath(VT) || in foldVSelectOfConstants()
9883 unsigned Elts = VT.getVectorNumElements(); in foldVSelectOfConstants()
9907 SDValue ExtendedCond = DAG.getNode(ExtendOpcode, DL, VT, Cond); in foldVSelectOfConstants()
9908 return DAG.getNode(ISD::ADD, DL, VT, ExtendedCond, N2); in foldVSelectOfConstants()
9915 SDValue ZextCond = DAG.getZExtOrTrunc(Cond, DL, VT); in foldVSelectOfConstants()
9916 SDValue ShAmtC = DAG.getConstant(Pow2C.exactLogBase2(), DL, VT); in foldVSelectOfConstants()
9917 return DAG.getNode(ISD::SHL, DL, VT, ZextCond, ShAmtC); in foldVSelectOfConstants()
9934 EVT VT = N->getValueType(0); in visitVSELECT() local
9945 return DAG.getSelect(DL, VT, F, N2, N1); in visitVSELECT()
9967 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) in visitVSELECT()
9968 return DAG.getNode(ISD::ABS, DL, VT, LHS); in visitVSELECT()
9970 SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, LHS, in visitVSELECT()
9971 DAG.getConstant(VT.getScalarSizeInBits() - 1, in visitVSELECT()
9972 DL, getShiftAmountTy(VT))); in visitVSELECT()
9973 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift); in visitVSELECT()
9976 return DAG.getNode(ISD::XOR, DL, VT, Add, Shift); in visitVSELECT()
9987 combineMinNumMaxNum(DL, VT, LHS, RHS, N1, N2, CC, TLI, DAG)) in visitVSELECT()
10022 if (hasOperation(ISD::UADDSAT, VT)) { in visitVSELECT()
10029 SatCC = ISD::getSetCCInverse(SatCC, VT.getScalarType()); in visitVSELECT()
10049 return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
10063 return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
10069 if (hasOperation(ISD::USUBSAT, VT)) { in visitVSELECT()
10076 SatCC = ISD::getSetCCInverse(SatCC, VT.getScalarType()); in visitVSELECT()
10094 return getTruncatedUSUBSAT(VT, LHS.getValueType(), LHS, RHS, DAG, in visitVSELECT()
10104 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
10121 OpRHS = DAG.getNode(ISD::SUB, DL, VT, in visitVSELECT()
10122 DAG.getConstant(0, DL, VT), OpRHS); in visitVSELECT()
10123 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
10138 OpRHS = DAG.getConstant(SplatValue, DL, VT); in visitVSELECT()
10139 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS); in visitVSELECT()
10226 EVT VT = N->getValueType(0); in visitSETCC() local
10276 return DAG.getFreeze(DAG.getSetCC(SDLoc(N), VT, N0, N1, Cond)); in visitSETCC()
10279 SDValue Combined = SimplifySetCC(VT, N->getOperand(0), N->getOperand(1), Cond, in visitSETCC()
10352 EVT VT = N->getValueType(0); in tryToFoldExtendSelectLoad() local
10376 if (!TLI.isLoadExtLegal(ExtLoadOpcode, VT, Load1->getMemoryVT()) || in tryToFoldExtendSelectLoad()
10377 !TLI.isLoadExtLegal(ExtLoadOpcode, VT, Load2->getMemoryVT())) in tryToFoldExtendSelectLoad()
10380 SDValue Ext1 = DAG.getNode(Opcode, DL, VT, Op1); in tryToFoldExtendSelectLoad()
10381 SDValue Ext2 = DAG.getNode(Opcode, DL, VT, Op2); in tryToFoldExtendSelectLoad()
10382 return DAG.getSelect(DL, VT, N0->getOperand(0), Ext1, Ext2); in tryToFoldExtendSelectLoad()
10395 EVT VT = N->getValueType(0); in tryToFoldExtendOfConstant() local
10407 return DAG.getNode(Opcode, DL, VT, N0); in tryToFoldExtendOfConstant()
10416 (Opcode != ISD::ZERO_EXTEND || !TLI.isZExtFree(N0.getValueType(), VT))) { in tryToFoldExtendOfConstant()
10429 return DAG.getSelect(DL, VT, N0->getOperand(0), in tryToFoldExtendOfConstant()
10430 DAG.getNode(FoldOpc, DL, VT, Op1), in tryToFoldExtendOfConstant()
10431 DAG.getNode(FoldOpc, DL, VT, Op2)); in tryToFoldExtendOfConstant()
10438 EVT SVT = VT.getScalarType(); in tryToFoldExtendOfConstant()
10439 if (!(VT.isVector() && (!LegalTypes || TLI.isTypeLegal(SVT)) && in tryToFoldExtendOfConstant()
10447 unsigned NumElts = VT.getVectorNumElements(); in tryToFoldExtendOfConstant()
10471 return DAG.getBuildVector(VT, DL, Elts); in tryToFoldExtendOfConstant()
10478 static bool ExtendUsesToFormExtLoad(EVT VT, SDNode *N, SDValue N0, in ExtendUsesToFormExtLoad() argument
10483 bool isTruncFree = TLI.isTruncateFree(VT, N0.getValueType()); in ExtendUsesToFormExtLoad()
10662 EVT VT = N->getValueType(0); in CombineZExtLogicopShiftLoad() local
10664 if (TLI.isZExtFree(OrigVT, VT)) in CombineZExtLogicopShiftLoad()
10672 (LegalOperations && !TLI.isOperationLegal(N0.getOpcode(), VT))) in CombineZExtLogicopShiftLoad()
10679 (LegalOperations && !TLI.isOperationLegal(N1.getOpcode(), VT))) in CombineZExtLogicopShiftLoad()
10687 if (!TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) || in CombineZExtLogicopShiftLoad()
10701 if (!ExtendUsesToFormExtLoad(VT, N1.getNode(), N1.getOperand(0), in CombineZExtLogicopShiftLoad()
10706 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(Load), VT, in CombineZExtLogicopShiftLoad()
10711 SDValue Shift = DAG.getNode(N1.getOpcode(), DL1, VT, ExtLoad, in CombineZExtLogicopShiftLoad()
10714 APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits()); in CombineZExtLogicopShiftLoad()
10716 SDValue And = DAG.getNode(N0.getOpcode(), DL0, VT, Shift, in CombineZExtLogicopShiftLoad()
10717 DAG.getConstant(Mask, DL0, VT)); in CombineZExtLogicopShiftLoad()
10749 EVT VT = Cast->getValueType(0); in matchVSelectOpSizesWithSetCC() local
10750 if (LegalOperations || !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT)) in matchVSelectOpSizesWithSetCC()
10761 if (SetCCVT.getSizeInBits() != VT.getSizeInBits()) in matchVSelectOpSizesWithSetCC()
10771 CastA = DAG.getNode(CastOpcode, DL, VT, A, Cast->getOperand(1)); in matchVSelectOpSizesWithSetCC()
10772 CastB = DAG.getNode(CastOpcode, DL, VT, B, Cast->getOperand(1)); in matchVSelectOpSizesWithSetCC()
10774 CastA = DAG.getNode(CastOpcode, DL, VT, A); in matchVSelectOpSizesWithSetCC()
10775 CastB = DAG.getNode(CastOpcode, DL, VT, B); in matchVSelectOpSizesWithSetCC()
10777 return DAG.getNode(ISD::VSELECT, DL, VT, SetCC, CastA, CastB); in matchVSelectOpSizesWithSetCC()
10783 const TargetLowering &TLI, EVT VT, in tryToFoldExtOfExtload() argument
10796 VT.isVector()) && in tryToFoldExtOfExtload()
10797 !TLI.isLoadExtLegal(ExtLoadType, VT, MemVT)) in tryToFoldExtOfExtload()
10801 DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(), in tryToFoldExtOfExtload()
10814 const TargetLowering &TLI, EVT VT, in tryToFoldExtOfLoad() argument
10820 ((LegalOperations || VT.isVector() || in tryToFoldExtOfLoad()
10822 !TLI.isLoadExtLegal(ExtLoadType, VT, N0.getValueType()))) in tryToFoldExtOfLoad()
10828 DoXform = ExtendUsesToFormExtLoad(VT, N, N0, ExtOpc, SetCCs, TLI); in tryToFoldExtOfLoad()
10829 if (VT.isVector()) in tryToFoldExtOfLoad()
10835 SDValue ExtLoad = DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(), in tryToFoldExtOfLoad()
10854 const TargetLowering &TLI, EVT VT, in tryToFoldExtOfMaskedLoad() argument
10865 if (!TLI.isLoadExtLegal(ExtLoadType, VT, Ld->getValueType(0))) in tryToFoldExtOfMaskedLoad()
10872 SDValue PassThru = DAG.getNode(ExtOpc, dl, VT, Ld->getPassThru()); in tryToFoldExtOfMaskedLoad()
10874 VT, dl, Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(), Ld->getMask(), in tryToFoldExtOfMaskedLoad()
10894 EVT VT = N->getValueType(0); in foldExtendedSignBitTest() local
10899 if (CC == ISD::SETGT && isAllOnesConstant(Ones) && VT == XVT) { in foldExtendedSignBitTest()
10904 unsigned ShCt = VT.getSizeInBits() - 1; in foldExtendedSignBitTest()
10906 if (!TLI.shouldAvoidTransformToShift(VT, ShCt)) { in foldExtendedSignBitTest()
10907 SDValue NotX = DAG.getNOT(DL, X, VT); in foldExtendedSignBitTest()
10908 SDValue ShiftAmount = DAG.getConstant(ShCt, DL, VT); in foldExtendedSignBitTest()
10911 return DAG.getNode(ShiftOpcode, DL, VT, NotX, ShiftAmount); in foldExtendedSignBitTest()
10925 EVT VT = N->getValueType(0); in foldSextSetcc() local
10932 if (VT.isVector() && !LegalOperations && in foldSextSetcc()
10944 if (VT.getSizeInBits() == SVT.getSizeInBits()) in foldSextSetcc()
10945 return DAG.getSetCC(DL, VT, N00, N01, CC); in foldSextSetcc()
10953 return DAG.getSExtOrTrunc(VsetCC, DL, VT); in foldSextSetcc()
10958 if (N0.hasOneUse() && TLI.isOperationLegalOrCustom(ISD::SETCC, VT) && in foldSextSetcc()
10976 TLI.isLoadExtLegal(LoadOpcode, VT, V.getValueType()))) in foldSextSetcc()
10990 if (User->getOpcode() != ExtOpcode || User->getValueType(0) != VT) in foldSextSetcc()
10997 SDValue Ext0 = DAG.getNode(ExtOpcode, DL, VT, N00); in foldSextSetcc()
10998 SDValue Ext1 = DAG.getNode(ExtOpcode, DL, VT, N01); in foldSextSetcc()
10999 return DAG.getSetCC(DL, VT, Ext0, Ext1, CC); in foldSextSetcc()
11017 ? DAG.getAllOnesConstant(DL, VT) in foldSextSetcc()
11018 : DAG.getBoolConstant(true, DL, VT, N00VT); in foldSextSetcc()
11019 SDValue Zero = DAG.getConstant(0, DL, VT); in foldSextSetcc()
11023 if (!VT.isVector() && !TLI.convertSelectOfConstantsToMath(VT)) { in foldSextSetcc()
11032 return DAG.getSelect(DL, VT, SetCC, ExtTrueVal, Zero); in foldSextSetcc()
11041 EVT VT = N->getValueType(0); in visitSIGN_EXTEND() local
11050 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N0.getOperand(0)); in visitSIGN_EXTEND()
11070 unsigned DestBits = VT.getScalarSizeInBits(); in visitSIGN_EXTEND()
11082 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op); in visitSIGN_EXTEND()
11087 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op); in visitSIGN_EXTEND()
11094 Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N0), VT, Op); in visitSIGN_EXTEND()
11096 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, Op); in visitSIGN_EXTEND()
11097 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Op, in visitSIGN_EXTEND()
11104 tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0, in visitSIGN_EXTEND()
11109 tryToFoldExtOfMaskedLoad(DAG, TLI, VT, N, N0, ISD::SEXTLOAD, in visitSIGN_EXTEND()
11120 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::SEXTLOAD)) in visitSIGN_EXTEND()
11129 (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) { in visitSIGN_EXTEND()
11132 if (TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, MemVT) && in visitSIGN_EXTEND()
11135 bool DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0), in visitSIGN_EXTEND()
11138 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN00), VT, in visitSIGN_EXTEND()
11142 APInt Mask = N0.getConstantOperandAPInt(1).sext(VT.getSizeInBits()); in visitSIGN_EXTEND()
11143 SDValue And = DAG.getNode(N0.getOpcode(), DL, VT, in visitSIGN_EXTEND()
11144 ExtLoad, DAG.getConstant(Mask, DL, VT)); in visitSIGN_EXTEND()
11174 if ((!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) && in visitSIGN_EXTEND()
11176 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0); in visitSIGN_EXTEND()
11186 TLI.isOperationLegalOrCustom(ISD::SUB, VT)) { in visitSIGN_EXTEND()
11187 SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(1).getOperand(0), DL, VT); in visitSIGN_EXTEND()
11188 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Zext); in visitSIGN_EXTEND()
11195 TLI.isOperationLegalOrCustom(ISD::ADD, VT)) { in visitSIGN_EXTEND()
11196 SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(0).getOperand(0), DL, VT); in visitSIGN_EXTEND()
11197 return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT)); in visitSIGN_EXTEND()
11203 (!LegalOperations || (TLI.isOperationLegal(ISD::ZERO_EXTEND, VT) && in visitSIGN_EXTEND()
11204 TLI.isOperationLegal(ISD::ADD, VT)))) { in visitSIGN_EXTEND()
11215 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NewXor); in visitSIGN_EXTEND()
11219 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitSIGN_EXTEND()
11220 return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT)); in visitSIGN_EXTEND()
11273 EVT VT = Extend->getValueType(0); in widenCtPop() local
11276 !TLI.isOperationLegalOrCustom(ISD::CTPOP, VT)) in widenCtPop()
11281 SDValue NewZext = DAG.getZExtOrTrunc(CtPop.getOperand(0), DL, VT); in widenCtPop()
11282 return DAG.getNode(ISD::CTPOP, DL, VT, NewZext); in widenCtPop()
11287 EVT VT = N->getValueType(0); in visitZERO_EXTEND() local
11295 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, in visitZERO_EXTEND()
11310 VT.getScalarSizeInBits())); in visitZERO_EXTEND()
11312 return DAG.getZExtOrTrunc(Op, SDLoc(N), VT); in visitZERO_EXTEND()
11334 if (SrcVT.bitsLT(VT) && VT.isVector()) { in visitZERO_EXTEND()
11336 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) { in visitZERO_EXTEND()
11340 SDValue ZExtOrTrunc = DAG.getZExtOrTrunc(Op, SDLoc(N), VT); in visitZERO_EXTEND()
11347 if (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT)) { in visitZERO_EXTEND()
11348 SDValue Op = DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT); in visitZERO_EXTEND()
11365 !TLI.isZExtFree(N0.getValueType(), VT))) { in visitZERO_EXTEND()
11367 X = DAG.getAnyExtOrTrunc(X, SDLoc(X), VT); in visitZERO_EXTEND()
11368 APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits()); in visitZERO_EXTEND()
11370 return DAG.getNode(ISD::AND, DL, VT, in visitZERO_EXTEND()
11371 X, DAG.getConstant(Mask, DL, VT)); in visitZERO_EXTEND()
11376 tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0, in visitZERO_EXTEND()
11381 tryToFoldExtOfMaskedLoad(DAG, TLI, VT, N, N0, ISD::ZEXTLOAD, in visitZERO_EXTEND()
11398 (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) { in visitZERO_EXTEND()
11401 if (TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) && in visitZERO_EXTEND()
11415 DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0), in visitZERO_EXTEND()
11418 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN00), VT, in visitZERO_EXTEND()
11422 APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits()); in visitZERO_EXTEND()
11424 SDValue And = DAG.getNode(N0.getOpcode(), DL, VT, in visitZERO_EXTEND()
11425 ExtLoad, DAG.getConstant(Mask, DL, VT)); in visitZERO_EXTEND()
11455 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::ZEXTLOAD)) in visitZERO_EXTEND()
11463 if (!LegalOperations && VT.isVector() && in visitZERO_EXTEND()
11475 if (VT.getSizeInBits() == N00VT.getSizeInBits()) { in visitZERO_EXTEND()
11477 SDValue VSetCC = DAG.getNode(ISD::SETCC, DL, VT, N0.getOperand(0), in visitZERO_EXTEND()
11489 return DAG.getZeroExtendInReg(DAG.getAnyExtOrTrunc(VsetCC, DL, VT), DL, in visitZERO_EXTEND()
11502 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, SCC); in visitZERO_EXTEND()
11524 if (Log2_32_Ceil(VT.getSizeInBits()) > ShAmt.getValueSizeInBits()) in visitZERO_EXTEND()
11527 return DAG.getNode(N0.getOpcode(), DL, VT, in visitZERO_EXTEND()
11528 DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)), in visitZERO_EXTEND()
11546 EVT VT = N->getValueType(0); in visitANY_EXTEND() local
11557 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0)); in visitANY_EXTEND()
11575 return DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT); in visitANY_EXTEND()
11586 X = DAG.getAnyExtOrTrunc(X, DL, VT); in visitANY_EXTEND()
11587 APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits()); in visitANY_EXTEND()
11588 return DAG.getNode(ISD::AND, DL, VT, in visitANY_EXTEND()
11589 X, DAG.getConstant(Mask, DL, VT)); in visitANY_EXTEND()
11595 if (VT.isVector()) { in visitANY_EXTEND()
11598 tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0, in visitANY_EXTEND()
11603 TLI.isLoadExtLegal(ISD::EXTLOAD, VT, N0.getValueType())) { in visitANY_EXTEND()
11608 ExtendUsesToFormExtLoad(VT, N, N0, ISD::ANY_EXTEND, SetCCs, TLI); in visitANY_EXTEND()
11611 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, in visitANY_EXTEND()
11638 if (!LegalOperations || TLI.isLoadExtLegal(ExtType, VT, MemVT)) { in visitANY_EXTEND()
11640 VT, LN0->getChain(), LN0->getBasePtr(), in visitANY_EXTEND()
11655 if (VT.isVector() && !LegalOperations) { in visitANY_EXTEND()
11665 if (VT.getSizeInBits() == N00VT.getSizeInBits()) in visitANY_EXTEND()
11666 return DAG.getSetCC(SDLoc(N), VT, N0.getOperand(0), in visitANY_EXTEND()
11678 return DAG.getAnyExtOrTrunc(VsetCC, SDLoc(N), VT); in visitANY_EXTEND()
11684 DL, N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, DL, VT), in visitANY_EXTEND()
11685 DAG.getConstant(0, DL, VT), in visitANY_EXTEND()
11804 EVT VT = N->getValueType(0); in ReduceLoadWidth() local
11805 EVT ExtVT = VT; in ReduceLoadWidth()
11808 if (VT.isVector()) in ReduceLoadWidth()
11836 VT.getScalarSizeInBits() - ShiftAmt); in ReduceLoadWidth()
11910 ExtVT == VT && TLI.isNarrowingProfitable(N0.getValueType(), VT)) { in ReduceLoadWidth()
11952 Load = DAG.getLoad(VT, DL, LN0->getChain(), NewPtr, in ReduceLoadWidth()
11956 Load = DAG.getExtLoad(ExtType, DL, VT, LN0->getChain(), NewPtr, in ReduceLoadWidth()
11970 ShImmTy = VT; in ReduceLoadWidth()
11975 if (ShLeftAmt >= VT.getScalarSizeInBits()) in ReduceLoadWidth()
11976 Result = DAG.getConstant(0, DL, VT); in ReduceLoadWidth()
11978 Result = DAG.getNode(ISD::SHL, DL, VT, in ReduceLoadWidth()
11992 SDValue ShiftC = DAG.getConstant(ShAmt, DL, VT); in ReduceLoadWidth()
11993 Result = DAG.getNode(ISD::SHL, DL, VT, Result, ShiftC); in ReduceLoadWidth()
12004 EVT VT = N->getValueType(0); in visitSIGN_EXTEND_INREG() local
12006 unsigned VTBits = VT.getScalarSizeInBits(); in visitSIGN_EXTEND_INREG()
12011 return DAG.getConstant(0, SDLoc(N), VT); in visitSIGN_EXTEND_INREG()
12015 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0, N1); in visitSIGN_EXTEND_INREG()
12024 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0.getOperand(0), in visitSIGN_EXTEND_INREG()
12036 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT))) in visitSIGN_EXTEND_INREG()
12037 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00); in visitSIGN_EXTEND_INREG()
12057 TLI.isOperationLegal(ISD::SIGN_EXTEND_VECTOR_INREG, VT))) in visitSIGN_EXTEND_INREG()
12058 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, SDLoc(N), VT, N00); in visitSIGN_EXTEND_INREG()
12066 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT))) in visitSIGN_EXTEND_INREG()
12067 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00, N1); in visitSIGN_EXTEND_INREG()
12094 return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0.getOperand(0), in visitSIGN_EXTEND_INREG()
12108 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT))) { in visitSIGN_EXTEND_INREG()
12110 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
12125 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT))) { in visitSIGN_EXTEND_INREG()
12127 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
12141 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, ExtVT)) { in visitSIGN_EXTEND_INREG()
12143 VT, SDLoc(N), Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(), in visitSIGN_EXTEND_INREG()
12161 DAG.getVTList(VT, MVT::Other), ExtVT, SDLoc(N), Ops, in visitSIGN_EXTEND_INREG()
12175 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, BSwap, N1); in visitSIGN_EXTEND_INREG()
12183 EVT VT = N->getValueType(0); in visitEXTEND_VECTOR_INREG() local
12187 return DAG.getConstant(0, SDLoc(N), VT); in visitEXTEND_VECTOR_INREG()
12200 EVT VT = N->getValueType(0); in visitTRUNCATE() local
12205 if (SrcVT == VT) in visitTRUNCATE()
12210 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0)); in visitTRUNCATE()
12214 SDValue C = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0); in visitTRUNCATE()
12224 if (N0.getOperand(0).getValueType().bitsLT(VT)) in visitTRUNCATE()
12225 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0)); in visitTRUNCATE()
12227 if (N0.getOperand(0).getValueType().bitsGT(VT)) in visitTRUNCATE()
12228 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0)); in visitTRUNCATE()
12249 LegalTypes && !LegalOperations && N0->hasOneUse() && VT != MVT::i1) { in visitTRUNCATE()
12276 TLI.isTruncateFree(SrcVT, VT)) { in visitTRUNCATE()
12279 SDValue TruncOp0 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1)); in visitTRUNCATE()
12280 SDValue TruncOp1 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(2)); in visitTRUNCATE()
12281 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, Cond, TruncOp0, TruncOp1); in visitTRUNCATE()
12287 (!LegalOperations || TLI.isOperationLegal(ISD::SHL, VT)) && in visitTRUNCATE()
12288 TLI.isTypeDesirableForOp(ISD::SHL, VT)) { in visitTRUNCATE()
12291 unsigned Size = VT.getScalarSizeInBits(); in visitTRUNCATE()
12294 EVT AmtVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); in visitTRUNCATE()
12296 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0)); in visitTRUNCATE()
12301 return DAG.getNode(ISD::SHL, SL, VT, Trunc, Amt); in visitTRUNCATE()
12305 if (SDValue V = foldSubToUSubSat(VT, N0.getNode())) in visitTRUNCATE()
12310 TLI.isTruncateFree(SrcVT.getScalarType(), VT.getScalarType()) && in visitTRUNCATE()
12312 (!LegalTypes || TLI.isTypeLegal(VT.getScalarType()))) { in visitTRUNCATE()
12314 EVT SVT = VT.getScalarType(); in visitTRUNCATE()
12320 return DAG.getBuildVector(VT, DL, TruncOps); in visitTRUNCATE()
12327 if (Level == AfterLegalizeVectorOps && VT.isVector() && in visitTRUNCATE()
12333 EVT TruncVecEltTy = VT.getVectorElementType(); in visitTRUNCATE()
12339 unsigned TruncVecNumElts = VT.getVectorNumElements(); in visitTRUNCATE()
12349 return DAG.getBuildVector(VT, SDLoc(N), Opnds); in visitTRUNCATE()
12358 if (!VT.isVector()) { in visitTRUNCATE()
12360 APInt::getLowBitsSet(N0.getValueSizeInBits(), VT.getSizeInBits()); in visitTRUNCATE()
12362 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Shorter); in visitTRUNCATE()
12367 if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getOpcode(), VT)) { in visitTRUNCATE()
12375 if (LN0->isSimple() && LN0->getMemoryVT().bitsLT(VT)) { in visitTRUNCATE()
12377 VT, LN0->getChain(), LN0->getBasePtr(), in visitTRUNCATE()
12406 VT.getVectorElementType(), in visitTRUNCATE()
12411 return DAG.getUNDEF(VT); in visitTRUNCATE()
12425 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Opnds); in visitTRUNCATE()
12433 if (N0.getOpcode() == ISD::BITCAST && !VT.isVector()) { in visitTRUNCATE()
12436 if (VecSrcVT.isVector() && VecSrcVT.getScalarType() == VT && in visitTRUNCATE()
12442 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, VT, VecSrc, in visitTRUNCATE()
12458 TLI.isOperationLegal(N0.getOpcode(), VT))) { in visitTRUNCATE()
12460 auto X = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0)); in visitTRUNCATE()
12461 auto Y = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1)); in visitTRUNCATE()
12462 auto VTs = DAG.getVTList(VT, N0->getValueType(1)); in visitTRUNCATE()
12476 VT.getVectorElementType()) in visitTRUNCATE()
12477 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N0->getOperand(0)), VT, in visitTRUNCATE()
12502 if (VT.isScalarInteger() || TLI.isOperationLegal(N0.getOpcode(), VT)) { in visitTRUNCATE()
12504 SDValue NarrowL = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
12505 SDValue NarrowR = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1)); in visitTRUNCATE()
12506 return DAG.getNode(N0.getOpcode(), DL, VT, NarrowL, NarrowR); in visitTRUNCATE()
12517 VT.getScalarSizeInBits() && in visitTRUNCATE()
12518 hasOperation(N0.getOpcode(), VT)) { in visitTRUNCATE()
12519 return getTruncatedUSUBSAT(VT, SrcVT, N0.getOperand(0), N0.getOperand(1), in visitTRUNCATE()
12537 SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) { in CombineConsecutiveLoads() argument
12558 VT.getTypeForEVT(*DAG.getContext())); in CombineConsecutiveLoads()
12561 (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT))) in CombineConsecutiveLoads()
12562 return DAG.getLoad(VT, SDLoc(N), LD1->getChain(), LD1->getBasePtr(), in CombineConsecutiveLoads()
12579 EVT VT = N->getValueType(0); in foldBitcastedFPLogic() local
12580 if (!VT.isFloatingPoint() || !TLI.hasBitPreservingFPLogic(VT)) in foldBitcastedFPLogic()
12587 if (VT.getScalarSizeInBits() != SourceVT.getScalarSizeInBits()) in foldBitcastedFPLogic()
12617 LogicOp0.getOperand(0).getValueType() == VT) { in foldBitcastedFPLogic()
12618 SDValue FPOp = DAG.getNode(FPOpcode, SDLoc(N), VT, LogicOp0.getOperand(0)); in foldBitcastedFPLogic()
12621 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, FPOp); in foldBitcastedFPLogic()
12630 EVT VT = N->getValueType(0); in visitBITCAST() local
12633 return DAG.getUNDEF(VT); in visitBITCAST()
12641 if (VT.isVector() && in visitBITCAST()
12643 (!LegalOperations && VT.isInteger() && N0.getValueType().isInteger() && in visitBITCAST()
12644 TLI.isTypeLegal(VT.getVectorElementType()))) && in visitBITCAST()
12648 VT.getVectorElementType()); in visitBITCAST()
12656 (isa<ConstantSDNode>(N0) && VT.isFloatingPoint() && !VT.isVector() && in visitBITCAST()
12657 TLI.isOperationLegal(ISD::ConstantFP, VT)) || in visitBITCAST()
12658 (isa<ConstantFPSDNode>(N0) && VT.isInteger() && !VT.isVector() && in visitBITCAST()
12659 TLI.isOperationLegal(ISD::Constant, VT))) { in visitBITCAST()
12660 SDValue C = DAG.getBitcast(VT, N0); in visitBITCAST()
12668 return DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
12675 TLI.hasBigEndianPartOrdering(VT, DAG.getDataLayout()) && in visitBITCAST()
12682 TLI.isOperationLegal(ISD::LOAD, VT))) { in visitBITCAST()
12685 if (TLI.isLoadBitCastBeneficial(N0.getValueType(), VT, DAG, in visitBITCAST()
12688 DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(), in visitBITCAST()
12713 N0.getNode()->hasOneUse() && VT.isInteger() && in visitBITCAST()
12714 !VT.isVector() && !N0.getValueType().isVector()) { in visitBITCAST()
12715 SDValue NewConv = DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
12720 assert(VT.getSizeInBits() == 128); in visitBITCAST()
12722 APInt::getSignMask(VT.getSizeInBits() / 2), SDLoc(N0), MVT::i64); in visitBITCAST()
12738 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit); in visitBITCAST()
12740 return DAG.getNode(ISD::XOR, DL, VT, NewConv, FlipBits); in visitBITCAST()
12742 APInt SignBit = APInt::getSignMask(VT.getSizeInBits()); in visitBITCAST()
12744 return DAG.getNode(ISD::XOR, DL, VT, in visitBITCAST()
12745 NewConv, DAG.getConstant(SignBit, DL, VT)); in visitBITCAST()
12747 return DAG.getNode(ISD::AND, DL, VT, in visitBITCAST()
12748 NewConv, DAG.getConstant(~SignBit, DL, VT)); in visitBITCAST()
12764 VT.isInteger() && !VT.isVector()) { in visitBITCAST()
12772 unsigned VTWidth = VT.getSizeInBits(); in visitBITCAST()
12774 X = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, X); in visitBITCAST()
12785 X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X); in visitBITCAST()
12790 APInt SignBit = APInt::getSignMask(VT.getSizeInBits() / 2); in visitBITCAST()
12791 SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
12793 SDValue X = DAG.getBitcast(VT, N0.getOperand(1)); in visitBITCAST()
12795 SDValue XorResult = DAG.getNode(ISD::XOR, SDLoc(N0), VT, Cst, X); in visitBITCAST()
12807 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit); in visitBITCAST()
12809 return DAG.getNode(ISD::XOR, SDLoc(N), VT, Cst, FlipBits); in visitBITCAST()
12811 APInt SignBit = APInt::getSignMask(VT.getSizeInBits()); in visitBITCAST()
12812 X = DAG.getNode(ISD::AND, SDLoc(X), VT, in visitBITCAST()
12813 X, DAG.getConstant(SignBit, SDLoc(X), VT)); in visitBITCAST()
12816 SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
12817 Cst = DAG.getNode(ISD::AND, SDLoc(Cst), VT, in visitBITCAST()
12818 Cst, DAG.getConstant(~SignBit, SDLoc(Cst), VT)); in visitBITCAST()
12821 return DAG.getNode(ISD::OR, SDLoc(N), VT, X, Cst); in visitBITCAST()
12827 if (SDValue CombineLD = CombineConsecutiveLoads(N0.getNode(), VT)) in visitBITCAST()
12834 if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT) && VT.isVector() && in visitBITCAST()
12836 VT.getVectorNumElements() >= N0.getValueType().getVectorNumElements() && in visitBITCAST()
12837 !(VT.getVectorNumElements() % N0.getValueType().getVectorNumElements())) { in visitBITCAST()
12844 Op.getOperand(0).getValueType() == VT) in visitBITCAST()
12848 return DAG.getBitcast(VT, Op); in visitBITCAST()
12861 VT.getVectorNumElements() / N0.getValueType().getVectorNumElements(); in visitBITCAST()
12868 TLI.buildLegalVectorShuffle(VT, SDLoc(N), SV0, SV1, NewMask, DAG); in visitBITCAST()
12877 EVT VT = N->getValueType(0); in visitBUILD_PAIR() local
12878 return CombineConsecutiveLoads(N, VT); in visitBUILD_PAIR()
12914 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, in ConstantFoldBITCASTofBUILD_VECTOR() local
12916 return DAG.getBuildVector(VT, SDLoc(BV), Ops); in ConstantFoldBITCASTofBUILD_VECTOR()
12971 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, Ops.size()); in ConstantFoldBITCASTofBUILD_VECTOR() local
12972 return DAG.getBuildVector(VT, DL, Ops); in ConstantFoldBITCASTofBUILD_VECTOR()
12978 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, in ConstantFoldBITCASTofBUILD_VECTOR() local
13002 return DAG.getBuildVector(VT, DL, Ops); in ConstantFoldBITCASTofBUILD_VECTOR()
13009 EVT VT = N->getValueType(0); in visitFADDForFMACombine() local
13019 TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT) && in visitFADDForFMACombine()
13020 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFADDForFMACombine()
13034 if (TLI.generateFMAsInMachineCombiner(VT, OptLevel)) in visitFADDForFMACombine()
13039 bool Aggressive = TLI.enableAggressiveFMAFusion(VT); in visitFADDForFMACombine()
13057 return DAG.getNode(PreferredFusedOpcode, SL, VT, N0.getOperand(0), in visitFADDForFMACombine()
13064 return DAG.getNode(PreferredFusedOpcode, SL, VT, N1.getOperand(0), in visitFADDForFMACombine()
13088 SDValue CDE = DAG.getNode(PreferredFusedOpcode, SL, VT, C, D, E); in visitFADDForFMACombine()
13089 return DAG.getNode(PreferredFusedOpcode, SL, VT, A, B, CDE); in visitFADDForFMACombine()
13098 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFADDForFMACombine()
13100 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
13101 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)), in visitFADDForFMACombine()
13102 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)), in visitFADDForFMACombine()
13112 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFADDForFMACombine()
13114 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
13115 DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(0)), in visitFADDForFMACombine()
13116 DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)), in visitFADDForFMACombine()
13127 return DAG.getNode(PreferredFusedOpcode, SL, VT, X, Y, in visitFADDForFMACombine()
13128 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
13129 DAG.getNode(ISD::FP_EXTEND, SL, VT, U), in visitFADDForFMACombine()
13130 DAG.getNode(ISD::FP_EXTEND, SL, VT, V), in visitFADDForFMACombine()
13138 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFADDForFMACombine()
13155 PreferredFusedOpcode, SL, VT, DAG.getNode(ISD::FP_EXTEND, SL, VT, X), in visitFADDForFMACombine()
13156 DAG.getNode(ISD::FP_EXTEND, SL, VT, Y), in visitFADDForFMACombine()
13157 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
13158 DAG.getNode(ISD::FP_EXTEND, SL, VT, U), in visitFADDForFMACombine()
13159 DAG.getNode(ISD::FP_EXTEND, SL, VT, V), Z)); in visitFADDForFMACombine()
13166 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFADDForFMACombine()
13182 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFADDForFMACombine()
13201 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFADDForFMACombine()
13218 EVT VT = N->getValueType(0); in visitFSUBForFMACombine() local
13227 TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT) && in visitFSUBForFMACombine()
13228 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFSUBForFMACombine()
13242 if (TLI.generateFMAsInMachineCombiner(VT, OptLevel)) in visitFSUBForFMACombine()
13247 bool Aggressive = TLI.enableAggressiveFMAFusion(VT); in visitFSUBForFMACombine()
13261 return DAG.getNode(PreferredFusedOpcode, SL, VT, XY.getOperand(0), in visitFSUBForFMACombine()
13262 XY.getOperand(1), DAG.getNode(ISD::FNEG, SL, VT, Z)); in visitFSUBForFMACombine()
13271 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
13272 DAG.getNode(ISD::FNEG, SL, VT, YZ.getOperand(0)), in visitFSUBForFMACombine()
13302 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
13303 DAG.getNode(ISD::FNEG, SL, VT, N00), N01, in visitFSUBForFMACombine()
13304 DAG.getNode(ISD::FNEG, SL, VT, N1)); in visitFSUBForFMACombine()
13314 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
13316 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
13317 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)), in visitFSUBForFMACombine()
13318 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)), in visitFSUBForFMACombine()
13319 DAG.getNode(ISD::FNEG, SL, VT, N1)); in visitFSUBForFMACombine()
13329 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
13332 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
13333 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
13334 DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(0))), in visitFSUBForFMACombine()
13335 DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)), N0); in visitFSUBForFMACombine()
13350 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
13353 ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
13354 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
13355 DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)), in visitFSUBForFMACombine()
13356 DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)), in visitFSUBForFMACombine()
13373 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
13376 ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
13377 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
13378 DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)), in visitFSUBForFMACombine()
13379 DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)), in visitFSUBForFMACombine()
13402 return DAG.getNode(PreferredFusedOpcode, SL, VT, N0.getOperand(0), in visitFSUBForFMACombine()
13404 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
13407 DAG.getNode(ISD::FNEG, SL, VT, N1))); in visitFSUBForFMACombine()
13418 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
13419 DAG.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)), N1.getOperand(1), in visitFSUBForFMACombine()
13420 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
13421 DAG.getNode(ISD::FNEG, SL, VT, N20), N21, N0)); in visitFSUBForFMACombine()
13432 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
13435 PreferredFusedOpcode, SL, VT, N0.getOperand(0), N0.getOperand(1), in visitFSUBForFMACombine()
13437 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
13438 DAG.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(0)), in visitFSUBForFMACombine()
13439 DAG.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(1)), in visitFSUBForFMACombine()
13440 DAG.getNode(ISD::FNEG, SL, VT, N1))); in visitFSUBForFMACombine()
13456 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
13459 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
13460 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)), in visitFSUBForFMACombine()
13461 DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)), in visitFSUBForFMACombine()
13463 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
13464 DAG.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(0)), in visitFSUBForFMACombine()
13465 DAG.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(1)), in visitFSUBForFMACombine()
13466 DAG.getNode(ISD::FNEG, SL, VT, N1))); in visitFSUBForFMACombine()
13478 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
13483 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
13484 DAG.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)), N1.getOperand(1), in visitFSUBForFMACombine()
13485 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
13486 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
13487 DAG.getNode(ISD::FP_EXTEND, SL, VT, N1200)), in visitFSUBForFMACombine()
13488 DAG.getNode(ISD::FP_EXTEND, SL, VT, N1201), N0)); in visitFSUBForFMACombine()
13505 TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT, in visitFSUBForFMACombine()
13510 PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
13511 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
13512 DAG.getNode(ISD::FP_EXTEND, SL, VT, N100)), in visitFSUBForFMACombine()
13513 DAG.getNode(ISD::FP_EXTEND, SL, VT, N101), in visitFSUBForFMACombine()
13514 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
13515 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
13516 DAG.getNode(ISD::FP_EXTEND, SL, VT, N1020)), in visitFSUBForFMACombine()
13517 DAG.getNode(ISD::FP_EXTEND, SL, VT, N1021), N0)); in visitFSUBForFMACombine()
13531 EVT VT = N->getValueType(0); in visitFMULForFMADistributiveCombine() local
13546 TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT) && in visitFMULForFMADistributiveCombine()
13547 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFMULForFMADistributiveCombine()
13560 bool Aggressive = TLI.enableAggressiveFMAFusion(VT); in visitFMULForFMADistributiveCombine()
13568 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y, in visitFMULForFMADistributiveCombine()
13571 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y, in visitFMULForFMADistributiveCombine()
13572 DAG.getNode(ISD::FNEG, SL, VT, Y)); in visitFMULForFMADistributiveCombine()
13591 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFMULForFMADistributiveCombine()
13592 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y, in visitFMULForFMADistributiveCombine()
13595 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFMULForFMADistributiveCombine()
13596 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y, in visitFMULForFMADistributiveCombine()
13597 DAG.getNode(ISD::FNEG, SL, VT, Y)); in visitFMULForFMADistributiveCombine()
13601 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y, in visitFMULForFMADistributiveCombine()
13602 DAG.getNode(ISD::FNEG, SL, VT, Y)); in visitFMULForFMADistributiveCombine()
13604 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y, in visitFMULForFMADistributiveCombine()
13624 EVT VT = N->getValueType(0); in visitFADD() local
13634 if (VT.isVector()) in visitFADD()
13640 return DAG.getNode(ISD::FADD, DL, VT, N0, N1); in visitFADD()
13644 return DAG.getNode(ISD::FADD, DL, VT, N1, N0); in visitFADD()
13656 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) in visitFADD()
13659 return DAG.getNode(ISD::FSUB, DL, VT, N0, NegN1); in visitFADD()
13662 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) in visitFADD()
13665 return DAG.getNode(ISD::FSUB, DL, VT, N1, NegN0); in visitFADD()
13677 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B); in visitFADD()
13678 return DAG.getNode(ISD::FSUB, DL, VT, N1, Add); in visitFADD()
13683 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B); in visitFADD()
13684 return DAG.getNode(ISD::FSUB, DL, VT, N0, Add); in visitFADD()
13695 return DAG.getConstantFP(0.0, DL, VT); in visitFADD()
13699 return DAG.getConstantFP(0.0, DL, VT); in visitFADD()
13711 SDValue NewC = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), N1); in visitFADD()
13712 return DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(0), NewC); in visitFADD()
13718 if (TLI.isOperationLegalOrCustom(ISD::FMUL, VT) && !N0CFP && !N1CFP) { in visitFADD()
13725 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), in visitFADD()
13726 DAG.getConstantFP(1.0, DL, VT)); in visitFADD()
13727 return DAG.getNode(ISD::FMUL, DL, VT, N1, NewCFP); in visitFADD()
13734 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), in visitFADD()
13735 DAG.getConstantFP(2.0, DL, VT)); in visitFADD()
13736 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), NewCFP); in visitFADD()
13746 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1), in visitFADD()
13747 DAG.getConstantFP(1.0, DL, VT)); in visitFADD()
13748 return DAG.getNode(ISD::FMUL, DL, VT, N0, NewCFP); in visitFADD()
13755 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1), in visitFADD()
13756 DAG.getConstantFP(2.0, DL, VT)); in visitFADD()
13757 return DAG.getNode(ISD::FMUL, DL, VT, N1.getOperand(0), NewCFP); in visitFADD()
13766 return DAG.getNode(ISD::FMUL, DL, VT, N1, in visitFADD()
13767 DAG.getConstantFP(3.0, DL, VT)); in visitFADD()
13776 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFADD()
13777 DAG.getConstantFP(3.0, DL, VT)); in visitFADD()
13786 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), in visitFADD()
13787 DAG.getConstantFP(4.0, DL, VT)); in visitFADD()
13804 EVT VT = N->getValueType(0); in visitSTRICT_FADD() local
13810 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::STRICT_FSUB, VT)) in visitSTRICT_FADD()
13813 return DAG.getNode(ISD::STRICT_FSUB, DL, DAG.getVTList(VT, ChainVT), in visitSTRICT_FADD()
13818 if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::STRICT_FSUB, VT)) in visitSTRICT_FADD()
13821 return DAG.getNode(ISD::STRICT_FSUB, DL, DAG.getVTList(VT, ChainVT), in visitSTRICT_FADD()
13832 EVT VT = N->getValueType(0); in visitFSUB() local
13842 if (VT.isVector()) in visitFSUB()
13848 return DAG.getNode(ISD::FSUB, DL, VT, N0, N1); in visitFSUB()
13864 return DAG.getConstantFP(0.0f, DL, VT); in visitFSUB()
13875 DenormalMode DenormMode = DAG.getDenormalMode(VT); in visitFSUB()
13880 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) in visitFSUB()
13881 return DAG.getNode(ISD::FNEG, DL, VT, N1); in visitFSUB()
13891 return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(1)); in visitFSUB()
13894 return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(0)); in visitFSUB()
13900 return DAG.getNode(ISD::FADD, DL, VT, N0, NegN1); in visitFSUB()
13916 EVT VT = N->getValueType(0); in visitFMUL() local
13926 if (VT.isVector()) { in visitFMUL()
13934 return DAG.getNode(ISD::FMUL, DL, VT, N0, N1); in visitFMUL()
13939 return DAG.getNode(ISD::FMUL, DL, VT, N1, N0); in visitFMUL()
13954 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, N01, N1); in visitFMUL()
13955 return DAG.getNode(ISD::FMUL, DL, VT, N00, MulConsts); in visitFMUL()
13963 const SDValue Two = DAG.getConstantFP(2.0, DL, VT); in visitFMUL()
13964 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, Two, N1); in visitFMUL()
13965 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), MulConsts); in visitFMUL()
13971 return DAG.getNode(ISD::FADD, DL, VT, N0, N0); in visitFMUL()
13975 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) in visitFMUL()
13976 return DAG.getNode(ISD::FNEG, DL, VT, N0); in visitFMUL()
13990 return DAG.getNode(ISD::FMUL, DL, VT, NegN0, NegN1); in visitFMUL()
13996 TLI.isOperationLegal(ISD::FABS, VT)) { in visitFMUL()
14027 TLI.isOperationLegal(ISD::FNEG, VT)) in visitFMUL()
14028 return DAG.getNode(ISD::FNEG, DL, VT, in visitFMUL()
14029 DAG.getNode(ISD::FABS, DL, VT, X)); in visitFMUL()
14031 return DAG.getNode(ISD::FABS, DL, VT, X); in visitFMUL()
14053 EVT VT = N->getValueType(0); in visitFMA() local
14066 return DAG.getNode(ISD::FMA, DL, VT, N0, N1, N2); in visitFMA()
14081 return DAG.getNode(ISD::FMA, DL, VT, NegN0, NegN1, N2); in visitFMA()
14091 return DAG.getNode(ISD::FADD, SDLoc(N), VT, N1, N2); in visitFMA()
14093 return DAG.getNode(ISD::FADD, SDLoc(N), VT, N0, N2); in visitFMA()
14098 return DAG.getNode(ISD::FMA, SDLoc(N), VT, N1, N0, N2); in visitFMA()
14105 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFMA()
14106 DAG.getNode(ISD::FADD, DL, VT, N1, N2.getOperand(1))); in visitFMA()
14113 return DAG.getNode(ISD::FMA, DL, VT, N0.getOperand(0), in visitFMA()
14114 DAG.getNode(ISD::FMUL, DL, VT, N1, N0.getOperand(1)), in visitFMA()
14122 return DAG.getNode(ISD::FADD, DL, VT, N0, N2); in visitFMA()
14125 (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))) { in visitFMA()
14126 SDValue RHSNeg = DAG.getNode(ISD::FNEG, DL, VT, N0); in visitFMA()
14128 return DAG.getNode(ISD::FADD, DL, VT, N2, RHSNeg); in visitFMA()
14133 (TLI.isOperationLegal(ISD::ConstantFP, VT) || in visitFMA()
14134 (N1.hasOneUse() && !TLI.isFPImmLegal(N1CFP->getValueAPF(), VT, in visitFMA()
14136 return DAG.getNode(ISD::FMA, DL, VT, N0.getOperand(0), in visitFMA()
14137 DAG.getNode(ISD::FNEG, DL, VT, N1), N2); in visitFMA()
14145 ISD::FMUL, DL, VT, N0, in visitFMA()
14146 DAG.getNode(ISD::FADD, DL, VT, N1, DAG.getConstantFP(1.0, DL, VT))); in visitFMA()
14152 ISD::FMUL, DL, VT, N0, in visitFMA()
14153 DAG.getNode(ISD::FADD, DL, VT, N1, DAG.getConstantFP(-1.0, DL, VT))); in visitFMA()
14159 if (!TLI.isFNegFree(VT)) in visitFMA()
14162 return DAG.getNode(ISD::FNEG, DL, VT, Neg); in visitFMA()
14195 EVT VT = N->getValueType(0); in combineRepeatedFPDivisors() local
14196 if (VT.isVector() && DAG.isSplatValue(N1)) in combineRepeatedFPDivisors()
14197 NumElts = VT.getVectorNumElements(); in combineRepeatedFPDivisors()
14227 SDValue FPOne = DAG.getConstantFP(1.0, DL, VT); in combineRepeatedFPDivisors()
14228 SDValue Reciprocal = DAG.getNode(ISD::FDIV, DL, VT, FPOne, N1, Flags); in combineRepeatedFPDivisors()
14234 SDValue NewNode = DAG.getNode(ISD::FMUL, SDLoc(U), VT, Dividend, in combineRepeatedFPDivisors()
14251 EVT VT = N->getValueType(0); in visitFDIV() local
14261 if (VT.isVector()) in visitFDIV()
14267 return DAG.getNode(ISD::FDIV, SDLoc(N), VT, N0, N1); in visitFDIV()
14289 TLI.isOperationLegal(ISD::ConstantFP, VT) || in visitFDIV()
14290 TLI.isFPImmLegal(Recip, VT, ForCodeSize))) in visitFDIV()
14291 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFDIV()
14292 DAG.getConstantFP(Recip, DL, VT)); in visitFDIV()
14299 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
14304 RV = DAG.getNode(ISD::FP_EXTEND, SDLoc(N1), VT, RV); in visitFDIV()
14306 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
14312 RV = DAG.getNode(ISD::FP_ROUND, SDLoc(N1), VT, RV, N1.getOperand(1)); in visitFDIV()
14314 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
14340 SDValue AA = DAG.getNode(ISD::FMUL, DL, VT, A, A); in visitFDIV()
14342 DAG.getNode(ISD::FMUL, DL, VT, AA, Sqrt.getOperand(0)); in visitFDIV()
14344 return DAG.getNode(ISD::FMUL, DL, VT, N0, Rsqrt); in visitFDIV()
14354 SDValue Div = DAG.getNode(ISD::FDIV, SDLoc(N1), VT, Rsqrt, Y); in visitFDIV()
14356 return DAG.getNode(ISD::FMUL, DL, VT, N0, Div); in visitFDIV()
14385 return DAG.getNode(ISD::FDIV, SDLoc(N), VT, NegN0, NegN1); in visitFDIV()
14395 EVT VT = N->getValueType(0); in visitFREM() local
14404 return DAG.getNode(ISD::FREM, SDLoc(N), VT, N0, N1); in visitFREM()
14468 EVT VT = N->getValueType(0); in visitFCOPYSIGN() local
14471 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1); in visitFCOPYSIGN()
14478 if (!LegalOperations || TLI.isOperationLegal(ISD::FABS, VT)) in visitFCOPYSIGN()
14479 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFCOPYSIGN()
14481 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) in visitFCOPYSIGN()
14482 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, in visitFCOPYSIGN()
14483 DAG.getNode(ISD::FABS, SDLoc(N0), VT, N0)); in visitFCOPYSIGN()
14492 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0.getOperand(0), N1); in visitFCOPYSIGN()
14496 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFCOPYSIGN()
14500 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1.getOperand(1)); in visitFCOPYSIGN()
14505 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1.getOperand(0)); in visitFCOPYSIGN()
14520 EVT VT = N->getValueType(0); in visitFPOW() local
14521 if ((VT == MVT::f32 && ExponentC->getValueAPF().isExactlyValue(1.0f/3.0f)) || in visitFPOW()
14522 (VT == MVT::f64 && ExponentC->getValueAPF().isExactlyValue(1.0/3.0))) { in visitFPOW()
14537 (!DAG.getTargetLoweringInfo().isOperationExpand(ISD::FPOW, VT) && in visitFPOW()
14538 DAG.getTargetLoweringInfo().isOperationExpand(ISD::FCBRT, VT))) in visitFPOW()
14541 return DAG.getNode(ISD::FCBRT, SDLoc(N), VT, N->getOperand(0)); in visitFPOW()
14566 if (!DAG.getTargetLoweringInfo().isOperationLegalOrCustom(ISD::FSQRT, VT)) in visitFPOW()
14576 SDValue Sqrt = DAG.getNode(ISD::FSQRT, DL, VT, N->getOperand(0)); in visitFPOW()
14577 SDValue SqrtSqrt = DAG.getNode(ISD::FSQRT, DL, VT, Sqrt); in visitFPOW()
14581 return DAG.getNode(ISD::FMUL, DL, VT, Sqrt, SqrtSqrt); in visitFPOW()
14603 EVT VT = N->getValueType(0); in foldFPToIntToFP() local
14604 if (!TLI.isOperationLegal(ISD::FTRUNC, VT) || in foldFPToIntToFP()
14612 N0.getOperand(0).getValueType() == VT) in foldFPToIntToFP()
14613 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0.getOperand(0)); in foldFPToIntToFP()
14616 N0.getOperand(0).getValueType() == VT) in foldFPToIntToFP()
14617 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0.getOperand(0)); in foldFPToIntToFP()
14624 EVT VT = N->getValueType(0); in visitSINT_TO_FP() local
14629 return DAG.getConstantFP(0.0, SDLoc(N), VT); in visitSINT_TO_FP()
14635 TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) in visitSINT_TO_FP()
14636 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); in visitSINT_TO_FP()
14644 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); in visitSINT_TO_FP()
14650 !VT.isVector() && in visitSINT_TO_FP()
14651 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitSINT_TO_FP()
14653 return DAG.getSelect(DL, VT, N0, DAG.getConstantFP(-1.0, DL, VT), in visitSINT_TO_FP()
14654 DAG.getConstantFP(0.0, DL, VT)); in visitSINT_TO_FP()
14660 N0.getOperand(0).getOpcode() == ISD::SETCC && !VT.isVector() && in visitSINT_TO_FP()
14661 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitSINT_TO_FP()
14663 return DAG.getSelect(DL, VT, N0.getOperand(0), in visitSINT_TO_FP()
14664 DAG.getConstantFP(1.0, DL, VT), in visitSINT_TO_FP()
14665 DAG.getConstantFP(0.0, DL, VT)); in visitSINT_TO_FP()
14676 EVT VT = N->getValueType(0); in visitUINT_TO_FP() local
14681 return DAG.getConstantFP(0.0, SDLoc(N), VT); in visitUINT_TO_FP()
14687 TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) in visitUINT_TO_FP()
14688 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); in visitUINT_TO_FP()
14696 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); in visitUINT_TO_FP()
14700 if (N0.getOpcode() == ISD::SETCC && !VT.isVector() && in visitUINT_TO_FP()
14701 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitUINT_TO_FP()
14703 return DAG.getSelect(DL, VT, N0, DAG.getConstantFP(1.0, DL, VT), in visitUINT_TO_FP()
14704 DAG.getConstantFP(0.0, DL, VT)); in visitUINT_TO_FP()
14716 EVT VT = N->getValueType(0); in FoldIntToFPToInt() local
14736 unsigned OutputSize = (int)VT.getScalarSizeInBits() - IsOutputSigned; in FoldIntToFPToInt()
14743 if (VT.getScalarSizeInBits() > SrcVT.getScalarSizeInBits()) { in FoldIntToFPToInt()
14746 return DAG.getNode(ExtOp, SDLoc(N), VT, Src); in FoldIntToFPToInt()
14748 if (VT.getScalarSizeInBits() < SrcVT.getScalarSizeInBits()) in FoldIntToFPToInt()
14749 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Src); in FoldIntToFPToInt()
14750 return DAG.getBitcast(VT, Src); in FoldIntToFPToInt()
14757 EVT VT = N->getValueType(0); in visitFP_TO_SINT() local
14761 return DAG.getUNDEF(VT); in visitFP_TO_SINT()
14765 return DAG.getNode(ISD::FP_TO_SINT, SDLoc(N), VT, N0); in visitFP_TO_SINT()
14772 EVT VT = N->getValueType(0); in visitFP_TO_UINT() local
14776 return DAG.getUNDEF(VT); in visitFP_TO_UINT()
14780 return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), VT, N0); in visitFP_TO_UINT()
14789 EVT VT = N->getValueType(0); in visitFP_ROUND() local
14793 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, N0, N1); in visitFP_ROUND()
14796 if (N0.getOpcode() == ISD::FP_EXTEND && VT == N0.getOperand(0).getValueType()) in visitFP_ROUND()
14811 if (N0.getOperand(0).getValueType() == MVT::f80 && VT == MVT::f16) in visitFP_ROUND()
14821 return DAG.getNode(ISD::FP_ROUND, DL, VT, N0.getOperand(0), in visitFP_ROUND()
14828 SDValue Tmp = DAG.getNode(ISD::FP_ROUND, SDLoc(N0), VT, in visitFP_ROUND()
14831 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, in visitFP_ROUND()
14843 EVT VT = N->getValueType(0); in visitFP_EXTEND() local
14852 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, N0); in visitFP_EXTEND()
14856 TLI.getOperationAction(ISD::FP16_TO_FP, VT) == TargetLowering::Legal) in visitFP_EXTEND()
14857 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), VT, N0.getOperand(0)); in visitFP_EXTEND()
14864 if (In.getValueType() == VT) return In; in visitFP_EXTEND()
14865 if (VT.bitsLT(In.getValueType())) in visitFP_EXTEND()
14866 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, in visitFP_EXTEND()
14868 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, In); in visitFP_EXTEND()
14873 TLI.isLoadExtLegal(ISD::EXTLOAD, VT, N0.getValueType())) { in visitFP_EXTEND()
14875 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, in visitFP_EXTEND()
14896 EVT VT = N->getValueType(0); in visitFCEIL() local
14900 return DAG.getNode(ISD::FCEIL, SDLoc(N), VT, N0); in visitFCEIL()
14907 EVT VT = N->getValueType(0); in visitFTRUNC() local
14911 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0); in visitFTRUNC()
14931 EVT VT = N->getValueType(0); in visitFFLOOR() local
14935 return DAG.getNode(ISD::FFLOOR, SDLoc(N), VT, N0); in visitFFLOOR()
14942 EVT VT = N->getValueType(0); in visitFNEG() local
14947 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0); in visitFNEG()
14960 return DAG.getNode(ISD::FSUB, SDLoc(N), VT, N0.getOperand(1), in visitFNEG()
14974 EVT VT = N->getValueType(0); in visitFMinMax() local
14986 return DAG.getConstantFP(Op(C0, C1), SDLoc(N), VT); in visitFMinMax()
14992 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0); in visitFMinMax()
15044 EVT VT = N->getValueType(0); in visitFABS() local
15048 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFABS()
15057 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0.getOperand(0)); in visitFABS()
15236 EVT VT = LD->getMemoryVT(); in getCombineLoadStoreParts() local
15237 if (!TLI.isIndexedLoadLegal(Inc, VT) && !TLI.isIndexedLoadLegal(Dec, VT)) in getCombineLoadStoreParts()
15243 EVT VT = ST->getMemoryVT(); in getCombineLoadStoreParts() local
15244 if (!TLI.isIndexedStoreLegal(Inc, VT) && !TLI.isIndexedStoreLegal(Dec, VT)) in getCombineLoadStoreParts()
15251 EVT VT = LD->getMemoryVT(); in getCombineLoadStoreParts() local
15252 if (!TLI.isIndexedMaskedLoadLegal(Inc, VT) && in getCombineLoadStoreParts()
15253 !TLI.isIndexedMaskedLoadLegal(Dec, VT)) in getCombineLoadStoreParts()
15260 EVT VT = ST->getMemoryVT(); in getCombineLoadStoreParts() local
15261 if (!TLI.isIndexedMaskedStoreLegal(Inc, VT) && in getCombineLoadStoreParts()
15262 !TLI.isIndexedMaskedStoreLegal(Dec, VT)) in getCombineLoadStoreParts()
16597 MVT VT = MVT::getIntegerVT(NumBytes * 8); in ShrinkLoadReplaceStoreWithStore() local
16599 if (!DC->isTypeLegal(VT)) in ShrinkLoadReplaceStoreWithStore()
16602 !TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, in ShrinkLoadReplaceStoreWithStore()
16629 IVal = DAG.getNode(ISD::TRUNCATE, SDLoc(IVal), VT, IVal); in ShrinkLoadReplaceStoreWithStore()
16650 EVT VT = Value.getValueType(); in ReduceLoadOpStoreWidth() local
16652 if (ST->isTruncatingStore() || VT.isVector() || !Value.hasOneUse()) in ReduceLoadOpStoreWidth()
16711 !TLI.isNarrowingProfitable(VT, NewVT))) { in ReduceLoadOpStoreWidth()
16774 EVT VT = LD->getMemoryVT(); in TransformFPLoadStorePair() local
16775 if (!VT.isFloatingPoint() || in TransformFPLoadStorePair()
16776 VT != ST->getMemoryVT() || in TransformFPLoadStorePair()
16783 TypeSize VTSize = VT.getSizeInBits(); in TransformFPLoadStorePair()
16793 !TLI.isDesirableToTransformToIntegerOp(ISD::LOAD, VT) || in TransformFPLoadStorePair()
16794 !TLI.isDesirableToTransformToIntegerOp(ISD::STORE, VT)) in TransformFPLoadStorePair()
18337 EVT VT = EVT::getIntegerVT(*DAG.getContext(), HalfValBitSize); in splitMergedValStore() local
18338 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Lo.getOperand(0)); in splitMergedValStore()
18339 Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Hi.getOperand(0)); in splitMergedValStore()
18441 EVT VT = DestVec.getValueType(); in combineInsertEltToShuffle() local
18447 unsigned ExtendRatio = VT.getSizeInBits() / SubVecVT.getSizeInBits(); in combineInsertEltToShuffle()
18482 return DAG.getBitcast(VT, Shuf); in combineInsertEltToShuffle()
18491 EVT VT = InVec.getValueType(); in visitINSERT_VECTOR_ELT() local
18495 if (IndexC && VT.isFixedLengthVector() && in visitINSERT_VECTOR_ELT()
18496 IndexC->getZExtValue() >= VT.getVectorNumElements()) in visitINSERT_VECTOR_ELT()
18497 return DAG.getUNDEF(VT); in visitINSERT_VECTOR_ELT()
18508 if (InVec.isUndef() && TLI.shouldSplatInsEltVarIndex(VT)) { in visitINSERT_VECTOR_ELT()
18509 if (VT.isScalableVector()) in visitINSERT_VECTOR_ELT()
18510 return DAG.getSplatVector(VT, DL, InVal); in visitINSERT_VECTOR_ELT()
18512 SmallVector<SDValue, 8> Ops(VT.getVectorNumElements(), InVal); in visitINSERT_VECTOR_ELT()
18513 return DAG.getBuildVector(VT, DL, Ops); in visitINSERT_VECTOR_ELT()
18519 if (VT.isScalableVector()) in visitINSERT_VECTOR_ELT()
18522 unsigned NumElts = VT.getVectorNumElements(); in visitINSERT_VECTOR_ELT()
18541 SDValue NewOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, in visitINSERT_VECTOR_ELT()
18545 VT, NewOp, InVec.getOperand(1), InVec.getOperand(2)); in visitINSERT_VECTOR_ELT()
18550 if (LegalOperations && !TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) in visitINSERT_VECTOR_ELT()
18578 return DAG.getBuildVector(VT, DL, Ops); in visitINSERT_VECTOR_ELT()
18693 EVT VT = ExtElt->getValueType(0); in scalarizeExtractedBinop() local
18694 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Index); in scalarizeExtractedBinop()
18695 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op1, Index); in scalarizeExtractedBinop()
18696 return DAG.getNode(Vec.getOpcode(), DL, VT, Ext0, Ext1); in scalarizeExtractedBinop()
19023 EVT VT = N->getValueType(0); in reduceBuildVecExtToExtBuildVec() local
19067 EVT OutScalarTy = VT.getScalarType(); in reduceBuildVecExtToExtBuildVec()
19088 unsigned NewBVElems = ElemRatio * VT.getVectorNumElements(); in reduceBuildVecExtToExtBuildVec()
19111 assert(VecVT.getSizeInBits() == VT.getSizeInBits() && in reduceBuildVecExtToExtBuildVec()
19116 TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))) in reduceBuildVecExtToExtBuildVec()
19125 return DAG.getBitcast(VT, BV); in reduceBuildVecExtToExtBuildVec()
19140 EVT VT = N->getValueType(0); in reduceBuildVecTruncToBitCast() local
19141 EVT OutScalarTy = VT.getScalarType(); in reduceBuildVecTruncToBitCast()
19200 if (Src.getValueType().getSizeInBits() != VT.getSizeInBits()) in reduceBuildVecTruncToBitCast()
19203 return DAG.getBitcast(VT, Src); in reduceBuildVecTruncToBitCast()
19212 EVT VT = N->getValueType(0); in createBuildVecShuffle() local
19216 unsigned NumElems = VT.getVectorNumElements(); in createBuildVecShuffle()
19223 uint64_t VTSize = VT.getFixedSizeInBits(); in createBuildVecShuffle()
19232 if (InVT1 != VT || InVT2 != VT) { in createBuildVecShuffle()
19241 VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in createBuildVecShuffle()
19244 if (!TLI.isExtractSubvectorCheap(VT, InVT1, NumElems)) in createBuildVecShuffle()
19250 VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1, in createBuildVecShuffle()
19252 VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1, ZeroIdx); in createBuildVecShuffle()
19282 VecIn2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in createBuildVecShuffle()
19320 Shuffle = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuffle, ZeroIdx); in createBuildVecShuffle()
19351 EVT VT = BV->getValueType(0); in reduceBuildVecToShuffleWithZero() local
19356 Zext.getValueSizeInBits() != VT.getScalarSizeInBits()) in reduceBuildVecToShuffleWithZero()
19365 Extract.getOperand(0).getValueSizeInBits() != VT.getSizeInBits()) in reduceBuildVecToShuffleWithZero()
19399 return DAG.getBitcast(VT, Shuf); in reduceBuildVecToShuffleWithZero()
19416 EVT VT = N->getValueType(0); in reduceBuildVecToShuffle() local
19419 if (!isTypeLegal(VT)) in reduceBuildVecToShuffle()
19426 if (LegalOperations && !TLI.isOperationLegal(ISD::VECTOR_SHUFFLE, VT)) in reduceBuildVecToShuffle()
19472 if (VT.getVectorElementType() != in reduceBuildVecToShuffle()
19594 Shuffles.push_back(VT.isInteger() ? DAG.getConstant(0, DL, VT) in reduceBuildVecToShuffle()
19595 : DAG.getConstantFP(0.0, DL, VT)); in reduceBuildVecToShuffle()
19621 Shuffles.push_back(DAG.getUNDEF(VT)); in reduceBuildVecToShuffle()
19625 Shuffles[CurSize] = DAG.getUNDEF(VT); in reduceBuildVecToShuffle()
19643 DAG.getVectorShuffle(VT, DL, Shuffles[Left], Shuffles[Right], Mask); in reduceBuildVecToShuffle()
19657 EVT VT = N->getValueType(0); in convertBuildVecZextToZext() local
19697 VT, In); in convertBuildVecZextToZext()
19701 EVT VT = N->getValueType(0); in visitBUILD_VECTOR() local
19705 return DAG.getUNDEF(VT); in visitBUILD_VECTOR()
19727 return DAG.getBitcast(VT, Concat); in visitBUILD_VECTOR()
19776 if (TLI.getOperationAction(ISD::SPLAT_VECTOR, VT) != TargetLowering::Expand) in visitBUILD_VECTOR()
19779 return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), VT, V); in visitBUILD_VECTOR()
19794 EVT VT = N->getValueType(0); in combineConcatVectorOfScalars() local
19842 VT.getSizeInBits() / SVT.getSizeInBits()); in combineConcatVectorOfScalars()
19843 return DAG.getBitcast(VT, DAG.getBuildVector(VecVT, DL, Ops)); in combineConcatVectorOfScalars()
19851 EVT VT = N->getValueType(0); in combineConcatVectorOfExtracts() local
19855 if (VT.isScalableVector()) in combineConcatVectorOfExtracts()
19858 int NumElts = VT.getVectorNumElements(); in combineConcatVectorOfExtracts()
19861 SDValue SV0 = DAG.getUNDEF(VT), SV1 = DAG.getUNDEF(VT); in combineConcatVectorOfExtracts()
19893 if (ExtVT.getSizeInBits() != VT.getSizeInBits()) in combineConcatVectorOfExtracts()
19920 return TLI.buildLegalVectorShuffle(VT, SDLoc(N), DAG.getBitcast(VT, SV0), in combineConcatVectorOfExtracts()
19921 DAG.getBitcast(VT, SV1), Mask, DAG); in combineConcatVectorOfExtracts()
19959 EVT VT = N->getValueType(0); in combineConcatVectorOfCasts() local
19968 !TLI.isTypeLegal(VT)) in combineConcatVectorOfCasts()
19973 if (!TLI.isOperationLegalOrCustom(CastOpcode, VT) || in combineConcatVectorOfCasts()
19984 return DAG.getNode(CastOpcode, DL, VT, NewConcat); in combineConcatVectorOfCasts()
19993 EVT VT = N->getValueType(0); in visitCONCAT_VECTORS() local
19995 return DAG.getUNDEF(VT); in visitCONCAT_VECTORS()
20009 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in visitCONCAT_VECTORS()
20039 if (VT.getSizeInBits() % SclTy.getSizeInBits()) in visitCONCAT_VECTORS()
20042 unsigned VNTNumElms = VT.getSizeInBits() / SclTy.getSizeInBits(); in visitCONCAT_VECTORS()
20051 return DAG.getBitcast(VT, Res); in visitCONCAT_VECTORS()
20064 EVT SVT = VT.getScalarType(); in visitCONCAT_VECTORS()
20099 assert(VT.getVectorNumElements() == Opnds.size() && in visitCONCAT_VECTORS()
20101 return DAG.getBuildVector(VT, SDLoc(N), Opnds); in visitCONCAT_VECTORS()
20109 if (Level < AfterLegalizeVectorOps && TLI.isTypeLegal(VT)) in visitCONCAT_VECTORS()
20254 EVT VT = Extract->getValueType(0); in narrowExtractedVectorBinOp() local
20256 assert(ExtractIndex % VT.getVectorNumElements() == 0 && in narrowExtractedVectorBinOp()
20261 unsigned NarrowWidth = VT.getSizeInBits(); in narrowExtractedVectorBinOp()
20282 unsigned ConcatOpNum = ExtractIndex / VT.getVectorNumElements(); in narrowExtractedVectorBinOp()
20295 return DAG.getBitcast(VT, NarrowBinOp); in narrowExtractedVectorBinOp()
20338 return DAG.getBitcast(VT, NarrowBinOp); in narrowExtractedVectorBinOp()
20359 EVT VT = Extract->getValueType(0); in narrowExtractedVectorLoad() local
20362 if (!VT.isByteSized()) in narrowExtractedVectorLoad()
20366 unsigned NumElts = VT.getVectorMinNumElements(); in narrowExtractedVectorLoad()
20374 TypeSize Offset = VT.getStoreSize() * (Index / NumElts); in narrowExtractedVectorLoad()
20377 if (!TLI.shouldReduceLoadWidth(Ld, Ld->getExtensionType(), VT)) in narrowExtractedVectorLoad()
20387 uint64_t StoreSize = MemoryLocation::getSizeOrUnknown(VT.getStoreSize()); in narrowExtractedVectorLoad()
20398 SDValue NewLd = DAG.getLoad(VT, DL, Ld->getChain(), NewAddr, MMO); in narrowExtractedVectorLoad()
20600 EVT VT = Shuf->getValueType(0); in foldShuffleOfConcatUndefs() local
20601 unsigned NumElts = VT.getVectorNumElements(); in foldShuffleOfConcatUndefs()
20620 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), in foldShuffleOfConcatUndefs()
20632 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Shuf0, Shuf1); in foldShuffleOfConcatUndefs()
20638 EVT VT = N->getValueType(0); in partitionShuffleOfConcats() local
20639 unsigned NumElts = VT.getVectorNumElements(); in partitionShuffleOfConcats()
20663 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, N0, N1); in partitionShuffleOfConcats()
20697 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in partitionShuffleOfConcats()
20719 EVT VT = SVN->getValueType(0); in combineShuffleOfScalars() local
20720 unsigned NumElts = VT.getVectorNumElements(); in combineShuffleOfScalars()
20753 SDValue Op = DAG.getUNDEF(VT.getScalarType()); in combineShuffleOfScalars()
20781 EVT SVT = VT.getScalarType(); in combineShuffleOfScalars()
20785 if (SVT != VT.getScalarType()) in combineShuffleOfScalars()
20790 return DAG.getBuildVector(VT, SDLoc(SVN), Ops); in combineShuffleOfScalars()
20801 EVT VT = SVN->getValueType(0); in combineShuffleToVectorExtend() local
20805 if (!VT.isInteger() || IsBigEndian) in combineShuffleToVectorExtend()
20808 unsigned NumElts = VT.getVectorNumElements(); in combineShuffleToVectorExtend()
20809 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in combineShuffleToVectorExtend()
20841 return DAG.getBitcast(VT, in combineShuffleToVectorExtend()
20856 EVT VT = SVN->getValueType(0); in combineTruncationShuffle() local
20860 if (!VT.isInteger() || IsBigEndian) in combineTruncationShuffle()
20873 unsigned NumElts = VT.getVectorNumElements(); in combineTruncationShuffle()
20874 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in combineTruncationShuffle()
20905 return DAG.getBitcast(VT, N00); in combineTruncationShuffle()
21007 EVT VT = OuterShuf->getValueType(0); in formSplatFromShuffles() local
21008 assert(VT == InnerShuf->getValueType(0) && "Expected matching shuffle types"); in formSplatFromShuffles()
21009 if (!DAG.getTargetLoweringInfo().isShuffleMaskLegal(CombinedMask, VT)) in formSplatFromShuffles()
21012 return DAG.getVectorShuffle(VT, SDLoc(OuterShuf), InnerShuf->getOperand(0), in formSplatFromShuffles()
21122 EVT VT = N->getValueType(0); in visitVECTOR_SHUFFLE() local
21123 unsigned NumElts = VT.getVectorNumElements(); in visitVECTOR_SHUFFLE()
21128 assert(N0.getValueType() == VT && "Vector shuffle must be normalized in DAG"); in visitVECTOR_SHUFFLE()
21132 return DAG.getUNDEF(VT); in visitVECTOR_SHUFFLE()
21144 return DAG.getVectorShuffle(VT, SDLoc(N), N0, DAG.getUNDEF(VT), NewMask); in visitVECTOR_SHUFFLE()
21164 return DAG.getVectorShuffle(VT, SDLoc(N), N0, N1, NewMask); in visitVECTOR_SHUFFLE()
21181 if (N0.hasOneUse() && TLI.isExtractVecEltCheap(VT, SplatIndex) && in visitVECTOR_SHUFFLE()
21187 EVT EltVT = VT.getScalarType(); in visitVECTOR_SHUFFLE()
21193 SDValue Insert = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, NewBO); in visitVECTOR_SHUFFLE()
21194 SmallVector<int, 16> ZeroMask(VT.getVectorNumElements(), 0); in visitVECTOR_SHUFFLE()
21195 return DAG.getVectorShuffle(VT, DL, Insert, DAG.getUNDEF(VT), ZeroMask); in visitVECTOR_SHUFFLE()
21240 if (V->getValueType(0) != VT) in visitVECTOR_SHUFFLE()
21241 NewBV = DAG.getBitcast(VT, NewBV); in visitVECTOR_SHUFFLE()
21288 if (TLI.isShuffleMaskLegal(NewMask, VT)) { in visitVECTOR_SHUFFLE()
21290 SDValue NewCat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, in visitVECTOR_SHUFFLE()
21292 return DAG.getVectorShuffle(VT, SDLoc(N), NewCat, N1, NewMask); in visitVECTOR_SHUFFLE()
21298 if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT)) in visitVECTOR_SHUFFLE()
21307 TLI.isTypeLegal(VT)) { in visitVECTOR_SHUFFLE()
21311 EVT SVT = VT.getScalarType(); in visitVECTOR_SHUFFLE()
21316 EVT ScaleVT = SVT.bitsLT(InnerSVT) ? VT : InnerVT; in visitVECTOR_SHUFFLE()
21351 VT, DAG.getVectorShuffle(ScaleVT, SDLoc(N), SV0, SV1, NewMask)); in visitVECTOR_SHUFFLE()
21362 [NumElts, &VT](bool Commute, ShuffleVectorSDNode *SVN, in visitVECTOR_SHUFFLE()
21466 if (TLI.isShuffleMaskLegal(Mask, VT)) in visitVECTOR_SHUFFLE()
21471 return TLI.isShuffleMaskLegal(Mask, VT); in visitVECTOR_SHUFFLE()
21474 if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT)) { in visitVECTOR_SHUFFLE()
21483 assert(N1->getOperand(0).getValueType() == VT && in visitVECTOR_SHUFFLE()
21517 assert(OtherSV->getOperand(0).getValueType() == VT && in visitVECTOR_SHUFFLE()
21526 return DAG.getUNDEF(VT); in visitVECTOR_SHUFFLE()
21528 return DAG.getVectorShuffle(VT, SDLoc(N), in visitVECTOR_SHUFFLE()
21529 SV0 ? SV0 : DAG.getUNDEF(VT), in visitVECTOR_SHUFFLE()
21530 SV1 ? SV1 : DAG.getUNDEF(VT), Mask); in visitVECTOR_SHUFFLE()
21551 if (Op00.getValueType() == VT && Op10.getValueType() == VT && in visitVECTOR_SHUFFLE()
21552 Op01.getValueType() == VT && Op11.getValueType() == VT && in visitVECTOR_SHUFFLE()
21602 VT, DL, LeftSV0 ? LeftSV0 : DAG.getUNDEF(VT), in visitVECTOR_SHUFFLE()
21603 LeftSV1 ? LeftSV1 : DAG.getUNDEF(VT), LeftMask); in visitVECTOR_SHUFFLE()
21605 VT, DL, RightSV0 ? RightSV0 : DAG.getUNDEF(VT), in visitVECTOR_SHUFFLE()
21606 RightSV1 ? RightSV1 : DAG.getUNDEF(VT), RightMask); in visitVECTOR_SHUFFLE()
21607 return DAG.getNode(SrcOpcode, DL, VT, LHS, RHS); in visitVECTOR_SHUFFLE()
21621 EVT VT = N->getValueType(0); in visitSCALAR_TO_VECTOR() local
21626 VT.isFixedLengthVector() && in visitSCALAR_TO_VECTOR()
21637 if (VT.getScalarType() != InVal.getValueType() && in visitSCALAR_TO_VECTOR()
21639 isTypeLegal(VT.getScalarType())) { in visitSCALAR_TO_VECTOR()
21641 DAG.getNode(ISD::TRUNCATE, SDLoc(InVal), VT.getScalarType(), InVal); in visitSCALAR_TO_VECTOR()
21642 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Val); in visitSCALAR_TO_VECTOR()
21644 if (VT.getScalarType() == InVecT.getScalarType() && in visitSCALAR_TO_VECTOR()
21645 VT.getVectorNumElements() <= InVecT.getVectorNumElements()) { in visitSCALAR_TO_VECTOR()
21652 if (VT == InVecT) in visitSCALAR_TO_VECTOR()
21655 if (VT.getVectorNumElements() != InVecT.getVectorNumElements()) { in visitSCALAR_TO_VECTOR()
21659 VT.getVectorNumElements()); in visitSCALAR_TO_VECTOR()
21672 EVT VT = N->getValueType(0); in visitINSERT_SUBVECTOR() local
21685 N1.getOperand(1) == N2 && N1.getOperand(0).getValueType() == VT) in visitINSERT_SUBVECTOR()
21696 VT.getVectorElementCount() && in visitINSERT_SUBVECTOR()
21698 VT.getSizeInBits()) { in visitINSERT_SUBVECTOR()
21699 return DAG.getBitcast(VT, N1.getOperand(0).getOperand(0)); in visitINSERT_SUBVECTOR()
21713 CN0VT.getVectorElementCount() == VT.getVectorElementCount()) { in visitINSERT_SUBVECTOR()
21716 return DAG.getBitcast(VT, NewINSERT); in visitINSERT_SUBVECTOR()
21726 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0.getOperand(0), in visitINSERT_SUBVECTOR()
21734 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0, in visitINSERT_SUBVECTOR()
21752 ElementCount NumElts = VT.getVectorElementCount(); in visitINSERT_SUBVECTOR()
21753 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in visitINSERT_SUBVECTOR()
21769 return DAG.getBitcast(VT, Res); in visitINSERT_SUBVECTOR()
21783 SDValue NewOp = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, in visitINSERT_SUBVECTOR()
21787 VT, NewOp, N0.getOperand(1), N0.getOperand(2)); in visitINSERT_SUBVECTOR()
21800 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in visitINSERT_SUBVECTOR()
21837 EVT VT = N0.getValueType(); in visitVECREDUCE() local
21841 if (VT.getVectorElementCount().isScalar()) { in visitVECREDUCE()
21844 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getVectorElementType(), N0, in visitVECREDUCE()
21856 if (!TLI.isOperationLegalOrCustom(Opcode, VT) && in visitVECREDUCE()
21857 TLI.isOperationLegalOrCustom(NewOpcode, VT) && in visitVECREDUCE()
21858 DAG.ComputeNumSignBits(N0) == VT.getScalarSizeInBits()) in visitVECREDUCE()
21872 EVT VT = N->getValueType(0); in XformToShuffleWithZero() local
21936 return DAG.getBitcast(VT, DAG.getVectorShuffle(ClearVT, DL, in XformToShuffleWithZero()
21960 EVT VT = N->getValueType(0); in scalarizeBinOpOfSplats() local
21961 EVT EltVT = VT.getVectorElementType(); in scalarizeBinOpOfSplats()
21973 !TLI.isExtractVecEltCheap(VT, Index0) || in scalarizeBinOpOfSplats()
21990 SmallVector<SDValue, 8> Ops(VT.getVectorNumElements(), DAG.getUNDEF(EltVT)); in scalarizeBinOpOfSplats()
21992 return DAG.getBuildVector(VT, DL, Ops); in scalarizeBinOpOfSplats()
21996 SmallVector<SDValue, 8> Ops(VT.getVectorNumElements(), ScalarBO); in scalarizeBinOpOfSplats()
21997 return DAG.getBuildVector(VT, DL, Ops); in scalarizeBinOpOfSplats()
22008 EVT VT = N->getValueType(0); in SimplifyVBinOp() local
22033 SDValue NewBinOp = DAG.getNode(Opcode, DL, VT, LHS.getOperand(0), in SimplifyVBinOp()
22036 return DAG.getVectorShuffle(VT, DL, NewBinOp, UndefV, Shuf0->getMask()); in SimplifyVBinOp()
22051 SDValue NewBinOp = DAG.getNode(Opcode, DL, VT, X, RHS, Flags); in SimplifyVBinOp()
22052 return DAG.getVectorShuffle(VT, DL, NewBinOp, DAG.getUNDEF(VT), in SimplifyVBinOp()
22061 SDValue NewBinOp = DAG.getNode(Opcode, DL, VT, LHS, X, Flags); in SimplifyVBinOp()
22062 return DAG.getVectorShuffle(VT, DL, NewBinOp, DAG.getUNDEF(VT), in SimplifyVBinOp()
22085 DAG.getNode(Opcode, DL, VT, DAG.getUNDEF(VT), DAG.getUNDEF(VT)); in SimplifyVBinOp()
22087 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, VecC, NarrowBO, Z); in SimplifyVBinOp()
22119 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in SimplifyVBinOp()
22430 EVT VT = N->getValueType(0); in foldSelectOfBinops() local
22444 DAG.getSelect(DL, VT, N0, N1.getOperand(0), N2.getOperand(0)); in foldSelectOfBinops()
22445 SDValue NewBinOp = DAG.getNode(BinOpc, DL, VT, NewSel, N1.getOperand(1)); in foldSelectOfBinops()
22455 VT == N1.getOperand(1).getValueType() && in foldSelectOfBinops()
22456 VT == N2.getOperand(1).getValueType()) { in foldSelectOfBinops()
22458 DAG.getSelect(DL, VT, N0, N1.getOperand(1), N2.getOperand(1)); in foldSelectOfBinops()
22459 SDValue NewBinOp = DAG.getNode(BinOpc, DL, VT, N1.getOperand(0), NewSel); in foldSelectOfBinops()
22472 EVT VT = N->getValueType(0); in foldSignChangeInBitcast() local
22474 bool IsFree = IsFabs ? TLI.isFAbsFree(VT) : TLI.isFNegFree(VT); in foldSignChangeInBitcast()
22506 return DAG.getBitcast(VT, Int); in foldSignChangeInBitcast()
22523 EVT VT = N2.getValueType(); in convertSelectOfFPConstantsToLoadOffset() local
22524 if (!TV || !FV || !TLI.isTypeLegal(VT)) in convertSelectOfFPConstantsToLoadOffset()
22528 if (TLI.getOperationAction(ISD::ConstantFP, VT) == TargetLowering::Legal || in convertSelectOfFPConstantsToLoadOffset()
22576 EVT VT = N2.getValueType(); in SimplifySelectCC() local
22605 N0->getValueType(0) == VT && isNullConstant(N1) && isNullConstant(N2)) { in SimplifySelectCC()
22612 if (!TLI.shouldAvoidTransformToShift(VT, ShCt)) { in SimplifySelectCC()
22616 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N0), VT, AndLHS, ShlAmt); in SimplifySelectCC()
22623 SDValue Shr = DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl, ShrAmt); in SimplifySelectCC()
22625 return DAG.getNode(ISD::AND, DL, VT, Shr, N3); in SimplifySelectCC()
22653 if (VT.bitsLT(SCC.getValueType())) in SimplifySelectCC()
22654 Temp = DAG.getZeroExtendInReg(SCC, SDLoc(N2), VT); in SimplifySelectCC()
22656 Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2), VT, SCC); in SimplifySelectCC()
22659 Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2), VT, SCC); in SimplifySelectCC()
22669 if (TLI.shouldAvoidTransformToShift(VT, ShCt)) in SimplifySelectCC()
22694 if (ValueOnZeroC->getAPIntValue() == VT.getSizeInBits()) { in SimplifySelectCC()
22700 (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ, VT))) in SimplifySelectCC()
22701 return DAG.getNode(ISD::CTTZ, DL, VT, N0); in SimplifySelectCC()
22707 (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ, VT))) in SimplifySelectCC()
22708 return DAG.getNode(ISD::CTLZ, DL, VT, N0); in SimplifySelectCC()
22717 SDValue DAGCombiner::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, in SimplifySetCC() argument
22722 return TLI.SimplifySetCC(VT, N0, N1, Cond, foldBooleans, DagCombineInfo, DL); in SimplifySetCC()
22789 EVT VT = V.getValueType(); in BuildLogBase2() local
22790 SDValue Ctlz = DAG.getNode(ISD::CTLZ, DL, VT, V); in BuildLogBase2()
22791 SDValue Base = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT); in BuildLogBase2()
22792 SDValue LogBase2 = DAG.getNode(ISD::SUB, DL, VT, Base, Ctlz); in BuildLogBase2()
22810 EVT VT = Op.getValueType(); in BuildDivEstimate() local
22811 if (VT.getScalarType() != MVT::f32 && VT.getScalarType() != MVT::f64) in BuildDivEstimate()
22816 int Enabled = TLI.getRecipEstimateDivEnabled(VT, MF); in BuildDivEstimate()
22822 int Iterations = TLI.getDivRefinementSteps(VT, MF); in BuildDivEstimate()
22828 SDValue FPOne = DAG.getConstantFP(1.0, DL, VT); in BuildDivEstimate()
22836 MulEst = DAG.getNode(ISD::FMUL, DL, VT, N, Est, Flags); in BuildDivEstimate()
22840 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Op, MulEst, Flags); in BuildDivEstimate()
22843 NewEst = DAG.getNode(ISD::FSUB, DL, VT, in BuildDivEstimate()
22847 NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags); in BuildDivEstimate()
22850 Est = DAG.getNode(ISD::FADD, DL, VT, MulEst, NewEst, Flags); in BuildDivEstimate()
22855 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, N, Flags); in BuildDivEstimate()
22874 EVT VT = Arg.getValueType(); in buildSqrtNROneConst() local
22876 SDValue ThreeHalves = DAG.getConstantFP(1.5, DL, VT); in buildSqrtNROneConst()
22880 SDValue HalfArg = DAG.getNode(ISD::FMUL, DL, VT, ThreeHalves, Arg, Flags); in buildSqrtNROneConst()
22881 HalfArg = DAG.getNode(ISD::FSUB, DL, VT, HalfArg, Arg, Flags); in buildSqrtNROneConst()
22885 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, Est, Flags); in buildSqrtNROneConst()
22886 NewEst = DAG.getNode(ISD::FMUL, DL, VT, HalfArg, NewEst, Flags); in buildSqrtNROneConst()
22887 NewEst = DAG.getNode(ISD::FSUB, DL, VT, ThreeHalves, NewEst, Flags); in buildSqrtNROneConst()
22888 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags); in buildSqrtNROneConst()
22893 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, Arg, Flags); in buildSqrtNROneConst()
22906 EVT VT = Arg.getValueType(); in buildSqrtNRTwoConst() local
22908 SDValue MinusThree = DAG.getConstantFP(-3.0, DL, VT); in buildSqrtNRTwoConst()
22909 SDValue MinusHalf = DAG.getConstantFP(-0.5, DL, VT); in buildSqrtNRTwoConst()
22918 SDValue AE = DAG.getNode(ISD::FMUL, DL, VT, Arg, Est, Flags); in buildSqrtNRTwoConst()
22919 SDValue AEE = DAG.getNode(ISD::FMUL, DL, VT, AE, Est, Flags); in buildSqrtNRTwoConst()
22920 SDValue RHS = DAG.getNode(ISD::FADD, DL, VT, AEE, MinusThree, Flags); in buildSqrtNRTwoConst()
22928 LHS = DAG.getNode(ISD::FMUL, DL, VT, Est, MinusHalf, Flags); in buildSqrtNRTwoConst()
22931 LHS = DAG.getNode(ISD::FMUL, DL, VT, AE, MinusHalf, Flags); in buildSqrtNRTwoConst()
22934 Est = DAG.getNode(ISD::FMUL, DL, VT, LHS, RHS, Flags); in buildSqrtNRTwoConst()
22949 EVT VT = Op.getValueType(); in buildSqrtEstimateImpl() local
22950 if (VT.getScalarType() != MVT::f32 && VT.getScalarType() != MVT::f64) in buildSqrtEstimateImpl()
22955 int Enabled = TLI.getRecipEstimateSqrtEnabled(VT, MF); in buildSqrtEstimateImpl()
22961 int Iterations = TLI.getSqrtRefinementSteps(VT, MF); in buildSqrtEstimateImpl()
22976 SDValue Test = TLI.getSqrtInputTest(Op, DAG, DAG.getDenormalMode(VT)); in buildSqrtEstimateImpl()
22982 Test.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT, DL, VT, in buildSqrtEstimateImpl()