Lines Matching refs:VT
168 for (MVT VT : MVT::all_valuetypes()) in DAGCombiner() local
169 if (EVT(VT).isSimple() && VT != MVT::Other && in DAGCombiner()
170 TLI.isTypeLegal(EVT(VT)) && in DAGCombiner()
171 VT.getSizeInBits() >= MaximumLegalStoreInBits) in DAGCombiner()
172 MaximumLegalStoreInBits = VT.getSizeInBits(); in DAGCombiner()
432 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
442 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
595 bool hasOperation(unsigned Opcode, EVT VT) { in hasOperation() argument
597 return TLI.isOperationLegal(Opcode, VT); in hasOperation()
598 return TLI.isOperationLegalOrCustom(Opcode, VT); in hasOperation()
616 bool isTypeLegal(const EVT &VT) { in isTypeLegal() argument
618 return TLI.isTypeLegal(VT); in isTypeLegal()
622 EVT getSetCCResultType(EVT VT) const { in getSetCCResultType()
623 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in getSetCCResultType()
705 EVT VT = Op.getValueType(); in isNegatibleForFree() local
709 TLI.isFPExtFree(VT, Op.getOperand(0).getValueType()))) in isNegatibleForFree()
723 return TLI.isOperationLegal(ISD::ConstantFP, VT) || in isNegatibleForFree()
724 TLI.isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT); in isNegatibleForFree()
731 if (LegalOperations && !TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) in isNegatibleForFree()
933 EVT VT = N0.getValueType(); in ReassociateOps() local
938 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, L, R)) in ReassociateOps()
939 return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode); in ReassociateOps()
945 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0.getOperand(0), N1); in ReassociateOps()
949 return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1)); in ReassociateOps()
958 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, R, L)) in ReassociateOps()
959 return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode); in ReassociateOps()
965 SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0, N1.getOperand(0)); in ReassociateOps()
969 return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1)); in ReassociateOps()
1074 EVT VT = Load->getValueType(0); in ReplaceLoadWithPromotedLoad() local
1075 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0)); in ReplaceLoadWithPromotedLoad()
1161 EVT VT = Op.getValueType(); in PromoteIntBinOp() local
1162 if (VT.isVector() || !VT.isInteger()) in PromoteIntBinOp()
1168 if (TLI.isTypeDesirableForOp(Opc, VT)) in PromoteIntBinOp()
1171 EVT PVT = VT; in PromoteIntBinOp()
1175 assert(PVT != VT && "Don't know what type to promote to!"); in PromoteIntBinOp()
1189 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1)); in PromoteIntBinOp()
1226 EVT VT = Op.getValueType(); in PromoteIntShiftOp() local
1227 if (VT.isVector() || !VT.isInteger()) in PromoteIntShiftOp()
1233 if (TLI.isTypeDesirableForOp(Opc, VT)) in PromoteIntShiftOp()
1236 EVT PVT = VT; in PromoteIntShiftOp()
1240 assert(PVT != VT && "Don't know what type to promote to!"); in PromoteIntShiftOp()
1259 DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, N0, N1)); in PromoteIntShiftOp()
1276 EVT VT = Op.getValueType(); in PromoteExtend() local
1277 if (VT.isVector() || !VT.isInteger()) in PromoteExtend()
1283 if (TLI.isTypeDesirableForOp(Opc, VT)) in PromoteExtend()
1286 EVT PVT = VT; in PromoteExtend()
1290 assert(PVT != VT && "Don't know what type to promote to!"); in PromoteExtend()
1295 return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0)); in PromoteExtend()
1307 EVT VT = Op.getValueType(); in PromoteLoad() local
1308 if (VT.isVector() || !VT.isInteger()) in PromoteLoad()
1314 if (TLI.isTypeDesirableForOp(Opc, VT)) in PromoteLoad()
1317 EVT PVT = VT; in PromoteLoad()
1321 assert(PVT != VT && "Don't know what type to promote to!"); in PromoteLoad()
1332 SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD); in PromoteLoad()
1914 EVT VT = Sel.getValueType(); in foldBinOpIntoSelect() local
1922 if (SelOpNo && VT != CBO.getValueType()) in foldBinOpIntoSelect()
1929 SDValue NewCT = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CT) in foldBinOpIntoSelect()
1930 : DAG.getNode(BinOpcode, DL, VT, CT, CBO); in foldBinOpIntoSelect()
1936 SDValue NewCF = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CF) in foldBinOpIntoSelect()
1937 : DAG.getNode(BinOpcode, DL, VT, CF, CBO); in foldBinOpIntoSelect()
1943 return DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF); in foldBinOpIntoSelect()
1977 EVT VT = C.getValueType(); in foldAddSubBoolOfMaskedVal() local
1979 SDValue LowBit = DAG.getZExtOrTrunc(SetCC.getOperand(0), DL, VT); in foldAddSubBoolOfMaskedVal()
1980 SDValue C1 = IsAdd ? DAG.getConstant(CN->getAPIntValue() + 1, DL, VT) : in foldAddSubBoolOfMaskedVal()
1981 DAG.getConstant(CN->getAPIntValue() - 1, DL, VT); in foldAddSubBoolOfMaskedVal()
1982 return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit); in foldAddSubBoolOfMaskedVal()
2006 EVT VT = ShiftOp.getValueType(); in foldAddSubOfSignBit() local
2009 if (!ShAmtC || ShAmtC->getZExtValue() != VT.getScalarSizeInBits() - 1) in foldAddSubOfSignBit()
2017 SDValue NewShift = DAG.getNode(ShOpcode, DL, VT, Not.getOperand(0), ShAmt); in foldAddSubOfSignBit()
2019 return DAG.getNode(ISD::ADD, DL, VT, NewShift, DAG.getConstant(NewC, DL, VT)); in foldAddSubOfSignBit()
2025 EVT VT = N0.getValueType(); in visitADD() local
2029 if (VT.isVector()) { in visitADD()
2050 return DAG.getNode(ISD::ADD, DL, VT, N1, N0); in visitADD()
2052 return DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, N0.getNode(), in visitADD()
2065 return DAG.getNode(ISD::SUB, DL, VT, in visitADD()
2066 DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)), in visitADD()
2079 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) && in visitADD()
2082 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not); in visitADD()
2091 SDValue Add0 = DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(1)); in visitADD()
2092 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add0); in visitADD()
2105 return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1)); in visitADD()
2109 return DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(1)); in visitADD()
2122 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), in visitADD()
2128 return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), in visitADD()
2135 return DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0).getOperand(0), in visitADD()
2146 return DAG.getNode(ISD::SUB, DL, VT, in visitADD()
2147 DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10), in visitADD()
2148 DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11)); in visitADD()
2161 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) && in visitADD()
2163 return DAG.getNode(ISD::OR, DL, VT, N0, N1); in visitADD()
2167 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in visitADD()
2183 EVT VT = N0.getValueType(); in visitADDSAT() local
2187 if (VT.isVector()) { in visitADDSAT()
2199 return DAG.getAllOnesConstant(DL, VT); in visitADDSAT()
2204 return DAG.getNode(Opcode, DL, VT, N1, N0); in visitADDSAT()
2206 return DAG.FoldConstantArithmetic(Opcode, DL, VT, N0.getNode(), in visitADDSAT()
2217 return DAG.getNode(ISD::ADD, DL, VT, N0, N1); in visitADDSAT()
2261 EVT VT = N0.getValueType(); in visitADDLike() local
2267 return DAG.getNode(ISD::SUB, DL, VT, N0, in visitADDLike()
2268 DAG.getNode(ISD::SHL, DL, VT, in visitADDLike()
2275 unsigned DestBits = VT.getScalarSizeInBits(); in visitADDLike()
2280 return DAG.getNode(ISD::SUB, DL, VT, N0, AndOp0); in visitADDLike()
2287 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitADDLike()
2288 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt); in visitADDLike()
2295 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0), in visitADDLike()
2296 DAG.getConstant(1, DL, VT)); in visitADDLike()
2297 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt); in visitADDLike()
2308 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT)) in visitADDLike()
2311 DAG.getVTList(VT, Carry.getValueType()), N0, in visitADDLike()
2312 DAG.getConstant(0, DL, VT), Carry); in visitADDLike()
2320 EVT VT = N0.getValueType(); in visitADDC() local
2325 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDC()
2341 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitADDC()
2347 static SDValue flipBoolean(SDValue V, const SDLoc &DL, EVT VT, in flipBoolean() argument
2350 switch (TLI.getBooleanContents(VT)) { in flipBoolean()
2353 Cst = DAG.getConstant(1, DL, VT); in flipBoolean()
2356 Cst = DAG.getConstant(-1, DL, VT); in flipBoolean()
2360 return DAG.getNode(ISD::XOR, DL, VT, V, Cst); in flipBoolean()
2363 static bool isBooleanFlip(SDValue V, EVT VT, const TargetLowering &TLI) { in isBooleanFlip() argument
2368 switch(TLI.getBooleanContents(VT)) { in isBooleanFlip()
2382 EVT VT = N0.getValueType(); in visitUADDO() local
2383 if (VT.isVector()) in visitUADDO()
2391 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitUADDO()
2406 return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), in visitUADDO()
2412 DAG.getConstant(0, DL, VT), in visitUADDO()
2428 auto VT = N0.getValueType(); in visitUADDOLike() local
2441 if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT)) in visitUADDOLike()
2444 DAG.getConstant(0, SDLoc(N), VT), Carry); in visitUADDOLike()
2491 EVT VT = N0.getValueType(); in visitADDCARRY() local
2492 SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT); in visitADDCARRY()
2494 return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt, in visitADDCARRY()
2495 DAG.getConstant(1, DL, VT)), in visitADDCARRY()
2568 static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT, in tryFoldToZero() argument
2570 if (!VT.isVector()) in tryFoldToZero()
2571 return DAG.getConstant(0, DL, VT); in tryFoldToZero()
2572 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) in tryFoldToZero()
2573 return DAG.getConstant(0, DL, VT); in tryFoldToZero()
2580 EVT VT = N0.getValueType(); in visitSUB() local
2584 if (VT.isVector()) { in visitSUB()
2596 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations); in visitSUB()
2600 return DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N0.getNode(), in visitSUB()
2611 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
2612 DAG.getConstant(-N1C->getAPIntValue(), DL, VT)); in visitSUB()
2616 unsigned BitWidth = VT.getScalarSizeInBits(); in visitSUB()
2625 if (!LegalOperations || TLI.isOperationLegal(NewSh, VT)) in visitSUB()
2626 return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1)); in visitSUB()
2647 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitSUB()
2651 return DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(1)); in visitSUB()
2670 SDValue NewC = DAG.getNode(ISD::SUB, DL, VT, N0, N11); in visitSUB()
2671 return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0)); in visitSUB()
2680 return DAG.getNode(N0.getOperand(1).getOpcode(), DL, VT, N0.getOperand(0), in visitSUB()
2686 return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), in visitSUB()
2692 return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), in visitSUB()
2697 return DAG.getNode(ISD::ADD, DL, VT, N0, in visitSUB()
2698 DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(1), in visitSUB()
2705 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, in visitSUB()
2708 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul); in visitSUB()
2712 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, in visitSUB()
2715 return DAG.getNode(ISD::ADD, DL, VT, N0, Mul); in visitSUB()
2732 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) { in visitSUB()
2737 unsigned OpSizeInBits = VT.getScalarSizeInBits(); in visitSUB()
2740 return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0); in visitSUB()
2750 return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT, in visitSUB()
2757 DL, VT); in visitSUB()
2764 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0), in visitSUB()
2765 DAG.getConstant(1, DL, VT)); in visitSUB()
2766 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt); in visitSUB()
2776 SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt); in visitSUB()
2777 return DAG.getNode(ISD::ADD, DL, VT, N0, SRA); in visitSUB()
2787 EVT VT = N0.getValueType(); in visitSUBSAT() local
2791 if (VT.isVector()) { in visitSUBSAT()
2801 return DAG.getConstant(0, DL, VT); in visitSUBSAT()
2805 return DAG.getConstant(0, DL, VT); in visitSUBSAT()
2810 return DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, N0.getNode(), in visitSUBSAT()
2824 EVT VT = N0.getValueType(); in visitSUBC() local
2829 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitSUBC()
2834 return CombineTo(N, DAG.getConstant(0, DL, VT), in visitSUBC()
2843 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0), in visitSUBC()
2852 EVT VT = N0.getValueType(); in visitUSUBO() local
2853 if (VT.isVector()) in visitUSUBO()
2861 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), in visitUSUBO()
2866 return CombineTo(N, DAG.getConstant(0, DL, VT), in visitUSUBO()
2875 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0), in visitUSUBO()
2911 EVT VT = N0.getValueType(); in visitMUL() local
2915 return DAG.getConstant(0, SDLoc(N), VT); in visitMUL()
2923 if (VT.isVector()) { in visitMUL()
2930 ConstValue0.getBitWidth() == VT.getScalarSizeInBits()) && in visitMUL()
2933 ConstValue1.getBitWidth() == VT.getScalarSizeInBits()) && in visitMUL()
2950 return DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT, in visitMUL()
2956 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0); in visitMUL()
2970 return DAG.getNode(ISD::SUB, DL, VT, in visitMUL()
2971 DAG.getConstant(0, DL, VT), N0); in visitMUL()
2976 (!VT.isVector() || Level <= AfterLegalizeVectorOps)) { in visitMUL()
2981 return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc); in visitMUL()
2989 return DAG.getNode(ISD::SUB, DL, VT, in visitMUL()
2990 DAG.getConstant(0, DL, VT), in visitMUL()
2991 DAG.getNode(ISD::SHL, DL, VT, N0, in visitMUL()
3003 if (N1IsConst && TLI.decomposeMulByConstant(VT, N1)) { in visitMUL()
3017 assert(ShAmt > 0 && ShAmt < VT.getScalarSizeInBits() && in visitMUL()
3020 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, N0, in visitMUL()
3021 DAG.getConstant(ShAmt, DL, VT)); in visitMUL()
3022 SDValue R = DAG.getNode(MathOp, DL, VT, Shl, N0); in visitMUL()
3024 R = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), R); in visitMUL()
3033 SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT, N1, N0.getOperand(1)); in visitMUL()
3035 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), C3); in visitMUL()
3055 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, Sh.getOperand(0), Y); in visitMUL()
3056 return DAG.getNode(ISD::SHL, SDLoc(N), VT, Mul, Sh.getOperand(1)); in visitMUL()
3065 return DAG.getNode(ISD::ADD, SDLoc(N), VT, in visitMUL()
3066 DAG.getNode(ISD::MUL, SDLoc(N0), VT, in visitMUL()
3068 DAG.getNode(ISD::MUL, SDLoc(N1), VT, in visitMUL()
3107 EVT VT = Node->getValueType(0); in useDivRem() local
3108 if (VT.isVector() || !VT.isInteger()) in useDivRem()
3111 if (!TLI.isTypeLegal(VT) && !TLI.isOperationCustom(DivRemOpc, VT)) in useDivRem()
3116 if (!TLI.isOperationLegalOrCustom(DivRemOpc, VT) && in useDivRem()
3124 if (TLI.isOperationLegalOrCustom(Opcode, VT)) in useDivRem()
3128 if (TLI.isOperationLegalOrCustom(OtherOpcode, VT)) in useDivRem()
3150 SDVTList VTs = DAG.getVTList(VT, VT); in useDivRem()
3171 EVT VT = N->getValueType(0); in simplifyDivRem() local
3184 return DAG.getUNDEF(VT); in simplifyDivRem()
3189 return DAG.getConstant(0, DL, VT); in simplifyDivRem()
3200 return DAG.getConstant(IsDiv ? 1 : 0, DL, VT); in simplifyDivRem()
3208 if ((N1C && N1C->isOne()) || (VT.getScalarType() == MVT::i1)) in simplifyDivRem()
3209 return IsDiv ? N0 : DAG.getConstant(0, DL, VT); in simplifyDivRem()
3217 EVT VT = N->getValueType(0); in visitSDIV() local
3218 EVT CCVT = getSetCCResultType(VT); in visitSDIV()
3221 if (VT.isVector()) in visitSDIV()
3231 return DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, N0C, N1C); in visitSDIV()
3234 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), N0); in visitSDIV()
3237 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ), in visitSDIV()
3238 DAG.getConstant(1, DL, VT), in visitSDIV()
3239 DAG.getConstant(0, DL, VT)); in visitSDIV()
3257 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1); in visitSDIV()
3258 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitSDIV()
3279 EVT VT = N->getValueType(0); in visitSDIVLike() local
3280 EVT CCVT = getSetCCResultType(VT); in visitSDIVLike()
3281 unsigned BitWidth = VT.getScalarSizeInBits(); in visitSDIVLike()
3307 SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1); in visitSDIVLike()
3314 SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0, in visitSDIVLike()
3319 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact); in visitSDIVLike()
3321 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl); in visitSDIVLike()
3323 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1); in visitSDIVLike()
3328 SDValue One = DAG.getConstant(1, DL, VT); in visitSDIVLike()
3329 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT); in visitSDIVLike()
3333 Sra = DAG.getSelect(DL, VT, IsOneOrAllOnes, N0, Sra); in visitSDIVLike()
3337 SDValue Zero = DAG.getConstant(0, DL, VT); in visitSDIVLike()
3338 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, Zero, Sra); in visitSDIVLike()
3342 SDValue Res = DAG.getSelect(DL, VT, IsNeg, Sub, Sra); in visitSDIVLike()
3361 EVT VT = N->getValueType(0); in visitUDIV() local
3362 EVT CCVT = getSetCCResultType(VT); in visitUDIV()
3365 if (VT.isVector()) in visitUDIV()
3375 if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT, in visitUDIV()
3380 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ), in visitUDIV()
3381 DAG.getConstant(1, DL, VT), in visitUDIV()
3382 DAG.getConstant(0, DL, VT)); in visitUDIV()
3395 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1); in visitUDIV()
3396 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitUDIV()
3417 EVT VT = N->getValueType(0); in visitUDIVLike() local
3428 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc); in visitUDIVLike()
3444 return DAG.getNode(ISD::SRL, DL, VT, N0, Add); in visitUDIVLike()
3463 EVT VT = N->getValueType(0); in visitREM() local
3464 EVT CCVT = getSetCCResultType(VT); in visitREM()
3473 if (SDValue Folded = DAG.FoldConstantArithmetic(Opcode, DL, VT, N0C, N1C)) in visitREM()
3477 return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ), in visitREM()
3478 DAG.getConstant(0, DL, VT), N0); in visitREM()
3490 return DAG.getNode(ISD::UREM, DL, VT, N0, N1); in visitREM()
3492 SDValue NegOne = DAG.getAllOnesConstant(DL, VT); in visitREM()
3495 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne); in visitREM()
3497 return DAG.getNode(ISD::AND, DL, VT, N0, Add); in visitREM()
3502 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne); in visitREM()
3504 return DAG.getNode(ISD::AND, DL, VT, N0, Add); in visitREM()
3517 if (DAG.isKnownNeverZero(N1) && !TLI.isIntDivCheap(VT, Attr)) { in visitREM()
3526 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1); in visitREM()
3527 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul); in visitREM()
3544 EVT VT = N->getValueType(0); in visitMULHS() local
3547 if (VT.isVector()) { in visitMULHS()
3566 return DAG.getConstant(0, DL, VT); in visitMULHS()
3570 if (VT.isSimple() && !VT.isVector()) { in visitMULHS()
3571 MVT Simple = VT.getSimpleVT(); in visitMULHS()
3581 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in visitMULHS()
3591 EVT VT = N->getValueType(0); in visitMULHU() local
3594 if (VT.isVector()) { in visitMULHU()
3610 return DAG.getConstant(0, DL, VT); in visitMULHU()
3614 DAG.isKnownToBeAPowerOfTwo(N1) && hasOperation(ISD::SRL, VT)) { in visitMULHU()
3616 unsigned NumEltBits = VT.getScalarSizeInBits(); in visitMULHU()
3619 ISD::SUB, DL, VT, DAG.getConstant(NumEltBits, DL, VT), LogBase2); in visitMULHU()
3622 return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc); in visitMULHU()
3627 if (VT.isSimple() && !VT.isVector()) { in visitMULHU()
3628 MVT Simple = VT.getSimpleVT(); in visitMULHU()
3638 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in visitMULHU()
3698 EVT VT = N->getValueType(0); in visitSMUL_LOHI() local
3703 if (VT.isSimple() && !VT.isVector()) { in visitSMUL_LOHI()
3704 MVT Simple = VT.getSimpleVT(); in visitSMUL_LOHI()
3715 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi); in visitSMUL_LOHI()
3717 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo); in visitSMUL_LOHI()
3729 EVT VT = N->getValueType(0); in visitUMUL_LOHI() local
3734 if (VT.isSimple() && !VT.isVector()) { in visitUMUL_LOHI()
3735 MVT Simple = VT.getSimpleVT(); in visitUMUL_LOHI()
3746 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi); in visitUMUL_LOHI()
3748 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo); in visitUMUL_LOHI()
3779 EVT VT = N0.getValueType(); in visitIMINMAX() local
3782 if (VT.isVector()) in visitIMINMAX()
3790 return DAG.FoldConstantArithmetic(N->getOpcode(), SDLoc(N), VT, N0C, N1C); in visitIMINMAX()
3795 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0); in visitIMINMAX()
3801 if (!TLI.isOperationLegal(Opcode, VT) && in visitIMINMAX()
3812 if (TLI.isOperationLegal(AltOpcode, VT)) in visitIMINMAX()
3813 return DAG.getNode(AltOpcode, SDLoc(N), VT, N0, N1); in visitIMINMAX()
3823 EVT VT = N0.getValueType(); in hoistLogicOpWithSameOpcodeHands() local
3853 if ((VT.isVector() || LegalOperations) && in hoistLogicOpWithSameOpcodeHands()
3863 return DAG.getNode(HandOpcode, DL, VT, Logic); in hoistLogicOpWithSameOpcodeHands()
3880 if (TLI.isZExtFree(VT, XVT) && TLI.isTruncateFree(XVT, VT)) in hoistLogicOpWithSameOpcodeHands()
3885 return DAG.getNode(HandOpcode, DL, VT, Logic); in hoistLogicOpWithSameOpcodeHands()
3897 return DAG.getNode(HandOpcode, DL, VT, Logic, N0.getOperand(1)); in hoistLogicOpWithSameOpcodeHands()
3906 return DAG.getNode(HandOpcode, DL, VT, Logic); in hoistLogicOpWithSameOpcodeHands()
3921 return DAG.getNode(HandOpcode, DL, VT, Logic); in hoistLogicOpWithSameOpcodeHands()
3955 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations); in hoistLogicOpWithSameOpcodeHands()
3959 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT, in hoistLogicOpWithSameOpcodeHands()
3961 return DAG.getVectorShuffle(VT, DL, Logic, ShOp, SVN0->getMask()); in hoistLogicOpWithSameOpcodeHands()
3968 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations); in hoistLogicOpWithSameOpcodeHands()
3972 SDValue Logic = DAG.getNode(LogicOpcode, DL, VT, N0.getOperand(1), in hoistLogicOpWithSameOpcodeHands()
3974 return DAG.getVectorShuffle(VT, DL, ShOp, Logic, SVN0->getMask()); in hoistLogicOpWithSameOpcodeHands()
3998 EVT VT = N0.getValueType(); in foldLogicOfSetCCs() local
4000 if (LegalOperations || VT.getScalarType() != MVT::i1) in foldLogicOfSetCCs()
4001 if (VT != getSetCCResultType(OpVT)) in foldLogicOfSetCCs()
4029 return DAG.getSetCC(DL, VT, Or, LR, CC1); in foldLogicOfSetCCs()
4048 return DAG.getSetCC(DL, VT, And, LR, CC1); in foldLogicOfSetCCs()
4062 return DAG.getSetCC(DL, VT, Add, Two, ISD::SETUGE); in foldLogicOfSetCCs()
4076 return DAG.getSetCC(DL, VT, Or, Zero, CC1); in foldLogicOfSetCCs()
4095 return DAG.getSetCC(DL, VT, LL, LR, NewCC); in foldLogicOfSetCCs()
4106 EVT VT = N1.getValueType(); in visitANDLike() local
4111 return DAG.getConstant(0, DL, VT); in visitANDLike()
4117 VT.getSizeInBits() <= 64) { in visitANDLike()
4127 SRLC.ult(VT.getSizeInBits()) && in visitANDLike()
4129 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(), in visitANDLike()
4136 DAG.getNode(ISD::ADD, DL0, VT, in visitANDLike()
4137 N0.getOperand(0), DAG.getConstant(ADDC, DL, VT)); in visitANDLike()
4154 unsigned Size = VT.getSizeInBits(); in visitANDLike()
4169 TLI.isNarrowingProfitable(VT, HalfVT) && in visitANDLike()
4172 TLI.isTruncateFree(VT, HalfVT) && in visitANDLike()
4173 TLI.isZExtFree(HalfVT, VT)) { in visitANDLike()
4191 return DAG.getNode(ISD::ZERO_EXTEND, SL, VT, And); in visitANDLike()
4360 EVT VT = Op.getOpcode() == ISD::AssertZext ? in SearchForAndLoads() local
4366 if (ExtVT.bitsGE(VT)) in SearchForAndLoads()
4388 MVT VT = SDValue(NodeToMask, i).getSimpleValueType(); in SearchForAndLoads() local
4389 if (VT != MVT::Glue && VT != MVT::Other) { in SearchForAndLoads()
4515 EVT VT = N->getValueType(0); in unfoldExtremeBitClearingToShifts() local
4518 SDValue T0 = DAG.getNode(InnerShift, DL, VT, X, Y); in unfoldExtremeBitClearingToShifts()
4520 SDValue T1 = DAG.getNode(OuterShift, DL, VT, T0, Y); in unfoldExtremeBitClearingToShifts()
4528 EVT VT = N1.getValueType(); in visitAND() local
4535 if (VT.isVector()) { in visitAND()
4560 return DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, N0C, N1C); in visitAND()
4564 return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0); in visitAND()
4569 unsigned BitWidth = VT.getScalarSizeInBits(); in visitAND()
4572 return DAG.getConstant(0, SDLoc(N), VT); in visitAND()
4582 if (VT.isVector()) in visitAND()
4647 EVT VT = Vector->getValueType(0); in visitAND() local
4648 unsigned BitWidth = VT.getScalarSizeInBits(); in visitAND()
4721 if (!VT.isVector() && N1C && (N0.getOpcode() == ISD::LOAD || in visitAND()
4766 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0)); in visitAND()
4785 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) { in visitAND()
4786 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, in visitAND()
4805 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) { in visitAND()
4806 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, in visitAND()
4833 EVT VT = N->getValueType(0); in MatchBSwapHWordLow() local
4834 if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16) in MatchBSwapHWordLow()
4836 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT)) in MatchBSwapHWordLow()
4914 unsigned OpSizeInBits = VT.getSizeInBits(); in MatchBSwapHWordLow()
4930 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00); in MatchBSwapHWordLow()
4933 Res = DAG.getNode(ISD::SRL, DL, VT, Res, in MatchBSwapHWordLow()
4935 getShiftAmountTy(VT))); in MatchBSwapHWordLow()
5040 EVT VT = N->getValueType(0); in MatchBSwapHWord() local
5041 if (VT != MVT::i32) in MatchBSwapHWord()
5043 if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT)) in MatchBSwapHWord()
5090 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, in MatchBSwapHWord()
5095 SDValue ShAmt = DAG.getConstant(16, DL, getShiftAmountTy(VT)); in MatchBSwapHWord()
5096 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT)) in MatchBSwapHWord()
5097 return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt); in MatchBSwapHWord()
5098 if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT)) in MatchBSwapHWord()
5099 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt); in MatchBSwapHWord()
5100 return DAG.getNode(ISD::OR, DL, VT, in MatchBSwapHWord()
5101 DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt), in MatchBSwapHWord()
5102 DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt)); in MatchBSwapHWord()
5108 EVT VT = N1.getValueType(); in visitORLike() local
5113 return DAG.getAllOnesConstant(DL, VT); in visitORLike()
5135 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, in visitORLike()
5137 return DAG.getNode(ISD::AND, DL, VT, X, in visitORLike()
5138 DAG.getConstant(LHSMask | RHSMask, DL, VT)); in visitORLike()
5150 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, in visitORLike()
5152 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X); in visitORLike()
5161 EVT VT = N1.getValueType(); in visitOR() local
5168 if (VT.isVector()) { in visitOR()
5191 TLI.isTypeLegal(VT)) { in visitOR()
5203 int NumElts = VT.getVectorNumElements(); in visitOR()
5240 bool LegalMask = TLI.isShuffleMaskLegal(Mask, VT); in visitOR()
5244 LegalMask = TLI.isShuffleMaskLegal(Mask, VT); in visitOR()
5248 return DAG.getVectorShuffle(VT, SDLoc(N), NewLHS, NewRHS, Mask); in visitOR()
5258 return DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, N0C, N1C); in visitOR()
5262 return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0); in visitOR()
5298 ISD::OR, SDLoc(N1), VT, N1.getNode(), N0.getOperand(1).getNode())) { in visitOR()
5299 SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1); in visitOR()
5301 return DAG.getNode(ISD::AND, SDLoc(N), VT, COR, IOR); in visitOR()
5586 EVT VT = Shifted.getValueType(); in MatchRotatePosNeg() local
5587 if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits(), DAG)) { in MatchRotatePosNeg()
5588 bool HasPos = TLI.isOperationLegalOrCustom(PosOpcode, VT); in MatchRotatePosNeg()
5589 return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted, in MatchRotatePosNeg()
5601 EVT VT = LHS.getValueType(); in MatchRotate() local
5602 if (!TLI.isTypeLegal(VT)) return nullptr; in MatchRotate()
5605 bool HasROTL = hasOperation(ISD::ROTL, VT); in MatchRotate()
5606 bool HasROTR = hasOperation(ISD::ROTR, VT); in MatchRotate()
5670 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in MatchRotate()
5683 SDValue Rot = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT, in MatchRotate()
5688 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT); in MatchRotate()
5692 SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt); in MatchRotate()
5693 Mask = DAG.getNode(ISD::AND, DL, VT, Mask, in MatchRotate()
5694 DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits)); in MatchRotate()
5697 SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt); in MatchRotate()
5698 Mask = DAG.getNode(ISD::AND, DL, VT, Mask, in MatchRotate()
5699 DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits)); in MatchRotate()
5702 Rot = DAG.getNode(ISD::AND, DL, VT, Rot, Mask); in MatchRotate()
5907 EVT VT = N->getValueType(0); in MatchLoadCombine() local
5908 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64) in MatchLoadCombine()
5910 unsigned ByteWidth = VT.getSizeInBits() / 8; in MatchLoadCombine()
5916 if (LegalOperations && !TLI.isOperationLegal(ISD::LOAD, VT)) in MatchLoadCombine()
6016 if (NeedsBswap && LegalOperations && !TLI.isOperationLegal(ISD::BSWAP, VT)) in MatchLoadCombine()
6022 VT, FirstLoad->getAddressSpace(), in MatchLoadCombine()
6028 DAG.getLoad(VT, SDLoc(N), Chain, FirstLoad->getBasePtr(), in MatchLoadCombine()
6035 return NeedsBswap ? DAG.getNode(ISD::BSWAP, SDLoc(N), VT, NewLoad) : NewLoad; in MatchLoadCombine()
6058 EVT VT = N->getValueType(0); in unfoldMaskedMerge() local
6106 SDValue NotX = DAG.getNOT(DL, X, VT); in unfoldMaskedMerge()
6107 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, NotX, M); in unfoldMaskedMerge()
6108 SDValue NotLHS = DAG.getNOT(DL, LHS, VT); in unfoldMaskedMerge()
6109 SDValue RHS = DAG.getNode(ISD::OR, DL, VT, M, Y); in unfoldMaskedMerge()
6110 return DAG.getNode(ISD::AND, DL, VT, NotLHS, RHS); in unfoldMaskedMerge()
6113 SDValue LHS = DAG.getNode(ISD::AND, DL, VT, X, M); in unfoldMaskedMerge()
6114 SDValue NotM = DAG.getNOT(DL, M, VT); in unfoldMaskedMerge()
6115 SDValue RHS = DAG.getNode(ISD::AND, DL, VT, Y, NotM); in unfoldMaskedMerge()
6117 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS); in unfoldMaskedMerge()
6123 EVT VT = N0.getValueType(); in visitXOR() local
6126 if (VT.isVector()) { in visitXOR()
6140 return DAG.getConstant(0, DL, VT); in visitXOR()
6150 return DAG.FoldConstantArithmetic(ISD::XOR, DL, VT, N0C, N1C); in visitXOR()
6154 return DAG.getNode(ISD::XOR, DL, VT, N1, N0); in visitXOR()
6178 return DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC); in visitXOR()
6194 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, V); in visitXOR()
6198 if (isOneConstant(N1) && VT == MVT::i1 && N0.hasOneUse() && in visitXOR()
6203 LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS in visitXOR()
6204 RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS in visitXOR()
6206 return DAG.getNode(NewOpcode, DL, VT, LHS, RHS); in visitXOR()
6215 LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS in visitXOR()
6216 RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS in visitXOR()
6218 return DAG.getNode(NewOpcode, DL, VT, LHS, RHS); in visitXOR()
6224 SDValue NotX = DAG.getNOT(SDLoc(X), X, VT); in visitXOR()
6226 return DAG.getNode(ISD::AND, DL, VT, NotX, N1); in visitXOR()
6232 unsigned BitWidth = VT.getScalarSizeInBits(); in visitXOR()
6244 SDValue Not = DAG.getNOT(DL, N0.getOperand(0), VT); in visitXOR()
6245 return DAG.getNode(N0Opcode, DL, VT, Not, N0.getOperand(1)); in visitXOR()
6252 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) { in visitXOR()
6259 unsigned OpSizeInBits = VT.getScalarSizeInBits(); in visitXOR()
6262 return DAG.getNode(ISD::ABS, DL, VT, S0); in visitXOR()
6269 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations); in visitXOR()
6289 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0Opcode == ISD::SHL && in visitXOR()
6291 return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT), in visitXOR()
6363 EVT VT = N->getValueType(0); in visitShiftByConstant() local
6387 VT, LHS->getOperand(0), N->getOperand(1)); in visitShiftByConstant()
6390 return DAG.getNode(LHS->getOpcode(), SDLoc(N), VT, NewShift, NewRHS); in visitShiftByConstant()
6419 EVT VT = N->getValueType(0); in visitRotate() local
6420 unsigned Bitsize = VT.getScalarSizeInBits(); in visitRotate()
6437 return DAG.getNode(N->getOpcode(), dl, VT, N0, in visitRotate()
6446 return DAG.getNode(N->getOpcode(), dl, VT, N0, NewOp1); in visitRotate()
6464 return DAG.getNode(N->getOpcode(), dl, VT, N0->getOperand(0), in visitRotate()
6478 EVT VT = N0.getValueType(); in visitSHL() local
6479 unsigned OpSizeInBits = VT.getScalarSizeInBits(); in visitSHL()
6482 if (VT.isVector()) { in visitSHL()
6498 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, in visitSHL()
6500 return DAG.getNode(ISD::AND, SDLoc(N), VT, N00, C); in visitSHL()
6511 return DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, N0C, N1C); in visitSHL()
6519 return DAG.getConstant(0, SDLoc(N), VT); in visitSHL()
6524 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, NewOp1); in visitSHL()
6540 return DAG.getConstant(0, SDLoc(N), VT); in visitSHL()
6553 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum); in visitSHL()
6578 return DAG.getConstant(0, DL, VT); in visitSHL()
6581 ISD::SHL, DL, VT, in visitSHL()
6582 DAG.getNode(N0.getOpcode(), DL, VT, N0Op0->getOperand(0)), in visitSHL()
6595 if (N0Op0C1->getAPIntValue().ult(VT.getScalarSizeInBits())) { in visitSHL()
6606 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL); in visitSHL()
6621 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), in visitSHL()
6623 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0), in visitSHL()
6643 Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), in visitSHL()
6648 Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), in visitSHL()
6652 return DAG.getNode(ISD::AND, DL, VT, Shift, in visitSHL()
6653 DAG.getConstant(Mask, DL, VT)); in visitSHL()
6662 SDValue AllBits = DAG.getAllOnesConstant(DL, VT); in visitSHL()
6663 SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1); in visitSHL()
6664 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), HiBitsMask); in visitSHL()
6676 SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1); in visitSHL()
6677 SDValue Shl1 = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1); in visitSHL()
6680 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, Shl0, Shl1); in visitSHL()
6687 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1); in visitSHL()
6689 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), Shl); in visitSHL()
6705 EVT VT = N0.getValueType(); in visitSRA() local
6706 unsigned OpSizeInBits = VT.getScalarSizeInBits(); in visitSRA()
6715 if (VT.isVector()) in visitSRA()
6724 return DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, N0C, N1C); in visitSRA()
6734 if (VT.isVector()) in visitSRA()
6736 ExtVT, VT.getVectorNumElements()); in visitSRA()
6739 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, in visitSRA()
6763 if (VT.isVector()) in visitSRA()
6767 return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0), ShiftValue); in visitSRA()
6784 if (VT.isVector()) in visitSRA()
6785 TruncVT = EVT::getVectorVT(Ctx, TruncVT, VT.getVectorNumElements()); in visitSRA()
6796 TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) && in visitSRA()
6797 TLI.isTruncateFree(VT, TruncVT)) { in visitSRA()
6801 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, in visitSRA()
6815 return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0, NewOp1); in visitSRA()
6838 return DAG.getNode(ISD::TRUNCATE, DL, VT, SRA); in visitSRA()
6849 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, N1); in visitSRA()
6864 EVT VT = N0.getValueType(); in visitSRL() local
6865 unsigned OpSizeInBits = VT.getScalarSizeInBits(); in visitSRL()
6868 if (VT.isVector()) in visitSRL()
6877 return DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, N0C, N1C); in visitSRL()
6885 return DAG.getConstant(0, SDLoc(N), VT); in visitSRL()
6897 return DAG.getConstant(0, SDLoc(N), VT); in visitSRL()
6910 return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Sum); in visitSRL()
6927 return DAG.getConstant(0, DL, VT); in visitSRL()
6928 return DAG.getNode(ISD::TRUNCATE, DL, VT, in visitSRL()
6942 DAG.getNode(ISD::SRL, DL, VT, DAG.getAllOnesConstant(DL, VT), N1); in visitSRL()
6944 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), Mask); in visitSRL()
6953 return DAG.getUNDEF(VT); in visitSRL()
6965 return DAG.getNode(ISD::AND, DL, VT, in visitSRL()
6966 DAG.getNode(ISD::ANY_EXTEND, DL, VT, SmallShift), in visitSRL()
6967 DAG.getConstant(Mask, DL, VT)); in visitSRL()
6975 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), N1); in visitSRL()
6985 if (Known.One.getBoolValue()) return DAG.getConstant(0, SDLoc(N0), VT); in visitSRL()
6990 if (UnknownBits == 0) return DAG.getConstant(1, SDLoc(N0), VT); in visitSRL()
7003 Op = DAG.getNode(ISD::SRL, DL, VT, Op, in visitSRL()
7010 return DAG.getNode(ISD::XOR, DL, VT, in visitSRL()
7011 Op, DAG.getConstant(1, DL, VT)); in visitSRL()
7019 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, NewOp1); in visitSRL()
7068 EVT VT = N->getValueType(0); in visitFunnelShift() local
7073 unsigned BitWidth = VT.getScalarSizeInBits(); in visitFunnelShift()
7086 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N0, N1, in visitFunnelShift()
7096 if (N0 == N1 && hasOperation(RotOpc, VT)) in visitFunnelShift()
7097 return DAG.getNode(RotOpc, SDLoc(N), VT, N0, N2); in visitFunnelShift()
7104 EVT VT = N->getValueType(0); in visitABS() local
7108 return DAG.getNode(ISD::ABS, SDLoc(N), VT, N0); in visitABS()
7120 EVT VT = N->getValueType(0); in visitBSWAP() local
7124 return DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N0); in visitBSWAP()
7133 EVT VT = N->getValueType(0); in visitBITREVERSE() local
7137 return DAG.getNode(ISD::BITREVERSE, SDLoc(N), VT, N0); in visitBITREVERSE()
7146 EVT VT = N->getValueType(0); in visitCTLZ() local
7150 return DAG.getNode(ISD::CTLZ, SDLoc(N), VT, N0); in visitCTLZ()
7153 if (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ_ZERO_UNDEF, VT)) { in visitCTLZ()
7155 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0); in visitCTLZ()
7163 EVT VT = N->getValueType(0); in visitCTLZ_ZERO_UNDEF() local
7167 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0); in visitCTLZ_ZERO_UNDEF()
7173 EVT VT = N->getValueType(0); in visitCTTZ() local
7177 return DAG.getNode(ISD::CTTZ, SDLoc(N), VT, N0); in visitCTTZ()
7180 if (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ_ZERO_UNDEF, VT)) { in visitCTTZ()
7182 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0); in visitCTTZ()
7190 EVT VT = N->getValueType(0); in visitCTTZ_ZERO_UNDEF() local
7194 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0); in visitCTTZ_ZERO_UNDEF()
7200 EVT VT = N->getValueType(0); in visitCTPOP() local
7204 return DAG.getNode(ISD::CTPOP, SDLoc(N), VT, N0); in visitCTPOP()
7212 EVT VT = LHS.getValueType(); in isLegalToCombineMinNumMaxNum() local
7214 return Options.NoSignedZerosFPMath && VT.isFloatingPoint() && in isLegalToCombineMinNumMaxNum()
7219 static SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS, in combineMinNumMaxNum() argument
7226 EVT TransformVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); in combineMinNumMaxNum()
7238 if (TLI.isOperationLegalOrCustom(IEEEOpcode, VT)) in combineMinNumMaxNum()
7239 return DAG.getNode(IEEEOpcode, DL, VT, LHS, RHS); in combineMinNumMaxNum()
7243 return DAG.getNode(Opcode, DL, VT, LHS, RHS); in combineMinNumMaxNum()
7253 if (TLI.isOperationLegalOrCustom(IEEEOpcode, VT)) in combineMinNumMaxNum()
7254 return DAG.getNode(IEEEOpcode, DL, VT, LHS, RHS); in combineMinNumMaxNum()
7258 return DAG.getNode(Opcode, DL, VT, LHS, RHS); in combineMinNumMaxNum()
7270 EVT VT = N->getValueType(0); in foldSelectOfConstants() local
7274 if (!VT.isInteger()) in foldSelectOfConstants()
7290 if (VT != MVT::i1) in foldSelectOfConstants()
7291 NotCond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, NotCond); in foldSelectOfConstants()
7297 if (VT != MVT::i1) in foldSelectOfConstants()
7298 NotCond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NotCond); in foldSelectOfConstants()
7303 if (VT != MVT::i1) in foldSelectOfConstants()
7304 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond); in foldSelectOfConstants()
7309 if (VT != MVT::i1) in foldSelectOfConstants()
7310 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond); in foldSelectOfConstants()
7317 if (TLI.convertSelectOfConstantsToMath(VT)) { in foldSelectOfConstants()
7320 if (VT != MVT::i1) in foldSelectOfConstants()
7321 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond); in foldSelectOfConstants()
7322 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2); in foldSelectOfConstants()
7326 if (VT != MVT::i1) in foldSelectOfConstants()
7327 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond); in foldSelectOfConstants()
7328 return DAG.getNode(ISD::ADD, DL, VT, Cond, N2); in foldSelectOfConstants()
7352 if (VT.bitsEq(CondVT)) in foldSelectOfConstants()
7354 return DAG.getZExtOrTrunc(NotCond, DL, VT); in foldSelectOfConstants()
7364 EVT VT = N->getValueType(0); in visitSELECT() local
7373 if (VT == VT0 && VT == MVT::i1 && (N0 == N1 || isOneConstant(N1))) in visitSELECT()
7374 return DAG.getNode(ISD::OR, DL, VT, N0, N2); in visitSELECT()
7380 if (VT == VT0 && VT == MVT::i1 && isNullConstant(N1)) { in visitSELECT()
7381 SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT); in visitSELECT()
7383 return DAG.getNode(ISD::AND, DL, VT, NOTNode, N2); in visitSELECT()
7386 if (VT == VT0 && VT == MVT::i1 && isOneConstant(N2)) { in visitSELECT()
7387 SDValue NOTNode = DAG.getNOT(SDLoc(N0), N0, VT); in visitSELECT()
7389 return DAG.getNode(ISD::OR, DL, VT, NOTNode, N1); in visitSELECT()
7393 if (VT == VT0 && VT == MVT::i1 && (N0 == N2 || isNullConstant(N2))) in visitSELECT()
7394 return DAG.getNode(ISD::AND, DL, VT, N0, N1); in visitSELECT()
7410 TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT); in visitSELECT()
7472 return DAG.getNode(ISD::SELECT, DL, VT, N0->getOperand(0), N2, N1); in visitSELECT()
7485 if (SDValue FMinMax = combineMinNumMaxNum(DL, VT, Cond0, Cond1, N1, N2, in visitSELECT()
7494 if (!LegalOperations && TLI.isOperationLegalOrCustom(ISD::UADDO, VT) && in visitSELECT()
7512 SDVTList VTs = DAG.getVTList(VT, VT0); in visitSELECT()
7514 return DAG.getSelect(DL, VT, UAO.getValue(1), N1, UAO.getValue(0)); in visitSELECT()
7518 if (TLI.isOperationLegal(ISD::SELECT_CC, VT) || in visitSELECT()
7519 (!LegalOperations && TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT))) in visitSELECT()
7520 return DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1, N2, in visitSELECT()
7553 EVT VT = N->getValueType(0); in ConvertSelectToConcatVector() local
7554 int NumElems = VT.getVectorNumElements(); in ConvertSelectToConcatVector()
7596 ISD::CONCAT_VECTORS, DL, VT, in ConvertSelectToConcatVector()
7667 EVT VT = Data.getValueType(); in visitMSTORE() local
7676 if (TLI.getTypeAction(*DAG.getContext(), VT) != in visitMSTORE()
7692 (Alignment == VT.getSizeInBits() / 8) ? Alignment / 2 : Alignment; in visitMSTORE()
7746 EVT VT = N->getValueType(0); in visitMGATHER() local
7749 if (TLI.getTypeAction(*DAG.getContext(), VT) != in visitMGATHER()
7761 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in visitMGATHER()
7801 SDValue GatherRes = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); in visitMGATHER()
7820 EVT VT = N->getValueType(0); in visitMLOAD() local
7823 if (TLI.getTypeAction(*DAG.getContext(), VT) != in visitMLOAD()
7883 SDValue LoadRes = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); in visitMLOAD()
7897 EVT VT = N->getValueType(0); in foldVSelectOfConstants() local
7899 !TLI.convertSelectOfConstantsToMath(VT) || in foldVSelectOfConstants()
7909 unsigned Elts = VT.getVectorNumElements(); in foldVSelectOfConstants()
7931 SDValue ExtendedCond = DAG.getNode(ExtendOpcode, DL, VT, Cond); in foldVSelectOfConstants()
7932 return DAG.getNode(ISD::ADD, DL, VT, ExtendedCond, N2); in foldVSelectOfConstants()
7971 EVT VT = LHS.getValueType(); in visitVSELECT() local
7972 if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) in visitVSELECT()
7973 return DAG.getNode(ISD::ABS, DL, VT, LHS); in visitVSELECT()
7976 ISD::SRA, DL, VT, LHS, in visitVSELECT()
7977 DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT)); in visitVSELECT()
7978 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift); in visitVSELECT()
7981 return DAG.getNode(ISD::XOR, DL, VT, Add, Shift); in visitVSELECT()
7990 EVT VT = N->getValueType(0); in visitVSELECT() local
7994 DL, VT, N0.getOperand(0), N0.getOperand(1), N1, N2, CC, TLI, DAG)) in visitVSELECT()
8150 EVT VT = N->getValueType(0); in tryToFoldExtendOfConstant() local
8161 return DAG.getNode(Opcode, SDLoc(N), VT, N0); in tryToFoldExtendOfConstant()
8166 EVT SVT = VT.getScalarType(); in tryToFoldExtendOfConstant()
8167 if (!(VT.isVector() && (!LegalTypes || TLI.isTypeLegal(SVT)) && in tryToFoldExtendOfConstant()
8175 unsigned NumElts = VT.getVectorNumElements(); in tryToFoldExtendOfConstant()
8200 return DAG.getBuildVector(VT, DL, Elts); in tryToFoldExtendOfConstant()
8207 static bool ExtendUsesToFormExtLoad(EVT VT, SDNode *N, SDValue N0, in ExtendUsesToFormExtLoad() argument
8212 bool isTruncFree = TLI.isTruncateFree(VT, N0.getValueType()); in ExtendUsesToFormExtLoad()
8389 EVT VT = N->getValueType(0); in CombineZExtLogicopShiftLoad() local
8396 (LegalOperations && !TLI.isOperationLegal(N0.getOpcode(), VT))) in CombineZExtLogicopShiftLoad()
8403 (LegalOperations && !TLI.isOperationLegal(N1.getOpcode(), VT))) in CombineZExtLogicopShiftLoad()
8411 if (!TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) || in CombineZExtLogicopShiftLoad()
8425 if (!ExtendUsesToFormExtLoad(VT, N1.getNode(), N1.getOperand(0), in CombineZExtLogicopShiftLoad()
8430 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(Load), VT, in CombineZExtLogicopShiftLoad()
8435 SDValue Shift = DAG.getNode(N1.getOpcode(), DL1, VT, ExtLoad, in CombineZExtLogicopShiftLoad()
8439 Mask = Mask.zext(VT.getSizeInBits()); in CombineZExtLogicopShiftLoad()
8441 SDValue And = DAG.getNode(N0.getOpcode(), DL0, VT, Shift, in CombineZExtLogicopShiftLoad()
8442 DAG.getConstant(Mask, DL0, VT)); in CombineZExtLogicopShiftLoad()
8470 EVT VT = Cast->getValueType(0); in matchVSelectOpSizesWithSetCC() local
8471 if (LegalOperations || !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT)) in matchVSelectOpSizesWithSetCC()
8482 if (SetCCVT.getSizeInBits() != VT.getSizeInBits()) in matchVSelectOpSizesWithSetCC()
8492 CastA = DAG.getNode(CastOpcode, DL, VT, A, Cast->getOperand(1)); in matchVSelectOpSizesWithSetCC()
8493 CastB = DAG.getNode(CastOpcode, DL, VT, B, Cast->getOperand(1)); in matchVSelectOpSizesWithSetCC()
8495 CastA = DAG.getNode(CastOpcode, DL, VT, A); in matchVSelectOpSizesWithSetCC()
8496 CastB = DAG.getNode(CastOpcode, DL, VT, B); in matchVSelectOpSizesWithSetCC()
8498 return DAG.getNode(ISD::VSELECT, DL, VT, SetCC, CastA, CastB); in matchVSelectOpSizesWithSetCC()
8504 const TargetLowering &TLI, EVT VT, in tryToFoldExtOfExtload() argument
8516 if ((LegalOperations || LN0->isVolatile() || VT.isVector()) && in tryToFoldExtOfExtload()
8517 !TLI.isLoadExtLegal(ExtLoadType, VT, MemVT)) in tryToFoldExtOfExtload()
8521 DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(), in tryToFoldExtOfExtload()
8532 const TargetLowering &TLI, EVT VT, in tryToFoldExtOfLoad() argument
8538 ((LegalOperations || VT.isVector() || in tryToFoldExtOfLoad()
8540 !TLI.isLoadExtLegal(ExtLoadType, VT, N0.getValueType()))) in tryToFoldExtOfLoad()
8546 DoXform = ExtendUsesToFormExtLoad(VT, N, N0, ExtOpc, SetCCs, TLI); in tryToFoldExtOfLoad()
8547 if (VT.isVector()) in tryToFoldExtOfLoad()
8553 SDValue ExtLoad = DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(), in tryToFoldExtOfLoad()
8583 EVT VT = N->getValueType(0); in foldExtendedSignBitTest() local
8588 if (CC == ISD::SETGT && isAllOnesConstant(Ones) && VT == XVT) { in foldExtendedSignBitTest()
8593 SDValue NotX = DAG.getNOT(DL, X, VT); in foldExtendedSignBitTest()
8594 SDValue ShiftAmount = DAG.getConstant(VT.getSizeInBits() - 1, DL, VT); in foldExtendedSignBitTest()
8596 return DAG.getNode(ShiftOpcode, DL, VT, NotX, ShiftAmount); in foldExtendedSignBitTest()
8603 EVT VT = N->getValueType(0); in visitSIGN_EXTEND() local
8612 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N0.getOperand(0)); in visitSIGN_EXTEND()
8632 unsigned DestBits = VT.getScalarSizeInBits(); in visitSIGN_EXTEND()
8644 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op); in visitSIGN_EXTEND()
8649 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op); in visitSIGN_EXTEND()
8656 Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N0), VT, Op); in visitSIGN_EXTEND()
8658 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, Op); in visitSIGN_EXTEND()
8659 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Op, in visitSIGN_EXTEND()
8666 tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0, in visitSIGN_EXTEND()
8677 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::SEXTLOAD)) in visitSIGN_EXTEND()
8686 (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) { in visitSIGN_EXTEND()
8689 if (TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, MemVT) && in visitSIGN_EXTEND()
8692 bool DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0), in visitSIGN_EXTEND()
8695 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN00), VT, in visitSIGN_EXTEND()
8700 Mask = Mask.sext(VT.getSizeInBits()); in visitSIGN_EXTEND()
8701 SDValue And = DAG.getNode(N0.getOpcode(), DL, VT, in visitSIGN_EXTEND()
8702 ExtLoad, DAG.getConstant(Mask, DL, VT)); in visitSIGN_EXTEND()
8736 if (VT.isVector() && !LegalOperations && in visitSIGN_EXTEND()
8751 if (VT.getSizeInBits() == SVT.getSizeInBits()) in visitSIGN_EXTEND()
8752 return DAG.getSetCC(DL, VT, N00, N01, CC); in visitSIGN_EXTEND()
8760 return DAG.getSExtOrTrunc(VsetCC, DL, VT); in visitSIGN_EXTEND()
8778 ? DAG.getAllOnesConstant(DL, VT) in visitSIGN_EXTEND()
8779 : DAG.getBoolConstant(true, DL, VT, N00VT); in visitSIGN_EXTEND()
8780 SDValue Zero = DAG.getConstant(0, DL, VT); in visitSIGN_EXTEND()
8785 if (!VT.isVector() && !TLI.convertSelectOfConstantsToMath(VT)) { in visitSIGN_EXTEND()
8794 return DAG.getSelect(DL, VT, SetCC, ExtTrueVal, Zero); in visitSIGN_EXTEND()
8800 if ((!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) && in visitSIGN_EXTEND()
8802 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0); in visitSIGN_EXTEND()
8845 EVT VT = N->getValueType(0); in visitZERO_EXTEND() local
8853 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, in visitZERO_EXTEND()
8868 VT.getScalarSizeInBits())); in visitZERO_EXTEND()
8870 return DAG.getZExtOrTrunc(Op, SDLoc(N), VT); in visitZERO_EXTEND()
8892 if (SrcVT.bitsLT(VT) && VT.isVector()) { in visitZERO_EXTEND()
8894 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) { in visitZERO_EXTEND()
8898 SDValue ZExtOrTrunc = DAG.getZExtOrTrunc(Op, SDLoc(N), VT); in visitZERO_EXTEND()
8905 if (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT)) { in visitZERO_EXTEND()
8906 SDValue Op = DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT); in visitZERO_EXTEND()
8923 !TLI.isZExtFree(N0.getValueType(), VT))) { in visitZERO_EXTEND()
8925 X = DAG.getAnyExtOrTrunc(X, SDLoc(X), VT); in visitZERO_EXTEND()
8927 Mask = Mask.zext(VT.getSizeInBits()); in visitZERO_EXTEND()
8929 return DAG.getNode(ISD::AND, DL, VT, in visitZERO_EXTEND()
8930 X, DAG.getConstant(Mask, DL, VT)); in visitZERO_EXTEND()
8935 tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0, in visitZERO_EXTEND()
8952 (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) { in visitZERO_EXTEND()
8955 if (TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT) && in visitZERO_EXTEND()
8969 DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0), in visitZERO_EXTEND()
8972 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN00), VT, in visitZERO_EXTEND()
8977 Mask = Mask.zext(VT.getSizeInBits()); in visitZERO_EXTEND()
8979 SDValue And = DAG.getNode(N0.getOpcode(), DL, VT, in visitZERO_EXTEND()
8980 ExtLoad, DAG.getConstant(Mask, DL, VT)); in visitZERO_EXTEND()
9010 DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::ZEXTLOAD)) in visitZERO_EXTEND()
9018 if (!LegalOperations && VT.isVector() && in visitZERO_EXTEND()
9030 SDValue VecOnes = DAG.getConstant(1, DL, VT); in visitZERO_EXTEND()
9031 if (VT.getSizeInBits() == N00VT.getSizeInBits()) { in visitZERO_EXTEND()
9033 SDValue VSetCC = DAG.getNode(ISD::SETCC, DL, VT, N0.getOperand(0), in visitZERO_EXTEND()
9035 return DAG.getNode(ISD::AND, DL, VT, VSetCC, VecOnes); in visitZERO_EXTEND()
9045 return DAG.getNode(ISD::AND, DL, VT, DAG.getSExtOrTrunc(VsetCC, DL, VT), in visitZERO_EXTEND()
9052 DL, N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, DL, VT), in visitZERO_EXTEND()
9053 DAG.getConstant(0, DL, VT), in visitZERO_EXTEND()
9078 if (VT.getSizeInBits() >= 256) in visitZERO_EXTEND()
9081 return DAG.getNode(N0.getOpcode(), DL, VT, in visitZERO_EXTEND()
9082 DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)), in visitZERO_EXTEND()
9094 EVT VT = N->getValueType(0); in visitANY_EXTEND() local
9105 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0)); in visitANY_EXTEND()
9123 return DAG.getAnyExtOrTrunc(N0.getOperand(0), SDLoc(N), VT); in visitANY_EXTEND()
9134 X = DAG.getAnyExtOrTrunc(X, DL, VT); in visitANY_EXTEND()
9136 Mask = Mask.zext(VT.getSizeInBits()); in visitANY_EXTEND()
9137 return DAG.getNode(ISD::AND, DL, VT, in visitANY_EXTEND()
9138 X, DAG.getConstant(Mask, DL, VT)); in visitANY_EXTEND()
9145 if (ISD::isNON_EXTLoad(N0.getNode()) && !VT.isVector() && in visitANY_EXTEND()
9147 TLI.isLoadExtLegal(ISD::EXTLOAD, VT, N0.getValueType())) { in visitANY_EXTEND()
9151 DoXform = ExtendUsesToFormExtLoad(VT, N, N0, ISD::ANY_EXTEND, SetCCs, in visitANY_EXTEND()
9155 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, in visitANY_EXTEND()
9182 if (!LegalOperations || TLI.isLoadExtLegal(ExtType, VT, MemVT)) { in visitANY_EXTEND()
9184 VT, LN0->getChain(), LN0->getBasePtr(), in visitANY_EXTEND()
9198 if (VT.isVector() && !LegalOperations) { in visitANY_EXTEND()
9208 if (VT.getSizeInBits() == N00VT.getSizeInBits()) in visitANY_EXTEND()
9209 return DAG.getSetCC(SDLoc(N), VT, N0.getOperand(0), in visitANY_EXTEND()
9221 return DAG.getAnyExtOrTrunc(VsetCC, SDLoc(N), VT); in visitANY_EXTEND()
9227 DL, N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, DL, VT), in visitANY_EXTEND()
9228 DAG.getConstant(0, DL, VT), in visitANY_EXTEND()
9302 EVT VT = N->getValueType(0); in ReduceLoadWidth() local
9303 EVT ExtVT = VT; in ReduceLoadWidth()
9306 if (VT.isVector()) in ReduceLoadWidth()
9334 VT.getSizeInBits() - ShiftAmt); in ReduceLoadWidth()
9410 ExtVT == VT && TLI.isNarrowingProfitable(N0.getValueType(), VT)) { in ReduceLoadWidth()
9451 Load = DAG.getLoad(VT, SDLoc(N0), LN0->getChain(), NewPtr, in ReduceLoadWidth()
9455 Load = DAG.getExtLoad(ExtType, SDLoc(N0), VT, LN0->getChain(), NewPtr, in ReduceLoadWidth()
9469 ShImmTy = VT; in ReduceLoadWidth()
9475 if (ShLeftAmt >= VT.getSizeInBits()) in ReduceLoadWidth()
9476 Result = DAG.getConstant(0, DL, VT); in ReduceLoadWidth()
9478 Result = DAG.getNode(ISD::SHL, DL, VT, in ReduceLoadWidth()
9492 SDValue ShiftC = DAG.getConstant(ShAmt, DL, VT); in ReduceLoadWidth()
9493 Result = DAG.getNode(ISD::SHL, DL, VT, Result, ShiftC); in ReduceLoadWidth()
9504 EVT VT = N->getValueType(0); in visitSIGN_EXTEND_INREG() local
9506 unsigned VTBits = VT.getScalarSizeInBits(); in visitSIGN_EXTEND_INREG()
9510 return DAG.getUNDEF(VT); in visitSIGN_EXTEND_INREG()
9514 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0, N1); in visitSIGN_EXTEND_INREG()
9523 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
9535 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT))) in visitSIGN_EXTEND_INREG()
9536 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00); in visitSIGN_EXTEND_INREG()
9545 TLI.isOperationLegal(ISD::SIGN_EXTEND_VECTOR_INREG, VT)) in visitSIGN_EXTEND_INREG()
9546 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
9555 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT))) in visitSIGN_EXTEND_INREG()
9556 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00, N1); in visitSIGN_EXTEND_INREG()
9583 return DAG.getNode(ISD::SRA, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
9597 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, EVT))) { in visitSIGN_EXTEND_INREG()
9599 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
9613 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, EVT))) { in visitSIGN_EXTEND_INREG()
9615 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
9628 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
9637 EVT VT = N->getValueType(0); in visitSIGN_EXTEND_VECTOR_INREG() local
9640 return DAG.getUNDEF(VT); in visitSIGN_EXTEND_VECTOR_INREG()
9653 EVT VT = N->getValueType(0); in visitZERO_EXTEND_VECTOR_INREG() local
9656 return DAG.getUNDEF(VT); in visitZERO_EXTEND_VECTOR_INREG()
9669 EVT VT = N->getValueType(0); in visitTRUNCATE() local
9678 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0)); in visitTRUNCATE()
9682 SDValue C = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0); in visitTRUNCATE()
9692 if (N0.getOperand(0).getValueType().bitsLT(VT)) in visitTRUNCATE()
9693 return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, N0.getOperand(0)); in visitTRUNCATE()
9695 if (N0.getOperand(0).getValueType().bitsGT(VT)) in visitTRUNCATE()
9696 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0)); in visitTRUNCATE()
9717 LegalTypes && !LegalOperations && N0->hasOneUse() && VT != MVT::i1) { in visitTRUNCATE()
9745 TLI.isTruncateFree(SrcVT, VT)) { in visitTRUNCATE()
9748 SDValue TruncOp0 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1)); in visitTRUNCATE()
9749 SDValue TruncOp1 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(2)); in visitTRUNCATE()
9750 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, Cond, TruncOp0, TruncOp1); in visitTRUNCATE()
9756 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::SHL, VT)) && in visitTRUNCATE()
9757 TLI.isTypeDesirableForOp(ISD::SHL, VT)) { in visitTRUNCATE()
9760 unsigned Size = VT.getScalarSizeInBits(); in visitTRUNCATE()
9763 EVT AmtVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); in visitTRUNCATE()
9765 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0)); in visitTRUNCATE()
9770 return DAG.getNode(ISD::SHL, SL, VT, Trunc, Amt); in visitTRUNCATE()
9778 if (Level == AfterLegalizeVectorOps && VT.isVector() && in visitTRUNCATE()
9784 EVT TruncVecEltTy = VT.getVectorElementType(); in visitTRUNCATE()
9790 unsigned TruncVecNumElts = VT.getVectorNumElements(); in visitTRUNCATE()
9800 return DAG.getBuildVector(VT, SDLoc(N), Opnds); in visitTRUNCATE()
9809 if (!VT.isVector()) { in visitTRUNCATE()
9811 APInt::getLowBitsSet(N0.getValueSizeInBits(), VT.getSizeInBits()); in visitTRUNCATE()
9813 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Shorter); in visitTRUNCATE()
9818 if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getOpcode(), VT)) { in visitTRUNCATE()
9827 LN0->getMemoryVT().getStoreSizeInBits() < VT.getSizeInBits()) { in visitTRUNCATE()
9829 VT, LN0->getChain(), LN0->getBasePtr(), in visitTRUNCATE()
9857 VT.getVectorElementType(), in visitTRUNCATE()
9862 return DAG.getUNDEF(VT); in visitTRUNCATE()
9876 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Opnds); in visitTRUNCATE()
9884 if (N0.getOpcode() == ISD::BITCAST && !VT.isVector()) { in visitTRUNCATE()
9887 if (SrcVT.isVector() && SrcVT.getScalarType() == VT && in visitTRUNCATE()
9894 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, VT, in visitTRUNCATE()
9900 if (!VT.isVector() && in visitTRUNCATE()
9909 (!LegalOperations || TLI.isOperationLegal(N0.getOpcode(), VT))) { in visitTRUNCATE()
9911 auto X = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0)); in visitTRUNCATE()
9912 auto Y = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1)); in visitTRUNCATE()
9913 auto VTs = DAG.getVTList(VT, N0->getValueType(1)); in visitTRUNCATE()
9927 VT.getVectorElementType()) in visitTRUNCATE()
9928 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N0->getOperand(0)), VT, in visitTRUNCATE()
9953 if (VT.isScalarInteger() || TLI.isOperationLegal(N0.getOpcode(), VT)) { in visitTRUNCATE()
9955 SDValue NarrowL = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0)); in visitTRUNCATE()
9956 SDValue NarrowR = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1)); in visitTRUNCATE()
9957 return DAG.getNode(N0.getOpcode(), DL, VT, NarrowL, NarrowR); in visitTRUNCATE()
9974 SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) { in CombineConsecutiveLoads() argument
9995 VT.getTypeForEVT(*DAG.getContext())); in CombineConsecutiveLoads()
9998 (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT))) in CombineConsecutiveLoads()
9999 return DAG.getLoad(VT, SDLoc(N), LD1->getChain(), LD1->getBasePtr(), in CombineConsecutiveLoads()
10016 EVT VT = N->getValueType(0); in foldBitcastedFPLogic() local
10017 if (!VT.isFloatingPoint() || !TLI.hasBitPreservingFPLogic(VT)) in foldBitcastedFPLogic()
10024 if (VT.getScalarSizeInBits() != SourceVT.getScalarSizeInBits()) in foldBitcastedFPLogic()
10054 LogicOp0.getOperand(0).getValueType() == VT) { in foldBitcastedFPLogic()
10055 SDValue FPOp = DAG.getNode(FPOpcode, SDLoc(N), VT, LogicOp0.getOperand(0)); in foldBitcastedFPLogic()
10058 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, FPOp); in foldBitcastedFPLogic()
10067 EVT VT = N->getValueType(0); in visitBITCAST() local
10070 return DAG.getUNDEF(VT); in visitBITCAST()
10080 VT.isVector() && cast<BuildVectorSDNode>(N0)->isConstant()) in visitBITCAST()
10082 VT.getVectorElementType()); in visitBITCAST()
10090 (isa<ConstantSDNode>(N0) && VT.isFloatingPoint() && !VT.isVector() && in visitBITCAST()
10091 TLI.isOperationLegal(ISD::ConstantFP, VT)) || in visitBITCAST()
10092 (isa<ConstantFPSDNode>(N0) && VT.isInteger() && !VT.isVector() && in visitBITCAST()
10093 TLI.isOperationLegal(ISD::Constant, VT))) { in visitBITCAST()
10094 SDValue C = DAG.getBitcast(VT, N0); in visitBITCAST()
10102 return DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
10109 TLI.hasBigEndianPartOrdering(VT, DAG.getDataLayout()) && in visitBITCAST()
10116 TLI.isOperationLegal(ISD::LOAD, VT)) && in visitBITCAST()
10117 TLI.isLoadBitCastBeneficial(N0.getValueType(), VT)) { in visitBITCAST()
10122 if (TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, in visitBITCAST()
10126 DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(), in visitBITCAST()
10151 N0.getNode()->hasOneUse() && VT.isInteger() && in visitBITCAST()
10152 !VT.isVector() && !N0.getValueType().isVector()) { in visitBITCAST()
10153 SDValue NewConv = DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
10158 assert(VT.getSizeInBits() == 128); in visitBITCAST()
10160 APInt::getSignMask(VT.getSizeInBits() / 2), SDLoc(N0), MVT::i64); in visitBITCAST()
10176 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit); in visitBITCAST()
10178 return DAG.getNode(ISD::XOR, DL, VT, NewConv, FlipBits); in visitBITCAST()
10180 APInt SignBit = APInt::getSignMask(VT.getSizeInBits()); in visitBITCAST()
10182 return DAG.getNode(ISD::XOR, DL, VT, in visitBITCAST()
10183 NewConv, DAG.getConstant(SignBit, DL, VT)); in visitBITCAST()
10185 return DAG.getNode(ISD::AND, DL, VT, in visitBITCAST()
10186 NewConv, DAG.getConstant(~SignBit, DL, VT)); in visitBITCAST()
10202 VT.isInteger() && !VT.isVector()) { in visitBITCAST()
10210 unsigned VTWidth = VT.getSizeInBits(); in visitBITCAST()
10212 X = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, X); in visitBITCAST()
10223 X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X); in visitBITCAST()
10228 APInt SignBit = APInt::getSignMask(VT.getSizeInBits() / 2); in visitBITCAST()
10229 SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
10231 SDValue X = DAG.getBitcast(VT, N0.getOperand(1)); in visitBITCAST()
10233 SDValue XorResult = DAG.getNode(ISD::XOR, SDLoc(N0), VT, Cst, X); in visitBITCAST()
10245 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit); in visitBITCAST()
10247 return DAG.getNode(ISD::XOR, SDLoc(N), VT, Cst, FlipBits); in visitBITCAST()
10249 APInt SignBit = APInt::getSignMask(VT.getSizeInBits()); in visitBITCAST()
10250 X = DAG.getNode(ISD::AND, SDLoc(X), VT, in visitBITCAST()
10251 X, DAG.getConstant(SignBit, SDLoc(X), VT)); in visitBITCAST()
10254 SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0)); in visitBITCAST()
10255 Cst = DAG.getNode(ISD::AND, SDLoc(Cst), VT, in visitBITCAST()
10256 Cst, DAG.getConstant(~SignBit, SDLoc(Cst), VT)); in visitBITCAST()
10259 return DAG.getNode(ISD::OR, SDLoc(N), VT, X, Cst); in visitBITCAST()
10265 if (SDValue CombineLD = CombineConsecutiveLoads(N0.getNode(), VT)) in visitBITCAST()
10272 if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT) && VT.isVector() && in visitBITCAST()
10274 VT.getVectorNumElements() >= N0.getValueType().getVectorNumElements() && in visitBITCAST()
10275 !(VT.getVectorNumElements() % N0.getValueType().getVectorNumElements())) { in visitBITCAST()
10282 Op.getOperand(0).getValueType() == VT) in visitBITCAST()
10286 return DAG.getBitcast(VT, Op); in visitBITCAST()
10299 VT.getVectorNumElements() / N0.getValueType().getVectorNumElements(); in visitBITCAST()
10305 bool LegalMask = TLI.isShuffleMaskLegal(NewMask, VT); in visitBITCAST()
10309 LegalMask = TLI.isShuffleMaskLegal(NewMask, VT); in visitBITCAST()
10313 return DAG.getVectorShuffle(VT, SDLoc(N), SV0, SV1, NewMask); in visitBITCAST()
10320 EVT VT = N->getValueType(0); in visitBUILD_PAIR() local
10321 return CombineConsecutiveLoads(N, VT); in visitBUILD_PAIR()
10348 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, in ConstantFoldBITCASTofBUILD_VECTOR() local
10350 return DAG.getBuildVector(VT, SDLoc(BV), Ops); in ConstantFoldBITCASTofBUILD_VECTOR()
10405 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, Ops.size()); in ConstantFoldBITCASTofBUILD_VECTOR() local
10406 return DAG.getBuildVector(VT, DL, Ops); in ConstantFoldBITCASTofBUILD_VECTOR()
10412 EVT VT = EVT::getVectorVT(*DAG.getContext(), DstEltVT, in ConstantFoldBITCASTofBUILD_VECTOR() local
10436 return DAG.getBuildVector(VT, DL, Ops); in ConstantFoldBITCASTofBUILD_VECTOR()
10448 EVT VT = N->getValueType(0); in visitFADDForFMACombine() local
10454 bool HasFMAD = (LegalOperations && TLI.isOperationLegal(ISD::FMAD, VT)); in visitFADDForFMACombine()
10458 TLI.isFMAFasterThanFMulAndFAdd(VT) && in visitFADDForFMACombine()
10459 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFADDForFMACombine()
10479 bool Aggressive = TLI.enableAggressiveFMAFusion(VT); in visitFADDForFMACombine()
10497 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
10504 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
10514 TLI.isFPExtFoldable(PreferredFusedOpcode, VT, N00.getValueType())) { in visitFADDForFMACombine()
10515 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
10516 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFADDForFMACombine()
10518 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFADDForFMACombine()
10528 TLI.isFPExtFoldable(PreferredFusedOpcode, VT, N10.getValueType())) { in visitFADDForFMACombine()
10529 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
10530 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFADDForFMACombine()
10532 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFADDForFMACombine()
10544 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
10546 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
10557 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
10559 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
10571 return DAG.getNode(PreferredFusedOpcode, SL, VT, X, Y, in visitFADDForFMACombine()
10572 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
10573 DAG.getNode(ISD::FP_EXTEND, SL, VT, U), in visitFADDForFMACombine()
10574 DAG.getNode(ISD::FP_EXTEND, SL, VT, V), in visitFADDForFMACombine()
10582 TLI.isFPExtFoldable(PreferredFusedOpcode, VT, N020.getValueType())) { in visitFADDForFMACombine()
10598 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
10599 DAG.getNode(ISD::FP_EXTEND, SL, VT, X), in visitFADDForFMACombine()
10600 DAG.getNode(ISD::FP_EXTEND, SL, VT, Y), in visitFADDForFMACombine()
10601 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFADDForFMACombine()
10602 DAG.getNode(ISD::FP_EXTEND, SL, VT, U), in visitFADDForFMACombine()
10603 DAG.getNode(ISD::FP_EXTEND, SL, VT, V), in visitFADDForFMACombine()
10611 TLI.isFPExtFoldable(PreferredFusedOpcode, VT, N00.getValueType())) { in visitFADDForFMACombine()
10626 TLI.isFPExtFoldable(PreferredFusedOpcode, VT, N120.getValueType())) { in visitFADDForFMACombine()
10644 TLI.isFPExtFoldable(PreferredFusedOpcode, VT, N10.getValueType())) { in visitFADDForFMACombine()
10660 EVT VT = N->getValueType(0); in visitFSUBForFMACombine() local
10665 bool HasFMAD = (LegalOperations && TLI.isOperationLegal(ISD::FMAD, VT)); in visitFSUBForFMACombine()
10669 TLI.isFMAFasterThanFMulAndFAdd(VT) && in visitFSUBForFMACombine()
10670 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFSUBForFMACombine()
10691 bool Aggressive = TLI.enableAggressiveFMAFusion(VT); in visitFSUBForFMACombine()
10703 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
10705 DAG.getNode(ISD::FNEG, SL, VT, N1), Flags); in visitFSUBForFMACombine()
10711 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
10712 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
10722 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
10723 DAG.getNode(ISD::FNEG, SL, VT, N00), N01, in visitFSUBForFMACombine()
10724 DAG.getNode(ISD::FNEG, SL, VT, N1), Flags); in visitFSUBForFMACombine()
10734 TLI.isFPExtFoldable(PreferredFusedOpcode, VT, N00.getValueType())) { in visitFSUBForFMACombine()
10735 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
10736 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFSUBForFMACombine()
10738 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFSUBForFMACombine()
10740 DAG.getNode(ISD::FNEG, SL, VT, N1), Flags); in visitFSUBForFMACombine()
10750 TLI.isFPExtFoldable(PreferredFusedOpcode, VT, N10.getValueType())) { in visitFSUBForFMACombine()
10751 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
10752 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
10753 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFSUBForFMACombine()
10755 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFSUBForFMACombine()
10772 TLI.isFPExtFoldable(PreferredFusedOpcode, VT, N00.getValueType())) { in visitFSUBForFMACombine()
10773 return DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
10774 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
10775 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFSUBForFMACombine()
10777 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFSUBForFMACombine()
10795 TLI.isFPExtFoldable(PreferredFusedOpcode, VT, N000.getValueType())) { in visitFSUBForFMACombine()
10796 return DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
10797 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
10798 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFSUBForFMACombine()
10800 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFSUBForFMACombine()
10814 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
10816 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
10819 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
10829 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
10830 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
10833 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
10834 DAG.getNode(ISD::FNEG, SL, VT, N20), in visitFSUBForFMACombine()
10846 TLI.isFPExtFoldable(PreferredFusedOpcode, VT, N020.getValueType())) { in visitFSUBForFMACombine()
10847 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
10849 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
10850 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFSUBForFMACombine()
10852 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFSUBForFMACombine()
10854 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
10871 TLI.isFPExtFoldable(PreferredFusedOpcode, VT, N00.getValueType())) { in visitFSUBForFMACombine()
10872 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
10873 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFSUBForFMACombine()
10875 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFSUBForFMACombine()
10877 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
10878 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFSUBForFMACombine()
10880 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFSUBForFMACombine()
10882 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
10894 TLI.isFPExtFoldable(PreferredFusedOpcode, VT, N120.getValueType())) { in visitFSUBForFMACombine()
10897 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
10898 DAG.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)), in visitFSUBForFMACombine()
10900 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
10901 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
10903 VT, N1200)), in visitFSUBForFMACombine()
10904 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFSUBForFMACombine()
10923 TLI.isFPExtFoldable(PreferredFusedOpcode, VT, CvtSrc.getValueType())) { in visitFSUBForFMACombine()
10926 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
10927 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
10928 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFSUBForFMACombine()
10930 DAG.getNode(ISD::FP_EXTEND, SL, VT, N101), in visitFSUBForFMACombine()
10931 DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFSUBForFMACombine()
10932 DAG.getNode(ISD::FNEG, SL, VT, in visitFSUBForFMACombine()
10934 VT, N1020)), in visitFSUBForFMACombine()
10935 DAG.getNode(ISD::FP_EXTEND, SL, VT, in visitFSUBForFMACombine()
10951 EVT VT = N->getValueType(0); in visitFMULForFMADistributiveCombine() local
10967 TLI.isFMAFasterThanFMulAndFAdd(VT) && in visitFMULForFMADistributiveCombine()
10968 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)); in visitFMULForFMADistributiveCombine()
10973 (LegalOperations && TLI.isOperationLegal(ISD::FMAD, VT)); in visitFMULForFMADistributiveCombine()
10981 bool Aggressive = TLI.enableAggressiveFMAFusion(VT); in visitFMULForFMADistributiveCombine()
10989 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y, in visitFMULForFMADistributiveCombine()
10992 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y, in visitFMULForFMADistributiveCombine()
10993 DAG.getNode(ISD::FNEG, SL, VT, Y), Flags); in visitFMULForFMADistributiveCombine()
11012 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFMULForFMADistributiveCombine()
11013 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y, in visitFMULForFMADistributiveCombine()
11016 return DAG.getNode(PreferredFusedOpcode, SL, VT, in visitFMULForFMADistributiveCombine()
11017 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y, in visitFMULForFMADistributiveCombine()
11018 DAG.getNode(ISD::FNEG, SL, VT, Y), Flags); in visitFMULForFMADistributiveCombine()
11022 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y, in visitFMULForFMADistributiveCombine()
11023 DAG.getNode(ISD::FNEG, SL, VT, Y), Flags); in visitFMULForFMADistributiveCombine()
11025 return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y, in visitFMULForFMADistributiveCombine()
11045 EVT VT = N->getValueType(0); in visitFADD() local
11051 if (VT.isVector()) in visitFADD()
11057 return DAG.getNode(ISD::FADD, DL, VT, N0, N1, Flags); in visitFADD()
11061 return DAG.getNode(ISD::FADD, DL, VT, N1, N0, Flags); in visitFADD()
11073 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) && in visitFADD()
11075 return DAG.getNode(ISD::FSUB, DL, VT, N0, in visitFADD()
11079 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) && in visitFADD()
11081 return DAG.getNode(ISD::FSUB, DL, VT, N1, in visitFADD()
11094 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B, Flags); in visitFADD()
11095 return DAG.getNode(ISD::FSUB, DL, VT, N1, Add, Flags); in visitFADD()
11100 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B, Flags); in visitFADD()
11101 return DAG.getNode(ISD::FSUB, DL, VT, N0, Add, Flags); in visitFADD()
11112 return DAG.getConstantFP(0.0, DL, VT); in visitFADD()
11116 return DAG.getConstantFP(0.0, DL, VT); in visitFADD()
11128 SDValue NewC = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), N1, Flags); in visitFADD()
11129 return DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(0), NewC, Flags); in visitFADD()
11135 if (TLI.isOperationLegalOrCustom(ISD::FMUL, VT) && !N0CFP && !N1CFP) { in visitFADD()
11142 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), in visitFADD()
11143 DAG.getConstantFP(1.0, DL, VT), Flags); in visitFADD()
11144 return DAG.getNode(ISD::FMUL, DL, VT, N1, NewCFP, Flags); in visitFADD()
11151 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), in visitFADD()
11152 DAG.getConstantFP(2.0, DL, VT), Flags); in visitFADD()
11153 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), NewCFP, Flags); in visitFADD()
11163 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1), in visitFADD()
11164 DAG.getConstantFP(1.0, DL, VT), Flags); in visitFADD()
11165 return DAG.getNode(ISD::FMUL, DL, VT, N0, NewCFP, Flags); in visitFADD()
11172 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1), in visitFADD()
11173 DAG.getConstantFP(2.0, DL, VT), Flags); in visitFADD()
11174 return DAG.getNode(ISD::FMUL, DL, VT, N1.getOperand(0), NewCFP, Flags); in visitFADD()
11183 return DAG.getNode(ISD::FMUL, DL, VT, in visitFADD()
11184 N1, DAG.getConstantFP(3.0, DL, VT), Flags); in visitFADD()
11193 return DAG.getNode(ISD::FMUL, DL, VT, in visitFADD()
11194 N0, DAG.getConstantFP(3.0, DL, VT), Flags); in visitFADD()
11203 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), in visitFADD()
11204 DAG.getConstantFP(4.0, DL, VT), Flags); in visitFADD()
11222 EVT VT = N->getValueType(0); in visitFSUB() local
11228 if (VT.isVector()) in visitFSUB()
11234 return DAG.getNode(ISD::FSUB, DL, VT, N0, N1, Flags); in visitFSUB()
11250 return DAG.getConstantFP(0.0f, DL, VT); in visitFSUB()
11259 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) in visitFSUB()
11260 return DAG.getNode(ISD::FNEG, DL, VT, N1, Flags); in visitFSUB()
11269 return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(1), Flags); in visitFSUB()
11272 return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(0), Flags); in visitFSUB()
11277 return DAG.getNode(ISD::FADD, DL, VT, N0, in visitFSUB()
11294 EVT VT = N->getValueType(0); in visitFMUL() local
11300 if (VT.isVector()) { in visitFMUL()
11308 return DAG.getNode(ISD::FMUL, DL, VT, N0, N1, Flags); in visitFMUL()
11313 return DAG.getNode(ISD::FMUL, DL, VT, N1, N0, Flags); in visitFMUL()
11339 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, N01, N1, Flags); in visitFMUL()
11340 return DAG.getNode(ISD::FMUL, DL, VT, N00, MulConsts, Flags); in visitFMUL()
11348 const SDValue Two = DAG.getConstantFP(2.0, DL, VT); in visitFMUL()
11349 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, Two, N1, Flags); in visitFMUL()
11350 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), MulConsts, Flags); in visitFMUL()
11356 return DAG.getNode(ISD::FADD, DL, VT, N0, N0, Flags); in visitFMUL()
11360 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) in visitFMUL()
11361 return DAG.getNode(ISD::FNEG, DL, VT, N0); in visitFMUL()
11369 return DAG.getNode(ISD::FMUL, DL, VT, in visitFMUL()
11380 TLI.isOperationLegal(ISD::FABS, VT)) { in visitFMUL()
11411 TLI.isOperationLegal(ISD::FNEG, VT)) in visitFMUL()
11412 return DAG.getNode(ISD::FNEG, DL, VT, in visitFMUL()
11413 DAG.getNode(ISD::FABS, DL, VT, X)); in visitFMUL()
11415 return DAG.getNode(ISD::FABS, DL, VT, X); in visitFMUL()
11437 EVT VT = N->getValueType(0); in visitFMA() local
11449 return DAG.getNode(ISD::FMA, DL, VT, N0, N1, N2); in visitFMA()
11460 return DAG.getNode(ISD::FADD, SDLoc(N), VT, N1, N2); in visitFMA()
11462 return DAG.getNode(ISD::FADD, SDLoc(N), VT, N0, N2); in visitFMA()
11467 return DAG.getNode(ISD::FMA, SDLoc(N), VT, N1, N0, N2); in visitFMA()
11474 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFMA()
11475 DAG.getNode(ISD::FADD, DL, VT, N1, N2.getOperand(1), in visitFMA()
11483 return DAG.getNode(ISD::FMA, DL, VT, in visitFMA()
11485 DAG.getNode(ISD::FMUL, DL, VT, N1, N0.getOperand(1), in visitFMA()
11496 return DAG.getNode(ISD::FADD, DL, VT, N0, N2); in visitFMA()
11499 (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))) { in visitFMA()
11500 SDValue RHSNeg = DAG.getNode(ISD::FNEG, DL, VT, N0); in visitFMA()
11503 return DAG.getNode(ISD::FADD, DL, VT, N2, RHSNeg); in visitFMA()
11508 (TLI.isOperationLegal(ISD::ConstantFP, VT) || in visitFMA()
11509 (N1.hasOneUse() && !TLI.isFPImmLegal(N1CFP->getValueAPF(), VT)))) { in visitFMA()
11510 return DAG.getNode(ISD::FMA, DL, VT, N0.getOperand(0), in visitFMA()
11511 DAG.getNode(ISD::FNEG, DL, VT, N1, Flags), N2); in visitFMA()
11518 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFMA()
11519 DAG.getNode(ISD::FADD, DL, VT, N1, in visitFMA()
11520 DAG.getConstantFP(1.0, DL, VT), Flags), in visitFMA()
11526 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFMA()
11527 DAG.getNode(ISD::FADD, DL, VT, N1, in visitFMA()
11528 DAG.getConstantFP(-1.0, DL, VT), Flags), in visitFMA()
11579 EVT VT = N->getValueType(0); in combineRepeatedFPDivisors() local
11581 SDValue FPOne = DAG.getConstantFP(1.0, DL, VT); in combineRepeatedFPDivisors()
11582 SDValue Reciprocal = DAG.getNode(ISD::FDIV, DL, VT, FPOne, N1, Flags); in combineRepeatedFPDivisors()
11588 SDValue NewNode = DAG.getNode(ISD::FMUL, SDLoc(U), VT, Dividend, in combineRepeatedFPDivisors()
11605 EVT VT = N->getValueType(0); in visitFDIV() local
11611 if (VT.isVector()) in visitFDIV()
11617 return DAG.getNode(ISD::FDIV, SDLoc(N), VT, N0, N1, Flags); in visitFDIV()
11636 TLI.isOperationLegal(ISD::ConstantFP, VT) || in visitFDIV()
11637 TLI.isFPImmLegal(Recip, VT))) in visitFDIV()
11638 return DAG.getNode(ISD::FMUL, DL, VT, N0, in visitFDIV()
11639 DAG.getConstantFP(Recip, DL, VT), Flags); in visitFDIV()
11646 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV, Flags); in visitFDIV()
11652 RV = DAG.getNode(ISD::FP_EXTEND, SDLoc(N1), VT, RV); in visitFDIV()
11654 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV, Flags); in visitFDIV()
11660 RV = DAG.getNode(ISD::FP_ROUND, SDLoc(N1), VT, RV, N1.getOperand(1)); in visitFDIV()
11662 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV, Flags); in visitFDIV()
11680 RV = DAG.getNode(ISD::FDIV, SDLoc(N1), VT, RV, OtherOp, Flags); in visitFDIV()
11682 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV, Flags); in visitFDIV()
11690 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV, Flags); in visitFDIV()
11700 return DAG.getNode(ISD::FDIV, SDLoc(N), VT, in visitFDIV()
11718 EVT VT = N->getValueType(0); in visitFREM() local
11722 return DAG.getNode(ISD::FREM, SDLoc(N), VT, N0, N1, N->getFlags()); in visitFREM()
11766 EVT VT = N->getValueType(0); in visitFCOPYSIGN() local
11769 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1); in visitFCOPYSIGN()
11776 if (!LegalOperations || TLI.isOperationLegal(ISD::FABS, VT)) in visitFCOPYSIGN()
11777 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFCOPYSIGN()
11779 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) in visitFCOPYSIGN()
11780 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, in visitFCOPYSIGN()
11781 DAG.getNode(ISD::FABS, SDLoc(N0), VT, N0)); in visitFCOPYSIGN()
11790 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0.getOperand(0), N1); in visitFCOPYSIGN()
11794 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFCOPYSIGN()
11798 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1.getOperand(1)); in visitFCOPYSIGN()
11803 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1.getOperand(0)); in visitFCOPYSIGN()
11817 EVT VT = N->getValueType(0); in visitFPOW() local
11818 if ((VT == MVT::f32 && ExponentC->getValueAPF().isExactlyValue(1.0f/3.0f)) || in visitFPOW()
11819 (VT == MVT::f64 && ExponentC->getValueAPF().isExactlyValue(1.0/3.0))) { in visitFPOW()
11834 (!DAG.getTargetLoweringInfo().isOperationExpand(ISD::FPOW, VT) && in visitFPOW()
11835 DAG.getTargetLoweringInfo().isOperationExpand(ISD::FCBRT, VT))) in visitFPOW()
11838 return DAG.getNode(ISD::FCBRT, SDLoc(N), VT, N->getOperand(0), Flags); in visitFPOW()
11857 if (!DAG.getTargetLoweringInfo().isOperationLegalOrCustom(ISD::FSQRT, VT)) in visitFPOW()
11867 SDValue Sqrt = DAG.getNode(ISD::FSQRT, DL, VT, N->getOperand(0), Flags); in visitFPOW()
11868 return DAG.getNode(ISD::FSQRT, DL, VT, Sqrt, Flags); in visitFPOW()
11890 EVT VT = N->getValueType(0); in foldFPToIntToFP() local
11891 if (!TLI.isOperationLegal(ISD::FTRUNC, VT) || in foldFPToIntToFP()
11899 N0.getOperand(0).getValueType() == VT) in foldFPToIntToFP()
11900 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0.getOperand(0)); in foldFPToIntToFP()
11903 N0.getOperand(0).getValueType() == VT) in foldFPToIntToFP()
11904 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0.getOperand(0)); in foldFPToIntToFP()
11911 EVT VT = N->getValueType(0); in visitSINT_TO_FP() local
11918 TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) in visitSINT_TO_FP()
11919 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); in visitSINT_TO_FP()
11927 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); in visitSINT_TO_FP()
11931 if (TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT) || !LegalOperations) { in visitSINT_TO_FP()
11934 !VT.isVector() && in visitSINT_TO_FP()
11936 TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitSINT_TO_FP()
11940 DAG.getConstantFP(-1.0, DL, VT), DAG.getConstantFP(0.0, DL, VT), in visitSINT_TO_FP()
11942 return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops); in visitSINT_TO_FP()
11948 N0.getOperand(0).getOpcode() == ISD::SETCC &&!VT.isVector() && in visitSINT_TO_FP()
11950 TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitSINT_TO_FP()
11954 DAG.getConstantFP(1.0, DL, VT), DAG.getConstantFP(0.0, DL, VT), in visitSINT_TO_FP()
11956 return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops); in visitSINT_TO_FP()
11968 EVT VT = N->getValueType(0); in visitUINT_TO_FP() local
11975 TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) in visitUINT_TO_FP()
11976 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); in visitUINT_TO_FP()
11984 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); in visitUINT_TO_FP()
11988 if (TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT) || !LegalOperations) { in visitUINT_TO_FP()
11990 if (N0.getOpcode() == ISD::SETCC && !VT.isVector() && in visitUINT_TO_FP()
11992 TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT))) { in visitUINT_TO_FP()
11996 DAG.getConstantFP(1.0, DL, VT), DAG.getConstantFP(0.0, DL, VT), in visitUINT_TO_FP()
11998 return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops); in visitUINT_TO_FP()
12011 EVT VT = N->getValueType(0); in FoldIntToFPToInt() local
12031 unsigned OutputSize = (int)VT.getScalarSizeInBits() - IsOutputSigned; in FoldIntToFPToInt()
12038 if (VT.getScalarSizeInBits() > SrcVT.getScalarSizeInBits()) { in FoldIntToFPToInt()
12041 return DAG.getNode(ExtOp, SDLoc(N), VT, Src); in FoldIntToFPToInt()
12043 if (VT.getScalarSizeInBits() < SrcVT.getScalarSizeInBits()) in FoldIntToFPToInt()
12044 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Src); in FoldIntToFPToInt()
12045 return DAG.getBitcast(VT, Src); in FoldIntToFPToInt()
12052 EVT VT = N->getValueType(0); in visitFP_TO_SINT() local
12056 return DAG.getNode(ISD::FP_TO_SINT, SDLoc(N), VT, N0); in visitFP_TO_SINT()
12063 EVT VT = N->getValueType(0); in visitFP_TO_UINT() local
12067 return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), VT, N0); in visitFP_TO_UINT()
12076 EVT VT = N->getValueType(0); in visitFP_ROUND() local
12080 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, N0, N1); in visitFP_ROUND()
12083 if (N0.getOpcode() == ISD::FP_EXTEND && VT == N0.getOperand(0).getValueType()) in visitFP_ROUND()
12098 if (N0.getOperand(0).getValueType() == MVT::f80 && VT == MVT::f16) in visitFP_ROUND()
12108 return DAG.getNode(ISD::FP_ROUND, DL, VT, N0.getOperand(0), in visitFP_ROUND()
12115 SDValue Tmp = DAG.getNode(ISD::FP_ROUND, SDLoc(N0), VT, in visitFP_ROUND()
12118 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, in visitFP_ROUND()
12130 EVT VT = N->getValueType(0); in visitFP_ROUND_INREG() local
12138 return DAG.getNode(ISD::FP_EXTEND, DL, VT, Round); in visitFP_ROUND_INREG()
12146 EVT VT = N->getValueType(0); in visitFP_EXTEND() local
12155 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, N0); in visitFP_EXTEND()
12159 TLI.getOperationAction(ISD::FP16_TO_FP, VT) == TargetLowering::Legal) in visitFP_EXTEND()
12160 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), VT, N0.getOperand(0)); in visitFP_EXTEND()
12167 if (In.getValueType() == VT) return In; in visitFP_EXTEND()
12168 if (VT.bitsLT(In.getValueType())) in visitFP_EXTEND()
12169 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, in visitFP_EXTEND()
12171 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, In); in visitFP_EXTEND()
12176 TLI.isLoadExtLegal(ISD::EXTLOAD, VT, N0.getValueType())) { in visitFP_EXTEND()
12178 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, in visitFP_EXTEND()
12199 EVT VT = N->getValueType(0); in visitFCEIL() local
12203 return DAG.getNode(ISD::FCEIL, SDLoc(N), VT, N0); in visitFCEIL()
12210 EVT VT = N->getValueType(0); in visitFTRUNC() local
12214 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0); in visitFTRUNC()
12234 EVT VT = N->getValueType(0); in visitFFLOOR() local
12238 return DAG.getNode(ISD::FFLOOR, SDLoc(N), VT, N0); in visitFFLOOR()
12246 EVT VT = N->getValueType(0); in visitFNEG() local
12250 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0); in visitFNEG()
12258 if (!TLI.isFNegFree(VT) && in visitFNEG()
12278 return DAG.getBitcast(VT, Int); in visitFNEG()
12284 (N0.getNode()->hasOneUse() || !TLI.isFNegFree(VT))) { in visitFNEG()
12290 (TLI.isFPImmLegal(CVal, VT) || in visitFNEG()
12291 TLI.isOperationLegal(ISD::ConstantFP, VT))) in visitFNEG()
12293 ISD::FMUL, SDLoc(N), VT, N0.getOperand(0), in visitFNEG()
12294 DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0.getOperand(1)), in visitFNEG()
12306 EVT VT = N->getValueType(0); in visitFMinMax() local
12313 return DAG.getConstantFP(Op(C0, C1), SDLoc(N), VT); in visitFMinMax()
12319 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0); in visitFMinMax()
12342 EVT VT = N->getValueType(0); in visitFABS() local
12346 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFABS()
12355 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0.getOperand(0)); in visitFABS()
12358 if (!TLI.isFAbsFree(VT) && N0.getOpcode() == ISD::BITCAST && N0.hasOneUse()) { in visitFABS()
12541 EVT VT; in canFoldInAddressingMode() local
12547 VT = LD->getMemoryVT(); in canFoldInAddressingMode()
12552 VT = ST->getMemoryVT(); in canFoldInAddressingMode()
12578 VT.getTypeForEVT(*DAG.getContext()), AS); in canFoldInAddressingMode()
12592 EVT VT; in CombineToPreIndexedLoadStore() local
12596 VT = LD->getMemoryVT(); in CombineToPreIndexedLoadStore()
12597 if (!TLI.isIndexedLoadLegal(ISD::PRE_INC, VT) && in CombineToPreIndexedLoadStore()
12598 !TLI.isIndexedLoadLegal(ISD::PRE_DEC, VT)) in CombineToPreIndexedLoadStore()
12604 VT = ST->getMemoryVT(); in CombineToPreIndexedLoadStore()
12605 if (!TLI.isIndexedStoreLegal(ISD::PRE_INC, VT) && in CombineToPreIndexedLoadStore()
12606 !TLI.isIndexedStoreLegal(ISD::PRE_DEC, VT)) in CombineToPreIndexedLoadStore()
12817 EVT VT; in CombineToPostIndexedLoadStore() local
12821 VT = LD->getMemoryVT(); in CombineToPostIndexedLoadStore()
12822 if (!TLI.isIndexedLoadLegal(ISD::POST_INC, VT) && in CombineToPostIndexedLoadStore()
12823 !TLI.isIndexedLoadLegal(ISD::POST_DEC, VT)) in CombineToPostIndexedLoadStore()
12829 VT = ST->getMemoryVT(); in CombineToPostIndexedLoadStore()
12830 if (!TLI.isIndexedStoreLegal(ISD::POST_INC, VT) && in CombineToPostIndexedLoadStore()
12831 !TLI.isIndexedStoreLegal(ISD::POST_DEC, VT)) in CombineToPostIndexedLoadStore()
13871 MVT VT = MVT::getIntegerVT(NumBytes*8); in ShrinkLoadReplaceStoreWithStore() local
13872 if (!DC->isTypeLegal(VT)) in ShrinkLoadReplaceStoreWithStore()
13902 IVal = DAG.getNode(ISD::TRUNCATE, SDLoc(IVal), VT, IVal); in ShrinkLoadReplaceStoreWithStore()
13923 EVT VT = Value.getValueType(); in ReduceLoadOpStoreWidth() local
13925 if (ST->isTruncatingStore() || VT.isVector() || !Value.hasOneUse()) in ReduceLoadOpStoreWidth()
13981 !TLI.isNarrowingProfitable(VT, NewVT))) { in ReduceLoadOpStoreWidth()
14048 EVT VT = LD->getMemoryVT(); in TransformFPLoadStorePair() local
14049 if (!VT.isFloatingPoint() || in TransformFPLoadStorePair()
14050 VT != ST->getMemoryVT() || in TransformFPLoadStorePair()
14057 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); in TransformFPLoadStorePair()
14060 !TLI.isDesirableToTransformToIntegerOp(ISD::LOAD, VT) || in TransformFPLoadStorePair()
14061 !TLI.isDesirableToTransformToIntegerOp(ISD::STORE, VT)) in TransformFPLoadStorePair()
15422 EVT VT = EVT::getIntegerVT(*DAG.getContext(), HalfValBitSize); in splitMergedValStore() local
15423 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Lo.getOperand(0)); in splitMergedValStore()
15424 Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Hi.getOperand(0)); in splitMergedValStore()
15456 EVT VT = DestVec.getValueType(); in combineInsertEltToShuffle() local
15458 unsigned ExtendRatio = VT.getSizeInBits() / SubVecVT.getSizeInBits(); in combineInsertEltToShuffle()
15493 return DAG.getBitcast(VT, Shuf); in combineInsertEltToShuffle()
15506 EVT VT = InVec.getValueType(); in visitINSERT_VECTOR_ELT() local
15507 unsigned NumElts = VT.getVectorNumElements(); in visitINSERT_VECTOR_ELT()
15519 if (InVec.isUndef() && TLI.shouldSplatInsEltVarIndex(VT)) { in visitINSERT_VECTOR_ELT()
15521 return DAG.getBuildVector(VT, DL, Ops); in visitINSERT_VECTOR_ELT()
15543 SDValue NewOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, in visitINSERT_VECTOR_ELT()
15547 VT, NewOp, InVec.getOperand(1), InVec.getOperand(2)); in visitINSERT_VECTOR_ELT()
15552 if (LegalOperations && !TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) in visitINSERT_VECTOR_ELT()
15580 return DAG.getBuildVector(VT, DL, Ops); in visitINSERT_VECTOR_ELT()
15692 EVT VT = ExtElt->getValueType(0); in scalarizeExtractedBinop() local
15693 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Index); in scalarizeExtractedBinop()
15694 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op1, Index); in scalarizeExtractedBinop()
15695 return DAG.getNode(Vec.getOpcode(), DL, VT, Ext0, Ext1); in scalarizeExtractedBinop()
15973 EVT VT = N->getValueType(0); in reduceBuildVecExtToExtBuildVec() local
16017 EVT OutScalarTy = VT.getScalarType(); in reduceBuildVecExtToExtBuildVec()
16033 unsigned NewBVElems = ElemRatio * VT.getVectorNumElements(); in reduceBuildVecExtToExtBuildVec()
16056 assert(VecVT.getSizeInBits() == VT.getSizeInBits() && in reduceBuildVecExtToExtBuildVec()
16061 TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))) in reduceBuildVecExtToExtBuildVec()
16070 return DAG.getBitcast(VT, BV); in reduceBuildVecExtToExtBuildVec()
16080 EVT VT = N->getValueType(0); in createBuildVecShuffle() local
16085 unsigned NumElems = VT.getVectorNumElements(); in createBuildVecShuffle()
16098 if (InVT1 != VT || InVT2 != VT) { in createBuildVecShuffle()
16099 if ((VT.getSizeInBits() % InVT1.getSizeInBits() == 0) && InVT1 == InVT2) { in createBuildVecShuffle()
16102 unsigned NumConcats = VT.getSizeInBits() / InVT1.getSizeInBits(); in createBuildVecShuffle()
16107 VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in createBuildVecShuffle()
16109 } else if (InVT1.getSizeInBits() == VT.getSizeInBits() * 2) { in createBuildVecShuffle()
16110 if (!TLI.isExtractSubvectorCheap(VT, InVT1, NumElems)) in createBuildVecShuffle()
16116 VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1, in createBuildVecShuffle()
16118 VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, VecIn1, ZeroIdx); in createBuildVecShuffle()
16147 } else if (InVT2.getSizeInBits() * 2 == VT.getSizeInBits() && in createBuildVecShuffle()
16148 InVT1.getSizeInBits() == VT.getSizeInBits()) { in createBuildVecShuffle()
16151 VecIn2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in createBuildVecShuffle()
16189 Shuffle = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuffle, ZeroIdx); in createBuildVecShuffle()
16220 EVT VT = BV->getValueType(0); in reduceBuildVecToShuffleWithZero() local
16225 Zext.getValueSizeInBits() != VT.getScalarSizeInBits()) in reduceBuildVecToShuffleWithZero()
16234 Extract.getOperand(0).getValueSizeInBits() != VT.getSizeInBits()) in reduceBuildVecToShuffleWithZero()
16269 return DAG.getBitcast(VT, Shuf); in reduceBuildVecToShuffleWithZero()
16277 EVT VT = N->getValueType(0); in reduceBuildVecToShuffle() local
16280 if (!isTypeLegal(VT)) in reduceBuildVecToShuffle()
16287 if (LegalOperations && !TLI.isOperationLegal(ISD::VECTOR_SHUFFLE, VT)) in reduceBuildVecToShuffle()
16330 if (VT.getVectorElementType() != in reduceBuildVecToShuffle()
16429 Shuffles.push_back(VT.isInteger() ? DAG.getConstant(0, DL, VT) in reduceBuildVecToShuffle()
16430 : DAG.getConstantFP(0.0, DL, VT)); in reduceBuildVecToShuffle()
16456 Shuffles.push_back(DAG.getUNDEF(VT)); in reduceBuildVecToShuffle()
16460 Shuffles[CurSize] = DAG.getUNDEF(VT); in reduceBuildVecToShuffle()
16478 DAG.getVectorShuffle(VT, DL, Shuffles[Left], Shuffles[Right], Mask); in reduceBuildVecToShuffle()
16493 EVT VT = N->getValueType(0); in convertBuildVecZextToZext() local
16529 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, In); in convertBuildVecZextToZext()
16533 EVT VT = N->getValueType(0); in visitBUILD_VECTOR() local
16537 return DAG.getUNDEF(VT); in visitBUILD_VECTOR()
16559 return DAG.getBitcast(VT, Concat); in visitBUILD_VECTOR()
16615 EVT VT = N->getValueType(0); in combineConcatVectorOfScalars() local
16663 VT.getSizeInBits() / SVT.getSizeInBits()); in combineConcatVectorOfScalars()
16664 return DAG.getBitcast(VT, DAG.getBuildVector(VecVT, DL, Ops)); in combineConcatVectorOfScalars()
16672 EVT VT = N->getValueType(0); in combineConcatVectorOfExtracts() local
16674 int NumElts = VT.getVectorNumElements(); in combineConcatVectorOfExtracts()
16677 SDValue SV0 = DAG.getUNDEF(VT), SV1 = DAG.getUNDEF(VT); in combineConcatVectorOfExtracts()
16712 if (ExtVT.getSizeInBits() != VT.getSizeInBits()) in combineConcatVectorOfExtracts()
16738 if (!DAG.getTargetLoweringInfo().isShuffleMaskLegal(Mask, VT)) in combineConcatVectorOfExtracts()
16741 return DAG.getVectorShuffle(VT, SDLoc(N), DAG.getBitcast(VT, SV0), in combineConcatVectorOfExtracts()
16742 DAG.getBitcast(VT, SV1), Mask); in combineConcatVectorOfExtracts()
16751 EVT VT = N->getValueType(0); in visitCONCAT_VECTORS() local
16753 return DAG.getUNDEF(VT); in visitCONCAT_VECTORS()
16789 if (VT.getSizeInBits() % SclTy.getSizeInBits()) in visitCONCAT_VECTORS()
16792 unsigned VNTNumElms = VT.getSizeInBits() / SclTy.getSizeInBits(); in visitCONCAT_VECTORS()
16801 return DAG.getBitcast(VT, Res); in visitCONCAT_VECTORS()
16814 EVT SVT = VT.getScalarType(); in visitCONCAT_VECTORS()
16849 assert(VT.getVectorNumElements() == Opnds.size() && in visitCONCAT_VECTORS()
16851 return DAG.getBuildVector(VT, SDLoc(N), Opnds); in visitCONCAT_VECTORS()
16859 if (Level < AfterLegalizeVectorOps && TLI.isTypeLegal(VT)) in visitCONCAT_VECTORS()
16933 EVT VT = Extract->getValueType(0); in narrowExtractedVectorBinOp() local
16935 assert(ExtractIndex % VT.getVectorNumElements() == 0 && in narrowExtractedVectorBinOp()
16940 unsigned NarrowWidth = VT.getSizeInBits(); in narrowExtractedVectorBinOp()
16963 unsigned ConcatOpNum = ExtractIndex / VT.getVectorNumElements(); in narrowExtractedVectorBinOp()
16977 return DAG.getBitcast(VT, NarrowBinOp); in narrowExtractedVectorBinOp()
17023 return DAG.getBitcast(VT, NarrowBinOp); in narrowExtractedVectorBinOp()
17040 EVT VT = Extract->getValueType(0); in narrowExtractedVectorLoad() local
17042 if (!TLI.shouldReduceLoadWidth(Ld, Ld->getExtensionType(), VT)) in narrowExtractedVectorLoad()
17049 unsigned Offset = ExtIdx->getZExtValue() * VT.getScalarType().getStoreSize(); in narrowExtractedVectorLoad()
17055 VT.getStoreSize()); in narrowExtractedVectorLoad()
17056 SDValue NewLd = DAG.getLoad(VT, DL, Ld->getChain(), NewAddr, MMO); in narrowExtractedVectorLoad()
17166 EVT VT = N->getValueType(0); in partitionShuffleOfConcats() local
17167 unsigned NumElts = VT.getVectorNumElements(); in partitionShuffleOfConcats()
17187 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, N0, N1); in partitionShuffleOfConcats()
17224 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in partitionShuffleOfConcats()
17246 EVT VT = SVN->getValueType(0); in combineShuffleOfScalars() local
17247 unsigned NumElts = VT.getVectorNumElements(); in combineShuffleOfScalars()
17280 SDValue Op = DAG.getUNDEF(VT.getScalarType()); in combineShuffleOfScalars()
17308 EVT SVT = VT.getScalarType(); in combineShuffleOfScalars()
17312 if (SVT != VT.getScalarType()) in combineShuffleOfScalars()
17317 return DAG.getBuildVector(VT, SDLoc(SVN), Ops); in combineShuffleOfScalars()
17328 EVT VT = SVN->getValueType(0); in combineShuffleToVectorExtend() local
17332 if (!VT.isInteger() || IsBigEndian) in combineShuffleToVectorExtend()
17335 unsigned NumElts = VT.getVectorNumElements(); in combineShuffleToVectorExtend()
17336 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in combineShuffleToVectorExtend()
17368 return DAG.getBitcast(VT, in combineShuffleToVectorExtend()
17383 EVT VT = SVN->getValueType(0); in combineTruncationShuffle() local
17387 if (!VT.isInteger() || IsBigEndian) in combineTruncationShuffle()
17400 unsigned NumElts = VT.getVectorNumElements(); in combineTruncationShuffle()
17401 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in combineTruncationShuffle()
17432 return DAG.getBitcast(VT, N00); in combineTruncationShuffle()
17565 EVT VT = N->getValueType(0); in visitVECTOR_SHUFFLE() local
17566 unsigned NumElts = VT.getVectorNumElements(); in visitVECTOR_SHUFFLE()
17571 assert(N0.getValueType() == VT && "Vector shuffle must be normalized in DAG"); in visitVECTOR_SHUFFLE()
17575 return DAG.getUNDEF(VT); in visitVECTOR_SHUFFLE()
17587 return DAG.getVectorShuffle(VT, SDLoc(N), N0, DAG.getUNDEF(VT), NewMask); in visitVECTOR_SHUFFLE()
17607 return DAG.getVectorShuffle(VT, SDLoc(N), N0, N1, NewMask); in visitVECTOR_SHUFFLE()
17664 if (V->getValueType(0) != VT) in visitVECTOR_SHUFFLE()
17665 NewBV = DAG.getBitcast(VT, NewBV); in visitVECTOR_SHUFFLE()
17693 if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT)) in visitVECTOR_SHUFFLE()
17702 TLI.isTypeLegal(VT)) { in visitVECTOR_SHUFFLE()
17716 EVT SVT = VT.getScalarType(); in visitVECTOR_SHUFFLE()
17721 EVT ScaleVT = SVT.bitsLT(InnerSVT) ? VT : InnerVT; in visitVECTOR_SHUFFLE()
17756 VT, DAG.getVectorShuffle(ScaleVT, SDLoc(N), SV0, SV1, NewMask)); in visitVECTOR_SHUFFLE()
17768 TLI.isTypeLegal(VT)) { in visitVECTOR_SHUFFLE()
17771 assert(N1->getOperand(0).getValueType() == VT && in visitVECTOR_SHUFFLE()
17791 Level < AfterLegalizeDAG && TLI.isTypeLegal(VT)) { in visitVECTOR_SHUFFLE()
17801 assert(OtherSV->getOperand(0).getValueType() == VT && in visitVECTOR_SHUFFLE()
17867 return DAG.getUNDEF(VT); in visitVECTOR_SHUFFLE()
17870 SV0 = DAG.getUNDEF(VT); in visitVECTOR_SHUFFLE()
17872 SV1 = DAG.getUNDEF(VT); in visitVECTOR_SHUFFLE()
17875 if (!TLI.isShuffleMaskLegal(Mask, VT)) { in visitVECTOR_SHUFFLE()
17878 if (!TLI.isShuffleMaskLegal(Mask, VT)) in visitVECTOR_SHUFFLE()
17890 return DAG.getVectorShuffle(VT, SDLoc(N), SV0, SV1, Mask); in visitVECTOR_SHUFFLE()
17898 EVT VT = N->getValueType(0); in visitSCALAR_TO_VECTOR() local
17913 if (VT.getScalarType() != InVal.getValueType() && in visitSCALAR_TO_VECTOR()
17915 isTypeLegal(VT.getScalarType())) { in visitSCALAR_TO_VECTOR()
17917 DAG.getNode(ISD::TRUNCATE, SDLoc(InVal), VT.getScalarType(), InVal); in visitSCALAR_TO_VECTOR()
17918 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Val); in visitSCALAR_TO_VECTOR()
17920 if (VT.getScalarType() == InVecT.getScalarType() && in visitSCALAR_TO_VECTOR()
17921 VT.getVectorNumElements() <= InVecT.getVectorNumElements() && in visitSCALAR_TO_VECTOR()
17922 TLI.isShuffleMaskLegal(NewMask, VT)) { in visitSCALAR_TO_VECTOR()
17927 if (VT == InVecT) in visitSCALAR_TO_VECTOR()
17930 if (VT.getVectorNumElements() != InVecT.getVectorNumElements()) { in visitSCALAR_TO_VECTOR()
17935 VT.getVectorNumElements()); in visitSCALAR_TO_VECTOR()
17948 EVT VT = N->getValueType(0); in visitINSERT_SUBVECTOR() local
17960 N1.getOperand(1) == N2 && N1.getOperand(0).getValueType() == VT) in visitINSERT_SUBVECTOR()
17971 VT.getVectorNumElements() && in visitINSERT_SUBVECTOR()
17973 VT.getSizeInBits()) { in visitINSERT_SUBVECTOR()
17974 return DAG.getBitcast(VT, N1.getOperand(0).getOperand(0)); in visitINSERT_SUBVECTOR()
17988 CN0VT.getVectorNumElements() == VT.getVectorNumElements()) { in visitINSERT_SUBVECTOR()
17991 return DAG.getBitcast(VT, NewINSERT); in visitINSERT_SUBVECTOR()
18001 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0.getOperand(0), in visitINSERT_SUBVECTOR()
18009 return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0, in visitINSERT_SUBVECTOR()
18027 SDValue NewOp = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, in visitINSERT_SUBVECTOR()
18031 VT, NewOp, N0.getOperand(1), N0.getOperand(2)); in visitINSERT_SUBVECTOR()
18044 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in visitINSERT_SUBVECTOR()
18086 EVT VT = N->getValueType(0); in XformToShuffleWithZero() local
18152 return DAG.getBitcast(VT, DAG.getVectorShuffle(ClearVT, DL, in XformToShuffleWithZero()
18195 EVT VT = N->getValueType(0); in SimplifyVBinOp() local
18197 SDValue NewBinOp = DAG.getNode(N->getOpcode(), SDLoc(N), VT, in SimplifyVBinOp()
18201 return DAG.getVectorShuffle(VT, SDLoc(N), NewBinOp, UndefVector, in SimplifyVBinOp()
18504 EVT VT = N2.getValueType(); in convertSelectOfFPConstantsToLoadOffset() local
18505 if (!TV || !FV || !TLI.isTypeLegal(VT)) in convertSelectOfFPConstantsToLoadOffset()
18509 if (TLI.getOperationAction(ISD::ConstantFP, VT) == TargetLowering::Legal || in convertSelectOfFPConstantsToLoadOffset()
18556 EVT VT = N2.getValueType(); in SimplifySelectCC() local
18586 N0->getValueType(0) == VT && isNullConstant(N1) && isNullConstant(N2)) { in SimplifySelectCC()
18595 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N0), VT, AndLHS, ShlAmt); in SimplifySelectCC()
18602 SDValue Shr = DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl, ShrAmt); in SimplifySelectCC()
18604 return DAG.getNode(ISD::AND, DL, VT, Shr, N3); in SimplifySelectCC()
18631 if (VT.bitsLT(SCC.getValueType())) in SimplifySelectCC()
18632 Temp = DAG.getZeroExtendInReg(SCC, SDLoc(N2), VT); in SimplifySelectCC()
18634 Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2), VT, SCC); in SimplifySelectCC()
18637 Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2), VT, SCC); in SimplifySelectCC()
18699 if (ValueOnZeroC->getAPIntValue() == VT.getSizeInBits()) { in SimplifySelectCC()
18705 (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ, VT))) in SimplifySelectCC()
18706 return DAG.getNode(ISD::CTTZ, DL, VT, N0); in SimplifySelectCC()
18712 (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ, VT))) in SimplifySelectCC()
18713 return DAG.getNode(ISD::CTLZ, DL, VT, N0); in SimplifySelectCC()
18722 SDValue DAGCombiner::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, in SimplifySetCC() argument
18727 return TLI.SimplifySetCC(VT, N0, N1, Cond, foldBooleans, DagCombineInfo, DL); in SimplifySetCC()
18794 EVT VT = V.getValueType(); in BuildLogBase2() local
18795 unsigned EltBits = VT.getScalarSizeInBits(); in BuildLogBase2()
18796 SDValue Ctlz = DAG.getNode(ISD::CTLZ, DL, VT, V); in BuildLogBase2()
18797 SDValue Base = DAG.getConstant(EltBits - 1, DL, VT); in BuildLogBase2()
18798 SDValue LogBase2 = DAG.getNode(ISD::SUB, DL, VT, Base, Ctlz); in BuildLogBase2()
18813 EVT VT = Op.getValueType(); in BuildReciprocalEstimate() local
18814 if (VT.getScalarType() != MVT::f32 && VT.getScalarType() != MVT::f64) in BuildReciprocalEstimate()
18819 int Enabled = TLI.getRecipEstimateDivEnabled(VT, MF); in BuildReciprocalEstimate()
18825 int Iterations = TLI.getDivRefinementSteps(VT, MF); in BuildReciprocalEstimate()
18830 EVT VT = Op.getValueType(); in BuildReciprocalEstimate() local
18832 SDValue FPOne = DAG.getConstantFP(1.0, DL, VT); in BuildReciprocalEstimate()
18836 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Op, Est, Flags); in BuildReciprocalEstimate()
18839 NewEst = DAG.getNode(ISD::FSUB, DL, VT, FPOne, NewEst, Flags); in BuildReciprocalEstimate()
18842 NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags); in BuildReciprocalEstimate()
18845 Est = DAG.getNode(ISD::FADD, DL, VT, Est, NewEst, Flags); in BuildReciprocalEstimate()
18864 EVT VT = Arg.getValueType(); in buildSqrtNROneConst() local
18866 SDValue ThreeHalves = DAG.getConstantFP(1.5, DL, VT); in buildSqrtNROneConst()
18870 SDValue HalfArg = DAG.getNode(ISD::FMUL, DL, VT, ThreeHalves, Arg, Flags); in buildSqrtNROneConst()
18873 HalfArg = DAG.getNode(ISD::FSUB, DL, VT, HalfArg, Arg, Flags); in buildSqrtNROneConst()
18878 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, Est, Flags); in buildSqrtNROneConst()
18881 NewEst = DAG.getNode(ISD::FMUL, DL, VT, HalfArg, NewEst, Flags); in buildSqrtNROneConst()
18884 NewEst = DAG.getNode(ISD::FSUB, DL, VT, ThreeHalves, NewEst, Flags); in buildSqrtNROneConst()
18887 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags); in buildSqrtNROneConst()
18893 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, Arg, Flags); in buildSqrtNROneConst()
18908 EVT VT = Arg.getValueType(); in buildSqrtNRTwoConst() local
18910 SDValue MinusThree = DAG.getConstantFP(-3.0, DL, VT); in buildSqrtNRTwoConst()
18911 SDValue MinusHalf = DAG.getConstantFP(-0.5, DL, VT); in buildSqrtNRTwoConst()
18920 SDValue AE = DAG.getNode(ISD::FMUL, DL, VT, Arg, Est, Flags); in buildSqrtNRTwoConst()
18923 SDValue AEE = DAG.getNode(ISD::FMUL, DL, VT, AE, Est, Flags); in buildSqrtNRTwoConst()
18926 SDValue RHS = DAG.getNode(ISD::FADD, DL, VT, AEE, MinusThree, Flags); in buildSqrtNRTwoConst()
18935 LHS = DAG.getNode(ISD::FMUL, DL, VT, Est, MinusHalf, Flags); in buildSqrtNRTwoConst()
18938 LHS = DAG.getNode(ISD::FMUL, DL, VT, AE, MinusHalf, Flags); in buildSqrtNRTwoConst()
18942 Est = DAG.getNode(ISD::FMUL, DL, VT, LHS, RHS, Flags); in buildSqrtNRTwoConst()
18958 EVT VT = Op.getValueType(); in buildSqrtEstimateImpl() local
18959 if (VT.getScalarType() != MVT::f32 && VT.getScalarType() != MVT::f64) in buildSqrtEstimateImpl()
18964 int Enabled = TLI.getRecipEstimateSqrtEnabled(VT, MF); in buildSqrtEstimateImpl()
18970 int Iterations = TLI.getSqrtRefinementSteps(VT, MF); in buildSqrtEstimateImpl()
18986 EVT VT = Op.getValueType(); in buildSqrtEstimateImpl() local
18988 EVT CCVT = getSetCCResultType(VT); in buildSqrtEstimateImpl()
18989 ISD::NodeType SelOpcode = VT.isVector() ? ISD::VSELECT : ISD::SELECT; in buildSqrtEstimateImpl()
18994 const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT); in buildSqrtEstimateImpl()
18996 SDValue NormC = DAG.getConstantFP(SmallestNorm, DL, VT); in buildSqrtEstimateImpl()
18997 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT); in buildSqrtEstimateImpl()
18998 SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op); in buildSqrtEstimateImpl()
19000 Est = DAG.getNode(SelOpcode, DL, VT, IsDenorm, FPZero, Est); in buildSqrtEstimateImpl()
19006 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT); in buildSqrtEstimateImpl()
19008 Est = DAG.getNode(SelOpcode, DL, VT, IsZero, FPZero, Est); in buildSqrtEstimateImpl()