Lines Matching refs:VT
205 EVT VT = getOptimalMemOpType(Op, FuncAttributes); in findOptimalMemOpLowering() local
207 if (VT == MVT::Other) { in findOptimalMemOpLowering()
211 VT = MVT::i64; in findOptimalMemOpLowering()
213 while (Op.getDstAlign() < (VT.getSizeInBits() / 8) && in findOptimalMemOpLowering()
214 !allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign())) in findOptimalMemOpLowering()
215 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1); in findOptimalMemOpLowering()
216 assert(VT.isInteger()); in findOptimalMemOpLowering()
226 if (VT.bitsGT(LVT)) in findOptimalMemOpLowering()
227 VT = LVT; in findOptimalMemOpLowering()
233 unsigned VTSize = VT.getSizeInBits() / 8; in findOptimalMemOpLowering()
236 EVT NewVT = VT; in findOptimalMemOpLowering()
240 if (VT.isVector() || VT.isFloatingPoint()) { in findOptimalMemOpLowering()
241 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32; in findOptimalMemOpLowering()
268 VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1), in findOptimalMemOpLowering()
273 VT = NewVT; in findOptimalMemOpLowering()
281 MemOps.push_back(VT); in findOptimalMemOpLowering()
290 void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT, in softenSetCCOperands() argument
296 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS, in softenSetCCOperands()
300 void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT, in softenSetCCOperands() argument
311 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) in softenSetCCOperands()
320 LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : in softenSetCCOperands()
321 (VT == MVT::f64) ? RTLIB::OEQ_F64 : in softenSetCCOperands()
322 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128; in softenSetCCOperands()
326 LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 : in softenSetCCOperands()
327 (VT == MVT::f64) ? RTLIB::UNE_F64 : in softenSetCCOperands()
328 (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128; in softenSetCCOperands()
332 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 : in softenSetCCOperands()
333 (VT == MVT::f64) ? RTLIB::OGE_F64 : in softenSetCCOperands()
334 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128; in softenSetCCOperands()
338 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : in softenSetCCOperands()
339 (VT == MVT::f64) ? RTLIB::OLT_F64 : in softenSetCCOperands()
340 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128; in softenSetCCOperands()
344 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 : in softenSetCCOperands()
345 (VT == MVT::f64) ? RTLIB::OLE_F64 : in softenSetCCOperands()
346 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128; in softenSetCCOperands()
350 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 : in softenSetCCOperands()
351 (VT == MVT::f64) ? RTLIB::OGT_F64 : in softenSetCCOperands()
352 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128; in softenSetCCOperands()
358 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : in softenSetCCOperands()
359 (VT == MVT::f64) ? RTLIB::UO_F64 : in softenSetCCOperands()
360 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128; in softenSetCCOperands()
367 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : in softenSetCCOperands()
368 (VT == MVT::f64) ? RTLIB::UO_F64 : in softenSetCCOperands()
369 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128; in softenSetCCOperands()
370 LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : in softenSetCCOperands()
371 (VT == MVT::f64) ? RTLIB::OEQ_F64 : in softenSetCCOperands()
372 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128; in softenSetCCOperands()
379 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 : in softenSetCCOperands()
380 (VT == MVT::f64) ? RTLIB::OGE_F64 : in softenSetCCOperands()
381 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128; in softenSetCCOperands()
384 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 : in softenSetCCOperands()
385 (VT == MVT::f64) ? RTLIB::OGT_F64 : in softenSetCCOperands()
386 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128; in softenSetCCOperands()
389 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 : in softenSetCCOperands()
390 (VT == MVT::f64) ? RTLIB::OLE_F64 : in softenSetCCOperands()
391 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128; in softenSetCCOperands()
394 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : in softenSetCCOperands()
395 (VT == MVT::f64) ? RTLIB::OLT_F64 : in softenSetCCOperands()
396 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128; in softenSetCCOperands()
545 EVT VT = Op.getValueType(); in ShrinkDemandedConstant() local
546 SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT); in ShrinkDemandedConstant()
547 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC); in ShrinkDemandedConstant()
561 EVT VT = Op.getValueType(); in ShrinkDemandedConstant() local
562 APInt DemandedElts = VT.isVector() in ShrinkDemandedConstant()
563 ? APInt::getAllOnes(VT.getVectorNumElements()) in ShrinkDemandedConstant()
580 EVT VT = Op.getValueType(); in ShrinkDemandedOp() local
585 if (VT.isVector()) in ShrinkDemandedOp()
600 if (TLI.isTruncateFree(VT, SmallVT) && TLI.isZExtFree(SmallVT, VT)) { in ShrinkDemandedOp()
607 SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, VT, X); in ShrinkDemandedOp()
651 EVT VT = Op.getValueType(); in SimplifyDemandedBits() local
656 APInt DemandedElts = VT.isFixedLengthVector() in SimplifyDemandedBits()
657 ? APInt::getAllOnes(VT.getVectorNumElements()) in SimplifyDemandedBits()
667 EVT VT = Op.getValueType(); in SimplifyMultipleUseDemandedBits() local
679 return DAG.getUNDEF(VT); in SimplifyMultipleUseDemandedBits()
687 if (VT.isScalableVector()) in SimplifyMultipleUseDemandedBits()
836 if (VT.isScalableVector()) in SimplifyMultipleUseDemandedBits()
852 if (VT.isScalableVector()) in SimplifyMultipleUseDemandedBits()
865 if (VT.isScalableVector()) in SimplifyMultipleUseDemandedBits()
879 assert(!VT.isScalableVector()); in SimplifyMultipleUseDemandedBits()
905 if (VT.isScalableVector()) in SimplifyMultipleUseDemandedBits()
920 EVT VT = Op.getValueType(); in SimplifyMultipleUseDemandedBits() local
924 APInt DemandedElts = VT.isFixedLengthVector() in SimplifyMultipleUseDemandedBits()
925 ? APInt::getAllOnes(VT.getVectorNumElements()) in SimplifyMultipleUseDemandedBits()
1045 EVT VT = Op.getValueType(); in combineShiftToAVG() local
1047 std::max<unsigned>(VT.getScalarSizeInBits() - KnownBits, 8); in combineShiftToAVG()
1049 if (VT.isVector()) in combineShiftToAVG()
1050 NVT = EVT::getVectorVT(*DAG.getContext(), NVT, VT.getVectorElementCount()); in combineShiftToAVG()
1054 if (!TLI.isOperationLegalOrCustom(AVGOpc, VT)) in combineShiftToAVG()
1060 NVT = VT; in combineShiftToAVG()
1069 return DAG.getExtOrTrunc(IsSigned, ResultAVG, DL, VT); in combineShiftToAVG()
1090 EVT VT = Op.getValueType(); in SimplifyDemandedBits() local
1093 assert((!VT.isFixedLengthVector() || NumElts == VT.getVectorNumElements()) && in SimplifyDemandedBits()
1135 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); in SimplifyDemandedBits()
1144 if (VT.isScalableVector()) in SimplifyDemandedBits()
1147 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); in SimplifyDemandedBits()
1195 if (VT.isScalableVector()) in SimplifyDemandedBits()
1233 if (VT.isScalableVector()) in SimplifyDemandedBits()
1270 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub, in SimplifyDemandedBits()
1278 if (VT.isScalableVector()) in SimplifyDemandedBits()
1297 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, in SimplifyDemandedBits()
1305 if (VT.isScalableVector()) in SimplifyDemandedBits()
1325 assert(!VT.isScalableVector()); in SimplifyDemandedBits()
1361 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask); in SimplifyDemandedBits()
1396 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1); in SimplifyDemandedBits()
1403 if (Op0.getOpcode() == ISD::INSERT_SUBVECTOR && !VT.isScalableVector() && in SimplifyDemandedBits()
1416 TLO.DAG.getNode(ISD::AND, dl, VT, Op0.getOperand(0), Op1); in SimplifyDemandedBits()
1418 TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, NewAnd, in SimplifyDemandedBits()
1441 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT)); in SimplifyDemandedBits()
1459 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1); in SimplifyDemandedBits()
1512 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1); in SimplifyDemandedBits()
1531 if (SDValue C12 = TLO.DAG.FoldConstantArithmetic(ISD::OR, dl, VT, in SimplifyDemandedBits()
1533 SDValue MaskX = TLO.DAG.getNode(ISD::AND, dl, VT, X, C12); in SimplifyDemandedBits()
1534 SDValue MaskY = TLO.DAG.getNode(ISD::AND, dl, VT, Y, C2); in SimplifyDemandedBits()
1536 Op, TLO.DAG.getNode(ISD::OR, dl, VT, MaskX, MaskY)); in SimplifyDemandedBits()
1574 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
1585 TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT); in SimplifyDemandedBits()
1586 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC)); in SimplifyDemandedBits()
1594 SDValue New = TLO.DAG.getNOT(dl, Op0, VT); in SimplifyDemandedBits()
1616 SDValue Not = TLO.DAG.getNOT(dl, Op0.getOperand(0), VT); in SimplifyDemandedBits()
1617 return TLO.CombineTo(Op, TLO.DAG.getNode(Op0Opcode, dl, VT, Not, in SimplifyDemandedBits()
1639 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1); in SimplifyDemandedBits()
1750 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA)); in SimplifyDemandedBits()
1768 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl)); in SimplifyDemandedBits()
1788 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, in SimplifyDemandedBits()
1791 Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA)); in SimplifyDemandedBits()
1821 SDValue NewOp = TLO.DAG.getNode(ISD::SHL, dl, VT, DemandedOp0, Op1); in SimplifyDemandedBits()
1836 if ((BitWidth % 2) == 0 && !VT.isVector() && ShAmt < HalfWidth && in SimplifyDemandedBits()
1839 if (isNarrowingProfitable(VT, HalfVT) && in SimplifyDemandedBits()
1841 isTruncateFree(VT, HalfVT) && isZExtFree(HalfVT, VT) && in SimplifyDemandedBits()
1857 TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, NewShift); in SimplifyDemandedBits()
1930 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA)); in SimplifyDemandedBits()
1944 if ((BitWidth % 2) == 0 && !VT.isVector()) { in SimplifyDemandedBits()
1947 if (isNarrowingProfitable(VT, HalfVT) && in SimplifyDemandedBits()
1949 isTruncateFree(VT, HalfVT) && isZExtFree(HalfVT, VT) && in SimplifyDemandedBits()
1959 Op, TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, NewShift)); in SimplifyDemandedBits()
1978 SDValue NewOp = TLO.DAG.getNode(ISD::SRL, dl, VT, DemandedOp0, Op1); in SimplifyDemandedBits()
2006 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
2026 if (VT.isVector()) in SimplifyDemandedBits()
2028 VT.getVectorElementCount()); in SimplifyDemandedBits()
2034 Op, TLO.DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT, in SimplifyDemandedBits()
2074 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags)); in SimplifyDemandedBits()
2081 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA)); in SimplifyDemandedBits()
2093 SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1); in SimplifyDemandedBits()
2146 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedOp0, in SimplifyDemandedBits()
2188 if ((!TLO.LegalOperations() || isOperationLegal(ISD::SHL, VT)) && in SimplifyDemandedBits()
2191 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
2193 if ((!TLO.LegalOperations() || isOperationLegal(ISD::SRL, VT)) && in SimplifyDemandedBits()
2196 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1)); in SimplifyDemandedBits()
2225 return TLO.CombineTo(Op, TLO.DAG.getNode(BitOp, SDLoc(Op), VT, Op0, Op1)); in SimplifyDemandedBits()
2290 if (!TLO.LegalOperations() || isOperationLegal(ShiftOpcode, VT)) { in SimplifyDemandedBits()
2291 EVT ShiftAmtTy = getShiftAmountTy(VT, DL); in SimplifyDemandedBits()
2294 SDValue NewOp = TLO.DAG.getNode(ShiftOpcode, dl, VT, Src, ShAmt); in SimplifyDemandedBits()
2311 if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector()) in SimplifyDemandedBits()
2312 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT, in SimplifyDemandedBits()
2334 getShiftAmountTy(VT, DL)); in SimplifyDemandedBits()
2336 TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt)); in SimplifyDemandedBits()
2391 if (VT.isScalableVector()) in SimplifyDemandedBits()
2406 VT.getSizeInBits() == SrcVT.getSizeInBits()) in SimplifyDemandedBits()
2407 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src)); in SimplifyDemandedBits()
2411 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT)) in SimplifyDemandedBits()
2412 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src)); in SimplifyDemandedBits()
2433 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc)); in SimplifyDemandedBits()
2437 if (VT.isScalableVector()) in SimplifyDemandedBits()
2459 VT.getSizeInBits() == SrcVT.getSizeInBits()) in SimplifyDemandedBits()
2460 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src)); in SimplifyDemandedBits()
2463 if (getBooleanContents(VT) != ZeroOrNegativeOneBooleanContent || in SimplifyDemandedBits()
2468 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT)) in SimplifyDemandedBits()
2469 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src)); in SimplifyDemandedBits()
2486 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT)) { in SimplifyDemandedBits()
2490 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src, Flags)); in SimplifyDemandedBits()
2497 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc)); in SimplifyDemandedBits()
2501 if (VT.isScalableVector()) in SimplifyDemandedBits()
2514 VT.getSizeInBits() == SrcVT.getSizeInBits()) in SimplifyDemandedBits()
2515 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src)); in SimplifyDemandedBits()
2529 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc)); in SimplifyDemandedBits()
2547 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc)); in SimplifyDemandedBits()
2557 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT)) in SimplifyDemandedBits()
2578 ShVal, dl, getShiftAmountTy(VT, DL, TLO.LegalTypes())); in SimplifyDemandedBits()
2580 TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0)); in SimplifyDemandedBits()
2582 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt)); in SimplifyDemandedBits()
2636 TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx); in SimplifyDemandedBits()
2647 if (VT.isScalableVector()) in SimplifyDemandedBits()
2655 if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() && in SimplifyDemandedBits()
2658 bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT); in SimplifyDemandedBits()
2660 if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 && in SimplifyDemandedBits()
2663 EVT Ty = OpVTLegal ? VT : MVT::i32; in SimplifyDemandedBits()
2669 Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign); in SimplifyDemandedBits()
2671 SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT); in SimplifyDemandedBits()
2673 TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt)); in SimplifyDemandedBits()
2734 SDValue NewOp = TLO.DAG.getBitcast(VT, DemandedSrc); in SimplifyDemandedBits()
2756 EVT ShiftAmtTy = getShiftAmountTy(VT, TLO.DAG.getDataLayout()); in SimplifyDemandedBits()
2758 SDValue Shl = TLO.DAG.getNode(ISD::SHL, dl, VT, Op.getOperand(0), AmtC); in SimplifyDemandedBits()
2766 SDValue One = TLO.DAG.getConstant(1, dl, VT); in SimplifyDemandedBits()
2767 SDValue And1 = TLO.DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0), One); in SimplifyDemandedBits()
2813 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags); in SimplifyDemandedBits()
2827 SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT); in SimplifyDemandedBits()
2832 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags); in SimplifyDemandedBits()
2856 EVT ShiftAmtTy = getShiftAmountTy(VT, TLO.DAG.getDataLayout()); in SimplifyDemandedBits()
2858 SDValue Shl = TLO.DAG.getNode(ISD::SHL, dl, VT, X, ShlAmtC); in SimplifyDemandedBits()
2859 SDValue Res = TLO.DAG.getNode(NT, dl, VT, Y, Shl); in SimplifyDemandedBits()
2863 if (isOperationLegalOrCustom(ISD::SHL, VT)) { in SimplifyDemandedBits()
2919 if (VT.isInteger()) in SimplifyDemandedBits()
2920 return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT)); in SimplifyDemandedBits()
2921 if (VT.isFloatingPoint()) in SimplifyDemandedBits()
2925 APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT)); in SimplifyDemandedBits()
2960 EVT VT = BO.getValueType(); in getKnownUndefForVectorBinop() local
2961 assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && in getKnownUndefForVectorBinop()
2964 EVT EltVT = VT.getVectorElementType(); in getKnownUndefForVectorBinop()
2965 unsigned NumElts = VT.isFixedLengthVector() ? VT.getVectorNumElements() : 1; in getKnownUndefForVectorBinop()
3007 EVT VT = Op.getValueType(); in SimplifyDemandedVectorElts() local
3011 assert(VT.isVector() && "Expected vector op"); in SimplifyDemandedVectorElts()
3020 if (VT.isScalableVector()) in SimplifyDemandedVectorElts()
3023 assert(VT.getVectorNumElements() == NumElts && in SimplifyDemandedVectorElts()
3039 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); in SimplifyDemandedVectorElts()
3047 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in SimplifyDemandedVectorElts()
3059 TLO.DAG.getNode(Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, in SimplifyDemandedVectorElts()
3070 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); in SimplifyDemandedVectorElts()
3206 return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops)); in SimplifyDemandedVectorElts()
3249 TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, DemandedSubOps); in SimplifyDemandedVectorElts()
3273 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in SimplifyDemandedVectorElts()
3274 TLO.DAG.getUNDEF(VT), Sub, in SimplifyDemandedVectorElts()
3292 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc, in SimplifyDemandedVectorElts()
3320 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc, in SimplifyDemandedVectorElts()
3445 buildLegalVectorShuffle(VT, DL, LHS, RHS, NewMask, TLO.DAG); in SimplifyDemandedVectorElts()
3486 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src)); in SimplifyDemandedVectorElts()
3492 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT)); in SimplifyDemandedVectorElts()
3510 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Fold)); in SimplifyDemandedVectorElts()
3612 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT)); in SimplifyDemandedVectorElts()
3638 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT)); in SimplifyDemandedVectorElts()
3662 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); in SimplifyDemandedVectorElts()
3762 TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, in buildLegalVectorShuffle() argument
3765 bool LegalMask = isShuffleMaskLegal(Mask, VT); in buildLegalVectorShuffle()
3769 LegalMask = isShuffleMaskLegal(Mask, VT); in buildLegalVectorShuffle()
3775 return DAG.getVectorShuffle(VT, DL, N0, N1, Mask); in buildLegalVectorShuffle()
3892 bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT, in isExtendedTrueVal() argument
3894 if (VT == MVT::i1) in isExtendedTrueVal()
3897 TargetLowering::BooleanContent Cnt = getBooleanContents(VT); in isExtendedTrueVal()
3912 SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1, in foldSetCCWithAnd() argument
3932 return DAG.getBoolExtOrTrunc(N0, DL, VT, OpVT); in foldSetCCWithAnd()
3950 return DAG.getSetCC(DL, VT, Trunc, Zero, in foldSetCCWithAnd()
3983 return DAG.getSetCC(DL, VT, N0, Zero, Cond); in foldSetCCWithAnd()
3999 return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond); in foldSetCCWithAnd()
4163 EVT VT = X.getValueType(); in optimizeSetCCByHoistingAndByConstFromLogicalShift() local
4167 SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y); in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4168 SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C); in optimizeSetCCByHoistingAndByConstFromLogicalShift()
4176 SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1, in foldSetCCWithBinOp() argument
4192 return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond); in foldSetCCWithBinOp()
4200 return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond); in foldSetCCWithBinOp()
4213 return DAG.getSetCC(DL, VT, X, YShl1, Cond); in foldSetCCWithBinOp()
4216 static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT, in simplifySetCCWithCTPOP() argument
4223 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() && in simplifySetCCWithCTPOP()
4256 return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC); in simplifySetCCWithCTPOP()
4276 SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond); in simplifySetCCWithCTPOP()
4284 return DAG.getSetCC(dl, VT, Xor, Add, CmpCond); in simplifySetCCWithCTPOP()
4290 static SDValue foldSetCCWithRotate(EVT VT, SDValue N0, SDValue N1, in foldSetCCWithRotate() argument
4310 return DAG.getSetCC(dl, VT, R, N1, Cond); in foldSetCCWithRotate()
4322 return DAG.getSetCC(dl, VT, NewOr, N1, Cond); in foldSetCCWithRotate()
4326 return DAG.getSetCC(dl, VT, NewOr, N1, Cond); in foldSetCCWithRotate()
4333 static SDValue foldSetCCWithFunnelShift(EVT VT, SDValue N0, SDValue N1, in foldSetCCWithFunnelShift() argument
4386 return DAG.getSetCC(dl, VT, NewOr, N1, Cond); in foldSetCCWithFunnelShift()
4393 return DAG.getSetCC(dl, VT, NewOr, N1, Cond); in foldSetCCWithFunnelShift()
4401 SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, in SimplifySetCC() argument
4411 if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl)) in SimplifySetCC()
4427 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC); in SimplifySetCC()
4438 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC); in SimplifySetCC()
4440 if (SDValue V = foldSetCCWithRotate(VT, N0, N1, Cond, dl, DAG)) in SimplifySetCC()
4443 if (SDValue V = foldSetCCWithFunnelShift(VT, N0, N1, Cond, dl, DAG)) in SimplifySetCC()
4450 if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG)) in SimplifySetCC()
4464 SDValue IsXZero = DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond); in SimplifySetCC()
4465 SDValue IsYZero = DAG.getSetCC(dl, VT, N0.getOperand(1), N1, Cond); in SimplifySetCC()
4467 return DAG.getNode(LogicOp, dl, VT, IsXZero, IsYZero); in SimplifySetCC()
4489 return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero, in SimplifySetCC()
4548 return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1), in SimplifySetCC()
4551 return DAG.getSetCC(dl, VT, Trunc, C, Cond); in SimplifySetCC()
4565 if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 && in SimplifySetCC()
4580 return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0), in SimplifySetCC()
4636 return DAG.getSetCC(dl, VT, in SimplifySetCC()
4657 return DAG.getConstant(0, dl, VT); in SimplifySetCC()
4661 return DAG.getConstant(1, dl, VT); in SimplifySetCC()
4665 return DAG.getConstant(C1.isNegative(), dl, VT); in SimplifySetCC()
4669 return DAG.getConstant(C1.isNonNegative(), dl, VT); in SimplifySetCC()
4692 return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType()); in SimplifySetCC()
4711 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT); in SimplifySetCC()
4721 return DAG.getSetCC(dl, VT, ZextOp, in SimplifySetCC()
4727 isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) && in SimplifySetCC()
4733 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0); in SimplifySetCC()
4739 return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC); in SimplifySetCC()
4766 return DAG.getSetCC(dl, VT, Val, N1, in SimplifySetCC()
4787 return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond); in SimplifySetCC()
4792 if (Op0.getValueType().bitsGT(VT)) in SimplifySetCC()
4793 Op0 = DAG.getNode(ISD::AND, dl, VT, in SimplifySetCC()
4794 DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)), in SimplifySetCC()
4795 DAG.getConstant(1, dl, VT)); in SimplifySetCC()
4796 else if (Op0.getValueType().bitsLT(VT)) in SimplifySetCC()
4797 Op0 = DAG.getNode(ISD::AND, dl, VT, in SimplifySetCC()
4798 DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)), in SimplifySetCC()
4799 DAG.getConstant(1, dl, VT)); in SimplifySetCC()
4801 return DAG.getSetCC(dl, VT, Op0, in SimplifySetCC()
4807 return DAG.getSetCC(dl, VT, Op0, in SimplifySetCC()
4822 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond); in SimplifySetCC()
4831 return DAG.getSetCC(dl, VT, N0.getOperand(0), in SimplifySetCC()
4837 optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl)) in SimplifySetCC()
4860 return DAG.getBoolConstant(true, dl, VT, OpVT); in SimplifySetCC()
4862 if (!VT.isVector()) { // TODO: Support this for vectors. in SimplifySetCC()
4867 isCondCodeLegal(NewCC, VT.getSimpleVT())) && in SimplifySetCC()
4870 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
4880 return DAG.getBoolConstant(true, dl, VT, OpVT); in SimplifySetCC()
4883 if (!VT.isVector()) { // TODO: Support this for vectors. in SimplifySetCC()
4887 isCondCodeLegal(NewCC, VT.getSimpleVT())) && in SimplifySetCC()
4890 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
4899 return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false in SimplifySetCC()
4902 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) { in SimplifySetCC()
4905 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE); in SimplifySetCC()
4909 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
4917 return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false in SimplifySetCC()
4920 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) { in SimplifySetCC()
4923 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE); in SimplifySetCC()
4927 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
4937 VT, N0, N1, Cond, DCI, dl)) in SimplifySetCC()
4984 return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond); in SimplifySetCC()
5005 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) { in SimplifySetCC()
5010 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
5018 return DAG.getSetCC(dl, VT, N0, in SimplifySetCC()
5035 if ((VT.getSizeInBits() == 1 || in SimplifySetCC()
5038 (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) && in SimplifySetCC()
5048 return DAG.getNode(ISD::TRUNCATE, dl, VT, in SimplifySetCC()
5058 return DAG.getNode(ISD::TRUNCATE, dl, VT, in SimplifySetCC()
5081 return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond); in SimplifySetCC()
5109 return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond); in SimplifySetCC()
5124 return DAG.getSetCC(dl, VT, N0, N0, Cond); in SimplifySetCC()
5132 return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond); in SimplifySetCC()
5146 return DAG.getNode(ISD::IS_FPCLASS, dl, VT, Op, in SimplifySetCC()
5169 return DAG.getSetCC(dl, VT, N0, N1, NewCond); in SimplifySetCC()
5183 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT); in SimplifySetCC()
5185 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT); in SimplifySetCC()
5192 return DAG.getSetCC(dl, VT, N0, N1, NewCond); in SimplifySetCC()
5203 return DAG.getSetCC(dl, VT, N1.getOperand(0), N0.getOperand(0), Cond); in SimplifySetCC()
5208 return DAG.getSetCC(dl, VT, Not, N0.getOperand(0), Cond); in SimplifySetCC()
5220 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond); in SimplifySetCC()
5222 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond); in SimplifySetCC()
5226 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0), in SimplifySetCC()
5229 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1), in SimplifySetCC()
5243 dl, VT, N0.getOperand(0), in SimplifySetCC()
5251 dl, VT, N0.getOperand(0), in SimplifySetCC()
5261 dl, VT, N0.getOperand(1), in SimplifySetCC()
5276 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI)) in SimplifySetCC()
5282 if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI)) in SimplifySetCC()
5285 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI)) in SimplifySetCC()
5294 if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttr(Attribute::MinSize)) { in SimplifySetCC()
5296 if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl)) in SimplifySetCC()
5299 if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl)) in SimplifySetCC()
5346 if (VT.getScalarType() != MVT::i1) { in SimplifySetCC()
5351 N0 = DAG.getNode(ExtendCode, dl, VT, N0); in SimplifySetCC()
5549 MVT VT) const { in getRegForInlineAsmConstraint()
5575 if (RI->isTypeLegalForClass(*RC, VT)) in getRegForInlineAsmConstraint()
5696 EVT VT = getAsmOperandValueType(DL, OpTy, true); in ParseConstraints() local
5697 OpInfo.ConstraintVT = VT.isSimple() ? VT.getSimpleVT() : MVT::Other; in ParseConstraints()
6025 EVT VT = N->getValueType(0); in BuildExactSDIV() local
6026 EVT SVT = VT.getScalarType(); in BuildExactSDIV()
6027 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); in BuildExactSDIV()
6059 Factor = DAG.getBuildVector(VT, dl, Factors); in BuildExactSDIV()
6065 Factor = DAG.getSplatVector(VT, dl, Factors[0]); in BuildExactSDIV()
6079 Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags); in BuildExactSDIV()
6083 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor); in BuildExactSDIV()
6119 EVT VT = N->getValueType(0); in buildSDIVPow2WithCMov() local
6123 SDValue Zero = DAG.getConstant(0, DL, VT); in buildSDIVPow2WithCMov()
6124 APInt Lg2Mask = APInt::getLowBitsSet(VT.getSizeInBits(), Lg2); in buildSDIVPow2WithCMov()
6125 SDValue Pow2MinusOne = DAG.getConstant(Lg2Mask, DL, VT); in buildSDIVPow2WithCMov()
6128 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in buildSDIVPow2WithCMov()
6130 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne); in buildSDIVPow2WithCMov()
6131 SDValue CMov = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0); in buildSDIVPow2WithCMov()
6139 DAG.getNode(ISD::SRA, DL, VT, CMov, DAG.getConstant(Lg2, DL, VT)); in buildSDIVPow2WithCMov()
6147 return DAG.getNode(ISD::SUB, DL, VT, Zero, SRA); in buildSDIVPow2WithCMov()
6158 EVT VT = N->getValueType(0); in BuildSDIV() local
6159 EVT SVT = VT.getScalarType(); in BuildSDIV()
6160 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in BuildSDIV()
6162 unsigned EltBits = VT.getScalarSizeInBits(); in BuildSDIV()
6167 if (!isTypeLegal(VT)) { in BuildSDIV()
6169 if (VT.isVector() || !VT.isSimple()) in BuildSDIV()
6174 if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger) in BuildSDIV()
6177 MulVT = getTypeToTransformTo(*DAG.getContext(), VT); in BuildSDIV()
6228 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors); in BuildSDIV()
6229 Factor = DAG.getBuildVector(VT, dl, Factors); in BuildSDIV()
6231 ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks); in BuildSDIV()
6237 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]); in BuildSDIV()
6238 Factor = DAG.getSplatVector(VT, dl, Factors[0]); in BuildSDIV()
6240 ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]); in BuildSDIV()
6254 if (!isTypeLegal(VT)) { in BuildSDIV()
6260 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildSDIV()
6263 if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization)) in BuildSDIV()
6264 return DAG.getNode(ISD::MULHS, dl, VT, X, Y); in BuildSDIV()
6265 if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) { in BuildSDIV()
6267 DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y); in BuildSDIV()
6271 unsigned Size = VT.getScalarSizeInBits(); in BuildSDIV()
6273 if (VT.isVector()) in BuildSDIV()
6275 VT.getVectorElementCount()); in BuildSDIV()
6282 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildSDIV()
6294 Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor); in BuildSDIV()
6296 Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor); in BuildSDIV()
6300 Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift); in BuildSDIV()
6305 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift); in BuildSDIV()
6307 T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask); in BuildSDIV()
6309 return DAG.getNode(ISD::ADD, dl, VT, Q, T); in BuildSDIV()
6320 EVT VT = N->getValueType(0); in BuildUDIV() local
6321 EVT SVT = VT.getScalarType(); in BuildUDIV()
6322 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in BuildUDIV()
6324 unsigned EltBits = VT.getScalarSizeInBits(); in BuildUDIV()
6329 if (!isTypeLegal(VT)) { in BuildUDIV()
6331 if (VT.isVector() || !VT.isSimple()) in BuildUDIV()
6336 if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger) in BuildUDIV()
6339 MulVT = getTypeToTransformTo(*DAG.getContext(), VT); in BuildUDIV()
6352 if (!VT.isVector() && isa<ConstantSDNode>(N1)) { in BuildUDIV()
6412 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors); in BuildUDIV()
6413 NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors); in BuildUDIV()
6420 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]); in BuildUDIV()
6421 NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]); in BuildUDIV()
6432 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift); in BuildUDIV()
6440 if (!isTypeLegal(VT)) { in BuildUDIV()
6446 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildUDIV()
6449 if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization)) in BuildUDIV()
6450 return DAG.getNode(ISD::MULHU, dl, VT, X, Y); in BuildUDIV()
6451 if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) { in BuildUDIV()
6453 DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y); in BuildUDIV()
6457 unsigned Size = VT.getScalarSizeInBits(); in BuildUDIV()
6459 if (VT.isVector()) in BuildUDIV()
6461 VT.getVectorElementCount()); in BuildUDIV()
6468 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y); in BuildUDIV()
6481 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q); in BuildUDIV()
6486 if (VT.isVector()) in BuildUDIV()
6489 NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT)); in BuildUDIV()
6493 Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q); in BuildUDIV()
6498 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift); in BuildUDIV()
6502 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in BuildUDIV()
6504 SDValue One = DAG.getConstant(1, dl, VT); in BuildUDIV()
6506 return DAG.getSelect(dl, VT, IsOne, N0, Q); in BuildUDIV()
6573 EVT VT = REMNode.getValueType(); in prepareUREMEqFold() local
6574 EVT SVT = VT.getScalarType(); in prepareUREMEqFold()
6575 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize()); in prepareUREMEqFold()
6579 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT)) in prepareUREMEqFold()
6699 PVal = DAG.getBuildVector(VT, DL, PAmts); in prepareUREMEqFold()
6701 QVal = DAG.getBuildVector(VT, DL, QAmts); in prepareUREMEqFold()
6706 PVal = DAG.getSplatVector(VT, DL, PAmts[0]); in prepareUREMEqFold()
6708 QVal = DAG.getSplatVector(VT, DL, QAmts[0]); in prepareUREMEqFold()
6716 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT)) in prepareUREMEqFold()
6720 N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode); in prepareUREMEqFold()
6724 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal); in prepareUREMEqFold()
6731 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT)) in prepareUREMEqFold()
6734 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal); in prepareUREMEqFold()
6748 assert(VT.isVector() && "Can/should only get here for vectors."); in prepareUREMEqFold()
6822 EVT VT = REMNode.getValueType(); in prepareSREMEqFold() local
6823 EVT SVT = VT.getScalarType(); in prepareSREMEqFold()
6824 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize()); in prepareSREMEqFold()
6829 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT)) in prepareSREMEqFold()
6957 PVal = DAG.getBuildVector(VT, DL, PAmts); in prepareSREMEqFold()
6958 AVal = DAG.getBuildVector(VT, DL, AAmts); in prepareSREMEqFold()
6960 QVal = DAG.getBuildVector(VT, DL, QAmts); in prepareSREMEqFold()
6966 PVal = DAG.getSplatVector(VT, DL, PAmts[0]); in prepareSREMEqFold()
6967 AVal = DAG.getSplatVector(VT, DL, AAmts[0]); in prepareSREMEqFold()
6969 QVal = DAG.getSplatVector(VT, DL, QAmts[0]); in prepareSREMEqFold()
6979 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal); in prepareSREMEqFold()
6984 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ADD, VT)) in prepareSREMEqFold()
6988 Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal); in prepareSREMEqFold()
6996 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT)) in prepareSREMEqFold()
6999 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal); in prepareSREMEqFold()
7015 assert(VT.isVector() && "Can/should only get here for vectors."); in prepareSREMEqFold()
7021 !isOperationLegalOrCustom(ISD::AND, VT) || in prepareSREMEqFold()
7022 !isCondCodeLegalOrCustom(Cond, VT.getSimpleVT()) || in prepareSREMEqFold()
7029 APInt::getSignedMinValue(SVT.getScalarSizeInBits()), DL, VT); in prepareSREMEqFold()
7031 APInt::getSignedMaxValue(SVT.getScalarSizeInBits()), DL, VT); in prepareSREMEqFold()
7033 DAG.getConstant(APInt::getZero(SVT.getScalarSizeInBits()), DL, VT); in prepareSREMEqFold()
7040 SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax); in prepareSREMEqFold()
7069 EVT VT = Op.getValueType(); in getSqrtInputTest() local
7070 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in getSqrtInputTest()
7071 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT); in getSqrtInputTest()
7084 const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT); in getSqrtInputTest()
7086 SDValue NormC = DAG.getConstantFP(SmallestNorm, DL, VT); in getSqrtInputTest()
7087 SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op); in getSqrtInputTest()
7109 EVT VT = Op.getValueType(); in getNegatedExpression() local
7115 isFPExtFree(VT, Op.getOperand(0).getValueType()); in getNegatedExpression()
7137 isOperationLegal(ISD::ConstantFP, VT) || in getNegatedExpression()
7138 isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT, in getNegatedExpression()
7146 SDValue CFP = DAG.getConstantFP(V, DL, VT); in getNegatedExpression()
7163 (isOperationLegal(ISD::ConstantFP, VT) && in getNegatedExpression()
7164 isOperationLegal(ISD::BUILD_VECTOR, VT)) || in getNegatedExpression()
7167 isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT, in getNegatedExpression()
7185 return DAG.getBuildVector(VT, DL, Ops); in getNegatedExpression()
7192 if (LegalOps && !isOperationLegalOrCustom(ISD::FSUB, VT)) in getNegatedExpression()
7215 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegX, Y, Flags); in getNegatedExpression()
7224 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegY, X, Flags); in getNegatedExpression()
7246 return DAG.getNode(ISD::FSUB, DL, VT, Y, X, Flags); in getNegatedExpression()
7271 SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, Flags); in getNegatedExpression()
7285 SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, Flags); in getNegatedExpression()
7327 SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, NegZ, Flags); in getNegatedExpression()
7336 SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, NegZ, Flags); in getNegatedExpression()
7348 return DAG.getNode(Opcode, DL, VT, NegV); in getNegatedExpression()
7353 return DAG.getNode(ISD::FP_ROUND, DL, VT, NegV, Op.getOperand(1)); in getNegatedExpression()
7388 return DAG.getSelect(DL, VT, Op.getOperand(0), NegLHS, NegRHS); in getNegatedExpression()
7399 bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, in expandMUL_LOHI() argument
7420 unsigned OuterBitSize = VT.getScalarSizeInBits(); in expandMUL_LOHI()
7470 if (!VT.isVector() && Opcode == ISD::MUL && in expandMUL_LOHI()
7483 SDValue Shift = DAG.getShiftAmountConstant(ShiftAmount, VT, dl); in expandMUL_LOHI()
7486 isOperationLegalOrCustom(ISD::SRL, VT) && in expandMUL_LOHI()
7488 LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift); in expandMUL_LOHI()
7490 RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift); in expandMUL_LOHI()
7513 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); in expandMUL_LOHI()
7514 Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi); in expandMUL_LOHI()
7515 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); in expandMUL_LOHI()
7516 return DAG.getNode(ISD::OR, dl, VT, Lo, Hi); in expandMUL_LOHI()
7519 SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi); in expandMUL_LOHI()
7525 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi)); in expandMUL_LOHI()
7531 EVT BoolType = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandMUL_LOHI()
7533 bool UseGlue = (isOperationLegalOrCustom(ISD::ADDC, VT) && in expandMUL_LOHI()
7534 isOperationLegalOrCustom(ISD::ADDE, VT)); in expandMUL_LOHI()
7536 Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next, in expandMUL_LOHI()
7539 Next = DAG.getNode(ISD::UADDO_CARRY, dl, DAG.getVTList(VT, BoolType), Next, in expandMUL_LOHI()
7544 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift); in expandMUL_LOHI()
7556 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi)); in expandMUL_LOHI()
7559 SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next, in expandMUL_LOHI()
7560 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL)); in expandMUL_LOHI()
7563 NextSub = DAG.getNode(ISD::SUB, dl, VT, Next, in expandMUL_LOHI()
7564 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL)); in expandMUL_LOHI()
7569 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift); in expandMUL_LOHI()
7616 EVT VT = N->getValueType(0); in expandDIVREMByConstant() local
7632 assert(VT.getScalarSizeInBits() == BitWidth && in expandDIVREMByConstant()
7730 SDValue Dividend = DAG.getNode(ISD::BUILD_PAIR, dl, VT, LL, LH); in expandDIVREMByConstant()
7731 SDValue Rem = DAG.getNode(ISD::BUILD_PAIR, dl, VT, RemL, RemH); in expandDIVREMByConstant()
7733 Dividend = DAG.getNode(ISD::SUB, dl, VT, Dividend, Rem); in expandDIVREMByConstant()
7742 SDValue Quotient = DAG.getNode(ISD::MUL, dl, VT, Dividend, in expandDIVREMByConstant()
7743 DAG.getConstant(MulFactor, dl, VT)); in expandDIVREMByConstant()
7777 EVT VT = Node->getValueType(0); in expandVPFunnelShift() local
7786 unsigned BW = VT.getScalarSizeInBits(); in expandVPFunnelShift()
7798 ShX = DAG.getNode(ISD::VP_SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt, Mask, in expandVPFunnelShift()
7800 ShY = DAG.getNode(ISD::VP_LSHR, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt, Mask, in expandVPFunnelShift()
7821 ShX = DAG.getNode(ISD::VP_SHL, DL, VT, X, ShAmt, Mask, VL); in expandVPFunnelShift()
7822 SDValue ShY1 = DAG.getNode(ISD::VP_LSHR, DL, VT, Y, One, Mask, VL); in expandVPFunnelShift()
7823 ShY = DAG.getNode(ISD::VP_LSHR, DL, VT, ShY1, InvShAmt, Mask, VL); in expandVPFunnelShift()
7825 SDValue ShX1 = DAG.getNode(ISD::VP_SHL, DL, VT, X, One, Mask, VL); in expandVPFunnelShift()
7826 ShX = DAG.getNode(ISD::VP_SHL, DL, VT, ShX1, InvShAmt, Mask, VL); in expandVPFunnelShift()
7827 ShY = DAG.getNode(ISD::VP_LSHR, DL, VT, Y, ShAmt, Mask, VL); in expandVPFunnelShift()
7830 return DAG.getNode(ISD::VP_OR, DL, VT, ShX, ShY, Mask, VL); in expandVPFunnelShift()
7838 EVT VT = Node->getValueType(0); in expandFunnelShift() local
7840 if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) || in expandFunnelShift()
7841 !isOperationLegalOrCustom(ISD::SRL, VT) || in expandFunnelShift()
7842 !isOperationLegalOrCustom(ISD::SUB, VT) || in expandFunnelShift()
7843 !isOperationLegalOrCustomOrPromote(ISD::OR, VT))) in expandFunnelShift()
7850 unsigned BW = VT.getScalarSizeInBits(); in expandFunnelShift()
7858 if (!isOperationLegalOrCustom(Node->getOpcode(), VT) && in expandFunnelShift()
7859 isOperationLegalOrCustom(RevOpcode, VT) && isPowerOf2_32(BW)) { in expandFunnelShift()
7864 Z = DAG.getNode(ISD::SUB, DL, VT, Zero, Z); in expandFunnelShift()
7870 Y = DAG.getNode(RevOpcode, DL, VT, X, Y, One); in expandFunnelShift()
7871 X = DAG.getNode(ISD::SRL, DL, VT, X, One); in expandFunnelShift()
7873 X = DAG.getNode(RevOpcode, DL, VT, X, Y, One); in expandFunnelShift()
7874 Y = DAG.getNode(ISD::SHL, DL, VT, Y, One); in expandFunnelShift()
7878 return DAG.getNode(RevOpcode, DL, VT, X, Y, Z); in expandFunnelShift()
7890 ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt); in expandFunnelShift()
7891 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt); in expandFunnelShift()
7909 ShX = DAG.getNode(ISD::SHL, DL, VT, X, ShAmt); in expandFunnelShift()
7910 SDValue ShY1 = DAG.getNode(ISD::SRL, DL, VT, Y, One); in expandFunnelShift()
7911 ShY = DAG.getNode(ISD::SRL, DL, VT, ShY1, InvShAmt); in expandFunnelShift()
7913 SDValue ShX1 = DAG.getNode(ISD::SHL, DL, VT, X, One); in expandFunnelShift()
7914 ShX = DAG.getNode(ISD::SHL, DL, VT, ShX1, InvShAmt); in expandFunnelShift()
7915 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt); in expandFunnelShift()
7918 return DAG.getNode(ISD::OR, DL, VT, ShX, ShY); in expandFunnelShift()
7924 EVT VT = Node->getValueType(0); in expandROT() local
7925 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in expandROT()
7936 if (!isOperationLegalOrCustom(Node->getOpcode(), VT) && in expandROT()
7937 isOperationLegalOrCustom(RevRot, VT) && isPowerOf2_32(EltSizeInBits)) { in expandROT()
7939 return DAG.getNode(RevRot, DL, VT, Op0, Sub); in expandROT()
7942 if (!AllowVectorOps && VT.isVector() && in expandROT()
7943 (!isOperationLegalOrCustom(ISD::SHL, VT) || in expandROT()
7944 !isOperationLegalOrCustom(ISD::SRL, VT) || in expandROT()
7945 !isOperationLegalOrCustom(ISD::SUB, VT) || in expandROT()
7946 !isOperationLegalOrCustomOrPromote(ISD::OR, VT) || in expandROT()
7947 !isOperationLegalOrCustomOrPromote(ISD::AND, VT))) in expandROT()
7960 ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt); in expandROT()
7962 HsVal = DAG.getNode(HsOpc, DL, VT, Op0, HsAmt); in expandROT()
7968 ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt); in expandROT()
7972 DAG.getNode(HsOpc, DL, VT, DAG.getNode(HsOpc, DL, VT, Op0, One), HsAmt); in expandROT()
7974 return DAG.getNode(ISD::OR, DL, VT, ShVal, HsVal); in expandROT()
7980 EVT VT = Node->getValueType(0); in expandShiftParts() local
7981 unsigned VTBits = VT.getScalarSizeInBits(); in expandShiftParts()
7999 SDValue Tmp1 = IsSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi, in expandShiftParts()
8001 : DAG.getConstant(0, dl, VT); in expandShiftParts()
8005 Tmp2 = DAG.getNode(ISD::FSHL, dl, VT, ShOpHi, ShOpLo, ShAmt); in expandShiftParts()
8006 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt); in expandShiftParts()
8008 Tmp2 = DAG.getNode(ISD::FSHR, dl, VT, ShOpHi, ShOpLo, ShAmt); in expandShiftParts()
8009 Tmp3 = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt); in expandShiftParts()
8021 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2); in expandShiftParts()
8022 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3); in expandShiftParts()
8024 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2); in expandShiftParts()
8025 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3); in expandShiftParts()
8282 EVT VT = Node->getValueType(0); in expandFMINNUM_FMAXNUM() local
8284 if (VT.isScalableVector()) in expandFMINNUM_FMAXNUM()
8288 if (isOperationLegalOrCustom(NewOp, VT)) { in expandFMINNUM_FMAXNUM()
8296 Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0, in expandFMINNUM_FMAXNUM()
8300 Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1, in expandFMINNUM_FMAXNUM()
8305 return DAG.getNode(NewOp, dl, VT, Quiet0, Quiet1, Node->getFlags()); in expandFMINNUM_FMAXNUM()
8319 if (isOperationLegalOrCustom(IEEE2018Op, VT)) in expandFMINNUM_FMAXNUM()
8320 return DAG.getNode(IEEE2018Op, dl, VT, Node->getOperand(0), in expandFMINNUM_FMAXNUM()
8616 static bool canExpandVectorCTPOP(const TargetLowering &TLI, EVT VT) { in canExpandVectorCTPOP() argument
8617 assert(VT.isVector() && "Expected vector type"); in canExpandVectorCTPOP()
8618 unsigned Len = VT.getScalarSizeInBits(); in canExpandVectorCTPOP()
8619 return TLI.isOperationLegalOrCustom(ISD::ADD, VT) && in canExpandVectorCTPOP()
8620 TLI.isOperationLegalOrCustom(ISD::SUB, VT) && in canExpandVectorCTPOP()
8621 TLI.isOperationLegalOrCustom(ISD::SRL, VT) && in canExpandVectorCTPOP()
8622 (Len == 8 || TLI.isOperationLegalOrCustom(ISD::MUL, VT)) && in canExpandVectorCTPOP()
8623 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT); in canExpandVectorCTPOP()
8628 EVT VT = Node->getValueType(0); in expandCTPOP() local
8629 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandCTPOP()
8631 unsigned Len = VT.getScalarSizeInBits(); in expandCTPOP()
8632 assert(VT.isInteger() && "CTPOP not implemented for this type."); in expandCTPOP()
8639 if (VT.isVector() && !canExpandVectorCTPOP(*this, VT)) in expandCTPOP()
8645 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT); in expandCTPOP()
8647 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT); in expandCTPOP()
8649 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT); in expandCTPOP()
8652 Op = DAG.getNode(ISD::SUB, dl, VT, Op, in expandCTPOP()
8653 DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8654 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8658 Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, dl, VT, Op, Mask33), in expandCTPOP()
8659 DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8660 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8664 Op = DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8665 DAG.getNode(ISD::ADD, dl, VT, Op, in expandCTPOP()
8666 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8676 if (Len == 16 && !VT.isVector()) { in expandCTPOP()
8678 return DAG.getNode(ISD::AND, dl, VT, in expandCTPOP()
8679 DAG.getNode(ISD::ADD, dl, VT, Op, in expandCTPOP()
8680 DAG.getNode(ISD::SRL, dl, VT, Op, in expandCTPOP()
8682 DAG.getConstant(0xFF, dl, VT)); in expandCTPOP()
8687 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT); in expandCTPOP()
8688 return DAG.getNode(ISD::SRL, dl, VT, in expandCTPOP()
8689 DAG.getNode(ISD::MUL, dl, VT, Op, Mask01), in expandCTPOP()
8695 EVT VT = Node->getValueType(0); in expandVPCTPOP() local
8696 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandVPCTPOP()
8700 unsigned Len = VT.getScalarSizeInBits(); in expandVPCTPOP()
8701 assert(VT.isInteger() && "VP_CTPOP not implemented for this type."); in expandVPCTPOP()
8710 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT); in expandVPCTPOP()
8712 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT); in expandVPCTPOP()
8714 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT); in expandVPCTPOP()
8719 Tmp1 = DAG.getNode(ISD::VP_AND, dl, VT, in expandVPCTPOP()
8720 DAG.getNode(ISD::VP_LSHR, dl, VT, Op, in expandVPCTPOP()
8723 Op = DAG.getNode(ISD::VP_SUB, dl, VT, Op, Tmp1, Mask, VL); in expandVPCTPOP()
8726 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Op, Mask33, Mask, VL); in expandVPCTPOP()
8727 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, in expandVPCTPOP()
8728 DAG.getNode(ISD::VP_LSHR, dl, VT, Op, in expandVPCTPOP()
8731 Op = DAG.getNode(ISD::VP_ADD, dl, VT, Tmp2, Tmp3, Mask, VL); in expandVPCTPOP()
8734 Tmp4 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(4, dl, ShVT), in expandVPCTPOP()
8736 Tmp5 = DAG.getNode(ISD::VP_ADD, dl, VT, Op, Tmp4, Mask, VL); in expandVPCTPOP()
8737 Op = DAG.getNode(ISD::VP_AND, dl, VT, Tmp5, Mask0F, Mask, VL); in expandVPCTPOP()
8744 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT); in expandVPCTPOP()
8745 return DAG.getNode(ISD::VP_LSHR, dl, VT, in expandVPCTPOP()
8746 DAG.getNode(ISD::VP_MUL, dl, VT, Op, Mask01, Mask, VL), in expandVPCTPOP()
8752 EVT VT = Node->getValueType(0); in expandCTLZ() local
8753 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandCTLZ()
8755 unsigned NumBitsPerElt = VT.getScalarSizeInBits(); in expandCTLZ()
8759 isOperationLegalOrCustom(ISD::CTLZ, VT)) in expandCTLZ()
8760 return DAG.getNode(ISD::CTLZ, dl, VT, Op); in expandCTLZ()
8763 if (isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) { in expandCTLZ()
8765 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandCTLZ()
8766 SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op); in expandCTLZ()
8767 SDValue Zero = DAG.getConstant(0, dl, VT); in expandCTLZ()
8769 return DAG.getSelect(dl, VT, SrcIsZero, in expandCTLZ()
8770 DAG.getConstant(NumBitsPerElt, dl, VT), CTLZ); in expandCTLZ()
8775 if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) || in expandCTLZ()
8776 (!isOperationLegalOrCustom(ISD::CTPOP, VT) && in expandCTLZ()
8777 !canExpandVectorCTPOP(*this, VT)) || in expandCTLZ()
8778 !isOperationLegalOrCustom(ISD::SRL, VT) || in expandCTLZ()
8779 !isOperationLegalOrCustomOrPromote(ISD::OR, VT))) in expandCTLZ()
8793 Op = DAG.getNode(ISD::OR, dl, VT, Op, in expandCTLZ()
8794 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp)); in expandCTLZ()
8796 Op = DAG.getNOT(dl, Op, VT); in expandCTLZ()
8797 return DAG.getNode(ISD::CTPOP, dl, VT, Op); in expandCTLZ()
8802 EVT VT = Node->getValueType(0); in expandVPCTLZ() local
8803 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandVPCTLZ()
8807 unsigned NumBitsPerElt = VT.getScalarSizeInBits(); in expandVPCTLZ()
8818 Op = DAG.getNode(ISD::VP_OR, dl, VT, Op, in expandVPCTLZ()
8819 DAG.getNode(ISD::VP_LSHR, dl, VT, Op, Tmp, Mask, VL), Mask, in expandVPCTLZ()
8822 Op = DAG.getNode(ISD::VP_XOR, dl, VT, Op, DAG.getConstant(-1, dl, VT), Mask, in expandVPCTLZ()
8824 return DAG.getNode(ISD::VP_CTPOP, dl, VT, Op, Mask, VL); in expandVPCTLZ()
8828 const SDLoc &DL, EVT VT, SDValue Op, in CTTZTableLookup() argument
8838 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op); in CTTZTableLookup()
8840 ISD::SRL, DL, VT, in CTTZTableLookup()
8841 DAG.getNode(ISD::MUL, DL, VT, DAG.getNode(ISD::AND, DL, VT, Op, Neg), in CTTZTableLookup()
8842 DAG.getConstant(DeBruijn, DL, VT)), in CTTZTableLookup()
8843 DAG.getConstant(ShiftAmt, DL, VT)); in CTTZTableLookup()
8857 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, DL, VT, DAG.getEntryNode(), in CTTZTableLookup()
8864 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in CTTZTableLookup()
8865 SDValue Zero = DAG.getConstant(0, DL, VT); in CTTZTableLookup()
8867 return DAG.getSelect(DL, VT, SrcIsZero, in CTTZTableLookup()
8868 DAG.getConstant(BitWidth, DL, VT), ExtLoad); in CTTZTableLookup()
8873 EVT VT = Node->getValueType(0); in expandCTTZ() local
8875 unsigned NumBitsPerElt = VT.getScalarSizeInBits(); in expandCTTZ()
8879 isOperationLegalOrCustom(ISD::CTTZ, VT)) in expandCTTZ()
8880 return DAG.getNode(ISD::CTTZ, dl, VT, Op); in expandCTTZ()
8883 if (isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) { in expandCTTZ()
8885 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandCTTZ()
8886 SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op); in expandCTTZ()
8887 SDValue Zero = DAG.getConstant(0, dl, VT); in expandCTTZ()
8889 return DAG.getSelect(dl, VT, SrcIsZero, in expandCTTZ()
8890 DAG.getConstant(NumBitsPerElt, dl, VT), CTTZ); in expandCTTZ()
8895 if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) || in expandCTTZ()
8896 (!isOperationLegalOrCustom(ISD::CTPOP, VT) && in expandCTTZ()
8897 !isOperationLegalOrCustom(ISD::CTLZ, VT) && in expandCTTZ()
8898 !canExpandVectorCTPOP(*this, VT)) || in expandCTTZ()
8899 !isOperationLegalOrCustom(ISD::SUB, VT) || in expandCTTZ()
8900 !isOperationLegalOrCustomOrPromote(ISD::AND, VT) || in expandCTTZ()
8901 !isOperationLegalOrCustomOrPromote(ISD::XOR, VT))) in expandCTTZ()
8905 if (!VT.isVector() && isOperationExpand(ISD::CTPOP, VT) && in expandCTTZ()
8906 !isOperationLegal(ISD::CTLZ, VT)) in expandCTTZ()
8907 if (SDValue V = CTTZTableLookup(Node, DAG, dl, VT, Op, NumBitsPerElt)) in expandCTTZ()
8915 ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT), in expandCTTZ()
8916 DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT))); in expandCTTZ()
8919 if (isOperationLegal(ISD::CTLZ, VT) && !isOperationLegal(ISD::CTPOP, VT)) { in expandCTTZ()
8920 return DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT), in expandCTTZ()
8921 DAG.getNode(ISD::CTLZ, dl, VT, Tmp)); in expandCTTZ()
8924 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp); in expandCTTZ()
8932 EVT VT = Node->getValueType(0); in expandVPCTTZ() local
8935 SDValue Not = DAG.getNode(ISD::VP_XOR, dl, VT, Op, in expandVPCTTZ()
8936 DAG.getConstant(-1, dl, VT), Mask, VL); in expandVPCTTZ()
8937 SDValue MinusOne = DAG.getNode(ISD::VP_SUB, dl, VT, Op, in expandVPCTTZ()
8938 DAG.getConstant(1, dl, VT), Mask, VL); in expandVPCTTZ()
8939 SDValue Tmp = DAG.getNode(ISD::VP_AND, dl, VT, Not, MinusOne, Mask, VL); in expandVPCTTZ()
8940 return DAG.getNode(ISD::VP_CTPOP, dl, VT, Tmp, Mask, VL); in expandVPCTTZ()
8946 EVT VT = N->getValueType(0); in expandABS() local
8947 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandABS()
8951 if (!IsNegative && isOperationLegal(ISD::SUB, VT) && in expandABS()
8952 isOperationLegal(ISD::SMAX, VT)) { in expandABS()
8953 SDValue Zero = DAG.getConstant(0, dl, VT); in expandABS()
8954 return DAG.getNode(ISD::SMAX, dl, VT, Op, in expandABS()
8955 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
8959 if (!IsNegative && isOperationLegal(ISD::SUB, VT) && in expandABS()
8960 isOperationLegal(ISD::UMIN, VT)) { in expandABS()
8961 SDValue Zero = DAG.getConstant(0, dl, VT); in expandABS()
8963 return DAG.getNode(ISD::UMIN, dl, VT, Op, in expandABS()
8964 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
8968 if (IsNegative && isOperationLegal(ISD::SUB, VT) && in expandABS()
8969 isOperationLegal(ISD::SMIN, VT)) { in expandABS()
8971 SDValue Zero = DAG.getConstant(0, dl, VT); in expandABS()
8972 return DAG.getNode(ISD::SMIN, dl, VT, Op, in expandABS()
8973 DAG.getNode(ISD::SUB, dl, VT, Zero, Op)); in expandABS()
8977 if (VT.isVector() && in expandABS()
8978 (!isOperationLegalOrCustom(ISD::SRA, VT) || in expandABS()
8979 (!IsNegative && !isOperationLegalOrCustom(ISD::ADD, VT)) || in expandABS()
8980 (IsNegative && !isOperationLegalOrCustom(ISD::SUB, VT)) || in expandABS()
8981 !isOperationLegalOrCustomOrPromote(ISD::XOR, VT))) in expandABS()
8986 DAG.getNode(ISD::SRA, dl, VT, Op, in expandABS()
8987 DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, ShVT)); in expandABS()
8988 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, Shift); in expandABS()
8992 return DAG.getNode(ISD::SUB, dl, VT, Xor, Shift); in expandABS()
8995 return DAG.getNode(ISD::SUB, dl, VT, Shift, Xor); in expandABS()
9000 EVT VT = N->getValueType(0); in expandABD() local
9009 if (isOperationLegal(MaxOpc, VT) && isOperationLegal(MinOpc, VT)) { in expandABD()
9010 SDValue Max = DAG.getNode(MaxOpc, dl, VT, LHS, RHS); in expandABD()
9011 SDValue Min = DAG.getNode(MinOpc, dl, VT, LHS, RHS); in expandABD()
9012 return DAG.getNode(ISD::SUB, dl, VT, Max, Min); in expandABD()
9016 if (!IsSigned && isOperationLegal(ISD::USUBSAT, VT)) in expandABD()
9017 return DAG.getNode(ISD::OR, dl, VT, in expandABD()
9018 DAG.getNode(ISD::USUBSAT, dl, VT, LHS, RHS), in expandABD()
9019 DAG.getNode(ISD::USUBSAT, dl, VT, RHS, LHS)); in expandABD()
9023 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandABD()
9026 return DAG.getSelect(dl, VT, Cmp, DAG.getNode(ISD::SUB, dl, VT, LHS, RHS), in expandABD()
9027 DAG.getNode(ISD::SUB, dl, VT, RHS, LHS)); in expandABD()
9032 EVT VT = N->getValueType(0); in expandBSWAP() local
9035 if (!VT.isSimple()) in expandBSWAP()
9038 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandBSWAP()
9040 switch (VT.getSimpleVT().getScalarType().SimpleTy) { in expandBSWAP()
9045 return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9047 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9048 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9049 DAG.getConstant(0xFF00, dl, VT)); in expandBSWAP()
9050 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9051 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9052 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT)); in expandBSWAP()
9053 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9054 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); in expandBSWAP()
9055 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); in expandBSWAP()
9056 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); in expandBSWAP()
9058 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); in expandBSWAP()
9059 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9060 DAG.getConstant(255ULL<<8, dl, VT)); in expandBSWAP()
9061 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Tmp7, DAG.getConstant(40, dl, SHVT)); in expandBSWAP()
9062 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9063 DAG.getConstant(255ULL<<16, dl, VT)); in expandBSWAP()
9064 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Tmp6, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9065 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Op, in expandBSWAP()
9066 DAG.getConstant(255ULL<<24, dl, VT)); in expandBSWAP()
9067 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Tmp5, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9068 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); in expandBSWAP()
9069 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, in expandBSWAP()
9070 DAG.getConstant(255ULL<<24, dl, VT)); in expandBSWAP()
9071 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); in expandBSWAP()
9072 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, in expandBSWAP()
9073 DAG.getConstant(255ULL<<16, dl, VT)); in expandBSWAP()
9074 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); in expandBSWAP()
9075 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, in expandBSWAP()
9076 DAG.getConstant(255ULL<<8, dl, VT)); in expandBSWAP()
9077 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); in expandBSWAP()
9078 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); in expandBSWAP()
9079 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); in expandBSWAP()
9080 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); in expandBSWAP()
9081 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); in expandBSWAP()
9082 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); in expandBSWAP()
9083 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); in expandBSWAP()
9084 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); in expandBSWAP()
9090 EVT VT = N->getValueType(0); in expandVPBSWAP() local
9095 if (!VT.isSimple()) in expandVPBSWAP()
9098 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandVPBSWAP()
9100 switch (VT.getSimpleVT().getScalarType().SimpleTy) { in expandVPBSWAP()
9104 Tmp1 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9106 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9108 return DAG.getNode(ISD::VP_OR, dl, VT, Tmp1, Tmp2, Mask, EVL); in expandVPBSWAP()
9110 Tmp4 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9112 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Op, DAG.getConstant(0xFF00, dl, VT), in expandVPBSWAP()
9114 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9116 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9118 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBSWAP()
9119 DAG.getConstant(0xFF00, dl, VT), Mask, EVL); in expandVPBSWAP()
9120 Tmp1 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9122 Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp3, Mask, EVL); in expandVPBSWAP()
9123 Tmp2 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp1, Mask, EVL); in expandVPBSWAP()
9124 return DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp2, Mask, EVL); in expandVPBSWAP()
9126 Tmp8 = DAG.getNode(ISD::VP_SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT), in expandVPBSWAP()
9128 Tmp7 = DAG.getNode(ISD::VP_AND, dl, VT, Op, in expandVPBSWAP()
9129 DAG.getConstant(255ULL << 8, dl, VT), Mask, EVL); in expandVPBSWAP()
9130 Tmp7 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp7, DAG.getConstant(40, dl, SHVT), in expandVPBSWAP()
9132 Tmp6 = DAG.getNode(ISD::VP_AND, dl, VT, Op, in expandVPBSWAP()
9133 DAG.getConstant(255ULL << 16, dl, VT), Mask, EVL); in expandVPBSWAP()
9134 Tmp6 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp6, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9136 Tmp5 = DAG.getNode(ISD::VP_AND, dl, VT, Op, in expandVPBSWAP()
9137 DAG.getConstant(255ULL << 24, dl, VT), Mask, EVL); in expandVPBSWAP()
9138 Tmp5 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp5, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9140 Tmp4 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(8, dl, SHVT), in expandVPBSWAP()
9142 Tmp4 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp4, in expandVPBSWAP()
9143 DAG.getConstant(255ULL << 24, dl, VT), Mask, EVL); in expandVPBSWAP()
9144 Tmp3 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(24, dl, SHVT), in expandVPBSWAP()
9146 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp3, in expandVPBSWAP()
9147 DAG.getConstant(255ULL << 16, dl, VT), Mask, EVL); in expandVPBSWAP()
9148 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(40, dl, SHVT), in expandVPBSWAP()
9150 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBSWAP()
9151 DAG.getConstant(255ULL << 8, dl, VT), Mask, EVL); in expandVPBSWAP()
9152 Tmp1 = DAG.getNode(ISD::VP_LSHR, dl, VT, Op, DAG.getConstant(56, dl, SHVT), in expandVPBSWAP()
9154 Tmp8 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp7, Mask, EVL); in expandVPBSWAP()
9155 Tmp6 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp6, Tmp5, Mask, EVL); in expandVPBSWAP()
9156 Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp3, Mask, EVL); in expandVPBSWAP()
9157 Tmp2 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp1, Mask, EVL); in expandVPBSWAP()
9158 Tmp8 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp6, Mask, EVL); in expandVPBSWAP()
9159 Tmp4 = DAG.getNode(ISD::VP_OR, dl, VT, Tmp4, Tmp2, Mask, EVL); in expandVPBSWAP()
9160 return DAG.getNode(ISD::VP_OR, dl, VT, Tmp8, Tmp4, Mask, EVL); in expandVPBSWAP()
9166 EVT VT = N->getValueType(0); in expandBITREVERSE() local
9168 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandBITREVERSE()
9169 unsigned Sz = VT.getScalarSizeInBits(); in expandBITREVERSE()
9183 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op); in expandBITREVERSE()
9186 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT)); in expandBITREVERSE()
9187 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask4, dl, VT)); in expandBITREVERSE()
9188 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT)); in expandBITREVERSE()
9189 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT)); in expandBITREVERSE()
9190 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
9193 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT)); in expandBITREVERSE()
9194 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask2, dl, VT)); in expandBITREVERSE()
9195 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT)); in expandBITREVERSE()
9196 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT)); in expandBITREVERSE()
9197 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
9200 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT)); in expandBITREVERSE()
9201 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask1, dl, VT)); in expandBITREVERSE()
9202 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT)); in expandBITREVERSE()
9203 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT)); in expandBITREVERSE()
9204 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); in expandBITREVERSE()
9208 Tmp = DAG.getConstant(0, dl, VT); in expandBITREVERSE()
9212 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT)); in expandBITREVERSE()
9215 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT)); in expandBITREVERSE()
9218 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT)); in expandBITREVERSE()
9219 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2); in expandBITREVERSE()
9229 EVT VT = N->getValueType(0); in expandVPBITREVERSE() local
9233 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout()); in expandVPBITREVERSE()
9234 unsigned Sz = VT.getScalarSizeInBits(); in expandVPBITREVERSE()
9248 Tmp = (Sz > 8 ? DAG.getNode(ISD::VP_BSWAP, dl, VT, Op, Mask, EVL) : Op); in expandVPBITREVERSE()
9251 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT), in expandVPBITREVERSE()
9253 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBITREVERSE()
9254 DAG.getConstant(Mask4, dl, VT), Mask, EVL); in expandVPBITREVERSE()
9255 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT), in expandVPBITREVERSE()
9257 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT), in expandVPBITREVERSE()
9259 Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL); in expandVPBITREVERSE()
9262 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT), in expandVPBITREVERSE()
9264 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBITREVERSE()
9265 DAG.getConstant(Mask2, dl, VT), Mask, EVL); in expandVPBITREVERSE()
9266 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT), in expandVPBITREVERSE()
9268 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT), in expandVPBITREVERSE()
9270 Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL); in expandVPBITREVERSE()
9273 Tmp2 = DAG.getNode(ISD::VP_LSHR, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT), in expandVPBITREVERSE()
9275 Tmp2 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp2, in expandVPBITREVERSE()
9276 DAG.getConstant(Mask1, dl, VT), Mask, EVL); in expandVPBITREVERSE()
9277 Tmp3 = DAG.getNode(ISD::VP_AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT), in expandVPBITREVERSE()
9279 Tmp3 = DAG.getNode(ISD::VP_SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT), in expandVPBITREVERSE()
9281 Tmp = DAG.getNode(ISD::VP_OR, dl, VT, Tmp2, Tmp3, Mask, EVL); in expandVPBITREVERSE()
9461 EVT VT = LD->getValueType(0); in expandUnalignedLoad() local
9466 if (VT.isFloatingPoint() || VT.isVector()) { in expandUnalignedLoad()
9480 if (LoadedVT != VT) in expandUnalignedLoad()
9481 Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND : in expandUnalignedLoad()
9482 ISD::ANY_EXTEND, dl, VT, Result); in expandUnalignedLoad()
9544 Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase, in expandUnalignedLoad()
9573 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(), in expandUnalignedLoad()
9578 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, in expandUnalignedLoad()
9583 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(), in expandUnalignedLoad()
9588 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, in expandUnalignedLoad()
9598 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount); in expandUnalignedLoad()
9599 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo); in expandUnalignedLoad()
9614 EVT VT = Val.getValueType(); in expandUnalignedStore() local
9621 EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); in expandUnalignedStore()
9717 ISD::AND, dl, VT, Lo, in expandUnalignedStore()
9718 DAG.getConstant(APInt::getLowBitsSet(VT.getSizeInBits(), NumBits), dl, in expandUnalignedStore()
9719 VT)); in expandUnalignedStore()
9720 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount); in expandUnalignedStore()
9901 EVT VT = Op.getOperand(0).getValueType(); in lowerCmpEqZeroToCtlzSrl() local
9903 if (VT.bitsLT(MVT::i32)) { in lowerCmpEqZeroToCtlzSrl()
9904 VT = MVT::i32; in lowerCmpEqZeroToCtlzSrl()
9905 Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0)); in lowerCmpEqZeroToCtlzSrl()
9907 unsigned Log2b = Log2_32(VT.getSizeInBits()); in lowerCmpEqZeroToCtlzSrl()
9908 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext); in lowerCmpEqZeroToCtlzSrl()
9909 SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz, in lowerCmpEqZeroToCtlzSrl()
9919 EVT VT = Op0.getValueType(); in expandIntMINMAX() local
9920 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandIntMINMAX()
9925 if (Opcode == ISD::UMAX && llvm::isOneOrOneSplat(Op1, true) && BoolVT == VT && in expandIntMINMAX()
9926 getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) { in expandIntMINMAX()
9928 SDValue Zero = DAG.getConstant(0, DL, VT); in expandIntMINMAX()
9929 return DAG.getNode(ISD::SUB, DL, VT, Op0, in expandIntMINMAX()
9930 DAG.getSetCC(DL, VT, Op0, Zero, ISD::SETEQ)); in expandIntMINMAX()
9935 if (Opcode == ISD::UMIN && isOperationLegal(ISD::SUB, VT) && in expandIntMINMAX()
9936 isOperationLegal(ISD::USUBSAT, VT)) { in expandIntMINMAX()
9937 return DAG.getNode(ISD::SUB, DL, VT, Op0, in expandIntMINMAX()
9938 DAG.getNode(ISD::USUBSAT, DL, VT, Op0, Op1)); in expandIntMINMAX()
9943 if (Opcode == ISD::UMAX && isOperationLegal(ISD::ADD, VT) && in expandIntMINMAX()
9944 isOperationLegal(ISD::USUBSAT, VT)) { in expandIntMINMAX()
9945 return DAG.getNode(ISD::ADD, DL, VT, Op0, in expandIntMINMAX()
9946 DAG.getNode(ISD::USUBSAT, DL, VT, Op1, Op0)); in expandIntMINMAX()
9951 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT)) in expandIntMINMAX()
9965 return DAG.getSelect(DL, VT, Cond, Op0, Op1); in expandIntMINMAX()
9972 return DAG.getSelect(DL, VT, Cond, Op1, Op0); in expandIntMINMAX()
9976 return DAG.getSelect(DL, VT, Cond, Op0, Op1); in expandIntMINMAX()
10001 EVT VT = LHS.getValueType(); in expandAddSubSat() local
10004 assert(VT == RHS.getValueType() && "Expected operands to be the same type"); in expandAddSubSat()
10005 assert(VT.isInteger() && "Expected operands to be integers"); in expandAddSubSat()
10008 if (Opcode == ISD::USUBSAT && isOperationLegal(ISD::UMAX, VT)) { in expandAddSubSat()
10009 SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS); in expandAddSubSat()
10010 return DAG.getNode(ISD::SUB, dl, VT, Max, RHS); in expandAddSubSat()
10014 if (Opcode == ISD::UADDSAT && isOperationLegal(ISD::UMIN, VT)) { in expandAddSubSat()
10015 SDValue InvRHS = DAG.getNOT(dl, RHS, VT); in expandAddSubSat()
10016 SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS); in expandAddSubSat()
10017 return DAG.getNode(ISD::ADD, dl, VT, Min, RHS); in expandAddSubSat()
10041 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT)) in expandAddSubSat()
10045 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandAddSubSat()
10046 SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); in expandAddSubSat()
10049 SDValue Zero = DAG.getConstant(0, dl, VT); in expandAddSubSat()
10050 SDValue AllOnes = DAG.getAllOnesConstant(dl, VT); in expandAddSubSat()
10053 if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) { in expandAddSubSat()
10055 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT); in expandAddSubSat()
10056 return DAG.getNode(ISD::OR, dl, VT, SumDiff, OverflowMask); in expandAddSubSat()
10059 return DAG.getSelect(dl, VT, Overflow, AllOnes, SumDiff); in expandAddSubSat()
10063 if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) { in expandAddSubSat()
10065 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT); in expandAddSubSat()
10066 SDValue Not = DAG.getNOT(dl, OverflowMask, VT); in expandAddSubSat()
10067 return DAG.getNode(ISD::AND, dl, VT, SumDiff, Not); in expandAddSubSat()
10070 return DAG.getSelect(dl, VT, Overflow, Zero, SumDiff); in expandAddSubSat()
10091 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); in expandAddSubSat()
10092 return DAG.getSelect(dl, VT, Overflow, SatMax, SumDiff); in expandAddSubSat()
10099 SDValue SatMin = DAG.getConstant(MinVal, dl, VT); in expandAddSubSat()
10100 return DAG.getSelect(dl, VT, Overflow, SatMin, SumDiff); in expandAddSubSat()
10106 SDValue SatMin = DAG.getConstant(MinVal, dl, VT); in expandAddSubSat()
10107 SDValue Shift = DAG.getNode(ISD::SRA, dl, VT, SumDiff, in expandAddSubSat()
10108 DAG.getConstant(BitWidth - 1, dl, VT)); in expandAddSubSat()
10109 Result = DAG.getNode(ISD::XOR, dl, VT, Shift, SatMin); in expandAddSubSat()
10110 return DAG.getSelect(dl, VT, Overflow, Result, SumDiff); in expandAddSubSat()
10118 EVT VT = LHS.getValueType(); in expandShlSat() local
10124 assert(VT == RHS.getValueType() && "Expected operands to be the same type"); in expandShlSat()
10125 assert(VT.isInteger() && "Expected operands to be integers"); in expandShlSat()
10127 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT)) in expandShlSat()
10132 unsigned BW = VT.getScalarSizeInBits(); in expandShlSat()
10133 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandShlSat()
10134 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, LHS, RHS); in expandShlSat()
10136 DAG.getNode(IsSigned ? ISD::SRA : ISD::SRL, dl, VT, Result, RHS); in expandShlSat()
10140 SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(BW), dl, VT); in expandShlSat()
10141 SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(BW), dl, VT); in expandShlSat()
10143 DAG.getSetCC(dl, BoolVT, LHS, DAG.getConstant(0, dl, VT), ISD::SETLT); in expandShlSat()
10144 SatVal = DAG.getSelect(dl, VT, Cond, SatMin, SatMax); in expandShlSat()
10146 SatVal = DAG.getConstant(APInt::getMaxValue(BW), dl, VT); in expandShlSat()
10149 return DAG.getSelect(dl, VT, Cond, SatVal, Result); in expandShlSat()
10163 EVT VT = LHS.getValueType(); in expandFixedPointMul() local
10169 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandFixedPointMul()
10170 unsigned VTSize = VT.getScalarSizeInBits(); in expandFixedPointMul()
10175 if (isOperationLegalOrCustom(ISD::MUL, VT)) in expandFixedPointMul()
10176 return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandFixedPointMul()
10177 } else if (Signed && isOperationLegalOrCustom(ISD::SMULO, VT)) { in expandFixedPointMul()
10179 DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); in expandFixedPointMul()
10182 SDValue Zero = DAG.getConstant(0, dl, VT); in expandFixedPointMul()
10186 SDValue SatMin = DAG.getConstant(MinVal, dl, VT); in expandFixedPointMul()
10187 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); in expandFixedPointMul()
10190 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS); in expandFixedPointMul()
10192 Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax); in expandFixedPointMul()
10193 return DAG.getSelect(dl, VT, Overflow, Result, Product); in expandFixedPointMul()
10194 } else if (!Signed && isOperationLegalOrCustom(ISD::UMULO, VT)) { in expandFixedPointMul()
10196 DAG.getNode(ISD::UMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); in expandFixedPointMul()
10201 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); in expandFixedPointMul()
10202 return DAG.getSelect(dl, VT, Overflow, SatMax, Product); in expandFixedPointMul()
10216 if (isOperationLegalOrCustom(LoHiOp, VT)) { in expandFixedPointMul()
10217 SDValue Result = DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS); in expandFixedPointMul()
10220 } else if (isOperationLegalOrCustom(HiOp, VT)) { in expandFixedPointMul()
10221 Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandFixedPointMul()
10222 Hi = DAG.getNode(HiOp, dl, VT, LHS, RHS); in expandFixedPointMul()
10223 } else if (VT.isVector()) { in expandFixedPointMul()
10238 EVT ShiftTy = getShiftAmountTy(VT, DAG.getDataLayout()); in expandFixedPointMul()
10239 SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo, in expandFixedPointMul()
10252 dl, VT); in expandFixedPointMul()
10254 DAG.getConstant(MaxVal, dl, VT), Result, in expandFixedPointMul()
10263 SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(VTSize), dl, VT); in expandFixedPointMul()
10264 SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(VTSize), dl, VT); in expandFixedPointMul()
10267 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, Lo, in expandFixedPointMul()
10272 SDValue Zero = DAG.getConstant(0, dl, VT); in expandFixedPointMul()
10276 return DAG.getSelect(dl, VT, Overflow, ResultIfOverflow, Result); in expandFixedPointMul()
10284 dl, VT); in expandFixedPointMul()
10290 dl, VT); in expandFixedPointMul()
10303 EVT VT = LHS.getValueType(); in expandFixedPointDiv() local
10306 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandFixedPointDiv()
10335 EVT ShiftTy = getShiftAmountTy(VT, DAG.getDataLayout()); in expandFixedPointDiv()
10337 LHS = DAG.getNode(ISD::SHL, dl, VT, LHS, in expandFixedPointDiv()
10340 RHS = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, VT, RHS, in expandFixedPointDiv()
10352 if (isTypeLegal(VT) && in expandFixedPointDiv()
10353 isOperationLegalOrCustom(ISD::SDIVREM, VT)) { in expandFixedPointDiv()
10355 DAG.getVTList(VT, VT), in expandFixedPointDiv()
10360 Quot = DAG.getNode(ISD::SDIV, dl, VT, in expandFixedPointDiv()
10362 Rem = DAG.getNode(ISD::SREM, dl, VT, in expandFixedPointDiv()
10365 SDValue Zero = DAG.getConstant(0, dl, VT); in expandFixedPointDiv()
10370 SDValue Sub1 = DAG.getNode(ISD::SUB, dl, VT, Quot, in expandFixedPointDiv()
10371 DAG.getConstant(1, dl, VT)); in expandFixedPointDiv()
10372 Quot = DAG.getSelect(dl, VT, in expandFixedPointDiv()
10376 Quot = DAG.getNode(ISD::UDIV, dl, VT, in expandFixedPointDiv()
10471 EVT VT = Node->getValueType(0); in expandMULO() local
10472 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in expandMULO()
10484 EVT ShiftAmtTy = getShiftAmountTy(VT, DAG.getDataLayout()); in expandMULO()
10486 Result = DAG.getNode(ISD::SHL, dl, VT, LHS, ShiftAmt); in expandMULO()
10489 dl, VT, Result, ShiftAmt), in expandMULO()
10495 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2); in expandMULO()
10496 if (VT.isVector()) in expandMULO()
10498 EVT::getVectorVT(*DAG.getContext(), WideVT, VT.getVectorElementCount()); in expandMULO()
10505 if (isOperationLegalOrCustom(Ops[isSigned][0], VT)) { in expandMULO()
10506 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in expandMULO()
10507 TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS); in expandMULO()
10508 } else if (isOperationLegalOrCustom(Ops[isSigned][1], VT)) { in expandMULO()
10509 BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS, in expandMULO()
10516 BottomHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul); in expandMULO()
10517 SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits(), dl, in expandMULO()
10519 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, in expandMULO()
10522 if (VT.isVector()) in expandMULO()
10545 unsigned LoSize = VT.getFixedSizeInBits(); in expandMULO()
10547 DAG.getNode(ISD::SRA, dl, VT, LHS, in expandMULO()
10551 DAG.getNode(ISD::SRA, dl, VT, RHS, in expandMULO()
10555 HiLHS = DAG.getConstant(0, dl, VT); in expandMULO()
10556 HiRHS = DAG.getConstant(0, dl, VT); in expandMULO()
10593 VT.getScalarSizeInBits() - 1, dl, in expandMULO()
10595 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt); in expandMULO()
10599 DAG.getConstant(0, dl, VT), ISD::SETNE); in expandMULO()
10616 EVT VT = Op.getValueType(); in expandVecReduce() local
10618 if (VT.isScalableVector()) in expandVecReduce()
10623 if (VT.isPow2VectorType()) { in expandVecReduce()
10624 while (VT.getVectorNumElements() > 1) { in expandVecReduce()
10625 EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext()); in expandVecReduce()
10632 VT = HalfVT; in expandVecReduce()
10636 EVT EltVT = VT.getVectorElementType(); in expandVecReduce()
10637 unsigned NumElts = VT.getVectorNumElements(); in expandVecReduce()
10658 EVT VT = VecOp.getValueType(); in expandVecReduceSeq() local
10659 EVT EltVT = VT.getVectorElementType(); in expandVecReduceSeq()
10661 if (VT.isScalableVector()) in expandVecReduceSeq()
10665 unsigned NumElts = VT.getVectorNumElements(); in expandVecReduceSeq()
10681 EVT VT = Node->getValueType(0); in expandREM() local
10688 if (isOperationLegalOrCustom(DivRemOpc, VT)) { in expandREM()
10689 SDVTList VTs = DAG.getVTList(VT, VT); in expandREM()
10693 if (isOperationLegalOrCustom(DivOpc, VT)) { in expandREM()
10695 SDValue Divide = DAG.getNode(DivOpc, dl, VT, Dividend, Divisor); in expandREM()
10696 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor); in expandREM()
10697 Result = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul); in expandREM()
10818 EVT VT = Node->getValueType(0); in expandVectorSplice() local
10835 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false); in expandVectorSplice()
10837 EVT MemVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), in expandVectorSplice()
10838 VT.getVectorElementCount() * 2); in expandVectorSplice()
10850 APInt(PtrVT.getFixedSizeInBits(), VT.getStoreSize().getKnownMinValue())); in expandVectorSplice()
10857 StackPtr = getVectorElementPointer(DAG, StackPtr, VT, Node->getOperand(2)); in expandVectorSplice()
10859 return DAG.getLoad(VT, DL, StoreV2, StackPtr, in expandVectorSplice()
10866 TypeSize EltByteSize = VT.getVectorElementType().getStoreSize(); in expandVectorSplice()
10870 if (TrailingElts > VT.getVectorMinNumElements()) { in expandVectorSplice()
10874 VT.getStoreSize().getKnownMinValue())); in expandVectorSplice()
10882 return DAG.getLoad(VT, DL, StoreV2, StackPtr2, in expandVectorSplice()
10886 bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT, in LegalizeSetCCCondCode() argument
11005 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1, Chain, IsSignaling); in LegalizeSetCCCondCode()
11006 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2, Chain, IsSignaling); in LegalizeSetCCCondCode()
11008 SetCC1 = DAG.getSetCCVP(dl, VT, LHS, RHS, CC1, Mask, EVL); in LegalizeSetCCCondCode()
11009 SetCC2 = DAG.getSetCCVP(dl, VT, LHS, RHS, CC2, Mask, EVL); in LegalizeSetCCCondCode()
11014 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1, Chain, IsSignaling); in LegalizeSetCCCondCode()
11015 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2, Chain, IsSignaling); in LegalizeSetCCCondCode()
11017 SetCC1 = DAG.getSetCCVP(dl, VT, LHS, LHS, CC1, Mask, EVL); in LegalizeSetCCCondCode()
11018 SetCC2 = DAG.getSetCCVP(dl, VT, RHS, RHS, CC2, Mask, EVL); in LegalizeSetCCCondCode()
11025 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2); in LegalizeSetCCCondCode()
11030 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2, Mask, EVL); in LegalizeSetCCCondCode()