111   MVT PtrVT = MVT::getIntegerVT(TM.getPointerSizeInBits(0));  in X86TargetLowering()
174   addRegisterClass(MVT::i8, &X86::GR8RegClass);  in X86TargetLowering()
175   addRegisterClass(MVT::i16, &X86::GR16RegClass);  in X86TargetLowering()
176   addRegisterClass(MVT::i32, &X86::GR32RegClass);  in X86TargetLowering()
178     addRegisterClass(MVT::i64, &X86::GR64RegClass);  in X86TargetLowering()
180   for (MVT VT : MVT::integer_valuetypes())  in X86TargetLowering()
181     setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);  in X86TargetLowering()
184   setTruncStoreAction(MVT::i64, MVT::i32, Expand);  in X86TargetLowering()
185   setTruncStoreAction(MVT::i64, MVT::i16, Expand);  in X86TargetLowering()
186   setTruncStoreAction(MVT::i64, MVT::i8 , Expand);  in X86TargetLowering()
187   setTruncStoreAction(MVT::i32, MVT::i16, Expand);  in X86TargetLowering()
188   setTruncStoreAction(MVT::i32, MVT::i8 , Expand);  in X86TargetLowering()
189   setTruncStoreAction(MVT::i16, MVT::i8,  Expand);  in X86TargetLowering()
191   setTruncStoreAction(MVT::f64, MVT::f32, Expand);  in X86TargetLowering()
194   for (auto VT : {MVT::f32, MVT::f64, MVT::f80}) {  in X86TargetLowering()
201     setOperationAction(ISD::ABS            , MVT::i16  , Custom);  in X86TargetLowering()
202     setOperationAction(ISD::ABS            , MVT::i32  , Custom);  in X86TargetLowering()
204       setOperationAction(ISD::ABS          , MVT::i64  , Custom);  in X86TargetLowering()
208   setOperationAction(ISD::SSUBSAT          , MVT::i8   , Custom);  in X86TargetLowering()
209   setOperationAction(ISD::SSUBSAT          , MVT::i16  , Custom);  in X86TargetLowering()
210   setOperationAction(ISD::SSUBSAT          , MVT::i32  , Custom);  in X86TargetLowering()
212     setOperationAction(ISD::SSUBSAT        , MVT::i64  , Custom);  in X86TargetLowering()
219     setOperationAction(ShiftOp             , MVT::i8   , Custom);  in X86TargetLowering()
220     setOperationAction(ShiftOp             , MVT::i16  , Custom);  in X86TargetLowering()
221     setOperationAction(ShiftOp             , MVT::i32  , ShiftDoubleAction);  in X86TargetLowering()
223       setOperationAction(ShiftOp           , MVT::i64  , ShiftDoubleAction);  in X86TargetLowering()
229     setOperationAction(ISD::UINT_TO_FP,        MVT::i8, Promote);  in X86TargetLowering()
230     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i8, Promote);  in X86TargetLowering()
231     setOperationAction(ISD::UINT_TO_FP,        MVT::i16, Promote);  in X86TargetLowering()
232     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i16, Promote);  in X86TargetLowering()
235     setOperationAction(ISD::UINT_TO_FP,        MVT::i32, Custom);  in X86TargetLowering()
236     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom);  in X86TargetLowering()
239     setOperationAction(ISD::UINT_TO_FP,        MVT::i64, Custom);  in X86TargetLowering()
240     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom);  in X86TargetLowering()
244     setOperationAction(ISD::SINT_TO_FP,        MVT::i8, Promote);  in X86TargetLowering()
245     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i8, Promote);  in X86TargetLowering()
248     setOperationAction(ISD::SINT_TO_FP,        MVT::i16, Custom);  in X86TargetLowering()
249     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i16, Custom);  in X86TargetLowering()
251     setOperationAction(ISD::SINT_TO_FP,        MVT::i32, Custom);  in X86TargetLowering()
252     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);  in X86TargetLowering()
255     setOperationAction(ISD::SINT_TO_FP,        MVT::i64, Custom);  in X86TargetLowering()
256     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);  in X86TargetLowering()
260     setOperationAction(ISD::FP_TO_SINT,        MVT::i8,  Promote);  in X86TargetLowering()
262     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i8,  Promote);  in X86TargetLowering()
263     setOperationAction(ISD::FP_TO_SINT,        MVT::i16, Custom);  in X86TargetLowering()
264     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i16, Custom);  in X86TargetLowering()
265     setOperationAction(ISD::FP_TO_SINT,        MVT::i32, Custom);  in X86TargetLowering()
266     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);  in X86TargetLowering()
269     setOperationAction(ISD::FP_TO_SINT,        MVT::i64, Custom);  in X86TargetLowering()
270     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);  in X86TargetLowering()
274     setOperationAction(ISD::FP_TO_UINT,        MVT::i8,  Promote);  in X86TargetLowering()
276     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i8,  Promote);  in X86TargetLowering()
277     setOperationAction(ISD::FP_TO_UINT,        MVT::i16, Promote);  in X86TargetLowering()
279     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i16, Promote);  in X86TargetLowering()
280     setOperationAction(ISD::FP_TO_UINT,        MVT::i32, Custom);  in X86TargetLowering()
281     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);  in X86TargetLowering()
282     setOperationAction(ISD::FP_TO_UINT,        MVT::i64, Custom);  in X86TargetLowering()
283     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom);  in X86TargetLowering()
285     setOperationAction(ISD::LRINT,             MVT::f32, Custom);  in X86TargetLowering()
286     setOperationAction(ISD::LRINT,             MVT::f64, Custom);  in X86TargetLowering()
287     setOperationAction(ISD::LLRINT,            MVT::f32, Custom);  in X86TargetLowering()
288     setOperationAction(ISD::LLRINT,            MVT::f64, Custom);  in X86TargetLowering()
291       setOperationAction(ISD::LRINT,  MVT::i64, Custom);  in X86TargetLowering()
292       setOperationAction(ISD::LLRINT, MVT::i64, Custom);  in X86TargetLowering()
299     for (MVT VT : { MVT::i8, MVT::i16, MVT::i32 }) {  in X86TargetLowering()
304       setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i64, Custom);  in X86TargetLowering()
305       setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i64, Custom);  in X86TargetLowering()
310   setOperationAction(ISD::ADDRSPACECAST, MVT::i32, Custom);  in X86TargetLowering()
311   setOperationAction(ISD::ADDRSPACECAST, MVT::i64, Custom);  in X86TargetLowering()
315     setOperationAction(ISD::BITCAST        , MVT::f32  , Expand);  in X86TargetLowering()
316     setOperationAction(ISD::BITCAST        , MVT::i32  , Expand);  in X86TargetLowering()
318       setOperationAction(ISD::BITCAST      , MVT::f64  , Expand);  in X86TargetLowering()
320       setOperationAction(ISD::BITCAST      , MVT::i64  , Expand);  in X86TargetLowering()
323     setOperationAction(ISD::BITCAST      , MVT::i64  , Custom);  in X86TargetLowering()
335   for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {  in X86TargetLowering()
344   setOperationAction(ISD::BR_JT            , MVT::Other, Expand);  in X86TargetLowering()
345   setOperationAction(ISD::BRCOND           , MVT::Other, Custom);  in X86TargetLowering()
346   for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128,  in X86TargetLowering()
347                    MVT::i8,  MVT::i16, MVT::i32, MVT::i64 }) {  in X86TargetLowering()
352     setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);  in X86TargetLowering()
353   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16  , Legal);  in X86TargetLowering()
354   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8   , Legal);  in X86TargetLowering()
355   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1   , Expand);  in X86TargetLowering()
357   setOperationAction(ISD::FREM             , MVT::f32  , Expand);  in X86TargetLowering()
358   setOperationAction(ISD::FREM             , MVT::f64  , Expand);  in X86TargetLowering()
359   setOperationAction(ISD::FREM             , MVT::f80  , Expand);  in X86TargetLowering()
360   setOperationAction(ISD::FREM             , MVT::f128 , Expand);  in X86TargetLowering()
363     setOperationAction(ISD::FLT_ROUNDS_    , MVT::i32  , Custom);  in X86TargetLowering()
364     setOperationAction(ISD::SET_ROUNDING   , MVT::Other, Custom);  in X86TargetLowering()
369   setOperationPromotedToType(ISD::CTTZ           , MVT::i8   , MVT::i32);  in X86TargetLowering()
370   setOperationPromotedToType(ISD::CTTZ_ZERO_UNDEF, MVT::i8   , MVT::i32);  in X86TargetLowering()
375     setOperationPromotedToType(ISD::CTTZ_ZERO_UNDEF, MVT::i16, MVT::i32);  in X86TargetLowering()
377     setOperationAction(ISD::CTTZ, MVT::i16, Custom);  in X86TargetLowering()
378     setOperationAction(ISD::CTTZ           , MVT::i32  , Custom);  in X86TargetLowering()
379     setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16  , Legal);  in X86TargetLowering()
380     setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32  , Legal);  in X86TargetLowering()
382       setOperationAction(ISD::CTTZ         , MVT::i64  , Custom);  in X86TargetLowering()
383       setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Legal);  in X86TargetLowering()
390     setOperationPromotedToType(ISD::CTLZ           , MVT::i8   , MVT::i32);  in X86TargetLowering()
391     setOperationPromotedToType(ISD::CTLZ_ZERO_UNDEF, MVT::i8   , MVT::i32);  in X86TargetLowering()
393     for (auto VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {  in X86TargetLowering()
394       if (VT == MVT::i64 && !Subtarget.is64Bit())  in X86TargetLowering()
407         Op, MVT::f32,  in X86TargetLowering()
410     setOperationAction(Op, MVT::f64, Expand);  in X86TargetLowering()
411     setOperationAction(Op, MVT::f80, Expand);  in X86TargetLowering()
412     setOperationAction(Op, MVT::f128, Expand);  in X86TargetLowering()
415   for (MVT VT : {MVT::f32, MVT::f64, MVT::f80, MVT::f128}) {  in X86TargetLowering()
416     setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);  in X86TargetLowering()
417     setLoadExtAction(ISD::EXTLOAD, VT, MVT::bf16, Expand);  in X86TargetLowering()
418     setTruncStoreAction(VT, MVT::f16, Expand);  in X86TargetLowering()
419     setTruncStoreAction(VT, MVT::bf16, Expand);  in X86TargetLowering()
425   setOperationAction(ISD::PARITY, MVT::i8, Custom);  in X86TargetLowering()
426   setOperationAction(ISD::PARITY, MVT::i16, Custom);  in X86TargetLowering()
427   setOperationAction(ISD::PARITY, MVT::i32, Custom);  in X86TargetLowering()
429     setOperationAction(ISD::PARITY, MVT::i64, Custom);  in X86TargetLowering()
431     setOperationPromotedToType(ISD::CTPOP, MVT::i8, MVT::i32);  in X86TargetLowering()
434     setOperationPromotedToType(ISD::CTPOP, MVT::i16, MVT::i32);  in X86TargetLowering()
436     setOperationAction(ISD::CTPOP          , MVT::i8   , Expand);  in X86TargetLowering()
437     setOperationAction(ISD::CTPOP          , MVT::i16  , Expand);  in X86TargetLowering()
438     setOperationAction(ISD::CTPOP          , MVT::i32  , Expand);  in X86TargetLowering()
440       setOperationAction(ISD::CTPOP        , MVT::i64  , Expand);  in X86TargetLowering()
442       setOperationAction(ISD::CTPOP        , MVT::i64  , Custom);  in X86TargetLowering()
445   setOperationAction(ISD::READCYCLECOUNTER , MVT::i64  , Custom);  in X86TargetLowering()
448     setOperationAction(ISD::BSWAP          , MVT::i16  , Expand);  in X86TargetLowering()
451   for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) {  in X86TargetLowering()
457   for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {  in X86TargetLowering()
458     if (VT == MVT::i64 && !Subtarget.is64Bit())  in X86TargetLowering()
465   setOperationAction(ISD::SELECT, MVT::x86mmx, Custom);  in X86TargetLowering()
466   setOperationAction(ISD::SELECT_CC, MVT::x86mmx, Expand);  in X86TargetLowering()
468   setOperationAction(ISD::EH_RETURN       , MVT::Other, Custom);  in X86TargetLowering()
471   setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);  in X86TargetLowering()
472   setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);  in X86TargetLowering()
473   setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);  in X86TargetLowering()
478   for (auto VT : { MVT::i32, MVT::i64 }) {  in X86TargetLowering()
479     if (VT == MVT::i64 && !Subtarget.is64Bit())  in X86TargetLowering()
490   for (auto VT : { MVT::i32, MVT::i64 }) {  in X86TargetLowering()
491     if (VT == MVT::i64 && !Subtarget.is64Bit())  in X86TargetLowering()
499     setOperationAction(ISD::PREFETCH      , MVT::Other, Legal);  in X86TargetLowering()
501   setOperationAction(ISD::ATOMIC_FENCE  , MVT::Other, Custom);  in X86TargetLowering()
504   for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {  in X86TargetLowering()
515     setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Custom);  in X86TargetLowering()
518     setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom);  in X86TargetLowering()
524     setOperationAction(ISD::EH_LABEL, MVT::Other, Expand);  in X86TargetLowering()
527   setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom);  in X86TargetLowering()
528   setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom);  in X86TargetLowering()
530   setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);  in X86TargetLowering()
531   setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);  in X86TargetLowering()
533   setOperationAction(ISD::TRAP, MVT::Other, Legal);  in X86TargetLowering()
534   setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);  in X86TargetLowering()
536     setOperationAction(ISD::UBSANTRAP, MVT::Other, Expand);  in X86TargetLowering()
538     setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal);  in X86TargetLowering()
541   setOperationAction(ISD::VASTART           , MVT::Other, Custom);  in X86TargetLowering()
542   setOperationAction(ISD::VAEND             , MVT::Other, Expand);  in X86TargetLowering()
544   setOperationAction(ISD::VAARG,  MVT::Other, Is64Bit ? Custom : Expand);  in X86TargetLowering()
545   setOperationAction(ISD::VACOPY, MVT::Other, Is64Bit ? Custom : Expand);  in X86TargetLowering()
547   setOperationAction(ISD::STACKSAVE,          MVT::Other, Expand);  in X86TargetLowering()
548   setOperationAction(ISD::STACKRESTORE,       MVT::Other, Expand);  in X86TargetLowering()
553   setOperationAction(ISD::GC_TRANSITION_START, MVT::Other, Custom);  in X86TargetLowering()
554   setOperationAction(ISD::GC_TRANSITION_END, MVT::Other, Custom);  in X86TargetLowering()
556   setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);  in X86TargetLowering()
558   auto setF16Action = [&] (MVT VT, LegalizeAction Action) {  in X86TargetLowering()
594     addRegisterClass(MVT::f16, Subtarget.hasAVX512() ? &X86::FR16XRegClass  in X86TargetLowering()
596     addRegisterClass(MVT::f32, Subtarget.hasAVX512() ? &X86::FR32XRegClass  in X86TargetLowering()
598     addRegisterClass(MVT::f64, Subtarget.hasAVX512() ? &X86::FR64XRegClass  in X86TargetLowering()
605     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);  in X86TargetLowering()
607     for (auto VT : { MVT::f32, MVT::f64 }) {  in X86TargetLowering()
628     setF16Action(MVT::f16, Promote);  in X86TargetLowering()
629     setOperationAction(ISD::FADD, MVT::f16, Promote);  in X86TargetLowering()
630     setOperationAction(ISD::FSUB, MVT::f16, Promote);  in X86TargetLowering()
631     setOperationAction(ISD::FMUL, MVT::f16, Promote);  in X86TargetLowering()
632     setOperationAction(ISD::FDIV, MVT::f16, Promote);  in X86TargetLowering()
633     setOperationAction(ISD::FP_ROUND, MVT::f16, LibCall);  in X86TargetLowering()
634     setOperationAction(ISD::FP_EXTEND, MVT::f32, LibCall);  in X86TargetLowering()
635     setOperationAction(ISD::FP_EXTEND, MVT::f64, Custom);  in X86TargetLowering()
637     setOperationAction(ISD::STRICT_FADD, MVT::f16, Promote);  in X86TargetLowering()
638     setOperationAction(ISD::STRICT_FSUB, MVT::f16, Promote);  in X86TargetLowering()
639     setOperationAction(ISD::STRICT_FMUL, MVT::f16, Promote);  in X86TargetLowering()
640     setOperationAction(ISD::STRICT_FDIV, MVT::f16, Promote);  in X86TargetLowering()
641     setOperationAction(ISD::STRICT_FMA, MVT::f16, Promote);  in X86TargetLowering()
642     setOperationAction(ISD::STRICT_FMINNUM, MVT::f16, Promote);  in X86TargetLowering()
643     setOperationAction(ISD::STRICT_FMAXNUM, MVT::f16, Promote);  in X86TargetLowering()
644     setOperationAction(ISD::STRICT_FMINIMUM, MVT::f16, Promote);  in X86TargetLowering()
645     setOperationAction(ISD::STRICT_FMAXIMUM, MVT::f16, Promote);  in X86TargetLowering()
646     setOperationAction(ISD::STRICT_FSQRT, MVT::f16, Promote);  in X86TargetLowering()
647     setOperationAction(ISD::STRICT_FPOW, MVT::f16, Promote);  in X86TargetLowering()
648     setOperationAction(ISD::STRICT_FLOG, MVT::f16, Promote);  in X86TargetLowering()
649     setOperationAction(ISD::STRICT_FLOG2, MVT::f16, Promote);  in X86TargetLowering()
650     setOperationAction(ISD::STRICT_FLOG10, MVT::f16, Promote);  in X86TargetLowering()
651     setOperationAction(ISD::STRICT_FEXP, MVT::f16, Promote);  in X86TargetLowering()
652     setOperationAction(ISD::STRICT_FEXP2, MVT::f16, Promote);  in X86TargetLowering()
653     setOperationAction(ISD::STRICT_FCEIL, MVT::f16, Promote);  in X86TargetLowering()
654     setOperationAction(ISD::STRICT_FFLOOR, MVT::f16, Promote);  in X86TargetLowering()
655     setOperationAction(ISD::STRICT_FNEARBYINT, MVT::f16, Promote);  in X86TargetLowering()
656     setOperationAction(ISD::STRICT_FRINT, MVT::f16, Promote);  in X86TargetLowering()
657     setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Promote);  in X86TargetLowering()
658     setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Promote);  in X86TargetLowering()
659     setOperationAction(ISD::STRICT_FROUND, MVT::f16, Promote);  in X86TargetLowering()
660     setOperationAction(ISD::STRICT_FROUNDEVEN, MVT::f16, Promote);  in X86TargetLowering()
661     setOperationAction(ISD::STRICT_FTRUNC, MVT::f16, Promote);  in X86TargetLowering()
662     setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, LibCall);  in X86TargetLowering()
663     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, LibCall);  in X86TargetLowering()
664     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Custom);  in X86TargetLowering()
670     setOperationAction(ISD::FGETSIGN, MVT::i64, Custom);  in X86TargetLowering()
671     setOperationAction(ISD::FGETSIGN, MVT::i32, Custom);  in X86TargetLowering()
677     addRegisterClass(MVT::f32, &X86::FR32RegClass);  in X86TargetLowering()
679       addRegisterClass(MVT::f64, &X86::RFP64RegClass);  in X86TargetLowering()
682     setOperationAction(ISD::FABS , MVT::f32, Custom);  in X86TargetLowering()
685     setOperationAction(ISD::FNEG , MVT::f32, Custom);  in X86TargetLowering()
688       setOperationAction(ISD::UNDEF, MVT::f64, Expand);  in X86TargetLowering()
692       setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);  in X86TargetLowering()
693     setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);  in X86TargetLowering()
696     setOperationAction(ISD::FSIN   , MVT::f32, Expand);  in X86TargetLowering()
697     setOperationAction(ISD::FCOS   , MVT::f32, Expand);  in X86TargetLowering()
698     setOperationAction(ISD::FSINCOS, MVT::f32, Expand);  in X86TargetLowering()
702       setOperationAction(ISD::FSIN, MVT::f64, Expand);  in X86TargetLowering()
703       setOperationAction(ISD::FCOS, MVT::f64, Expand);  in X86TargetLowering()
704       setOperationAction(ISD::FSINCOS, MVT::f64, Expand);  in X86TargetLowering()
709     addRegisterClass(MVT::f64, &X86::RFP64RegClass);  in X86TargetLowering()
710     addRegisterClass(MVT::f32, &X86::RFP32RegClass);  in X86TargetLowering()
712     for (auto VT : { MVT::f32, MVT::f64 }) {  in X86TargetLowering()
724   if (isTypeLegal(MVT::f32)) {  in X86TargetLowering()
725     if (UseX87 && (getRegClassFor(MVT::f32) == &X86::RFP32RegClass)) {  in X86TargetLowering()
734   if (isTypeLegal(MVT::f64)) {  in X86TargetLowering()
735     if (UseX87 && getRegClassFor(MVT::f64) == &X86::RFP64RegClass) {  in X86TargetLowering()
744   if (isTypeLegal(MVT::f16))  in X86TargetLowering()
748   setOperationAction(ISD::STRICT_FADD,      MVT::f32, Legal);  in X86TargetLowering()
749   setOperationAction(ISD::STRICT_FADD,      MVT::f64, Legal);  in X86TargetLowering()
750   setOperationAction(ISD::STRICT_FSUB,      MVT::f32, Legal);  in X86TargetLowering()
751   setOperationAction(ISD::STRICT_FSUB,      MVT::f64, Legal);  in X86TargetLowering()
752   setOperationAction(ISD::STRICT_FMUL,      MVT::f32, Legal);  in X86TargetLowering()
753   setOperationAction(ISD::STRICT_FMUL,      MVT::f64, Legal);  in X86TargetLowering()
754   setOperationAction(ISD::STRICT_FDIV,      MVT::f32, Legal);  in X86TargetLowering()
755   setOperationAction(ISD::STRICT_FDIV,      MVT::f64, Legal);  in X86TargetLowering()
756   setOperationAction(ISD::STRICT_FP_ROUND,  MVT::f32, Legal);  in X86TargetLowering()
757   setOperationAction(ISD::STRICT_FP_ROUND,  MVT::f64, Legal);  in X86TargetLowering()
758   setOperationAction(ISD::STRICT_FSQRT,     MVT::f32, Legal);  in X86TargetLowering()
759   setOperationAction(ISD::STRICT_FSQRT,     MVT::f64, Legal);  in X86TargetLowering()
762   setOperationAction(ISD::FMA, MVT::f64, Expand);  in X86TargetLowering()
763   setOperationAction(ISD::FMA, MVT::f32, Expand);  in X86TargetLowering()
767     addRegisterClass(MVT::f80, &X86::RFP80RegClass);  in X86TargetLowering()
768     setOperationAction(ISD::UNDEF,     MVT::f80, Expand);  in X86TargetLowering()
769     setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand);  in X86TargetLowering()
786     setOperationAction(ISD::FSIN   , MVT::f80, Expand);  in X86TargetLowering()
787     setOperationAction(ISD::FCOS   , MVT::f80, Expand);  in X86TargetLowering()
788     setOperationAction(ISD::FSINCOS, MVT::f80, Expand);  in X86TargetLowering()
790     setOperationAction(ISD::FFLOOR, MVT::f80, Expand);  in X86TargetLowering()
791     setOperationAction(ISD::FCEIL,  MVT::f80, Expand);  in X86TargetLowering()
792     setOperationAction(ISD::FTRUNC, MVT::f80, Expand);  in X86TargetLowering()
793     setOperationAction(ISD::FRINT,  MVT::f80, Expand);  in X86TargetLowering()
794     setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand);  in X86TargetLowering()
795     setOperationAction(ISD::FMA, MVT::f80, Expand);  in X86TargetLowering()
796     setOperationAction(ISD::LROUND, MVT::f80, Expand);  in X86TargetLowering()
797     setOperationAction(ISD::LLROUND, MVT::f80, Expand);  in X86TargetLowering()
798     setOperationAction(ISD::LRINT, MVT::f80, Custom);  in X86TargetLowering()
799     setOperationAction(ISD::LLRINT, MVT::f80, Custom);  in X86TargetLowering()
802     setOperationAction(ISD::STRICT_FADD     , MVT::f80, Legal);  in X86TargetLowering()
803     setOperationAction(ISD::STRICT_FSUB     , MVT::f80, Legal);  in X86TargetLowering()
804     setOperationAction(ISD::STRICT_FMUL     , MVT::f80, Legal);  in X86TargetLowering()
805     setOperationAction(ISD::STRICT_FDIV     , MVT::f80, Legal);  in X86TargetLowering()
806     setOperationAction(ISD::STRICT_FSQRT    , MVT::f80, Legal);  in X86TargetLowering()
807     if (isTypeLegal(MVT::f16)) {  in X86TargetLowering()
808       setOperationAction(ISD::FP_EXTEND, MVT::f80, Custom);  in X86TargetLowering()
809       setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f80, Custom);  in X86TargetLowering()
811       setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f80, Legal);  in X86TargetLowering()
815     setOperationAction(ISD::STRICT_FP_ROUND, MVT::f80, Legal);  in X86TargetLowering()
820     addRegisterClass(MVT::f128, Subtarget.hasVLX() ? &X86::VR128XRegClass  in X86TargetLowering()
825     setOperationAction(ISD::FADD,        MVT::f128, LibCall);  in X86TargetLowering()
826     setOperationAction(ISD::STRICT_FADD, MVT::f128, LibCall);  in X86TargetLowering()
827     setOperationAction(ISD::FSUB,        MVT::f128, LibCall);  in X86TargetLowering()
828     setOperationAction(ISD::STRICT_FSUB, MVT::f128, LibCall);  in X86TargetLowering()
829     setOperationAction(ISD::FDIV,        MVT::f128, LibCall);  in X86TargetLowering()
830     setOperationAction(ISD::STRICT_FDIV, MVT::f128, LibCall);  in X86TargetLowering()
831     setOperationAction(ISD::FMUL,        MVT::f128, LibCall);  in X86TargetLowering()
832     setOperationAction(ISD::STRICT_FMUL, MVT::f128, LibCall);  in X86TargetLowering()
833     setOperationAction(ISD::FMA,         MVT::f128, LibCall);  in X86TargetLowering()
834     setOperationAction(ISD::STRICT_FMA,  MVT::f128, LibCall);  in X86TargetLowering()
836     setOperationAction(ISD::FABS, MVT::f128, Custom);  in X86TargetLowering()
837     setOperationAction(ISD::FNEG, MVT::f128, Custom);  in X86TargetLowering()
838     setOperationAction(ISD::FCOPYSIGN, MVT::f128, Custom);  in X86TargetLowering()
840     setOperationAction(ISD::FSIN,         MVT::f128, LibCall);  in X86TargetLowering()
841     setOperationAction(ISD::STRICT_FSIN,  MVT::f128, LibCall);  in X86TargetLowering()
842     setOperationAction(ISD::FCOS,         MVT::f128, LibCall);  in X86TargetLowering()
843     setOperationAction(ISD::STRICT_FCOS,  MVT::f128, LibCall);  in X86TargetLowering()
844     setOperationAction(ISD::FSINCOS,      MVT::f128, LibCall);  in X86TargetLowering()
846     setOperationAction(ISD::FSQRT,        MVT::f128, LibCall);  in X86TargetLowering()
847     setOperationAction(ISD::STRICT_FSQRT, MVT::f128, LibCall);  in X86TargetLowering()
849     setOperationAction(ISD::FP_EXTEND,        MVT::f128, Custom);  in X86TargetLowering()
850     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Custom);  in X86TargetLowering()
854     if (isTypeLegal(MVT::f32)) {  in X86TargetLowering()
855       setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);  in X86TargetLowering()
856       setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom);  in X86TargetLowering()
858     if (isTypeLegal(MVT::f64)) {  in X86TargetLowering()
859       setOperationAction(ISD::FP_ROUND, MVT::f64, Custom);  in X86TargetLowering()
860       setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Custom);  in X86TargetLowering()
862     if (isTypeLegal(MVT::f80)) {  in X86TargetLowering()
863       setOperationAction(ISD::FP_ROUND, MVT::f80, Custom);  in X86TargetLowering()
864       setOperationAction(ISD::STRICT_FP_ROUND, MVT::f80, Custom);  in X86TargetLowering()
867     setOperationAction(ISD::SETCC, MVT::f128, Custom);  in X86TargetLowering()
869     setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f32, Expand);  in X86TargetLowering()
870     setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f64, Expand);  in X86TargetLowering()
871     setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f80, Expand);  in X86TargetLowering()
872     setTruncStoreAction(MVT::f128, MVT::f32, Expand);  in X86TargetLowering()
873     setTruncStoreAction(MVT::f128, MVT::f64, Expand);  in X86TargetLowering()
874     setTruncStoreAction(MVT::f128, MVT::f80, Expand);  in X86TargetLowering()
878   setOperationAction(ISD::FPOW             , MVT::f32  , Expand);  in X86TargetLowering()
879   setOperationAction(ISD::FPOW             , MVT::f64  , Expand);  in X86TargetLowering()
880   setOperationAction(ISD::FPOW             , MVT::f80  , Expand);  in X86TargetLowering()
881   setOperationAction(ISD::FPOW             , MVT::f128 , Expand);  in X86TargetLowering()
883   setOperationAction(ISD::FLOG, MVT::f80, Expand);  in X86TargetLowering()
884   setOperationAction(ISD::FLOG2, MVT::f80, Expand);  in X86TargetLowering()
885   setOperationAction(ISD::FLOG10, MVT::f80, Expand);  in X86TargetLowering()
886   setOperationAction(ISD::FEXP, MVT::f80, Expand);  in X86TargetLowering()
887   setOperationAction(ISD::FEXP2, MVT::f80, Expand);  in X86TargetLowering()
888   setOperationAction(ISD::FMINNUM, MVT::f80, Expand);  in X86TargetLowering()
889   setOperationAction(ISD::FMAXNUM, MVT::f80, Expand);  in X86TargetLowering()
892   for (auto VT : { MVT::v8f16, MVT::v16f16, MVT::v32f16,  in X86TargetLowering()
893                    MVT::v4f32, MVT::v8f32,  MVT::v16f32,  in X86TargetLowering()
894                    MVT::v2f64, MVT::v4f64,  MVT::v8f64 }) {  in X86TargetLowering()
911   for (MVT VT : MVT::fixedlen_vector_valuetypes()) {  in X86TargetLowering()
949     for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {  in X86TargetLowering()
959       if (VT.getVectorElementType() == MVT::i1)  in X86TargetLowering()
964       if (VT.getVectorElementType() == MVT::f16 ||  in X86TargetLowering()
965           VT.getVectorElementType() == MVT::bf16)  in X86TargetLowering()
973     addRegisterClass(MVT::x86mmx, &X86::VR64RegClass);  in X86TargetLowering()
978     addRegisterClass(MVT::v4f32, Subtarget.hasVLX() ? &X86::VR128XRegClass  in X86TargetLowering()
981     setOperationAction(ISD::FNEG,               MVT::v4f32, Custom);  in X86TargetLowering()
982     setOperationAction(ISD::FABS,               MVT::v4f32, Custom);  in X86TargetLowering()
983     setOperationAction(ISD::FCOPYSIGN,          MVT::v4f32, Custom);  in X86TargetLowering()
984     setOperationAction(ISD::BUILD_VECTOR,       MVT::v4f32, Custom);  in X86TargetLowering()
985     setOperationAction(ISD::VECTOR_SHUFFLE,     MVT::v4f32, Custom);  in X86TargetLowering()
986     setOperationAction(ISD::VSELECT,            MVT::v4f32, Custom);  in X86TargetLowering()
987     setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);  in X86TargetLowering()
988     setOperationAction(ISD::SELECT,             MVT::v4f32, Custom);  in X86TargetLowering()
990     setOperationAction(ISD::LOAD,               MVT::v2f32, Custom);  in X86TargetLowering()
991     setOperationAction(ISD::STORE,              MVT::v2f32, Custom);  in X86TargetLowering()
993     setOperationAction(ISD::STRICT_FADD,        MVT::v4f32, Legal);  in X86TargetLowering()
994     setOperationAction(ISD::STRICT_FSUB,        MVT::v4f32, Legal);  in X86TargetLowering()
995     setOperationAction(ISD::STRICT_FMUL,        MVT::v4f32, Legal);  in X86TargetLowering()
996     setOperationAction(ISD::STRICT_FDIV,        MVT::v4f32, Legal);  in X86TargetLowering()
997     setOperationAction(ISD::STRICT_FSQRT,       MVT::v4f32, Legal);  in X86TargetLowering()
1001     addRegisterClass(MVT::v2f64, Subtarget.hasVLX() ? &X86::VR128XRegClass  in X86TargetLowering()
1006     addRegisterClass(MVT::v16i8, Subtarget.hasVLX() ? &X86::VR128XRegClass  in X86TargetLowering()
1008     addRegisterClass(MVT::v8i16, Subtarget.hasVLX() ? &X86::VR128XRegClass  in X86TargetLowering()
1010     addRegisterClass(MVT::v8f16, Subtarget.hasVLX() ? &X86::VR128XRegClass  in X86TargetLowering()
1012     addRegisterClass(MVT::v4i32, Subtarget.hasVLX() ? &X86::VR128XRegClass  in X86TargetLowering()
1014     addRegisterClass(MVT::v2i64, Subtarget.hasVLX() ? &X86::VR128XRegClass  in X86TargetLowering()
1017     for (auto VT : { MVT::v2i8, MVT::v4i8, MVT::v8i8,  in X86TargetLowering()
1018                      MVT::v2i16, MVT::v4i16, MVT::v2i32 }) {  in X86TargetLowering()
1025     setOperationAction(ISD::MUL,                MVT::v2i8,  Custom);  in X86TargetLowering()
1026     setOperationAction(ISD::MUL,                MVT::v4i8,  Custom);  in X86TargetLowering()
1027     setOperationAction(ISD::MUL,                MVT::v8i8,  Custom);  in X86TargetLowering()
1029     setOperationAction(ISD::MUL,                MVT::v16i8, Custom);  in X86TargetLowering()
1030     setOperationAction(ISD::MUL,                MVT::v4i32, Custom);  in X86TargetLowering()
1031     setOperationAction(ISD::MUL,                MVT::v2i64, Custom);  in X86TargetLowering()
1032     setOperationAction(ISD::MULHU,              MVT::v4i32, Custom);  in X86TargetLowering()
1033     setOperationAction(ISD::MULHS,              MVT::v4i32, Custom);  in X86TargetLowering()
1034     setOperationAction(ISD::MULHU,              MVT::v16i8, Custom);  in X86TargetLowering()
1035     setOperationAction(ISD::MULHS,              MVT::v16i8, Custom);  in X86TargetLowering()
1036     setOperationAction(ISD::MULHU,              MVT::v8i16, Legal);  in X86TargetLowering()
1037     setOperationAction(ISD::MULHS,              MVT::v8i16, Legal);  in X86TargetLowering()
1038     setOperationAction(ISD::MUL,                MVT::v8i16, Legal);  in X86TargetLowering()
1039     setOperationAction(ISD::AVGCEILU,           MVT::v16i8, Legal);  in X86TargetLowering()
1040     setOperationAction(ISD::AVGCEILU,           MVT::v8i16, Legal);  in X86TargetLowering()
1042     setOperationAction(ISD::SMULO,              MVT::v16i8, Custom);  in X86TargetLowering()
1043     setOperationAction(ISD::UMULO,              MVT::v16i8, Custom);  in X86TargetLowering()
1044     setOperationAction(ISD::UMULO,              MVT::v2i32, Custom);  in X86TargetLowering()
1046     setOperationAction(ISD::FNEG,               MVT::v2f64, Custom);  in X86TargetLowering()
1047     setOperationAction(ISD::FABS,               MVT::v2f64, Custom);  in X86TargetLowering()
1048     setOperationAction(ISD::FCOPYSIGN,          MVT::v2f64, Custom);  in X86TargetLowering()
1050     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {  in X86TargetLowering()
1051       setOperationAction(ISD::SMAX, VT, VT == MVT::v8i16 ? Legal : Custom);  in X86TargetLowering()
1052       setOperationAction(ISD::SMIN, VT, VT == MVT::v8i16 ? Legal : Custom);  in X86TargetLowering()
1053       setOperationAction(ISD::UMAX, VT, VT == MVT::v16i8 ? Legal : Custom);  in X86TargetLowering()
1054       setOperationAction(ISD::UMIN, VT, VT == MVT::v16i8 ? Legal : Custom);  in X86TargetLowering()
1057     setOperationAction(ISD::UADDSAT,            MVT::v16i8, Legal);  in X86TargetLowering()
1058     setOperationAction(ISD::SADDSAT,            MVT::v16i8, Legal);  in X86TargetLowering()
1059     setOperationAction(ISD::USUBSAT,            MVT::v16i8, Legal);  in X86TargetLowering()
1060     setOperationAction(ISD::SSUBSAT,            MVT::v16i8, Legal);  in X86TargetLowering()
1061     setOperationAction(ISD::UADDSAT,            MVT::v8i16, Legal);  in X86TargetLowering()
1062     setOperationAction(ISD::SADDSAT,            MVT::v8i16, Legal);  in X86TargetLowering()
1063     setOperationAction(ISD::USUBSAT,            MVT::v8i16, Legal);  in X86TargetLowering()
1064     setOperationAction(ISD::SSUBSAT,            MVT::v8i16, Legal);  in X86TargetLowering()
1065     setOperationAction(ISD::USUBSAT,            MVT::v4i32, Custom);  in X86TargetLowering()
1066     setOperationAction(ISD::USUBSAT,            MVT::v2i64, Custom);  in X86TargetLowering()
1068     setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v16i8, Custom);  in X86TargetLowering()
1069     setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v8i16, Custom);  in X86TargetLowering()
1070     setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v4i32, Custom);  in X86TargetLowering()
1071     setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v4f32, Custom);  in X86TargetLowering()
1073     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {  in X86TargetLowering()
1086     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {  in X86TargetLowering()
1094     for (auto VT : { MVT::v8f16, MVT::v2f64, MVT::v2i64 }) {  in X86TargetLowering()
1099       if (VT == MVT::v2i64 && !Subtarget.is64Bit())  in X86TargetLowering()
1105     setF16Action(MVT::v8f16, Expand);  in X86TargetLowering()
1106     setOperationAction(ISD::FADD, MVT::v8f16, Expand);  in X86TargetLowering()
1107     setOperationAction(ISD::FSUB, MVT::v8f16, Expand);  in X86TargetLowering()
1108     setOperationAction(ISD::FMUL, MVT::v8f16, Expand);  in X86TargetLowering()
1109     setOperationAction(ISD::FDIV, MVT::v8f16, Expand);  in X86TargetLowering()
1112     setOperationAction(ISD::SELECT,             MVT::v2f64, Custom);  in X86TargetLowering()
1113     setOperationAction(ISD::SELECT,             MVT::v2i64, Custom);  in X86TargetLowering()
1114     setOperationAction(ISD::SELECT,             MVT::v4i32, Custom);  in X86TargetLowering()
1115     setOperationAction(ISD::SELECT,             MVT::v8i16, Custom);  in X86TargetLowering()
1116     setOperationAction(ISD::SELECT,             MVT::v8f16, Custom);  in X86TargetLowering()
1117     setOperationAction(ISD::SELECT,             MVT::v16i8, Custom);  in X86TargetLowering()
1119     setOperationAction(ISD::FP_TO_SINT,         MVT::v4i32, Custom);  in X86TargetLowering()
1120     setOperationAction(ISD::FP_TO_UINT,         MVT::v4i32, Custom);  in X86TargetLowering()
1121     setOperationAction(ISD::FP_TO_SINT,         MVT::v2i32, Custom);  in X86TargetLowering()
1122     setOperationAction(ISD::FP_TO_UINT,         MVT::v2i32, Custom);  in X86TargetLowering()
1123     setOperationAction(ISD::STRICT_FP_TO_SINT,  MVT::v4i32, Custom);  in X86TargetLowering()
1124     setOperationAction(ISD::STRICT_FP_TO_SINT,  MVT::v2i32, Custom);  in X86TargetLowering()
1127     for (auto VT : {MVT::v2i8, MVT::v4i8, MVT::v8i8, MVT::v2i16, MVT::v4i16}) {  in X86TargetLowering()
1134     setOperationAction(ISD::SINT_TO_FP,         MVT::v4i32, Custom);  in X86TargetLowering()
1135     setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v4i32, Custom);  in X86TargetLowering()
1136     setOperationAction(ISD::SINT_TO_FP,         MVT::v2i32, Custom);  in X86TargetLowering()
1137     setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v2i32, Custom);  in X86TargetLowering()
1139     setOperationAction(ISD::UINT_TO_FP,         MVT::v2i32, Custom);  in X86TargetLowering()
1140     setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v2i32, Custom);  in X86TargetLowering()
1142     setOperationAction(ISD::UINT_TO_FP,         MVT::v4i32, Custom);  in X86TargetLowering()
1143     setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v4i32, Custom);  in X86TargetLowering()
1146     setOperationAction(ISD::SINT_TO_FP,         MVT::v2f32, Custom);  in X86TargetLowering()
1147     setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v2f32, Custom);  in X86TargetLowering()
1148     setOperationAction(ISD::UINT_TO_FP,         MVT::v2f32, Custom);  in X86TargetLowering()
1149     setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v2f32, Custom);  in X86TargetLowering()
1151     setOperationAction(ISD::FP_EXTEND,          MVT::v2f32, Custom);  in X86TargetLowering()
1152     setOperationAction(ISD::STRICT_FP_EXTEND,   MVT::v2f32, Custom);  in X86TargetLowering()
1153     setOperationAction(ISD::FP_ROUND,           MVT::v2f32, Custom);  in X86TargetLowering()
1154     setOperationAction(ISD::STRICT_FP_ROUND,    MVT::v2f32, Custom);  in X86TargetLowering()
1159     setOperationAction(ISD::LOAD,               MVT::v2i32, Custom);  in X86TargetLowering()
1160     setOperationAction(ISD::LOAD,               MVT::v4i16, Custom);  in X86TargetLowering()
1161     setOperationAction(ISD::LOAD,               MVT::v8i8,  Custom);  in X86TargetLowering()
1162     setOperationAction(ISD::STORE,              MVT::v2i32, Custom);  in X86TargetLowering()
1163     setOperationAction(ISD::STORE,              MVT::v4i16, Custom);  in X86TargetLowering()
1164     setOperationAction(ISD::STORE,              MVT::v8i8,  Custom);  in X86TargetLowering()
1167     setOperationAction(ISD::STORE,              MVT::v2i16, Custom);  in X86TargetLowering()
1168     setOperationAction(ISD::STORE,              MVT::v4i8,  Custom);  in X86TargetLowering()
1170     setOperationAction(ISD::BITCAST,            MVT::v2i32, Custom);  in X86TargetLowering()
1171     setOperationAction(ISD::BITCAST,            MVT::v4i16, Custom);  in X86TargetLowering()
1172     setOperationAction(ISD::BITCAST,            MVT::v8i8,  Custom);  in X86TargetLowering()
1174       setOperationAction(ISD::BITCAST, MVT::v16i1, Custom);  in X86TargetLowering()
1176     setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v2i64, Custom);  in X86TargetLowering()
1177     setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i32, Custom);  in X86TargetLowering()
1178     setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v8i16, Custom);  in X86TargetLowering()
1180     setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom);  in X86TargetLowering()
1182     setOperationAction(ISD::TRUNCATE,    MVT::v2i8,  Custom);  in X86TargetLowering()
1183     setOperationAction(ISD::TRUNCATE,    MVT::v2i16, Custom);  in X86TargetLowering()
1184     setOperationAction(ISD::TRUNCATE,    MVT::v2i32, Custom);  in X86TargetLowering()
1185     setOperationAction(ISD::TRUNCATE,    MVT::v4i8,  Custom);  in X86TargetLowering()
1186     setOperationAction(ISD::TRUNCATE,    MVT::v4i16, Custom);  in X86TargetLowering()
1187     setOperationAction(ISD::TRUNCATE,    MVT::v8i8,  Custom);  in X86TargetLowering()
1191     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {  in X86TargetLowering()
1195       if (VT == MVT::v2i64) continue;  in X86TargetLowering()
1202     setOperationAction(ISD::STRICT_FSQRT,       MVT::v2f64, Legal);  in X86TargetLowering()
1203     setOperationAction(ISD::STRICT_FADD,        MVT::v2f64, Legal);  in X86TargetLowering()
1204     setOperationAction(ISD::STRICT_FSUB,        MVT::v2f64, Legal);  in X86TargetLowering()
1205     setOperationAction(ISD::STRICT_FMUL,        MVT::v2f64, Legal);  in X86TargetLowering()
1206     setOperationAction(ISD::STRICT_FDIV,        MVT::v2f64, Legal);  in X86TargetLowering()
1210     setOperationAction(ISD::ABS,                MVT::v16i8, Legal);  in X86TargetLowering()
1211     setOperationAction(ISD::ABS,                MVT::v8i16, Legal);  in X86TargetLowering()
1212     setOperationAction(ISD::ABS,                MVT::v4i32, Legal);  in X86TargetLowering()
1213     setOperationAction(ISD::BITREVERSE,         MVT::v16i8, Custom);  in X86TargetLowering()
1214     setOperationAction(ISD::CTLZ,               MVT::v16i8, Custom);  in X86TargetLowering()
1215     setOperationAction(ISD::CTLZ,               MVT::v8i16, Custom);  in X86TargetLowering()
1216     setOperationAction(ISD::CTLZ,               MVT::v4i32, Custom);  in X86TargetLowering()
1217     setOperationAction(ISD::CTLZ,               MVT::v2i64, Custom);  in X86TargetLowering()
1220     setOperationAction(ISD::ADD,                MVT::i16, Custom);  in X86TargetLowering()
1221     setOperationAction(ISD::ADD,                MVT::i32, Custom);  in X86TargetLowering()
1222     setOperationAction(ISD::SUB,                MVT::i16, Custom);  in X86TargetLowering()
1223     setOperationAction(ISD::SUB,                MVT::i32, Custom);  in X86TargetLowering()
1227     for (MVT RoundedTy : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {  in X86TargetLowering()
1244     setOperationAction(ISD::SMAX,               MVT::v16i8, Legal);  in X86TargetLowering()
1245     setOperationAction(ISD::SMAX,               MVT::v4i32, Legal);  in X86TargetLowering()
1246     setOperationAction(ISD::UMAX,               MVT::v8i16, Legal);  in X86TargetLowering()
1247     setOperationAction(ISD::UMAX,               MVT::v4i32, Legal);  in X86TargetLowering()
1248     setOperationAction(ISD::SMIN,               MVT::v16i8, Legal);  in X86TargetLowering()
1249     setOperationAction(ISD::SMIN,               MVT::v4i32, Legal);  in X86TargetLowering()
1250     setOperationAction(ISD::UMIN,               MVT::v8i16, Legal);  in X86TargetLowering()
1251     setOperationAction(ISD::UMIN,               MVT::v4i32, Legal);  in X86TargetLowering()
1253     setOperationAction(ISD::UADDSAT,            MVT::v4i32, Custom);  in X86TargetLowering()
1254     setOperationAction(ISD::SADDSAT,            MVT::v2i64, Custom);  in X86TargetLowering()
1255     setOperationAction(ISD::SSUBSAT,            MVT::v2i64, Custom);  in X86TargetLowering()
1258     setOperationAction(ISD::MUL,                MVT::v4i32, Legal);  in X86TargetLowering()
1259     setOperationAction(ISD::SMULO,              MVT::v2i32, Custom);  in X86TargetLowering()
1263     setOperationAction(ISD::VSELECT,            MVT::v16i8, Legal);  in X86TargetLowering()
1267     for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {  in X86TargetLowering()
1274       setLoadExtAction(LoadExtOp, MVT::v8i16, MVT::v8i8,  Legal);  in X86TargetLowering()
1275       setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i8,  Legal);  in X86TargetLowering()
1276       setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i8,  Legal);  in X86TargetLowering()
1277       setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i16, Legal);  in X86TargetLowering()
1278       setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i16, Legal);  in X86TargetLowering()
1279       setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i32, Legal);  in X86TargetLowering()
1285       setOperationAction(ISD::UINT_TO_FP,        MVT::v4i64, Custom);  in X86TargetLowering()
1286       setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i64, Custom);  in X86TargetLowering()
1289       setOperationAction(ISD::SINT_TO_FP,        MVT::v4i64, Custom);  in X86TargetLowering()
1290       setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i64, Custom);  in X86TargetLowering()
1295     setOperationAction(ISD::UADDSAT,            MVT::v2i64, Custom);  in X86TargetLowering()
1299     for (auto VT : { MVT::v16i8, MVT::v8i16,  MVT::v4i32, MVT::v2i64,  in X86TargetLowering()
1300                      MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {  in X86TargetLowering()
1306     for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 })  in X86TargetLowering()
1309     for (auto VT : { MVT::v16i8, MVT::v8i16,  MVT::v4i32, MVT::v2i64,  in X86TargetLowering()
1310                      MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 })  in X86TargetLowering()
1317     addRegisterClass(MVT::v32i8,  Subtarget.hasVLX() ? &X86::VR256XRegClass  in X86TargetLowering()
1319     addRegisterClass(MVT::v16i16, Subtarget.hasVLX() ? &X86::VR256XRegClass  in X86TargetLowering()
1321     addRegisterClass(MVT::v16f16, Subtarget.hasVLX() ? &X86::VR256XRegClass  in X86TargetLowering()
1323     addRegisterClass(MVT::v8i32,  Subtarget.hasVLX() ? &X86::VR256XRegClass  in X86TargetLowering()
1325     addRegisterClass(MVT::v8f32,  Subtarget.hasVLX() ? &X86::VR256XRegClass  in X86TargetLowering()
1327     addRegisterClass(MVT::v4i64,  Subtarget.hasVLX() ? &X86::VR256XRegClass  in X86TargetLowering()
1329     addRegisterClass(MVT::v4f64,  Subtarget.hasVLX() ? &X86::VR256XRegClass  in X86TargetLowering()
1332     for (auto VT : { MVT::v8f32, MVT::v4f64 }) {  in X86TargetLowering()
1355     setOperationPromotedToType(ISD::FP_TO_SINT,        MVT::v8i16, MVT::v8i32);  in X86TargetLowering()
1356     setOperationPromotedToType(ISD::FP_TO_UINT,        MVT::v8i16, MVT::v8i32);  in X86TargetLowering()
1357     setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i16, MVT::v8i32);  in X86TargetLowering()
1358     setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i16, MVT::v8i32);  in X86TargetLowering()
1359     setOperationAction(ISD::FP_TO_SINT,                MVT::v8i32, Custom);  in X86TargetLowering()
1360     setOperationAction(ISD::FP_TO_UINT,                MVT::v8i32, Custom);  in X86TargetLowering()
1361     setOperationAction(ISD::STRICT_FP_TO_SINT,         MVT::v8i32, Custom);  in X86TargetLowering()
1363     setOperationAction(ISD::SINT_TO_FP,         MVT::v8i32, Custom);  in X86TargetLowering()
1364     setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v8i32, Custom);  in X86TargetLowering()
1365     setOperationAction(ISD::FP_EXTEND,          MVT::v8f32, Expand);  in X86TargetLowering()
1366     setOperationAction(ISD::FP_ROUND,           MVT::v8f16, Expand);  in X86TargetLowering()
1367     setOperationAction(ISD::FP_EXTEND,          MVT::v4f64, Custom);  in X86TargetLowering()
1368     setOperationAction(ISD::STRICT_FP_EXTEND,   MVT::v4f64, Custom);  in X86TargetLowering()
1370     setOperationAction(ISD::STRICT_FP_ROUND,    MVT::v4f32, Legal);  in X86TargetLowering()
1371     setOperationAction(ISD::STRICT_FADD,        MVT::v8f32, Legal);  in X86TargetLowering()
1372     setOperationAction(ISD::STRICT_FADD,        MVT::v4f64, Legal);  in X86TargetLowering()
1373     setOperationAction(ISD::STRICT_FSUB,        MVT::v8f32, Legal);  in X86TargetLowering()
1374     setOperationAction(ISD::STRICT_FSUB,        MVT::v4f64, Legal);  in X86TargetLowering()
1375     setOperationAction(ISD::STRICT_FMUL,        MVT::v8f32, Legal);  in X86TargetLowering()
1376     setOperationAction(ISD::STRICT_FMUL,        MVT::v4f64, Legal);  in X86TargetLowering()
1377     setOperationAction(ISD::STRICT_FDIV,        MVT::v8f32, Legal);  in X86TargetLowering()
1378     setOperationAction(ISD::STRICT_FDIV,        MVT::v4f64, Legal);  in X86TargetLowering()
1379     setOperationAction(ISD::STRICT_FSQRT,       MVT::v8f32, Legal);  in X86TargetLowering()
1380     setOperationAction(ISD::STRICT_FSQRT,       MVT::v4f64, Legal);  in X86TargetLowering()
1383       setOperationAction(ISD::BITCAST, MVT::v32i1, Custom);  in X86TargetLowering()
1387     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {  in X86TargetLowering()
1391       if (VT == MVT::v4i64) continue;  in X86TargetLowering()
1399     setOperationAction(ISD::SIGN_EXTEND,       MVT::v8i64,  Custom);  in X86TargetLowering()
1400     setOperationAction(ISD::SIGN_EXTEND,       MVT::v16i32, Custom);  in X86TargetLowering()
1401     setOperationAction(ISD::ZERO_EXTEND,       MVT::v8i64,  Custom);  in X86TargetLowering()
1402     setOperationAction(ISD::ZERO_EXTEND,       MVT::v16i32, Custom);  in X86TargetLowering()
1404     setOperationAction(ISD::SELECT,            MVT::v4f64, Custom);  in X86TargetLowering()
1405     setOperationAction(ISD::SELECT,            MVT::v4i64, Custom);  in X86TargetLowering()
1406     setOperationAction(ISD::SELECT,            MVT::v8i32, Custom);  in X86TargetLowering()
1407     setOperationAction(ISD::SELECT,            MVT::v16i16, Custom);  in X86TargetLowering()
1408     setOperationAction(ISD::SELECT,            MVT::v16f16, Custom);  in X86TargetLowering()
1409     setOperationAction(ISD::SELECT,            MVT::v32i8, Custom);  in X86TargetLowering()
1410     setOperationAction(ISD::SELECT,            MVT::v8f32, Custom);  in X86TargetLowering()
1412     for (auto VT : { MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {  in X86TargetLowering()
1418     setOperationAction(ISD::TRUNCATE,          MVT::v16i8, Custom);  in X86TargetLowering()
1419     setOperationAction(ISD::TRUNCATE,          MVT::v8i16, Custom);  in X86TargetLowering()
1420     setOperationAction(ISD::TRUNCATE,          MVT::v4i32, Custom);  in X86TargetLowering()
1421     setOperationAction(ISD::BITREVERSE,        MVT::v32i8, Custom);  in X86TargetLowering()
1423     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {  in X86TargetLowering()
1437       for (auto VT : { MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32,  in X86TargetLowering()
1438                        MVT::v2f64, MVT::v4f64 }) {  in X86TargetLowering()
1444     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {  in X86TargetLowering()
1449     setOperationAction(ISD::MUL,       MVT::v4i64,  Custom);  in X86TargetLowering()
1450     setOperationAction(ISD::MUL,       MVT::v8i32,  HasInt256 ? Legal : Custom);  in X86TargetLowering()
1451     setOperationAction(ISD::MUL,       MVT::v16i16, HasInt256 ? Legal : Custom);  in X86TargetLowering()
1452     setOperationAction(ISD::MUL,       MVT::v32i8,  Custom);  in X86TargetLowering()
1454     setOperationAction(ISD::MULHU,     MVT::v8i32,  Custom);  in X86TargetLowering()
1455     setOperationAction(ISD::MULHS,     MVT::v8i32,  Custom);  in X86TargetLowering()
1456     setOperationAction(ISD::MULHU,     MVT::v16i16, HasInt256 ? Legal : Custom);  in X86TargetLowering()
1457     setOperationAction(ISD::MULHS,     MVT::v16i16, HasInt256 ? Legal : Custom);  in X86TargetLowering()
1458     setOperationAction(ISD::MULHU,     MVT::v32i8,  Custom);  in X86TargetLowering()
1459     setOperationAction(ISD::MULHS,     MVT::v32i8,  Custom);  in X86TargetLowering()
1460     setOperationAction(ISD::AVGCEILU,  MVT::v16i16, HasInt256 ? Legal : Custom);  in X86TargetLowering()
1461     setOperationAction(ISD::AVGCEILU,  MVT::v32i8,  HasInt256 ? Legal : Custom);  in X86TargetLowering()
1463     setOperationAction(ISD::SMULO,     MVT::v32i8, Custom);  in X86TargetLowering()
1464     setOperationAction(ISD::UMULO,     MVT::v32i8, Custom);  in X86TargetLowering()
1466     setOperationAction(ISD::ABS,       MVT::v4i64,  Custom);  in X86TargetLowering()
1467     setOperationAction(ISD::SMAX,      MVT::v4i64,  Custom);  in X86TargetLowering()
1468     setOperationAction(ISD::UMAX,      MVT::v4i64,  Custom);  in X86TargetLowering()
1469     setOperationAction(ISD::SMIN,      MVT::v4i64,  Custom);  in X86TargetLowering()
1470     setOperationAction(ISD::UMIN,      MVT::v4i64,  Custom);  in X86TargetLowering()
1472     setOperationAction(ISD::UADDSAT,   MVT::v32i8,  HasInt256 ? Legal : Custom);  in X86TargetLowering()
1473     setOperationAction(ISD::SADDSAT,   MVT::v32i8,  HasInt256 ? Legal : Custom);  in X86TargetLowering()
1474     setOperationAction(ISD::USUBSAT,   MVT::v32i8,  HasInt256 ? Legal : Custom);  in X86TargetLowering()
1475     setOperationAction(ISD::SSUBSAT,   MVT::v32i8,  HasInt256 ? Legal : Custom);  in X86TargetLowering()
1476     setOperationAction(ISD::UADDSAT,   MVT::v16i16, HasInt256 ? Legal : Custom);  in X86TargetLowering()
1477     setOperationAction(ISD::SADDSAT,   MVT::v16i16, HasInt256 ? Legal : Custom);  in X86TargetLowering()
1478     setOperationAction(ISD::USUBSAT,   MVT::v16i16, HasInt256 ? Legal : Custom);  in X86TargetLowering()
1479     setOperationAction(ISD::SSUBSAT,   MVT::v16i16, HasInt256 ? Legal : Custom);  in X86TargetLowering()
1480     setOperationAction(ISD::UADDSAT,   MVT::v8i32, Custom);  in X86TargetLowering()
1481     setOperationAction(ISD::USUBSAT,   MVT::v8i32, Custom);  in X86TargetLowering()
1482     setOperationAction(ISD::UADDSAT,   MVT::v4i64, Custom);  in X86TargetLowering()
1483     setOperationAction(ISD::USUBSAT,   MVT::v4i64, Custom);  in X86TargetLowering()
1485     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32 }) {  in X86TargetLowering()
1493     for (auto VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64}) {  in X86TargetLowering()
1501       setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Custom);  in X86TargetLowering()
1502       setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i32, Custom);  in X86TargetLowering()
1506         setLoadExtAction(LoadExtOp, MVT::v16i16, MVT::v16i8, Legal);  in X86TargetLowering()
1507         setLoadExtAction(LoadExtOp, MVT::v8i32,  MVT::v8i8,  Legal);  in X86TargetLowering()
1508         setLoadExtAction(LoadExtOp, MVT::v4i64,  MVT::v4i8,  Legal);  in X86TargetLowering()
1509         setLoadExtAction(LoadExtOp, MVT::v8i32,  MVT::v8i16, Legal);  in X86TargetLowering()
1510         setLoadExtAction(LoadExtOp, MVT::v4i64,  MVT::v4i16, Legal);  in X86TargetLowering()
1511         setLoadExtAction(LoadExtOp, MVT::v4i64,  MVT::v4i32, Legal);  in X86TargetLowering()
1515     for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,  in X86TargetLowering()
1516                      MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 }) {  in X86TargetLowering()
1523     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,  in X86TargetLowering()
1524                      MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {  in X86TargetLowering()
1529     for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,  in X86TargetLowering()
1530                     MVT::v16f16, MVT::v8f32, MVT::v4f64 }) {  in X86TargetLowering()
1541     setF16Action(MVT::v16f16, Expand);  in X86TargetLowering()
1542     setOperationAction(ISD::FADD, MVT::v16f16, Expand);  in X86TargetLowering()
1543     setOperationAction(ISD::FSUB, MVT::v16f16, Expand);  in X86TargetLowering()
1544     setOperationAction(ISD::FMUL, MVT::v16f16, Expand);  in X86TargetLowering()
1545     setOperationAction(ISD::FDIV, MVT::v16f16, Expand);  in X86TargetLowering()
1548       setOperationAction(ISD::VSELECT, MVT::v32i8, Legal);  in X86TargetLowering()
1551       setOperationAction(ISD::MGATHER, MVT::v2f32, Custom);  in X86TargetLowering()
1552       setOperationAction(ISD::MGATHER, MVT::v2i32, Custom);  in X86TargetLowering()
1554       for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,  in X86TargetLowering()
1555                        MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })  in X86TargetLowering()
1562     for (MVT VT : { MVT::f16, MVT::v2f16, MVT::v4f16, MVT::v8f16 }) {  in X86TargetLowering()
1566     for (MVT VT : { MVT::f32, MVT::v2f32, MVT::v4f32 }) {  in X86TargetLowering()
1571       setOperationPromotedToType(Opc, MVT::v8f16, MVT::v8f32);  in X86TargetLowering()
1572       setOperationPromotedToType(Opc, MVT::v16f16, MVT::v16f32);  in X86TargetLowering()
1575     setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Legal);  in X86TargetLowering()
1576     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f32, Legal);  in X86TargetLowering()
1583     addRegisterClass(MVT::v1i1,   &X86::VK1RegClass);  in X86TargetLowering()
1584     addRegisterClass(MVT::v2i1,   &X86::VK2RegClass);  in X86TargetLowering()
1585     addRegisterClass(MVT::v4i1,   &X86::VK4RegClass);  in X86TargetLowering()
1586     addRegisterClass(MVT::v8i1,   &X86::VK8RegClass);  in X86TargetLowering()
1587     addRegisterClass(MVT::v16i1,  &X86::VK16RegClass);  in X86TargetLowering()
1589     setOperationAction(ISD::SELECT,             MVT::v1i1, Custom);  in X86TargetLowering()
1590     setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v1i1, Custom);  in X86TargetLowering()
1591     setOperationAction(ISD::BUILD_VECTOR,       MVT::v1i1, Custom);  in X86TargetLowering()
1593     setOperationPromotedToType(ISD::FP_TO_SINT,        MVT::v8i1,  MVT::v8i32);  in X86TargetLowering()
1594     setOperationPromotedToType(ISD::FP_TO_UINT,        MVT::v8i1,  MVT::v8i32);  in X86TargetLowering()
1595     setOperationPromotedToType(ISD::FP_TO_SINT,        MVT::v4i1,  MVT::v4i32);  in X86TargetLowering()
1596     setOperationPromotedToType(ISD::FP_TO_UINT,        MVT::v4i1,  MVT::v4i32);  in X86TargetLowering()
1597     setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i1,  MVT::v8i32);  in X86TargetLowering()
1598     setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i1,  MVT::v8i32);  in X86TargetLowering()
1599     setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v4i1,  MVT::v4i32);  in X86TargetLowering()
1600     setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v4i1,  MVT::v4i32);  in X86TargetLowering()
1601     setOperationAction(ISD::FP_TO_SINT,                MVT::v2i1,  Custom);  in X86TargetLowering()
1602     setOperationAction(ISD::FP_TO_UINT,                MVT::v2i1,  Custom);  in X86TargetLowering()
1603     setOperationAction(ISD::STRICT_FP_TO_SINT,         MVT::v2i1,  Custom);  in X86TargetLowering()
1604     setOperationAction(ISD::STRICT_FP_TO_UINT,         MVT::v2i1,  Custom);  in X86TargetLowering()
1608       setOperationAction(ISD::LOAD, MVT::v1i1, Custom);  in X86TargetLowering()
1609       setOperationAction(ISD::LOAD, MVT::v2i1, Custom);  in X86TargetLowering()
1610       setOperationAction(ISD::LOAD, MVT::v4i1, Custom);  in X86TargetLowering()
1611       setOperationAction(ISD::LOAD, MVT::v8i1, Custom);  in X86TargetLowering()
1613       setOperationAction(ISD::STORE, MVT::v1i1, Custom);  in X86TargetLowering()
1614       setOperationAction(ISD::STORE, MVT::v2i1, Custom);  in X86TargetLowering()
1615       setOperationAction(ISD::STORE, MVT::v4i1, Custom);  in X86TargetLowering()
1616       setOperationAction(ISD::STORE, MVT::v8i1, Custom);  in X86TargetLowering()
1620     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {  in X86TargetLowering()
1626     for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 })  in X86TargetLowering()
1629     for (auto VT : { MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 }) {  in X86TargetLowering()
1644     for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1 })  in X86TargetLowering()
1654     addRegisterClass(MVT::v16i32, &X86::VR512RegClass);  in X86TargetLowering()
1655     addRegisterClass(MVT::v16f32, &X86::VR512RegClass);  in X86TargetLowering()
1656     addRegisterClass(MVT::v8i64,  &X86::VR512RegClass);  in X86TargetLowering()
1657     addRegisterClass(MVT::v8f64,  &X86::VR512RegClass);  in X86TargetLowering()
1658     addRegisterClass(MVT::v32i16, &X86::VR512RegClass);  in X86TargetLowering()
1659     addRegisterClass(MVT::v32f16, &X86::VR512RegClass);  in X86TargetLowering()
1660     addRegisterClass(MVT::v64i8,  &X86::VR512RegClass);  in X86TargetLowering()
1663       setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i8,  Legal);  in X86TargetLowering()
1664       setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i16, Legal);  in X86TargetLowering()
1665       setLoadExtAction(ExtType, MVT::v8i64,  MVT::v8i8,   Legal);  in X86TargetLowering()
1666       setLoadExtAction(ExtType, MVT::v8i64,  MVT::v8i16,  Legal);  in X86TargetLowering()
1667       setLoadExtAction(ExtType, MVT::v8i64,  MVT::v8i32,  Legal);  in X86TargetLowering()
1669         setLoadExtAction(ExtType, MVT::v32i16, MVT::v32i8, Legal);  in X86TargetLowering()
1672     for (MVT VT : { MVT::v16f32, MVT::v8f64 }) {  in X86TargetLowering()
1680     for (MVT VT : { MVT::v16i1, MVT::v16i8, MVT::v16i16 }) {  in X86TargetLowering()
1681       setOperationPromotedToType(ISD::FP_TO_SINT       , VT, MVT::v16i32);  in X86TargetLowering()
1682       setOperationPromotedToType(ISD::FP_TO_UINT       , VT, MVT::v16i32);  in X86TargetLowering()
1683       setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, VT, MVT::v16i32);  in X86TargetLowering()
1684       setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, VT, MVT::v16i32);  in X86TargetLowering()
1686     setOperationAction(ISD::FP_TO_SINT,        MVT::v16i32, Custom);  in X86TargetLowering()
1687     setOperationAction(ISD::FP_TO_UINT,        MVT::v16i32, Custom);  in X86TargetLowering()
1688     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v16i32, Custom);  in X86TargetLowering()
1689     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v16i32, Custom);  in X86TargetLowering()
1690     setOperationAction(ISD::SINT_TO_FP,        MVT::v16i32, Custom);  in X86TargetLowering()
1691     setOperationAction(ISD::UINT_TO_FP,        MVT::v16i32, Custom);  in X86TargetLowering()
1692     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v16i32, Custom);  in X86TargetLowering()
1693     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v16i32, Custom);  in X86TargetLowering()
1694     setOperationAction(ISD::FP_EXTEND,         MVT::v8f64,  Custom);  in X86TargetLowering()
1695     setOperationAction(ISD::STRICT_FP_EXTEND,  MVT::v8f64,  Custom);  in X86TargetLowering()
1697     setOperationAction(ISD::STRICT_FADD,      MVT::v16f32, Legal);  in X86TargetLowering()
1698     setOperationAction(ISD::STRICT_FADD,      MVT::v8f64,  Legal);  in X86TargetLowering()
1699     setOperationAction(ISD::STRICT_FSUB,      MVT::v16f32, Legal);  in X86TargetLowering()
1700     setOperationAction(ISD::STRICT_FSUB,      MVT::v8f64,  Legal);  in X86TargetLowering()
1701     setOperationAction(ISD::STRICT_FMUL,      MVT::v16f32, Legal);  in X86TargetLowering()
1702     setOperationAction(ISD::STRICT_FMUL,      MVT::v8f64,  Legal);  in X86TargetLowering()
1703     setOperationAction(ISD::STRICT_FDIV,      MVT::v16f32, Legal);  in X86TargetLowering()
1704     setOperationAction(ISD::STRICT_FDIV,      MVT::v8f64,  Legal);  in X86TargetLowering()
1705     setOperationAction(ISD::STRICT_FSQRT,     MVT::v16f32, Legal);  in X86TargetLowering()
1706     setOperationAction(ISD::STRICT_FSQRT,     MVT::v8f64,  Legal);  in X86TargetLowering()
1707     setOperationAction(ISD::STRICT_FP_ROUND,  MVT::v8f32,  Legal);  in X86TargetLowering()
1709     setTruncStoreAction(MVT::v8i64,   MVT::v8i8,   Legal);  in X86TargetLowering()
1710     setTruncStoreAction(MVT::v8i64,   MVT::v8i16,  Legal);  in X86TargetLowering()
1711     setTruncStoreAction(MVT::v8i64,   MVT::v8i32,  Legal);  in X86TargetLowering()
1712     setTruncStoreAction(MVT::v16i32,  MVT::v16i8,  Legal);  in X86TargetLowering()
1713     setTruncStoreAction(MVT::v16i32,  MVT::v16i16, Legal);  in X86TargetLowering()
1715       setTruncStoreAction(MVT::v32i16,  MVT::v32i8, Legal);  in X86TargetLowering()
1721       for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,  in X86TargetLowering()
1722            MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64}) {  in X86TargetLowering()
1728     setOperationAction(ISD::TRUNCATE,    MVT::v8i32,  Legal);  in X86TargetLowering()
1729     setOperationAction(ISD::TRUNCATE,    MVT::v16i16, Legal);  in X86TargetLowering()
1730     setOperationAction(ISD::TRUNCATE,    MVT::v32i8,  HasBWI ? Legal : Custom);  in X86TargetLowering()
1731     setOperationAction(ISD::TRUNCATE,    MVT::v16i64, Custom);  in X86TargetLowering()
1732     setOperationAction(ISD::ZERO_EXTEND, MVT::v32i16, Custom);  in X86TargetLowering()
1733     setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);  in X86TargetLowering()
1734     setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64,  Custom);  in X86TargetLowering()
1735     setOperationAction(ISD::ANY_EXTEND,  MVT::v32i16, Custom);  in X86TargetLowering()
1736     setOperationAction(ISD::ANY_EXTEND,  MVT::v16i32, Custom);  in X86TargetLowering()
1737     setOperationAction(ISD::ANY_EXTEND,  MVT::v8i64,  Custom);  in X86TargetLowering()
1738     setOperationAction(ISD::SIGN_EXTEND, MVT::v32i16, Custom);  in X86TargetLowering()
1739     setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);  in X86TargetLowering()
1740     setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64,  Custom);  in X86TargetLowering()
1744       setOperationAction(ISD::SIGN_EXTEND,        MVT::v64i8, Custom);  in X86TargetLowering()
1745       setOperationAction(ISD::ZERO_EXTEND,        MVT::v64i8, Custom);  in X86TargetLowering()
1746       setOperationAction(ISD::ANY_EXTEND,         MVT::v64i8, Custom);  in X86TargetLowering()
1749     for (auto VT : { MVT::v16f32, MVT::v8f64 }) {  in X86TargetLowering()
1766     for (auto VT : {MVT::v32i16, MVT::v16i32, MVT::v8i64}) {  in X86TargetLowering()
1771     setOperationAction(ISD::ADD, MVT::v32i16, HasBWI ? Legal : Custom);  in X86TargetLowering()
1772     setOperationAction(ISD::SUB, MVT::v32i16, HasBWI ? Legal : Custom);  in X86TargetLowering()
1773     setOperationAction(ISD::ADD, MVT::v64i8,  HasBWI ? Legal : Custom);  in X86TargetLowering()
1774     setOperationAction(ISD::SUB, MVT::v64i8,  HasBWI ? Legal : Custom);  in X86TargetLowering()
1776     setOperationAction(ISD::MUL, MVT::v8i64,  Custom);  in X86TargetLowering()
1777     setOperationAction(ISD::MUL, MVT::v16i32, Legal);  in X86TargetLowering()
1778     setOperationAction(ISD::MUL, MVT::v32i16, HasBWI ? Legal : Custom);  in X86TargetLowering()
1779     setOperationAction(ISD::MUL, MVT::v64i8,  Custom);  in X86TargetLowering()
1781     setOperationAction(ISD::MULHU, MVT::v16i32, Custom);  in X86TargetLowering()
1782     setOperationAction(ISD::MULHS, MVT::v16i32, Custom);  in X86TargetLowering()
1783     setOperationAction(ISD::MULHS, MVT::v32i16, HasBWI ? Legal : Custom);  in X86TargetLowering()
1784     setOperationAction(ISD::MULHU, MVT::v32i16, HasBWI ? Legal : Custom);  in X86TargetLowering()
1785     setOperationAction(ISD::MULHS, MVT::v64i8,  Custom);  in X86TargetLowering()
1786     setOperationAction(ISD::MULHU, MVT::v64i8,  Custom);  in X86TargetLowering()
1787     setOperationAction(ISD::AVGCEILU, MVT::v32i16, HasBWI ? Legal : Custom);  in X86TargetLowering()
1788     setOperationAction(ISD::AVGCEILU, MVT::v64i8,  HasBWI ? Legal : Custom);  in X86TargetLowering()
1790     setOperationAction(ISD::SMULO, MVT::v64i8, Custom);  in X86TargetLowering()
1791     setOperationAction(ISD::UMULO, MVT::v64i8, Custom);  in X86TargetLowering()
1793     setOperationAction(ISD::BITREVERSE, MVT::v64i8,  Custom);  in X86TargetLowering()
1795     for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64 }) {  in X86TargetLowering()
1808     for (auto VT : { MVT::v16i32, MVT::v8i64 }) {  in X86TargetLowering()
1819     for (auto VT : { MVT::v64i8, MVT::v32i16 }) {  in X86TargetLowering()
1833     setOperationAction(ISD::FSHL,       MVT::v64i8, Custom);  in X86TargetLowering()
1834     setOperationAction(ISD::FSHR,       MVT::v64i8, Custom);  in X86TargetLowering()
1835     setOperationAction(ISD::FSHL,      MVT::v32i16, Custom);  in X86TargetLowering()
1836     setOperationAction(ISD::FSHR,      MVT::v32i16, Custom);  in X86TargetLowering()
1837     setOperationAction(ISD::FSHL,      MVT::v16i32, Custom);  in X86TargetLowering()
1838     setOperationAction(ISD::FSHR,      MVT::v16i32, Custom);  in X86TargetLowering()
1844         setOperationAction(Opc,           MVT::v8i64, Custom);  in X86TargetLowering()
1845       setOperationAction(ISD::MUL,        MVT::v8i64, Legal);  in X86TargetLowering()
1850       for (auto VT : { MVT::v16i32, MVT::v8i64} ) {  in X86TargetLowering()
1856       for (auto VT : { MVT::v16i32, MVT::v8i64 })  in X86TargetLowering()
1863     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,  in X86TargetLowering()
1864                      MVT::v16f16, MVT::v8f32, MVT::v4f64 })  in X86TargetLowering()
1867     for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64,  in X86TargetLowering()
1868                      MVT::v32f16, MVT::v16f32, MVT::v8f64 }) {  in X86TargetLowering()
1879     setF16Action(MVT::v32f16, Expand);  in X86TargetLowering()
1880     setOperationAction(ISD::FP_ROUND, MVT::v16f16, Custom);  in X86TargetLowering()
1881     setOperationAction(ISD::STRICT_FP_ROUND, MVT::v16f16, Custom);  in X86TargetLowering()
1882     setOperationAction(ISD::FP_EXTEND, MVT::v16f32, Legal);  in X86TargetLowering()
1883     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v16f32, Legal);  in X86TargetLowering()
1885       setOperationPromotedToType(Opc, MVT::v16f16, MVT::v16f32);  in X86TargetLowering()
1886       setOperationPromotedToType(Opc, MVT::v32f16, MVT::v32f32);  in X86TargetLowering()
1889     for (auto VT : { MVT::v16i32, MVT::v8i64, MVT::v16f32, MVT::v8f64 }) {  in X86TargetLowering()
1896       for (auto VT : { MVT::v64i8, MVT::v32i16 }) {  in X86TargetLowering()
1901       setOperationAction(ISD::STORE, MVT::v32i16, Custom);  in X86TargetLowering()
1902       setOperationAction(ISD::STORE, MVT::v64i8,  Custom);  in X86TargetLowering()
1906       for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64,  in X86TargetLowering()
1907                        MVT::v16i16, MVT::v8i32, MVT::v4i64,  in X86TargetLowering()
1908                        MVT::v32i16, MVT::v16i32, MVT::v8i64 }) {  in X86TargetLowering()
1913       setOperationAction(ISD::ROTL, MVT::v32i16, Custom);  in X86TargetLowering()
1914       setOperationAction(ISD::ROTR, MVT::v8i16,  Custom);  in X86TargetLowering()
1915       setOperationAction(ISD::ROTR, MVT::v16i16, Custom);  in X86TargetLowering()
1916       setOperationAction(ISD::ROTR, MVT::v32i16, Custom);  in X86TargetLowering()
1927     setOperationAction(ISD::STRICT_FP_TO_UINT,  MVT::v8i32, Custom);  in X86TargetLowering()
1928     setOperationAction(ISD::STRICT_FP_TO_UINT,  MVT::v4i32, Custom);  in X86TargetLowering()
1929     setOperationAction(ISD::STRICT_FP_TO_UINT,  MVT::v2i32, Custom);  in X86TargetLowering()
1934       assert(isOperationCustom(ISD::UINT_TO_FP, MVT::v2f32) &&  in X86TargetLowering()
1935              isOperationCustom(ISD::STRICT_UINT_TO_FP, MVT::v2f32) &&  in X86TargetLowering()
1938       setOperationAction(ISD::FP_TO_SINT,        MVT::v2f32, Custom);  in X86TargetLowering()
1939       setOperationAction(ISD::FP_TO_UINT,        MVT::v2f32, Custom);  in X86TargetLowering()
1940       setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f32, Custom);  in X86TargetLowering()
1941       setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f32, Custom);  in X86TargetLowering()
1944     for (auto VT : { MVT::v2i64, MVT::v4i64 }) {  in X86TargetLowering()
1952     for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {  in X86TargetLowering()
1958     setOperationAction(ISD::MSCATTER, MVT::v2f32, Custom);  in X86TargetLowering()
1959     setOperationAction(ISD::MSCATTER, MVT::v2i32, Custom);  in X86TargetLowering()
1961     for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,  in X86TargetLowering()
1962                      MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })  in X86TargetLowering()
1969         setOperationAction(Opc, MVT::v2i64, Custom);  in X86TargetLowering()
1970         setOperationAction(Opc, MVT::v4i64, Custom);  in X86TargetLowering()
1972       setOperationAction(ISD::MUL, MVT::v2i64, Legal);  in X86TargetLowering()
1973       setOperationAction(ISD::MUL, MVT::v4i64, Legal);  in X86TargetLowering()
1977       for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {  in X86TargetLowering()
1983       for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 })  in X86TargetLowering()
1992     addRegisterClass(MVT::v32i1,  &X86::VK32RegClass);  in X86TargetLowering()
1993     addRegisterClass(MVT::v64i1,  &X86::VK64RegClass);  in X86TargetLowering()
1995     for (auto VT : { MVT::v32i1, MVT::v64i1 }) {  in X86TargetLowering()
2008     for (auto VT : { MVT::v16i1, MVT::v32i1 })  in X86TargetLowering()
2012     setOperationAction(ISD::SIGN_EXTEND,        MVT::v32i8, Custom);  in X86TargetLowering()
2013     setOperationAction(ISD::ZERO_EXTEND,        MVT::v32i8, Custom);  in X86TargetLowering()
2014     setOperationAction(ISD::ANY_EXTEND,         MVT::v32i8, Custom);  in X86TargetLowering()
2016     for (auto VT : { MVT::v32i8, MVT::v16i8, MVT::v16i16, MVT::v8i16 }) {  in X86TargetLowering()
2026       for (auto VT : { MVT::v16i8, MVT::v32i8, MVT::v8i16, MVT::v16i16 })  in X86TargetLowering()
2032     auto setGroup = [&] (MVT VT) {  in X86TargetLowering()
2072     setGroup(MVT::f16);  in X86TargetLowering()
2073     setOperationAction(ISD::FREM,                 MVT::f16, Promote);  in X86TargetLowering()
2074     setOperationAction(ISD::STRICT_FREM,          MVT::f16, Promote);  in X86TargetLowering()
2075     setOperationAction(ISD::SELECT_CC,            MVT::f16, Expand);  in X86TargetLowering()
2076     setOperationAction(ISD::BR_CC,                MVT::f16, Expand);  in X86TargetLowering()
2077     setOperationAction(ISD::SETCC,                MVT::f16, Custom);  in X86TargetLowering()
2078     setOperationAction(ISD::STRICT_FSETCC,        MVT::f16, Custom);  in X86TargetLowering()
2079     setOperationAction(ISD::STRICT_FSETCCS,       MVT::f16, Custom);  in X86TargetLowering()
2080     setOperationAction(ISD::FROUND,               MVT::f16, Custom);  in X86TargetLowering()
2081     setOperationAction(ISD::STRICT_FROUND,        MVT::f16, Promote);  in X86TargetLowering()
2082     setOperationAction(ISD::FROUNDEVEN,           MVT::f16, Legal);  in X86TargetLowering()
2083     setOperationAction(ISD::STRICT_FROUNDEVEN,    MVT::f16, Legal);  in X86TargetLowering()
2084     setOperationAction(ISD::FP_ROUND,             MVT::f16, Custom);  in X86TargetLowering()
2085     setOperationAction(ISD::STRICT_FP_ROUND,      MVT::f16, Custom);  in X86TargetLowering()
2086     setOperationAction(ISD::FP_EXTEND,            MVT::f32, Legal);  in X86TargetLowering()
2087     setOperationAction(ISD::STRICT_FP_EXTEND,     MVT::f32, Legal);  in X86TargetLowering()
2089     setCondCodeAction(ISD::SETOEQ, MVT::f16, Expand);  in X86TargetLowering()
2090     setCondCodeAction(ISD::SETUNE, MVT::f16, Expand);  in X86TargetLowering()
2093       setGroup(MVT::v32f16);  in X86TargetLowering()
2094       setOperationAction(ISD::SCALAR_TO_VECTOR,       MVT::v32f16, Custom);  in X86TargetLowering()
2095       setOperationAction(ISD::SINT_TO_FP,             MVT::v32i16, Legal);  in X86TargetLowering()
2096       setOperationAction(ISD::STRICT_SINT_TO_FP,      MVT::v32i16, Legal);  in X86TargetLowering()
2097       setOperationAction(ISD::UINT_TO_FP,             MVT::v32i16, Legal);  in X86TargetLowering()
2098       setOperationAction(ISD::STRICT_UINT_TO_FP,      MVT::v32i16, Legal);  in X86TargetLowering()
2099       setOperationAction(ISD::FP_ROUND,               MVT::v16f16, Legal);  in X86TargetLowering()
2100       setOperationAction(ISD::STRICT_FP_ROUND,        MVT::v16f16, Legal);  in X86TargetLowering()
2101       setOperationAction(ISD::FP_EXTEND,              MVT::v16f32, Legal);  in X86TargetLowering()
2102       setOperationAction(ISD::STRICT_FP_EXTEND,       MVT::v16f32, Legal);  in X86TargetLowering()
2103       setOperationAction(ISD::FP_EXTEND,              MVT::v8f64,  Legal);  in X86TargetLowering()
2104       setOperationAction(ISD::STRICT_FP_EXTEND,       MVT::v8f64,  Legal);  in X86TargetLowering()
2105       setOperationAction(ISD::INSERT_VECTOR_ELT,      MVT::v32f16, Custom);  in X86TargetLowering()
2107       setOperationAction(ISD::FP_TO_SINT,             MVT::v32i16, Custom);  in X86TargetLowering()
2108       setOperationAction(ISD::STRICT_FP_TO_SINT,      MVT::v32i16, Custom);  in X86TargetLowering()
2109       setOperationAction(ISD::FP_TO_UINT,             MVT::v32i16, Custom);  in X86TargetLowering()
2110       setOperationAction(ISD::STRICT_FP_TO_UINT,      MVT::v32i16, Custom);  in X86TargetLowering()
2111       setOperationPromotedToType(ISD::FP_TO_SINT,     MVT::v32i8,  MVT::v32i16);  in X86TargetLowering()
2112       setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v32i8,  in X86TargetLowering()
2113                                  MVT::v32i16);  in X86TargetLowering()
2114       setOperationPromotedToType(ISD::FP_TO_UINT,     MVT::v32i8,  MVT::v32i16);  in X86TargetLowering()
2115       setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v32i8,  in X86TargetLowering()
2116                                  MVT::v32i16);  in X86TargetLowering()
2117       setOperationPromotedToType(ISD::FP_TO_SINT,     MVT::v32i1,  MVT::v32i16);  in X86TargetLowering()
2118       setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v32i1,  in X86TargetLowering()
2119                                  MVT::v32i16);  in X86TargetLowering()
2120       setOperationPromotedToType(ISD::FP_TO_UINT,     MVT::v32i1,  MVT::v32i16);  in X86TargetLowering()
2121       setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v32i1,  in X86TargetLowering()
2122                                  MVT::v32i16);  in X86TargetLowering()
2124       setOperationAction(ISD::EXTRACT_SUBVECTOR,      MVT::v16f16, Legal);  in X86TargetLowering()
2125       setOperationAction(ISD::INSERT_SUBVECTOR,       MVT::v32f16, Legal);  in X86TargetLowering()
2126       setOperationAction(ISD::CONCAT_VECTORS,         MVT::v32f16, Custom);  in X86TargetLowering()
2128       setLoadExtAction(ISD::EXTLOAD, MVT::v8f64,  MVT::v8f16,  Legal);  in X86TargetLowering()
2129       setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16f16, Legal);  in X86TargetLowering()
2131       setOperationAction(ISD::STRICT_FSETCC,      MVT::v32i1, Custom);  in X86TargetLowering()
2132       setOperationAction(ISD::STRICT_FSETCCS,     MVT::v32i1, Custom);  in X86TargetLowering()
2136       setGroup(MVT::v8f16);  in X86TargetLowering()
2137       setGroup(MVT::v16f16);  in X86TargetLowering()
2139       setOperationAction(ISD::SCALAR_TO_VECTOR,   MVT::v8f16,  Legal);  in X86TargetLowering()
2140       setOperationAction(ISD::SCALAR_TO_VECTOR,   MVT::v16f16, Custom);  in X86TargetLowering()
2141       setOperationAction(ISD::SINT_TO_FP,         MVT::v16i16, Legal);  in X86TargetLowering()
2142       setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v16i16, Legal);  in X86TargetLowering()
2143       setOperationAction(ISD::SINT_TO_FP,         MVT::v8i16,  Legal);  in X86TargetLowering()
2144       setOperationAction(ISD::STRICT_SINT_TO_FP,  MVT::v8i16,  Legal);  in X86TargetLowering()
2145       setOperationAction(ISD::UINT_TO_FP,         MVT::v16i16, Legal);  in X86TargetLowering()
2146       setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v16i16, Legal);  in X86TargetLowering()
2147       setOperationAction(ISD::UINT_TO_FP,         MVT::v8i16,  Legal);  in X86TargetLowering()
2148       setOperationAction(ISD::STRICT_UINT_TO_FP,  MVT::v8i16,  Legal);  in X86TargetLowering()
2150       setOperationAction(ISD::FP_TO_SINT,         MVT::v8i16, Custom);  in X86TargetLowering()
2151       setOperationAction(ISD::STRICT_FP_TO_SINT,  MVT::v8i16, Custom);  in X86TargetLowering()
2152       setOperationAction(ISD::FP_TO_UINT,         MVT::v8i16, Custom);  in X86TargetLowering()
2153       setOperationAction(ISD::STRICT_FP_TO_UINT,  MVT::v8i16, Custom);  in X86TargetLowering()
2154       setOperationAction(ISD::FP_ROUND,           MVT::v8f16, Legal);  in X86TargetLowering()
2155       setOperationAction(ISD::STRICT_FP_ROUND,    MVT::v8f16, Legal);  in X86TargetLowering()
2156       setOperationAction(ISD::FP_EXTEND,          MVT::v8f32, Legal);  in X86TargetLowering()
2157       setOperationAction(ISD::STRICT_FP_EXTEND,   MVT::v8f32, Legal);  in X86TargetLowering()
2158       setOperationAction(ISD::FP_EXTEND,          MVT::v4f64, Legal);  in X86TargetLowering()
2159       setOperationAction(ISD::STRICT_FP_EXTEND,   MVT::v4f64, Legal);  in X86TargetLowering()
2162       setOperationAction(ISD::INSERT_VECTOR_ELT,    MVT::v8f16,  Custom);  in X86TargetLowering()
2163       setOperationAction(ISD::INSERT_VECTOR_ELT,    MVT::v16f16, Custom);  in X86TargetLowering()
2165       setOperationAction(ISD::EXTRACT_SUBVECTOR,    MVT::v8f16, Legal);  in X86TargetLowering()
2166       setOperationAction(ISD::INSERT_SUBVECTOR,     MVT::v16f16, Legal);  in X86TargetLowering()
2167       setOperationAction(ISD::CONCAT_VECTORS,       MVT::v16f16, Custom);  in X86TargetLowering()
2169       setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Legal);  in X86TargetLowering()
2170       setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Legal);  in X86TargetLowering()
2171       setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Legal);  in X86TargetLowering()
2172       setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Legal);  in X86TargetLowering()
2175       setOperationAction(ISD::LOAD,  MVT::v4f16, Custom);  in X86TargetLowering()
2176       setOperationAction(ISD::STORE, MVT::v4f16, Custom);  in X86TargetLowering()
2181     setTruncStoreAction(MVT::v4i64, MVT::v4i8,  Legal);  in X86TargetLowering()
2182     setTruncStoreAction(MVT::v4i64, MVT::v4i16, Legal);  in X86TargetLowering()
2183     setTruncStoreAction(MVT::v4i64, MVT::v4i32, Legal);  in X86TargetLowering()
2184     setTruncStoreAction(MVT::v8i32, MVT::v8i8,  Legal);  in X86TargetLowering()
2185     setTruncStoreAction(MVT::v8i32, MVT::v8i16, Legal);  in X86TargetLowering()
2187     setTruncStoreAction(MVT::v2i64, MVT::v2i8,  Legal);  in X86TargetLowering()
2188     setTruncStoreAction(MVT::v2i64, MVT::v2i16, Legal);  in X86TargetLowering()
2189     setTruncStoreAction(MVT::v2i64, MVT::v2i32, Legal);  in X86TargetLowering()
2190     setTruncStoreAction(MVT::v4i32, MVT::v4i8,  Legal);  in X86TargetLowering()
2191     setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal);  in X86TargetLowering()
2194       setTruncStoreAction(MVT::v16i16,  MVT::v16i8, Legal);  in X86TargetLowering()
2195       setTruncStoreAction(MVT::v8i16,   MVT::v8i8,  Legal);  in X86TargetLowering()
2200       setOperationAction(ISD::FP_TO_SINT,        MVT::v2f16, Custom);  in X86TargetLowering()
2201       setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f16, Custom);  in X86TargetLowering()
2202       setOperationAction(ISD::FP_TO_UINT,        MVT::v2f16, Custom);  in X86TargetLowering()
2203       setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f16, Custom);  in X86TargetLowering()
2204       setOperationAction(ISD::FP_TO_SINT,        MVT::v4f16, Custom);  in X86TargetLowering()
2205       setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4f16, Custom);  in X86TargetLowering()
2206       setOperationAction(ISD::FP_TO_UINT,        MVT::v4f16, Custom);  in X86TargetLowering()
2207       setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4f16, Custom);  in X86TargetLowering()
2209       setOperationAction(ISD::SINT_TO_FP,        MVT::v2f16, Custom);  in X86TargetLowering()
2210       setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2f16, Custom);  in X86TargetLowering()
2211       setOperationAction(ISD::UINT_TO_FP,        MVT::v2f16, Custom);  in X86TargetLowering()
2212       setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2f16, Custom);  in X86TargetLowering()
2213       setOperationAction(ISD::SINT_TO_FP,        MVT::v4f16, Custom);  in X86TargetLowering()
2214       setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4f16, Custom);  in X86TargetLowering()
2215       setOperationAction(ISD::UINT_TO_FP,        MVT::v4f16, Custom);  in X86TargetLowering()
2216       setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4f16, Custom);  in X86TargetLowering()
2218       setOperationAction(ISD::FP_ROUND,          MVT::v2f16, Custom);  in X86TargetLowering()
2219       setOperationAction(ISD::STRICT_FP_ROUND,   MVT::v2f16, Custom);  in X86TargetLowering()
2220       setOperationAction(ISD::FP_ROUND,          MVT::v4f16, Custom);  in X86TargetLowering()
2221       setOperationAction(ISD::STRICT_FP_ROUND,   MVT::v4f16, Custom);  in X86TargetLowering()
2223       setOperationAction(ISD::FP_EXTEND,         MVT::v2f16, Custom);  in X86TargetLowering()
2224       setOperationAction(ISD::STRICT_FP_EXTEND,  MVT::v2f16, Custom);  in X86TargetLowering()
2225       setOperationAction(ISD::FP_EXTEND,         MVT::v4f16, Custom);  in X86TargetLowering()
2226       setOperationAction(ISD::STRICT_FP_EXTEND,  MVT::v4f16, Custom);  in X86TargetLowering()
2229     setOperationAction(ISD::TRUNCATE, MVT::v16i32, Custom);  in X86TargetLowering()
2230     setOperationAction(ISD::TRUNCATE, MVT::v8i64, Custom);  in X86TargetLowering()
2231     setOperationAction(ISD::TRUNCATE, MVT::v16i64, Custom);  in X86TargetLowering()
2235     addRegisterClass(MVT::x86amx, &X86::TILERegClass);  in X86TargetLowering()
2239   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);  in X86TargetLowering()
2240   setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);  in X86TargetLowering()
2241   setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);  in X86TargetLowering()
2243     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);  in X86TargetLowering()
2252   for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {  in X86TargetLowering()
2253     if (VT == MVT::i64 && !Subtarget.is64Bit())  in X86TargetLowering()
2286     setOperationAction(ISD::FSINCOS, MVT::f64, Custom);  in X86TargetLowering()
2287     setOperationAction(ISD::FSINCOS, MVT::f32, Custom);  in X86TargetLowering()
2291     setOperationAction(ISD::SDIV, MVT::i128, Custom);  in X86TargetLowering()
2292     setOperationAction(ISD::UDIV, MVT::i128, Custom);  in X86TargetLowering()
2293     setOperationAction(ISD::SREM, MVT::i128, Custom);  in X86TargetLowering()
2294     setOperationAction(ISD::UREM, MVT::i128, Custom);  in X86TargetLowering()
2295     setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom);  in X86TargetLowering()
2296     setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom);  in X86TargetLowering()
2297     setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom);  in X86TargetLowering()
2298     setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom);  in X86TargetLowering()
2299     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i128, Custom);  in X86TargetLowering()
2300     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i128, Custom);  in X86TargetLowering()
2301     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, Custom);  in X86TargetLowering()
2302     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, Custom);  in X86TargetLowering()
2321       if (isOperationExpand(Op, MVT::f32))  in X86TargetLowering()
2322         setOperationAction(Op, MVT::f32, Promote);  in X86TargetLowering()
2425 X86TargetLowering::getPreferredVectorAction(MVT VT) const {  in getPreferredVectorAction()
2426   if ((VT == MVT::v32i1 || VT == MVT::v64i1) && Subtarget.hasAVX512() &&  in getPreferredVectorAction()
2431       !Subtarget.hasF16C() && VT.getVectorElementType() == MVT::f16)  in getPreferredVectorAction()
2435       VT.getVectorElementType() != MVT::i1)  in getPreferredVectorAction()
2441 static std::pair<MVT, unsigned>
2447     return {MVT::v2i64, 1};  in handleMaskRegisterForCallingConv()
2449     return {MVT::v4i32, 1};  in handleMaskRegisterForCallingConv()
2452     return {MVT::v8i16, 1};  in handleMaskRegisterForCallingConv()
2455     return {MVT::v16i8, 1};  in handleMaskRegisterForCallingConv()
2459     return {MVT::v32i8, 1};  in handleMaskRegisterForCallingConv()
2463       return {MVT::v64i8, 1};  in handleMaskRegisterForCallingConv()
2464     return {MVT::v32i8, 2};  in handleMaskRegisterForCallingConv()
2470     return {MVT::i8, NumElts};  in handleMaskRegisterForCallingConv()
2472   return {MVT::INVALID_SIMPLE_VALUE_TYPE, 0};  in handleMaskRegisterForCallingConv()
2475 MVT X86TargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,  in getRegisterTypeForCallingConv()
2479     if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512()) {  in getRegisterTypeForCallingConv()
2482       MVT RegisterVT;  in getRegisterTypeForCallingConv()
2486       if (RegisterVT != MVT::INVALID_SIMPLE_VALUE_TYPE)  in getRegisterTypeForCallingConv()
2490     if (VT.getVectorElementType() == MVT::f16 && VT.getVectorNumElements() < 8)  in getRegisterTypeForCallingConv()
2491       return MVT::v8f16;  in getRegisterTypeForCallingConv()
2495   if ((VT == MVT::f64 || VT == MVT::f80) && !Subtarget.is64Bit() &&  in getRegisterTypeForCallingConv()
2497     return MVT::i32;  in getRegisterTypeForCallingConv()
2499   if (VT.isVector() && VT.getVectorElementType() == MVT::bf16)  in getRegisterTypeForCallingConv()
2510     if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512()) {  in getNumRegistersForCallingConv()
2513       MVT RegisterVT;  in getNumRegistersForCallingConv()
2517       if (RegisterVT != MVT::INVALID_SIMPLE_VALUE_TYPE)  in getNumRegistersForCallingConv()
2521     if (VT.getVectorElementType() == MVT::f16 && VT.getVectorNumElements() < 8)  in getNumRegistersForCallingConv()
2528     if (VT == MVT::f64)  in getNumRegistersForCallingConv()
2530     if (VT == MVT::f80)  in getNumRegistersForCallingConv()
2534   if (VT.isVector() && VT.getVectorElementType() == MVT::bf16)  in getNumRegistersForCallingConv()
2543     unsigned &NumIntermediates, MVT &RegisterVT) const {  in getVectorTypeBreakdownForCallingConv()
2545   if (VT.isVector() && VT.getVectorElementType() == MVT::i1 &&  in getVectorTypeBreakdownForCallingConv()
2550     RegisterVT = MVT::i8;  in getVectorTypeBreakdownForCallingConv()
2551     IntermediateVT = MVT::i1;  in getVectorTypeBreakdownForCallingConv()
2557   if (VT == MVT::v64i1 && Subtarget.hasBWI() && !Subtarget.useAVX512Regs() &&  in getVectorTypeBreakdownForCallingConv()
2559     RegisterVT = MVT::v32i8;  in getVectorTypeBreakdownForCallingConv()
2560     IntermediateVT = MVT::v32i1;  in getVectorTypeBreakdownForCallingConv()
2573     return MVT::i8;  in getSetCCResultType()
2583       return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());  in getSetCCResultType()
2589       MVT EltVT = LegalVT.getSimpleVT().getVectorElementType();  in getSetCCResultType()
2591         return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());  in getSetCCResultType()
2655         return Subtarget.hasBWI() ? MVT::v64i8 : MVT::v16i32;  in getOptimalMemOpType()
2665         return MVT::v32i8;  in getOptimalMemOpType()
2668         return MVT::v16i8;  in getOptimalMemOpType()
2673         return MVT::v4f32;  in getOptimalMemOpType()
2682       return MVT::f64;  in getOptimalMemOpType()
2689     return MVT::i64;  in getOptimalMemOpType()
2690   return MVT::i32;  in getOptimalMemOpType()
2693 bool X86TargetLowering::isSafeMemOpType(MVT VT) const {  in isSafeMemOpType()
2694   if (VT == MVT::f32)  in isSafeMemOpType()
2696   if (VT == MVT::f64)  in isSafeMemOpType()
2748     unsigned NumParts, MVT PartVT, Optional<CallingConv::ID> CC) const {  in splitValueIntoRegisterParts()
2751   if (IsABIRegCopy && ValueVT == MVT::bf16 && PartVT == MVT::f32) {  in splitValueIntoRegisterParts()
2754     Val = DAG.getNode(ISD::BITCAST, DL, MVT::getIntegerVT(ValueBits), Val);  in splitValueIntoRegisterParts()
2755     Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::getIntegerVT(PartBits), Val);  in splitValueIntoRegisterParts()
2765     MVT PartVT, EVT ValueVT, Optional<CallingConv::ID> CC) const {  in joinRegisterPartsIntoValue()
2767   if (IsABIRegCopy && ValueVT == MVT::bf16 && PartVT == MVT::f32) {  in joinRegisterPartsIntoValue()
2772     Val = DAG.getNode(ISD::BITCAST, DL, MVT::getIntegerVT(PartBits), Val);  in joinRegisterPartsIntoValue()
2773     Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::getIntegerVT(ValueBits), Val);  in joinRegisterPartsIntoValue()
2849                                            MVT VT) const {  in findRepresentativeClass()
2855   case MVT::i8: case MVT::i16: case MVT::i32: case MVT::i64:  in findRepresentativeClass()
2858   case MVT::x86mmx:  in findRepresentativeClass()
2861   case MVT::f32: case MVT::f64:  in findRepresentativeClass()
2862   case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64:  in findRepresentativeClass()
2863   case MVT::v4f32: case MVT::v2f64:  in findRepresentativeClass()
2864   case MVT::v32i8: case MVT::v16i16: case MVT::v8i32: case MVT::v4i64:  in findRepresentativeClass()
2865   case MVT::v8f32: case MVT::v4f64:  in findRepresentativeClass()
2866   case MVT::v64i8: case MVT::v32i16: case MVT::v16i32: case MVT::v8i64:  in findRepresentativeClass()
2867   case MVT::v16f32: case MVT::v8f64:  in findRepresentativeClass()
3030   if (ValVT == MVT::v1i1)  in lowerMasksToReg()
3034   if ((ValVT == MVT::v8i1 && (ValLoc == MVT::i8 || ValLoc == MVT::i32)) ||  in lowerMasksToReg()
3035       (ValVT == MVT::v16i1 && (ValLoc == MVT::i16 || ValLoc == MVT::i32))) {  in lowerMasksToReg()
3039     EVT TempValLoc = ValVT == MVT::v8i1 ? MVT::i8 : MVT::i16;  in lowerMasksToReg()
3041     if (ValLoc == MVT::i32)  in lowerMasksToReg()
3046   if ((ValVT == MVT::v32i1 && ValLoc == MVT::i32) ||  in lowerMasksToReg()
3047       (ValVT == MVT::v64i1 && ValLoc == MVT::i64)) {  in lowerMasksToReg()
3063   assert(Arg.getValueType() == MVT::i64 && "Expecting 64 bit value");  in Passv64i1ArgInRegs()
3068   Arg = DAG.getBitcast(MVT::i64, Arg);  in Passv64i1ArgInRegs()
3072   Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, Dl, MVT::i32, Arg,  in Passv64i1ArgInRegs()
3073                    DAG.getConstant(0, Dl, MVT::i32));  in Passv64i1ArgInRegs()
3074   Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, Dl, MVT::i32, Arg,  in Passv64i1ArgInRegs()
3075                    DAG.getConstant(1, Dl, MVT::i32));  in Passv64i1ArgInRegs()
3123       if (ValVT.isVector() && ValVT.getVectorElementType() == MVT::i1)  in LowerReturn()
3141                ValVT == MVT::f64) {  in LowerReturn()
3155         ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy);  in LowerReturn()
3164       if (ValVT == MVT::x86mmx) {  in LowerReturn()
3166           ValToCopy = DAG.getBitcast(MVT::i64, ValToCopy);  in LowerReturn()
3167           ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64,  in LowerReturn()
3172             ValToCopy = DAG.getBitcast(MVT::v4f32, ValToCopy);  in LowerReturn()
3178       assert(VA.getValVT() == MVT::v64i1 &&  in LowerReturn()
3197                    MVT::i32));  in LowerReturn()
3268         RetOps.push_back(DAG.getRegister(*I, MVT::i64));  in LowerReturn()
3283   return DAG.getNode(opcode, dl, MVT::Other, RetOps);  in LowerReturn()
3295     if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue)  in isUsedByReturnOnly()
3310         U->getOperand(U->getNumOperands() - 1).getValueType() != MVT::Glue)  in isUsedByReturnOnly()
3324   MVT ReturnMVT = MVT::i32;  in getTypeForExtReturn()
3327   if (VT == MVT::i1 || (!Darwin && (VT == MVT::i8 || VT == MVT::i16))) {  in getTypeForExtReturn()
3333     ReturnMVT = MVT::i8;  in getTypeForExtReturn()
3355   assert(VA.getValVT() == MVT::v64i1 &&  in getv64i1Argument()
3373     ArgValueLo = DAG.getCopyFromReg(Root, Dl, Reg, MVT::i32);  in getv64i1Argument()
3375     ArgValueHi = DAG.getCopyFromReg(Root, Dl, Reg, MVT::i32);  in getv64i1Argument()
3380       DAG.getCopyFromReg(Root, Dl, VA.getLocReg(), MVT::i32, *InFlag);  in getv64i1Argument()
3383       DAG.getCopyFromReg(Root, Dl, NextVA.getLocReg(), MVT::i32, *InFlag);  in getv64i1Argument()
3388   Lo = DAG.getBitcast(MVT::v32i1, ArgValueLo);  in getv64i1Argument()
3391   Hi = DAG.getBitcast(MVT::v32i1, ArgValueHi);  in getv64i1Argument()
3394   return DAG.getNode(ISD::CONCAT_VECTORS, Dl, MVT::v64i1, Lo, Hi);  in getv64i1Argument()
3405   if (ValVT == MVT::v1i1)  in lowerRegToMasks()
3406     return DAG.getNode(ISD::SCALAR_TO_VECTOR, Dl, MVT::v1i1, ValReturned);  in lowerRegToMasks()
3408   if (ValVT == MVT::v64i1) {  in lowerRegToMasks()
3410     assert(ValLoc == MVT::i64 && "Expecting only i64 locations");  in lowerRegToMasks()
3413     MVT maskLen;  in lowerRegToMasks()
3415     case MVT::v8i1:  in lowerRegToMasks()
3416       maskLen = MVT::i8;  in lowerRegToMasks()
3418     case MVT::v16i1:  in lowerRegToMasks()
3419       maskLen = MVT::i16;  in lowerRegToMasks()
3421     case MVT::v32i1:  in lowerRegToMasks()
3422       maskLen = MVT::i32;  in lowerRegToMasks()
3473                CopyVT == MVT::f64) {  in LowerCallResult()
3488       CopyVT = MVT::f80;  in LowerCallResult()
3494       assert(VA.getValVT() == MVT::v64i1 &&  in LowerCallResult()
3512           VA.getValVT().getScalarType() == MVT::i1 &&  in LowerCallResult()
3513           ((VA.getLocVT() == MVT::i64) || (VA.getLocVT() == MVT::i32) ||  in LowerCallResult()
3514            (VA.getLocVT() == MVT::i16) || (VA.getLocVT() == MVT::i8))) {  in LowerCallResult()
3648   MVT PtrVT = getPointerTy(DAG.getDataLayout());  in LowerMemArgument()
3654       VA.isExtInLoc() && VA.getValVT().getScalarType() == MVT::i1 &&  in LowerMemArgument()
3744       ValVT != MVT::f80)  in LowerMemArgument()
3903       LiveGPRs.push_back(DAG.getCopyFromReg(Chain, DL, GPR, MVT::i64));  in createVarArgAreaAndStoreRegisters()
3908       ALVal = DAG.getCopyFromReg(Chain, DL, AL, MVT::i8);  in createVarArgAreaAndStoreRegisters()
3915         LiveXMMRegs.push_back(DAG.getRegister(Reg, MVT::v4f32));  in createVarArgAreaAndStoreRegisters()
3945           DAG.getTargetConstant(FuncInfo->getVarArgsFPOffset(), DL, MVT::i32));  in createVarArgAreaAndStoreRegisters()
3954                                                DL, DAG.getVTList(MVT::Other),  in createVarArgAreaAndStoreRegisters()
3955                                                SaveXMMOps, MVT::i8, StoreMMO));  in createVarArgAreaAndStoreRegisters()
3959       Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);  in createVarArgAreaAndStoreRegisters()
3965   MVT VecVT = MVT::Other;  in forwardMustTailParameters()
3970     VecVT = MVT::v16f32;  in forwardMustTailParameters()
3972     VecVT = MVT::v8f32;  in forwardMustTailParameters()
3974     VecVT = MVT::v4f32;  in forwardMustTailParameters()
3977   SmallVector<MVT, 2> RegParmTypes;  in forwardMustTailParameters()
3978   MVT IntVT = is64Bit() ? MVT::i64 : MVT::i32;  in forwardMustTailParameters()
3980   if (VecVT != MVT::Other)  in forwardMustTailParameters()
3991     Forwards.push_back(ForwardedRegister(ALVReg, X86::AL, MVT::i8));  in forwardMustTailParameters()
4069             VA.getValVT() == MVT::v64i1 &&  in LowerFormalArguments()
4078         if (RegVT == MVT::i8)  in LowerFormalArguments()
4080         else if (RegVT == MVT::i16)  in LowerFormalArguments()
4082         else if (RegVT == MVT::i32)  in LowerFormalArguments()
4084         else if (Is64Bit && RegVT == MVT::i64)  in LowerFormalArguments()
4086         else if (RegVT == MVT::f16)  in LowerFormalArguments()
4088         else if (RegVT == MVT::f32)  in LowerFormalArguments()
4090         else if (RegVT == MVT::f64)  in LowerFormalArguments()
4092         else if (RegVT == MVT::f80)  in LowerFormalArguments()
4094         else if (RegVT == MVT::f128)  in LowerFormalArguments()
4102         else if (RegVT == MVT::x86mmx)  in LowerFormalArguments()
4104         else if (RegVT == MVT::v1i1)  in LowerFormalArguments()
4106         else if (RegVT == MVT::v8i1)  in LowerFormalArguments()
4108         else if (RegVT == MVT::v16i1)  in LowerFormalArguments()
4110         else if (RegVT == MVT::v32i1)  in LowerFormalArguments()
4112         else if (RegVT == MVT::v64i1)  in LowerFormalArguments()
4135         if (RegVT.isVector() && VA.getValVT().getScalarType() != MVT::i1)  in LowerFormalArguments()
4138                  VA.getValVT().getScalarType() == MVT::i1 &&  in LowerFormalArguments()
4139                  ((VA.getLocVT() == MVT::i64) || (VA.getLocVT() == MVT::i32) ||  in LowerFormalArguments()
4140                   (VA.getLocVT() == MVT::i16) || (VA.getLocVT() == MVT::i8))) {  in LowerFormalArguments()
4169                                   DAG.getFrameIndex(FI, MVT::i32),  in LowerFormalArguments()
4171         Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, St, Chain);  in LowerFormalArguments()
4187       MVT PtrTy = getPointerTy(DAG.getDataLayout());  in LowerFormalArguments()
4192       Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain);  in LowerFormalArguments()
4271       Arg.getSimpleValueType() != MVT::f80)  in LowerMemOpCallTo()
4314 static SDValue getMOVL(SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1,  in getMOVL()
4511           Arg.getValueType().getVectorElementType() == MVT::i1)  in LowerCall()
4515         Arg = DAG.getBitcast(MVT::i64, Arg);  in LowerCall()
4516         Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg);  in LowerCall()
4517         Arg = getMOVL(DAG, dl, MVT::v2i64, DAG.getUNDEF(MVT::v2i64), Arg);  in LowerCall()
4553       assert(VA.getValVT() == MVT::v64i1 &&  in LowerCall()
4586     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);  in LowerCall()
4638                                                         MVT::i8)));  in LowerCall()
4713       Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2);  in LowerCall()
4744              Callee.getValueType() == MVT::i32) {  in LowerCall()
4746     Callee = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Callee);  in LowerCall()
4750   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);  in LowerCall()
4764     Ops.push_back(DAG.getTargetConstant(FPDiff, dl, MVT::i32));  in LowerCall()
5271 bool X86::mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT,  in mayFoldLoadIntoBroadcastFromMem()
5561 static bool useVPTERNLOG(const X86Subtarget &Subtarget, MVT VT) {  in useVPTERNLOG()
5630     MVT VT  = MVT::getVT(I.getArgOperand(1)->getType());  in getTgtMemIntrinsic()
5631     MVT ScalarVT = MVT::INVALID_SIMPLE_VALUE_TYPE;  in getTgtMemIntrinsic()
5633       ScalarVT = MVT::i8;  in getTgtMemIntrinsic()
5635       ScalarVT = MVT::i16;  in getTgtMemIntrinsic()
5637       ScalarVT = MVT::i32;  in getTgtMemIntrinsic()
5639     Info.memVT = MVT::getVectorVT(ScalarVT, VT.getVectorNumElements());  in getTgtMemIntrinsic()
5648     MVT DataVT = MVT::getVT(I.getType());  in getTgtMemIntrinsic()
5649     MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());  in getTgtMemIntrinsic()
5652     Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);  in getTgtMemIntrinsic()
5660     MVT DataVT = MVT::getVT(I.getArgOperand(3)->getType());  in getTgtMemIntrinsic()
5661     MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());  in getTgtMemIntrinsic()
5664     Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);  in getTgtMemIntrinsic()
5738   bool IsFPSetCC = CmpOpVT.isFloatingPoint() && CmpOpVT != MVT::f128;  in reduceSelectOfFPConstantLoads()
5790   if (ResVT.getVectorElementType() == MVT::i1)  in isExtractSubvectorCheap()
5835   return VT == MVT::f32 || VT == MVT::f64 || VT.isVector();  in hasBitPreservingFPLogic()
5842   return !Subtarget.hasSSE2() || VT == MVT::f80;  in ShouldShrinkFPConstant()
5846   return (VT == MVT::f64 && Subtarget.hasSSE2()) ||  in isScalarFPTypeInSSEReg()
5847          (VT == MVT::f32 && Subtarget.hasSSE1()) || VT == MVT::f16;  in isScalarFPTypeInSSEReg()
5854       BitcastVT.getVectorElementType() == MVT::i1)  in isLoadBitCastBeneficial()
5857   if (!Subtarget.hasDQI() && BitcastVT == MVT::v8i1 && LoadVT == MVT::i8)  in isLoadBitCastBeneficial()
5905   if (VT != MVT::i32 && VT != MVT::i64)  in hasAndNotCompare()
5922   if (VT == MVT::v4i32)  in hasAndNot()
5982   if (VT == MVT::i64 && !Subtarget.is64Bit())  in shouldFoldMaskToVariableShiftPair()
6002 MVT X86TargetLowering::hasFastEqualityCompare(unsigned NumBits) const {  in hasFastEqualityCompare()
6003   MVT VT = MVT::getIntegerVT(NumBits);  in hasFastEqualityCompare()
6008   if (NumBits == 128 && isTypeLegal(MVT::v16i8))  in hasFastEqualityCompare()
6009     return MVT::v16i8;  in hasFastEqualityCompare()
6012   if (NumBits == 256 && isTypeLegal(MVT::v32i8))  in hasFastEqualityCompare()
6013     return MVT::v32i8;  in hasFastEqualityCompare()
6019   return MVT::INVALID_SIMPLE_VALUE_TYPE;  in hasFastEqualityCompare()
6257 static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG,  in getConstVector()
6263   MVT ConstVecVT = VT;  in getConstVector()
6265   bool In64BitMode = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64);  in getConstVector()
6266   if (!In64BitMode && VT.getVectorElementType() == MVT::i64) {  in getConstVector()
6267     ConstVecVT = MVT::getVectorVT(MVT::i32, NumElts * 2);  in getConstVector()
6271   MVT EltVT = ConstVecVT.getVectorElementType();  in getConstVector()
6288                               MVT VT, SelectionDAG &DAG, const SDLoc &dl) {  in getConstVector()
6294   MVT ConstVecVT = VT;  in getConstVector()
6296   bool In64BitMode = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64);  in getConstVector()
6297   if (!In64BitMode && VT.getVectorElementType() == MVT::i64) {  in getConstVector()
6298     ConstVecVT = MVT::getVectorVT(MVT::i32, NumElts * 2);  in getConstVector()
6302   MVT EltVT = ConstVecVT.getVectorElementType();  in getConstVector()
6313     } else if (EltVT == MVT::f32) {  in getConstVector()
6316     } else if (EltVT == MVT::f64) {  in getConstVector()
6329 static SDValue getZeroVector(MVT VT, const X86Subtarget &Subtarget,  in getZeroVector()
6332           VT.getVectorElementType() == MVT::i1) &&  in getZeroVector()
6340     Vec = DAG.getConstantFP(+0.0, dl, MVT::v4f32);  in getZeroVector()
6343   } else if (VT.getVectorElementType() == MVT::i1) {  in getZeroVector()
6349     Vec = DAG.getConstant(0, dl, MVT::getVectorVT(MVT::i32, Num32BitElts));  in getZeroVector()
6460 static SDValue widenSubVector(MVT VT, SDValue Vec, bool ZeroNewElements,  in widenSubVector()
6481   MVT SVT = Vec.getSimpleValueType().getScalarType();  in widenSubVector()
6482   MVT VT = MVT::getVectorVT(SVT, WideNumElts);  in widenSubVector()
6662 static SDValue getAVX512Node(unsigned Opcode, const SDLoc &DL, MVT VT,  in getAVX512Node()
6666   MVT SVT = VT.getScalarType();  in getAVX512Node()
6670   auto MakeBroadcastOp = [&](SDValue Op, MVT OpVT, MVT DstVT) {  in getAVX512Node()
6694   MVT DstVT = VT;  in getAVX512Node()
6696     DstVT = MVT::getVectorVT(SVT, 512 / SVT.getSizeInBits());  in getAVX512Node()
6701     MVT OpVT = Op.getSimpleValueType();  in getAVX512Node()
6742   MVT OpVT = Op.getSimpleValueType();  in insert1BitVector()
6747   MVT WideOpVT = OpVT;  in insert1BitVector()
6749     WideOpVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1;  in insert1BitVector()
6761   MVT SubVecVT = SubVec.getSimpleValueType();  in insert1BitVector()
6771     SDValue ShiftBits = DAG.getTargetConstant(SubVecNumElems, dl, MVT::i8);  in insert1BitVector()
6790                          DAG.getTargetConstant(IdxVal, dl, MVT::i8));  in insert1BitVector()
6800                            DAG.getTargetConstant(IdxVal, dl, MVT::i8));  in insert1BitVector()
6806                            DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));  in insert1BitVector()
6809                              DAG.getTargetConstant(ShiftRight, dl, MVT::i8));  in insert1BitVector()
6817                          DAG.getTargetConstant(IdxVal, dl, MVT::i8));  in insert1BitVector()
6830       SDValue ShiftBits = DAG.getTargetConstant(NumElems - IdxVal, dl, MVT::i8);  in insert1BitVector()
6849   if (WideOpVT != MVT::v64i1 || Subtarget.is64Bit()) {  in insert1BitVector()
6852     SDValue CMask0 = DAG.getConstant(Mask0, dl, MVT::getIntegerVT(NumElems));  in insert1BitVector()
6856                          DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));  in insert1BitVector()
6858                          DAG.getTargetConstant(ShiftRight, dl, MVT::i8));  in insert1BitVector()
6867                        DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));  in insert1BitVector()
6869                        DAG.getTargetConstant(ShiftRight, dl, MVT::i8));  in insert1BitVector()
6874                             DAG.getTargetConstant(LowShift, dl, MVT::i8));  in insert1BitVector()
6876                     DAG.getTargetConstant(LowShift, dl, MVT::i8));  in insert1BitVector()
6881                             DAG.getTargetConstant(HighShift, dl, MVT::i8));  in insert1BitVector()
6883                     DAG.getTargetConstant(HighShift, dl, MVT::i8));  in insert1BitVector()
6914   SDValue Vec = DAG.getConstant(Ones, dl, MVT::getVectorVT(MVT::i32, NumElts));  in getOnesVector()
7023 void llvm::createSplat2ShuffleMask(MVT VT, SmallVectorImpl<int> &Mask,  in createSplat2ShuffleMask()
7075                        const SDLoc &dl, MVT VT, SDValue LHS, SDValue RHS,  in getPack()
7077   MVT OpVT = LHS.getSimpleValueType();  in getPack()
7115   SDValue Amt = DAG.getTargetConstant(EltSizeInBits, dl, MVT::i8);  in getPack()
7145   MVT VT = V2.getSimpleValueType();  in getShuffleVectorZeroOrUndef()
7543 static void createPackShuffleMask(MVT VT, SmallVectorImpl<int> &Mask,  in createPackShuffleMask()
7624 static bool getTargetShuffleMask(SDNode *N, MVT VT, bool AllowSentinelZero,  in getTargetShuffleMask()
7706     assert((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) &&  in getTargetShuffleMask()
7717     assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");  in getTargetShuffleMask()
7727     assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");  in getTargetShuffleMask()
7734     assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");  in getTargetShuffleMask()
7786     assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");  in getTargetShuffleMask()
7926 static bool getTargetShuffleMask(SDNode *N, MVT VT, bool AllowSentinelZero,  in getTargetShuffleMask()
8031   MVT VT = N.getSimpleValueType();  in getTargetShuffleAndZeroables()
8202   MVT VT = N.getSimpleValueType();  in getFauxShuffleMask()
8701   SDVTList Tys = DAG.getVTList(VT, MVT::Other);  in getBROADCAST_LOAD()
8735     MVT ShufVT = VT.getSimpleVT();  in getShuffleScalarElt()
8736     MVT ShufSVT = ShufVT.getVectorElementType();  in getShuffleScalarElt()
8819   MVT VT = Op.getSimpleValueType();  in LowerBuildVectorAsInsert()
8821   assert(((VT == MVT::v8i16 && Subtarget.hasSSE2()) ||  in LowerBuildVectorAsInsert()
8822           ((VT == MVT::v16i8 || VT == MVT::v4i32) && Subtarget.hasSSE41())) &&  in LowerBuildVectorAsInsert()
8843         V = DAG.getAnyExtOrTrunc(Op.getOperand(i), dl, MVT::i32);  in LowerBuildVectorAsInsert()
8844         V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, V);  in LowerBuildVectorAsInsert()
8883         Elt = DAG.getZExtOrTrunc(Op.getOperand(i), dl, MVT::i32);  in LowerBuildVectorv16i8()
8885         Elt = DAG.getAnyExtOrTrunc(Op.getOperand(i), dl, MVT::i32);  in LowerBuildVectorv16i8()
8891         NextElt = DAG.getZExtOrTrunc(NextElt, dl, MVT::i32);  in LowerBuildVectorv16i8()
8893         NextElt = DAG.getAnyExtOrTrunc(NextElt, dl, MVT::i32);  in LowerBuildVectorv16i8()
8894       NextElt = DAG.getNode(ISD::SHL, dl, MVT::i32, NextElt,  in LowerBuildVectorv16i8()
8895                             DAG.getConstant(8, dl, MVT::i8));  in LowerBuildVectorv16i8()
8897         Elt = DAG.getNode(ISD::OR, dl, MVT::i32, NextElt, Elt);  in LowerBuildVectorv16i8()
8907         V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl);  in LowerBuildVectorv16i8()
8909         V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Elt);  in LowerBuildVectorv16i8()
8910         V = DAG.getBitcast(MVT::v8i16, V);  in LowerBuildVectorv16i8()
8914     Elt = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Elt);  in LowerBuildVectorv16i8()
8915     V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, Elt,  in LowerBuildVectorv16i8()
8919   return DAG.getBitcast(MVT::v16i8, V);  in LowerBuildVectorv16i8()
8947     MVT VT = Op.getSimpleValueType();  in LowerBuildVectorv4x32()
8948     MVT EltVT = VT.getVectorElementType();  in LowerBuildVectorv4x32()
8953     SDValue NewBV = DAG.getBitcast(MVT::v2f64, DAG.getBuildVector(VT, DL, Ops));  in LowerBuildVectorv4x32()
8954     SDValue Dup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, NewBV);  in LowerBuildVectorv4x32()
8980     MVT VT = Elt.getOperand(0).getSimpleValueType();  in LowerBuildVectorv4x32()
8991   MVT VT = V1.getSimpleValueType();  in LowerBuildVectorv4x32()
9046   if (V1.getSimpleValueType() != MVT::v4f32)  in LowerBuildVectorv4x32()
9047     V1 = DAG.getBitcast(MVT::v4f32, V1);  in LowerBuildVectorv4x32()
9048   if (V2.getSimpleValueType() != MVT::v4f32)  in LowerBuildVectorv4x32()
9049     V2 = DAG.getBitcast(MVT::v4f32, V2);  in LowerBuildVectorv4x32()
9057   SDValue Result = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,  in LowerBuildVectorv4x32()
9067   MVT ShVT = MVT::v16i8;  in getVShift()
9071   SDValue ShiftVal = DAG.getTargetConstant(NumBits / 8, dl, MVT::i8);  in getVShift()
9075 static SDValue LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, const SDLoc &dl,  in LowerAsSplatVectorLoad()
9086     if (PVT != MVT::i32 && PVT != MVT::f32)  in LowerAsSplatVectorLoad()
9387     MVT VecSVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(LoadSizeInBits)  in EltsFromConsecutiveLoads()
9388                                       : MVT::getIntegerVT(LoadSizeInBits);  in EltsFromConsecutiveLoads()
9389     MVT VecVT = MVT::getVectorVT(VecSVT, VT.getSizeInBits() / LoadSizeInBits);  in EltsFromConsecutiveLoads()
9392     if (!Subtarget.hasSSE2() && VT == MVT::v4f32)  in EltsFromConsecutiveLoads()
9393       VecVT = MVT::v4f32;  in EltsFromConsecutiveLoads()
9395       SDVTList Tys = DAG.getVTList(VecVT, MVT::Other);  in EltsFromConsecutiveLoads()
9441           VT.isInteger() && (RepeatSize != 64 || TLI.isTypeLegal(MVT::i64))  in EltsFromConsecutiveLoads()
9496 static Constant *getConstantVector(MVT VT, const APInt &SplatValue,  in getConstantVector()
9560   MVT VT = BVOp->getSimpleValueType(0);  in lowerBuildVectorAsBroadcast()
9598       MVT MaskVT = BOperand.getSimpleValueType();  in lowerBuildVectorAsBroadcast()
9599       MVT EltType = MVT::getIntegerVT(VT.getScalarSizeInBits() * SeqLen);  in lowerBuildVectorAsBroadcast()
9600       if ((EltType == MVT::i64 && MaskVT == MVT::v8i1) ||  // for broadcastmb2q  in lowerBuildVectorAsBroadcast()
9601           (EltType == MVT::i32 && MaskVT == MVT::v16i1)) { // for broadcastmw2d  in lowerBuildVectorAsBroadcast()
9602         MVT BcstVT = MVT::getVectorVT(EltType, NumElts / SeqLen);  in lowerBuildVectorAsBroadcast()
9605           BcstVT = MVT::getVectorVT(EltType, Scale * (NumElts / SeqLen));  in lowerBuildVectorAsBroadcast()
9631       MVT PVT = TLI.getPointerTy(DAG.getDataLayout());  in lowerBuildVectorAsBroadcast()
9637           MVT CVT = MVT::getIntegerVT(SplatBitSize);  in lowerBuildVectorAsBroadcast()
9645               DAG.getVTList(MVT::getVectorVT(CVT, Repeat), MVT::Other);  in lowerBuildVectorAsBroadcast()
9660           MVT VVT = MVT::getVectorVT(VT.getScalarType(), NumElm);  in lowerBuildVectorAsBroadcast()
9662           SDVTList Tys = DAG.getVTList(VT, MVT::Other);  in lowerBuildVectorAsBroadcast()
9723         CVT == MVT::f16 ||  in lowerBuildVectorAsBroadcast()
9738       SDVTList Tys = DAG.getVTList(VT, MVT::Other);  in lowerBuildVectorAsBroadcast()
9763     SDVTList Tys = DAG.getVTList(VT, MVT::Other);  in lowerBuildVectorAsBroadcast()
9777     SDVTList Tys = DAG.getVTList(VT, MVT::Other);  in lowerBuildVectorAsBroadcast()
9816   MVT ShuffleVecVT = ShuffleVec.getSimpleValueType();  in getUnderlyingExtractedFromVec()
9829   MVT VT = Op.getSimpleValueType();  in buildFromShuffleMostly()
9904   MVT VT = Op.getSimpleValueType();  in LowerBUILD_VECTORvXi1()
9905   assert((VT.getVectorElementType() == MVT::i1) &&  in LowerBUILD_VECTORvXi1()
9941     assert(Cond.getValueType() == MVT::i8 && "Unexpected VT!");  in LowerBUILD_VECTORvXi1()
9943       Cond = DAG.getNode(ISD::AND, dl, MVT::i8, Cond,  in LowerBUILD_VECTORvXi1()
9944                          DAG.getConstant(1, dl, MVT::i8));  in LowerBUILD_VECTORvXi1()
9947     if (VT == MVT::v64i1 && !Subtarget.is64Bit()) {  in LowerBUILD_VECTORvXi1()
9948       SDValue Select = DAG.getSelect(dl, MVT::i32, Cond,  in LowerBUILD_VECTORvXi1()
9949                                      DAG.getAllOnesConstant(dl, MVT::i32),  in LowerBUILD_VECTORvXi1()
9950                                      DAG.getConstant(0, dl, MVT::i32));  in LowerBUILD_VECTORvXi1()
9951       Select = DAG.getBitcast(MVT::v32i1, Select);  in LowerBUILD_VECTORvXi1()
9952       return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Select, Select);  in LowerBUILD_VECTORvXi1()
9954       MVT ImmVT = MVT::getIntegerVT(std::max((unsigned)VT.getSizeInBits(), 8U));  in LowerBUILD_VECTORvXi1()
9958       MVT VecVT = VT.getSizeInBits() >= 8 ? VT : MVT::v8i1;  in LowerBUILD_VECTORvXi1()
9968     if (VT == MVT::v64i1 && !Subtarget.is64Bit()) {  in LowerBUILD_VECTORvXi1()
9969       SDValue ImmL = DAG.getConstant(Lo_32(Immediate), dl, MVT::i32);  in LowerBUILD_VECTORvXi1()
9970       SDValue ImmH = DAG.getConstant(Hi_32(Immediate), dl, MVT::i32);  in LowerBUILD_VECTORvXi1()
9971       ImmL = DAG.getBitcast(MVT::v32i1, ImmL);  in LowerBUILD_VECTORvXi1()
9972       ImmH = DAG.getBitcast(MVT::v32i1, ImmH);  in LowerBUILD_VECTORvXi1()
9973       DstVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, ImmL, ImmH);  in LowerBUILD_VECTORvXi1()
9975       MVT ImmVT = MVT::getIntegerVT(std::max((unsigned)VT.getSizeInBits(), 8U));  in LowerBUILD_VECTORvXi1()
9977       MVT VecVT = VT.getSizeInBits() >= 8 ? VT : MVT::v8i1;  in LowerBUILD_VECTORvXi1()
10145   MVT VT = V0.getSimpleValueType();  in ExpandHorizontalBinOp()
10154   MVT NewVT = V0_LO.getSimpleValueType();  in ExpandHorizontalBinOp()
10188   MVT VT = BV->getSimpleValueType(0);  in isAddSubOrSubAdd()
10347   MVT VT = BV->getSimpleValueType(0);  in lowerToAddSubOrFMAddSub()
10380   MVT VT = BV->getSimpleValueType(0);  in isHopBuildVector()
10470   MVT VT = BV->getSimpleValueType(0);  in getHopForBuildVector()
10491     MVT HalfVT = VT.getHalfNumVectorElementsVT();  in getHopForBuildVector()
10514   MVT VT = BV->getSimpleValueType(0);  in LowerToHorizontalOp()
10515   if (((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget.hasSSE3()) ||  in LowerToHorizontalOp()
10516       ((VT == MVT::v8i16 || VT == MVT::v4i32) && Subtarget.hasSSSE3()) ||  in LowerToHorizontalOp()
10517       ((VT == MVT::v8f32 || VT == MVT::v4f64) && Subtarget.hasAVX()) ||  in LowerToHorizontalOp()
10518       ((VT == MVT::v16i16 || VT == MVT::v8i32) && Subtarget.hasAVX2())) {  in LowerToHorizontalOp()
10544   if (VT == MVT::v8i32 || VT == MVT::v16i16) {  in LowerToHorizontalOp()
10584   if (VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 ||  in LowerToHorizontalOp()
10585       VT == MVT::v16i16) {  in LowerToHorizontalOp()
10630   MVT VT = Op->getSimpleValueType(0);  in lowerBuildVectorToBitOp()
10707   MVT VT = Op.getSimpleValueType();  in materializeVectorConstant()
10717     if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32)  in materializeVectorConstant()
10729 static SDValue createVariablePermute(MVT VT, SDValue SrcVec, SDValue IndicesVec,  in createVariablePermute()
10732   MVT ShuffleVT = VT;  in createVariablePermute()
10760       VT = MVT::getVectorVT(VT.getScalarType(), Scale * NumElts);  in createVariablePermute()
10804   case MVT::v16i8:  in createVariablePermute()
10808   case MVT::v8i16:  in createVariablePermute()
10813       ShuffleVT = MVT::v16i8;  in createVariablePermute()
10816   case MVT::v4f32:  in createVariablePermute()
10817   case MVT::v4i32:  in createVariablePermute()
10820       ShuffleVT = MVT::v4f32;  in createVariablePermute()
10823       ShuffleVT = MVT::v16i8;  in createVariablePermute()
10826   case MVT::v2f64:  in createVariablePermute()
10827   case MVT::v2i64:  in createVariablePermute()
10832       ShuffleVT = MVT::v2f64;  in createVariablePermute()
10843   case MVT::v32i8:  in createVariablePermute()
10853           DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, LoIdx),  in createVariablePermute()
10854           DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, HiIdx));  in createVariablePermute()
10873       return SplitOpsAndApply(DAG, Subtarget, DL, MVT::v32i8, Ops,  in createVariablePermute()
10877   case MVT::v16i16:  in createVariablePermute()
10885                   MVT::v32i8, DAG.getBitcast(MVT::v32i8, SrcVec),  in createVariablePermute()
10886                   DAG.getBitcast(MVT::v32i8, IndicesVec), DL, DAG, Subtarget));  in createVariablePermute()
10889   case MVT::v8f32:  in createVariablePermute()
10890   case MVT::v8i32:  in createVariablePermute()
10894       SrcVec = DAG.getBitcast(MVT::v8f32, SrcVec);  in createVariablePermute()
10895       SDValue LoLo = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,  in createVariablePermute()
10897       SDValue HiHi = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,  in createVariablePermute()
10901             VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v8f32, LoLo, HiHi,  in createVariablePermute()
10902                             IndicesVec, DAG.getTargetConstant(0, DL, MVT::i8)));  in createVariablePermute()
10906           DL, IndicesVec, DAG.getConstant(3, DL, MVT::v8i32),  in createVariablePermute()
10907           DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, HiHi, IndicesVec),  in createVariablePermute()
10908           DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, LoLo, IndicesVec),  in createVariablePermute()
10913   case MVT::v4i64:  in createVariablePermute()
10914   case MVT::v4f64:  in createVariablePermute()
10917         MVT WidenSrcVT = MVT::getVectorVT(VT.getScalarType(), 8);  in createVariablePermute()
10920         IndicesVec = widenSubVector(MVT::v8i64, IndicesVec, false, Subtarget,  in createVariablePermute()
10928       SrcVec = DAG.getBitcast(MVT::v4f64, SrcVec);  in createVariablePermute()
10930           DAG.getVectorShuffle(MVT::v4f64, DL, SrcVec, SrcVec, {0, 1, 0, 1});  in createVariablePermute()
10932           DAG.getVectorShuffle(MVT::v4f64, DL, SrcVec, SrcVec, {2, 3, 2, 3});  in createVariablePermute()
10937             VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v4f64, LoLo, HiHi,  in createVariablePermute()
10938                             IndicesVec, DAG.getTargetConstant(0, DL, MVT::i8)));  in createVariablePermute()
10942           DL, IndicesVec, DAG.getConstant(2, DL, MVT::v4i64),  in createVariablePermute()
10943           DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, HiHi, IndicesVec),  in createVariablePermute()
10944           DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, LoLo, IndicesVec),  in createVariablePermute()
10949   case MVT::v64i8:  in createVariablePermute()
10953   case MVT::v32i16:  in createVariablePermute()
10957   case MVT::v16f32:  in createVariablePermute()
10958   case MVT::v16i32:  in createVariablePermute()
10959   case MVT::v8f64:  in createVariablePermute()
10960   case MVT::v8i64:  in createVariablePermute()
11038   MVT VT = V.getSimpleValueType();  in LowerBUILD_VECTORAsVariablePermute()
11046   MVT VT = Op.getSimpleValueType();  in LowerBUILD_VECTOR()
11047   MVT EltVT = VT.getVectorElementType();  in LowerBUILD_VECTOR()
11051   if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512())  in LowerBUILD_VECTOR()
11102       MVT LowerVT = MVT::getVectorVT(EltVT, NumElems - UpperElems);  in LowerBUILD_VECTOR()
11194       if (EltVT == MVT::i32 || EltVT == MVT::f16 || EltVT == MVT::f32 ||  in LowerBUILD_VECTOR()
11195           EltVT == MVT::f64 || (EltVT == MVT::i64 && Subtarget.is64Bit()) ||  in LowerBUILD_VECTOR()
11196           (EltVT == MVT::i16 && Subtarget.hasFP16())) {  in LowerBUILD_VECTOR()
11208       if (EltVT == MVT::i16 || EltVT == MVT::i8) {  in LowerBUILD_VECTOR()
11209         Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item);  in LowerBUILD_VECTOR()
11210         MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32);  in LowerBUILD_VECTOR()
11287       MVT WideEltVT = VT.isFloatingPoint() ? MVT::f64 : MVT::i64;  in LowerBUILD_VECTOR()
11288       MVT NarrowVT = MVT::getVectorVT(EltVT, 4);  in LowerBUILD_VECTOR()
11290       SDValue NewBV = DAG.getBitcast(MVT::getVectorVT(WideEltVT, 2),  in LowerBUILD_VECTOR()
11293       MVT BcastVT = MVT::getVectorVT(WideEltVT, NumElems / 2);  in LowerBUILD_VECTOR()
11302     MVT HVT = MVT::getVectorVT(EltVT, NumElems / 2);  in LowerBUILD_VECTOR()
11332   if (EltVT == MVT::i16 && NumElems == 8)  in LowerBUILD_VECTOR()
11389   if (Subtarget.hasSSE41() && EltVT != MVT::f16) {  in LowerBUILD_VECTOR()
11440   MVT ResVT = Op.getSimpleValueType();  in LowerAVXCONCAT_VECTORS()
11464     MVT HalfVT = ResVT.getHalfNumVectorElementsVT();  in LowerAVXCONCAT_VECTORS()
11477   MVT SubVT = Op.getOperand(0).getSimpleValueType();  in LowerAVXCONCAT_VECTORS()
11499   MVT ResVT = Op.getSimpleValueType();  in LowerCONCAT_VECTORSvXi1()
11525     MVT ShiftVT = ResVT;  in LowerCONCAT_VECTORSvXi1()
11527       ShiftVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1;  in LowerCONCAT_VECTORSvXi1()
11535                      DAG.getTargetConstant(Idx * SubVecNumElts, dl, MVT::i8));  in LowerCONCAT_VECTORSvXi1()
11553     MVT HalfVT = ResVT.getHalfNumVectorElementsVT();  in LowerCONCAT_VECTORSvXi1()
11577   MVT VT = Op.getSimpleValueType();  in LowerCONCAT_VECTORS()
11578   if (VT.getVectorElementType() == MVT::i1)  in LowerCONCAT_VECTORS()
11640 static bool is128BitLaneCrossingShuffleMask(MVT VT, ArrayRef<int> Mask) {  in is128BitLaneCrossingShuffleMask()
11684 static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT,  in isRepeatedShuffleMask()
11714 is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask,  in is128BitLaneRepeatedShuffleMask()
11720 is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask) {  in is128BitLaneRepeatedShuffleMask()
11727 is256BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask,  in is256BitLaneRepeatedShuffleMask()
11771 static bool isRepeatedTargetShuffleMask(unsigned LaneSizeInBits, MVT VT,  in isRepeatedTargetShuffleMask()
11811       MVT VT = Op.getSimpleValueType();  in IsElementEquivalent()
11872 static bool isTargetShuffleEquivalent(MVT VT, ArrayRef<int> Mask,  in isTargetShuffleEquivalent()
11931 static bool isUnpackWdShuffleMask(ArrayRef<int> Mask, MVT VT,  in isUnpackWdShuffleMask()
11933   if (VT != MVT::v8i32 && VT != MVT::v8f32)  in isUnpackWdShuffleMask()
11937   createUnpackShuffleMask(MVT::v8i16, Unpcklwd, /* Lo = */ true,  in isUnpackWdShuffleMask()
11940   createUnpackShuffleMask(MVT::v8i16, Unpckhwd, /* Lo = */ false,  in isUnpackWdShuffleMask()
11950   MVT EltVT = MVT::getIntegerVT(128 / Mask.size());  in is128BitUnpackShuffleMask()
11951   MVT VT = MVT::getVectorVT(EltVT, Mask.size());  in is128BitUnpackShuffleMask()
12016   return DAG.getTargetConstant(getV4X86ShuffleImm(Mask), DL, MVT::i8);  in getV4X86ShuffleImm8ForMask()
12052 static SDValue lowerShuffleWithPSHUFB(const SDLoc &DL, MVT VT,  in lowerShuffleWithPSHUFB()
12068   SDValue ZeroMask = DAG.getConstant(0x80, DL, MVT::i8);  in lowerShuffleWithPSHUFB()
12074       PSHUFBMask[i] = DAG.getUNDEF(MVT::i8);  in lowerShuffleWithPSHUFB()
12095     PSHUFBMask[i] = DAG.getConstant(M, DL, MVT::i8);  in lowerShuffleWithPSHUFB()
12099   MVT I8VT = MVT::getVectorVT(MVT::i8, NumBytes);  in lowerShuffleWithPSHUFB()
12105 static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
12110 static SDValue lowerShuffleToEXPAND(const SDLoc &DL, MVT VT,  in lowerShuffleToEXPAND()
12120   MVT IntegerType =  in lowerShuffleToEXPAND()
12121       MVT::getIntegerVT(std::max((int)VT.getVectorNumElements(), 8));  in lowerShuffleToEXPAND()
12126   SDValue VMask = getMaskNode(MaskNode, MVT::getVectorVT(MVT::i1, NumElts),  in lowerShuffleToEXPAND()
12133 static bool matchShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2,  in matchShuffleWithUNPCK()
12175     if ((Subtarget.hasSSE41() || VT == MVT::v2i64 || VT == MVT::v2f64) &&  in matchShuffleWithUNPCK()
12222 static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT,  in lowerShuffleWithUNPCK()
12249 static SDValue lowerShuffleWithUNPCK256(const SDLoc &DL, MVT VT,  in lowerShuffleWithUNPCK256()
12267   V1 = DAG.getVectorShuffle(MVT::v4f64, DL, DAG.getBitcast(MVT::v4f64, V1),  in lowerShuffleWithUNPCK256()
12268                             DAG.getUNDEF(MVT::v4f64), {0, 2, 1, 3});  in lowerShuffleWithUNPCK256()
12275 static bool matchShuffleAsVTRUNC(MVT &SrcVT, MVT &DstVT, MVT VT,  in matchShuffleAsVTRUNC()
12295     SrcVT = MVT::getIntegerVT(EltSizeInBits * Scale);  in matchShuffleAsVTRUNC()
12296     SrcVT = MVT::getVectorVT(SrcVT, NumSrcElts);  in matchShuffleAsVTRUNC()
12297     DstVT = MVT::getIntegerVT(EltSizeInBits);  in matchShuffleAsVTRUNC()
12300       DstVT = MVT::getVectorVT(DstVT, NumSrcElts);  in matchShuffleAsVTRUNC()
12303       DstVT = MVT::getVectorVT(DstVT, 128 / EltSizeInBits);  in matchShuffleAsVTRUNC()
12313 static SDValue getAVX512TruncNode(const SDLoc &DL, MVT DstVT, SDValue Src,  in getAVX512TruncNode()
12316   MVT SrcVT = Src.getSimpleValueType();  in getAVX512TruncNode()
12317   MVT DstSVT = DstVT.getScalarType();  in getAVX512TruncNode()
12330     MVT TruncVT = MVT::getVectorVT(DstSVT, NumSrcElts);  in getAVX512TruncNode()
12336     MVT TruncVT = MVT::getVectorVT(DstSVT, NumSrcElts);  in getAVX512TruncNode()
12350   MVT TruncVT = MVT::getVectorVT(DstSVT, 128 / DstEltSizeInBits);  in getAVX512TruncNode()
12374 static SDValue lowerShuffleWithVPMOV(const SDLoc &DL, MVT VT, SDValue V1,  in lowerShuffleWithVPMOV()
12379   assert((VT == MVT::v16i8 || VT == MVT::v8i16) && "Unexpected VTRUNC type");  in lowerShuffleWithVPMOV()
12404     MVT SrcVT = Src.getSimpleValueType();  in lowerShuffleWithVPMOV()
12405     if (SrcVT.getVectorElementType() == MVT::i16 && VT == MVT::v16i8 &&  in lowerShuffleWithVPMOV()
12417 static SDValue lowerShuffleAsVTRUNC(const SDLoc &DL, MVT VT, SDValue V1,  in lowerShuffleAsVTRUNC()
12454     MVT ConcatVT = MVT::getVectorVT(VT.getScalarType(), NumElts * 2);  in lowerShuffleAsVTRUNC()
12457     MVT SrcSVT = MVT::getIntegerVT(SrcEltBits);  in lowerShuffleAsVTRUNC()
12458     MVT SrcVT = MVT::getVectorVT(SrcSVT, NumSrcElts);  in lowerShuffleAsVTRUNC()
12542 static bool matchShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2,  in matchShuffleWithPACK()
12552   auto MatchPACK = [&](SDValue N1, SDValue N2, MVT PackVT) {  in matchShuffleWithPACK()
12592     MVT PackSVT = MVT::getIntegerVT(BitSize << NumStages);  in matchShuffleWithPACK()
12593     MVT PackVT = MVT::getVectorVT(PackSVT, NumElts >> NumStages);  in matchShuffleWithPACK()
12613 static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,  in lowerShuffleWithPACK()
12616   MVT PackVT;  in lowerShuffleWithPACK()
12644     MVT SrcSVT = MVT::getIntegerVT(SrcEltBits);  in lowerShuffleWithPACK()
12645     MVT DstSVT = MVT::getIntegerVT(SrcEltBits / 2);  in lowerShuffleWithPACK()
12646     MVT SrcVT = MVT::getVectorVT(SrcSVT, NumSrcElts);  in lowerShuffleWithPACK()
12647     MVT DstVT = MVT::getVectorVT(DstSVT, NumSrcElts * 2);  in lowerShuffleWithPACK()
12662 static SDValue lowerShuffleAsBitMask(const SDLoc &DL, MVT VT, SDValue V1,  in lowerShuffleAsBitMask()
12667   MVT MaskVT = VT;  in lowerShuffleAsBitMask()
12668   MVT EltVT = VT.getVectorElementType();  in lowerShuffleAsBitMask()
12671   if (EltVT == MVT::i64 && !Subtarget.is64Bit()) {  in lowerShuffleAsBitMask()
12672     EltVT = MVT::f64;  in lowerShuffleAsBitMask()
12673     MaskVT = MVT::getVectorVT(EltVT, Mask.size());  in lowerShuffleAsBitMask()
12676   MVT LogicVT = VT;  in lowerShuffleAsBitMask()
12677   if (EltVT == MVT::f32 || EltVT == MVT::f64) {  in lowerShuffleAsBitMask()
12683         MVT::getVectorVT(EltVT == MVT::f64 ? MVT::i64 : MVT::i32, Mask.size());  in lowerShuffleAsBitMask()
12718 static SDValue lowerShuffleAsBitBlend(const SDLoc &DL, MVT VT, SDValue V1,  in lowerShuffleAsBitBlend()
12722   MVT EltVT = VT.getVectorElementType();  in lowerShuffleAsBitBlend()
12806 static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,  in lowerShuffleAsBlend()
12827   case MVT::v4i64:  in lowerShuffleAsBlend()
12828   case MVT::v8i32:  in lowerShuffleAsBlend()
12831   case MVT::v4f64:  in lowerShuffleAsBlend()
12832   case MVT::v8f32:  in lowerShuffleAsBlend()
12835   case MVT::v2f64:  in lowerShuffleAsBlend()
12836   case MVT::v2i64:  in lowerShuffleAsBlend()
12837   case MVT::v4f32:  in lowerShuffleAsBlend()
12838   case MVT::v4i32:  in lowerShuffleAsBlend()
12839   case MVT::v8i16:  in lowerShuffleAsBlend()
12842                        DAG.getTargetConstant(BlendMask, DL, MVT::i8));  in lowerShuffleAsBlend()
12843   case MVT::v16i16: {  in lowerShuffleAsBlend()
12846     if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) {  in lowerShuffleAsBlend()
12853       return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,  in lowerShuffleAsBlend()
12854                          DAG.getTargetConstant(BlendMask, DL, MVT::i8));  in lowerShuffleAsBlend()
12862       SDValue Lo = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,  in lowerShuffleAsBlend()
12863                                DAG.getTargetConstant(LoMask, DL, MVT::i8));  in lowerShuffleAsBlend()
12864       SDValue Hi = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,  in lowerShuffleAsBlend()
12865                                DAG.getTargetConstant(HiMask, DL, MVT::i8));  in lowerShuffleAsBlend()
12867           MVT::v16i16, DL, Lo, Hi,  in lowerShuffleAsBlend()
12872   case MVT::v32i8:  in lowerShuffleAsBlend()
12875   case MVT::v16i8: {  in lowerShuffleAsBlend()
12884       MVT IntegerType = MVT::getIntegerVT(std::max<unsigned>(NumElts, 8));  in lowerShuffleAsBlend()
12900     MVT BlendVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);  in lowerShuffleAsBlend()
12926             Mask[i] < 0 ? DAG.getUNDEF(MVT::i8)  in lowerShuffleAsBlend()
12928                                           MVT::i8));  in lowerShuffleAsBlend()
12937   case MVT::v16f32:  in lowerShuffleAsBlend()
12938   case MVT::v8f64:  in lowerShuffleAsBlend()
12939   case MVT::v8i64:  in lowerShuffleAsBlend()
12940   case MVT::v16i32:  in lowerShuffleAsBlend()
12941   case MVT::v32i16:  in lowerShuffleAsBlend()
12942   case MVT::v64i8: {  in lowerShuffleAsBlend()
12953     MVT IntegerType = MVT::getIntegerVT(std::max<unsigned>(NumElts, 8));  in lowerShuffleAsBlend()
12967 static SDValue lowerShuffleAsBlendAndPermute(const SDLoc &DL, MVT VT,  in lowerShuffleAsBlendAndPermute()
13006 static SDValue lowerShuffleAsUNPCKAndPermute(const SDLoc &DL, MVT VT,  in lowerShuffleAsUNPCKAndPermute()
13070     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,  in lowerShuffleAsByteRotateAndPermute()
13125     MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);  in lowerShuffleAsByteRotateAndPermute()
13129                         DAG.getTargetConstant(Scale * RotAmt, DL, MVT::i8)));  in lowerShuffleAsByteRotateAndPermute()
13169     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,  in lowerShuffleAsDecomposedShuffleMerge()
13299 static int matchShuffleAsBitRotate(MVT &RotateVT, int EltSizeInBits,  in matchShuffleAsBitRotate()
13314     MVT RotateSVT = MVT::getIntegerVT(EltSizeInBits * NumSubElts);  in matchShuffleAsBitRotate()
13315     RotateVT = MVT::getVectorVT(RotateSVT, NumElts / NumSubElts);  in matchShuffleAsBitRotate()
13323 static SDValue lowerShuffleAsBitRotate(const SDLoc &DL, MVT VT, SDValue V1,  in lowerShuffleAsBitRotate()
13334   MVT RotateVT;  in lowerShuffleAsBitRotate()
13351                               DAG.getTargetConstant(ShlAmt, DL, MVT::i8));  in lowerShuffleAsBitRotate()
13353                               DAG.getTargetConstant(SrlAmt, DL, MVT::i8));  in lowerShuffleAsBitRotate()
13360                   DAG.getTargetConstant(RotateAmt, DL, MVT::i8));  in lowerShuffleAsBitRotate()
13452 static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,  in matchShuffleAsByteRotate()
13474 static SDValue lowerShuffleAsByteRotate(const SDLoc &DL, MVT VT, SDValue V1,  in lowerShuffleAsByteRotate()
13487   MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);  in lowerShuffleAsByteRotate()
13497                         DAG.getTargetConstant(ByteRotation, DL, MVT::i8)));  in lowerShuffleAsByteRotate()
13504   assert(ByteVT == MVT::v16i8 &&  in lowerShuffleAsByteRotate()
13512       DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Lo,  in lowerShuffleAsByteRotate()
13513                   DAG.getTargetConstant(LoByteShift, DL, MVT::i8));  in lowerShuffleAsByteRotate()
13515       DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Hi,  in lowerShuffleAsByteRotate()
13516                   DAG.getTargetConstant(HiByteShift, DL, MVT::i8));  in lowerShuffleAsByteRotate()
13518                         DAG.getNode(ISD::OR, DL, MVT::v16i8, LoShift, HiShift));  in lowerShuffleAsByteRotate()
13531 static SDValue lowerShuffleAsVALIGN(const SDLoc &DL, MVT VT, SDValue V1,  in lowerShuffleAsVALIGN()
13535   assert((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) &&  in lowerShuffleAsVALIGN()
13548                      DAG.getTargetConstant(Rotation, DL, MVT::i8));  in lowerShuffleAsVALIGN()
13552 static SDValue lowerShuffleAsByteShiftMask(const SDLoc &DL, MVT VT, SDValue V1,  in lowerShuffleAsByteShiftMask()
13579   Res = DAG.getBitcast(MVT::v16i8, Res);  in lowerShuffleAsByteShiftMask()
13588     Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,  in lowerShuffleAsByteShiftMask()
13589                       DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));  in lowerShuffleAsByteShiftMask()
13590     Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,  in lowerShuffleAsByteShiftMask()
13591                       DAG.getTargetConstant(Scale * ZeroHi, DL, MVT::i8));  in lowerShuffleAsByteShiftMask()
13594     Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,  in lowerShuffleAsByteShiftMask()
13595                       DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));  in lowerShuffleAsByteShiftMask()
13596     Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,  in lowerShuffleAsByteShiftMask()
13597                       DAG.getTargetConstant(Scale * ZeroLo, DL, MVT::i8));  in lowerShuffleAsByteShiftMask()
13603     Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,  in lowerShuffleAsByteShiftMask()
13604                       DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));  in lowerShuffleAsByteShiftMask()
13606     Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,  in lowerShuffleAsByteShiftMask()
13607                       DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));  in lowerShuffleAsByteShiftMask()
13608     Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,  in lowerShuffleAsByteShiftMask()
13609                       DAG.getTargetConstant(Scale * ZeroLo, DL, MVT::i8));  in lowerShuffleAsByteShiftMask()
13639 static int matchShuffleAsShift(MVT &ShiftVT, unsigned &Opcode,  in matchShuffleAsShift()
13675     MVT ShiftSVT = MVT::getIntegerVT(ScalarSizeInBits * Scale);  in matchShuffleAsShift()
13676     ShiftVT = ByteShift ? MVT::getVectorVT(MVT::i8, SizeInBits / 8)  in matchShuffleAsShift()
13677                         : MVT::getVectorVT(ShiftSVT, Size / Scale);  in matchShuffleAsShift()
13701 static SDValue lowerShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1,  in lowerShuffleAsShift()
13709   MVT ShiftVT;  in lowerShuffleAsShift()
13731                   DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));  in lowerShuffleAsShift()
13737 static bool matchShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2,  in matchShuffleAsEXTRQ()
13793 static bool matchShuffleAsINSERTQ(MVT VT, SDValue &V1, SDValue &V2,  in matchShuffleAsINSERTQ()
13859 static SDValue lowerShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1,  in lowerShuffleWithSSE4A()
13865                        DAG.getTargetConstant(BitLen, DL, MVT::i8),  in lowerShuffleWithSSE4A()
13866                        DAG.getTargetConstant(BitIdx, DL, MVT::i8));  in lowerShuffleWithSSE4A()
13871                        DAG.getTargetConstant(BitLen, DL, MVT::i8),  in lowerShuffleWithSSE4A()
13872                        DAG.getTargetConstant(BitIdx, DL, MVT::i8));  in lowerShuffleWithSSE4A()
13887     const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV,  in lowerShuffleAsSpecificZeroOrAnyExtend()
13926     MVT ExtVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Scale),  in lowerShuffleAsSpecificZeroOrAnyExtend()
13942         VT, DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,  in lowerShuffleAsSpecificZeroOrAnyExtend()
13943                         DAG.getBitcast(MVT::v4i32, InputV),  in lowerShuffleAsSpecificZeroOrAnyExtend()
13949     InputV = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,  in lowerShuffleAsSpecificZeroOrAnyExtend()
13950                          DAG.getBitcast(MVT::v4i32, InputV),  in lowerShuffleAsSpecificZeroOrAnyExtend()
13955         VT, DAG.getNode(OddEvenOp, DL, MVT::v8i16,  in lowerShuffleAsSpecificZeroOrAnyExtend()
13956                         DAG.getBitcast(MVT::v8i16, InputV),  in lowerShuffleAsSpecificZeroOrAnyExtend()
13968         MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,  in lowerShuffleAsSpecificZeroOrAnyExtend()
13969                                 DAG.getTargetConstant(EltBits, DL, MVT::i8),  in lowerShuffleAsSpecificZeroOrAnyExtend()
13970                                 DAG.getTargetConstant(LoIdx, DL, MVT::i8)));  in lowerShuffleAsSpecificZeroOrAnyExtend()
13977         MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,  in lowerShuffleAsSpecificZeroOrAnyExtend()
13978                                 DAG.getTargetConstant(EltBits, DL, MVT::i8),  in lowerShuffleAsSpecificZeroOrAnyExtend()
13979                                 DAG.getTargetConstant(HiIdx, DL, MVT::i8)));  in lowerShuffleAsSpecificZeroOrAnyExtend()
13981                           DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, Lo, Hi));  in lowerShuffleAsSpecificZeroOrAnyExtend()
13993         PSHUFBMask[i] = DAG.getConstant(Idx, DL, MVT::i8);  in lowerShuffleAsSpecificZeroOrAnyExtend()
13997           AnyExt ? DAG.getUNDEF(MVT::i8) : DAG.getConstant(0x80, DL, MVT::i8);  in lowerShuffleAsSpecificZeroOrAnyExtend()
13999     InputV = DAG.getBitcast(MVT::v16i8, InputV);  in lowerShuffleAsSpecificZeroOrAnyExtend()
14001         VT, DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV,  in lowerShuffleAsSpecificZeroOrAnyExtend()
14002                         DAG.getBuildVector(MVT::v16i8, DL, PSHUFBMask)));  in lowerShuffleAsSpecificZeroOrAnyExtend()
14024     MVT InputVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits), NumElements);  in lowerShuffleAsSpecificZeroOrAnyExtend()
14049     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,  in lowerShuffleAsZeroOrAnyExtend()
14155     V = DAG.getBitcast(MVT::v2i64, V);  in lowerShuffleAsZeroOrAnyExtend()
14156     V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v2i64, V);  in lowerShuffleAsZeroOrAnyExtend()
14169   MVT VT = V.getSimpleValueType();  in getScalarValueForVectorElement()
14170   MVT EltVT = VT.getVectorElementType();  in getScalarValueForVectorElement()
14175   MVT NewVT = V.getSimpleValueType();  in getScalarValueForVectorElement()
14202   return VT.getScalarType() == MVT::f16 && !Subtarget.hasFP16();  in isSoftFP16()
14215     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,  in lowerShuffleAsElementInsertion()
14218   MVT ExtVT = VT;  in lowerShuffleAsElementInsertion()
14219   MVT EltVT = VT.getVectorElementType();  in lowerShuffleAsElementInsertion()
14244     if (EltVT == MVT::i8 || (EltVT == MVT::i16 && !Subtarget.hasFP16())) {  in lowerShuffleAsElementInsertion()
14251       ExtVT = MVT::getVectorVT(MVT::i32, ExtVT.getSizeInBits() / 32);  in lowerShuffleAsElementInsertion()
14252       V2S = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, V2S);  in lowerShuffleAsElementInsertion()
14255   } else if (Mask[V2Index] != (int)Mask.size() || EltVT == MVT::i8 ||  in lowerShuffleAsElementInsertion()
14256              EltVT == MVT::i16) {  in lowerShuffleAsElementInsertion()
14278     if (EltVT == MVT::f16)  in lowerShuffleAsElementInsertion()
14280     else if (EltVT == MVT::f32)  in lowerShuffleAsElementInsertion()
14282     else if (EltVT == MVT::f64)  in lowerShuffleAsElementInsertion()
14307       V2 = DAG.getBitcast(MVT::v16i8, V2);  in lowerShuffleAsElementInsertion()
14308       V2 = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, V2,  in lowerShuffleAsElementInsertion()
14310                            V2Index * EltVT.getSizeInBits() / 8, DL, MVT::i8));  in lowerShuffleAsElementInsertion()
14321 static SDValue lowerShuffleAsTruncBroadcast(const SDLoc &DL, MVT VT, SDValue V0,  in lowerShuffleAsTruncBroadcast()
14328   MVT EltVT = VT.getVectorElementType();  in lowerShuffleAsTruncBroadcast()
14329   MVT V0VT = V0.getSimpleValueType();  in lowerShuffleAsTruncBroadcast()
14334   MVT V0EltVT = V0VT.getVectorElementType();  in lowerShuffleAsTruncBroadcast()
14364                          DAG.getConstant(OffsetIdx * EltSize, DL, MVT::i8));  in lowerShuffleAsTruncBroadcast()
14398   MVT VT = N0.getSimpleValueType();  in lowerShuffleOfExtractsAsVperm()
14411   MVT WideVT = WideVec.getSimpleValueType();  in lowerShuffleOfExtractsAsVperm()
14449 static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1,  in lowerShuffleAsBroadcast()
14453   if (!((Subtarget.hasSSE3() && VT == MVT::v2f64) ||  in lowerShuffleAsBroadcast()
14461   unsigned Opcode = (VT == MVT::v2f64 && !Subtarget.hasAVX2())  in lowerShuffleAsBroadcast()
14554     MVT SVT = VT.getScalarType();  in lowerShuffleAsBroadcast()
14564       SDVTList Tys = DAG.getVTList(VT, MVT::Other);  in lowerShuffleAsBroadcast()
14573     assert(SVT == MVT::f64 && "Unexpected VT!");  in lowerShuffleAsBroadcast()
14589     if (VT == MVT::v4f64 || VT == MVT::v4i64)  in lowerShuffleAsBroadcast()
14606     V = DAG.getBitcast(MVT::f64, V);  in lowerShuffleAsBroadcast()
14608       V = DAG.getNode(X86ISD::VBROADCAST, DL, MVT::v2f64, V);  in lowerShuffleAsBroadcast()
14611     V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V);  in lowerShuffleAsBroadcast()
14618     MVT BroadcastVT = MVT::getVectorVT(V.getSimpleValueType(),  in lowerShuffleAsBroadcast()
14632   MVT CastVT = MVT::getVectorVT(VT.getVectorElementType(), NumSrcElts);  in lowerShuffleAsBroadcast()
14706       VA = DAG.getUNDEF(MVT::v4f32);  in matchShuffleAsInsertPS()
14733   assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");  in lowerShuffleAsInsertPS()
14734   assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");  in lowerShuffleAsInsertPS()
14742   return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,  in lowerShuffleAsInsertPS()
14743                      DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));  in lowerShuffleAsInsertPS()
14756     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,  in lowerShuffleAsPermuteAndUnpack()
14809     MVT UnpackVT = MVT::getVectorVT(MVT::getIntegerVT(ScalarSize), Size / Scale);  in lowerShuffleAsPermuteAndUnpack()
14874   assert(V1.getSimpleValueType() == MVT::v2f64 && "Bad operand type!");  in lowerV2F64Shuffle()
14875   assert(V2.getSimpleValueType() == MVT::v2f64 && "Bad operand type!");  in lowerV2F64Shuffle()
14880     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2f64, V1, V2,  in lowerV2F64Shuffle()
14891       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1,  in lowerV2F64Shuffle()
14892                          DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));  in lowerV2F64Shuffle()
14896         X86ISD::SHUFP, DL, MVT::v2f64,  in lowerV2F64Shuffle()
14897         Mask[0] == SM_SentinelUndef ? DAG.getUNDEF(MVT::v2f64) : V1,  in lowerV2F64Shuffle()
14898         Mask[1] == SM_SentinelUndef ? DAG.getUNDEF(MVT::v2f64) : V1,  in lowerV2F64Shuffle()
14899         DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));  in lowerV2F64Shuffle()
14913           DL, MVT::v2f64, V1, V2, Mask, Zeroable, Subtarget, DAG))  in lowerV2F64Shuffle()
14920           DL, MVT::v2f64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))  in lowerV2F64Shuffle()
14931           X86ISD::MOVSD, DL, MVT::v2f64, V2,  in lowerV2F64Shuffle()
14932           DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V1S));  in lowerV2F64Shuffle()
14935     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask,  in lowerV2F64Shuffle()
14940   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2f64, Mask, V1, V2, DAG))  in lowerV2F64Shuffle()
14944   return DAG.getNode(X86ISD::SHUFP, DL, MVT::v2f64, V1, V2,  in lowerV2F64Shuffle()
14945                      DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));  in lowerV2F64Shuffle()
14958   assert(V1.getSimpleValueType() == MVT::v2i64 && "Bad operand type!");  in lowerV2I64Shuffle()
14959   assert(V2.getSimpleValueType() == MVT::v2i64 && "Bad operand type!");  in lowerV2I64Shuffle()
14964     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2i64, V1, V2,  in lowerV2I64Shuffle()
14971     V1 = DAG.getBitcast(MVT::v4i32, V1);  in lowerV2I64Shuffle()
14977         MVT::v2i64,  in lowerV2I64Shuffle()
14978         DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,  in lowerV2I64Shuffle()
14991   if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v2i64, V1, V2, Mask,  in lowerV2I64Shuffle()
14998           DL, MVT::v2i64, V1, V2, Mask, Zeroable, Subtarget, DAG))  in lowerV2I64Shuffle()
15004           DL, MVT::v2i64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))  in lowerV2I64Shuffle()
15011     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2i64, V1, V2, Mask,  in lowerV2I64Shuffle()
15016   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2i64, Mask, V1, V2, DAG))  in lowerV2I64Shuffle()
15023       if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v2i64, V1, V2, Mask,  in lowerV2I64Shuffle()
15027     if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v2i64, V1, V2, Mask,  in lowerV2I64Shuffle()
15035     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v2i64, V1, V2, Mask,  in lowerV2I64Shuffle()
15042   V1 = DAG.getBitcast(MVT::v2f64, V1);  in lowerV2I64Shuffle()
15043   V2 = DAG.getBitcast(MVT::v2f64, V2);  in lowerV2I64Shuffle()
15044   return DAG.getBitcast(MVT::v2i64,  in lowerV2I64Shuffle()
15045                         DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask));  in lowerV2I64Shuffle()
15053 static SDValue lowerShuffleWithSHUFPS(const SDLoc &DL, MVT VT,  in lowerShuffleWithSHUFPS()
15149   assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");  in lowerV4F32Shuffle()
15150   assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");  in lowerV4F32Shuffle()
15157     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f32, V1, V2,  in lowerV4F32Shuffle()
15164         return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v4f32, V1);  in lowerV4F32Shuffle()
15166         return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v4f32, V1);  in lowerV4F32Shuffle()
15172       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f32, V1,  in lowerV4F32Shuffle()
15180         return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V1);  in lowerV4F32Shuffle()
15182         return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V1, V1);  in lowerV4F32Shuffle()
15187     return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1,  in lowerV4F32Shuffle()
15202             DL, MVT::v4f32, V1, V2, Mask, Zeroable, Subtarget, DAG))  in lowerV4F32Shuffle()
15206     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f32, V1, V2, Mask,  in lowerV4F32Shuffle()
15215       if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, MVT::v4f32, V1,  in lowerV4F32Shuffle()
15224       return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V2);  in lowerV4F32Shuffle()
15226       return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V2, V1);  in lowerV4F32Shuffle()
15230   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f32, Mask, V1, V2, DAG))  in lowerV4F32Shuffle()
15234   return lowerShuffleWithSHUFPS(DL, MVT::v4f32, Mask, V1, V2, DAG);  in lowerV4F32Shuffle()
15245   assert(V1.getSimpleValueType() == MVT::v4i32 && "Bad operand type!");  in lowerV4I32Shuffle()
15246   assert(V2.getSimpleValueType() == MVT::v4i32 && "Bad operand type!");  in lowerV4I32Shuffle()
15252   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v4i32, V1, V2, Mask,  in lowerV4I32Shuffle()
15261       if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i32, V1, V2,  in lowerV4I32Shuffle()
15278     return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,  in lowerV4I32Shuffle()
15287   if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask,  in lowerV4I32Shuffle()
15294             DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG))  in lowerV4I32Shuffle()
15301     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i32, V1, V2, Mask,  in lowerV4I32Shuffle()
15305   if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v4i32, V1, V2, Mask,  in lowerV4I32Shuffle()
15310   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i32, Mask, V1, V2, DAG))  in lowerV4I32Shuffle()
15317       if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v4i32, V1, V2, Mask,  in lowerV4I32Shuffle()
15321     if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i32, V1, V2, Mask,  in lowerV4I32Shuffle()
15333       return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i32, V1, V2, Mask,  in lowerV4I32Shuffle()
15337     if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v4i32, V1, V2,  in lowerV4I32Shuffle()
15347   SDValue CastV1 = DAG.getBitcast(MVT::v4f32, V1);  in lowerV4I32Shuffle()
15348   SDValue CastV2 = DAG.getBitcast(MVT::v4f32, V2);  in lowerV4I32Shuffle()
15349   SDValue ShufPS = DAG.getVectorShuffle(MVT::v4f32, DL, CastV1, CastV2, Mask);  in lowerV4I32Shuffle()
15350   return DAG.getBitcast(MVT::v4i32, ShufPS);  in lowerV4I32Shuffle()
15370     const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef<int> Mask,  in lowerV8I16GeneralSingleInputShuffle()
15372   assert(VT.getVectorElementType() == MVT::i16 && "Bad input type!");  in lowerV8I16GeneralSingleInputShuffle()
15373   MVT PSHUFDVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2);  in lowerV8I16GeneralSingleInputShuffle()
15567               MVT::getVectorVT(MVT::i16, V.getValueSizeInBits() / 16), V,  in lowerV8I16GeneralSingleInputShuffle()
15864     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,  in lowerShuffleAsBlendOfPSHUFBs()
15873   SmallVector<SDValue, 64> V1Mask(NumBytes, DAG.getUNDEF(MVT::i8));  in lowerShuffleAsBlendOfPSHUFBs()
15874   SmallVector<SDValue, 64> V2Mask(NumBytes, DAG.getUNDEF(MVT::i8));  in lowerShuffleAsBlendOfPSHUFBs()
15889     V1Mask[i] = DAG.getConstant(V1Idx, DL, MVT::i8);  in lowerShuffleAsBlendOfPSHUFBs()
15890     V2Mask[i] = DAG.getConstant(V2Idx, DL, MVT::i8);  in lowerShuffleAsBlendOfPSHUFBs()
15895   MVT ShufVT = MVT::getVectorVT(MVT::i8, NumBytes);  in lowerShuffleAsBlendOfPSHUFBs()
15930   assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad operand type!");  in lowerV8I16Shuffle()
15931   assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad operand type!");  in lowerV8I16Shuffle()
15936   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i16, V1, V2, Mask,  in lowerV8I16Shuffle()
15941   if (SDValue V = lowerShuffleWithVPMOV(DL, MVT::v8i16, V1, V2, Mask, Zeroable,  in lowerV8I16Shuffle()
15949     if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i16, V1, V1, Mask,  in lowerV8I16Shuffle()
15954     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i16, V1, V2,  in lowerV8I16Shuffle()
15959     if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v8i16, V1, Mask,  in lowerV8I16Shuffle()
15964     if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))  in lowerV8I16Shuffle()
15968     if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,  in lowerV8I16Shuffle()
15973     if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V1, Mask,  in lowerV8I16Shuffle()
15979     return lowerV8I16GeneralSingleInputShuffle(DL, MVT::v8i16, V1, MutableMask,  in lowerV8I16Shuffle()
15988   if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i16, V1, V2, Mask,  in lowerV8I16Shuffle()
15994     if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v8i16, V1, V2, Mask,  in lowerV8I16Shuffle()
16001             DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG))  in lowerV8I16Shuffle()
16008     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i16, V1, V2, Mask,  in lowerV8I16Shuffle()
16012   if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v8i16, V1, V2, Mask,  in lowerV8I16Shuffle()
16017   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))  in lowerV8I16Shuffle()
16021   if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,  in lowerV8I16Shuffle()
16026   if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v8i16, V1, V2, Mask, Zeroable,  in lowerV8I16Shuffle()
16031   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V2, Mask,  in lowerV8I16Shuffle()
16036           lowerShuffleAsBitBlend(DL, MVT::v8i16, V1, V2, Mask, DAG))  in lowerV8I16Shuffle()
16040   if (SDValue V = lowerShuffleAsByteShiftMask(DL, MVT::v8i16, V1, V2, Mask,  in lowerV8I16Shuffle()
16055       V1V2 = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1V2,  in lowerV8I16Shuffle()
16056                          getZeroVector(MVT::v16i16, Subtarget, DAG, DL),  in lowerV8I16Shuffle()
16057                          DAG.getTargetConstant(0xEE, DL, MVT::i8));  in lowerV8I16Shuffle()
16058       V1V2 = DAG.getBitcast(MVT::v8i32, V1V2);  in lowerV8I16Shuffle()
16063                                             DAG.getConstant(0, DL, MVT::i32));  in lowerV8I16Shuffle()
16065         DWordClearOps[i] = DAG.getConstant(0xFFFF, DL, MVT::i32);  in lowerV8I16Shuffle()
16067           DAG.getBuildVector(MVT::v4i32, DL, DWordClearOps);  in lowerV8I16Shuffle()
16068       V1 = DAG.getNode(ISD::AND, DL, MVT::v4i32, DAG.getBitcast(MVT::v4i32, V1),  in lowerV8I16Shuffle()
16070       V2 = DAG.getNode(ISD::AND, DL, MVT::v4i32, DAG.getBitcast(MVT::v4i32, V2),  in lowerV8I16Shuffle()
16074     SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v8i16, V1, V2);  in lowerV8I16Shuffle()
16076       Result = DAG.getBitcast(MVT::v4i32, Result);  in lowerV8I16Shuffle()
16077       Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v8i16, Result, Result);  in lowerV8I16Shuffle()
16086     V1 = DAG.getNode(HasSSE41 ? X86ISD::VSRLI : X86ISD::VSRAI, DL, MVT::v4i32,  in lowerV8I16Shuffle()
16087                      DAG.getBitcast(MVT::v4i32, V1),  in lowerV8I16Shuffle()
16088                      DAG.getTargetConstant(16, DL, MVT::i8));  in lowerV8I16Shuffle()
16089     V2 = DAG.getNode(HasSSE41 ? X86ISD::VSRLI : X86ISD::VSRAI, DL, MVT::v4i32,  in lowerV8I16Shuffle()
16090                      DAG.getBitcast(MVT::v4i32, V2),  in lowerV8I16Shuffle()
16091                      DAG.getTargetConstant(16, DL, MVT::i8));  in lowerV8I16Shuffle()
16093                        MVT::v8i16, V1, V2);  in lowerV8I16Shuffle()
16097   if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v8i16, V1, V2,  in lowerV8I16Shuffle()
16105     return lowerShuffleAsBlendOfPSHUFBs(DL, MVT::v8i16, V1, V2, Mask,  in lowerV8I16Shuffle()
16111   return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8i16, V1, V2,  in lowerV8I16Shuffle()
16120   assert(V1.getSimpleValueType() == MVT::v8f16 && "Bad operand type!");  in lowerV8F16Shuffle()
16121   assert(V2.getSimpleValueType() == MVT::v8f16 && "Bad operand type!");  in lowerV8F16Shuffle()
16128       if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8f16, V1, V2,  in lowerV8F16Shuffle()
16134               DL, MVT::v8f16, V1, V2, Mask, Zeroable, Subtarget, DAG))  in lowerV8F16Shuffle()
16138   V1 = DAG.getBitcast(MVT::v8i16, V1);  in lowerV8F16Shuffle()
16139   V2 = DAG.getBitcast(MVT::v8i16, V2);  in lowerV8F16Shuffle()
16140   return DAG.getBitcast(MVT::v8f16,  in lowerV8F16Shuffle()
16141                         DAG.getVectorShuffle(MVT::v8i16, DL, V1, V2, Mask));  in lowerV8F16Shuffle()
16147 static SDValue lowerShuffleWithPERMV(const SDLoc &DL, MVT VT,  in lowerShuffleWithPERMV()
16151   MVT MaskVT = VT.changeTypeToInteger();  in lowerShuffleWithPERMV()
16153   MVT ShuffleVT = VT;  in lowerShuffleWithPERMV()
16195   assert(V1.getSimpleValueType() == MVT::v16i8 && "Bad operand type!");  in lowerV16I8Shuffle()
16196   assert(V2.getSimpleValueType() == MVT::v16i8 && "Bad operand type!");  in lowerV16I8Shuffle()
16200   if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v16i8, V1, V2, Mask,  in lowerV16I8Shuffle()
16205   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i8, V1, V2, Mask,  in lowerV16I8Shuffle()
16210   if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i8, Mask, V1, V2, DAG,  in lowerV16I8Shuffle()
16215   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v16i8, V1, V2, Mask,  in lowerV16I8Shuffle()
16220   if (SDValue V = lowerShuffleWithVPMOV(DL, MVT::v16i8, V1, V2, Mask, Zeroable,  in lowerV16I8Shuffle()
16224   if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v16i8, V1, V2, Mask, Zeroable,  in lowerV16I8Shuffle()
16230     if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask,  in lowerV16I8Shuffle()
16239     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i8, V1, V2,  in lowerV16I8Shuffle()
16244     if (SDValue Rotate = lowerShuffleAsBitRotate(DL, MVT::v16i8, V1, Mask,  in lowerV16I8Shuffle()
16248     if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))  in lowerV16I8Shuffle()
16313           MVT::v16i8,  in lowerV16I8Shuffle()
16314           DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1),  in lowerV16I8Shuffle()
16315                                DAG.getUNDEF(MVT::v8i16), PreDupI16Shuffle));  in lowerV16I8Shuffle()
16326                        MVT::v16i8, EvenInUse ? V1 : DAG.getUNDEF(MVT::v16i8),  in lowerV16I8Shuffle()
16327                        OddInUse ? V1 : DAG.getUNDEF(MVT::v16i8));  in lowerV16I8Shuffle()
16341           MVT::v16i8,  in lowerV16I8Shuffle()
16342           DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1),  in lowerV16I8Shuffle()
16343                                DAG.getUNDEF(MVT::v8i16), PostDupI16Shuffle));  in lowerV16I8Shuffle()
16349   if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v16i8, V1, V2, Mask,  in lowerV16I8Shuffle()
16354   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))  in lowerV16I8Shuffle()
16358   if (SDValue V = lowerShuffleAsByteShiftMask(DL, MVT::v16i8, V1, V2, Mask,  in lowerV16I8Shuffle()
16387         DL, MVT::v16i8, V1, V2, Mask, Zeroable, DAG, V1InUse, V2InUse);  in lowerV16I8Shuffle()
16394         if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i8, V1, V2, Mask,  in lowerV16I8Shuffle()
16407               DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))  in lowerV16I8Shuffle()
16412         return lowerShuffleWithPERMV(DL, MVT::v16i8, Mask, V1, V2, Subtarget,  in lowerV16I8Shuffle()
16417         SDValue MaskNode = getConstVector(Mask, MVT::v16i8, DAG, DL, true);  in lowerV16I8Shuffle()
16418         return DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, V1, V2, MaskNode);  in lowerV16I8Shuffle()
16424               DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))  in lowerV16I8Shuffle()
16434             DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG))  in lowerV16I8Shuffle()
16437   if (SDValue Blend = lowerShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG))  in lowerV16I8Shuffle()
16455     SmallVector<SDValue, 8> WordClearOps(8, DAG.getConstant(0, DL, MVT::i16));  in lowerV16I8Shuffle()
16457       WordClearOps[i] = DAG.getConstant(0xFF, DL, MVT::i16);  in lowerV16I8Shuffle()
16458     SDValue WordClearMask = DAG.getBuildVector(MVT::v8i16, DL, WordClearOps);  in lowerV16I8Shuffle()
16459     V1 = DAG.getNode(ISD::AND, DL, MVT::v8i16, DAG.getBitcast(MVT::v8i16, V1),  in lowerV16I8Shuffle()
16462       V2 = DAG.getNode(ISD::AND, DL, MVT::v8i16, DAG.getBitcast(MVT::v8i16, V2),  in lowerV16I8Shuffle()
16466     SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1,  in lowerV16I8Shuffle()
16469       Result = DAG.getBitcast(MVT::v8i16, Result);  in lowerV16I8Shuffle()
16470       Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result);  in lowerV16I8Shuffle()
16477     V1 = DAG.getNode(X86ISD::VSRLI, DL, MVT::v8i16,  in lowerV16I8Shuffle()
16478                      DAG.getBitcast(MVT::v8i16, V1),  in lowerV16I8Shuffle()
16479                      DAG.getTargetConstant(8, DL, MVT::i8));  in lowerV16I8Shuffle()
16481       V2 = DAG.getNode(X86ISD::VSRLI, DL, MVT::v8i16,  in lowerV16I8Shuffle()
16482                        DAG.getBitcast(MVT::v8i16, V2),  in lowerV16I8Shuffle()
16483                        DAG.getTargetConstant(8, DL, MVT::i8));  in lowerV16I8Shuffle()
16484     return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1,  in lowerV16I8Shuffle()
16490     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v16i8, V1, V2, Mask,  in lowerV16I8Shuffle()
16511     VLoHalf = DAG.getBitcast(MVT::v8i16, V);  in lowerV16I8Shuffle()
16512     VLoHalf = DAG.getNode(ISD::AND, DL, MVT::v8i16, VLoHalf,  in lowerV16I8Shuffle()
16513                           DAG.getConstant(0x00FF, DL, MVT::v8i16));  in lowerV16I8Shuffle()
16516     VHiHalf = DAG.getUNDEF(MVT::v8i16);  in lowerV16I8Shuffle()
16528     SDValue Zero = getZeroVector(MVT::v16i8, Subtarget, DAG, DL);  in lowerV16I8Shuffle()
16531         MVT::v8i16, DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero));  in lowerV16I8Shuffle()
16533         MVT::v8i16, DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero));  in lowerV16I8Shuffle()
16536   SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask);  in lowerV16I8Shuffle()
16537   SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask);  in lowerV16I8Shuffle()
16539   return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV);  in lowerV16I8Shuffle()
16547                                   MVT VT, SDValue V1, SDValue V2,  in lower128BitShuffle()
16552   case MVT::v2i64:  in lower128BitShuffle()
16554   case MVT::v2f64:  in lower128BitShuffle()
16556   case MVT::v4i32:  in lower128BitShuffle()
16558   case MVT::v4f32:  in lower128BitShuffle()
16560   case MVT::v8i16:  in lower128BitShuffle()
16562   case MVT::v8f16:  in lower128BitShuffle()
16564   case MVT::v16i8:  in lower128BitShuffle()
16577 static SDValue splitAndLowerShuffle(const SDLoc &DL, MVT VT, SDValue V1,  in splitAndLowerShuffle()
16590   MVT ScalarVT = VT.getVectorElementType();  in splitAndLowerShuffle()
16591   MVT SplitVT = MVT::getVectorVT(ScalarVT, SplitNumElements);  in splitAndLowerShuffle()
16679 static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1,  in lowerShuffleAsSplitOrBlend()
16732 static SDValue lowerShuffleAsLanePermuteAndSHUFP(const SDLoc &DL, MVT VT,  in lowerShuffleAsLanePermuteAndSHUFP()
16736   assert(VT == MVT::v4f64 && "Only for v4f64 shuffles");  in lowerShuffleAsLanePermuteAndSHUFP()
16757                      DAG.getTargetConstant(SHUFPMask, DL, MVT::i8));  in lowerShuffleAsLanePermuteAndSHUFP()
16769     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,  in lowerShuffleAsLanePermuteAndPermute()
16872     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,  in lowerShuffleAsLanePermuteAndShuffle()
16882   if (VT == MVT::v4f64 &&  in lowerShuffleAsLanePermuteAndShuffle()
16925   MVT PVT = VT.isFloatingPoint() ? MVT::v4f64 : MVT::v4i64;  in lowerShuffleAsLanePermuteAndShuffle()
16934 static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1,  in lowerV2X128Shuffle()
16945       MVT MemVT = VT.getHalfNumVectorElementsVT();  in lowerV2X128Shuffle()
16969     MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2);  in lowerV2X128Shuffle()
16997         MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2);  in lowerV2X128Shuffle()
17012                            DAG.getTargetConstant(PermMask, DL, MVT::i8));  in lowerV2X128Shuffle()
17044                      DAG.getTargetConstant(PermMask, DL, MVT::i8));  in lowerV2X128Shuffle()
17055     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,  in lowerShuffleAsLanePermuteAndRepeatedMask()
17281   MVT VT = V1.getSimpleValueType();  in getShuffleHalfVectors()
17282   MVT HalfVT = VT.getHalfNumVectorElementsVT();  in getShuffleHalfVectors()
17314 static SDValue lowerShuffleWithUndefHalf(const SDLoc &DL, MVT VT, SDValue V1,  in lowerShuffleWithUndefHalf()
17330   MVT HalfVT = VT.getHalfNumVectorElementsVT();  in lowerShuffleWithUndefHalf()
17441     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,  in lowerShuffleAsRepeatedMaskAndLanePermute()
17615         (!OnlyLowestElts && V2.isUndef() && VT == MVT::v32i8) ? 4 : 2;  in lowerShuffleAsRepeatedMaskAndLanePermute()
17617   if (Subtarget.hasBWI() && VT == MVT::v64i8)  in lowerShuffleAsRepeatedMaskAndLanePermute()
17627 static bool matchShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2,  in matchShuffleWithSHUFPD()
17672 static SDValue lowerShuffleWithSHUFPD(const SDLoc &DL, MVT VT, SDValue V1,  in lowerShuffleWithSHUFPD()
17677   assert((VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v8f64) &&  in lowerShuffleWithSHUFPD()
17693                      DAG.getTargetConstant(Immediate, DL, MVT::i8));  in lowerShuffleWithSHUFPD()
17699 static SDValue lowerShuffleAsVTRUNCAndUnpack(const SDLoc &DL, MVT VT,  in lowerShuffleAsVTRUNCAndUnpack()
17704   assert(VT == MVT::v32i8 && "Unexpected type!");  in lowerShuffleAsVTRUNCAndUnpack()
17714   V1 = DAG.getBitcast(MVT::v4i64, V1);  in lowerShuffleAsVTRUNCAndUnpack()
17715   V2 = DAG.getBitcast(MVT::v4i64, V2);  in lowerShuffleAsVTRUNCAndUnpack()
17717   V1 = DAG.getNode(X86ISD::VTRUNC, DL, MVT::v16i8, V1);  in lowerShuffleAsVTRUNCAndUnpack()
17718   V2 = DAG.getNode(X86ISD::VTRUNC, DL, MVT::v16i8, V2);  in lowerShuffleAsVTRUNCAndUnpack()
17722   SDValue Unpack = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2,  in lowerShuffleAsVTRUNCAndUnpack()
17726   return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v32i8,  in lowerShuffleAsVTRUNCAndUnpack()
17727                      DAG.getConstant(0, DL, MVT::v32i8), Unpack,  in lowerShuffleAsVTRUNCAndUnpack()
17740   assert(V1.getSimpleValueType() == MVT::v4f64 && "Bad operand type!");  in lowerV4F64Shuffle()
17741   assert(V2.getSimpleValueType() == MVT::v4f64 && "Bad operand type!");  in lowerV4F64Shuffle()
17744   if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4f64, V1, V2, Mask, Zeroable,  in lowerV4F64Shuffle()
17750     if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f64, V1, V2,  in lowerV4F64Shuffle()
17756       return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v4f64, V1);  in lowerV4F64Shuffle()
17758     if (!is128BitLaneCrossingShuffleMask(MVT::v4f64, Mask)) {  in lowerV4F64Shuffle()
17763       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1,  in lowerV4F64Shuffle()
17764                          DAG.getTargetConstant(VPERMILPMask, DL, MVT::i8));  in lowerV4F64Shuffle()
17769       return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4f64, V1,  in lowerV4F64Shuffle()
17775             DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))  in lowerV4F64Shuffle()
17779     if (SDValue V = lowerShuffleAsLanePermuteAndPermute(DL, MVT::v4f64, V1, V2,  in lowerV4F64Shuffle()
17784     return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v4f64, V1, V2, Mask,  in lowerV4F64Shuffle()
17789   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f64, Mask, V1, V2, DAG))  in lowerV4F64Shuffle()
17792   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask,  in lowerV4F64Shuffle()
17797   if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v4f64, V1, V2, Mask,  in lowerV4F64Shuffle()
17808   if (is128BitLaneCrossingShuffleMask(MVT::v4f64, Mask) &&  in lowerV4F64Shuffle()
17812     return lowerShuffleAsLanePermuteAndSHUFP(DL, MVT::v4f64, V1, V2, Mask, DAG);  in lowerV4F64Shuffle()
17817     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4f64, V1, V2, Mask,  in lowerV4F64Shuffle()
17823           DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))  in lowerV4F64Shuffle()
17832             DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))  in lowerV4F64Shuffle()
17837     if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v4f64, Zeroable, Mask, V1, V2,  in lowerV4F64Shuffle()
17844     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4f64, V1, V2, Mask,  in lowerV4F64Shuffle()
17848   return lowerShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask,  in lowerV4F64Shuffle()
17860   assert(V1.getSimpleValueType() == MVT::v4i64 && "Bad operand type!");  in lowerV4I64Shuffle()
17861   assert(V2.getSimpleValueType() == MVT::v4i64 && "Bad operand type!");  in lowerV4I64Shuffle()
17865   if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4i64, V1, V2, Mask, Zeroable,  in lowerV4I64Shuffle()
17869   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i64, V1, V2, Mask,  in lowerV4I64Shuffle()
17874   if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i64, V1, V2, Mask,  in lowerV4I64Shuffle()
17882     if (is128BitLaneRepeatedShuffleMask(MVT::v4i64, Mask, RepeatedMask)) {  in lowerV4I64Shuffle()
17886           MVT::v4i64,  in lowerV4I64Shuffle()
17887           DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32,  in lowerV4I64Shuffle()
17888                       DAG.getBitcast(MVT::v8i32, V1),  in lowerV4I64Shuffle()
17894     return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64, V1,  in lowerV4I64Shuffle()
17899   if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask,  in lowerV4I64Shuffle()
17905     if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v4i64, V1, V2, Mask,  in lowerV4I64Shuffle()
17909     if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v4i64, Zeroable, Mask, V1, V2,  in lowerV4I64Shuffle()
17915   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i64, V1, V2, Mask,  in lowerV4I64Shuffle()
17920   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i64, Mask, V1, V2, DAG))  in lowerV4I64Shuffle()
17929     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i64, V1, V2, Mask,  in lowerV4I64Shuffle()
17935           DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))  in lowerV4I64Shuffle()
17940           lowerShuffleAsBlendAndPermute(DL, MVT::v4i64, V1, V2, Mask, DAG))  in lowerV4I64Shuffle()
17949             DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))  in lowerV4I64Shuffle()
17953   return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i64, V1, V2, Mask,  in lowerV4I64Shuffle()
17965   assert(V1.getSimpleValueType() == MVT::v8f32 && "Bad operand type!");  in lowerV8F32Shuffle()
17966   assert(V2.getSimpleValueType() == MVT::v8f32 && "Bad operand type!");  in lowerV8F32Shuffle()
17969   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask,  in lowerV8F32Shuffle()
17974   if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8f32, V1, V2, Mask,  in lowerV8F32Shuffle()
17981   if (is128BitLaneRepeatedShuffleMask(MVT::v8f32, Mask, RepeatedMask)) {  in lowerV8F32Shuffle()
17987       return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v8f32, V1);  in lowerV8F32Shuffle()
17989       return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v8f32, V1);  in lowerV8F32Shuffle()
17992       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, V1,  in lowerV8F32Shuffle()
17996     if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8f32, Mask, V1, V2, DAG))  in lowerV8F32Shuffle()
18001     return lowerShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask, V1, V2, DAG);  in lowerV8F32Shuffle()
18007           DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))  in lowerV8F32Shuffle()
18013     if (!is128BitLaneCrossingShuffleMask(MVT::v8f32, Mask)) {  in lowerV8F32Shuffle()
18014       SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);  in lowerV8F32Shuffle()
18015       return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, V1, VPermMask);  in lowerV8F32Shuffle()
18018       SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);  in lowerV8F32Shuffle()
18019       return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8f32, VPermMask, V1);  in lowerV8F32Shuffle()
18022     return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v8f32, V1, V2, Mask,  in lowerV8F32Shuffle()
18029           DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))  in lowerV8F32Shuffle()
18034     if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8f32, Zeroable, Mask, V1, V2,  in lowerV8F32Shuffle()
18041   if (!Subtarget.hasAVX512() && isUnpackWdShuffleMask(Mask, MVT::v8f32, DAG))  in lowerV8F32Shuffle()
18042     return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, Subtarget,  in lowerV8F32Shuffle()
18048     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8f32, V1, V2, Mask,  in lowerV8F32Shuffle()
18052   return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask,  in lowerV8F32Shuffle()
18064   assert(V1.getSimpleValueType() == MVT::v8i32 && "Bad operand type!");  in lowerV8I32Shuffle()
18065   assert(V2.getSimpleValueType() == MVT::v8i32 && "Bad operand type!");  in lowerV8I32Shuffle()
18072   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2, Mask,  in lowerV8I32Shuffle()
18079   if (isUnpackWdShuffleMask(Mask, MVT::v8i32, DAG) && !V2.isUndef() &&  in lowerV8I32Shuffle()
18081     return lowerShuffleAsSplitOrBlend(DL, MVT::v8i32, V1, V2, Mask, Subtarget,  in lowerV8I32Shuffle()
18084   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask,  in lowerV8I32Shuffle()
18089   if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i32, V1, V2, Mask,  in lowerV8I32Shuffle()
18098       is128BitLaneRepeatedShuffleMask(MVT::v8i32, Mask, RepeatedMask);  in lowerV8I32Shuffle()
18102       return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, V1,  in lowerV8I32Shuffle()
18106     if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i32, Mask, V1, V2, DAG))  in lowerV8I32Shuffle()
18111   if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask,  in lowerV8I32Shuffle()
18117     if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v8i32, V1, V2, Mask,  in lowerV8I32Shuffle()
18121     if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i32, Zeroable, Mask, V1, V2,  in lowerV8I32Shuffle()
18127   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i32, V1, V2, Mask,  in lowerV8I32Shuffle()
18134           DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))  in lowerV8I32Shuffle()
18140     if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v8i32, Mask, V1, V2, DAG))  in lowerV8I32Shuffle()
18145     SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);  in lowerV8I32Shuffle()
18146     return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8i32, VPermMask, V1);  in lowerV8I32Shuffle()
18153     SDValue CastV1 = DAG.getBitcast(MVT::v8f32, V1);  in lowerV8I32Shuffle()
18154     SDValue CastV2 = DAG.getBitcast(MVT::v8f32, V2);  in lowerV8I32Shuffle()
18155     SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask,  in lowerV8I32Shuffle()
18157     return DAG.getBitcast(MVT::v8i32, ShufPS);  in lowerV8I32Shuffle()
18163           DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))  in lowerV8I32Shuffle()
18167   return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8i32, V1, V2, Mask,  in lowerV8I32Shuffle()
18179   assert(V1.getSimpleValueType() == MVT::v16i16 && "Bad operand type!");  in lowerV16I16Shuffle()
18180   assert(V2.getSimpleValueType() == MVT::v16i16 && "Bad operand type!");  in lowerV16I16Shuffle()
18188           DL, MVT::v16i16, V1, V2, Mask, Zeroable, Subtarget, DAG))  in lowerV16I16Shuffle()
18192   if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i16, V1, V2, Mask,  in lowerV16I16Shuffle()
18196   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i16, V1, V2, Mask,  in lowerV16I16Shuffle()
18201   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i16, Mask, V1, V2, DAG))  in lowerV16I16Shuffle()
18205   if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i16, Mask, V1, V2, DAG,  in lowerV16I16Shuffle()
18210   if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v16i16, V1, V2, Mask, Zeroable,  in lowerV16I16Shuffle()
18215   if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v16i16, V1, V2, Mask,  in lowerV16I16Shuffle()
18220   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i16, V1, V2, Mask,  in lowerV16I16Shuffle()
18227           DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))  in lowerV16I16Shuffle()
18233             lowerShuffleAsBitRotate(DL, MVT::v16i16, V1, Mask, Subtarget, DAG))  in lowerV16I16Shuffle()
18238     if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v16i16, Mask, V1, V2, DAG))  in lowerV16I16Shuffle()
18243     if (is128BitLaneCrossingShuffleMask(MVT::v16i16, Mask)) {  in lowerV16I16Shuffle()
18245               DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget))  in lowerV16I16Shuffle()
18248       return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v16i16, V1, V2, Mask,  in lowerV16I16Shuffle()
18253     if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) {  in lowerV16I16Shuffle()
18258           DL, MVT::v16i16, V1, RepeatedMask, Subtarget, DAG);  in lowerV16I16Shuffle()
18262   if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v16i16, Mask, V1, V2,  in lowerV16I16Shuffle()
18268     return lowerShuffleWithPERMV(DL, MVT::v16i16, Mask, V1, V2, Subtarget, DAG);  in lowerV16I16Shuffle()
18273           DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))  in lowerV16I16Shuffle()
18278           DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget))  in lowerV16I16Shuffle()
18282   return lowerShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask,  in lowerV16I16Shuffle()
18294   assert(V1.getSimpleValueType() == MVT::v32i8 && "Bad operand type!");  in lowerV32I8Shuffle()
18295   assert(V2.getSimpleValueType() == MVT::v32i8 && "Bad operand type!");  in lowerV32I8Shuffle()
18302   if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v32i8, V1, V2, Mask,  in lowerV32I8Shuffle()
18307   if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v32i8, V1, V2, Mask,  in lowerV32I8Shuffle()
18311   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask,  in lowerV32I8Shuffle()
18316   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i8, Mask, V1, V2, DAG))  in lowerV32I8Shuffle()
18320   if (SDValue V = lowerShuffleWithPACK(DL, MVT::v32i8, Mask, V1, V2, DAG,  in lowerV32I8Shuffle()
18325   if (SDValue V = lowerShuffleAsVTRUNC(DL, MVT::v32i8, V1, V2, Mask, Zeroable,  in lowerV32I8Shuffle()
18330   if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v32i8, V1, V2, Mask,  in lowerV32I8Shuffle()
18335   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i8, V1, V2, Mask,  in lowerV32I8Shuffle()
18342             lowerShuffleAsBitRotate(DL, MVT::v32i8, V1, Mask, Subtarget, DAG))  in lowerV32I8Shuffle()
18348           DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))  in lowerV32I8Shuffle()
18353   if (V2.isUndef() && is128BitLaneCrossingShuffleMask(MVT::v32i8, Mask)) {  in lowerV32I8Shuffle()
18356     if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v32i8, Mask, V1, V2, DAG))  in lowerV32I8Shuffle()
18360             DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))  in lowerV32I8Shuffle()
18363     return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v32i8, V1, V2, Mask,  in lowerV32I8Shuffle()
18367   if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i8, Mask, V1, V2,  in lowerV32I8Shuffle()
18373     return lowerShuffleWithPERMV(DL, MVT::v32i8, Mask, V1, V2, Subtarget, DAG);  in lowerV32I8Shuffle()
18378           DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))  in lowerV32I8Shuffle()
18383           DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))  in lowerV32I8Shuffle()
18390     if (SDValue V = lowerShuffleAsVTRUNCAndUnpack(DL, MVT::v32i8, V1, V2,  in lowerV32I8Shuffle()
18395   return lowerShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask,  in lowerV32I8Shuffle()
18404 static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,  in lower256BitShuffle()
18442     MVT FpVT = MVT::getVectorVT(MVT::getFloatingPointVT(ElementBits),  in lower256BitShuffle()
18449   if (VT == MVT::v16f16) {  in lower256BitShuffle()
18450     V1 = DAG.getBitcast(MVT::v16i16, V1);  in lower256BitShuffle()
18451     V2 = DAG.getBitcast(MVT::v16i16, V2);  in lower256BitShuffle()
18452     return DAG.getBitcast(MVT::v16f16,  in lower256BitShuffle()
18453                           DAG.getVectorShuffle(MVT::v16i16, DL, V1, V2, Mask));  in lower256BitShuffle()
18457   case MVT::v4f64:  in lower256BitShuffle()
18459   case MVT::v4i64:  in lower256BitShuffle()
18461   case MVT::v8f32:  in lower256BitShuffle()
18463   case MVT::v8i32:  in lower256BitShuffle()
18465   case MVT::v16i16:  in lower256BitShuffle()
18467   case MVT::v32i8:  in lower256BitShuffle()
18476 static SDValue lowerV4X128Shuffle(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,  in lowerV4X128Shuffle()
18497     MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);  in lowerV4X128Shuffle()
18510     MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 4);  in lowerV4X128Shuffle()
18542     MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2);  in lowerV4X128Shuffle()
18580                      DAG.getTargetConstant(PermMask, DL, MVT::i8));  in lowerV4X128Shuffle()
18588   assert(V1.getSimpleValueType() == MVT::v8f64 && "Bad operand type!");  in lowerV8F64Shuffle()
18589   assert(V2.getSimpleValueType() == MVT::v8f64 && "Bad operand type!");  in lowerV8F64Shuffle()
18595       return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v8f64, V1);  in lowerV8F64Shuffle()
18597     if (!is128BitLaneCrossingShuffleMask(MVT::v8f64, Mask)) {  in lowerV8F64Shuffle()
18604       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f64, V1,  in lowerV8F64Shuffle()
18605                          DAG.getTargetConstant(VPERMILPMask, DL, MVT::i8));  in lowerV8F64Shuffle()
18609     if (is256BitLaneRepeatedShuffleMask(MVT::v8f64, Mask, RepeatedMask))  in lowerV8F64Shuffle()
18610       return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8f64, V1,  in lowerV8F64Shuffle()
18614   if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8f64, Mask, Zeroable, V1,  in lowerV8F64Shuffle()
18618   if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8f64, Mask, V1, V2, DAG))  in lowerV8F64Shuffle()
18622   if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v8f64, V1, V2, Mask,  in lowerV8F64Shuffle()
18626   if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8f64, Zeroable, Mask, V1, V2,  in lowerV8F64Shuffle()
18630   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f64, V1, V2, Mask,  in lowerV8F64Shuffle()
18634   return lowerShuffleWithPERMV(DL, MVT::v8f64, Mask, V1, V2, Subtarget, DAG);  in lowerV8F64Shuffle()
18642   assert(V1.getSimpleValueType() == MVT::v16f32 && "Bad operand type!");  in lowerV16F32Shuffle()
18643   assert(V2.getSimpleValueType() == MVT::v16f32 && "Bad operand type!");  in lowerV16F32Shuffle()
18649   if (is128BitLaneRepeatedShuffleMask(MVT::v16f32, Mask, RepeatedMask)) {  in lowerV16F32Shuffle()
18654       return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v16f32, V1);  in lowerV16F32Shuffle()
18656       return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v16f32, V1);  in lowerV16F32Shuffle()
18659       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v16f32, V1,  in lowerV16F32Shuffle()
18663     if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16f32, Mask, V1, V2, DAG))  in lowerV16F32Shuffle()
18666     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask,  in lowerV16F32Shuffle()
18671     return lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask, V1, V2, DAG);  in lowerV16F32Shuffle()
18677           DL, MVT::v16f32, V1, V2, Mask, Subtarget, DAG))  in lowerV16F32Shuffle()
18683       !is128BitLaneCrossingShuffleMask(MVT::v16f32, Mask)) {  in lowerV16F32Shuffle()
18684     SDValue VPermMask = getConstVector(Mask, MVT::v16i32, DAG, DL, true);  in lowerV16F32Shuffle()
18685     return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v16f32, V1, VPermMask);  in lowerV16F32Shuffle()
18689   if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v16f32, Zeroable, Mask,  in lowerV16F32Shuffle()
18693   return lowerShuffleWithPERMV(DL, MVT::v16f32, Mask, V1, V2, Subtarget, DAG);  in lowerV16F32Shuffle()
18701   assert(V1.getSimpleValueType() == MVT::v8i64 && "Bad operand type!");  in lowerV8I64Shuffle()
18702   assert(V2.getSimpleValueType() == MVT::v8i64 && "Bad operand type!");  in lowerV8I64Shuffle()
18710     if (is128BitLaneRepeatedShuffleMask(MVT::v8i64, Mask, Repeated128Mask)) {  in lowerV8I64Shuffle()
18714           MVT::v8i64,  in lowerV8I64Shuffle()
18715           DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32,  in lowerV8I64Shuffle()
18716                       DAG.getBitcast(MVT::v16i32, V1),  in lowerV8I64Shuffle()
18721     if (is256BitLaneRepeatedShuffleMask(MVT::v8i64, Mask, Repeated256Mask))  in lowerV8I64Shuffle()
18722       return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8i64, V1,  in lowerV8I64Shuffle()
18726   if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8i64, Mask, Zeroable, V1,  in lowerV8I64Shuffle()
18731   if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i64, V1, V2, Mask,  in lowerV8I64Shuffle()
18736   if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v8i64, V1, V2, Mask,  in lowerV8I64Shuffle()
18742     if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i64, V1, V2, Mask,  in lowerV8I64Shuffle()
18746   if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, Mask, V1, V2, DAG))  in lowerV8I64Shuffle()
18750   if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i64, Zeroable, Mask, V1, V2,  in lowerV8I64Shuffle()
18754   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i64, V1, V2, Mask,  in lowerV8I64Shuffle()
18758   return lowerShuffleWithPERMV(DL, MVT::v8i64, Mask, V1, V2, Subtarget, DAG);  in lowerV8I64Shuffle()
18766   assert(V1.getSimpleValueType() == MVT::v16i32 && "Bad operand type!");  in lowerV16I32Shuffle()
18767   assert(V2.getSimpleValueType() == MVT::v16i32 && "Bad operand type!");  in lowerV16I32Shuffle()
18774           DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG))  in lowerV16I32Shuffle()
18782       is128BitLaneRepeatedShuffleMask(MVT::v16i32, Mask, RepeatedMask);  in lowerV16I32Shuffle()
18786       return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32, V1,  in lowerV16I32Shuffle()
18790     if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i32, Mask, V1, V2, DAG))  in lowerV16I32Shuffle()
18795   if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v16i32, V1, V2, Mask,  in lowerV16I32Shuffle()
18800   if (SDValue Rotate = lowerShuffleAsVALIGN(DL, MVT::v16i32, V1, V2, Mask,  in lowerV16I32Shuffle()
18806     if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i32, V1, V2, Mask,  in lowerV16I32Shuffle()
18813     SDValue CastV1 = DAG.getBitcast(MVT::v16f32, V1);  in lowerV16I32Shuffle()
18814     SDValue CastV2 = DAG.getBitcast(MVT::v16f32, V2);  in lowerV16I32Shuffle()
18815     SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask,  in lowerV16I32Shuffle()
18817     return DAG.getBitcast(MVT::v16i32, ShufPS);  in lowerV16I32Shuffle()
18823           DL, MVT::v16i32, V1, V2, Mask, Subtarget, DAG))  in lowerV16I32Shuffle()
18827   if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v16i32, Zeroable, Mask, V1, V2,  in lowerV16I32Shuffle()
18831   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i32, V1, V2, Mask,  in lowerV16I32Shuffle()
18835   return lowerShuffleWithPERMV(DL, MVT::v16i32, Mask, V1, V2, Subtarget, DAG);  in lowerV16I32Shuffle()
18843   assert(V1.getSimpleValueType() == MVT::v32i16 && "Bad operand type!");  in lowerV32I16Shuffle()
18844   assert(V2.getSimpleValueType() == MVT::v32i16 && "Bad operand type!");  in lowerV32I16Shuffle()
18852           DL, MVT::v32i16, V1, V2, Mask, Zeroable, Subtarget, DAG))  in lowerV32I16Shuffle()
18856   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i16, Mask, V1, V2, DAG))  in lowerV32I16Shuffle()
18861           lowerShuffleWithPACK(DL, MVT::v32i16, Mask, V1, V2, DAG, Subtarget))  in lowerV32I16Shuffle()
18865   if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v32i16, V1, V2, Mask,  in lowerV32I16Shuffle()
18870   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i16, V1, V2, Mask,  in lowerV32I16Shuffle()
18877             lowerShuffleAsBitRotate(DL, MVT::v32i16, V1, Mask, Subtarget, DAG))  in lowerV32I16Shuffle()
18881     if (is128BitLaneRepeatedShuffleMask(MVT::v32i16, Mask, RepeatedMask)) {  in lowerV32I16Shuffle()
18885       return lowerV8I16GeneralSingleInputShuffle(DL, MVT::v32i16, V1,  in lowerV32I16Shuffle()
18890   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i16, V1, V2, Mask,  in lowerV32I16Shuffle()
18894   if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i16, Mask, V1, V2,  in lowerV32I16Shuffle()
18898   return lowerShuffleWithPERMV(DL, MVT::v32i16, Mask, V1, V2, Subtarget, DAG);  in lowerV32I16Shuffle()
18906   assert(V1.getSimpleValueType() == MVT::v64i8 && "Bad operand type!");  in lowerV64I8Shuffle()
18907   assert(V2.getSimpleValueType() == MVT::v64i8 && "Bad operand type!");  in lowerV64I8Shuffle()
18915           DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget, DAG))  in lowerV64I8Shuffle()
18919   if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v64i8, Mask, V1, V2, DAG))  in lowerV64I8Shuffle()
18923   if (SDValue V = lowerShuffleWithPACK(DL, MVT::v64i8, Mask, V1, V2, DAG,  in lowerV64I8Shuffle()
18928   if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v64i8, V1, V2, Mask,  in lowerV64I8Shuffle()
18933   if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v64i8, V1, V2, Mask,  in lowerV64I8Shuffle()
18940             lowerShuffleAsBitRotate(DL, MVT::v64i8, V1, Mask, Subtarget, DAG))  in lowerV64I8Shuffle()
18944   if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v64i8, V1, V2, Mask,  in lowerV64I8Shuffle()
18948   if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v64i8, Mask, V1, V2,  in lowerV64I8Shuffle()
18955           DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))  in lowerV64I8Shuffle()
18959           DL, MVT::v64i8, V1, V2, Mask, DAG, Subtarget))  in lowerV64I8Shuffle()
18962   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v64i8, V1, V2, Mask,  in lowerV64I8Shuffle()
18966   if (!is128BitLaneCrossingShuffleMask(MVT::v64i8, Mask)) {  in lowerV64I8Shuffle()
18969     if (SDValue V = lowerShuffleAsByteRotateAndPermute(DL, MVT::v64i8, V1, V2,  in lowerV64I8Shuffle()
18976     return lowerShuffleAsBlendOfPSHUFBs(DL, MVT::v64i8, V1, V2, Mask, Zeroable,  in lowerV64I8Shuffle()
18984             DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))  in lowerV64I8Shuffle()
18989     return lowerShuffleWithPERMV(DL, MVT::v64i8, Mask, V1, V2, Subtarget, DAG);  in lowerV64I8Shuffle()
18991   return splitAndLowerShuffle(DL, MVT::v64i8, V1, V2, Mask, DAG);  in lowerV64I8Shuffle()
19000                                   MVT VT, SDValue V1, SDValue V2,  in lower512BitShuffle()
19027   if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) {  in lower512BitShuffle()
19039   if (VT == MVT::v32f16) {  in lower512BitShuffle()
19040     V1 = DAG.getBitcast(MVT::v32i16, V1);  in lower512BitShuffle()
19041     V2 = DAG.getBitcast(MVT::v32i16, V2);  in lower512BitShuffle()
19042     return DAG.getBitcast(MVT::v32f16,  in lower512BitShuffle()
19043                           DAG.getVectorShuffle(MVT::v32i16, DL, V1, V2, Mask));  in lower512BitShuffle()
19051   case MVT::v8f64:  in lower512BitShuffle()
19053   case MVT::v16f32:  in lower512BitShuffle()
19055   case MVT::v8i64:  in lower512BitShuffle()
19057   case MVT::v16i32:  in lower512BitShuffle()
19059   case MVT::v32i16:  in lower512BitShuffle()
19061   case MVT::v64i8:  in lower512BitShuffle()
19070                                          MVT VT, SDValue V1, SDValue V2,  in lower1BitShuffleAsKSHIFTR()
19100   MVT WideVT = VT;  in lower1BitShuffleAsKSHIFTR()
19102     WideVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1;  in lower1BitShuffleAsKSHIFTR()
19107                     DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));  in lower1BitShuffleAsKSHIFTR()
19150                                 MVT VT, SDValue V1, SDValue V2,  in lower1BitShuffle()
19183     MVT ExtractVT = MVT::getVectorVT(MVT::i1, SubvecElts);  in lower1BitShuffle()
19203       MVT WideVT = VT;  in lower1BitShuffle()
19205         WideVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1;  in lower1BitShuffle()
19214                           DAG.getTargetConstant(WideElts - NumElts, DL, MVT::i8));  in lower1BitShuffle()
19220                         DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));  in lower1BitShuffle()
19240   MVT ExtVT;  in lower1BitShuffle()
19244   case MVT::v2i1:  in lower1BitShuffle()
19245     ExtVT = MVT::v2i64;  in lower1BitShuffle()
19247   case MVT::v4i1:  in lower1BitShuffle()
19248     ExtVT = MVT::v4i32;  in lower1BitShuffle()
19250   case MVT::v8i1:  in lower1BitShuffle()
19253     ExtVT = Subtarget.hasVLX() ? MVT::v8i32 : MVT::v8i64;  in lower1BitShuffle()
19255   case MVT::v16i1:  in lower1BitShuffle()
19258     ExtVT = Subtarget.canExtendTo512DQ() ? MVT::v16i32 : MVT::v16i16;  in lower1BitShuffle()
19260   case MVT::v32i1:  in lower1BitShuffle()
19264     ExtVT = Subtarget.canExtendTo512BW() ? MVT::v32i16 : MVT::v32i8;  in lower1BitShuffle()
19266   case MVT::v64i1:  in lower1BitShuffle()
19271     ExtVT = MVT::v64i8;  in lower1BitShuffle()
19372   MVT VT = Op.getSimpleValueType();  in lowerVECTOR_SHUFFLE()
19375   bool Is1BitVector = (VT.getVectorElementType() == MVT::i1);  in lowerVECTOR_SHUFFLE()
19437     MVT NewEltVT = VT.isFloatingPoint()  in lowerVECTOR_SHUFFLE()
19438                        ? MVT::getFloatingPointVT(VT.getScalarSizeInBits() * 2)  in lowerVECTOR_SHUFFLE()
19439                        : MVT::getIntegerVT(VT.getScalarSizeInBits() * 2);  in lowerVECTOR_SHUFFLE()
19441     MVT NewVT = MVT::getVectorVT(NewEltVT, NewNumElts);  in lowerVECTOR_SHUFFLE()
19512   MVT VT = Op.getSimpleValueType();  in lowerVSELECTtoVectorShuffle()
19531   MVT VT = Op.getSimpleValueType();  in LowerVSELECT()
19533     MVT NVT = VT.changeVectorElementTypeToInteger();  in LowerVSELECT()
19553   MVT CondVT = Cond.getSimpleValueType();  in LowerVSELECT()
19566   if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI())  in LowerVSELECT()
19574     MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);  in LowerVSELECT()
19588     MVT NewCondSVT = MVT::getIntegerVT(EltSize);  in LowerVSELECT()
19589     MVT NewCondVT = MVT::getVectorVT(NewCondSVT, NumElts);  in LowerVSELECT()
19602   case MVT::v32i8:  in LowerVSELECT()
19609   case MVT::v8i16:  in LowerVSELECT()
19610   case MVT::v16i16: {  in LowerVSELECT()
19612     MVT CastVT = MVT::getVectorVT(MVT::i8, NumElts * 2);  in LowerVSELECT()
19623   MVT VT = Op.getSimpleValueType();  in LowerEXTRACT_VECTOR_ELT_SSE4()
19637       return DAG.getNode(ISD::TRUNCATE, dl, MVT::i8,  in LowerEXTRACT_VECTOR_ELT_SSE4()
19638                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,  in LowerEXTRACT_VECTOR_ELT_SSE4()
19639                                      DAG.getBitcast(MVT::v4i32, Vec), Idx));  in LowerEXTRACT_VECTOR_ELT_SSE4()
19642     SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32, Vec,  in LowerEXTRACT_VECTOR_ELT_SSE4()
19643                                   DAG.getTargetConstant(IdxVal, dl, MVT::i8));  in LowerEXTRACT_VECTOR_ELT_SSE4()
19647   if (VT == MVT::f32) {  in LowerEXTRACT_VECTOR_ELT_SSE4()
19658          User->getValueType(0) != MVT::i32))  in LowerEXTRACT_VECTOR_ELT_SSE4()
19660     SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,  in LowerEXTRACT_VECTOR_ELT_SSE4()
19661                                   DAG.getBitcast(MVT::v4i32, Vec), Idx);  in LowerEXTRACT_VECTOR_ELT_SSE4()
19662     return DAG.getBitcast(MVT::f32, Extract);  in LowerEXTRACT_VECTOR_ELT_SSE4()
19665   if (VT == MVT::i32 || VT == MVT::i64)  in LowerEXTRACT_VECTOR_ELT_SSE4()
19677   MVT VecVT = Vec.getSimpleValueType();  in ExtractBitFromMaskVector()
19680   MVT EltVT = Op.getSimpleValueType();  in ExtractBitFromMaskVector()
19691     MVT ExtEltVT = (NumElts <= 8) ? MVT::getIntegerVT(128 / NumElts) : MVT::i8;  in ExtractBitFromMaskVector()
19692     MVT ExtVecVT = MVT::getVectorVT(ExtEltVT, NumElts);  in ExtractBitFromMaskVector()
19704   MVT WideVecVT = VecVT;  in ExtractBitFromMaskVector()
19706     WideVecVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1;  in ExtractBitFromMaskVector()
19714                     DAG.getTargetConstant(IdxVal, dl, MVT::i8));  in ExtractBitFromMaskVector()
19725   MVT VecVT = Vec.getSimpleValueType();  in LowerEXTRACT_VECTOR_ELT()
19729   if (VecVT.getVectorElementType() == MVT::i1)  in LowerEXTRACT_VECTOR_ELT()
19773     MVT EltVT = VecVT.getVectorElementType();  in LowerEXTRACT_VECTOR_ELT()
19787   MVT VT = Op.getSimpleValueType();  in LowerEXTRACT_VECTOR_ELT()
19789   if (VT == MVT::i16) {  in LowerEXTRACT_VECTOR_ELT()
19797       return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16,  in LowerEXTRACT_VECTOR_ELT()
19798                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,  in LowerEXTRACT_VECTOR_ELT()
19799                                      DAG.getBitcast(MVT::v4i32, Vec), Idx));  in LowerEXTRACT_VECTOR_ELT()
19802     SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, Vec,  in LowerEXTRACT_VECTOR_ELT()
19803                                   DAG.getTargetConstant(IdxVal, dl, MVT::i8));  in LowerEXTRACT_VECTOR_ELT()
19818       SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,  in LowerEXTRACT_VECTOR_ELT()
19819                                 DAG.getBitcast(MVT::v4i32, Vec),  in LowerEXTRACT_VECTOR_ELT()
19823         Res = DAG.getNode(ISD::SRL, dl, MVT::i32, Res,  in LowerEXTRACT_VECTOR_ELT()
19824                           DAG.getConstant(ShiftVal, dl, MVT::i8));  in LowerEXTRACT_VECTOR_ELT()
19829     SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16,  in LowerEXTRACT_VECTOR_ELT()
19830                               DAG.getBitcast(MVT::v8i16, Vec),  in LowerEXTRACT_VECTOR_ELT()
19834       Res = DAG.getNode(ISD::SRL, dl, MVT::i16, Res,  in LowerEXTRACT_VECTOR_ELT()
19835                         DAG.getConstant(ShiftVal, dl, MVT::i8));  in LowerEXTRACT_VECTOR_ELT()
19839   if (VT == MVT::f16 || VT.getSizeInBits() == 32) {  in LowerEXTRACT_VECTOR_ELT()
19878   MVT VecVT = Vec.getSimpleValueType();  in InsertBitToMaskVector()
19884     MVT ExtEltVT = (NumElts <= 8) ? MVT::getIntegerVT(128 / NumElts) : MVT::i8;  in InsertBitToMaskVector()
19885     MVT ExtVecVT = MVT::getVectorVT(ExtEltVT, NumElts);  in InsertBitToMaskVector()
19893   SDValue EltInVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i1, Elt);  in InsertBitToMaskVector()
19899   MVT VT = Op.getSimpleValueType();  in LowerINSERT_VECTOR_ELT()
19900   MVT EltVT = VT.getVectorElementType();  in LowerINSERT_VECTOR_ELT()
19904   if (EltVT == MVT::i1)  in LowerINSERT_VECTOR_ELT()
19922     MVT IdxSVT = MVT::getIntegerVT(EltSizeInBits);  in LowerINSERT_VECTOR_ELT()
19923     MVT IdxVT = MVT::getVectorVT(IdxSVT, NumElts);  in LowerINSERT_VECTOR_ELT()
19952         ((VT == MVT::v16i8 && !Subtarget.hasSSE41()) ||  in LowerINSERT_VECTOR_ELT()
19953          ((VT == MVT::v32i8 || VT == MVT::v16i16) && !Subtarget.hasInt256()))) {  in LowerINSERT_VECTOR_ELT()
19983       if ((Subtarget.hasAVX() && (EltVT == MVT::f64 || EltVT == MVT::f32)) ||  in LowerINSERT_VECTOR_ELT()
19984           (Subtarget.hasAVX2() && (EltVT == MVT::i32 || EltVT == MVT::i64))) {  in LowerINSERT_VECTOR_ELT()
19987                            DAG.getTargetConstant(1, dl, MVT::i8));  in LowerINSERT_VECTOR_ELT()
20026     if (EltVT == MVT::i32 || EltVT == MVT::f32 || EltVT == MVT::f64 ||  in LowerINSERT_VECTOR_ELT()
20027         EltVT == MVT::f16 || EltVT == MVT::i64) {  in LowerINSERT_VECTOR_ELT()
20034     if (EltVT == MVT::i16 || EltVT == MVT::i8) {  in LowerINSERT_VECTOR_ELT()
20035       N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, N1);  in LowerINSERT_VECTOR_ELT()
20036       MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32);  in LowerINSERT_VECTOR_ELT()
20045   if (VT == MVT::v8i16 || (VT == MVT::v16i8 && Subtarget.hasSSE41())) {  in LowerINSERT_VECTOR_ELT()
20047     if (VT == MVT::v8i16) {  in LowerINSERT_VECTOR_ELT()
20051       assert(VT == MVT::v16i8 && "PINSRB requires v16i8 vector");  in LowerINSERT_VECTOR_ELT()
20056     assert(N1.getValueType() != MVT::i32 && "Unexpected VT");  in LowerINSERT_VECTOR_ELT()
20057     N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1);  in LowerINSERT_VECTOR_ELT()
20058     N2 = DAG.getTargetConstant(IdxVal, dl, MVT::i8);  in LowerINSERT_VECTOR_ELT()
20063     if (EltVT == MVT::f32) {  in LowerINSERT_VECTOR_ELT()
20082         N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);  in LowerINSERT_VECTOR_ELT()
20084                            DAG.getTargetConstant(1, dl, MVT::i8));  in LowerINSERT_VECTOR_ELT()
20087       N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);  in LowerINSERT_VECTOR_ELT()
20089                          DAG.getTargetConstant(IdxVal << 4, dl, MVT::i8));  in LowerINSERT_VECTOR_ELT()
20093     if (EltVT == MVT::i32 || EltVT == MVT::i64)  in LowerINSERT_VECTOR_ELT()
20103   MVT OpVT = Op.getSimpleValueType();  in LowerSCALAR_TO_VECTOR()
20115     MVT VT128 = MVT::getVectorVT(OpVT.getVectorElementType(),  in LowerSCALAR_TO_VECTOR()
20123   assert(OpVT.is128BitVector() && OpVT.isInteger() && OpVT != MVT::v2i64 &&  in LowerSCALAR_TO_VECTOR()
20128   if (OpVT == MVT::v4i32 || (OpVT == MVT::v8i16 && Subtarget.hasFP16()))  in LowerSCALAR_TO_VECTOR()
20131   SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0));  in LowerSCALAR_TO_VECTOR()
20133       OpVT, DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, AnyExt));  in LowerSCALAR_TO_VECTOR()
20141   assert(Op.getSimpleValueType().getVectorElementType() == MVT::i1);  in LowerINSERT_SUBVECTOR()
20148   assert(Op.getSimpleValueType().getVectorElementType() == MVT::i1 &&  in LowerEXTRACT_SUBVECTOR()
20158   MVT VecVT = Vec.getSimpleValueType();  in LowerEXTRACT_SUBVECTOR()
20162   MVT WideVecVT = VecVT;  in LowerEXTRACT_SUBVECTOR()
20164     WideVecVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1;  in LowerEXTRACT_SUBVECTOR()
20172                     DAG.getTargetConstant(IdxVal, dl, MVT::i8));  in LowerEXTRACT_SUBVECTOR()
20359   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);  in GetTLSADDR()
20570     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);  in LowerGlobalTLSAddress()
20629                              MachinePointerInfo(), MVT::i32);  in LowerGlobalTLSAddress()
20635           DAG.getConstant(Log2_64_Ceil(DL.getPointerSize()), dl, MVT::i8);  in LowerGlobalTLSAddress()
20678   MVT SrcVT = Src.getSimpleValueType();  in LowerI64IntToFP_AVX512DQ()
20679   MVT VT = Op.getSimpleValueType();  in LowerI64IntToFP_AVX512DQ()
20681    if (!Subtarget.hasDQI() || SrcVT != MVT::i64 || Subtarget.is64Bit() ||  in LowerI64IntToFP_AVX512DQ()
20682        (VT != MVT::f32 && VT != MVT::f64))  in LowerI64IntToFP_AVX512DQ()
20689   MVT VecInVT = MVT::getVectorVT(MVT::i64, NumElts);  in LowerI64IntToFP_AVX512DQ()
20690   MVT VecVT = MVT::getVectorVT(VT, NumElts);  in LowerI64IntToFP_AVX512DQ()
20695     SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, {VecVT, MVT::Other},  in LowerI64IntToFP_AVX512DQ()
20719   MVT SrcVT = Src.getSimpleValueType();  in LowerI64IntToFP16()
20720   MVT VT = Op.getSimpleValueType();  in LowerI64IntToFP16()
20722   if (SrcVT != MVT::i64 || Subtarget.is64Bit() || VT != MVT::f16)  in LowerI64IntToFP16()
20730   SDValue InVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Src);  in LowerI64IntToFP16()
20732     SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, {MVT::v2f16, MVT::Other},  in LowerI64IntToFP16()
20740   SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, MVT::v2f16, InVec);  in LowerI64IntToFP16()
20746 static bool useVectorCast(unsigned Opcode, MVT FromVT, MVT ToVT,  in useVectorCast()
20751       if (!Subtarget.hasSSE2() || FromVT != MVT::v4i32)  in useVectorCast()
20754       return ToVT == MVT::v4f32 || (Subtarget.hasAVX() && ToVT == MVT::v4f64);  in useVectorCast()
20758       if (!Subtarget.hasAVX512() || FromVT != MVT::v4i32)  in useVectorCast()
20761       return ToVT == MVT::v4f32 || ToVT == MVT::v4f64;  in useVectorCast()
20776   MVT DestVT = Cast.getSimpleValueType();  in vectorizeExtractedCast()
20783   MVT FromVT = VecOp.getSimpleValueType();  in vectorizeExtractedCast()
20785   MVT Vec128VT = MVT::getVectorVT(FromVT.getScalarType(), NumEltsInXMM);  in vectorizeExtractedCast()
20786   MVT ToVT = MVT::getVectorVT(DestVT, NumEltsInXMM);  in vectorizeExtractedCast()
20817   MVT VT = CastToFP.getSimpleValueType();  in lowerFPToIntToFP()
20821   MVT IntVT = CastToInt.getSimpleValueType();  in lowerFPToIntToFP()
20823   MVT SrcVT = X.getSimpleValueType();  in lowerFPToIntToFP()
20824   if (SrcVT != MVT::f32 && SrcVT != MVT::f64)  in lowerFPToIntToFP()
20829   if (!Subtarget.hasSSE2() || (VT != MVT::f32 && VT != MVT::f64) ||  in lowerFPToIntToFP()
20830       IntVT != MVT::i32)  in lowerFPToIntToFP()
20836   MVT VecSrcVT = MVT::getVectorVT(SrcVT, 128 / SrcSize);  in lowerFPToIntToFP()
20837   MVT VecIntVT = MVT::getVectorVT(IntVT, 128 / IntSize);  in lowerFPToIntToFP()
20838   MVT VecVT = MVT::getVectorVT(VT, 128 / VTSize);  in lowerFPToIntToFP()
20864   MVT VT = Op->getSimpleValueType(0);  in lowerINT_TO_FP_vXi64()
20870     assert((Src.getSimpleValueType() == MVT::v2i64 ||  in lowerINT_TO_FP_vXi64()
20871             Src.getSimpleValueType() == MVT::v4i64) &&  in lowerINT_TO_FP_vXi64()
20875     assert((VT == MVT::v4f32 || VT == MVT::v2f64 || VT == MVT::v4f64) &&  in lowerINT_TO_FP_vXi64()
20877     MVT WideVT = VT == MVT::v4f32 ? MVT::v8f32 : MVT::v8f64;  in lowerINT_TO_FP_vXi64()
20881     SDValue Tmp = IsStrict ? DAG.getConstant(0, DL, MVT::v8i64)  in lowerINT_TO_FP_vXi64()
20882                            : DAG.getUNDEF(MVT::v8i64);  in lowerINT_TO_FP_vXi64()
20883     Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i64, Tmp, Src,  in lowerINT_TO_FP_vXi64()
20887       Res = DAG.getNode(Op.getOpcode(), DL, {WideVT, MVT::Other},  in lowerINT_TO_FP_vXi64()
20904   if (VT != MVT::v4f32 || IsSigned)  in lowerINT_TO_FP_vXi64()
20907   SDValue Zero = DAG.getConstant(0, DL, MVT::v4i64);  in lowerINT_TO_FP_vXi64()
20908   SDValue One  = DAG.getConstant(1, DL, MVT::v4i64);  in lowerINT_TO_FP_vXi64()
20909   SDValue Sign = DAG.getNode(ISD::OR, DL, MVT::v4i64,  in lowerINT_TO_FP_vXi64()
20910                              DAG.getNode(ISD::SRL, DL, MVT::v4i64, Src, One),  in lowerINT_TO_FP_vXi64()
20911                              DAG.getNode(ISD::AND, DL, MVT::v4i64, Src, One));  in lowerINT_TO_FP_vXi64()
20912   SDValue IsNeg = DAG.getSetCC(DL, MVT::v4i64, Src, Zero, ISD::SETLT);  in lowerINT_TO_FP_vXi64()
20913   SDValue SignSrc = DAG.getSelect(DL, MVT::v4i64, IsNeg, Sign, Src);  in lowerINT_TO_FP_vXi64()
20917     SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, SignSrc,  in lowerINT_TO_FP_vXi64()
20921           DAG.getNode(ISD::STRICT_SINT_TO_FP, DL, {MVT::f32, MVT::Other},  in lowerINT_TO_FP_vXi64()
20925       SignCvts[i] = DAG.getNode(ISD::SINT_TO_FP, DL, MVT::f32, Elt);  in lowerINT_TO_FP_vXi64()
20932     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);  in lowerINT_TO_FP_vXi64()
20933     Slow = DAG.getNode(ISD::STRICT_FADD, DL, {MVT::v4f32, MVT::Other},  in lowerINT_TO_FP_vXi64()
20937     Slow = DAG.getNode(ISD::FADD, DL, MVT::v4f32, SignCvt, SignCvt);  in lowerINT_TO_FP_vXi64()
20940   IsNeg = DAG.getNode(ISD::TRUNCATE, DL, MVT::v4i32, IsNeg);  in lowerINT_TO_FP_vXi64()
20941   SDValue Cvt = DAG.getSelect(DL, MVT::v4f32, IsNeg, Slow, SignCvt);  in lowerINT_TO_FP_vXi64()
20953   MVT VT = Op.getSimpleValueType();  in promoteXINT_TO_FP()
20954   MVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32;  in promoteXINT_TO_FP()
20960         ISD::STRICT_FP_ROUND, dl, {VT, MVT::Other},  in promoteXINT_TO_FP()
20962          DAG.getNode(Op.getOpcode(), dl, {NVT, MVT::Other}, {Chain, Src}),  in promoteXINT_TO_FP()
20968 static bool isLegalConversion(MVT VT, bool IsSigned,  in isLegalConversion()
20970   if (VT == MVT::v4i32 && Subtarget.hasSSE2() && IsSigned)  in isLegalConversion()
20972   if (VT == MVT::v8i32 && Subtarget.hasAVX() && IsSigned)  in isLegalConversion()
20974   if (Subtarget.hasVLX() && (VT == MVT::v4i32 || VT == MVT::v8i32))  in isLegalConversion()
20977     if (VT == MVT::v16i32)  in isLegalConversion()
20979     if (VT == MVT::v8i64 && Subtarget.hasDQI())  in isLegalConversion()
20983       (VT == MVT::v2i64 || VT == MVT::v4i64))  in isLegalConversion()
20994   MVT SrcVT = Src.getSimpleValueType();  in LowerSINT_TO_FP()
20995   MVT VT = Op.getSimpleValueType();  in LowerSINT_TO_FP()
21003   if (Subtarget.isTargetWin64() && SrcVT == MVT::i128)  in LowerSINT_TO_FP()
21013     if (SrcVT == MVT::v2i32 && VT == MVT::v2f64) {  in LowerSINT_TO_FP()
21018             X86ISD::STRICT_CVTSI2P, dl, {VT, MVT::Other},  in LowerSINT_TO_FP()
21019             {Chain, DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,  in LowerSINT_TO_FP()
21022                          DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,  in LowerSINT_TO_FP()
21025     if (SrcVT == MVT::v2i64 || SrcVT == MVT::v4i64)  in LowerSINT_TO_FP()
21031   assert(SrcVT <= MVT::i64 && SrcVT >= MVT::i16 &&  in LowerSINT_TO_FP()
21038   if (SrcVT == MVT::i32 && UseSSEReg)  in LowerSINT_TO_FP()
21040   if (SrcVT == MVT::i64 && UseSSEReg && Subtarget.is64Bit())  in LowerSINT_TO_FP()
21049   if (SrcVT == MVT::i16 && (UseSSEReg || VT == MVT::f128)) {  in LowerSINT_TO_FP()
21050     SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, Src);  in LowerSINT_TO_FP()
21052       return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},  in LowerSINT_TO_FP()
21058   if (VT == MVT::f128)  in LowerSINT_TO_FP()
21062   if (SrcVT == MVT::i64 && Subtarget.hasSSE2() && !Subtarget.is64Bit())  in LowerSINT_TO_FP()
21066     ValueToStore = DAG.getBitcast(MVT::f64, ValueToStore);  in LowerSINT_TO_FP()
21093     Tys = DAG.getVTList(MVT::f80, MVT::Other);  in BuildFILD()
21095     Tys = DAG.getVTList(DstVT, MVT::Other);  in BuildFILD()
21110     Tys = DAG.getVTList(MVT::Other);  in BuildFILD()
21178       DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Op.getOperand(0));  in LowerUINT_TO_FP_i64()
21180       MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0,  in LowerUINT_TO_FP_i64()
21183       getUnpackl(DAG, dl, MVT::v4i32, DAG.getBitcast(MVT::v4i32, XR1), CLod0);  in LowerUINT_TO_FP_i64()
21186       MVT::v2f64, dl, CLod0.getValue(1), CPIdx1,  in LowerUINT_TO_FP_i64()
21188   SDValue XR2F = DAG.getBitcast(MVT::v2f64, Unpck1);  in LowerUINT_TO_FP_i64()
21190   SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1);  in LowerUINT_TO_FP_i64()
21195     Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub);  in LowerUINT_TO_FP_i64()
21197     SDValue Shuffle = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, Sub, {1,-1});  in LowerUINT_TO_FP_i64()
21198     Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64, Shuffle, Sub);  in LowerUINT_TO_FP_i64()
21200   Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result,  in LowerUINT_TO_FP_i64()
21212                                    MVT::f64);  in LowerUINT_TO_FP_i32()
21216       DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Op.getOperand(OpNo));  in LowerUINT_TO_FP_i32()
21223       ISD::OR, dl, MVT::v2i64,  in LowerUINT_TO_FP_i32()
21224       DAG.getBitcast(MVT::v2i64, Load),  in LowerUINT_TO_FP_i32()
21225       DAG.getBitcast(MVT::v2i64,  in LowerUINT_TO_FP_i32()
21226                      DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, Bias)));  in LowerUINT_TO_FP_i32()
21228       DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,  in LowerUINT_TO_FP_i32()
21229                   DAG.getBitcast(MVT::v2f64, Or), DAG.getIntPtrConstant(0, dl));  in LowerUINT_TO_FP_i32()
21235     SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},  in LowerUINT_TO_FP_i32()
21250   SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias);  in LowerUINT_TO_FP_i32()
21259   if (Op.getSimpleValueType() != MVT::v2f64)  in lowerUINT_TO_FP_v2i32()
21265   assert(N0.getSimpleValueType() == MVT::v2i32 && "Unexpected input type");  in lowerUINT_TO_FP_v2i32()
21273       N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,  in lowerUINT_TO_FP_v2i32()
21274                        DAG.getConstant(0, DL, MVT::v2i32));  in lowerUINT_TO_FP_v2i32()
21275       SDValue Res = DAG.getNode(Op->getOpcode(), DL, {MVT::v4f64, MVT::Other},  in lowerUINT_TO_FP_v2i32()
21278       Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2f64, Res,  in lowerUINT_TO_FP_v2i32()
21284     N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,  in lowerUINT_TO_FP_v2i32()
21285                      DAG.getUNDEF(MVT::v2i32));  in lowerUINT_TO_FP_v2i32()
21287       return DAG.getNode(X86ISD::STRICT_CVTUI2P, DL, {MVT::v2f64, MVT::Other},  in lowerUINT_TO_FP_v2i32()
21289     return DAG.getNode(X86ISD::CVTUI2P, DL, MVT::v2f64, N0);  in lowerUINT_TO_FP_v2i32()
21296   SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v2i64, N0);  in lowerUINT_TO_FP_v2i32()
21298       DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL), DL, MVT::v2f64);  in lowerUINT_TO_FP_v2i32()
21299   SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v2i64, ZExtIn,  in lowerUINT_TO_FP_v2i32()
21300                            DAG.getBitcast(MVT::v2i64, VBias));  in lowerUINT_TO_FP_v2i32()
21301   Or = DAG.getBitcast(MVT::v2f64, Or);  in lowerUINT_TO_FP_v2i32()
21304     return DAG.getNode(ISD::STRICT_FSUB, DL, {MVT::v2f64, MVT::Other},  in lowerUINT_TO_FP_v2i32()
21306   return DAG.getNode(ISD::FSUB, DL, MVT::v2f64, Or, VBias);  in lowerUINT_TO_FP_v2i32()
21314   MVT VecIntVT = V.getSimpleValueType();  in lowerUINT_TO_FP_vXi32()
21315   assert((VecIntVT == MVT::v4i32 || VecIntVT == MVT::v8i32) &&  in lowerUINT_TO_FP_vXi32()
21321     MVT VT = Op->getSimpleValueType(0);  in lowerUINT_TO_FP_vXi32()
21324     if (VT == MVT::v8f64)  in lowerUINT_TO_FP_vXi32()
21327     assert((VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v4f64) &&  in lowerUINT_TO_FP_vXi32()
21329     MVT WideVT = VT == MVT::v4f64 ? MVT::v8f64 : MVT::v16f32;  in lowerUINT_TO_FP_vXi32()
21330     MVT WideIntVT = VT == MVT::v4f64 ? MVT::v8i32 : MVT::v16i32;  in lowerUINT_TO_FP_vXi32()
21339       Res = DAG.getNode(ISD::STRICT_UINT_TO_FP, DL, {WideVT, MVT::Other},  in lowerUINT_TO_FP_vXi32()
21354   if (Subtarget.hasAVX() && VecIntVT == MVT::v4i32 &&  in lowerUINT_TO_FP_vXi32()
21355       Op->getSimpleValueType(0) == MVT::v4f64) {  in lowerUINT_TO_FP_vXi32()
21356     SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i64, V);  in lowerUINT_TO_FP_vXi32()
21362     SDVTList Tys = DAG.getVTList(MVT::v4f64, MVT::Other);  in lowerUINT_TO_FP_vXi32()
21365         X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::f64,  in lowerUINT_TO_FP_vXi32()
21369     SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v4i64, ZExtIn,  in lowerUINT_TO_FP_vXi32()
21370                              DAG.getBitcast(MVT::v4i64, VBias));  in lowerUINT_TO_FP_vXi32()
21371     Or = DAG.getBitcast(MVT::v4f64, Or);  in lowerUINT_TO_FP_vXi32()
21374       return DAG.getNode(ISD::STRICT_FSUB, DL, {MVT::v4f64, MVT::Other},  in lowerUINT_TO_FP_vXi32()
21376     return DAG.getNode(ISD::FSUB, DL, MVT::v4f64, Or, VBias);  in lowerUINT_TO_FP_vXi32()
21391   bool Is128 = VecIntVT == MVT::v4i32;  in lowerUINT_TO_FP_vXi32()
21392   MVT VecFloatVT = Is128 ? MVT::v4f32 : MVT::v8f32;  in lowerUINT_TO_FP_vXi32()
21416     MVT VecI16VT = Is128 ? MVT::v8i16 : MVT::v16i16;  in lowerUINT_TO_FP_vXi32()
21423                       VecCstLowBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));  in lowerUINT_TO_FP_vXi32()
21431                        VecCstHighBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));  in lowerUINT_TO_FP_vXi32()
21456     SDValue FHigh = DAG.getNode(ISD::STRICT_FSUB, DL, {VecFloatVT, MVT::Other},  in lowerUINT_TO_FP_vXi32()
21458     return DAG.getNode(ISD::STRICT_FADD, DL, {VecFloatVT, MVT::Other},  in lowerUINT_TO_FP_vXi32()
21471   MVT SrcVT = N0.getSimpleValueType();  in lowerUINT_TO_FP_vec()
21477   case MVT::v2i32:  in lowerUINT_TO_FP_vec()
21479   case MVT::v4i32:  in lowerUINT_TO_FP_vec()
21480   case MVT::v8i32:  in lowerUINT_TO_FP_vec()
21482   case MVT::v2i64:  in lowerUINT_TO_FP_vec()
21483   case MVT::v4i64:  in lowerUINT_TO_FP_vec()
21495   MVT SrcVT = Src.getSimpleValueType();  in LowerUINT_TO_FP()
21496   MVT DstVT = Op->getSimpleValueType(0);  in LowerUINT_TO_FP()
21500   if (DstVT == MVT::f128)  in LowerUINT_TO_FP()
21511   if (Subtarget.isTargetWin64() && SrcVT == MVT::i128)  in LowerUINT_TO_FP()
21518       (SrcVT == MVT::i32 || (SrcVT == MVT::i64 && Subtarget.is64Bit()))) {  in LowerUINT_TO_FP()
21525   if (SrcVT == MVT::i32 && Subtarget.is64Bit()) {  in LowerUINT_TO_FP()
21526     Src = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Src);  in LowerUINT_TO_FP()
21528       return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {DstVT, MVT::Other},  in LowerUINT_TO_FP()
21540   if (SrcVT == MVT::i64 && DstVT == MVT::f64 && Subtarget.hasSSE2() &&  in LowerUINT_TO_FP()
21545   if (SrcVT == MVT::i32 && Subtarget.hasSSE2() && DstVT != MVT::f80 &&  in LowerUINT_TO_FP()
21548   if (Subtarget.is64Bit() && SrcVT == MVT::i64 &&  in LowerUINT_TO_FP()
21549       (DstVT == MVT::f32 || DstVT == MVT::f64))  in LowerUINT_TO_FP()
21553   SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64, 8);  in LowerUINT_TO_FP()
21558   if (SrcVT == MVT::i32) {  in LowerUINT_TO_FP()
21562     SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, dl, MVT::i32),  in LowerUINT_TO_FP()
21565         BuildFILD(DstVT, MVT::i64, dl, Store2, StackSlot, MPI, SlotAlign, DAG);  in LowerUINT_TO_FP()
21572   assert(SrcVT == MVT::i64 && "Unexpected type in UINT_TO_FP");  in LowerUINT_TO_FP()
21578     ValueToStore = DAG.getBitcast(MVT::f64, ValueToStore);  in LowerUINT_TO_FP()
21585   SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);  in LowerUINT_TO_FP()
21588       DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops, MVT::i64, MPI,  in LowerUINT_TO_FP()
21595       dl, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i64),  in LowerUINT_TO_FP()
21596       Op.getOperand(OpNo), DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);  in LowerUINT_TO_FP()
21612       ISD::EXTLOAD, dl, MVT::f80, Chain, FudgePtr,  in LowerUINT_TO_FP()
21613       MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,  in LowerUINT_TO_FP()
21619     SDValue Add = DAG.getNode(ISD::STRICT_FADD, dl, {MVT::f80, MVT::Other},  in LowerUINT_TO_FP()
21622     if (DstVT == MVT::f80)  in LowerUINT_TO_FP()
21624     return DAG.getNode(ISD::STRICT_FP_ROUND, dl, {DstVT, MVT::Other},  in LowerUINT_TO_FP()
21627   SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::f80, Fild, Fudge);  in LowerUINT_TO_FP()
21649   if (TheVT != MVT::f32 && TheVT != MVT::f64 && TheVT != MVT::f80) {  in FP_TO_INTHelper()
21658   bool UnsignedFixup = !IsSigned && DstTy == MVT::i64;  in FP_TO_INTHelper()
21662   if (!IsSigned && DstTy != MVT::i64) {  in FP_TO_INTHelper()
21665     assert(DstTy == MVT::i32 && "Unexpected FP_TO_UINT");  in FP_TO_INTHelper()
21666     DstTy = MVT::i64;  in FP_TO_INTHelper()
21669   assert(DstTy.getSimpleVT() <= MVT::i64 &&  in FP_TO_INTHelper()
21670          DstTy.getSimpleVT() >= MVT::i16 &&  in FP_TO_INTHelper()
21706     if (TheVT == MVT::f64)  in FP_TO_INTHelper()
21710     else if (TheVT == MVT::f80)  in FP_TO_INTHelper()
21741     SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Cmp);  in FP_TO_INTHelper()
21742     SDValue Const63 = DAG.getConstant(63, DL, MVT::i8);  in FP_TO_INTHelper()
21743     Adjust = DAG.getNode(ISD::SHL, DL, MVT::i64, Zext, Const63);  in FP_TO_INTHelper()
21749       Value = DAG.getNode(ISD::STRICT_FSUB, DL, { TheVT, MVT::Other},  in FP_TO_INTHelper()
21761     assert(DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!");  in FP_TO_INTHelper()
21763     SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);  in FP_TO_INTHelper()
21779                                          DAG.getVTList(MVT::Other),  in FP_TO_INTHelper()
21787     Res = DAG.getNode(ISD::XOR, DL, MVT::i64, Res, Adjust);  in FP_TO_INTHelper()
21794   MVT VT = Op.getSimpleValueType();  in LowerAVXExtend()
21796   MVT InVT = In.getSimpleValueType();  in LowerAVXExtend()
21805   assert((VT.getVectorElementType() == MVT::i16 ||  in LowerAVXExtend()
21806           VT.getVectorElementType() == MVT::i32 ||  in LowerAVXExtend()
21807           VT.getVectorElementType() == MVT::i64) &&  in LowerAVXExtend()
21809   assert((InVT.getVectorElementType() == MVT::i8 ||  in LowerAVXExtend()
21810           InVT.getVectorElementType() == MVT::i16 ||  in LowerAVXExtend()
21811           InVT.getVectorElementType() == MVT::i32) &&  in LowerAVXExtend()
21816   if (VT == MVT::v32i16 && !Subtarget.hasBWI()) {  in LowerAVXExtend()
21817     assert(InVT == MVT::v32i8 && "Unexpected VT!");  in LowerAVXExtend()
21836   MVT HalfVT = VT.getHalfNumVectorElementsVT();  in LowerAVXExtend()
21855 static SDValue SplitAndExtendv16i1(unsigned ExtOpc, MVT VT, SDValue In,  in SplitAndExtendv16i1()
21857   assert((VT == MVT::v16i8 || VT == MVT::v16i16) && "Unexpected VT.");  in SplitAndExtendv16i1()
21858   SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In,  in SplitAndExtendv16i1()
21860   SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In,  in SplitAndExtendv16i1()
21862   Lo = DAG.getNode(ExtOpc, dl, MVT::v8i16, Lo);  in SplitAndExtendv16i1()
21863   Hi = DAG.getNode(ExtOpc, dl, MVT::v8i16, Hi);  in SplitAndExtendv16i1()
21864   SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i16, Lo, Hi);  in SplitAndExtendv16i1()
21871   MVT VT = Op->getSimpleValueType(0);  in LowerZERO_EXTEND_Mask()
21873   MVT InVT = In.getSimpleValueType();  in LowerZERO_EXTEND_Mask()
21874   assert(InVT.getVectorElementType() == MVT::i1 && "Unexpected input type!");  in LowerZERO_EXTEND_Mask()
21880   if (VT.getVectorElementType() != MVT::i8) {  in LowerZERO_EXTEND_Mask()
21887   MVT ExtVT = VT;  in LowerZERO_EXTEND_Mask()
21893     ExtVT = MVT::getVectorVT(MVT::i32, NumElts);  in LowerZERO_EXTEND_Mask()
21897   MVT WideVT = ExtVT;  in LowerZERO_EXTEND_Mask()
21900     InVT = MVT::getVectorVT(MVT::i1, NumElts);  in LowerZERO_EXTEND_Mask()
21903     WideVT = MVT::getVectorVT(ExtVT.getVectorElementType(),  in LowerZERO_EXTEND_Mask()
21914     WideVT = MVT::getVectorVT(MVT::i8, NumElts);  in LowerZERO_EXTEND_Mask()
21929   MVT SVT = In.getSimpleValueType();  in LowerZERO_EXTEND()
21931   if (SVT.getVectorElementType() == MVT::i1)  in LowerZERO_EXTEND()
21979   EVT InVT = MVT::i16, OutVT = MVT::i8;  in truncateVectorWithPACK()
21982     InVT = MVT::i32;  in truncateVectorWithPACK()
21983     OutVT = MVT::i16;  in truncateVectorWithPACK()
22051   MVT VT = Op.getSimpleValueType();  in LowerTruncateVecI1()
22053   MVT InVT = In.getSimpleValueType();  in LowerTruncateVecI1()
22055   assert(VT.getVectorElementType() == MVT::i1 && "Unexpected vector type.");  in LowerTruncateVecI1()
22065         MVT ExtVT = MVT::getVectorVT(MVT::i16, InVT.getSizeInBits()/16);  in LowerTruncateVecI1()
22090       if (InVT == MVT::v16i8) {  in LowerTruncateVecI1()
22091         Lo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, MVT::v8i32, In);  in LowerTruncateVecI1()
22095         Hi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, MVT::v8i32, Hi);  in LowerTruncateVecI1()
22097         assert(InVT == MVT::v16i16 && "Unexpected VT!");  in LowerTruncateVecI1()
22103       Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Lo);  in LowerTruncateVecI1()
22104       Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Hi);  in LowerTruncateVecI1()
22110     MVT EltVT = Subtarget.hasVLX() ? MVT::i32 : MVT::getIntegerVT(512/NumElts);  in LowerTruncateVecI1()
22111     MVT ExtVT = MVT::getVectorVT(EltVT, NumElts);  in LowerTruncateVecI1()
22130   MVT VT = Op.getSimpleValueType();  in LowerTRUNCATE()
22132   MVT InVT = In.getSimpleValueType();  in LowerTRUNCATE()
22141     if ((InVT == MVT::v8i64 || InVT == MVT::v16i32 || InVT == MVT::v16i64) &&  in LowerTRUNCATE()
22143       assert((InVT == MVT::v16i64 || Subtarget.hasVLX()) &&  in LowerTRUNCATE()
22163   if (VT.getVectorElementType() == MVT::i1)  in LowerTRUNCATE()
22168     if (InVT == MVT::v32i16 && !Subtarget.hasBWI()) {  in LowerTRUNCATE()
22169       assert(VT == MVT::v32i8 && "Unexpected VT!");  in LowerTRUNCATE()
22177     if (InVT != MVT::v16i16 || Subtarget.hasBWI() ||  in LowerTRUNCATE()
22204   if ((VT == MVT::v4i32) && (InVT == MVT::v4i64)) {  in LowerTRUNCATE()
22208       In = DAG.getBitcast(MVT::v8i32, In);  in LowerTRUNCATE()
22209       In = DAG.getVectorShuffle(MVT::v8i32, DL, In, In, ShufMask);  in LowerTRUNCATE()
22214     SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,  in LowerTRUNCATE()
22216     SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,  in LowerTRUNCATE()
22219     return DAG.getVectorShuffle(VT, DL, DAG.getBitcast(MVT::v4i32, OpLo),  in LowerTRUNCATE()
22220                                 DAG.getBitcast(MVT::v4i32, OpHi), ShufMask);  in LowerTRUNCATE()
22223   if ((VT == MVT::v8i16) && (InVT == MVT::v8i32)) {  in LowerTRUNCATE()
22231       In = DAG.getBitcast(MVT::v32i8, In);  in LowerTRUNCATE()
22232       In = DAG.getVectorShuffle(MVT::v32i8, DL, In, In, ShufMask1);  in LowerTRUNCATE()
22233       In = DAG.getBitcast(MVT::v4i64, In);  in LowerTRUNCATE()
22236       In = DAG.getVectorShuffle(MVT::v4i64, DL, In, In, ShufMask2);  in LowerTRUNCATE()
22237       In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,  in LowerTRUNCATE()
22239       return DAG.getBitcast(MVT::v8i16, In);  in LowerTRUNCATE()
22242     SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,  in LowerTRUNCATE()
22244     SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,  in LowerTRUNCATE()
22250     OpLo = DAG.getBitcast(MVT::v8i16, OpLo);  in LowerTRUNCATE()
22251     OpHi = DAG.getBitcast(MVT::v8i16, OpHi);  in LowerTRUNCATE()
22253     OpLo = DAG.getVectorShuffle(MVT::v8i16, DL, OpLo, OpLo, ShufMask1);  in LowerTRUNCATE()
22254     OpHi = DAG.getVectorShuffle(MVT::v8i16, DL, OpHi, OpHi, ShufMask1);  in LowerTRUNCATE()
22256     OpLo = DAG.getBitcast(MVT::v4i32, OpLo);  in LowerTRUNCATE()
22257     OpHi = DAG.getBitcast(MVT::v4i32, OpHi);  in LowerTRUNCATE()
22261     SDValue res = DAG.getVectorShuffle(MVT::v4i32, DL, OpLo, OpHi, ShufMask2);  in LowerTRUNCATE()
22262     return DAG.getBitcast(MVT::v8i16, res);  in LowerTRUNCATE()
22265   if (VT == MVT::v16i8 && InVT == MVT::v16i16) {  in LowerTRUNCATE()
22269     SDValue InLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i16, In,  in LowerTRUNCATE()
22271     SDValue InHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i16, In,  in LowerTRUNCATE()
22281 static SDValue expandFP_TO_UINT_SSE(MVT VT, SDValue Src, const SDLoc &dl,  in expandFP_TO_UINT_SSE()
22284   MVT SrcVT = Src.getSimpleValueType();  in expandFP_TO_UINT_SSE()
22305   if (VT == MVT::v8i32 && !Subtarget.hasAVX2()) {  in expandFP_TO_UINT_SSE()
22312                   DAG.getTargetConstant(DstBits - 1, dl, MVT::i8));  in expandFP_TO_UINT_SSE()
22321   MVT VT = Op->getSimpleValueType(0);  in LowerFP_TO_INT()
22324   MVT SrcVT = Src.getSimpleValueType();  in LowerFP_TO_INT()
22329     MVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32;  in LowerFP_TO_INT()
22331       return DAG.getNode(Op.getOpcode(), dl, {VT, MVT::Other},  in LowerFP_TO_INT()
22333                                              {NVT, MVT::Other}, {Chain, Src})});  in LowerFP_TO_INT()
22341     if (VT == MVT::v2i1 && SrcVT == MVT::v2f64) {  in LowerFP_TO_INT()
22342       MVT ResVT = MVT::v4i32;  in LowerFP_TO_INT()
22343       MVT TruncVT = MVT::v4i1;  in LowerFP_TO_INT()
22353         ResVT = MVT::v8i32;  in LowerFP_TO_INT()
22354         TruncVT = MVT::v8i1;  in LowerFP_TO_INT()
22359         SDValue Tmp = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v8f64)  in LowerFP_TO_INT()
22360                                : DAG.getUNDEF(MVT::v8f64);  in LowerFP_TO_INT()
22361         Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8f64, Tmp, Src,  in LowerFP_TO_INT()
22365         Res = DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {Chain, Src});  in LowerFP_TO_INT()
22372       Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i1, Res,  in LowerFP_TO_INT()
22379     if (Subtarget.hasFP16() && SrcVT.getVectorElementType() == MVT::f16) {  in LowerFP_TO_INT()
22380       if (VT == MVT::v8i16 || VT == MVT::v16i16 || VT == MVT::v32i16)  in LowerFP_TO_INT()
22383       MVT ResVT = VT;  in LowerFP_TO_INT()
22384       MVT EleVT = VT.getVectorElementType();  in LowerFP_TO_INT()
22385       if (EleVT != MVT::i64)  in LowerFP_TO_INT()
22386         ResVT = EleVT == MVT::i32 ? MVT::v4i32 : MVT::v8i16;  in LowerFP_TO_INT()
22388       if (SrcVT != MVT::v8f16) {  in LowerFP_TO_INT()
22391         SmallVector<SDValue, 4> Ops(SrcVT == MVT::v2f16 ? 4 : 2, Tmp);  in LowerFP_TO_INT()
22393         Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f16, Ops);  in LowerFP_TO_INT()
22399                           dl, {ResVT, MVT::Other}, {Chain, Src});  in LowerFP_TO_INT()
22408         ResVT = MVT::getVectorVT(EleVT, 8);  in LowerFP_TO_INT()
22421     if (VT == MVT::v8i16 && (SrcVT == MVT::v8f32 || SrcVT == MVT::v8f64)) {  in LowerFP_TO_INT()
22425                           dl, {MVT::v8i32, MVT::Other}, {Chain, Src});  in LowerFP_TO_INT()
22429                           MVT::v8i32, Src);  in LowerFP_TO_INT()
22433       Res = DAG.getNode(ISD::TRUNCATE, dl, MVT::v8i16, Res);  in LowerFP_TO_INT()
22441     if (VT == MVT::v8i32 && SrcVT == MVT::v8f64) {  in LowerFP_TO_INT()
22448     if ((VT == MVT::v4i32 || VT == MVT::v8i32) &&  in LowerFP_TO_INT()
22449         (SrcVT == MVT::v4f64 || SrcVT == MVT::v4f32 || SrcVT == MVT::v8f32) &&  in LowerFP_TO_INT()
22453       MVT WideVT = SrcVT == MVT::v4f64 ? MVT::v8f64 : MVT::v16f32;  in LowerFP_TO_INT()
22454       MVT ResVT = SrcVT == MVT::v4f64 ? MVT::v8i32 : MVT::v16i32;  in LowerFP_TO_INT()
22464         Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, dl, {ResVT, MVT::Other},  in LowerFP_TO_INT()
22480     if ((VT == MVT::v2i64 || VT == MVT::v4i64) &&  in LowerFP_TO_INT()
22481         (SrcVT == MVT::v2f64 || SrcVT == MVT::v4f64 || SrcVT == MVT::v4f32) &&  in LowerFP_TO_INT()
22484       MVT WideVT = SrcVT == MVT::v4f32 ? MVT::v8f32 : MVT::v8f64;  in LowerFP_TO_INT()
22494         Res = DAG.getNode(Op.getOpcode(), dl, {MVT::v8i64, MVT::Other},  in LowerFP_TO_INT()
22498         Res = DAG.getNode(Op.getOpcode(), dl, MVT::v8i64, Src);  in LowerFP_TO_INT()
22509     if (VT == MVT::v2i64 && SrcVT == MVT::v2f32) {  in LowerFP_TO_INT()
22516         SDValue Zero = DAG.getConstantFP(0.0, dl, MVT::v2f32);  in LowerFP_TO_INT()
22517         SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f32,  in LowerFP_TO_INT()
22519         Tmp = DAG.getNode(Op.getOpcode(), dl, {MVT::v8i64, MVT::Other},  in LowerFP_TO_INT()
22522         Tmp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i64, Tmp,  in LowerFP_TO_INT()
22528       SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,  in LowerFP_TO_INT()
22529                                 DAG.getUNDEF(MVT::v2f32));  in LowerFP_TO_INT()
22533         return DAG.getNode(Opc, dl, {VT, MVT::Other}, {Op->getOperand(0), Tmp});  in LowerFP_TO_INT()
22541     if ((VT == MVT::v4i32 && SrcVT == MVT::v4f32) ||  in LowerFP_TO_INT()
22542         (VT == MVT::v4i32 && SrcVT == MVT::v4f64) ||  in LowerFP_TO_INT()
22543         (VT == MVT::v8i32 && SrcVT == MVT::v8f32)) {  in LowerFP_TO_INT()
22562     if (!IsStrict && ((VT == MVT::i32 && !Subtarget.is64Bit()) ||  in LowerFP_TO_INT()
22563                       (VT == MVT::i64 && Subtarget.is64Bit()))) {  in LowerFP_TO_INT()
22568       MVT SrcVecVT = MVT::getVectorVT(SrcVT, 128 / SrcVT.getScalarSizeInBits());  in LowerFP_TO_INT()
22588           ISD::SRA, dl, VT, Small, DAG.getConstant(DstBits - 1, dl, MVT::i8));  in LowerFP_TO_INT()
22594     if (VT == MVT::i64)  in LowerFP_TO_INT()
22597     assert(VT == MVT::i32 && "Unexpected VT!");  in LowerFP_TO_INT()
22604         Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {MVT::i64, MVT::Other},  in LowerFP_TO_INT()
22608         Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i64, Src);  in LowerFP_TO_INT()
22625   if (VT == MVT::i16 && (UseSSEReg || SrcVT == MVT::f128)) {  in LowerFP_TO_INT()
22628       Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {MVT::i32, MVT::Other},  in LowerFP_TO_INT()
22632       Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Src);  in LowerFP_TO_INT()
22645   if (SrcVT == MVT::f128) {  in LowerFP_TO_INT()
22675   MVT SrcVT = Src.getSimpleValueType();  in LowerLRINT_LLRINT()
22677   if (SrcVT == MVT::f16)  in LowerLRINT_LLRINT()
22693   if (SrcVT != MVT::f32 && SrcVT != MVT::f64 && SrcVT != MVT::f80) {  in LRINT_LLRINTHelper()
22713     assert(DstVT == MVT::i64 && "Invalid LRINT/LLRINT to lower!");  in LRINT_LLRINTHelper()
22715     SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);  in LRINT_LLRINTHelper()
22724   Chain = DAG.getMemIntrinsicNode(X86ISD::FIST, DL, DAG.getVTList(MVT::Other),  in LRINT_LLRINTHelper()
22763     TmpVT = MVT::i32;  in LowerFP_TO_INT_SAT()
22770     TmpVT = MVT::i64;  in LowerFP_TO_INT_SAT()
22885   MVT VT = Op.getSimpleValueType();  in LowerFP_EXTEND()
22888   MVT SVT = In.getSimpleValueType();  in LowerFP_EXTEND()
22890   if (VT == MVT::f128 || (SVT == MVT::f16 && VT == MVT::f80))  in LowerFP_EXTEND()
22893   if (SVT == MVT::f16) {  in LowerFP_EXTEND()
22897     if (VT != MVT::f32) {  in LowerFP_EXTEND()
22900             ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other},  in LowerFP_EXTEND()
22902                                 {MVT::f32, MVT::Other}, {Chain, In})});  in LowerFP_EXTEND()
22905                          DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, In));  in LowerFP_EXTEND()
22911     In = DAG.getBitcast(MVT::i16, In);  in LowerFP_EXTEND()
22912     In = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v8i16,  in LowerFP_EXTEND()
22913                      getZeroVector(MVT::v8i16, Subtarget, DAG, DL), In,  in LowerFP_EXTEND()
22917       Res = DAG.getNode(X86ISD::STRICT_CVTPH2PS, DL, {MVT::v4f32, MVT::Other},  in LowerFP_EXTEND()
22921       Res = DAG.getNode(X86ISD::CVTPH2PS, DL, MVT::v4f32, In,  in LowerFP_EXTEND()
22922                         DAG.getTargetConstant(4, DL, MVT::i32));  in LowerFP_EXTEND()
22924     Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, Res,  in LowerFP_EXTEND()
22934   if (SVT.getVectorElementType() == MVT::f16) {  in LowerFP_EXTEND()
22936     if (SVT == MVT::v2f16)  in LowerFP_EXTEND()
22937       In = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f16, In,  in LowerFP_EXTEND()
22938                        DAG.getUNDEF(MVT::v2f16));  in LowerFP_EXTEND()
22939     SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8f16, In,  in LowerFP_EXTEND()
22940                               DAG.getUNDEF(MVT::v4f16));  in LowerFP_EXTEND()
22942       return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other},  in LowerFP_EXTEND()
22945   } else if (VT == MVT::v4f64 || VT == MVT::v8f64) {  in LowerFP_EXTEND()
22949   assert(SVT == MVT::v2f32 && "Only customize MVT::v2f32 type legalization!");  in LowerFP_EXTEND()
22952       DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32, In, DAG.getUNDEF(SVT));  in LowerFP_EXTEND()
22954     return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other},  in LowerFP_EXTEND()
22965   MVT VT = Op.getSimpleValueType();  in LowerFP_ROUND()
22966   MVT SVT = In.getSimpleValueType();  in LowerFP_ROUND()
22968   if (SVT == MVT::f128 || (VT == MVT::f16 && SVT == MVT::f80))  in LowerFP_ROUND()
22971   if (VT.getScalarType() == MVT::f16 && !Subtarget.hasFP16()) {  in LowerFP_ROUND()
22972     if (!Subtarget.hasF16C() || SVT.getScalarType() != MVT::f32)  in LowerFP_ROUND()
22980                                         MVT::i32);  in LowerFP_ROUND()
22982       Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v4f32,  in LowerFP_ROUND()
22983                         DAG.getConstantFP(0, DL, MVT::v4f32), In,  in LowerFP_ROUND()
22985       Res = DAG.getNode(X86ISD::STRICT_CVTPS2PH, DL, {MVT::v8i16, MVT::Other},  in LowerFP_ROUND()
22990       Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, In);  in LowerFP_ROUND()
22991       Res = DAG.getNode(X86ISD::CVTPS2PH, DL, MVT::v8i16, Res, Rnd);  in LowerFP_ROUND()
22994     Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i16, Res,  in LowerFP_ROUND()
22996     Res = DAG.getBitcast(MVT::f16, Res);  in LowerFP_ROUND()
23010   assert(Src.getValueType() == MVT::i16 && Op.getValueType() == MVT::f32 &&  in LowerFP16_TO_FP()
23014   SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16,  in LowerFP16_TO_FP()
23015                             DAG.getConstant(0, dl, MVT::v8i16), Src,  in LowerFP16_TO_FP()
23020     Res = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {MVT::v4f32, MVT::Other},  in LowerFP16_TO_FP()
23024     Res = DAG.getNode(X86ISD::CVTPH2PS, dl, MVT::v4f32, Res);  in LowerFP16_TO_FP()
23027   Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,  in LowerFP16_TO_FP()
23039   assert(Src.getValueType() == MVT::f32 && Op.getValueType() == MVT::i16 &&  in LowerFP_TO_FP16()
23045     Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v4f32,  in LowerFP_TO_FP16()
23046                       DAG.getConstantFP(0, dl, MVT::v4f32), Src,  in LowerFP_TO_FP16()
23049         X86ISD::STRICT_CVTPS2PH, dl, {MVT::v8i16, MVT::Other},  in LowerFP_TO_FP16()
23050         {Op.getOperand(0), Res, DAG.getTargetConstant(4, dl, MVT::i32)});  in LowerFP_TO_FP16()
23054     Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, Src);  in LowerFP_TO_FP16()
23055     Res = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Res,  in LowerFP_TO_FP16()
23056                       DAG.getTargetConstant(4, dl, MVT::i32));  in LowerFP_TO_FP16()
23059   Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Res,  in LowerFP_TO_FP16()
23073       RTLIB::getFPROUND(Op.getOperand(0).getValueType(), MVT::bf16);  in LowerFP_TO_BF16()
23075       makeLibCall(DAG, LC, MVT::f32, Op.getOperand(0), CallOptions, DL).first;  in LowerFP_TO_BF16()
23076   return DAG.getNode(ISD::TRUNCATE, DL, MVT::i16,  in LowerFP_TO_BF16()
23077                      DAG.getBitcast(MVT::i32, Res));  in LowerFP_TO_BF16()
23155   assert((Op.getValueType() == MVT::f32 || Op.getValueType() == MVT::f64) &&  in lowerFaddFsub()
23167   MVT VT = Op.getSimpleValueType();  in LowerFROUND()
23200   MVT VT = Op.getSimpleValueType();  in LowerFABSorFNEG()
23202   bool IsF128 = (VT == MVT::f128);  in LowerFABSorFNEG()
23203   assert(VT.isFloatingPoint() && VT != MVT::f80 &&  in LowerFABSorFNEG()
23216   MVT LogicVT = VT;  in LowerFABSorFNEG()
23218     LogicVT = (VT == MVT::f64)   ? MVT::v2f64  in LowerFABSorFNEG()
23219               : (VT == MVT::f32) ? MVT::v4f32  in LowerFABSorFNEG()
23220                                  : MVT::v8f16;  in LowerFABSorFNEG()
23253   MVT VT = Op.getSimpleValueType();  in LowerFCOPYSIGN()
23264   bool IsF128 = (VT == MVT::f128);  in LowerFCOPYSIGN()
23265   assert(VT.isFloatingPoint() && VT != MVT::f80 &&  in LowerFCOPYSIGN()
23277   MVT LogicVT = VT;  in LowerFCOPYSIGN()
23279     LogicVT = (VT == MVT::f64)   ? MVT::v2f64  in LowerFCOPYSIGN()
23280               : (VT == MVT::f32) ? MVT::v4f32  in LowerFCOPYSIGN()
23281                                  : MVT::v8f16;  in LowerFCOPYSIGN()
23319   MVT VT = Op.getSimpleValueType();  in LowerFGETSIGN()
23321   MVT OpVT = N0.getSimpleValueType();  in LowerFGETSIGN()
23322   assert((OpVT == MVT::f32 || OpVT == MVT::f64) &&  in LowerFGETSIGN()
23326   MVT VecVT = (OpVT == MVT::f32 ? MVT::v4f32 : MVT::v2f64);  in LowerFGETSIGN()
23328   Res = DAG.getNode(X86ISD::MOVMSK, dl, MVT::i32, Res);  in LowerFGETSIGN()
23342     Src = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Src);  in getBT()
23352   if (Src.getValueType() == MVT::i64 &&  in getBT()
23354     Src = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Src);  in getBT()
23372   return DAG.getNode(X86ISD::BT, DL, MVT::i32, Src, BitNo);  in getBT()
23378   return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,  in getSETCC()
23379                      DAG.getTargetConstant(Cond, dl, MVT::i8), EFLAGS);  in getSETCC()
23391   EVT VT = MVT::Other;  in matchScalarReduction()
23482     return DAG.getNode(X86ISD::CMP, DL, MVT::i32,  in LowerVectorAllZero()
23501     MVT TestVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;  in LowerVectorAllZero()
23503     return DAG.getNode(X86ISD::PTEST, DL, MVT::i32, V, V);  in LowerVectorAllZero()
23511   V = DAG.getBitcast(MVT::v16i8, MaskBits(V));  in LowerVectorAllZero()
23512   V = DAG.getNode(X86ISD::PCMPEQ, DL, MVT::v16i8, V,  in LowerVectorAllZero()
23513                   getZeroVector(MVT::v16i8, Subtarget, DAG, DL));  in LowerVectorAllZero()
23514   V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);  in LowerVectorAllZero()
23515   return DAG.getNode(X86ISD::CMP, DL, MVT::i32, V,  in LowerVectorAllZero()
23516                      DAG.getConstant(0xFFFF, DL, MVT::i32));  in LowerVectorAllZero()
23574       X86CC = DAG.getTargetConstant(CCode, DL, MVT::i8);  in MatchVectorAllZeroTest()
23586         X86CC = DAG.getTargetConstant(CCode, DL, MVT::i8);  in MatchVectorAllZeroTest()
23667     return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,  in EmitTest()
23714     SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);  in EmitTest()
23724     return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,  in EmitTest()
23727   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);  in EmitTest()
23745   assert((CmpVT == MVT::i8 || CmpVT == MVT::i16 ||  in EmitCmp()
23746           CmpVT == MVT::i32 || CmpVT == MVT::i64) && "Unexpected VT!");  in EmitCmp()
23750   if (CmpVT == MVT::i16 && !Subtarget.isAtom() &&  in EmitCmp()
23771       CmpVT = MVT::i32;  in EmitCmp()
23779   if (CmpVT == MVT::i64 && isa<ConstantSDNode>(Op1) && !isX86CCSigned(X86CC) &&  in EmitCmp()
23783     CmpVT = MVT::i32;  in EmitCmp()
23792     SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32);  in EmitCmp()
23801     SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32);  in EmitCmp()
23807   SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32);  in EmitCmp()
23817   if (VT.getScalarType() == MVT::f16)  in isFsqrtCheap()
23847   if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||  in getSqrtEstimate()
23848       (VT == MVT::v4f32 && Subtarget.hasSSE1() && Reciprocal) ||  in getSqrtEstimate()
23849       (VT == MVT::v4f32 && Subtarget.hasSSE2() && !Reciprocal) ||  in getSqrtEstimate()
23850       (VT == MVT::v8f32 && Subtarget.hasAVX()) ||  in getSqrtEstimate()
23851       (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) {  in getSqrtEstimate()
23857     unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RSQRT14 : X86ISD::FRSQRT;  in getSqrtEstimate()
23864   if (VT.getScalarType() == MVT::f16 && isTypeLegal(VT) &&  in getSqrtEstimate()
23870     if (VT == MVT::f16) {  in getSqrtEstimate()
23872       SDValue Undef = DAG.getUNDEF(MVT::v8f16);  in getSqrtEstimate()
23873       Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, Op);  in getSqrtEstimate()
23874       Op = DAG.getNode(X86ISD::RSQRT14S, DL, MVT::v8f16, Undef, Op);  in getSqrtEstimate()
23875       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Op, Zero);  in getSqrtEstimate()
23898   if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||  in getRecipEstimate()
23899       (VT == MVT::v4f32 && Subtarget.hasSSE1()) ||  in getRecipEstimate()
23900       (VT == MVT::v8f32 && Subtarget.hasAVX()) ||  in getRecipEstimate()
23901       (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) {  in getRecipEstimate()
23905     if (VT == MVT::f32 && Enabled == ReciprocalEstimate::Unspecified)  in getRecipEstimate()
23912     unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RCP14 : X86ISD::FRCP;  in getRecipEstimate()
23916   if (VT.getScalarType() == MVT::f16 && isTypeLegal(VT) &&  in getRecipEstimate()
23921     if (VT == MVT::f16) {  in getRecipEstimate()
23923       SDValue Undef = DAG.getUNDEF(MVT::v8f16);  in getRecipEstimate()
23924       Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f16, Op);  in getRecipEstimate()
23925       Op = DAG.getNode(X86ISD::RCP14S, DL, MVT::v8f16, Undef, Op);  in getRecipEstimate()
23926       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Op, Zero);  in getRecipEstimate()
23963   if (VT != MVT::i16 && VT != MVT::i32 &&  in BuildSDIVPow2()
23964       !(Subtarget.is64Bit() && VT == MVT::i64))  in BuildSDIVPow2()
23980   SDValue Cmp = DAG.getSetCC(DL, MVT::i8, N0, Zero, ISD::SETLT);  in BuildSDIVPow2()
23990       DAG.getNode(ISD::SRA, DL, VT, CMov, DAG.getConstant(Lg2, DL, MVT::i8));  in BuildSDIVPow2()
24167   MVT VT = Op.getSimpleValueType();  in LowerIntVSETCC_AVX512()
24170   assert(VT.getVectorElementType() == MVT::i1 &&  in LowerIntVSETCC_AVX512()
24193   MVT VT = V.getSimpleValueType();  in incDecVectorConstant()
24194   MVT EltVT = VT.getVectorElementType();  in incDecVectorConstant()
24218 static SDValue LowerVSETCCWithSUBUS(SDValue Op0, SDValue Op1, MVT VT,  in LowerVSETCCWithSUBUS()
24225   MVT VET = VT.getVectorElementType();  in LowerVSETCCWithSUBUS()
24226   if (VET != MVT::i8 && VET != MVT::i16)  in LowerVSETCCWithSUBUS()
24278   MVT VT = Op->getSimpleValueType(0);  in LowerVSETCC()
24284     MVT EltVT = Op0.getSimpleValueType().getVectorElementType();  in LowerVSETCC()
24285     assert(EltVT == MVT::f16 || EltVT == MVT::f32 || EltVT == MVT::f64);  in LowerVSETCC()
24297     if (Subtarget.hasAVX512() && VT.getVectorElementType() == MVT::i1 &&  in LowerVSETCC()
24302       assert(Num <= 16 || (Num == 32 && EltVT == MVT::f16));  in LowerVSETCC()
24330             Opc, dl, {VT, MVT::Other},  in LowerVSETCC()
24331             {Chain, Op0, Op1, DAG.getTargetConstant(1, dl, MVT::i8)}); // LT_OS  in LowerVSETCC()
24360               Opc, dl, {VT, MVT::Other},  in LowerVSETCC()
24361               {Chain, Op0, Op1, DAG.getTargetConstant(CC0, dl, MVT::i8)});  in LowerVSETCC()
24363               Opc, dl, {VT, MVT::Other},  in LowerVSETCC()
24364               {Chain, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8)});  in LowerVSETCC()
24365           Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Cmp0.getValue(1),  in LowerVSETCC()
24369               Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC0, dl, MVT::i8));  in LowerVSETCC()
24371               Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8));  in LowerVSETCC()
24377               Opc, dl, {VT, MVT::Other},  in LowerVSETCC()
24378               {Chain, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)});  in LowerVSETCC()
24382               Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8));  in LowerVSETCC()
24390             Opc, dl, {VT, MVT::Other},  in LowerVSETCC()
24391             {Chain, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)});  in LowerVSETCC()
24395             Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8));  in LowerVSETCC()
24421   MVT VTOp0 = Op0.getSimpleValueType();  in LowerVSETCC()
24434   if (VT.getVectorElementType() == MVT::i1) {  in LowerVSETCC()
24465                        DAG.getTargetConstant(CmpMode, dl, MVT::i8));  in LowerVSETCC()
24597   if (VT == MVT::v2i64) {  in LowerVSETCC()
24604         Op0 = DAG.getConstant(0, dl, MVT::v4i32);  in LowerVSETCC()
24605         Op1 = DAG.getBitcast(MVT::v4i32, Op1);  in LowerVSETCC()
24607         SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);  in LowerVSETCC()
24609         SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);  in LowerVSETCC()
24615         Op0 = DAG.getBitcast(MVT::v4i32, Op0);  in LowerVSETCC()
24616         Op1 = DAG.getConstant(-1, dl, MVT::v4i32);  in LowerVSETCC()
24618         SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);  in LowerVSETCC()
24620         SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);  in LowerVSETCC()
24630                                    dl, MVT::v2i64);  in LowerVSETCC()
24632       Op0 = DAG.getNode(ISD::XOR, dl, MVT::v2i64, Op0, SB);  in LowerVSETCC()
24633       Op1 = DAG.getNode(ISD::XOR, dl, MVT::v2i64, Op1, SB);  in LowerVSETCC()
24636       Op0 = DAG.getBitcast(MVT::v4i32, Op0);  in LowerVSETCC()
24637       Op1 = DAG.getBitcast(MVT::v4i32, Op1);  in LowerVSETCC()
24640       SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);  in LowerVSETCC()
24641       SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1);  in LowerVSETCC()
24646       SDValue EQHi = DAG.getVectorShuffle(MVT::v4i32, dl, EQ, EQ, MaskHi);  in LowerVSETCC()
24647       SDValue GTLo = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo);  in LowerVSETCC()
24648       SDValue GTHi = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);  in LowerVSETCC()
24650       SDValue Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, EQHi, GTLo);  in LowerVSETCC()
24651       Result = DAG.getNode(ISD::OR, dl, MVT::v4i32, Result, GTHi);  in LowerVSETCC()
24654         Result = DAG.getNOT(dl, Result, MVT::v4i32);  in LowerVSETCC()
24665       Op0 = DAG.getBitcast(MVT::v4i32, Op0);  in LowerVSETCC()
24666       Op1 = DAG.getBitcast(MVT::v4i32, Op1);  in LowerVSETCC()
24669       SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1);  in LowerVSETCC()
24673       SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Result, Result, Mask);  in LowerVSETCC()
24674       Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, Result, Shuf);  in LowerVSETCC()
24677         Result = DAG.getNOT(dl, Result, MVT::v4i32);  in LowerVSETCC()
24686     MVT EltVT = VT.getVectorElementType();  in LowerVSETCC()
24716   MVT VT = Op0.getSimpleValueType();  in EmitAVX512Test()
24717   if (!(Subtarget.hasAVX512() && VT == MVT::v16i1) &&  in EmitAVX512Test()
24718       !(Subtarget.hasDQI() && VT == MVT::v8i1) &&  in EmitAVX512Test()
24719       !(Subtarget.hasBWI() && (VT == MVT::v32i1 || VT == MVT::v64i1)))  in EmitAVX512Test()
24733   if (Subtarget.hasDQI() && (VT == MVT::v8i1 || VT == MVT::v16i1))  in EmitAVX512Test()
24735   if (Subtarget.hasBWI() && (VT == MVT::v32i1 || VT == MVT::v64i1))  in EmitAVX512Test()
24742     X86CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8);  in EmitAVX512Test()
24743     return DAG.getNode(X86ISD::KTEST, dl, MVT::i32, LHS, RHS);  in EmitAVX512Test()
24754   X86CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8);  in EmitAVX512Test()
24755   return DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS);  in EmitAVX512Test()
24772       X86CC = DAG.getTargetConstant(X86CondCode, dl, MVT::i8);  in emitFlagsForSetcc()
24801         X86CC = DAG.getTargetConstant(CCode, dl, MVT::i8);  in emitFlagsForSetcc()
24813       SDVTList VTs = DAG.getVTList(Op0.getValueType(), MVT::i32);  in emitFlagsForSetcc()
24819       X86CC = DAG.getTargetConstant(CCode, dl, MVT::i8);  in emitFlagsForSetcc()
24829   X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8);  in emitFlagsForSetcc()
24837   MVT VT = Op->getSimpleValueType(0);  in LowerSETCC()
24841   assert(VT == MVT::i8 && "SetCC type must be 8-bit integer");  in LowerSETCC()
24854   if (Op0.getValueType() == MVT::f128) {  in LowerSETCC()
24855     softenSetCCOperands(DAG, MVT::f128, Op0, Op1, CC, dl, Op0, Op1, Chain,  in LowerSETCC()
24898     SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS);  in LowerSETCC()
24912                     dl, {MVT::i32, MVT::Other}, {Chain, Op0, Op1});  in LowerSETCC()
24915     EFLAGS = DAG.getNode(X86ISD::FCMP, dl, MVT::i32, Op0, Op1);  in LowerSETCC()
24918   SDValue X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8);  in LowerSETCC()
24919   SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS);  in LowerSETCC()
24935   Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),  in LowerSETCCCARRY()
24938   SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);  in LowerSETCCCARRY()
24985     SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);  in getX86XALUOOp()
25004   assert(Op->getValueType(1) == MVT::i8 && "Unexpected VT!");  in LowerXALUO()
25039   MVT VT = Op1.getSimpleValueType();  in LowerSELECT()
25043     MVT NVT = VT.changeTypeToInteger();  in LowerSELECT()
25062           DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CondOp0, CondOp1,  in LowerSELECT()
25063                       DAG.getTargetConstant(SSECC, DL, MVT::i8));  in LowerSELECT()
25070                                 DAG.getTargetConstant(SSECC, DL, MVT::i8));  in LowerSELECT()
25088         MVT VecVT = VT == MVT::f32 ? MVT::v4f32 : MVT::v2f64;  in LowerSELECT()
25093         MVT VCmpVT = VT == MVT::f32 ? MVT::v4i32 : MVT::v2i64;  in LowerSELECT()
25109     SDValue Cmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, Cond);  in LowerSELECT()
25149     if (Subtarget.canUseCMOV() && (VT == MVT::i32 || VT == MVT::i64) &&  in LowerSELECT()
25156       SDVTList CmpVTs = DAG.getVTList(CmpOp0.getValueType(), MVT::i32);  in LowerSELECT()
25174                                 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),  in LowerSELECT()
25213     } else if ((VT == MVT::i32 || VT == MVT::i64) && isNullConstant(Op2) &&  in LowerSELECT()
25263     CC = DAG.getTargetConstant(X86Cond, DL, MVT::i8);  in LowerSELECT()
25277         CC = DAG.getTargetConstant(X86CondCode, DL, MVT::i8);  in LowerSELECT()
25285     CC = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);  in LowerSELECT()
25301                       DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), Cond);  in LowerSELECT()
25311   if (Op.getValueType() == MVT::i8 &&  in LowerSELECT()
25329   if ((Op.getValueType() == MVT::i8 && Subtarget.canUseCMOV()) ||  in LowerSELECT()
25330       (Op.getValueType() == MVT::i16 && !X86::mayFoldLoad(Op1, Subtarget) &&  in LowerSELECT()
25332     Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1);  in LowerSELECT()
25333     Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2);  in LowerSELECT()
25335     SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, MVT::i32, Ops);  in LowerSELECT()
25348   MVT VT = Op->getSimpleValueType(0);  in LowerSIGN_EXTEND_Mask()
25350   MVT InVT = In.getSimpleValueType();  in LowerSIGN_EXTEND_Mask()
25351   assert(InVT.getVectorElementType() == MVT::i1 && "Unexpected input type!");  in LowerSIGN_EXTEND_Mask()
25352   MVT VTElt = VT.getVectorElementType();  in LowerSIGN_EXTEND_Mask()
25358   MVT ExtVT = VT;  in LowerSIGN_EXTEND_Mask()
25364     ExtVT = MVT::getVectorVT(MVT::i32, NumElts);  in LowerSIGN_EXTEND_Mask()
25368   MVT WideVT = ExtVT;  in LowerSIGN_EXTEND_Mask()
25371     InVT = MVT::getVectorVT(MVT::i1, NumElts);  in LowerSIGN_EXTEND_Mask()
25374     WideVT = MVT::getVectorVT(ExtVT.getVectorElementType(), NumElts);  in LowerSIGN_EXTEND_Mask()
25378   MVT WideEltVT = WideVT.getVectorElementType();  in LowerSIGN_EXTEND_Mask()
25390     WideVT = MVT::getVectorVT(VTElt, NumElts);  in LowerSIGN_EXTEND_Mask()
25405   MVT InVT = In.getSimpleValueType();  in LowerANY_EXTEND()
25407   if (InVT.getVectorElementType() == MVT::i1)  in LowerANY_EXTEND()
25422   MVT VT = Op->getSimpleValueType(0);  in LowerEXTEND_VECTOR_INREG()
25423   MVT InVT = In.getSimpleValueType();  in LowerEXTEND_VECTOR_INREG()
25425   MVT SVT = VT.getVectorElementType();  in LowerEXTEND_VECTOR_INREG()
25426   MVT InSVT = InVT.getVectorElementType();  in LowerEXTEND_VECTOR_INREG()
25429   if (SVT != MVT::i64 && SVT != MVT::i32 && SVT != MVT::i16)  in LowerEXTEND_VECTOR_INREG()
25431   if (InSVT != MVT::i32 && InSVT != MVT::i16 && InSVT != MVT::i8)  in LowerEXTEND_VECTOR_INREG()
25472     MVT HalfVT = VT.getHalfNumVectorElementsVT();  in LowerEXTEND_VECTOR_INREG()
25496   if (InVT != MVT::v4i32) {  in LowerEXTEND_VECTOR_INREG()
25497     MVT DestVT = VT == MVT::v2i64 ? MVT::v4i32 : VT;  in LowerEXTEND_VECTOR_INREG()
25516                           DAG.getTargetConstant(SignExtShift, dl, MVT::i8));  in LowerEXTEND_VECTOR_INREG()
25519   if (VT == MVT::v2i64) {  in LowerEXTEND_VECTOR_INREG()
25520     assert(Curr.getValueType() == MVT::v4i32 && "Unexpected input VT");  in LowerEXTEND_VECTOR_INREG()
25521     SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32);  in LowerEXTEND_VECTOR_INREG()
25522     SDValue Sign = DAG.getSetCC(dl, MVT::v4i32, Zero, Curr, ISD::SETGT);  in LowerEXTEND_VECTOR_INREG()
25523     SignExt = DAG.getVectorShuffle(MVT::v4i32, dl, SignExt, Sign, {0, 4, 1, 5});  in LowerEXTEND_VECTOR_INREG()
25532   MVT VT = Op->getSimpleValueType(0);  in LowerSIGN_EXTEND()
25534   MVT InVT = In.getSimpleValueType();  in LowerSIGN_EXTEND()
25537   if (InVT.getVectorElementType() == MVT::i1)  in LowerSIGN_EXTEND()
25543   assert((VT.getVectorElementType() == MVT::i16 ||  in LowerSIGN_EXTEND()
25544           VT.getVectorElementType() == MVT::i32 ||  in LowerSIGN_EXTEND()
25545           VT.getVectorElementType() == MVT::i64) &&  in LowerSIGN_EXTEND()
25547   assert((InVT.getVectorElementType() == MVT::i8 ||  in LowerSIGN_EXTEND()
25548           InVT.getVectorElementType() == MVT::i16 ||  in LowerSIGN_EXTEND()
25549           InVT.getVectorElementType() == MVT::i32) &&  in LowerSIGN_EXTEND()
25552   if (VT == MVT::v32i16 && !Subtarget.hasBWI()) {  in LowerSIGN_EXTEND()
25553     assert(InVT == MVT::v32i8 && "Unexpected VT!");  in LowerSIGN_EXTEND()
25568   MVT HalfVT = VT.getHalfNumVectorElementsVT();  in LowerSIGN_EXTEND()
25612   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Ch0, Ch1);  in splitVectorStore()
25617 static SDValue scalarizeVectorStore(StoreSDNode *Store, MVT StoreVT,  in scalarizeVectorStore()
25630   MVT StoreSVT = StoreVT.getScalarType();  in scalarizeVectorStore()
25648   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores);  in scalarizeVectorStore()
25659       StoredVal.getValueType().getVectorElementType() == MVT::i1) {  in LowerStore()
25667     StoredVal = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1,  in LowerStore()
25668                             DAG.getUNDEF(MVT::v16i1), StoredVal,  in LowerStore()
25670     StoredVal = DAG.getBitcast(MVT::i16, StoredVal);  in LowerStore()
25671     StoredVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, StoredVal);  in LowerStore()
25689   MVT StoreVT = StoredVal.getSimpleValueType();  in LowerStore()
25691       ((StoreVT == MVT::v32i16 || StoreVT == MVT::v64i8) &&  in LowerStore()
25716     MVT StVT = Subtarget.is64Bit() && StoreVT.isInteger() ? MVT::i64 : MVT::f64;  in LowerStore()
25717     MVT CastVT = MVT::getVectorVT(StVT, 2);  in LowerStore()
25727   SDVTList Tys = DAG.getVTList(MVT::Other);  in LowerStore()
25729   return DAG.getMemIntrinsicNode(X86ISD::VEXTRACT_STORE, dl, Tys, Ops, MVT::i64,  in LowerStore()
25742   MVT RegVT = Op.getSimpleValueType();  in LowerLoad()
25751   if (RegVT.getVectorElementType() == MVT::i1) {  in LowerLoad()
25757     SDValue NewLd = DAG.getLoad(MVT::i8, dl, Ld->getChain(), Ld->getBasePtr(),  in LowerLoad()
25764     SDValue Val = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, NewLd);  in LowerLoad()
25766                       DAG.getBitcast(MVT::v16i1, Val),  in LowerLoad()
25794       Cond.getOperand(0).getValueType() != MVT::f128 &&  in LowerBRCOND()
25813       SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);  in LowerBRCOND()
25814       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,  in LowerBRCOND()
25821       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,  in LowerBRCOND()
25845               DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);  in LowerBRCOND()
25846           SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);  in LowerBRCOND()
25847           Chain = DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest,  in LowerBRCOND()
25849           CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);  in LowerBRCOND()
25850           return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,  in LowerBRCOND()
25858       SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);  in LowerBRCOND()
25859       SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);  in LowerBRCOND()
25861           DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, Cmp);  in LowerBRCOND()
25862       CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);  in LowerBRCOND()
25863       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,  in LowerBRCOND()
25868       SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);  in LowerBRCOND()
25869       SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);  in LowerBRCOND()
25870       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,  in LowerBRCOND()
25880     SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);  in LowerBRCOND()
25881     return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,  in LowerBRCOND()
25901   return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,  in LowerBRCOND()
25932   MVT SPTy = getPointerTy(DAG.getDataLayout());  in LowerDYNAMIC_STACKALLOC()
25981     SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);  in LowerDYNAMIC_STACKALLOC()
26033       DAG.getConstant(FuncInfo->getVarArgsGPOffset(), DL, MVT::i32), FIN,  in LowerVASTART()
26041       DAG.getConstant(FuncInfo->getVarArgsFPOffset(), DL, MVT::i32), FIN,  in LowerVASTART()
26060   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);  in LowerVASTART()
26087   assert(ArgVT != MVT::f80 && "va_arg for f80 not yet implemented");  in LowerVAARG()
26106                        DAG.getTargetConstant(ArgSize, dl, MVT::i32),  in LowerVAARG()
26107                        DAG.getTargetConstant(ArgMode, dl, MVT::i8),  in LowerVAARG()
26108                        DAG.getTargetConstant(Align, dl, MVT::i32)};  in LowerVAARG()
26109   SDVTList VTs = DAG.getVTList(getPointerTy(DAG.getDataLayout()), MVT::Other);  in LowerVAARG()
26112       VTs, InstOps, MVT::i64, MachinePointerInfo(SV),  in LowerVAARG()
26166 static SDValue getTargetVShiftByConstNode(unsigned Opc, const SDLoc &dl, MVT VT,  in getTargetVShiftByConstNode()
26169   MVT ElementType = VT.getVectorElementType();  in getTargetVShiftByConstNode()
26214                      DAG.getTargetConstant(ShiftAmt, dl, MVT::i8));  in getTargetVShiftByConstNode()
26218 static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT,  in getTargetVShiftNode()
26222   MVT AmtVT = ShAmt.getSimpleValueType();  in getTargetVShiftNode()
26253       ShAmt = DAG.getZExtOrTrunc(ShAmt.getOperand(0), dl, MVT::i32);  in getTargetVShiftNode()
26254       ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, ShAmt);  in getTargetVShiftNode()
26255       ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, dl, MVT::v4i32, ShAmt);  in getTargetVShiftNode()
26256       AmtVT = MVT::v4i32;  in getTargetVShiftNode()
26284     if (AmtVT == MVT::v4i32 && (ShAmt.getOpcode() == X86ISD::VBROADCAST ||  in getTargetVShiftNode()
26286       ShAmt = DAG.getNode(X86ISD::VZEXT_MOVL, SDLoc(ShAmt), MVT::v4i32, ShAmt);  in getTargetVShiftNode()
26289                           MVT::v2i64, ShAmt);  in getTargetVShiftNode()
26292           (128 - AmtVT.getScalarSizeInBits()) / 8, SDLoc(ShAmt), MVT::i8);  in getTargetVShiftNode()
26293       ShAmt = DAG.getBitcast(MVT::v16i8, ShAmt);  in getTargetVShiftNode()
26294       ShAmt = DAG.getNode(X86ISD::VSHLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt,  in getTargetVShiftNode()
26296       ShAmt = DAG.getNode(X86ISD::VSRLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt,  in getTargetVShiftNode()
26306   MVT EltVT = VT.getVectorElementType();  in getTargetVShiftNode()
26307   MVT ShVT = MVT::getVectorVT(EltVT, 128 / EltVT.getSizeInBits());  in getTargetVShiftNode()
26315 static SDValue getMaskNode(SDValue Mask, MVT MaskVT,  in getMaskNode()
26326   if (Mask.getSimpleValueType() == MVT::i64 && Subtarget.is32Bit()) {  in getMaskNode()
26327     assert(MaskVT == MVT::v64i1 && "Expected v64i1 mask!");  in getMaskNode()
26331     Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Mask,  in getMaskNode()
26332                         DAG.getConstant(0, dl, MVT::i32));  in getMaskNode()
26333     Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Mask,  in getMaskNode()
26334                         DAG.getConstant(1, dl, MVT::i32));  in getMaskNode()
26336     Lo = DAG.getBitcast(MVT::v32i1, Lo);  in getMaskNode()
26337     Hi = DAG.getBitcast(MVT::v32i1, Hi);  in getMaskNode()
26339     return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi);  in getMaskNode()
26341     MVT BitcastVT = MVT::getVectorVT(MVT::i1,  in getMaskNode()
26358   MVT VT = Op.getSimpleValueType();  in getVectorMaskingNode()
26359   MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements());  in getVectorMaskingNode()
26389   MVT VT = Op.getSimpleValueType();  in getScalarMaskingNode()
26392   assert(Mask.getValueType() == MVT::i8 && "Unexpect type");  in getScalarMaskingNode()
26393   SDValue IMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v1i1,  in getScalarMaskingNode()
26394                               DAG.getBitcast(MVT::v8i1, Mask),  in getScalarMaskingNode()
26435   MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());  in recoverFramePointer()
26507   MVT VT = Op.getSimpleValueType();  in LowerINTRINSIC_WO_CHAIN()
26526                              DAG.getTargetConstant(RC, dl, MVT::i32));  in LowerINTRINSIC_WO_CHAIN()
26559                              DAG.getTargetConstant(RC, dl, MVT::i32));  in LowerINTRINSIC_WO_CHAIN()
26588           Src3.getValueType() != MVT::i8) {  in LowerINTRINSIC_WO_CHAIN()
26590             cast<ConstantSDNode>(Src3)->getZExtValue() & 0xff, dl, MVT::i8);  in LowerINTRINSIC_WO_CHAIN()
26603                              DAG.getTargetConstant(RC, dl, MVT::i32));  in LowerINTRINSIC_WO_CHAIN()
26614       if (Src4.getValueType() != MVT::i8) {  in LowerINTRINSIC_WO_CHAIN()
26616             cast<ConstantSDNode>(Src4)->getZExtValue() & 0xff, dl, MVT::i8);  in LowerINTRINSIC_WO_CHAIN()
26637                           Src, DAG.getTargetConstant(RC, dl, MVT::i32)),  in LowerINTRINSIC_WO_CHAIN()
26680                             DAG.getTargetConstant(RC, dl, MVT::i32)),  in LowerINTRINSIC_WO_CHAIN()
26718                             DAG.getTargetConstant(RC, dl, MVT::i32));  in LowerINTRINSIC_WO_CHAIN()
26752                               DAG.getTargetConstant(RC, dl, MVT::i32));  in LowerINTRINSIC_WO_CHAIN()
26838       MVT VT = Op.getSimpleValueType();  in LowerINTRINSIC_WO_CHAIN()
26853                               DAG.getTargetConstant(RC, dl, MVT::i32));  in LowerINTRINSIC_WO_CHAIN()
26870       SDValue FPclass = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Imm);  in LowerINTRINSIC_WO_CHAIN()
26875       SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,  in LowerINTRINSIC_WO_CHAIN()
26876                                 DAG.getConstant(0, dl, MVT::v8i1),  in LowerINTRINSIC_WO_CHAIN()
26878       return DAG.getBitcast(MVT::i8, Ins);  in LowerINTRINSIC_WO_CHAIN()
26882       MVT MaskVT = Op.getSimpleValueType();  in LowerINTRINSIC_WO_CHAIN()
26910           Cmp = DAG.getNode(IntrData->Opc1, dl, MVT::v1i1, Src1, Src2, CC, Sae);  in LowerINTRINSIC_WO_CHAIN()
26916         Cmp = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Src2, CC);  in LowerINTRINSIC_WO_CHAIN()
26922       SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,  in LowerINTRINSIC_WO_CHAIN()
26923                                 DAG.getConstant(0, dl, MVT::v8i1),  in LowerINTRINSIC_WO_CHAIN()
26925       return DAG.getBitcast(MVT::i8, Ins);  in LowerINTRINSIC_WO_CHAIN()
26935       SDValue Comi = DAG.getNode(IntrData->Opc0, dl, MVT::i32, LHS, RHS);  in LowerINTRINSIC_WO_CHAIN()
26941         SetCC = DAG.getNode(ISD::AND, dl, MVT::i8, SetCC, SetNP);  in LowerINTRINSIC_WO_CHAIN()
26947         SetCC = DAG.getNode(ISD::OR, dl, MVT::i8, SetCC, SetP);  in LowerINTRINSIC_WO_CHAIN()
26962       return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);  in LowerINTRINSIC_WO_CHAIN()
26972         FCmp = DAG.getNode(X86ISD::FSETCCM, dl, MVT::v1i1, LHS, RHS,  in LowerINTRINSIC_WO_CHAIN()
26973                            DAG.getTargetConstant(CondVal, dl, MVT::i8));  in LowerINTRINSIC_WO_CHAIN()
26975         FCmp = DAG.getNode(X86ISD::FSETCCM_SAE, dl, MVT::v1i1, LHS, RHS,  in LowerINTRINSIC_WO_CHAIN()
26976                            DAG.getTargetConstant(CondVal, dl, MVT::i8), Sae);  in LowerINTRINSIC_WO_CHAIN()
26981       SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1,  in LowerINTRINSIC_WO_CHAIN()
26982                                 DAG.getConstant(0, dl, MVT::v16i1),  in LowerINTRINSIC_WO_CHAIN()
26984       return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32,  in LowerINTRINSIC_WO_CHAIN()
26985                          DAG.getBitcast(MVT::i16, Ins));  in LowerINTRINSIC_WO_CHAIN()
26990       assert(ShAmt.getValueType() == MVT::i32 &&  in LowerINTRINSIC_WO_CHAIN()
26999       ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, ShAmt);  in LowerINTRINSIC_WO_CHAIN()
27050           DAG.getTargetConstant(Round->getZExtValue() & 0xf, dl, MVT::i32);  in LowerINTRINSIC_WO_CHAIN()
27060           DAG.getTargetConstant(Round->getZExtValue() & 0xf, dl, MVT::i32);  in LowerINTRINSIC_WO_CHAIN()
27075       SDVTList CFVTs = DAG.getVTList(Op->getValueType(0), MVT::i32);  in LowerINTRINSIC_WO_CHAIN()
27076       SDVTList VTs = DAG.getVTList(Op.getOperand(2).getValueType(), MVT::i32);  in LowerINTRINSIC_WO_CHAIN()
27086                                     DAG.getConstant(-1, dl, MVT::i8));  in LowerINTRINSIC_WO_CHAIN()
27105       MVT SrcVT = Src.getSimpleValueType();  in LowerINTRINSIC_WO_CHAIN()
27106       MVT MaskVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorNumElements());  in LowerINTRINSIC_WO_CHAIN()
27120       MVT SrcVT = Src.getSimpleValueType();  in LowerINTRINSIC_WO_CHAIN()
27121       MVT MaskVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorNumElements());  in LowerINTRINSIC_WO_CHAIN()
27235     SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS);  in LowerINTRINSIC_WO_CHAIN()
27237     return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);  in LowerINTRINSIC_WO_CHAIN()
27296     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);  in LowerINTRINSIC_WO_CHAIN()
27299     return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);  in LowerINTRINSIC_WO_CHAIN()
27311     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);  in LowerINTRINSIC_WO_CHAIN()
27324     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);  in LowerINTRINSIC_WO_CHAIN()
27331     MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());  in LowerINTRINSIC_WO_CHAIN()
27387     MVT MaskVT = Op.getSimpleValueType();  in LowerINTRINSIC_WO_CHAIN()
27389     SDVTList VTs = DAG.getVTList(MVT::Untyped, MVT::Other);  in LowerINTRINSIC_WO_CHAIN()
27422                          DAG.getTargetConstant(ShiftAmount, DL, MVT::i32));  in LowerINTRINSIC_WO_CHAIN()
27457     ShAmt = DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, ShAmt);  in LowerINTRINSIC_WO_CHAIN()
27492   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Other);  in getAVX2GatherNode()
27515   MVT VT = Op.getSimpleValueType();  in getGatherNode()
27526   MVT MaskVT = MVT::getVectorVT(MVT::i1, MinElts);  in getGatherNode()
27533   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Other);  in getGatherNode()
27563   MVT MaskVT = MVT::getVectorVT(MVT::i1, MinElts);  in getScatterNode()
27572   SDVTList VTs = DAG.getVTList(MVT::Other);  in getScatterNode()
27592   SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32);  in getPrefetchNode()
27593   SDValue Segment = DAG.getRegister(0, MVT::i32);  in getPrefetchNode()
27594   MVT MaskVT =  in getPrefetchNode()
27595     MVT::getVectorVT(MVT::i1, Index.getSimpleValueType().getVectorNumElements());  in getPrefetchNode()
27598   SDNode *Res = DAG.getMachineNode(Opc, dl, MVT::Other, Ops);  in getPrefetchNode()
27625   SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);  in expandIntrinsicWChainHelper()
27634     LO = DAG.getCopyFromReg(Chain, DL, X86::RAX, MVT::i64, SDValue(N1, 1));  in expandIntrinsicWChainHelper()
27635     HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64,  in expandIntrinsicWChainHelper()
27638     LO = DAG.getCopyFromReg(Chain, DL, X86::EAX, MVT::i32, SDValue(N1, 1));  in expandIntrinsicWChainHelper()
27639     HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32,  in expandIntrinsicWChainHelper()
27647     SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI,  in expandIntrinsicWChainHelper()
27648                               DAG.getConstant(32, DL, MVT::i8));  in expandIntrinsicWChainHelper()
27649     Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp));  in expandIntrinsicWChainHelper()
27656   SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops);  in expandIntrinsicWChainHelper()
27681   SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32, Glue);  in getReadTimeStampCounter()
27736   SDVTList VTs = DAG.getVTList(MVT::Other);  in EmitTruncSStore()
27748   SDVTList VTs = DAG.getVTList(MVT::Other);  in EmitMaskedTruncSStore()
27769         SDValue CopyRBP = DAG.getCopyFromReg(Chain, dl, X86::RBP, MVT::i64);  in LowerINTRINSIC_W_CHAIN()
27771             SDValue(DAG.getMachineNode(X86::SUB64ri8, dl, MVT::i64, CopyRBP,  in LowerINTRINSIC_W_CHAIN()
27772                                        DAG.getTargetConstant(8, dl, MVT::i32)),  in LowerINTRINSIC_W_CHAIN()
27784             DAG.getFrameIndex(*X86FI->getSwiftAsyncContextFrameIdx(), MVT::i32);  in LowerINTRINSIC_W_CHAIN()
27797       SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);  in LowerINTRINSIC_W_CHAIN()
27800                          DAG.getConstant(0, dl, MVT::i32));  in LowerINTRINSIC_W_CHAIN()
27806       return DAG.getNode(X86ISD::WRPKRU, dl, MVT::Other,  in LowerINTRINSIC_W_CHAIN()
27808                          DAG.getConstant(0, dl, MVT::i32),  in LowerINTRINSIC_W_CHAIN()
27809                          DAG.getConstant(0, dl, MVT::i32));  in LowerINTRINSIC_W_CHAIN()
27835       SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);  in LowerINTRINSIC_W_CHAIN()
27863       SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);  in LowerINTRINSIC_W_CHAIN()
27885       SDVTList VTs = DAG.getVTList(MVT::v2i64, MVT::i32, MVT::Other);  in LowerINTRINSIC_W_CHAIN()
27922           {MVT::i32, MVT::v2i64, MVT::v2i64, MVT::v2i64, MVT::v2i64, MVT::v2i64,  in LowerINTRINSIC_W_CHAIN()
27923            MVT::v2i64, MVT::v2i64, MVT::v2i64, MVT::Other});  in LowerINTRINSIC_W_CHAIN()
27964       SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);  in LowerINTRINSIC_W_CHAIN()
27974       MVT VT = Op.getSimpleValueType();  in LowerINTRINSIC_W_CHAIN()
27981       SDValue Size = DAG.getConstant(VT.getScalarSizeInBits(), DL, MVT::i32);  in LowerINTRINSIC_W_CHAIN()
27984           DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::Other),  in LowerINTRINSIC_W_CHAIN()
28004     SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32, MVT::Other);  in LowerINTRINSIC_W_CHAIN()
28011                      DAG.getTargetConstant(X86::COND_B, dl, MVT::i8),  in LowerINTRINSIC_W_CHAIN()
28089     SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Other);  in LowerINTRINSIC_W_CHAIN()
28117       MVT MaskVT = MVT::getVectorVT(MVT::i1, MemVT.getVectorNumElements());  in LowerINTRINSIC_W_CHAIN()
28132       MVT MaskVT = MVT::getVectorVT(MVT::i1, MemVT.getVectorNumElements());  in LowerINTRINSIC_W_CHAIN()
28206   assert(((FrameReg == X86::RBP && VT == MVT::i64) ||  in LowerFRAMEADDR()
28207           (FrameReg == X86::EBP && VT == MVT::i32)) &&  in LowerFRAMEADDR()
28284   assert(((FrameReg == X86::RBP && PtrVT == MVT::i64) ||  in LowerEH_RETURN()
28285           (FrameReg == X86::EBP && PtrVT == MVT::i32)) &&  in LowerEH_RETURN()
28288   Register StoreAddrReg = (PtrVT == MVT::i64) ? X86::RCX : X86::ECX;  in LowerEH_RETURN()
28297   return DAG.getNode(X86ISD::EH_RETURN, dl, MVT::Other, Chain,  in LowerEH_RETURN()
28315                      DAG.getVTList(MVT::i32, MVT::Other),  in lowerEH_SJLJ_SETJMP()
28322   return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other,  in lowerEH_SJLJ_LONGJMP()
28329   return DAG.getNode(X86ISD::EH_SJLJ_SETUP_DISPATCH, DL, MVT::Other,  in lowerEH_SJLJ_SETUP_DISPATCH()
28363     OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),  in LowerINIT_TRAMPOLINE()
28366     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,  in LowerINIT_TRAMPOLINE()
28367                        DAG.getConstant(2, dl, MVT::i64));  in LowerINIT_TRAMPOLINE()
28374     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,  in LowerINIT_TRAMPOLINE()
28375                        DAG.getConstant(10, dl, MVT::i64));  in LowerINIT_TRAMPOLINE()
28376     OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),  in LowerINIT_TRAMPOLINE()
28379     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,  in LowerINIT_TRAMPOLINE()
28380                        DAG.getConstant(12, dl, MVT::i64));  in LowerINIT_TRAMPOLINE()
28386     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,  in LowerINIT_TRAMPOLINE()
28387                        DAG.getConstant(20, dl, MVT::i64));  in LowerINIT_TRAMPOLINE()
28388     OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),  in LowerINIT_TRAMPOLINE()
28392     Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,  in LowerINIT_TRAMPOLINE()
28393                        DAG.getConstant(22, dl, MVT::i64));  in LowerINIT_TRAMPOLINE()
28394     OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, dl, MVT::i8),  in LowerINIT_TRAMPOLINE()
28397     return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);  in LowerINIT_TRAMPOLINE()
28450     Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,  in LowerINIT_TRAMPOLINE()
28451                        DAG.getConstant(10, dl, MVT::i32));  in LowerINIT_TRAMPOLINE()
28452     Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr);  in LowerINIT_TRAMPOLINE()
28458         DAG.getStore(Root, dl, DAG.getConstant(MOV32ri | N86Reg, dl, MVT::i8),  in LowerINIT_TRAMPOLINE()
28461     Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,  in LowerINIT_TRAMPOLINE()
28462                        DAG.getConstant(1, dl, MVT::i32));  in LowerINIT_TRAMPOLINE()
28467     Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,  in LowerINIT_TRAMPOLINE()
28468                        DAG.getConstant(5, dl, MVT::i32));  in LowerINIT_TRAMPOLINE()
28470         DAG.getStore(Root, dl, DAG.getConstant(JMP, dl, MVT::i8), Addr,  in LowerINIT_TRAMPOLINE()
28473     Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,  in LowerINIT_TRAMPOLINE()
28474                        DAG.getConstant(6, dl, MVT::i32));  in LowerINIT_TRAMPOLINE()
28478     return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);  in LowerINIT_TRAMPOLINE()
28507   MVT VT = Op.getSimpleValueType();  in LowerFLT_ROUNDS_()
28520                                   DAG.getVTList(MVT::Other), Ops, MVT::i16, MPI,  in LowerFLT_ROUNDS_()
28524   SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MPI, Align(2));  in LowerFLT_ROUNDS_()
28529     DAG.getNode(ISD::SRL, DL, MVT::i16,  in LowerFLT_ROUNDS_()
28530                 DAG.getNode(ISD::AND, DL, MVT::i16,  in LowerFLT_ROUNDS_()
28531                             CWD, DAG.getConstant(0xc00, DL, MVT::i16)),  in LowerFLT_ROUNDS_()
28532                 DAG.getConstant(9, DL, MVT::i8));  in LowerFLT_ROUNDS_()
28533   Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, Shift);  in LowerFLT_ROUNDS_()
28535   SDValue LUT = DAG.getConstant(0x2d, DL, MVT::i32);  in LowerFLT_ROUNDS_()
28537     DAG.getNode(ISD::AND, DL, MVT::i32,  in LowerFLT_ROUNDS_()
28538                 DAG.getNode(ISD::SRL, DL, MVT::i32, LUT, Shift),  in LowerFLT_ROUNDS_()
28539                 DAG.getConstant(3, DL, MVT::i32));  in LowerFLT_ROUNDS_()
28564       X86ISD::FNSTCW16m, DL, DAG.getVTList(MVT::Other), Ops, MVT::i16, MMO);  in LowerSET_ROUNDING()
28567   SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MPI);  in LowerSET_ROUNDING()
28569   CWD = DAG.getNode(ISD::AND, DL, MVT::i16, CWD.getValue(0),  in LowerSET_ROUNDING()
28570                     DAG.getConstant(0xf3ff, DL, MVT::i16));  in LowerSET_ROUNDING()
28586     RMBits = DAG.getConstant(FieldVal, DL, MVT::i16);  in LowerSET_ROUNDING()
28601         DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,  in LowerSET_ROUNDING()
28602                     DAG.getNode(ISD::ADD, DL, MVT::i32,  in LowerSET_ROUNDING()
28603                                 DAG.getNode(ISD::SHL, DL, MVT::i32, NewRM,  in LowerSET_ROUNDING()
28604                                             DAG.getConstant(1, DL, MVT::i8)),  in LowerSET_ROUNDING()
28605                                 DAG.getConstant(4, DL, MVT::i32)));  in LowerSET_ROUNDING()
28607         DAG.getNode(ISD::SHL, DL, MVT::i16, DAG.getConstant(0xc9, DL, MVT::i16),  in LowerSET_ROUNDING()
28609     RMBits = DAG.getNode(ISD::AND, DL, MVT::i16, Shifted,  in LowerSET_ROUNDING()
28610                          DAG.getConstant(0xc00, DL, MVT::i16));  in LowerSET_ROUNDING()
28614   CWD = DAG.getNode(ISD::OR, DL, MVT::i16, CWD, RMBits);  in LowerSET_ROUNDING()
28622       X86ISD::FLDCW16m, DL, DAG.getVTList(MVT::Other), OpsLD, MVT::i16, MMOL);  in LowerSET_ROUNDING()
28629         ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,  in LowerSET_ROUNDING()
28630         DAG.getTargetConstant(Intrinsic::x86_sse_stmxcsr, DL, MVT::i32),  in LowerSET_ROUNDING()
28634     SDValue CWD = DAG.getLoad(MVT::i32, DL, Chain, StackSlot, MPI);  in LowerSET_ROUNDING()
28636     CWD = DAG.getNode(ISD::AND, DL, MVT::i32, CWD.getValue(0),  in LowerSET_ROUNDING()
28637                       DAG.getConstant(0xffff9fff, DL, MVT::i32));  in LowerSET_ROUNDING()
28640     RMBits = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, RMBits);  in LowerSET_ROUNDING()
28641     RMBits = DAG.getNode(ISD::SHL, DL, MVT::i32, RMBits,  in LowerSET_ROUNDING()
28642                          DAG.getConstant(3, DL, MVT::i8));  in LowerSET_ROUNDING()
28645     CWD = DAG.getNode(ISD::OR, DL, MVT::i32, CWD, RMBits);  in LowerSET_ROUNDING()
28650         ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Chain,  in LowerSET_ROUNDING()
28651         DAG.getTargetConstant(Intrinsic::x86_sse_ldmxcsr, DL, MVT::i32),  in LowerSET_ROUNDING()
28668   MVT VT = Op.getSimpleValueType();  in LowerVectorCTLZ_AVX512CDI()
28669   MVT EltVT = VT.getVectorElementType();  in LowerVectorCTLZ_AVX512CDI()
28672   assert((EltVT == MVT::i8 || EltVT == MVT::i16) &&  in LowerVectorCTLZ_AVX512CDI()
28680   MVT NewVT = MVT::getVectorVT(MVT::i32, NumElems);  in LowerVectorCTLZ_AVX512CDI()
28697   MVT VT = Op.getSimpleValueType();  in LowerVectorCTLZInRegLUT()
28700   MVT CurrVT = MVT::getVectorVT(MVT::i8, NumBytes);  in LowerVectorCTLZInRegLUT()
28710     LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8));  in LowerVectorCTLZInRegLUT()
28726     MVT MaskVT = MVT::getVectorVT(MVT::i1, CurrVT.getVectorNumElements());  in LowerVectorCTLZInRegLUT()
28746     MVT NextSVT = MVT::getIntegerVT(CurrScalarSizeInBits * 2);  in LowerVectorCTLZInRegLUT()
28747     MVT NextVT = MVT::getVectorVT(NextSVT, CurrNumElts / 2);  in LowerVectorCTLZInRegLUT()
28752       MVT MaskVT = MVT::getVectorVT(MVT::i1, CurrVT.getVectorNumElements());  in LowerVectorCTLZInRegLUT()
28779   MVT VT = Op.getSimpleValueType();  in LowerVectorCTLZ()
28783       (Subtarget.canExtendTo512DQ() || VT.getVectorElementType() != MVT::i8))  in LowerVectorCTLZ()
28800   MVT VT = Op.getSimpleValueType();  in LowerCTLZ()
28801   MVT OpVT = VT;  in LowerCTLZ()
28810   if (VT == MVT::i8) {  in LowerCTLZ()
28812     OpVT = MVT::i32;  in LowerCTLZ()
28817   SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);  in LowerCTLZ()
28823                      DAG.getTargetConstant(X86::COND_E, dl, MVT::i8),  in LowerCTLZ()
28832   if (VT == MVT::i8)  in LowerCTLZ()
28833     Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);  in LowerCTLZ()
28839   MVT VT = Op.getSimpleValueType();  in LowerCTTZ()
28848   SDVTList VTs = DAG.getVTList(VT, MVT::i32);  in LowerCTTZ()
28853                    DAG.getTargetConstant(X86::COND_E, dl, MVT::i8),  in LowerCTTZ()
28860   MVT VT = Op.getSimpleValueType();  in lowerAddSub()
28861   if (VT == MVT::i16 || VT == MVT::i32)  in lowerAddSub()
28864   if (VT == MVT::v32i16 || VT == MVT::v64i8)  in lowerAddSub()
28875   MVT VT = Op.getSimpleValueType();  in LowerADDSAT_SUBSAT()
28880   if (VT == MVT::v32i16 || VT == MVT::v64i8 ||  in LowerADDSAT_SUBSAT()
28922       (!VT.isVector() || VT == MVT::v2i64)) {  in LowerADDSAT_SUBSAT()
28945   MVT VT = Op.getSimpleValueType();  in LowerABS()
28946   if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) {  in LowerABS()
28951     SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),  in LowerABS()
28953     SDValue Ops[] = {N0, Neg, DAG.getTargetConstant(X86::COND_NS, DL, MVT::i8),  in LowerABS()
28959   if ((VT == MVT::v2i64 || VT == MVT::v4i64) && Subtarget.hasSSE41()) {  in LowerABS()
28973   if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI())  in LowerABS()
28982   MVT VT = Op.getSimpleValueType();  in LowerAVG()
28988   if (VT == MVT::v32i16 || VT == MVT::v64i8)  in LowerAVG()
28997   MVT VT = Op.getSimpleValueType();  in LowerMINMAX()
29003   if (VT == MVT::v32i16 || VT == MVT::v64i8)  in LowerMINMAX()
29013   MVT VT = Op.getSimpleValueType();  in LowerMUL()
29019   if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI())  in LowerMUL()
29027   if (VT == MVT::v16i8 || VT == MVT::v32i8 || VT == MVT::v64i8) {  in LowerMUL()
29030     if ((VT == MVT::v16i8 && Subtarget.hasInt256()) ||  in LowerMUL()
29031         (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) {  in LowerMUL()
29032       MVT ExVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements());  in LowerMUL()
29040     MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts / 2);  in LowerMUL()
29057                                                MVT::i16));  in LowerMUL()
29059                                                MVT::i16));  in LowerMUL()
29077   if (VT == MVT::v4i32) {  in LowerMUL()
29087     SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,  in LowerMUL()
29088                                 DAG.getBitcast(MVT::v2i64, A),  in LowerMUL()
29089                                 DAG.getBitcast(MVT::v2i64, B));  in LowerMUL()
29091     SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,  in LowerMUL()
29092                                DAG.getBitcast(MVT::v2i64, Aodds),  in LowerMUL()
29093                                DAG.getBitcast(MVT::v2i64, Bodds));  in LowerMUL()
29104   assert((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) &&  in LowerMUL()
29154                                      MVT VT, bool IsSigned,  in LowervXi8MulWithUNPCK()
29172   MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts / 2);  in LowervXi8MulWithUNPCK()
29194           LoOp = DAG.getAnyExtOrTrunc(LoOp, dl, MVT::i16);  in LowervXi8MulWithUNPCK()
29195           HiOp = DAG.getAnyExtOrTrunc(HiOp, dl, MVT::i16);  in LowervXi8MulWithUNPCK()
29196           LoOp = DAG.getNode(ISD::SHL, dl, MVT::i16, LoOp,  in LowervXi8MulWithUNPCK()
29197                              DAG.getConstant(8, dl, MVT::i16));  in LowervXi8MulWithUNPCK()
29198           HiOp = DAG.getNode(ISD::SHL, dl, MVT::i16, HiOp,  in LowervXi8MulWithUNPCK()
29199                              DAG.getConstant(8, dl, MVT::i16));  in LowervXi8MulWithUNPCK()
29201           LoOp = DAG.getZExtOrTrunc(LoOp, dl, MVT::i16);  in LowervXi8MulWithUNPCK()
29202           HiOp = DAG.getZExtOrTrunc(HiOp, dl, MVT::i16);  in LowervXi8MulWithUNPCK()
29235   MVT VT = Op.getSimpleValueType();  in LowerMULH()
29245   if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI())  in LowerMULH()
29248   if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) {  in LowerMULH()
29249     assert((VT == MVT::v4i32 && Subtarget.hasSSE2()) ||  in LowerMULH()
29250            (VT == MVT::v8i32 && Subtarget.hasInt256()) ||  in LowerMULH()
29251            (VT == MVT::v16i32 && Subtarget.hasAVX512()));  in LowerMULH()
29276     MVT MulVT = MVT::getVectorVT(MVT::i64, NumElts / 2);  in LowerMULH()
29314   assert((VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) ||  in LowerMULH()
29315          (VT == MVT::v64i8 && Subtarget.hasBWI())) &&  in LowerMULH()
29324   if ((VT == MVT::v16i8 && Subtarget.hasInt256()) ||  in LowerMULH()
29325       (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) {  in LowerMULH()
29326     MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts);  in LowerMULH()
29341   MVT VT = Op.getSimpleValueType();  in LowerMULO()
29353   if ((VT == MVT::v32i8 && !Subtarget.hasInt256()) ||  in LowerMULO()
29354       (VT == MVT::v64i8 && !Subtarget.hasBWI())) {  in LowerMULO()
29384   if ((VT == MVT::v16i8 && Subtarget.hasInt256()) ||  in LowerMULO()
29385       (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) {  in LowerMULO()
29387     MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts);  in LowerMULO()
29398       if (OvfVT.getVectorElementType() == MVT::i1 &&  in LowerMULO()
29411           High = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v16i32, High);  in LowerMULO()
29412           LowSign = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v16i32, LowSign);  in LowerMULO()
29426       if (OvfVT.getVectorElementType() == MVT::i1 &&  in LowerMULO()
29432           High = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v16i32, High);  in LowerMULO()
29517           static_cast<EVT>(MVT::v2i64).getTypeForEVT(*DAG.getContext()), Callee,  in LowerWin64_i128OP()
29556       makeLibCall(DAG, LC, MVT::v2i64, Arg, CallOptions, dl, Chain);  in LowerWin64_FP_TO_INT128()
29600 static bool supportedVectorShiftWithImm(MVT VT, const X86Subtarget &Subtarget,  in supportedVectorShiftWithImm()
29616                            (VT != MVT::v2i64 && VT != MVT::v4i64));  in supportedVectorShiftWithImm()
29623 bool supportedVectorShiftWithBaseAmnt(MVT VT, const X86Subtarget &Subtarget,  in supportedVectorShiftWithBaseAmnt()
29630 static bool supportedVectorVarShift(MVT VT, const X86Subtarget &Subtarget,  in supportedVectorVarShift()
29647   bool AShift = LShift &&  VT != MVT::v2i64 && VT != MVT::v4i64;  in supportedVectorVarShift()
29653   MVT VT = Op.getSimpleValueType();  in LowerShiftByScalarImmediate()
29660     assert((VT == MVT::v2i64 || VT == MVT::v4i64) && "Unexpected SRA type");  in LowerShiftByScalarImmediate()
29661     MVT ExVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() * 2);  in LowerShiftByScalarImmediate()
29666       assert((VT != MVT::v4i64 || Subtarget.hasInt256()) &&  in LowerShiftByScalarImmediate()
29677       if (VT == MVT::v2i64)  in LowerShiftByScalarImmediate()
29679       if (VT == MVT::v4i64)  in LowerShiftByScalarImmediate()
29689       if (VT == MVT::v2i64)  in LowerShiftByScalarImmediate()
29691       if (VT == MVT::v4i64)  in LowerShiftByScalarImmediate()
29713   if (((!Subtarget.hasXOP() && VT == MVT::v2i64) ||  in LowerShiftByScalarImmediate()
29714        (Subtarget.hasInt256() && VT == MVT::v4i64)) &&  in LowerShiftByScalarImmediate()
29718   if (VT == MVT::v16i8 || (Subtarget.hasInt256() && VT == MVT::v32i8) ||  in LowerShiftByScalarImmediate()
29719       (Subtarget.hasBWI() && VT == MVT::v64i8)) {  in LowerShiftByScalarImmediate()
29721     MVT ShiftVT = MVT::getVectorVT(MVT::i16, NumElts / 2);  in LowerShiftByScalarImmediate()
29738         assert(VT == MVT::v64i8 && "Unexpected element type!");  in LowerShiftByScalarImmediate()
29739         SDValue CMP = DAG.getSetCC(dl, MVT::v64i1, Zeros, R, ISD::SETGT);  in LowerShiftByScalarImmediate()
29746     if (VT == MVT::v16i8 && Subtarget.hasXOP())  in LowerShiftByScalarImmediate()
29784   MVT VT = Op.getSimpleValueType();  in LowerShiftByScalarVariable()
29798     if (((VT == MVT::v16i8 && !Subtarget.canExtendTo512DQ()) ||  in LowerShiftByScalarVariable()
29799          (VT == MVT::v32i8 && !Subtarget.canExtendTo512BW()) ||  in LowerShiftByScalarVariable()
29800          VT == MVT::v64i8) &&  in LowerShiftByScalarVariable()
29803       MVT ExtVT = MVT::getVectorVT(MVT::i16, NumElts / 2);  in LowerShiftByScalarVariable()
29849   MVT VT = Amt.getSimpleValueType();  in convertShiftLeftToScale()
29850   if (!(VT == MVT::v8i16 || VT == MVT::v4i32 ||  in convertShiftLeftToScale()
29851         (Subtarget.hasInt256() && VT == MVT::v16i16) ||  in convertShiftLeftToScale()
29852         (Subtarget.hasAVX512() && VT == MVT::v32i16) ||  in convertShiftLeftToScale()
29853         (!Subtarget.hasAVX512() && VT == MVT::v16i8) ||  in convertShiftLeftToScale()
29854         (Subtarget.hasInt256() && VT == MVT::v32i8) ||  in convertShiftLeftToScale()
29855         (Subtarget.hasBWI() && VT == MVT::v64i8)))  in convertShiftLeftToScale()
29858   MVT SVT = VT.getVectorElementType();  in convertShiftLeftToScale()
29878   if (VT == MVT::v4i32) {  in convertShiftLeftToScale()
29882     Amt = DAG.getBitcast(MVT::v4f32, Amt);  in convertShiftLeftToScale()
29887   if (VT == MVT::v8i16 && !Subtarget.hasAVX2()) {  in convertShiftLeftToScale()
29889     SDValue Lo = DAG.getBitcast(MVT::v4i32, getUnpackl(DAG, dl, VT, Amt, Z));  in convertShiftLeftToScale()
29890     SDValue Hi = DAG.getBitcast(MVT::v4i32, getUnpackh(DAG, dl, VT, Amt, Z));  in convertShiftLeftToScale()
29903   MVT VT = Op.getSimpleValueType();  in LowerShift()
29929   if (((VT == MVT::v2i64 && !Subtarget.hasXOP()) ||  in LowerShift()
29930        (VT == MVT::v4i64 && Subtarget.hasInt256())) &&  in LowerShift()
29942   if (Subtarget.hasXOP() && (VT == MVT::v2i64 || VT == MVT::v4i32 ||  in LowerShift()
29943                              VT == MVT::v8i16 || VT == MVT::v16i8)) {  in LowerShift()
29956   if (VT == MVT::v2i64 && Opc != ISD::SRA) {  in LowerShift()
29975   if (ConstantAmt && (VT == MVT::v8i16 || VT == MVT::v4i32 ||  in LowerShift()
29976                       (VT == MVT::v16i16 && Subtarget.hasInt256()))) {  in LowerShift()
30001         (VT != MVT::v16i16 ||  in LowerShift()
30003         (VT == MVT::v4i32 || Subtarget.hasSSE41() || Opc != ISD::SHL ||  in LowerShift()
30021   if (Opc == ISD::SHL && !(VT == MVT::v32i8 && (Subtarget.hasXOP() ||  in LowerShift()
30029       (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256()))) {  in LowerShift()
30045       (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256())) &&  in LowerShift()
30069   if (VT == MVT::v4i32) {  in LowerShift()
30088         SDValue Amt01 = DAG.getBitcast(MVT::v8i16, Amt);  in LowerShift()
30089         SDValue Amt23 = DAG.getVectorShuffle(MVT::v8i16, dl, Amt01, Amt01,  in LowerShift()
30093         Amt0 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt01, Msk02);  in LowerShift()
30094         Amt1 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt01, Msk13);  in LowerShift()
30095         Amt2 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt23, Msk02);  in LowerShift()
30096         Amt3 = DAG.getNode(X86ISD::PSHUFLW, dl, MVT::v8i16, Amt23, Msk13);  in LowerShift()
30122   if ((Subtarget.hasInt256() && VT == MVT::v8i16) ||  in LowerShift()
30123       (Subtarget.canExtendTo512DQ() && VT == MVT::v16i16) ||  in LowerShift()
30124       (Subtarget.canExtendTo512DQ() && VT == MVT::v16i8) ||  in LowerShift()
30125       (Subtarget.canExtendTo512BW() && VT == MVT::v32i8) ||  in LowerShift()
30126       (Subtarget.hasBWI() && Subtarget.hasVLX() && VT == MVT::v16i8)) {  in LowerShift()
30127     assert((!Subtarget.hasBWI() || VT == MVT::v32i8 || VT == MVT::v16i8) &&  in LowerShift()
30129     MVT EvtSVT = Subtarget.hasBWI() ? MVT::i16 : MVT::i32;  in LowerShift()
30130     MVT ExtVT = MVT::getVectorVT(EvtSVT, VT.getVectorNumElements());  in LowerShift()
30141       (VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) ||  in LowerShift()
30142        (VT == MVT::v64i8 && Subtarget.hasBWI())) &&  in LowerShift()
30145     SDValue Cst8 = DAG.getTargetConstant(8, dl, MVT::i8);  in LowerShift()
30149     MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts);  in LowerShift()
30156     if (VT == MVT::v16i8 && Subtarget.hasInt256()) {  in LowerShift()
30172     MVT VT16 = MVT::getVectorVT(MVT::i16, NumElts / 2);  in LowerShift()
30187   if (VT == MVT::v16i8 ||  in LowerShift()
30188       (VT == MVT::v32i8 && Subtarget.hasInt256() && !Subtarget.hasXOP()) ||  in LowerShift()
30189       (VT == MVT::v64i8 && Subtarget.hasBWI())) {  in LowerShift()
30190     MVT ExtVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements() / 2);  in LowerShift()
30192     auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) {  in LowerShift()
30197         MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements());  in LowerShift()
30296   if (Subtarget.hasInt256() && !Subtarget.hasXOP() && VT == MVT::v16i16) {  in LowerShift()
30297     MVT ExtVT = MVT::v8i32;  in LowerShift()
30314   if (VT == MVT::v8i16) {  in LowerShift()
30324         MVT ExtVT = MVT::getVectorVT(MVT::i8, VT.getVectorNumElements() * 2);  in LowerShift()
30382   if (VT == MVT::v32i16 || VT == MVT::v64i8)  in LowerShift()
30390   MVT VT = Op.getSimpleValueType();  in LowerFunnelShift()
30411         SDValue Imm = DAG.getTargetConstant(ShiftAmt, DL, MVT::i8);  in LowerFunnelShift()
30418     assert((VT == MVT::v16i8 || VT == MVT::v32i8 || VT == MVT::v64i8 ||  in LowerFunnelShift()
30419             VT == MVT::v8i16 || VT == MVT::v16i16 || VT == MVT::v32i16 ||  in LowerFunnelShift()
30420             VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) &&  in LowerFunnelShift()
30438     MVT ExtSVT = MVT::getIntegerVT(2 * EltSizeInBits);  in LowerFunnelShift()
30439     MVT ExtVT = MVT::getVectorVT(ExtSVT, NumElts / 2);  in LowerFunnelShift()
30470     MVT WideSVT = MVT::getIntegerVT(  in LowerFunnelShift()
30472     MVT WideVT = MVT::getVectorVT(WideSVT, NumElts);  in LowerFunnelShift()
30513       (VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) &&  in LowerFunnelShift()
30522   if ((VT == MVT::i8 || (ExpandFunnel && VT == MVT::i16)) &&  in LowerFunnelShift()
30526     Op0 = DAG.getAnyExtOrTrunc(Op0, DL, MVT::i32);  in LowerFunnelShift()
30527     Op1 = DAG.getZExtOrTrunc(Op1, DL, MVT::i32);  in LowerFunnelShift()
30529     SDValue Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Op0, HiShift);  in LowerFunnelShift()
30530     Res = DAG.getNode(ISD::OR, DL, MVT::i32, Res, Op1);  in LowerFunnelShift()
30532       Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, Amt);  in LowerFunnelShift()
30534       Res = DAG.getNode(ISD::SHL, DL, MVT::i32, Res, Amt);  in LowerFunnelShift()
30535       Res = DAG.getNode(ISD::SRL, DL, MVT::i32, Res, HiShift);  in LowerFunnelShift()
30540   if (VT == MVT::i8 || ExpandFunnel)  in LowerFunnelShift()
30544   if (VT == MVT::i16) {  in LowerFunnelShift()
30556   MVT VT = Op.getSimpleValueType();  in LowerRotate()
30582                          DAG.getTargetConstant(RotAmt, DL, MVT::i8));  in LowerRotate()
30624                          DAG.getTargetConstant(RotAmt, DL, MVT::i8));  in LowerRotate()
30640       (VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 ||  in LowerRotate()
30641        ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8) &&  in LowerRotate()
30643        ((VT == MVT::v32i16 || VT == MVT::v64i8) && Subtarget.useBWIRegs())) &&  in LowerRotate()
30646   MVT ExtSVT = MVT::getIntegerVT(2 * EltSizeInBits);  in LowerRotate()
30647   MVT ExtVT = MVT::getVectorVT(ExtSVT, NumElts / 2);  in LowerRotate()
30678     MVT WideVT =  in LowerRotate()
30679         MVT::getVectorVT(Subtarget.hasBWI() ? MVT::i16 : MVT::i32, NumElts);  in LowerRotate()
30715     assert((VT == MVT::v16i8 || VT == MVT::v32i8) && "Unsupported vXi8 type");  in LowerRotate()
30718     auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) {  in LowerRotate()
30824   assert(VT == MVT::v4i32 && "Only v4i32 vector rotate expected");  in LowerRotate()
30829   SDValue Res02 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,  in LowerRotate()
30830                               DAG.getBitcast(MVT::v2i64, R),  in LowerRotate()
30831                               DAG.getBitcast(MVT::v2i64, Scale));  in LowerRotate()
30832   SDValue Res13 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,  in LowerRotate()
30833                               DAG.getBitcast(MVT::v2i64, R13),  in LowerRotate()
30834                               DAG.getBitcast(MVT::v2i64, Scale13));  in LowerRotate()
31106     SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);  in emitLockedStackOp()
31108       DAG.getRegister(X86::RSP, MVT::i64),                  // Base  in emitLockedStackOp()
31109       DAG.getTargetConstant(1, DL, MVT::i8),                // Scale  in emitLockedStackOp()
31110       DAG.getRegister(0, MVT::i64),                         // Index  in emitLockedStackOp()
31111       DAG.getTargetConstant(SPOffset, DL, MVT::i32),        // Disp  in emitLockedStackOp()
31112       DAG.getRegister(0, MVT::i16),                         // Segment.  in emitLockedStackOp()
31115     SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,  in emitLockedStackOp()
31116                                      MVT::Other, Ops);  in emitLockedStackOp()
31120   SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);  in emitLockedStackOp()
31122     DAG.getRegister(X86::ESP, MVT::i32),            // Base  in emitLockedStackOp()
31123     DAG.getTargetConstant(1, DL, MVT::i8),          // Scale  in emitLockedStackOp()
31124     DAG.getRegister(0, MVT::i32),                   // Index  in emitLockedStackOp()
31125     DAG.getTargetConstant(SPOffset, DL, MVT::i32),  // Disp  in emitLockedStackOp()
31126     DAG.getRegister(0, MVT::i16),                   // Segment.  in emitLockedStackOp()
31130   SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,  in emitLockedStackOp()
31131                                    MVT::Other, Ops);  in emitLockedStackOp()
31148       return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));  in LowerATOMIC_FENCE()
31155   return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));  in LowerATOMIC_FENCE()
31160   MVT T = Op.getSimpleValueType();  in LowerCMP_SWAP()
31166   case MVT::i8:  Reg = X86::AL;  size = 1; break;  in LowerCMP_SWAP()
31167   case MVT::i16: Reg = X86::AX;  size = 2; break;  in LowerCMP_SWAP()
31168   case MVT::i32: Reg = X86::EAX; size = 4; break;  in LowerCMP_SWAP()
31169   case MVT::i64:  in LowerCMP_SWAP()
31179                     DAG.getTargetConstant(size, DL, MVT::i8),  in LowerCMP_SWAP()
31181   SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);  in LowerCMP_SWAP()
31189                                       MVT::i32, cpOut.getValue(2));  in LowerCMP_SWAP()
31199   MVT InVT = V.getSimpleValueType();  in getPMOVMSKB()
31201   if (InVT == MVT::v64i8) {  in getPMOVMSKB()
31206     Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Lo);  in getPMOVMSKB()
31207     Hi = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Hi);  in getPMOVMSKB()
31208     Hi = DAG.getNode(ISD::SHL, DL, MVT::i64, Hi,  in getPMOVMSKB()
31209                      DAG.getConstant(32, DL, MVT::i8));  in getPMOVMSKB()
31210     return DAG.getNode(ISD::OR, DL, MVT::i64, Lo, Hi);  in getPMOVMSKB()
31212   if (InVT == MVT::v32i8 && !Subtarget.hasInt256()) {  in getPMOVMSKB()
31215     Lo = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Lo);  in getPMOVMSKB()
31216     Hi = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Hi);  in getPMOVMSKB()
31217     Hi = DAG.getNode(ISD::SHL, DL, MVT::i32, Hi,  in getPMOVMSKB()
31218                      DAG.getConstant(16, DL, MVT::i8));  in getPMOVMSKB()
31219     return DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi);  in getPMOVMSKB()
31222   return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);  in getPMOVMSKB()
31228   MVT SrcVT = Src.getSimpleValueType();  in LowerBITCAST()
31229   MVT DstVT = Op.getSimpleValueType();  in LowerBITCAST()
31233   if (SrcVT == MVT::i64 && DstVT == MVT::v64i1) {  in LowerBITCAST()
31237     SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Src,  in LowerBITCAST()
31239     Lo = DAG.getBitcast(MVT::v32i1, Lo);  in LowerBITCAST()
31240     SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Src,  in LowerBITCAST()
31242     Hi = DAG.getBitcast(MVT::v32i1, Hi);  in LowerBITCAST()
31243     return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi);  in LowerBITCAST()
31247   if ((SrcVT == MVT::v16i1 || SrcVT == MVT::v32i1) && DstVT.isScalarInteger()) {  in LowerBITCAST()
31249     MVT SExtVT = SrcVT == MVT::v16i1 ? MVT::v16i8 : MVT::v32i8;  in LowerBITCAST()
31256   assert((SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8 ||  in LowerBITCAST()
31257           SrcVT == MVT::i64) && "Unexpected VT!");  in LowerBITCAST()
31260   if (!(DstVT == MVT::f64 && SrcVT == MVT::i64) &&  in LowerBITCAST()
31261       !(DstVT == MVT::x86mmx && SrcVT.isVector()))  in LowerBITCAST()
31269     MVT NewVT = MVT::getVectorVT(SrcVT.getVectorElementType(),  in LowerBITCAST()
31274     assert(SrcVT == MVT::i64 && !Subtarget.is64Bit() &&  in LowerBITCAST()
31276     Src = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Src);  in LowerBITCAST()
31279   MVT V2X64VT = DstVT == MVT::f64 ? MVT::v2f64 : MVT::v2i64;  in LowerBITCAST()
31282   if (DstVT == MVT::x86mmx)  in LowerBITCAST()
31295 static SDValue LowerHorizontalByteSum(SDValue V, MVT VT,  in LowerHorizontalByteSum()
31299   MVT ByteVecVT = V.getSimpleValueType();  in LowerHorizontalByteSum()
31300   MVT EltVT = VT.getVectorElementType();  in LowerHorizontalByteSum()
31301   assert(ByteVecVT.getVectorElementType() == MVT::i8 &&  in LowerHorizontalByteSum()
31303   assert(EltVT != MVT::i8 &&  in LowerHorizontalByteSum()
31310   if (EltVT == MVT::i64) {  in LowerHorizontalByteSum()
31312     MVT SadVecVT = MVT::getVectorVT(MVT::i64, VecSize / 64);  in LowerHorizontalByteSum()
31317   if (EltVT == MVT::i32) {  in LowerHorizontalByteSum()
31330     MVT SadVecVT = MVT::getVectorVT(MVT::i64, VecSize / 64);  in LowerHorizontalByteSum()
31337     MVT ShortVecVT = MVT::getVectorVT(MVT::i16, VecSize / 16);  in LowerHorizontalByteSum()
31346   assert(EltVT == MVT::i16 && "Unknown how to handle type");  in LowerHorizontalByteSum()
31362   MVT VT = Op.getSimpleValueType();  in LowerVectorCTPOPInRegLUT()
31363   MVT EltVT = VT.getVectorElementType();  in LowerVectorCTPOPInRegLUT()
31366   assert(EltVT == MVT::i8 && "Only vXi8 vector CTPOP lowering supported.");  in LowerVectorCTPOPInRegLUT()
31385     LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8));  in LowerVectorCTPOPInRegLUT()
31408   MVT VT = Op.getSimpleValueType();  in LowerVectorCTPOP()
31417     assert((VT.getVectorElementType() == MVT::i8 ||  in LowerVectorCTPOP()
31418             VT.getVectorElementType() == MVT::i16) && "Unexpected type");  in LowerVectorCTPOP()
31420       MVT NewVT = MVT::getVectorVT(MVT::i32, NumElems);  in LowerVectorCTPOP()
31436   if (VT.getScalarType() != MVT::i8) {  in LowerVectorCTPOP()
31437     MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);  in LowerVectorCTPOP()
31458   MVT VT = Op.getSimpleValueType();  in LowerBITREVERSE_XOP()
31465     MVT VecVT = MVT::getVectorVT(VT, 128 / VT.getSizeInBits());  in LowerBITREVERSE_XOP()
31491       MaskElts.push_back(DAG.getConstant(PermuteByte, DL, MVT::i8));  in LowerBITREVERSE_XOP()
31495   SDValue Mask = DAG.getBuildVector(MVT::v16i8, DL, MaskElts);  in LowerBITREVERSE_XOP()
31496   SDValue Res = DAG.getBitcast(MVT::v16i8, In);  in LowerBITREVERSE_XOP()
31497   Res = DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, DAG.getUNDEF(MVT::v16i8),  in LowerBITREVERSE_XOP()
31504   MVT VT = Op.getSimpleValueType();  in LowerBITREVERSE()
31514   assert(VT.getScalarType() == MVT::i8 &&  in LowerBITREVERSE()
31518   if (VT == MVT::v64i8 && !Subtarget.hasBWI())  in LowerBITREVERSE()
31522   if (VT == MVT::v32i8 && !Subtarget.hasInt256())  in LowerBITREVERSE()
31529     MVT MatrixVT = MVT::getVectorVT(MVT::i64, NumElts / 8);  in LowerBITREVERSE()
31533                        DAG.getTargetConstant(0, DL, MVT::i8));  in LowerBITREVERSE()
31556     LoMaskElts.push_back(DAG.getConstant(LoLUT[i % 16], DL, MVT::i8));  in LowerBITREVERSE()
31557     HiMaskElts.push_back(DAG.getConstant(HiLUT[i % 16], DL, MVT::i8));  in LowerBITREVERSE()
31571   MVT VT = Op.getSimpleValueType();  in LowerPARITY()
31574   if (VT == MVT::i8 ||  in LowerPARITY()
31576     X = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X);  in LowerPARITY()
31577     SDValue Flags = DAG.getNode(X86ISD::CMP, DL, MVT::i32, X,  in LowerPARITY()
31578                                 DAG.getConstant(0, DL, MVT::i8));  in LowerPARITY()
31589   if (VT == MVT::i64) {  in LowerPARITY()
31591     SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32,  in LowerPARITY()
31592                              DAG.getNode(ISD::SRL, DL, MVT::i64, X,  in LowerPARITY()
31593                                          DAG.getConstant(32, DL, MVT::i8)));  in LowerPARITY()
31594     SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, X);  in LowerPARITY()
31595     X = DAG.getNode(ISD::XOR, DL, MVT::i32, Lo, Hi);  in LowerPARITY()
31598   if (VT != MVT::i16) {  in LowerPARITY()
31600     SDValue Hi16 = DAG.getNode(ISD::SRL, DL, MVT::i32, X,  in LowerPARITY()
31601                                DAG.getConstant(16, DL, MVT::i8));  in LowerPARITY()
31602     X = DAG.getNode(ISD::XOR, DL, MVT::i32, X, Hi16);  in LowerPARITY()
31605     X = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, X);  in LowerPARITY()
31611       ISD::TRUNCATE, DL, MVT::i8,  in LowerPARITY()
31612       DAG.getNode(ISD::SRL, DL, MVT::i32, X, DAG.getConstant(8, DL, MVT::i8)));  in LowerPARITY()
31613   SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X);  in LowerPARITY()
31614   SDVTList VTs = DAG.getVTList(MVT::i8, MVT::i32);  in LowerPARITY()
31649       NewOpc, SDLoc(N), DAG.getVTList(MVT::i32, MVT::Other),  in lowerAtomicArithWithLOCK()
31662   MVT VT = N->getSimpleValueType(0);  in lowerAtomicArith()
31704     SDValue NewChain = DAG.getNode(X86ISD::MEMBARRIER, DL, MVT::Other, Chain);  in lowerAtomicArith()
31734   if (VT == MVT::i64 && !IsTypeLegal) {  in LowerATOMIC_STORE()
31743         SDValue SclToVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64,  in LowerATOMIC_STORE()
31745         MVT StVT = Subtarget.hasSSE2() ? MVT::v2i64 : MVT::v4f32;  in LowerATOMIC_STORE()
31747         SDVTList Tys = DAG.getVTList(MVT::Other);  in LowerATOMIC_STORE()
31750                                         MVT::i64, Node->getMemOperand());  in LowerATOMIC_STORE()
31754         SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64);  in LowerATOMIC_STORE()
31761         SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);  in LowerATOMIC_STORE()
31764             DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, LdOps, MVT::i64, MPI,  in LowerATOMIC_STORE()
31771             DAG.getMemIntrinsicNode(X86ISD::FIST, dl, DAG.getVTList(MVT::Other),  in LowerATOMIC_STORE()
31772                                     StoreOps, MVT::i64, Node->getMemOperand());  in LowerATOMIC_STORE()
31799   MVT VT = N->getSimpleValueType(0);  in LowerADDSUBCARRY()
31806   SDVTList VTs = DAG.getVTList(VT, MVT::i32);  in LowerADDSUBCARRY()
31812   Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),  in LowerADDSUBCARRY()
31823   if (N->getValueType(1) == MVT::i1)  in LowerADDSUBCARRY()
31824     SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);  in LowerADDSUBCARRY()
31850   bool isF64 = ArgVT == MVT::f64;  in LowerFSINCOS()
31885 static SDValue ExtendToType(SDValue InOp, MVT NVT, SelectionDAG &DAG,  in ExtendToType()
31888   MVT InVT = InOp.getSimpleValueType();  in ExtendToType()
31941   MVT VT = Src.getSimpleValueType();  in LowerMSCATTER()
31951   if (VT == MVT::v2f32 || VT == MVT::v2i32) {  in LowerMSCATTER()
31952     assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type");  in LowerMSCATTER()
31954     if (Index.getValueType() == MVT::v2i64 && Subtarget.hasVLX()) {  in LowerMSCATTER()
31958       SDVTList VTs = DAG.getVTList(MVT::Other);  in LowerMSCATTER()
31966   MVT IndexVT = Index.getSimpleValueType();  in LowerMSCATTER()
31970   if (IndexVT == MVT::v2i32)  in LowerMSCATTER()
31982     VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);  in LowerMSCATTER()
31983     IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(), NumElts);  in LowerMSCATTER()
31984     MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);  in LowerMSCATTER()
31991   SDVTList VTs = DAG.getVTList(MVT::Other);  in LowerMSCATTER()
32001   MVT VT = Op.getSimpleValueType();  in LowerMLOAD()
32002   MVT ScalarVT = VT.getScalarType();  in LowerMLOAD()
32004   MVT MaskVT = Mask.getSimpleValueType();  in LowerMLOAD()
32009   if (MaskVT.getVectorElementType() != MVT::i1) {  in LowerMLOAD()
32035               (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) &&  in LowerMLOAD()
32041   MVT WideDataVT = MVT::getVectorVT(ScalarVT, NumEltsInWideVec);  in LowerMLOAD()
32045   assert(Mask.getSimpleValueType().getScalarType() == MVT::i1 &&  in LowerMLOAD()
32048   MVT WideMaskVT = MVT::getVectorVT(MVT::i1, NumEltsInWideVec);  in LowerMLOAD()
32067   MVT VT = DataToStore.getSimpleValueType();  in LowerMSTORE()
32068   MVT ScalarVT = VT.getScalarType();  in LowerMSTORE()
32083               (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) &&  in LowerMSTORE()
32089   MVT WideDataVT = MVT::getVectorVT(ScalarVT, NumEltsInWideVec);  in LowerMSTORE()
32092   assert(Mask.getSimpleValueType().getScalarType() == MVT::i1 &&  in LowerMSTORE()
32095   MVT WideMaskVT = MVT::getVectorVT(MVT::i1, NumEltsInWideVec);  in LowerMSTORE()
32112   MVT VT = Op.getSimpleValueType();  in LowerMGATHER()
32116   MVT IndexVT = Index.getSimpleValueType();  in LowerMGATHER()
32121   if (IndexVT == MVT::v2i32)  in LowerMGATHER()
32126   MVT OrigVT = VT;  in LowerMGATHER()
32135     VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);  in LowerMGATHER()
32136     IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(), NumElts);  in LowerMGATHER()
32137     MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);  in LowerMGATHER()
32151       X86ISD::MGATHER, dl, DAG.getVTList(VT, MVT::Other), Ops, N->getMemoryVT(),  in LowerMGATHER()
32161   MVT DstVT = Op.getSimpleValueType();  in LowerADDRSPACECAST()
32169   if (SrcAS == X86AS::PTR32_UPTR && DstVT == MVT::i64) {  in LowerADDRSPACECAST()
32171   } else if (DstVT == MVT::i64) {  in LowerADDRSPACECAST()
32173   } else if (DstVT == MVT::i32) {  in LowerADDRSPACECAST()
32194   SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);  in LowerGC_TRANSITION()
32391     Lo = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {LoVT, MVT::Other},  in ReplaceNodeResults()
32393     Hi = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {HiVT, MVT::Other},  in ReplaceNodeResults()
32395     SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,  in ReplaceNodeResults()
32406     assert(N->getValueType(0) == MVT::i64 && "Unexpected VT!");  in ReplaceNodeResults()
32411     if (isTypeLegal(MVT::v2i64) && !NoImplicitFloatOps) {  in ReplaceNodeResults()
32413           DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, N->getOperand(0));  in ReplaceNodeResults()
32414       Wide = DAG.getNode(ISD::CTPOP, dl, MVT::v2i64, Wide);  in ReplaceNodeResults()
32417       Wide = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Wide);  in ReplaceNodeResults()
32418       Wide = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, Wide,  in ReplaceNodeResults()
32420       Wide = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Wide);  in ReplaceNodeResults()
32428            VT.getVectorElementType() == MVT::i8 && "Unexpected VT!");  in ReplaceNodeResults()
32431     MVT MulVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements());  in ReplaceNodeResults()
32439     Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i8, ConcatOps);  in ReplaceNodeResults()
32447            VT == MVT::v2i32 && "Unexpected VT!");  in ReplaceNodeResults()
32450     SDValue Op0 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(0));  in ReplaceNodeResults()
32451     SDValue Op1 = DAG.getNode(ExtOpc, dl, MVT::v2i64, N->getOperand(1));  in ReplaceNodeResults()
32452     SDValue Res = DAG.getNode(ISD::MUL, dl, MVT::v2i64, Op0, Op1);  in ReplaceNodeResults()
32455     SDValue Hi = DAG.getBitcast(MVT::v4i32, Res);  in ReplaceNodeResults()
32456     Hi = DAG.getVectorShuffle(MVT::v4i32, dl, Hi, Hi, {1, 3, -1, -1});  in ReplaceNodeResults()
32474     Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Res,  in ReplaceNodeResults()
32515     assert(VT == MVT::v2f32 && "Unexpected type (!= v2f32) on FMIN/FMAX.");  in ReplaceNodeResults()
32517     SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,  in ReplaceNodeResults()
32519     SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,  in ReplaceNodeResults()
32521     Results.push_back(DAG.getNode(N->getOpcode(), dl, MVT::v4f32, LHS, RHS));  in ReplaceNodeResults()
32554     MVT VT = N->getSimpleValueType(0);  in ReplaceNodeResults()
32561     MVT WidenVT = getTypeToTransformTo(*DAG.getContext(), VT).getSimpleVT();  in ReplaceNodeResults()
32570       MVT InEltVT = InVT.getSimpleVT().getVectorElementType();  in ReplaceNodeResults()
32595       if (InVT == MVT::v4i64 && VT == MVT::v4i8 && isTypeLegal(MVT::v8i64)) {  in ReplaceNodeResults()
32596         In = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i64, In,  in ReplaceNodeResults()
32597                          DAG.getUNDEF(MVT::v4i64));  in ReplaceNodeResults()
32602     if (Subtarget.hasVLX() && InVT == MVT::v8i64 && VT == MVT::v8i8 &&  in ReplaceNodeResults()
32604         isTypeLegal(MVT::v4i64)) {  in ReplaceNodeResults()
32610       Lo = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Lo);  in ReplaceNodeResults()
32611       Hi = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Hi);  in ReplaceNodeResults()
32612       SDValue Res = DAG.getVectorShuffle(MVT::v16i8, dl, Lo, Hi,  in ReplaceNodeResults()
32624     assert(N->getValueType(0) == MVT::v8i8 &&  in ReplaceNodeResults()
32632     if (!Subtarget.hasSSE41() && VT == MVT::v4i64 &&  in ReplaceNodeResults()
32633         (InVT == MVT::v4i16 || InVT == MVT::v4i8)){  in ReplaceNodeResults()
32641       In = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, In);  in ReplaceNodeResults()
32644       SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32);  in ReplaceNodeResults()
32645       SDValue SignBits = DAG.getSetCC(dl, MVT::v4i32, Zero, In, ISD::SETGT);  in ReplaceNodeResults()
32649       SDValue Lo = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,  in ReplaceNodeResults()
32651       Lo = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Lo);  in ReplaceNodeResults()
32652       SDValue Hi = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,  in ReplaceNodeResults()
32654       Hi = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Hi);  in ReplaceNodeResults()
32661     if (VT == MVT::v16i32 || VT == MVT::v8i64) {  in ReplaceNodeResults()
32713       EVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32;  in ReplaceNodeResults()
32716             DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other},  in ReplaceNodeResults()
32718                                             {NVT, MVT::Other}, {Chain, Src})});  in ReplaceNodeResults()
32732         SrcVT.getVectorElementType() == MVT::f16) {  in ReplaceNodeResults()
32734       EVT ResVT = EleVT == MVT::i32 ? MVT::v4i32 : MVT::v8i16;  in ReplaceNodeResults()
32736       if (SrcVT != MVT::v8f16) {  in ReplaceNodeResults()
32739         SmallVector<SDValue, 4> Ops(SrcVT == MVT::v2f16 ? 4 : 2, Tmp);  in ReplaceNodeResults()
32741         Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8f16, Ops);  in ReplaceNodeResults()
32748             DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {N->getOperand(0), Src});  in ReplaceNodeResults()
32757         MVT TmpVT = MVT::getVectorVT(EleVT.getSimpleVT(), 8);  in ReplaceNodeResults()
32762         MVT ConcatVT = MVT::getVectorVT(EleVT.getSimpleVT(), 8 * NumConcats);  in ReplaceNodeResults()
32781       MVT PromoteVT = MVT::getVectorVT(MVT::getIntegerVT(NewEltWidth),  in ReplaceNodeResults()
32786         Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {PromoteVT, MVT::Other},  in ReplaceNodeResults()
32794       if (PromoteVT == MVT::v2i32)  in ReplaceNodeResults()
32795         Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Res,  in ReplaceNodeResults()
32796                           DAG.getUNDEF(MVT::v2i32));  in ReplaceNodeResults()
32802       if (PromoteVT == MVT::v2i32)  in ReplaceNodeResults()
32803         Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i32, Res,  in ReplaceNodeResults()
32811       MVT ConcatVT = MVT::getVectorVT(VT.getSimpleVT().getVectorElementType(),  in ReplaceNodeResults()
32823     if (VT == MVT::v2i32) {  in ReplaceNodeResults()
32829       if (Src.getValueType() == MVT::v2f64) {  in ReplaceNodeResults()
32832               expandFP_TO_UINT_SSE(MVT::v4i32, Src, dl, DAG, Subtarget);  in ReplaceNodeResults()
32852           Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f64, Src,  in ReplaceNodeResults()
32853                             DAG.getConstantFP(0.0, dl, MVT::v2f64));  in ReplaceNodeResults()
32859           Res = DAG.getNode(Opc, dl, {MVT::v4i32, MVT::Other},  in ReplaceNodeResults()
32863           Res = DAG.getNode(Opc, dl, MVT::v4i32, Src);  in ReplaceNodeResults()
32873       if (Src.getValueType() == MVT::v2f32 && IsStrict) {  in ReplaceNodeResults()
32874         Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,  in ReplaceNodeResults()
32875                           DAG.getConstantFP(0.0, dl, MVT::v2f32));  in ReplaceNodeResults()
32876         SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4i32, MVT::Other},  in ReplaceNodeResults()
32890     if ((Subtarget.hasDQI() && VT == MVT::i64 &&  in ReplaceNodeResults()
32891          (SrcVT == MVT::f32 || SrcVT == MVT::f64)) ||  in ReplaceNodeResults()
32892         (Subtarget.hasFP16() && SrcVT == MVT::f16)) {  in ReplaceNodeResults()
32898       MVT VecVT = MVT::getVectorVT(MVT::i64, NumElts);  in ReplaceNodeResults()
32899       MVT VecInVT = MVT::getVectorVT(SrcVT.getSimpleVT(), SrcElts);  in ReplaceNodeResults()
32914         SDVTList Tys = DAG.getVTList(VecVT, MVT::Other);  in ReplaceNodeResults()
32926     if (VT == MVT::i128 && Subtarget.isTargetWin64()) {  in ReplaceNodeResults()
32958     if (VT.getVectorElementType() == MVT::f16 && Subtarget.hasFP16() &&  in ReplaceNodeResults()
32960       if (Src.getValueType().getVectorElementType() == MVT::i16)  in ReplaceNodeResults()
32963       if (VT == MVT::v2f16 && Src.getValueType() == MVT::v2i32)  in ReplaceNodeResults()
32964         Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,  in ReplaceNodeResults()
32965                           IsStrict ? DAG.getConstant(0, dl, MVT::v2i32)  in ReplaceNodeResults()
32966                                    : DAG.getUNDEF(MVT::v2i32));  in ReplaceNodeResults()
32970         SDValue Res = DAG.getNode(Opc, dl, {MVT::v8f16, MVT::Other},  in ReplaceNodeResults()
32976         Results.push_back(DAG.getNode(Opc, dl, MVT::v8f16, Src));  in ReplaceNodeResults()
32980     if (VT != MVT::v2f32)  in ReplaceNodeResults()
32983     if (Subtarget.hasDQI() && Subtarget.hasVLX() && SrcVT == MVT::v2i64) {  in ReplaceNodeResults()
32987         SDValue Res = DAG.getNode(Opc, dl, {MVT::v4f32, MVT::Other},  in ReplaceNodeResults()
32993         Results.push_back(DAG.getNode(Opc, dl, MVT::v4f32, Src));  in ReplaceNodeResults()
32997     if (SrcVT == MVT::v2i64 && !IsSigned && Subtarget.is64Bit() &&  in ReplaceNodeResults()
33004       SDValue IsNeg = DAG.getSetCC(dl, MVT::v2i64, Src, Zero, ISD::SETLT);  in ReplaceNodeResults()
33006       SmallVector<SDValue, 4> SignCvts(4, DAG.getConstantFP(0.0, dl, MVT::f32));  in ReplaceNodeResults()
33008         SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64,  in ReplaceNodeResults()
33012               DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {MVT::f32, MVT::Other},  in ReplaceNodeResults()
33015           SignCvts[i] = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Elt);  in ReplaceNodeResults()
33017       SDValue SignCvt = DAG.getBuildVector(MVT::v4f32, dl, SignCvts);  in ReplaceNodeResults()
33020         Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,  in ReplaceNodeResults()
33022         Slow = DAG.getNode(ISD::STRICT_FADD, dl, {MVT::v4f32, MVT::Other},  in ReplaceNodeResults()
33026         Slow = DAG.getNode(ISD::FADD, dl, MVT::v4f32, SignCvt, SignCvt);  in ReplaceNodeResults()
33028       IsNeg = DAG.getBitcast(MVT::v4i32, IsNeg);  in ReplaceNodeResults()
33030           DAG.getVectorShuffle(MVT::v4i32, dl, IsNeg, IsNeg, {1, 3, -1, -1});  in ReplaceNodeResults()
33031       SDValue Cvt = DAG.getSelect(dl, MVT::v4f32, IsNeg, Slow, SignCvt);  in ReplaceNodeResults()
33038     if (SrcVT != MVT::v2i32)  in ReplaceNodeResults()
33047       Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,  in ReplaceNodeResults()
33048                         DAG.getConstant(0, dl, MVT::v2i32));  in ReplaceNodeResults()
33049       SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4f32, MVT::Other},  in ReplaceNodeResults()
33057     SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64, Src);  in ReplaceNodeResults()
33059         DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL), dl, MVT::v2f64);  in ReplaceNodeResults()
33060     SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn,  in ReplaceNodeResults()
33061                              DAG.getBitcast(MVT::v2i64, VBias));  in ReplaceNodeResults()
33062     Or = DAG.getBitcast(MVT::v2f64, Or);  in ReplaceNodeResults()
33064       SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::v2f64, MVT::Other},  in ReplaceNodeResults()
33067                                 {MVT::v4f32, MVT::Other},  in ReplaceNodeResults()
33073       SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias);  in ReplaceNodeResults()
33074       Results.push_back(DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, Sub));  in ReplaceNodeResults()
33087     if (VT == MVT::v2f16 && Src.getValueType() == MVT::v2f32) {  in ReplaceNodeResults()
33088       SDValue Ext = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v2f32)  in ReplaceNodeResults()
33089                              : DAG.getUNDEF(MVT::v2f32);  in ReplaceNodeResults()
33090       Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src, Ext);  in ReplaceNodeResults()
33092     if (!Subtarget.hasFP16() && VT.getVectorElementType() == MVT::f16) {  in ReplaceNodeResults()
33094       if (SrcVT.getVectorElementType() != MVT::f32)  in ReplaceNodeResults()
33098         V = DAG.getNode(X86ISD::STRICT_CVTPS2PH, dl, {MVT::v8i16, MVT::Other},  in ReplaceNodeResults()
33101         V = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Src, Rnd);  in ReplaceNodeResults()
33103       Results.push_back(DAG.getBitcast(MVT::v8f16, V));  in ReplaceNodeResults()
33110     EVT NewVT = VT.getVectorElementType() == MVT::f16 ? MVT::v8f16 : MVT::v4f32;  in ReplaceNodeResults()
33112       V = DAG.getNode(X86ISD::STRICT_VFPROUND, dl, {NewVT, MVT::Other},  in ReplaceNodeResults()
33125     assert(N->getValueType(0) == MVT::v2f32 &&  in ReplaceNodeResults()
33131     SDValue Ext = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v2f16)  in ReplaceNodeResults()
33132                            : DAG.getUNDEF(MVT::v2f16);  in ReplaceNodeResults()
33133     SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f16, Src, Ext);  in ReplaceNodeResults()
33135       V = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {MVT::v4f32, MVT::Other},  in ReplaceNodeResults()
33138       V = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, V);  in ReplaceNodeResults()
33174     assert((T == MVT::i64 || T == MVT::i128) && "can only expand cmpxchg pair");  in ReplaceNodeResults()
33175     bool Regs64bit = T == MVT::i128;  in ReplaceNodeResults()
33178     MVT HalfT = Regs64bit ? MVT::i64 : MVT::i32;  in ReplaceNodeResults()
33206     SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);  in ReplaceNodeResults()
33231                                         MVT::i32, cpOutH.getValue(2));  in ReplaceNodeResults()
33241     assert(N->getValueType(0) == MVT::i64 && "Unexpected VT!");  in ReplaceNodeResults()
33250         MVT LdVT = Subtarget.hasSSE2() ? MVT::v2i64 : MVT::v4f32;  in ReplaceNodeResults()
33251         SDVTList Tys = DAG.getVTList(LdVT, MVT::Other);  in ReplaceNodeResults()
33254                                              MVT::i64, Node->getMemOperand());  in ReplaceNodeResults()
33256           SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld,  in ReplaceNodeResults()
33265         SDValue Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2f32, Ld,  in ReplaceNodeResults()
33267         Res = DAG.getBitcast(MVT::i64, Res);  in ReplaceNodeResults()
33275         SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);  in ReplaceNodeResults()
33278                                                  dl, Tys, Ops, MVT::i64,  in ReplaceNodeResults()
33286         SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64);  in ReplaceNodeResults()
33292             X86ISD::FIST, dl, DAG.getVTList(MVT::Other), StoreOps, MVT::i64,  in ReplaceNodeResults()
33298         Result = DAG.getLoad(MVT::i64, dl, Chain, StackPtr, MPI);  in ReplaceNodeResults()
33331     if (SrcVT == MVT::v64i1 && DstVT == MVT::i64 && Subtarget.hasBWI()) {  in ReplaceNodeResults()
33335       Lo = DAG.getBitcast(MVT::i32, Lo);  in ReplaceNodeResults()
33336       Hi = DAG.getBitcast(MVT::i32, Hi);  in ReplaceNodeResults()
33337       SDValue Res = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);  in ReplaceNodeResults()
33342     if (DstVT.isVector() && SrcVT == MVT::x86mmx) {  in ReplaceNodeResults()
33348       SDValue Res = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64,  in ReplaceNodeResults()
33359     if ((VT == MVT::v2f32 || VT == MVT::v2i32) &&  in ReplaceNodeResults()
33363       if (Index.getValueType() != MVT::v2i64)  in ReplaceNodeResults()
33369       assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type");  in ReplaceNodeResults()
33376         Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i1, Mask,  in ReplaceNodeResults()
33377                            DAG.getUNDEF(MVT::v2i1));  in ReplaceNodeResults()
33378         Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Mask);  in ReplaceNodeResults()
33383           X86ISD::MGATHER, dl, DAG.getVTList(WideVT, MVT::Other), Ops,  in ReplaceNodeResults()
33395     MVT VT = N->getSimpleValueType(0);  in ReplaceNodeResults()
33403       MVT LdVT = Subtarget.is64Bit() && VT.isInteger() ? MVT::i64 : MVT::f64;  in ReplaceNodeResults()
33408       MVT VecVT = MVT::getVectorVT(LdVT, 2);  in ReplaceNodeResults()
33417     SDVTList Tys = DAG.getVTList(MVT::v4f32, MVT::Other);  in ReplaceNodeResults()
33420                                           MVT::i64, Ld->getMemOperand());  in ReplaceNodeResults()
33431     assert(N->getValueType(0) == MVT::i64 && "Unexpected VT!");  in ReplaceNodeResults()
34002   return VT1 == MVT::i32 && VT2 == MVT::i64 && Subtarget.is64Bit();  in isZExtFree()
34019   case MVT::i8:  in isZExtFree()
34020   case MVT::i16:  in isZExtFree()
34021   case MVT::i32:  in isZExtFree()
34099   if (SrcVT.getScalarType() == MVT::i1)  in isVectorLoadExtDesirable()
34116   case MVT::f16:  in isFMAFasterThanFMulAndFAdd()
34118   case MVT::f32:  in isFMAFasterThanFMulAndFAdd()
34119   case MVT::f64:  in isFMAFasterThanFMulAndFAdd()
34130   return !(VT1 == MVT::i32 && VT2 == MVT::i16);  in isNarrowingProfitable()
34156   if (VT.getSimpleVT().getScalarType() == MVT::i1)  in isShuffleMaskLegal()
34173     if (VT == MVT::v32i8 || VT == MVT::v16i16)  in isVectorClearMaskLegal()
34189 MVT X86TargetLowering::getPreferredSwitchConditionType(LLVMContext &Context,  in getPreferredSwitchConditionType()
34194     return MVT::i32;  in getPreferredSwitchConditionType()
34351   const TargetRegisterClass *OffsetRegClass = getRegClassFor(MVT::i32);  in EmitVAARGWithCustomInserter()
35514   MVT PVT = getPointerTy(MF->getDataLayout());  in emitSetJmpShadowStackFix()
35517   unsigned XorRROpc = (PVT == MVT::i64) ? X86::XOR64rr : X86::XOR32rr;  in emitSetJmpShadowStackFix()
35525   unsigned RdsspOpc = (PVT == MVT::i64) ? X86::RDSSPQ : X86::RDSSPD;  in emitSetJmpShadowStackFix()
35529   unsigned PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;  in emitSetJmpShadowStackFix()
35566   assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!");  in emitEHSjLjSetJmp()
35573   MVT PVT = getPointerTy(MF->getDataLayout());  in emitEHSjLjSetJmp()
35574   assert((PVT == MVT::i64 || PVT == MVT::i32) &&  in emitEHSjLjSetJmp()
35618     PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;  in emitEHSjLjSetJmp()
35638     PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi;  in emitEHSjLjSetJmp()
35715   MVT PVT = getPointerTy(MF->getDataLayout());  in emitLongJmpShadowStackFix()
35769   if (PVT == MVT::i64) {  in emitLongJmpShadowStackFix()
35780   unsigned RdsspOpc = (PVT == MVT::i64) ? X86::RDSSPQ : X86::RDSSPD;  in emitLongJmpShadowStackFix()
35785   unsigned TestRROpc = (PVT == MVT::i64) ? X86::TEST64rr : X86::TEST32rr;  in emitLongJmpShadowStackFix()
35795   unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm;  in emitLongJmpShadowStackFix()
35813   unsigned SubRROpc = (PVT == MVT::i64) ? X86::SUB64rr : X86::SUB32rr;  in emitLongJmpShadowStackFix()
35824   unsigned ShrRIOpc = (PVT == MVT::i64) ? X86::SHR64ri : X86::SHR32ri;  in emitLongJmpShadowStackFix()
35825   unsigned Offset = (PVT == MVT::i64) ? 3 : 2;  in emitLongJmpShadowStackFix()
35832   unsigned IncsspOpc = (PVT == MVT::i64) ? X86::INCSSPQ : X86::INCSSPD;  in emitLongJmpShadowStackFix()
35847   unsigned ShlR1Opc = (PVT == MVT::i64) ? X86::SHL64r1 : X86::SHL32r1;  in emitLongJmpShadowStackFix()
35854   unsigned MovRIOpc = (PVT == MVT::i64) ? X86::MOV64ri32 : X86::MOV32ri;  in emitLongJmpShadowStackFix()
35873   unsigned DecROpc = (PVT == MVT::i64) ? X86::DEC64r : X86::DEC32r;  in emitLongJmpShadowStackFix()
35896   MVT PVT = getPointerTy(MF->getDataLayout());  in emitEHSjLjLongJmp()
35897   assert((PVT == MVT::i64 || PVT == MVT::i32) &&  in emitEHSjLjLongJmp()
35901     (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass;  in emitEHSjLjLongJmp()
35905   Register FP = (PVT == MVT::i64) ? X86::RBP : X86::EBP;  in emitEHSjLjLongJmp()
35913   unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm;  in emitEHSjLjLongJmp()
35914   unsigned IJmpOpc = (PVT == MVT::i64) ? X86::JMP64r : X86::JMP32r;  in emitEHSjLjLongJmp()
35976   MVT PVT = getPointerTy(MF->getDataLayout());  in SetupEntryBlockForSjLj()
35977   assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!");  in SetupEntryBlockForSjLj()
35986     Op = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi;  in SetupEntryBlockForSjLj()
35989         (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass;  in SetupEntryBlockForSjLj()
35991     Op = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;  in SetupEntryBlockForSjLj()
36470     MVT SPTy = getPointerTy(MF->getDataLayout());  in EmitInstrWithCustomInserter()
36881     assert(VT.getScalarType() == MVT::i64 &&  in computeKnownBitsForTargetNode()
36882            Op.getOperand(0).getValueType().getScalarType() == MVT::i8 &&  in computeKnownBitsForTargetNode()
37090     MVT SrcVT = Src.getSimpleValueType();  in ComputeNumSignBitsForTargetNode()
37148     if (VT == MVT::f32 || VT == MVT::f64 ||  in ComputeNumSignBitsForTargetNode()
37149         ((VT == MVT::v4f32 || VT == MVT::v2f64) && DemandedElts == 1))  in ComputeNumSignBitsForTargetNode()
37236 static SDValue narrowLoadToVZLoad(LoadSDNode *LN, MVT MemVT, MVT VT,  in narrowLoadToVZLoad()
37242   SDVTList Tys = DAG.getVTList(VT, MVT::Other);  in narrowLoadToVZLoad()
37252 static bool matchUnaryShuffle(MVT MaskVT, ArrayRef<int> Mask,  in matchUnaryShuffle()
37256                               MVT &SrcVT, MVT &DstVT) {  in matchUnaryShuffle()
37268         SrcVT = DstVT = MaskVT.changeVectorElementType(MVT::f16);  in matchUnaryShuffle()
37270         SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT;  in matchUnaryShuffle()
37295         MVT ScalarTy = MaskVT.isInteger() ? MaskVT.getScalarType() :  in matchUnaryShuffle()
37296                                             MVT::getIntegerVT(MaskEltSize);  in matchUnaryShuffle()
37297         SrcVT = MVT::getVectorVT(ScalarTy, SrcSize / MaskEltSize);  in matchUnaryShuffle()
37303         DstVT = MVT::getIntegerVT(Scale * MaskEltSize);  in matchUnaryShuffle()
37304         DstVT = MVT::getVectorVT(DstVT, NumDstElts);  in matchUnaryShuffle()
37317       SrcVT = DstVT = MaskVT.changeVectorElementType(MVT::f16);  in matchUnaryShuffle()
37319       SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT;  in matchUnaryShuffle()
37329       SrcVT = DstVT = MVT::v2f64;  in matchUnaryShuffle()
37334       SrcVT = DstVT = MVT::v4f32;  in matchUnaryShuffle()
37339       SrcVT = DstVT = MVT::v4f32;  in matchUnaryShuffle()
37348       SrcVT = DstVT = MVT::v4f64;  in matchUnaryShuffle()
37354       SrcVT = DstVT = MVT::v8f32;  in matchUnaryShuffle()
37360       SrcVT = DstVT = MVT::v8f32;  in matchUnaryShuffle()
37371       SrcVT = DstVT = MVT::v8f64;  in matchUnaryShuffle()
37378       SrcVT = DstVT = MVT::v16f32;  in matchUnaryShuffle()
37385       SrcVT = DstVT = MVT::v16f32;  in matchUnaryShuffle()
37396 static bool matchUnaryPermuteShuffle(MVT MaskVT, ArrayRef<int> Mask,  in matchUnaryPermuteShuffle()
37401                                      unsigned &Shuffle, MVT &ShuffleVT,  in matchUnaryPermuteShuffle()
37406   MVT MaskEltVT = MVT::getIntegerVT(MaskScalarSizeInBits);  in matchUnaryPermuteShuffle()
37416         ShuffleVT = (AllowFloatDomain ? MVT::v4f64 : MVT::v4i64);  in matchUnaryPermuteShuffle()
37422         if (is256BitLaneRepeatedShuffleMask(MVT::v8f64, Mask, RepeatedMask)) {  in matchUnaryPermuteShuffle()
37424           ShuffleVT = (AllowFloatDomain ? MVT::v8f64 : MVT::v8i64);  in matchUnaryPermuteShuffle()
37432       ShuffleVT = MVT::getVectorVT(MVT::f64, Mask.size());  in matchUnaryPermuteShuffle()
37458       ShuffleVT = (AllowIntDomain ? MVT::i32 : MVT::f32);  in matchUnaryPermuteShuffle()
37459       ShuffleVT = MVT::getVectorVT(ShuffleVT, InputSizeInBits / 32);  in matchUnaryPermuteShuffle()
37479         ShuffleVT = MVT::getVectorVT(MVT::i16, InputSizeInBits / 16);  in matchUnaryPermuteShuffle()
37493         ShuffleVT = MVT::getVectorVT(MVT::i16, InputSizeInBits / 16);  in matchUnaryPermuteShuffle()
37533 static bool matchBinaryShuffle(MVT MaskVT, ArrayRef<int> Mask,  in matchBinaryShuffle()
37537                                unsigned &Shuffle, MVT &SrcVT, MVT &DstVT,  in matchBinaryShuffle()
37547       V1 = (SM_SentinelUndef == Mask[0] ? DAG.getUNDEF(MVT::v4f32) : V1);  in matchBinaryShuffle()
37549       SrcVT = DstVT = Subtarget.hasSSE2() ? MVT::v2f64 : MVT::v4f32;  in matchBinaryShuffle()
37556       SrcVT = DstVT = Subtarget.hasSSE2() ? MVT::v2f64 : MVT::v4f32;  in matchBinaryShuffle()
37563       SrcVT = DstVT = MVT::v2f64;  in matchBinaryShuffle()
37569       SrcVT = DstVT = MVT::v4f32;  in matchBinaryShuffle()
37576       SrcVT = DstVT = MVT::v8f16;  in matchBinaryShuffle()
37582   if (((MaskVT == MVT::v8i16 || MaskVT == MVT::v16i8) && Subtarget.hasSSE2()) ||  in matchBinaryShuffle()
37583       ((MaskVT == MVT::v16i16 || MaskVT == MVT::v32i8) && Subtarget.hasInt256()) ||  in matchBinaryShuffle()
37584       ((MaskVT == MVT::v32i16 || MaskVT == MVT::v64i8) && Subtarget.hasBWI())) {  in matchBinaryShuffle()
37593   if ((MaskVT == MVT::v4f32 && Subtarget.hasSSE1()) ||  in matchBinaryShuffle()
37602         SrcVT = DstVT = (32 == EltSizeInBits ? MVT::v8f32 : MVT::v4f64);  in matchBinaryShuffle()
37700     MVT MaskVT, ArrayRef<int> Mask, const APInt &Zeroable,  in matchBinaryPermuteShuffle()
37703     unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm) {  in matchBinaryPermuteShuffle()
37717           ShuffleVT = MVT::getVectorVT(MVT::i64, MaskVT.getSizeInBits() / 64);  in matchBinaryPermuteShuffle()
37719           ShuffleVT = MVT::getVectorVT(MVT::i32, MaskVT.getSizeInBits() / 32);  in matchBinaryPermuteShuffle()
37733       ShuffleVT = MVT::getVectorVT(MVT::i8, MaskVT.getSizeInBits() / 8);  in matchBinaryPermuteShuffle()
37742       (MaskVT == MVT::v16i16 && Subtarget.hasAVX2())) {  in matchBinaryPermuteShuffle()
37748       if (MaskVT == MVT::v16i16) {  in matchBinaryPermuteShuffle()
37782     ShuffleVT = MVT::v4f32;  in matchBinaryPermuteShuffle()
37797       ShuffleVT = MVT::getVectorVT(MVT::f64, MaskVT.getSizeInBits() / 64);  in matchBinaryPermuteShuffle()
37842         ShuffleVT = MVT::getVectorVT(MVT::f32, MaskVT.getSizeInBits() / 32);  in matchBinaryPermuteShuffle()
37854     ShuffleVT = MVT::v4f32;  in matchBinaryPermuteShuffle()
37888   MVT RootVT = Root.getSimpleValueType();  in combineX86ShuffleChain()
37893   auto CanonicalizeShuffleInput = [&](MVT VT, SDValue Op) {  in combineX86ShuffleChain()
37908   MVT VT1 = V1.getSimpleValueType();  in combineX86ShuffleChain()
37909   MVT VT2 = V2.getSimpleValueType();  in combineX86ShuffleChain()
37989     auto MatchSHUF128 = [&](MVT ShuffleVT, const SDLoc &DL,  in combineX86ShuffleChain()
38016                          DAG.getTargetConstant(PermMask, DL, MVT::i8));  in combineX86ShuffleChain()
38034       MVT ShuffleVT = (FloatDomain ? MVT::v8f64 : MVT::v8i64);  in combineX86ShuffleChain()
38082           DAG.getUNDEF(RootVT), DAG.getTargetConstant(PermMask, DL, MVT::i8));  in combineX86ShuffleChain()
38102                           DAG.getTargetConstant(PermMask, DL, MVT::i8));  in combineX86ShuffleChain()
38133   MVT MaskVT = FloatDomain ? MVT::getFloatingPointVT(MaskEltSizeInBits)  in combineX86ShuffleChain()
38134                            : MVT::getIntegerVT(MaskEltSizeInBits);  in combineX86ShuffleChain()
38135   MaskVT = MVT::getVectorVT(MaskVT, NumMaskElts);  in combineX86ShuffleChain()
38142   MVT ShuffleSrcVT, ShuffleVT;  in combineX86ShuffleChain()
38203                         DAG.getTargetConstant(PermuteImm, DL, MVT::i8));  in combineX86ShuffleChain()
38221         Res = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32,  in combineX86ShuffleChain()
38222                           CanonicalizeShuffleInput(MVT::v4f32, SrcV1),  in combineX86ShuffleChain()
38223                           CanonicalizeShuffleInput(MVT::v4f32, SrcV2),  in combineX86ShuffleChain()
38224                           DAG.getTargetConstant(PermuteImm, DL, MVT::i8));  in combineX86ShuffleChain()
38235       Res = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32,  in combineX86ShuffleChain()
38236                         CanonicalizeShuffleInput(MVT::v4f32, V1),  in combineX86ShuffleChain()
38237                         CanonicalizeShuffleInput(MVT::v4f32, V2),  in combineX86ShuffleChain()
38238                         DAG.getTargetConstant(PermuteImm, DL, MVT::i8));  in combineX86ShuffleChain()
38268                       DAG.getTargetConstant(PermuteImm, DL, MVT::i8));  in combineX86ShuffleChain()
38273   MVT IntMaskVT = MVT::getIntegerVT(MaskEltSizeInBits);  in combineX86ShuffleChain()
38274   IntMaskVT = MVT::getVectorVT(IntMaskVT, NumMaskElts);  in combineX86ShuffleChain()
38285                         DAG.getTargetConstant(BitLen, DL, MVT::i8),  in combineX86ShuffleChain()
38286                         DAG.getTargetConstant(BitIdx, DL, MVT::i8));  in combineX86ShuffleChain()
38296                         DAG.getTargetConstant(BitLen, DL, MVT::i8),  in combineX86ShuffleChain()
38297                         DAG.getTargetConstant(BitIdx, DL, MVT::i8));  in combineX86ShuffleChain()
38332       ShuffleSrcVT = MVT::getIntegerVT(MaskEltSizeInBits * 2);  in combineX86ShuffleChain()
38333       ShuffleSrcVT = MVT::getVectorVT(ShuffleSrcVT, NumMaskElts / 2);  in combineX86ShuffleChain()
38336       ShuffleSrcVT = MVT::getIntegerVT(MaskEltSizeInBits * 2);  in combineX86ShuffleChain()
38337       ShuffleSrcVT = MVT::getVectorVT(ShuffleSrcVT, NumMaskElts);  in combineX86ShuffleChain()
38369           (MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) {  in combineX86ShuffleChain()
38377            (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 ||  in combineX86ShuffleChain()
38378             MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) ||  in combineX86ShuffleChain()
38380            (MaskVT == MVT::v16i16 || MaskVT == MVT::v32i16)) ||  in combineX86ShuffleChain()
38382            (MaskVT == MVT::v32i8 || MaskVT == MVT::v64i8))) {  in combineX86ShuffleChain()
38394           (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 ||  in combineX86ShuffleChain()
38395            MaskVT == MVT::v4f64 || MaskVT == MVT::v4i64 ||  in combineX86ShuffleChain()
38396            MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32 ||  in combineX86ShuffleChain()
38397            MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) ||  in combineX86ShuffleChain()
38399           (MaskVT == MVT::v16i16 || MaskVT == MVT::v32i16)) ||  in combineX86ShuffleChain()
38401           (MaskVT == MVT::v32i8 || MaskVT == MVT::v64i8)))) {  in combineX86ShuffleChain()
38424           (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 ||  in combineX86ShuffleChain()
38425            MaskVT == MVT::v4f64 || MaskVT == MVT::v4i64 ||  in combineX86ShuffleChain()
38426            MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32 ||  in combineX86ShuffleChain()
38427            MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) ||  in combineX86ShuffleChain()
38429           (MaskVT == MVT::v16i16 || MaskVT == MVT::v32i16)) ||  in combineX86ShuffleChain()
38431           (MaskVT == MVT::v32i8 || MaskVT == MVT::v64i8)))) {  in combineX86ShuffleChain()
38471       ((MaskVT == MVT::v8f32 && Subtarget.hasAVX()) ||  in combineX86ShuffleChain()
38472        (MaskVT == MVT::v16f32 && Subtarget.hasAVX512()))) {  in combineX86ShuffleChain()
38476           M < 0 ? DAG.getUNDEF(MVT::i32) : DAG.getConstant(M % 4, DL, MVT::i32);  in combineX86ShuffleChain()
38488       (MaskVT == MVT::v2f64 || MaskVT == MVT::v4f64 || MaskVT == MVT::v4f32 ||  in combineX86ShuffleChain()
38489        MaskVT == MVT::v8f32)) {  in combineX86ShuffleChain()
38516                       DAG.getTargetConstant(M2ZImm, DL, MVT::i8));  in combineX86ShuffleChain()
38535         PSHUFBMask.push_back(DAG.getUNDEF(MVT::i8));  in combineX86ShuffleChain()
38539         PSHUFBMask.push_back(DAG.getConstant(0x80, DL, MVT::i8));  in combineX86ShuffleChain()
38544       PSHUFBMask.push_back(DAG.getConstant(M, DL, MVT::i8));  in combineX86ShuffleChain()
38546     MVT ByteVT = MVT::getVectorVT(MVT::i8, NumBytes);  in combineX86ShuffleChain()
38567         VPPERMMask.push_back(DAG.getUNDEF(MVT::i8));  in combineX86ShuffleChain()
38571         VPPERMMask.push_back(DAG.getConstant(0x80, DL, MVT::i8));  in combineX86ShuffleChain()
38575       VPPERMMask.push_back(DAG.getConstant(M, DL, MVT::i8));  in combineX86ShuffleChain()
38577     MVT ByteVT = MVT::v16i8;  in combineX86ShuffleChain()
38596         (MaskVT == MVT::v2f64 || MaskVT == MVT::v4f64 || MaskVT == MVT::v8f64 ||  in combineX86ShuffleChain()
38597          MaskVT == MVT::v2i64 || MaskVT == MVT::v4i64 || MaskVT == MVT::v8i64 ||  in combineX86ShuffleChain()
38598          MaskVT == MVT::v4f32 || MaskVT == MVT::v4i32 || MaskVT == MVT::v8f32 ||  in combineX86ShuffleChain()
38599          MaskVT == MVT::v8i32 || MaskVT == MVT::v16f32 ||  in combineX86ShuffleChain()
38600          MaskVT == MVT::v16i32)) ||  in combineX86ShuffleChain()
38602         (MaskVT == MVT::v8i16 || MaskVT == MVT::v16i16 ||  in combineX86ShuffleChain()
38603          MaskVT == MVT::v32i16)) ||  in combineX86ShuffleChain()
38605         (MaskVT == MVT::v16i8 || MaskVT == MVT::v32i8 ||  in combineX86ShuffleChain()
38606          MaskVT == MVT::v64i8)))) {  in combineX86ShuffleChain()
38764   MVT SrcVT = BC0.getOperand(0).getSimpleValueType();  in canonicalizeShuffleMaskWithHorizOp()
38829           MVT ShuffleVT = MVT::getVectorVT(MVT::f32, RootSizeInBits / 32);  in canonicalizeShuffleMaskWithHorizOp()
38923   MVT VT = Root.getSimpleValueType();  in combineX86ShufflesConstants()
38993   MVT MaskSVT;  in combineX86ShufflesConstants()
38995     MaskSVT = MVT::getFloatingPointVT(MaskSizeInBits);  in combineX86ShufflesConstants()
38997     MaskSVT = MVT::getIntegerVT(MaskSizeInBits);  in combineX86ShufflesConstants()
38999   MVT MaskVT = MVT::getVectorVT(MaskSVT, NumMaskElts);  in combineX86ShufflesConstants()
39053   MVT RootVT = Root.getSimpleValueType();  in combineX86ShufflesRecursively()
39071   if (VT.getVectorElementType() == MVT::f16)  in combineX86ShufflesRecursively()
39436   MVT VT = N.getSimpleValueType();  in getPSHUFShuffleMask()
39528       if (V.getSimpleValueType().getVectorElementType() != MVT::i8 &&  in combineRedundantDWordShuffle()
39529           V.getSimpleValueType().getVectorElementType() != MVT::i16)  in combineRedundantDWordShuffle()
39608 static SDValue combineCommutableSHUFP(SDValue N, MVT VT, const SDLoc &DL,  in combineCommutableSHUFP()
39611   if (VT != MVT::v4f32 && VT != MVT::v8f32 && VT != MVT::v16f32)  in combineCommutableSHUFP()
39627                        DAG.getTargetConstant(Imm, DL, MVT::i8));  in combineCommutableSHUFP()
39635                          DAG.getTargetConstant(Imm ^ 0xAA, DL, MVT::i8));  in combineCommutableSHUFP()
39645                            DAG.getTargetConstant(Imm ^ 0xAA, DL, MVT::i8));  in combineCommutableSHUFP()
39648                          DAG.getTargetConstant(Imm ^ 0x0A, DL, MVT::i8));  in combineCommutableSHUFP()
39651                          DAG.getTargetConstant(Imm ^ 0xA0, DL, MVT::i8));  in combineCommutableSHUFP()
39800   MVT VT = V.getSimpleValueType();  in canonicalizeLaneShuffleWithRepeatedOps()
39822     if (SrcVT0 == MVT::v4f64) {  in canonicalizeLaneShuffleWithRepeatedOps()
39851   MVT VT = N.getSimpleValueType();  in combineTargetShuffle()
39863     if (VT == MVT::v2f64 && Src.hasOneUse() &&  in combineTargetShuffle()
39866       if (SDValue VZLoad = narrowLoadToVZLoad(LN, MVT::f64, MVT::v2f64, DAG)) {  in combineTargetShuffle()
39867         SDValue Movddup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, VZLoad);  in combineTargetShuffle()
39945       SDVTList Tys = DAG.getVTList(VT, MVT::Other);  in combineTargetShuffle()
39966     if (SrcVT == MVT::i16 && Src.getOpcode() == ISD::TRUNCATE &&  in combineTargetShuffle()
39977           SDVTList Tys = DAG.getVTList(VT, MVT::Other);  in combineTargetShuffle()
39980               X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::i16,  in combineTargetShuffle()
39995           SDVTList Tys = DAG.getVTList(VT, MVT::Other);  in combineTargetShuffle()
40020           SDVTList Tys = DAG.getVTList(VT, MVT::Other);  in combineTargetShuffle()
40025               X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, MVT::i16,  in combineTargetShuffle()
40041         SDVTList Tys = DAG.getVTList(VT, MVT::Other);  in combineTargetShuffle()
40054     if ((SrcVT == MVT::v2f64 || SrcVT == MVT::v4f32 || SrcVT == MVT::v2i64 ||  in combineTargetShuffle()
40055          SrcVT == MVT::v4i32) &&  in combineTargetShuffle()
40060         SDVTList Tys = DAG.getVTList(VT, MVT::Other);  in combineTargetShuffle()
40097         SDVTList Tys = DAG.getVTList(VT, MVT::Other);  in combineTargetShuffle()
40114         N0.getOperand(0).getValueType() == MVT::i64) {  in combineTargetShuffle()
40118         SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, In);  in combineTargetShuffle()
40119         MVT VecVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() * 2);  in combineTargetShuffle()
40140         MVT PVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());  in combineTargetShuffle()
40160         MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(),  in combineTargetShuffle()
40181       MVT SrcVT = N0.getOperand(0).getSimpleValueType();  in combineTargetShuffle()
40191                             DAG.getTargetConstant(BlendMask, DL, MVT::i8)));  in combineTargetShuffle()
40200     if (VT == MVT::v4f32) {  in combineTargetShuffle()
40285         MVT SubVT = VT.getHalfNumVectorElementsVT();  in combineTargetShuffle()
40317         MVT SVT = VT.getVectorElementType();  in combineTargetShuffle()
40330     assert(VT == MVT::v4f32 && "INSERTPS ValueType must be MVT::v4f32");  in combineTargetShuffle()
40341                          DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));  in combineTargetShuffle()
40346                          DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));  in combineTargetShuffle()
40358                            DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));  in combineTargetShuffle()
40366                          DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));  in combineTargetShuffle()
40413                            DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));  in combineTargetShuffle()
40422         SDValue Load = DAG.getLoad(MVT::f32, DL, MemIntr->getChain(),  in combineTargetShuffle()
40428                            DAG.getTargetConstant(InsertPSMask & 0x3f, DL, MVT::i8));  in combineTargetShuffle()
40451     assert(VT.getVectorElementType() == MVT::i16 && "Bad word shuffle type!");  in combineTargetShuffle()
40461       MVT DVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2);  in combineTargetShuffle()
40623   MVT VT = N->getSimpleValueType(0);  in combineShuffleToFMAddSub()
40668   MVT VT = N->getSimpleValueType(0);  in combineShuffleToAddSubOrFMAddSub()
40690   if (VT.getVectorElementType() == MVT::f16)  in combineShuffleToAddSubOrFMAddSub()
40710   if (VT.getVectorElementType() != MVT::i32 &&  in combineShuffleOfConcatUndef()
40711       VT.getVectorElementType() != MVT::i64 &&  in combineShuffleOfConcatUndef()
40712       VT.getVectorElementType() != MVT::f32 &&  in combineShuffleOfConcatUndef()
40713       VT.getVectorElementType() != MVT::f64)  in combineShuffleOfConcatUndef()
40747   MVT VT = Shuf->getSimpleValueType(0);  in narrowShuffle()
40957     assert(VT.getScalarType() == MVT::i64 &&  in SimplifyDemandedVectorEltsForTargetNode()
40959            LHS.getValueType().getScalarType() == MVT::i8 &&  in SimplifyDemandedVectorEltsForTargetNode()
40984     MVT AmtVT = Amt.getSimpleValueType();  in SimplifyDemandedVectorEltsForTargetNode()
41074         SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);  in SimplifyDemandedVectorEltsForTargetNode()
41113         SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);  in SimplifyDemandedVectorEltsForTargetNode()
41191     MVT SrcVT = Src.getSimpleValueType();  in SimplifyDemandedVectorEltsForTargetNode()
41275     MVT SrcVT = Src.getSimpleValueType();  in SimplifyDemandedVectorEltsForTargetNode()
41317     MVT SrcVT = Src.getSimpleValueType();  in SimplifyDemandedVectorEltsForTargetNode()
41389       SDVTList Tys = TLO.DAG.getVTList(BcstVT, MVT::Other);  in SimplifyDemandedVectorEltsForTargetNode()
41448       if (VT == MVT::v4f64 || VT == MVT::v4i64) {  in SimplifyDemandedVectorEltsForTargetNode()
41510       MVT ExtVT = VT.getSimpleVT();  in SimplifyDemandedVectorEltsForTargetNode()
41511       ExtVT = MVT::getVectorVT(ExtVT.getScalarType(),  in SimplifyDemandedVectorEltsForTargetNode()
41626     MVT SrcVT = Src.getSimpleValueType();  in SimplifyDemandedBitsForTargetNode()
41697             TLO.DAG.getTargetConstant(std::abs(Diff), SDLoc(Op), MVT::i8));  in SimplifyDemandedBitsForTargetNode()
41816     MVT VecVT = Vec.getSimpleValueType();  in SimplifyDemandedBitsForTargetNode()
41855     MVT VecVT = Vec.getSimpleValueType();  in SimplifyDemandedBitsForTargetNode()
41913     MVT SrcVT = Src.getSimpleValueType();  in SimplifyDemandedBitsForTargetNode()
41925       MVT NewSrcVT = MVT::getIntegerVT(BitWidth / 2);  in SimplifyDemandedBitsForTargetNode()
41928       MVT NewVT = MVT::getVectorVT(NewSrcVT, VT.getVectorNumElements() * 2);  in SimplifyDemandedBitsForTargetNode()
41944     MVT SrcVT = Src.getSimpleValueType();  in SimplifyDemandedBitsForTargetNode()
42084     MVT VecVT = Vec.getSimpleValueType();  in SimplifyMultipleUseDemandedBitsForTargetNode()
42216   if (SrcVT != MVT::v4i1)  in adjustBitcastSrcVectorSSE1()
42221     if (Src.getOperand(0).getValueType() == MVT::v4i32 &&  in adjustBitcastSrcVectorSSE1()
42226         return DAG.getBitcast(MVT::v4f32, Op0);  in adjustBitcastSrcVectorSSE1()
42228           Op0.getOperand(0).getValueType() == MVT::v4f32)  in adjustBitcastSrcVectorSSE1()
42238       return DAG.getNode(getAltBitOpcode(Src.getOpcode()), DL, MVT::v4f32, Op0,  in adjustBitcastSrcVectorSSE1()
42274   if (!SrcVT.isSimple() || SrcVT.getScalarType() != MVT::i1)  in combineBitcastvxi1()
42281       V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32,  in combineBitcastvxi1()
42282                       DAG.getBitcast(MVT::v4f32, V));  in combineBitcastvxi1()
42292                       (Src.getOperand(0).getValueType() == MVT::v16i8 ||  in combineBitcastvxi1()
42293                        Src.getOperand(0).getValueType() == MVT::v32i8 ||  in combineBitcastvxi1()
42294                        Src.getOperand(0).getValueType() == MVT::v64i8);  in combineBitcastvxi1()
42304         (EltVT == MVT::i8 || EltVT == MVT::i32 || EltVT == MVT::i64))  in combineBitcastvxi1()
42323   MVT SExtVT;  in combineBitcastvxi1()
42328   case MVT::v2i1:  in combineBitcastvxi1()
42329     SExtVT = MVT::v2i64;  in combineBitcastvxi1()
42331   case MVT::v4i1:  in combineBitcastvxi1()
42332     SExtVT = MVT::v4i32;  in combineBitcastvxi1()
42337       SExtVT = MVT::v4i64;  in combineBitcastvxi1()
42341   case MVT::v8i1:  in combineBitcastvxi1()
42342     SExtVT = MVT::v8i16;  in combineBitcastvxi1()
42350       SExtVT = MVT::v8i32;  in combineBitcastvxi1()
42354   case MVT::v16i1:  in combineBitcastvxi1()
42355     SExtVT = MVT::v16i8;  in combineBitcastvxi1()
42361   case MVT::v32i1:  in combineBitcastvxi1()
42362     SExtVT = MVT::v32i8;  in combineBitcastvxi1()
42364   case MVT::v64i1:  in combineBitcastvxi1()
42370       SExtVT = MVT::v64i8;  in combineBitcastvxi1()
42375       SExtVT = MVT::v64i8;  in combineBitcastvxi1()
42384   if (SExtVT == MVT::v16i8 || SExtVT == MVT::v32i8 || SExtVT == MVT::v64i8) {  in combineBitcastvxi1()
42387     if (SExtVT == MVT::v8i16)  in combineBitcastvxi1()
42388       V = DAG.getNode(X86ISD::PACKSS, DL, MVT::v16i8, V,  in combineBitcastvxi1()
42389                       DAG.getUNDEF(MVT::v8i16));  in combineBitcastvxi1()
42390     V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);  in combineBitcastvxi1()
42402   assert(SrcVT.getVectorElementType() == MVT::i1 &&  in combinevXi1ConstantToInteger()
42443   if (!(SrcVT.isVector() && SrcVT.getVectorElementType() == MVT::i1 &&  in combineCastedMaskArithmetic()
42445       !(DstVT.isVector() && DstVT.getVectorElementType() == MVT::i1 &&  in combineCastedMaskArithmetic()
42482       return DAG.getUNDEF(MVT::x86mmx);  in createMMXBuildVector()
42485         V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, V);  in createMMXBuildVector()
42486         V = DAG.getBitcast(MVT::v2i64, V);  in createMMXBuildVector()
42487         return DAG.getNode(X86ISD::MOVDQ2Q, DL, MVT::x86mmx, V);  in createMMXBuildVector()
42489       V = DAG.getBitcast(MVT::i32, V);  in createMMXBuildVector()
42491       V = DAG.getAnyExtOrTrunc(V, DL, MVT::i32);  in createMMXBuildVector()
42493     return DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, V);  in createMMXBuildVector()
42504       return DAG.getUNDEF(MVT::x86mmx);  in createMMXBuildVector()
42512             ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx,  in createMMXBuildVector()
42520           ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx,  in createMMXBuildVector()
42523           Splat, DAG.getTargetConstant(ShufMask, DL, MVT::i8));  in createMMXBuildVector()
42541       Ops[i / 2] = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx, Intrin,  in createMMXBuildVector()
42571         EVT::getVectorVT(*DAG.getContext(), MVT::i1, Src.getValueSizeInBits());  in combineBitcastToBoolVector()
42583     EVT NewSrcVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,  in combineBitcastToBoolVector()
42606     if ((VT == MVT::v8i1 && !Subtarget.hasDQI()) ||  in combineBitcastToBoolVector()
42607         ((VT == MVT::v32i1 || VT == MVT::v64i1) && !Subtarget.hasBWI()))  in combineBitcastToBoolVector()
42614             DAG.getTargetConstant(Amt->getZExtValue(), DL, MVT::i8));  in combineBitcastToBoolVector()
42642     if ((VT == MVT::v4i1 || VT == MVT::v2i1) && SrcVT.isScalarInteger() &&  in combineBitcast()
42644       N0 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i8, N0);  in combineBitcast()
42645       N0 = DAG.getBitcast(MVT::v8i1, N0);  in combineBitcast()
42652     if ((SrcVT == MVT::v4i1 || SrcVT == MVT::v2i1) && VT.isScalarInteger() &&  in combineBitcast()
42668           N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops);  in combineBitcast()
42669           N0 = DAG.getBitcast(MVT::i8, N0);  in combineBitcast()
42677       N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops);  in combineBitcast()
42678       N0 = DAG.getBitcast(MVT::i8, N0);  in combineBitcast()
42684     if (VT.isVector() && VT.getScalarType() == MVT::i1 &&  in combineBitcast()
42697   if (VT == MVT::i8 && SrcVT == MVT::v8i1 && Subtarget.hasAVX512() &&  in combineBitcast()
42699       N0.getOperand(0).getValueType() == MVT::v16i1 &&  in combineBitcast()
42702                        DAG.getBitcast(MVT::i16, N0.getOperand(0)));  in combineBitcast()
42714       MVT MemVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(MemSize)  in combineBitcast()
42715                                        : MVT::getIntegerVT(MemSize);  in combineBitcast()
42716       MVT LoadVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(SrcVTSize)  in combineBitcast()
42717                                         : MVT::getIntegerVT(SrcVTSize);  in combineBitcast()
42718       LoadVT = MVT::getVectorVT(LoadVT, SrcVT.getVectorNumElements());  in combineBitcast()
42720       SDVTList Tys = DAG.getVTList(LoadVT, MVT::Other);  in combineBitcast()
42733   if (VT == MVT::x86mmx) {  in combineBitcast()
42742                            DAG.getConstant(EltBits[0].trunc(32), DL, MVT::i32));  in combineBitcast()
42746       return DAG.getBitcast(VT, DAG.getConstantFP(F64, DL, MVT::f64));  in combineBitcast()
42751         (SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8) &&  in combineBitcast()
42762         N00 = LowUndef ? DAG.getAnyExtOrTrunc(N00, dl, MVT::i32)  in combineBitcast()
42763                        : DAG.getZExtOrTrunc(N00, dl, MVT::i32);  in combineBitcast()
42772         (SrcVT == MVT::v2f32 || SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 ||  in combineBitcast()
42773          SrcVT == MVT::v8i8))  in combineBitcast()
42783                            DAG.getBitcast(MVT::v2i64, N00));  in combineBitcast()
42787     if (SrcVT == MVT::v2i32 && N0.getOpcode() == ISD::FP_TO_SINT) {  in combineBitcast()
42789       SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,  in combineBitcast()
42790                                 DAG.getUNDEF(MVT::v2i32));  in combineBitcast()
42792                          DAG.getBitcast(MVT::v2i64, Res));  in combineBitcast()
42799       SrcVT.isVector() && SrcVT.getVectorElementType() == MVT::i1 &&  in combineBitcast()
42805       VT.isVector() && VT.getVectorElementType() == MVT::i1 &&  in combineBitcast()
42818       VT.isVector() && VT.getVectorElementType() == MVT::i1 &&  in combineBitcast()
42829       MVT MovmskVT = MovmskIn.getSimpleValueType();  in combineBitcast()
42835           (Subtarget.hasBWI() || MovmskVT.getVectorElementType() != MVT::i8)) {  in combineBitcast()
42839         MVT CmpVT = MVT::getVectorVT(MVT::i1, MovMskElts);  in combineBitcast()
42874   if (!((Subtarget.hasSSE1() && VT == MVT::f32) ||  in combineBitcast()
42875         (Subtarget.hasSSE2() && VT == MVT::f64) ||  in combineBitcast()
42876         (Subtarget.hasFP16() && VT == MVT::f16) ||  in combineBitcast()
42951       Op0.getOperand(0).getValueType().getVectorElementType() != MVT::i8 ||  in detectZextAbsDiff()
42953       Op1.getOperand(0).getValueType().getVectorElementType() != MVT::i8)  in detectZextAbsDiff()
42963   MVT Vi8VT =  in createVPDPBUSD()
42964       MVT::getVectorVT(MVT::i8, LHS.getValueType().getVectorElementCount());  in createVPDPBUSD()
42984   MVT ExtendedVT = MVT::getVectorVT(MVT::i8, RegSize / 8);  in createVPDPBUSD()
42993     MVT VT = MVT::getVectorVT(MVT::i32, Ops[0].getValueSizeInBits() / 32);  in createVPDPBUSD()
42996   MVT DpVT = MVT::getVectorVT(MVT::i32, RegSize / 32);  in createVPDPBUSD()
43017   MVT ExtendedVT = MVT::getVectorVT(MVT::i8, RegSize / 8);  in createPSADBW()
43025     MVT VT = MVT::getVectorVT(MVT::i64, Ops[0].getValueSizeInBits() / 64);  in createPSADBW()
43028   MVT SadVT = MVT::getVectorVT(MVT::i64, RegSize / 64);  in createPSADBW()
43042   if (ExtractVT != MVT::i16 && ExtractVT != MVT::i8)  in combineMinMaxReduction()
43067   assert(((SrcVT == MVT::v8i16 && ExtractVT == MVT::i16) ||  in combineMinMaxReduction()
43068           (SrcVT == MVT::v16i8 && ExtractVT == MVT::i8)) &&  in combineMinMaxReduction()
43089   if (ExtractVT == MVT::i8) {  in combineMinMaxReduction()
43091         SrcVT, DL, MinPos, DAG.getConstant(0, DL, MVT::v16i8),  in combineMinMaxReduction()
43097   MinPos = DAG.getBitcast(MVT::v8i16, MinPos);  in combineMinMaxReduction()
43098   MinPos = DAG.getNode(X86ISD::PHMINPOS, DL, MVT::v8i16, MinPos);  in combineMinMaxReduction()
43117   if (ExtractVT != MVT::i64 && ExtractVT != MVT::i32 && ExtractVT != MVT::i16 &&  in combinePredicateReduction()
43118       ExtractVT != MVT::i8 && ExtractVT != MVT::i1)  in combinePredicateReduction()
43124   if (!Match && ExtractVT == MVT::i1)  in combinePredicateReduction()
43141   if (ExtractVT == MVT::i1) {  in combinePredicateReduction()
43155         if (VecSVT != MVT::i8) {  in combinePredicateReduction()
43157           EVT CmpVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, NumElts);  in combinePredicateReduction()
43158           MatchVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElts);  in combinePredicateReduction()
43177     Movmsk = DAG.getZExtOrTrunc(Movmsk, DL, NumElts > 32 ? MVT::i64 : MVT::i32);  in combinePredicateReduction()
43204     MVT MaskSrcVT;  in combinePredicateReduction()
43206       MaskSrcVT = MVT::getVectorVT(MVT::getFloatingPointVT(BitWidth),  in combinePredicateReduction()
43209       MaskSrcVT = MVT::getVectorVT(MVT::i8, MatchSizeInBits / 8);  in combinePredicateReduction()
43218   MVT CmpVT = NumElts == 64 ? MVT::i64 : MVT::i32;  in combinePredicateReduction()
43256   if (ExtractVT != MVT::i32)  in combineVPDPBUSDPattern()
43328   if (ExtractVT != MVT::i32 && ExtractVT != MVT::i64)  in combineBasicSADPattern()
43412   if (SrcSVT == MVT::i1 || !isa<ConstantSDNode>(Idx))  in combineExtractWithShuffle()
43483     MVT ExtractVT = MVT::getVectorVT(SrcSVT.getSimpleVT(), 128 / SrcEltBits);  in combineExtractWithShuffle()
43495         (VecSVT == MVT::i8 || VecSVT == MVT::i16 || VecSVT == MVT::i32 ||  in combineExtractWithShuffle()
43496          VecSVT == MVT::i64)) {  in combineExtractWithShuffle()
43505     if ((VecVT == MVT::v4i32 || VecVT == MVT::v2i64) &&  in combineExtractWithShuffle()
43511     if ((VecVT == MVT::v8i16 && Subtarget.hasSSE2()) ||  in combineExtractWithShuffle()
43512         (VecVT == MVT::v16i8 && Subtarget.hasSSE41())) {  in combineExtractWithShuffle()
43513       unsigned OpCode = (VecVT == MVT::v8i16 ? X86ISD::PEXTRW : X86ISD::PEXTRB);  in combineExtractWithShuffle()
43514       return DAG.getNode(OpCode, dl, MVT::i32, DAG.getBitcast(VecVT, Vec),  in combineExtractWithShuffle()
43515                          DAG.getTargetConstant(Idx, dl, MVT::i8));  in combineExtractWithShuffle()
43609   if (Vec.getOpcode() == ISD::SETCC && VT == MVT::i1) {  in scalarizeExtEltFP()
43611     if (OpVT != MVT::f32 && OpVT != MVT::f64)  in scalarizeExtEltFP()
43623   if (!(VT == MVT::f16 && Subtarget.hasFP16()) && VT != MVT::f32 &&  in scalarizeExtEltFP()
43624       VT != MVT::f64)  in scalarizeExtEltFP()
43635       Vec.getOperand(0).getValueType().getScalarType() == MVT::i1 &&  in scalarizeExtEltFP()
43723     if (V.getValueType() == MVT::v4i8) {  in combineArithReduction()
43725         V = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v4i32,  in combineArithReduction()
43726                         DAG.getConstant(0, DL, MVT::v4i32),  in combineArithReduction()
43727                         DAG.getBitcast(MVT::i32, V),  in combineArithReduction()
43729         return DAG.getBitcast(MVT::v16i8, V);  in combineArithReduction()
43731       V = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, V,  in combineArithReduction()
43732                       ZeroExtend ? DAG.getConstant(0, DL, MVT::v4i8)  in combineArithReduction()
43733                                  : DAG.getUNDEF(MVT::v4i8));  in combineArithReduction()
43735     return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V,  in combineArithReduction()
43736                        DAG.getUNDEF(MVT::v8i8));  in combineArithReduction()
43741     if (VT != MVT::i8 || NumElts < 4 || !isPowerOf2_32(NumElts))  in combineArithReduction()
43744       EVT WideVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts / 2);  in combineArithReduction()
43756       Rdx = getUnpackl(DAG, DL, MVT::v16i8, Rdx, DAG.getUNDEF(MVT::v16i8));  in combineArithReduction()
43757       Rdx = DAG.getBitcast(MVT::v8i16, Rdx);  in combineArithReduction()
43760       Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,  in combineArithReduction()
43761                         DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,  in combineArithReduction()
43763     Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,  in combineArithReduction()
43764                       DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,  in combineArithReduction()
43766     Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,  in combineArithReduction()
43767                       DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,  in combineArithReduction()
43769     Rdx = DAG.getBitcast(MVT::v16i8, Rdx);  in combineArithReduction()
43774   if (VecVT == MVT::v4i8 || VecVT == MVT::v8i8) {  in combineArithReduction()
43776     Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,  in combineArithReduction()
43777                       DAG.getConstant(0, DL, MVT::v16i8));  in combineArithReduction()
43778     Rdx = DAG.getBitcast(MVT::v16i8, Rdx);  in combineArithReduction()
43787   if (VT == MVT::i8) {  in combineArithReduction()
43794     assert(VecVT == MVT::v16i8 && "v16i8 reduction expected");  in combineArithReduction()
43797         MVT::v16i8, DL, Rdx, Rdx,  in combineArithReduction()
43799     Rdx = DAG.getNode(ISD::ADD, DL, MVT::v16i8, Rdx, Hi);  in combineArithReduction()
43800     Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,  in combineArithReduction()
43801                       getZeroVector(MVT::v16i8, Subtarget, DAG, DL));  in combineArithReduction()
43802     Rdx = DAG.getBitcast(MVT::v16i8, Rdx);  in combineArithReduction()
43814     EVT ByteVT = VecVT.changeVectorElementType(MVT::i8);  in combineArithReduction()
43822       MVT VT = MVT::getVectorVT(MVT::i64, Ops[0].getValueSizeInBits() / 64);  in combineArithReduction()
43826     MVT SadVT = MVT::getVectorVT(MVT::i64, Rdx.getValueSizeInBits() / 64);  in combineArithReduction()
43836     assert(Rdx.getValueType() == MVT::v2i64 && "v2i64 reduction expected");  in combineArithReduction()
43839       SDValue RdxHi = DAG.getVectorShuffle(MVT::v2i64, DL, Rdx, Rdx, {1, -1});  in combineArithReduction()
43840       Rdx = DAG.getNode(ISD::ADD, DL, MVT::v2i64, Rdx, RdxHi);  in combineArithReduction()
43843     VecVT = MVT::getVectorVT(VT.getSimpleVT(), 128 / VT.getSizeInBits());  in combineArithReduction()
43858   if (((VecVT == MVT::v16i16 || VecVT == MVT::v8i32) && Subtarget.hasSSSE3()) ||  in combineArithReduction()
43859       ((VecVT == MVT::v8f32 || VecVT == MVT::v4f64) && Subtarget.hasSSE3())) {  in combineArithReduction()
43866   if (!((VecVT == MVT::v8i16 || VecVT == MVT::v4i32) && Subtarget.hasSSSE3()) &&  in combineArithReduction()
43867       !((VecVT == MVT::v4f32 || VecVT == MVT::v2f64) && Subtarget.hasSSE3()))  in combineArithReduction()
43941       VT == MVT::i64 && SrcVT == MVT::v1i64 && isNullConstant(EltIdx)) {  in combineExtractVectorElt()
43945     if (MMXSrc.getValueType() == MVT::x86mmx)  in combineExtractVectorElt()
43951       VT == MVT::i32 && SrcVT == MVT::v2i32 && isNullConstant(EltIdx)) {  in combineExtractVectorElt()
43955     if (MMXSrc.getValueType() == MVT::x86mmx)  in combineExtractVectorElt()
43956       return DAG.getNode(X86ISD::MMX_MOVD2W, dl, MVT::i32, MMXSrc);  in combineExtractVectorElt()
43993   if (SrcVT.getScalarType() == MVT::i1) {  in combineExtractVectorElt()
44000           Use->getValueType(0) == MVT::i1) {  in combineExtractVectorElt()
44016           SDValue MaskIdx = DAG.getZExtOrTrunc(Use->getOperand(1), dl, MVT::i8);  in combineExtractVectorElt()
44020           Res = DAG.getSetCC(dl, MVT::i1, Res, Mask, ISD::SETEQ);  in combineExtractVectorElt()
44081   if (SVT != MVT::i64 && SVT != MVT::i32 && SVT != MVT::i16 && SVT != MVT::i8)  in combineToExtendBoolVectorInReg()
44083   if (InSVT != MVT::i1 || N0.getOpcode() != ISD::BITCAST)  in combineToExtendBoolVectorInReg()
44112              (SclVT == MVT::i8 || SclVT == MVT::i16 || SclVT == MVT::i32)) {  in combineToExtendBoolVectorInReg()
44146   EVT CCVT = VT.changeVectorElementType(MVT::i1);  in combineToExtendBoolVectorInReg()
44253     if (CondVT.getScalarType() == MVT::i1)  in combineVSelectWithAllOnesOrZeros()
44316   if (Cond.getValueType() != MVT::i1)  in combineSelectOfTwoConstants()
44341       ((VT == MVT::i32 || VT == MVT::i64) &&  in combineSelectOfTwoConstants()
44348       Cond = DAG.getNOT(DL, Cond, MVT::i1);  in combineSelectOfTwoConstants()
44402   if (VT.getVectorElementType() == MVT::i16)  in combineVSelectToBLENDV()
44408   if (VT == MVT::v32i8 && !Subtarget.hasAVX2())  in combineVSelectToBLENDV()
44551       (!CondConstantVector || CondVT.getScalarType() == MVT::i8) &&  in combineSelect()
44572     MVT SimpleVT = VT.getSimpleVT();  in combineSelect()
44604       VT != MVT::f80 && VT != MVT::f128 && !isSoftFP16(VT, Subtarget) &&  in combineSelect()
44605       (TLI.isTypeLegal(VT) || VT == MVT::v2f32) &&  in combineSelect()
44607        (Subtarget.hasSSE1() && VT.getScalarType() == MVT::f32))) {  in combineSelect()
44757       Cond.getOpcode() == ISD::SETCC && (VT == MVT::f32 || VT == MVT::f64)) {  in combineSelect()
44765       AndNode = DAG.getZExtOrTrunc(AndNode, DL, MVT::i8);  in combineSelect()
44778       CondVT.getVectorElementType() == MVT::i1 &&  in combineSelect()
44779       (VT.getVectorElementType() == MVT::i8 ||  in combineSelect()
44780        VT.getVectorElementType() == MVT::i16)) {  in combineSelect()
44790       CondVT.getVectorElementType() == MVT::i1) {  in combineSelect()
44807       EVT SrcCondVT = SrcVT.changeVectorElementType(MVT::i1);  in combineSelect()
44898       Subtarget.hasAVX512() && CondVT.getVectorElementType() == MVT::i1 &&  in combineSelect()
44910       CondVT.getVectorElementType() == MVT::i1 && Cond.hasOneUse() &&  in combineSelect()
44934   if (CondVT.getScalarType() != MVT::i1) {  in combineSelect()
44953       VT.getVectorElementType() == MVT::i1 &&  in combineSelect()
44954       (DCI.isBeforeLegalize() || (VT != MVT::v64i1 || Subtarget.is64Bit()))) {  in combineSelect()
44985       Cond.hasOneUse() && CondVT.getVectorElementType() == MVT::i1 &&  in combineSelect()
45478         assert(VT == MVT::i32 && "Expected i32 EFLAGS comparison result");  in combinePTESTCC()
45487             MVT MovmskVT = BCVT.is128BitVector() ? MVT::v16i8 : MVT::v32i8;  in combinePTESTCC()
45490             Res = DAG.getNode(ISD::AND, DL, MVT::i32, Res,  in combinePTESTCC()
45491                               DAG.getConstant(0xAAAAAAAA, DL, MVT::i32));  in combinePTESTCC()
45495           return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Res,  in combinePTESTCC()
45496                              DAG.getConstant(0, DL, MVT::i32));  in combinePTESTCC()
45521                              DAG.getBitcast(MVT::v4i64, Src0),  in combinePTESTCC()
45522                              DAG.getBitcast(MVT::v4i64, Src1));  in combinePTESTCC()
45538   if (EFLAGS.getValueType() != MVT::i32)  in combineSetCCMOVMSK()
45561   MVT VecVT = Vec.getSimpleValueType();  in combineSetCCMOVMSK()
45589     MVT BCVT = BC.getSimpleValueType();  in combineSetCCMOVMSK()
45597       return DAG.getNode(X86ISD::CMP, DL, MVT::i32,  in combineSetCCMOVMSK()
45598                          DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, BC),  in combineSetCCMOVMSK()
45599                          DAG.getConstant(CmpMask, DL, MVT::i32));  in combineSetCCMOVMSK()
45618       return DAG.getNode(X86ISD::CMP, DL, MVT::i32,  in combineSetCCMOVMSK()
45619                          DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V),  in combineSetCCMOVMSK()
45620                          DAG.getConstant(CmpMask, DL, MVT::i32));  in combineSetCCMOVMSK()
45629     MVT TestVT = VecVT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;  in combineSetCCMOVMSK()
45637         return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V);  in combineSetCCMOVMSK()
45652         return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V);  in combineSetCCMOVMSK()
45662   if (Vec.getOpcode() == X86ISD::PACKSS && VecVT == MVT::v16i8) {  in combineSetCCMOVMSK()
45670       SDValue Result = DAG.getBitcast(MVT::v16i8, VecOp0);  in combineSetCCMOVMSK()
45671       Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);  in combineSetCCMOVMSK()
45672       Result = DAG.getZExtOrTrunc(Result, DL, MVT::i16);  in combineSetCCMOVMSK()
45674         Result = DAG.getNode(ISD::AND, DL, MVT::i16, Result,  in combineSetCCMOVMSK()
45675                              DAG.getConstant(0xAAAA, DL, MVT::i16));  in combineSetCCMOVMSK()
45677       return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result,  in combineSetCCMOVMSK()
45678                          DAG.getConstant(0, DL, MVT::i16));  in combineSetCCMOVMSK()
45691           V = DAG.getBitcast(MVT::v4i64, V);  in combineSetCCMOVMSK()
45692           return DAG.getNode(X86ISD::PTEST, SDLoc(EFLAGS), MVT::i32, V, V);  in combineSetCCMOVMSK()
45694         Result = DAG.getBitcast(MVT::v32i8, Result);  in combineSetCCMOVMSK()
45695         Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);  in combineSetCCMOVMSK()
45700           Result = DAG.getNode(ISD::AND, DL, MVT::i32, Result,  in combineSetCCMOVMSK()
45701                                DAG.getConstant(0xAAAAAAAA, DL, MVT::i32));  in combineSetCCMOVMSK()
45703         return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result,  in combineSetCCMOVMSK()
45704                            DAG.getConstant(CmpMask, DL, MVT::i32));  in combineSetCCMOVMSK()
45726       Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);  in combineSetCCMOVMSK()
45729       return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result,  in combineSetCCMOVMSK()
45777     if (!(FalseOp.getValueType() == MVT::f80 ||  in combineCMov()
45778           (FalseOp.getValueType() == MVT::f64 && !Subtarget.hasSSE2()) ||  in combineCMov()
45779           (FalseOp.getValueType() == MVT::f32 && !Subtarget.hasSSE1())) ||  in combineCMov()
45781       SDValue Ops[] = {FalseOp, TrueOp, DAG.getTargetConstant(CC, DL, MVT::i8),  in combineCMov()
45811                            DAG.getConstant(ShAmt, DL, MVT::i8));  in combineCMov()
45830       if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) {  in combineCMov()
45903                          DAG.getTargetConstant(CC, DL, MVT::i8), Cond};  in combineCMov()
45938                         DAG.getTargetConstant(CC0, DL, MVT::i8), Flags};  in combineCMov()
45940       SDValue Ops[] = {LCMOV, TrueOp, DAG.getTargetConstant(CC1, DL, MVT::i8),  in combineCMov()
45975                       DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8), Cond);  in combineCMov()
46075   EVT ReducedVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts);  in reduceVMULWidth()
46089   EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts / 2);  in reduceVMULWidth()
46125                          DAG.getConstant(Shift, DL, MVT::i8));  in combineMulSpecial()
46194                                    DAG.getConstant(ShiftAmt, DL, MVT::i8));  in combineMulSpecial()
46196                                    DAG.getConstant(ScaleShift, DL, MVT::i8));  in combineMulSpecial()
46218   if (!VT.isVector() || VT.getVectorElementType() != MVT::i32)  in combineMulToPMADDWD()
46279         EVT ExtVT = VT.changeVectorElementType(MVT::i16);  in combineMulToPMADDWD()
46309     MVT ResVT = MVT::getVectorVT(MVT::i32, Ops[0].getValueSizeInBits() / 32);  in combineMulToPMADDWD()
46310     MVT OpVT = MVT::getVectorVT(MVT::i16, Ops[0].getValueSizeInBits() / 16);  in combineMulToPMADDWD()
46327   if (!VT.isVector() || VT.getVectorElementType() != MVT::i64 ||  in combineMulToPMULDQ()
46387   if (VT != MVT::i64 && VT != MVT::i32)  in combineMul()
46441                            DAG.getConstant(Log2_64(MulAmt1), DL, MVT::i8));  in combineMul()
46448                            DAG.getConstant(Log2_64(MulAmt2), DL, MVT::i8));  in combineMul()
46462            C->getZExtValue() != (VT == MVT::i64 ? UINT64_MAX : UINT32_MAX) &&  in combineMul()
46471                                       MVT::i8)));  in combineMul()
46480                                            DL, MVT::i8));  in combineMul()
46490                                            DL, MVT::i8));  in combineMul()
46497                                            DL, MVT::i8));  in combineMul()
46552   if (MulVT.getVectorElementType() != MVT::i16 || RHS.getValueType() != MulVT)  in combineShiftToPMULH()
46656   for (MVT SVT : { MVT::i8, MVT::i16, MVT::i32 }) {  in combineShiftRightArithmetic()
46764           MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f32 : MVT::v4i32;  in combineHorizOpWithShuffle()
46825         MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f32 : MVT::v4i32;  in combineHorizOpWithShuffle()
46861         MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f64 : MVT::v4i64;  in combineHorizOpWithShuffle()
46954       N0.getOpcode() == ISD::TRUNCATE && N1.isUndef() && VT == MVT::v16i8 &&  in combineVectorPack()
46955       N0.getOperand(0).getValueType() == MVT::v8i32) {  in combineVectorPack()
46964       SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i32,  in combineVectorPack()
46965                                    N0.getOperand(0), DAG.getUNDEF(MVT::v8i32));  in combineVectorPack()
47016     MVT VT = N->getSimpleValueType(0);  in combineVectorHADDSUB()
47035         MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32);  in combineVectorHADDSUB()
47100   assert(N->getOperand(1).getValueType() == MVT::i8 &&  in combineVectorShiftImm()
47144                        DAG.getTargetConstant(NewShiftVal, SDLoc(N), MVT::i8));  in combineVectorShiftImm()
47192   assert(((N->getOpcode() == X86ISD::PINSRB && VT == MVT::v16i8) ||  in combineVectorInsert()
47193           (N->getOpcode() == X86ISD::PINSRW && VT == MVT::v8i16) ||  in combineVectorInsert()
47240     if (VT == MVT::f32 || VT == MVT::f64 ||  in combineCompareEqual()
47241         (VT == MVT::f16 && Subtarget.hasFP16())) {  in combineCompareEqual()
47280                 DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CMP00, CMP01,  in combineCompareEqual()
47281                             DAG.getTargetConstant(x86cc, DL, MVT::i8));  in combineCompareEqual()
47284             SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v16i1,  in combineCompareEqual()
47285                                       DAG.getConstant(0, DL, MVT::v16i1),  in combineCompareEqual()
47287             return DAG.getZExtOrTrunc(DAG.getBitcast(MVT::i16, Ins), DL,  in combineCompareEqual()
47292                           CMP01, DAG.getTargetConstant(x86cc, DL, MVT::i8));  in combineCompareEqual()
47294           bool is64BitFP = (CMP00.getValueType() == MVT::f64);  in combineCompareEqual()
47295           MVT IntVT = is64BitFP ? MVT::i64 : MVT::i32;  in combineCompareEqual()
47303             SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64,  in combineCompareEqual()
47305             SDValue Vector32 = DAG.getBitcast(MVT::v4f32, Vector64);  in combineCompareEqual()
47306             OnesOrZeroesF = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32,  in combineCompareEqual()
47308             IntVT = MVT::i32;  in combineCompareEqual()
47314           SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,  in combineCompareEqual()
47328   MVT VT = N->getSimpleValueType(0);  in combineAndNotIntoANDNP()
47493   if (N00Type != N10Type || !((Subtarget.hasSSE1() && N00Type == MVT::f32) ||  in convertIntLogicToFPLogic()
47494                               (Subtarget.hasSSE2() && N00Type == MVT::f64) ||  in convertIntLogicToFPLogic()
47495                               (Subtarget.hasFP16() && N00Type == MVT::f16)))  in convertIntLogicToFPLogic()
47504   if (VT != MVT::i1 || N0.getOpcode() != ISD::SETCC || !N0.hasOneUse() ||  in convertIntLogicToFPLogic()
47523   EVT BoolVecVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElts);  in convertIntLogicToFPLogic()
47568   return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);  in combineBitOpWithMOVMSK()
47673   SDValue ShAmt = DAG.getTargetConstant(EltBitWidth - ShiftVal, DL, MVT::i8);  in combineAndMaskToShift()
47698 static bool hasBZHI(const X86Subtarget &Subtarget, MVT VT) {  in hasBZHI()
47721   MVT VT = Node->getSimpleValueType(0);  in combineAndLoadToBZHI()
47774           SDValue SizeC = DAG.getConstant(VT.getSizeInBits(), dl, MVT::i32);  in combineAndLoadToBZHI()
47780           Index = DAG.getZExtOrTrunc(Index, dl, MVT::i32);  in combineAndLoadToBZHI()
47782           SDValue Sub = DAG.getNode(ISD::SUB, dl, MVT::i32, SizeC, Index);  in combineAndLoadToBZHI()
47783           Sub = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Sub);  in combineAndLoadToBZHI()
47823   if (!SrcVT.isVector() || SrcVT.getVectorElementType() != MVT::i1 ||  in combineScalarAndWithMaskSetcc()
47870   if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) {  in combineAnd()
47871     return DAG.getBitcast(MVT::v4i32,  in combineAnd()
47872                           DAG.getNode(X86ISD::FAND, dl, MVT::v4f32,  in combineAnd()
47873                                       DAG.getBitcast(MVT::v4f32, N0),  in combineAnd()
47874                                       DAG.getBitcast(MVT::v4f32, N1)));  in combineAnd()
47878   if (VT == MVT::i64 && Subtarget.is64Bit() && !isa<ConstantSDNode>(N1)) {  in combineAnd()
47882       SDValue LHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N0);  in combineAnd()
47883       SDValue RHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N1);  in combineAnd()
47884       return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64,  in combineAnd()
47885                          DAG.getNode(ISD::AND, dl, MVT::i32, LHS, RHS));  in combineAnd()
47891   if (VT == MVT::i1) {  in combineAnd()
47906         return DAG.getSetCC(dl, MVT::i1, Mask, PartialBits, ISD::SETEQ);  in combineAnd()
48086   MVT VT = N->getSimpleValueType(0);  in canonicalizeBitSelect()
48125     MVT OpSVT = EltSizeInBits == 32 ? MVT::i32 : MVT::i64;  in canonicalizeBitSelect()
48126     MVT OpVT =  in canonicalizeBitSelect()
48127         MVT::getVectorVT(OpSVT, VT.getSizeInBits() / OpSVT.getSizeInBits());  in canonicalizeBitSelect()
48131     SDValue Imm = DAG.getTargetConstant(0xCA, DL, MVT::i8);  in canonicalizeBitSelect()
48224   MVT BlendVT = VT.is256BitVector() ? MVT::v32i8 : MVT::v16i8;  in combineLogicBlendIntoPBLENDV()
48247   SDValue Trunc = DAG.getZExtOrTrunc(Clz, dl, MVT::i32);  in lowerX86CmpEqZeroToCtlzSrl()
48248   SDValue Scc = DAG.getNode(ISD::SRL, dl, MVT::i32, Trunc,  in lowerX86CmpEqZeroToCtlzSrl()
48249                             DAG.getConstant(Log2b, dl, MVT::i8));  in lowerX86CmpEqZeroToCtlzSrl()
48273   if (!N->hasOneUse() || !N->getSimpleValueType(0).bitsGE(MVT::i32) ||  in combineOrCmpEqZeroToCtlzSrl()
48283            N->getOperand(1).getValueType().bitsGE(MVT::i32);  in combineOrCmpEqZeroToCtlzSrl()
48313     Ret = DAG.getNode(ISD::OR, SDLoc(OR), MVT::i32, NewLHS, NewRHS);  in combineOrCmpEqZeroToCtlzSrl()
48329     Ret = DAG.getNode(ISD::OR, SDLoc(OR), MVT::i32, Ret, NewRHS);  in combineOrCmpEqZeroToCtlzSrl()
48398   if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) {  in combineOr()
48399     return DAG.getBitcast(MVT::v4i32,  in combineOr()
48400                           DAG.getNode(X86ISD::FOR, dl, MVT::v4f32,  in combineOr()
48401                                       DAG.getBitcast(MVT::v4f32, N0),  in combineOr()
48402                                       DAG.getBitcast(MVT::v4f32, N1)));  in combineOr()
48407   if (VT == MVT::i1) {  in combineOr()
48423         return DAG.getSetCC(dl, MVT::i1, Mask, ZeroBits, ISD::SETNE);  in combineOr()
48506   if (!Subtarget.hasBMI() && VT.isScalarInteger() && VT != MVT::i1)  in combineOr()
48520   if (ResultType != MVT::i8 && ResultType != MVT::i1)  in foldXorTruncShiftIntoCmp()
48541   if (ShiftTy != MVT::i16 && ShiftTy != MVT::i32 && ShiftTy != MVT::i64)  in foldXorTruncShiftIntoCmp()
48580   case MVT::v16i8:  in foldVectorXorShiftIntoCmp()
48581   case MVT::v8i16:  in foldVectorXorShiftIntoCmp()
48582   case MVT::v4i32:  in foldVectorXorShiftIntoCmp()
48583   case MVT::v2i64: if (!Subtarget.hasSSE2()) return SDValue(); break;  in foldVectorXorShiftIntoCmp()
48584   case MVT::v32i8:  in foldVectorXorShiftIntoCmp()
48585   case MVT::v16i16:  in foldVectorXorShiftIntoCmp()
48586   case MVT::v8i32:  in foldVectorXorShiftIntoCmp()
48587   case MVT::v4i64: if (!Subtarget.hasAVX2()) return SDValue(); break;  in foldVectorXorShiftIntoCmp()
48722       InVT == MVT::v16i32 && VT == MVT::v16i8) {  in combineTruncateWithSat()
48725       SDValue Mid = truncateVectorWithPACK(X86ISD::PACKUS, MVT::v16i16, USatVal,  in combineTruncateWithSat()
48738   bool PreferAVX512 = ((Subtarget.hasAVX512() && InSVT == MVT::i32) ||  in combineTruncateWithSat()
48739                        (Subtarget.hasBWI() && InSVT == MVT::i16)) &&  in combineTruncateWithSat()
48746       (SVT == MVT::i8 || SVT == MVT::i16) &&  in combineTruncateWithSat()
48747       (InSVT == MVT::i16 || InSVT == MVT::i32)) {  in combineTruncateWithSat()
48752       if (SVT == MVT::i8 && InSVT == MVT::i32) {  in combineTruncateWithSat()
48753         EVT MidVT = VT.changeVectorElementType(MVT::i16);  in combineTruncateWithSat()
48761       } else if (SVT == MVT::i8 || Subtarget.hasSSE41())  in combineTruncateWithSat()
48771   if (TLI.isTypeLegal(InVT) && InVT.isVector() && SVT != MVT::i1 &&  in combineTruncateWithSat()
48772       Subtarget.hasAVX512() && (InSVT != MVT::i16 || Subtarget.hasBWI()) &&  in combineTruncateWithSat()
48773       (SVT == MVT::i32 || SVT == MVT::i16 || SVT == MVT::i8)) {  in combineTruncateWithSat()
48821   if (!((ScalarVT == MVT::i8 || ScalarVT == MVT::i16) && NumElems >= 2))  in detectAVGPattern()
48904   if (IsConstVectorInRange(Operands[1], 1, ScalarVT == MVT::i8 ? 256 : 65536) &&  in detectAVGPattern()
48999     SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,  in combineLoad()
49009       RegVT.getScalarType() == MVT::i1 && DCI.isBeforeLegalize()) {  in combineLoad()
49049     MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());  in combineLoad()
49074   if (!BV || BV->getValueType(0).getVectorElementType() != MVT::i1)  in getOneTrueElt()
49151   if (EltVT == MVT::i64 && !Subtarget.is64Bit()) {  in reduceMaskedLoadToScalarLoad()
49152     EltVT = MVT::f64;  in reduceMaskedLoadToScalarLoad()
49284   if (EltVT == MVT::i64 && !Subtarget.is64Bit()) {  in reduceMaskedStoreToScalarStore()
49285     EltVT = MVT::f64;  in reduceMaskedStoreToScalarStore()
49358       VT.getVectorElementType() == MVT::i1) {  in combineStore()
49370   if (VT == MVT::v1i1 && VT == StVT && Subtarget.hasAVX512() &&  in combineStore()
49372       StoredVal.getOperand(0).getValueType() == MVT::i8) {  in combineStore()
49375     Val = DAG.getZeroExtendInReg(Val, dl, MVT::i1);  in combineStore()
49383   if ((VT == MVT::v1i1 || VT == MVT::v2i1 || VT == MVT::v4i1) && VT == StVT &&  in combineStore()
49389     StoredVal = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops);  in combineStore()
49396   if ((VT == MVT::v8i1 || VT == MVT::v16i1 || VT == MVT::v32i1 ||  in combineStore()
49397        VT == MVT::v64i1) && VT == StVT && TLI.isTypeLegal(VT) &&  in combineStore()
49400     if (!DCI.isBeforeLegalize() && VT == MVT::v64i1 && !Subtarget.is64Bit()) {  in combineStore()
49401       SDValue Lo = DAG.getBuildVector(MVT::v32i1, dl,  in combineStore()
49404       SDValue Hi = DAG.getBuildVector(MVT::v32i1, dl,  in combineStore()
49420       return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ch0, Ch1);  in combineStore()
49458       MVT NTVT = Subtarget.hasSSE4A()  in combineStore()
49459                      ? MVT::v2f64  in combineStore()
49460                      : (TLI.isTypeLegal(MVT::i64) ? MVT::v2i64 : MVT::v4i32);  in combineStore()
49467   if (!St->isTruncatingStore() && VT == MVT::v16i8 && !Subtarget.hasBWI() &&  in combineStore()
49469       St->getValue().getOperand(0).getValueType() == MVT::v16i16 &&  in combineStore()
49470       TLI.isTruncStoreLegal(MVT::v16i32, MVT::v16i8) &&  in combineStore()
49472     SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::v16i32,  in combineStore()
49475                              MVT::v16i8, St->getMemOperand());  in combineStore()
49506         MVT DstVT = Trunc.getSimpleValueType();  in combineStore()
49507         MVT SrcVT = Src.getSimpleValueType();  in combineStore()
49510         MVT TruncVT = MVT::getVectorVT(DstVT.getScalarType(), NumSrcElts);  in combineStore()
49553     MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());  in combineStore()
49576   if ((VT == MVT::i64 && F64IsLegal && !Subtarget.is64Bit()) &&  in combineStore()
49592     SDValue NewLd = DAG.getLoad(MVT::f64, LdDL, Ld->getChain(),  in combineStore()
49607   if (VT == MVT::i64 && F64IsLegal && !Subtarget.is64Bit() &&  in combineStore()
49612     EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64, VecSize / 64);  in combineStore()
49614     SDValue NewExtract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,  in combineStore()
49630   MVT VT = StoredVal.getSimpleValueType();  in combineVEXTRACT_STORE()
49677   MVT VT = LHS.getSimpleValueType();  in isHorizontalBinOp()
49858     if ((Subtarget.hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||  in combineToHorizontalAddSub()
49859         (Subtarget.hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) {  in combineToHorizontalAddSub()
49875     if (Subtarget.hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32 ||  in combineToHorizontalAddSub()
49876                                  VT == MVT::v16i16 || VT == MVT::v8i32)) {  in combineToHorizontalAddSub()
49987     assert(N->getSimpleValueType(0).getScalarType() == MVT::f32 &&  in combineFaddCFmul()
50005   if (VT != MVT::v8f16 && VT != MVT::v16f16 && VT != MVT::v32f16)  in combineFaddCFmul()
50046   MVT CVT = MVT::getVectorVT(MVT::f32, VT.getVectorNumElements() / 2);  in combineFaddCFmul()
50125     if (SrcVT.getScalarType() == MVT::i64 &&  in combineTruncatedArithmetic()
50200   if (!((InSVT == MVT::i16 || InSVT == MVT::i32 || InSVT == MVT::i64) &&  in combineVectorTruncation()
50201         (OutSVT == MVT::i8 || OutSVT == MVT::i16) && isPowerOf2_32(NumElems) &&  in combineVectorTruncation()
50207     if (InSVT == MVT::i16)  in combineVectorTruncation()
50209     if (InSVT == MVT::i32 &&  in combineVectorTruncation()
50210         (OutSVT == MVT::i8 || !Subtarget.hasSSE41() || Subtarget.hasInt256()))  in combineVectorTruncation()
50218   if (Subtarget.hasSSE41() || OutSVT == MVT::i8)  in combineVectorTruncation()
50220   if (InSVT == MVT::i32)  in combineVectorTruncation()
50243   MVT VT = N->getValueType(0).getSimpleVT();  in combineVectorSignBitsTruncation()
50244   MVT SVT = VT.getScalarType();  in combineVectorSignBitsTruncation()
50246   MVT InVT = In.getValueType().getSimpleVT();  in combineVectorSignBitsTruncation()
50247   MVT InSVT = InVT.getScalarType();  in combineVectorSignBitsTruncation()
50252   if (SVT != MVT::i8 && SVT != MVT::i16 && SVT != MVT::i32)  in combineVectorSignBitsTruncation()
50254   if (InSVT != MVT::i16 && InSVT != MVT::i32 && InSVT != MVT::i64)  in combineVectorSignBitsTruncation()
50258   if (SVT == MVT::i32 && VT.getSizeInBits() < 128)  in combineVectorSignBitsTruncation()
50291   if (SVT == MVT::i32 && NumSignBits != InSVT.getSizeInBits())  in combineVectorSignBitsTruncation()
50333   if (!VT.isVector() || VT.getVectorElementType() != MVT::i16)  in combinePMULH()
50382     EVT BCVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,  in combinePMULH()
50413   if (ScalarVT != MVT::i16 || NumElems < 8 || !isPowerOf2_32(NumElems))  in detectPMADDUBSW()
50454   if (N00.getValueType().getVectorElementType() != MVT::i8 ||  in detectPMADDUBSW()
50455       N01.getValueType().getVectorElementType() != MVT::i8 ||  in detectPMADDUBSW()
50456       N10.getValueType().getVectorElementType() != MVT::i8 ||  in detectPMADDUBSW()
50457       N11.getValueType().getVectorElementType() != MVT::i8)  in detectPMADDUBSW()
50525     assert(InVT.getScalarType() == MVT::i8 &&  in detectPMADDUBSW()
50528     EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,  in detectPMADDUBSW()
50564   if (Src.getOpcode() == ISD::BITCAST && VT == MVT::i32) {  in combineTruncate()
50566     if (BCSrc.getValueType() == MVT::x86mmx)  in combineTruncate()
50567       return DAG.getNode(X86ISD::MMX_MOVD2W, DL, MVT::i32, BCSrc);  in combineTruncate()
50765   if (Arg.getOpcode() == ISD::FMUL && (SVT == MVT::f32 || SVT == MVT::f64) &&  in combineFneg()
50807         !(SVT == MVT::f32 || SVT == MVT::f64) ||  in getNegatedExpression()
50851   MVT VT = N->getSimpleValueType(0);  in lowerX86FPLogicOp()
50859   MVT IntSVT = MVT::getIntegerVT(IntBits);  in lowerX86FPLogicOp()
50860   MVT IntVT = MVT::getVectorVT(IntSVT, VT.getSizeInBits() / IntBits);  in lowerX86FPLogicOp()
50900   if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) {  in combineXor()
50901     return DAG.getBitcast(MVT::v4i32,  in combineXor()
50902                           DAG.getNode(X86ISD::FXOR, SDLoc(N), MVT::v4f32,  in combineXor()
50903                                       DAG.getBitcast(MVT::v4f32, N0),  in combineXor()
50904                                       DAG.getBitcast(MVT::v4f32, N1)));  in combineXor()
50932       N0.getOperand(0).getValueType().getVectorElementType() == MVT::i1 &&  in combineXor()
50941       VT.getVectorElementType() == MVT::i1 &&  in combineXor()
51016   if (!((VT == MVT::f32 && Subtarget.hasSSE1()) ||  in combineFAndFNotToFAndn()
51017         (VT == MVT::f64 && Subtarget.hasSSE2()) ||  in combineFAndFNotToFAndn()
51018         (VT == MVT::v4f32 && Subtarget.hasSSE1() && !Subtarget.hasSSE2())))  in combineFAndFNotToFAndn()
51120   if (!((Subtarget.hasSSE1() && VT == MVT::f32) ||  in combineFMinNumFMaxNum()
51121         (Subtarget.hasSSE2() && VT == MVT::f64) ||  in combineFMinNumFMaxNum()
51122         (Subtarget.hasFP16() && VT == MVT::f16) ||  in combineFMinNumFMaxNum()
51189   MVT InVT = In.getSimpleValueType();  in combineX86INT_TO_FP()
51195     MVT MemVT = MVT::getIntegerVT(NumBits);  in combineX86INT_TO_FP()
51196     MVT LoadVT = MVT::getVectorVT(MemVT, 128 / NumBits);  in combineX86INT_TO_FP()
51218   MVT InVT = In.getSimpleValueType();  in combineCVTP2I_CVTTP2I()
51224     MVT MemVT = MVT::getFloatingPointVT(NumBits);  in combineCVTP2I_CVTTP2I()
51225     MVT LoadVT = MVT::getVectorVT(MemVT, 128 / NumBits);  in combineCVTP2I_CVTTP2I()
51230             DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other},  in combineCVTP2I_CVTTP2I()
51253   MVT VT = N->getSimpleValueType(0);  in combineAndnp()
51360   if (N->getValueType(0) == MVT::v4f32 && Src.getValueType() == MVT::v8i16) {  in combineCVTPH2PS()
51372       if (SDValue VZLoad = narrowLoadToVZLoad(LN, MVT::i64, MVT::v2i64, DAG)) {  in combineCVTPH2PS()
51376               N->getOpcode(), dl, {MVT::v4f32, MVT::Other},  in combineCVTPH2PS()
51377               {N->getOperand(0), DAG.getBitcast(MVT::v8i16, VZLoad)});  in combineCVTPH2PS()
51380           SDValue Convert = DAG.getNode(N->getOpcode(), dl, MVT::v4f32,  in combineCVTPH2PS()
51381                                         DAG.getBitcast(MVT::v8i16, VZLoad));  in combineCVTPH2PS()
51405   if (ExtraVT != MVT::i8 && ExtraVT != MVT::i16)  in combineSextInRegCmov()
51442   if (DstVT == MVT::i16) {  in combineSextInRegCmov()
51443     CMovVT = MVT::i32;  in combineSextInRegCmov()
51475   if (VT == MVT::v4i64 && (N0.getOpcode() == ISD::ANY_EXTEND ||  in combineSignExtendInReg()
51490     if (N00.getValueType() == MVT::v4i32 && ExtraVT.getSizeInBits() < 128) {  in combineSignExtendInReg()
51492           DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32, N00, N1);  in combineSignExtendInReg()
51493       return DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i64, Tmp);  in combineSignExtendInReg()
51512   if (VT != MVT::i64)  in promoteExtBeforeAdd()
51594   if (TargetVT != MVT::i32 && TargetVT != MVT::i64)  in combineToExtendCMOV()
51599   if (VT != MVT::i16 && !(ExtendOpcode == ISD::SIGN_EXTEND && VT == MVT::i32))  in combineToExtendCMOV()
51605   if (TargetVT == MVT::i64 && ExtendOpcode != ISD::SIGN_EXTEND)  in combineToExtendCMOV()
51606     ExtendVT = MVT::i32;  in combineToExtendCMOV()
51635   if (SVT != MVT::i8 && SVT != MVT::i16 && SVT != MVT::i32 &&  in combineExtSetcc()
51636       SVT != MVT::i64 && SVT != MVT::f32 && SVT != MVT::f64)  in combineExtSetcc()
51640   if (N0.getOperand(0).getValueType().getVectorElementType() == MVT::f16)  in combineExtSetcc()
51744   if (((ScalarVT != MVT::f32 && ScalarVT != MVT::f64) ||  in combineFMA()
51746       !(ScalarVT == MVT::f16 && Subtarget.hasFP16()))  in combineFMA()
51789     return DAG.getNode(NewOpcode, dl, {VT, MVT::Other},  in combineFMA()
51979     EVT VecVT = MVT::v16i8;  in combineVectorSizedSetCCEquality()
51980     EVT CmpVT = PreferKOT ? MVT::v16i1 : VecVT;  in combineVectorSizedSetCCEquality()
51982       VecVT = MVT::v32i8;  in combineVectorSizedSetCCEquality()
51983       CmpVT = PreferKOT ? MVT::v32i1 : VecVT;  in combineVectorSizedSetCCEquality()
51989         VecVT = MVT::v64i8;  in combineVectorSizedSetCCEquality()
51990         CmpVT = MVT::v64i1;  in combineVectorSizedSetCCEquality()
51994         VecVT = MVT::v16i32;  in combineVectorSizedSetCCEquality()
51995         CmpVT = MVT::v16i1;  in combineVectorSizedSetCCEquality()
51997                  OpSize == 256 ? MVT::v8i32 : MVT::v4i32;  in combineVectorSizedSetCCEquality()
52010             TmpCastVT = NeedsAVX512FCast ? MVT::v4i32 : MVT::v16i8;  in combineVectorSizedSetCCEquality()
52014             TmpCastVT = NeedsAVX512FCast ? MVT::v8i32 : MVT::v32i8;  in combineVectorSizedSetCCEquality()
52048       EVT KRegVT = CmpVT == MVT::v64i1 ? MVT::i64 :  in combineVectorSizedSetCCEquality()
52049                    CmpVT == MVT::v32i1 ? MVT::i32 : MVT::i16;  in combineVectorSizedSetCCEquality()
52054       SDValue BCCmp = DAG.getBitcast(OpSize == 256 ? MVT::v4i64 : MVT::v2i64,  in combineVectorSizedSetCCEquality()
52056       SDValue PT = DAG.getNode(X86ISD::PTEST, DL, MVT::i32, BCCmp, BCCmp);  in combineVectorSizedSetCCEquality()
52064     assert(Cmp.getValueType() == MVT::v16i8 &&  in combineVectorSizedSetCCEquality()
52066     SDValue MovMsk = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Cmp);  in combineVectorSizedSetCCEquality()
52067     SDValue FFFFs = DAG.getConstant(0xFFFF, DL, MVT::i32);  in combineVectorSizedSetCCEquality()
52088     if (VT == MVT::i1 && isNullConstant(RHS)) {  in combineSetCC()
52093                            DAG.getNode(X86ISD::SETCC, DL, MVT::i8, X86CC, V));  in combineSetCC()
52152   if (VT.isVector() && VT.getVectorElementType() == MVT::i1 &&  in combineSetCC()
52167         (Op0.getOperand(0).getValueType().getVectorElementType() == MVT::i1);  in combineSetCC()
52192       VT.getVectorElementType() == MVT::i1 &&  in combineSetCC()
52193       (OpVT.getVectorElementType() == MVT::i8 ||  in combineSetCC()
52194        OpVT.getVectorElementType() == MVT::i16)) {  in combineSetCC()
52201   if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32 &&  in combineSetCC()
52202       LHS.getValueType() == MVT::v4f32)  in combineSetCC()
52222   MVT SrcVT = Src.getSimpleValueType();  in combineMOVMSK()
52223   MVT VT = N->getSimpleValueType(0);  in combineMOVMSK()
52227   assert(VT == MVT::i32 && NumElts <= NumBits && "Unexpected MOVMSK types");  in combineMOVMSK()
52279       MVT ShiftVT = SrcVT;  in combineMOVMSK()
52280       if (ShiftVT.getScalarType() == MVT::i8) {  in combineMOVMSK()
52282         ShiftVT = MVT::getVectorVT(MVT::i16, NumElts / 2);  in combineMOVMSK()
52399         EVT NewVT = Index.getValueType().changeVectorElementType(MVT::i32);  in combineGatherScatter()
52413       EVT NewVT = Index.getValueType().changeVectorElementType(MVT::i32);  in combineGatherScatter()
52466       MVT EltVT = IndexWidth > 32 ? MVT::i64 : MVT::i32;  in combineGatherScatter()
52513     SDValue Cond = DAG.getTargetConstant(CC, DL, MVT::i8);  in combineBrCond()
52560       SourceConst = DAG.getNode(N->getOpcode(), DL, {VT, MVT::Other},  in combineVectorCompareAndMaskUnaryOp()
52622   if (InVT.isVector() && VT.getVectorElementType() == MVT::f16) {  in combineUIntToFP()
52628                                  ScalarSize < 16   ? MVT::i16  in combineUIntToFP()
52629                                  : ScalarSize < 32 ? MVT::i32  in combineUIntToFP()
52630                                                    : MVT::i64,  in combineUIntToFP()
52634       return DAG.getNode(ISD::STRICT_UINT_TO_FP, dl, {VT, MVT::Other},  in combineUIntToFP()
52643       VT.getScalarType() != MVT::f16) {  in combineUIntToFP()
52645     EVT DstVT = InVT.changeVectorElementType(MVT::i32);  in combineUIntToFP()
52650       return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},  in combineUIntToFP()
52660       return DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(N), {VT, MVT::Other},  in combineUIntToFP()
52685   if (InVT.isVector() && VT.getVectorElementType() == MVT::f16) {  in combineSIntToFP()
52691                                  ScalarSize < 16   ? MVT::i16  in combineSIntToFP()
52692                                  : ScalarSize < 32 ? MVT::i32  in combineSIntToFP()
52693                                                    : MVT::i64,  in combineSIntToFP()
52697       return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},  in combineSIntToFP()
52706       VT.getScalarType() != MVT::f16) {  in combineSIntToFP()
52708     EVT DstVT = InVT.changeVectorElementType(MVT::i32);  in combineSIntToFP()
52711       return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},  in combineSIntToFP()
52723       EVT TruncVT = MVT::i32;  in combineSIntToFP()
52727       if (DCI.isBeforeLegalize() || TruncVT != MVT::v2i32) {  in combineSIntToFP()
52730           return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},  in combineSIntToFP()
52736       assert(InVT == MVT::v2i64 && "Unexpected VT!");  in combineSIntToFP()
52737       SDValue Cast = DAG.getBitcast(MVT::v4i32, Op0);  in combineSIntToFP()
52738       SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Cast, Cast,  in combineSIntToFP()
52741         return DAG.getNode(X86ISD::STRICT_CVTSI2P, dl, {VT, MVT::Other},  in combineSIntToFP()
52754     if (VT == MVT::f16 || VT == MVT::f128)  in combineSIntToFP()
52759     if (Subtarget.hasDQI() && VT != MVT::f80)  in combineSIntToFP()
52763         Op0.hasOneUse() && !Subtarget.is64Bit() && InVT == MVT::i64) {  in combineSIntToFP()
52783   assert(Flags.getValueType() == MVT::i32 && "Unexpected VT!");  in needCarryOrOverflowFlag()
52816   assert(Flags.getValueType() == MVT::i32 && "Unexpected VT!");  in onlyZeroFlagUsed()
52880                          DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),  in combineAddOrSubToADCOrSBB()
52897                            DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),  in combineAddOrSubToADCOrSBB()
52907                        DAG.getVTList(VT, MVT::i32), X,  in combineAddOrSubToADCOrSBB()
52929                          DAG.getVTList(VT, MVT::i32), X,  in combineAddOrSubToADCOrSBB()
52938                        DAG.getVTList(VT, MVT::i32), X,  in combineAddOrSubToADCOrSBB()
52959                          DAG.getVTList(VT, MVT::i32), X,  in combineAddOrSubToADCOrSBB()
52985       SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32);  in combineAddOrSubToADCOrSBB()
52988                          DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),  in combineAddOrSubToADCOrSBB()
52999       SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32);  in combineAddOrSubToADCOrSBB()
53002                          DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),  in combineAddOrSubToADCOrSBB()
53009   SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32);  in combineAddOrSubToADCOrSBB()
53013   SDVTList VTs = DAG.getVTList(VT, MVT::i32);  in combineAddOrSubToADCOrSBB()
53080         return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,  in combineCMP()
53092       return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Src,  in combineCMP()
53109   if (OpVT == MVT::i32 && DAG.MaskedValueIsZero(Op, UpperBits) &&  in combineCMP()
53111     return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,  in combineCMP()
53150   SDVTList VTs = DAG.getVTList(VT, MVT::i32);  in combineCMP()
53155     return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,  in combineCMP()
53170   MVT VT = LHS.getSimpleValueType();  in combineX86AddSub()
53177     return DAG.getMergeValues({Res, DAG.getConstant(0, DL, MVT::i32)}, DL);  in combineX86AddSub()
53206     MVT VT = N->getSimpleValueType(0);  in combineSBB()
53207     SDVTList VTs = DAG.getVTList(VT, MVT::i32);  in combineSBB()
53248                     DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), CarryIn),  in combineADC()
53265     MVT VT = N->getSimpleValueType(0);  in combineADC()
53266     SDVTList VTs = DAG.getVTList(VT, MVT::i32);  in combineADC()
53301   if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 ||  in matchPMADDWD()
53369   EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,  in matchPMADDWD()
53378     EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,  in matchPMADDWD()
53397   if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 ||  in matchPMADDWD_2()
53423   if (InVT.getVectorElementType() != MVT::i16 || N01.getValueType() != InVT ||  in matchPMADDWD_2()
53500     assert(OpVT.getScalarType() == MVT::i16 &&  in matchPMADDWD_2()
53503     EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,  in matchPMADDWD_2()
53510   EVT OutVT16 = EVT::getVectorVT(*DAG.getContext(), MVT::i16,  in matchPMADDWD_2()
53538   MVT OpVT = N0.getOperand(0).getSimpleValueType();  in combineAddOfPMADDWD()
53667         Op0.getOperand(0).getValueType().getVectorElementType() == MVT::i1 &&  in combineAdd()
53674         Op1.getOperand(0).getValueType().getVectorElementType() == MVT::i1 &&  in combineAdd()
53726   MVT VT = N->getSimpleValueType(0);  in combineSubABS()
53795   MVT VT = N->getSimpleValueType(0);  in combineVectorCompare()
53811 static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,  in combineConcatVectorOps()
53856     if (Op0.getOpcode() == X86ISD::MOVDDUP && VT == MVT::v4f64 &&  in combineConcatVectorOps()
53861                          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f64,  in combineConcatVectorOps()
53901                            DAG.getTargetConstant(0x31, DL, MVT::i8));  in combineConcatVectorOps()
53912     auto ConcatSubOperand = [&](MVT VT, ArrayRef<SDValue> SubOps, unsigned I) {  in combineConcatVectorOps()
53918     auto IsConcatFree = [](MVT VT, ArrayRef<SDValue> SubOps, unsigned Op) {  in combineConcatVectorOps()
53933       if (!IsSplat && VT == MVT::v4f64 && llvm::all_of(Ops, [](SDValue Op) {  in combineConcatVectorOps()
53950       if (!IsSplat && VT.getScalarType() == MVT::f32 &&  in combineConcatVectorOps()
53970       if (!IsSplat && NumOps == 2 && (VT == MVT::v8f32 || VT == MVT::v8i32) &&  in combineConcatVectorOps()
53972         SDValue Res = DAG.getBitcast(MVT::v8f32, ConcatSubOperand(VT, Ops, 0));  in combineConcatVectorOps()
53973         Res = DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, Res,  in combineConcatVectorOps()
53977       if (!IsSplat && NumOps == 2 && VT == MVT::v4f64) {  in combineConcatVectorOps()
53983                            DAG.getTargetConstant(Idx, DL, MVT::i8));  in combineConcatVectorOps()
53996         MVT OpVT = Op0.getSimpleValueType();  in combineConcatVectorOps()
54018           MVT IntMaskSVT = MVT::getIntegerVT(VT.getScalarSizeInBits());  in combineConcatVectorOps()
54019           MVT IntMaskVT = MVT::getVectorVT(IntMaskSVT, NumOps * NumSrcElts);  in combineConcatVectorOps()
54029       if (VT == MVT::v4i64 && !Subtarget.hasInt256() &&  in combineConcatVectorOps()
54033         SDValue Res = DAG.getBitcast(MVT::v8i32, ConcatSubOperand(VT, Ops, 0));  in combineConcatVectorOps()
54034         SDValue Zero = getZeroVector(MVT::v8i32, Subtarget, DAG, DL);  in combineConcatVectorOps()
54036           Res = DAG.getVectorShuffle(MVT::v8i32, DL, Res, Zero,  in combineConcatVectorOps()
54039           Res = DAG.getVectorShuffle(MVT::v8i32, DL, Res, Zero,  in combineConcatVectorOps()
54076         MVT SrcVT = Op0.getOperand(0).getSimpleValueType();  in combineConcatVectorOps()
54077         SrcVT = MVT::getVectorVT(SrcVT.getScalarType(),  in combineConcatVectorOps()
54092         MVT SrcVT = Op0.getOperand(0).getSimpleValueType();  in combineConcatVectorOps()
54093         SrcVT = MVT::getVectorVT(SrcVT.getScalarType(),  in combineConcatVectorOps()
54170   if (VT.getVectorElementType() == MVT::i1)  in combineCONCAT_VECTORS()
54189   MVT OpVT = N->getSimpleValueType(0);  in combineINSERT_SUBVECTOR()
54191   bool IsI1Vector = OpVT.getVectorElementType() == MVT::i1;  in combineINSERT_SUBVECTOR()
54198   MVT SubVecVT = SubVec.getSimpleValueType();  in combineINSERT_SUBVECTOR()
54292     SDVTList Tys = DAG.getVTList(OpVT, MVT::Other);  in combineINSERT_SUBVECTOR()
54334   MVT VT = Ext->getSimpleValueType(0);  in narrowExtractedVectorSelect()
54338   MVT SelCondVT = Sel.getOperand(0).getSimpleValueType();  in narrowExtractedVectorSelect()
54342   MVT WideVT = Ext->getOperand(0).getSimpleValueType();  in narrowExtractedVectorSelect()
54343   MVT SelVT = Sel.getSimpleValueType();  in narrowExtractedVectorSelect()
54367   MVT NarrowSelVT = MVT::getVectorVT(SelVT.getVectorElementType(), NarrowElts);  in narrowExtractedVectorSelect()
54392   MVT VT = N->getSimpleValueType(0);  in combineEXTRACT_SUBVECTOR()
54431     if (VT.getScalarType() == MVT::i1)  in combineEXTRACT_SUBVECTOR()
54444   if (VT.getVectorElementType() != MVT::i1 &&  in combineEXTRACT_SUBVECTOR()
54498     if (IdxVal == 0 && VT == MVT::v2f64 && InVecVT == MVT::v4f64) {  in combineEXTRACT_SUBVECTOR()
54501           InVec.getOperand(0).getValueType() == MVT::v4i32) {  in combineEXTRACT_SUBVECTOR()
54506           InVec.getOperand(0).getValueType() == MVT::v4i32) {  in combineEXTRACT_SUBVECTOR()
54511           InVec.getOperand(0).getValueType() == MVT::v4f32) {  in combineEXTRACT_SUBVECTOR()
54581   if (VT == MVT::v1i1 && Src.getOpcode() == ISD::AND && Src.hasOneUse())  in combineScalarToVector()
54584         return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1,  in combineScalarToVector()
54588   if (VT == MVT::v1i1 && Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&  in combineScalarToVector()
54590       Src.getOperand(0).getValueType().getVectorElementType() == MVT::i1)  in combineScalarToVector()
54598   if (VT == MVT::v2i64 || VT == MVT::v2f64) {  in combineScalarToVector()
54600       if (Op.getValueType() != MVT::i64 || !Op.hasOneUse())  in combineScalarToVector()
54613           VT, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32,  in combineScalarToVector()
54614                           DAG.getAnyExtOrTrunc(ExtSrc, DL, MVT::i32)));  in combineScalarToVector()
54618   if (VT == MVT::v2i64 && Src.getOpcode() == ISD::BITCAST &&  in combineScalarToVector()
54619       Src.getOperand(0).getValueType() == MVT::x86mmx)  in combineScalarToVector()
54671   if (N->getValueType(0) == MVT::v2i64 && LHS.hasOneUse() &&  in combinePMULDQ()
54674       LHS.getOperand(0).getValueType() == MVT::v4i32) {  in combinePMULDQ()
54676     LHS = DAG.getVectorShuffle(MVT::v4i32, dl, LHS.getOperand(0),  in combinePMULDQ()
54678     LHS = DAG.getBitcast(MVT::v2i64, LHS);  in combinePMULDQ()
54679     return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS);  in combinePMULDQ()
54681   if (N->getValueType(0) == MVT::v2i64 && RHS.hasOneUse() &&  in combinePMULDQ()
54684       RHS.getOperand(0).getValueType() == MVT::v4i32) {  in combinePMULDQ()
54686     RHS = DAG.getVectorShuffle(MVT::v4i32, dl, RHS.getOperand(0),  in combinePMULDQ()
54688     RHS = DAG.getBitcast(MVT::v2i64, RHS);  in combinePMULDQ()
54689     return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS);  in combinePMULDQ()
54731       MVT SVT = In.getSimpleValueType().getVectorElementType();  in combineEXTEND_VECTOR_INREG()
54812   if (N->getValueType(0) != MVT::f32 ||  in combineFP16_TO_FP()
54813       N->getOperand(0).getOperand(0).getValueType() != MVT::f32)  in combineFP16_TO_FP()
54817   SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32,  in combineFP16_TO_FP()
54819   Res = DAG.getNode(X86ISD::CVTPS2PH, dl, MVT::v8i16, Res,  in combineFP16_TO_FP()
54820                     DAG.getTargetConstant(4, dl, MVT::i32));  in combineFP16_TO_FP()
54821   Res = DAG.getNode(X86ISD::CVTPH2PS, dl, MVT::v4f32, Res);  in combineFP16_TO_FP()
54822   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,  in combineFP16_TO_FP()
54839   if (!SrcVT.isVector() || SrcVT.getVectorElementType() != MVT::f16)  in combineFP_EXTEND()
54842   if (VT.getVectorElementType() != MVT::f32 &&  in combineFP_EXTEND()
54843       VT.getVectorElementType() != MVT::f64)  in combineFP_EXTEND()
54863     Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, Ops);  in combineFP_EXTEND()
54867   EVT CvtVT = EVT::getVectorVT(*DAG.getContext(), MVT::f32,  in combineFP_EXTEND()
54871     Cvt = DAG.getNode(X86ISD::STRICT_CVTPH2PS, dl, {CvtVT, MVT::Other},  in combineFP_EXTEND()
54880     Cvt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2f32, Cvt,  in combineFP_EXTEND()
54887       Cvt = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {VT, MVT::Other},  in combineFP_EXTEND()
54950   if (!VT.isVector() || VT.getVectorElementType() != MVT::f16 ||  in combineFP_ROUND()
54951       SrcVT.getVectorElementType() != MVT::f32)  in combineFP_ROUND()
54962     Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,  in combineFP_ROUND()
54966   EVT CvtVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,  in combineFP_ROUND()
54969   SDValue Rnd = DAG.getTargetConstant(4, dl, MVT::i32);  in combineFP_ROUND()
54971     Cvt = DAG.getNode(X86ISD::STRICT_CVTPS2PH, dl, {CvtVT, MVT::Other},  in combineFP_ROUND()
55001       SDValue NewLd = DAG.getLoad(MVT::x86mmx, SDLoc(N), LN->getChain(),  in combineMOVDQ2Q()
55214   if (Opc == ISD::SHL && VT.isVector() && VT.getVectorElementType() == MVT::i8)  in isTypeDesirableForOp()
55225   if ((Opc == ISD::MUL || Opc == ISD::SHL) && VT == MVT::i8)  in isTypeDesirableForOp()
55230   if (VT == MVT::i16) {  in isTypeDesirableForOp()
55265     return DAG.getNode(X86ISD::NT_BRIND, dl, MVT::Other, Value, Addr);  in expandIndirectJTBranch()
55273   bool Is8BitMulByConstant = VT == MVT::i8 && Op.getOpcode() == ISD::MUL &&  in IsDesirableToPromoteOp()
55280   if (VT != MVT::i16 && !Is8BitMulByConstant)  in IsDesirableToPromoteOp()
55348   PVT = MVT::i32;  in IsDesirableToPromoteOp()
55740     Flag = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32, Flag);  in LowerAsmOutputForConstraint()
55743     Flag = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32);  in LowerAsmOutputForConstraint()
55836         Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op), MVT::i64);  in LowerAsmOperandForConstraint()
55862       BooleanContent BCont = getBooleanContents(MVT::i64);  in LowerAsmOperandForConstraint()
55867       Result = DAG.getTargetConstant(ExtVal, SDLoc(Op), MVT::i64);  in LowerAsmOperandForConstraint()
55933                                                 MVT VT) const {  in getRegForInlineAsmConstraint()
55953         if (VT == MVT::i1)  in getRegForInlineAsmConstraint()
55955         if (VT == MVT::i8)  in getRegForInlineAsmConstraint()
55957         if (VT == MVT::i16)  in getRegForInlineAsmConstraint()
55961         if (VT == MVT::i32)  in getRegForInlineAsmConstraint()
55963         if (VT == MVT::i64)  in getRegForInlineAsmConstraint()
55969         if (VT == MVT::i8 || VT == MVT::i1)  in getRegForInlineAsmConstraint()
55971         if (VT == MVT::i16)  in getRegForInlineAsmConstraint()
55973         if (VT == MVT::i32 || VT == MVT::f32)  in getRegForInlineAsmConstraint()
55975         if (VT != MVT::f80 && !VT.isVector())  in getRegForInlineAsmConstraint()
55982       if (VT == MVT::i8 || VT == MVT::i1)  in getRegForInlineAsmConstraint()
55984       if (VT == MVT::i16)  in getRegForInlineAsmConstraint()
55986       if (VT == MVT::i32 || VT == MVT::f32 ||  in getRegForInlineAsmConstraint()
55989       if (VT != MVT::f80 && !VT.isVector())  in getRegForInlineAsmConstraint()
55994       if (VT == MVT::i8 || VT == MVT::i1)  in getRegForInlineAsmConstraint()
55996       if (VT == MVT::i16)  in getRegForInlineAsmConstraint()
55998       if (VT == MVT::i32 || VT == MVT::f32 ||  in getRegForInlineAsmConstraint()
56001       if (VT != MVT::f80 && !VT.isVector())  in getRegForInlineAsmConstraint()
56005       if (VT == MVT::i8 || VT == MVT::i1)  in getRegForInlineAsmConstraint()
56007       if (VT == MVT::i16)  in getRegForInlineAsmConstraint()
56009       if (VT == MVT::i32 || VT == MVT::f32 ||  in getRegForInlineAsmConstraint()
56012       if (VT != MVT::f80 && !VT.isVector())  in getRegForInlineAsmConstraint()
56018       if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT))  in getRegForInlineAsmConstraint()
56020       if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT))  in getRegForInlineAsmConstraint()
56022       if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f80)  in getRegForInlineAsmConstraint()
56036       case MVT::f16:  in getRegForInlineAsmConstraint()
56040       case MVT::f32:  in getRegForInlineAsmConstraint()
56041       case MVT::i32:  in getRegForInlineAsmConstraint()
56045       case MVT::f64:  in getRegForInlineAsmConstraint()
56046       case MVT::i64:  in getRegForInlineAsmConstraint()
56050       case MVT::i128:  in getRegForInlineAsmConstraint()
56058       case MVT::v8f16:  in getRegForInlineAsmConstraint()
56062       case MVT::f128:  in getRegForInlineAsmConstraint()
56063       case MVT::v16i8:  in getRegForInlineAsmConstraint()
56064       case MVT::v8i16:  in getRegForInlineAsmConstraint()
56065       case MVT::v4i32:  in getRegForInlineAsmConstraint()
56066       case MVT::v2i64:  in getRegForInlineAsmConstraint()
56067       case MVT::v4f32:  in getRegForInlineAsmConstraint()
56068       case MVT::v2f64:  in getRegForInlineAsmConstraint()
56073       case MVT::v16f16:  in getRegForInlineAsmConstraint()
56077       case MVT::v32i8:  in getRegForInlineAsmConstraint()
56078       case MVT::v16i16:  in getRegForInlineAsmConstraint()
56079       case MVT::v8i32:  in getRegForInlineAsmConstraint()
56080       case MVT::v4i64:  in getRegForInlineAsmConstraint()
56081       case MVT::v8f32:  in getRegForInlineAsmConstraint()
56082       case MVT::v4f64:  in getRegForInlineAsmConstraint()
56088       case MVT::v32f16:  in getRegForInlineAsmConstraint()
56092       case MVT::v64i8:  in getRegForInlineAsmConstraint()
56093       case MVT::v32i16:  in getRegForInlineAsmConstraint()
56094       case MVT::v8f64:  in getRegForInlineAsmConstraint()
56095       case MVT::v16f32:  in getRegForInlineAsmConstraint()
56096       case MVT::v16i32:  in getRegForInlineAsmConstraint()
56097       case MVT::v8i64:  in getRegForInlineAsmConstraint()
56121       case MVT::f16:  in getRegForInlineAsmConstraint()
56125       case MVT::f32:  in getRegForInlineAsmConstraint()
56126       case MVT::i32:  in getRegForInlineAsmConstraint()
56128       case MVT::f64:  in getRegForInlineAsmConstraint()
56129       case MVT::i64:  in getRegForInlineAsmConstraint()
56131       case MVT::v8f16:  in getRegForInlineAsmConstraint()
56135       case MVT::f128:  in getRegForInlineAsmConstraint()
56136       case MVT::v16i8:  in getRegForInlineAsmConstraint()
56137       case MVT::v8i16:  in getRegForInlineAsmConstraint()
56138       case MVT::v4i32:  in getRegForInlineAsmConstraint()
56139       case MVT::v2i64:  in getRegForInlineAsmConstraint()
56140       case MVT::v4f32:  in getRegForInlineAsmConstraint()
56141       case MVT::v2f64:  in getRegForInlineAsmConstraint()
56144       case MVT::v16f16:  in getRegForInlineAsmConstraint()
56148       case MVT::v32i8:  in getRegForInlineAsmConstraint()
56149       case MVT::v16i16:  in getRegForInlineAsmConstraint()
56150       case MVT::v8i32:  in getRegForInlineAsmConstraint()
56151       case MVT::v4i64:  in getRegForInlineAsmConstraint()
56152       case MVT::v8f32:  in getRegForInlineAsmConstraint()
56153       case MVT::v4f64:  in getRegForInlineAsmConstraint()
56157       case MVT::v32f16:  in getRegForInlineAsmConstraint()
56161       case MVT::v64i8:  in getRegForInlineAsmConstraint()
56162       case MVT::v32i16:  in getRegForInlineAsmConstraint()
56163       case MVT::v8f64:  in getRegForInlineAsmConstraint()
56164       case MVT::v16f32:  in getRegForInlineAsmConstraint()
56165       case MVT::v16i32:  in getRegForInlineAsmConstraint()
56166       case MVT::v8i64:  in getRegForInlineAsmConstraint()
56175         if (VT == MVT::i1)  in getRegForInlineAsmConstraint()
56177         if (VT == MVT::i8)  in getRegForInlineAsmConstraint()
56179         if (VT == MVT::i16)  in getRegForInlineAsmConstraint()
56183         if (VT == MVT::i32)  in getRegForInlineAsmConstraint()
56185         if (VT == MVT::i64)  in getRegForInlineAsmConstraint()
56204     if (VT == MVT::Other || VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f80) {  in getRegForInlineAsmConstraint()
56231         VT == MVT::Other)  in getRegForInlineAsmConstraint()
56260   if (TRI->isTypeLegalForClass(*Res.second, VT) || VT == MVT::Other)  in getRegForInlineAsmConstraint()
56318     if (VT == MVT::f16)  in getRegForInlineAsmConstraint()
56320     else if (VT == MVT::f32 || VT == MVT::i32)  in getRegForInlineAsmConstraint()
56322     else if (VT == MVT::f64 || VT == MVT::i64)  in getRegForInlineAsmConstraint()
56336     if (VT == MVT::i1)  in getRegForInlineAsmConstraint()
56338     else if (VT == MVT::i8)  in getRegForInlineAsmConstraint()
56340     else if (VT == MVT::i16)  in getRegForInlineAsmConstraint()
56342     else if (VT == MVT::i32)  in getRegForInlineAsmConstraint()
56344     else if (VT == MVT::i64)  in getRegForInlineAsmConstraint()