169   for (MVT VT : MVT::integer_valuetypes())  in X86TargetLowering()  local
170     setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);  in X86TargetLowering()
328   for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {  in X86TargetLowering()
329     setOperationAction(ISD::MULHS, VT, Expand);  in X86TargetLowering()
330     setOperationAction(ISD::MULHU, VT, Expand);  in X86TargetLowering()
331     setOperationAction(ISD::SDIV, VT, Expand);  in X86TargetLowering()
332     setOperationAction(ISD::UDIV, VT, Expand);  in X86TargetLowering()
333     setOperationAction(ISD::SREM, VT, Expand);  in X86TargetLowering()
334     setOperationAction(ISD::UREM, VT, Expand);  in X86TargetLowering()
339   for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128,  in X86TargetLowering()
341     setOperationAction(ISD::BR_CC,     VT, Expand);  in X86TargetLowering()
342     setOperationAction(ISD::SELECT_CC, VT, Expand);  in X86TargetLowering()
428   for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) {  in X86TargetLowering()
429     setOperationAction(ISD::SELECT, VT, Custom);  in X86TargetLowering()
430     setOperationAction(ISD::SETCC, VT, Custom);  in X86TargetLowering()
432   for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {  in X86TargetLowering()
433     if (VT == MVT::i64 && !Subtarget.is64Bit())  in X86TargetLowering()
435     setOperationAction(ISD::SELECT, VT, Custom);  in X86TargetLowering()
436     setOperationAction(ISD::SETCC,  VT, Custom);  in X86TargetLowering()
453   for (auto VT : { MVT::i32, MVT::i64 }) {  in X86TargetLowering()
454     if (VT == MVT::i64 && !Subtarget.is64Bit())  in X86TargetLowering()
456     setOperationAction(ISD::ConstantPool    , VT, Custom);  in X86TargetLowering()
457     setOperationAction(ISD::JumpTable       , VT, Custom);  in X86TargetLowering()
458     setOperationAction(ISD::GlobalAddress   , VT, Custom);  in X86TargetLowering()
459     setOperationAction(ISD::GlobalTLSAddress, VT, Custom);  in X86TargetLowering()
460     setOperationAction(ISD::ExternalSymbol  , VT, Custom);  in X86TargetLowering()
461     setOperationAction(ISD::BlockAddress    , VT, Custom);  in X86TargetLowering()
465   for (auto VT : { MVT::i32, MVT::i64 }) {  in X86TargetLowering()
466     if (VT == MVT::i64 && !Subtarget.is64Bit())  in X86TargetLowering()
468     setOperationAction(ISD::SHL_PARTS, VT, Custom);  in X86TargetLowering()
469     setOperationAction(ISD::SRA_PARTS, VT, Custom);  in X86TargetLowering()
470     setOperationAction(ISD::SRL_PARTS, VT, Custom);  in X86TargetLowering()
479   for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {  in X86TargetLowering()
480     setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Custom);  in X86TargetLowering()
481     setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);  in X86TargetLowering()
482     setOperationAction(ISD::ATOMIC_LOAD_ADD, VT, Custom);  in X86TargetLowering()
483     setOperationAction(ISD::ATOMIC_LOAD_OR, VT, Custom);  in X86TargetLowering()
484     setOperationAction(ISD::ATOMIC_LOAD_XOR, VT, Custom);  in X86TargetLowering()
485     setOperationAction(ISD::ATOMIC_LOAD_AND, VT, Custom);  in X86TargetLowering()
486     setOperationAction(ISD::ATOMIC_STORE, VT, Custom);  in X86TargetLowering()
533     for (auto VT : { MVT::f32, MVT::f64 }) {  in X86TargetLowering()
535       setOperationAction(ISD::FABS, VT, Custom);  in X86TargetLowering()
538       setOperationAction(ISD::FNEG, VT, Custom);  in X86TargetLowering()
541       setOperationAction(ISD::FCOPYSIGN, VT, Custom);  in X86TargetLowering()
544       setOperationAction(ISD::FADD, VT, Custom);  in X86TargetLowering()
545       setOperationAction(ISD::FSUB, VT, Custom);  in X86TargetLowering()
548       setOperationAction(ISD::FSIN   , VT, Expand);  in X86TargetLowering()
549       setOperationAction(ISD::FCOS   , VT, Expand);  in X86TargetLowering()
550       setOperationAction(ISD::FSINCOS, VT, Expand);  in X86TargetLowering()
595     for (auto VT : { MVT::f32, MVT::f64 }) {  in X86TargetLowering()
596       setOperationAction(ISD::UNDEF,     VT, Expand);  in X86TargetLowering()
597       setOperationAction(ISD::FCOPYSIGN, VT, Expand);  in X86TargetLowering()
600       setOperationAction(ISD::FSIN   , VT, Expand);  in X86TargetLowering()
601       setOperationAction(ISD::FCOS   , VT, Expand);  in X86TargetLowering()
602       setOperationAction(ISD::FSINCOS, VT, Expand);  in X86TargetLowering()
687   for (auto VT : { MVT::v4f32, MVT::v8f32, MVT::v16f32,  in X86TargetLowering()
689     setOperationAction(ISD::FSIN,      VT, Expand);  in X86TargetLowering()
690     setOperationAction(ISD::FSINCOS,   VT, Expand);  in X86TargetLowering()
691     setOperationAction(ISD::FCOS,      VT, Expand);  in X86TargetLowering()
692     setOperationAction(ISD::FREM,      VT, Expand);  in X86TargetLowering()
693     setOperationAction(ISD::FCOPYSIGN, VT, Expand);  in X86TargetLowering()
694     setOperationAction(ISD::FPOW,      VT, Expand);  in X86TargetLowering()
695     setOperationAction(ISD::FLOG,      VT, Expand);  in X86TargetLowering()
696     setOperationAction(ISD::FLOG2,     VT, Expand);  in X86TargetLowering()
697     setOperationAction(ISD::FLOG10,    VT, Expand);  in X86TargetLowering()
698     setOperationAction(ISD::FEXP,      VT, Expand);  in X86TargetLowering()
699     setOperationAction(ISD::FEXP2,     VT, Expand);  in X86TargetLowering()
705   for (MVT VT : MVT::vector_valuetypes()) {  in X86TargetLowering()  local
706     setOperationAction(ISD::SDIV, VT, Expand);  in X86TargetLowering()
707     setOperationAction(ISD::UDIV, VT, Expand);  in X86TargetLowering()
708     setOperationAction(ISD::SREM, VT, Expand);  in X86TargetLowering()
709     setOperationAction(ISD::UREM, VT, Expand);  in X86TargetLowering()
710     setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT,Expand);  in X86TargetLowering()
711     setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand);  in X86TargetLowering()
712     setOperationAction(ISD::EXTRACT_SUBVECTOR, VT,Expand);  in X86TargetLowering()
713     setOperationAction(ISD::INSERT_SUBVECTOR, VT,Expand);  in X86TargetLowering()
714     setOperationAction(ISD::FMA,  VT, Expand);  in X86TargetLowering()
715     setOperationAction(ISD::FFLOOR, VT, Expand);  in X86TargetLowering()
716     setOperationAction(ISD::FCEIL, VT, Expand);  in X86TargetLowering()
717     setOperationAction(ISD::FTRUNC, VT, Expand);  in X86TargetLowering()
718     setOperationAction(ISD::FRINT, VT, Expand);  in X86TargetLowering()
719     setOperationAction(ISD::FNEARBYINT, VT, Expand);  in X86TargetLowering()
720     setOperationAction(ISD::SMUL_LOHI, VT, Expand);  in X86TargetLowering()
721     setOperationAction(ISD::MULHS, VT, Expand);  in X86TargetLowering()
722     setOperationAction(ISD::UMUL_LOHI, VT, Expand);  in X86TargetLowering()
723     setOperationAction(ISD::MULHU, VT, Expand);  in X86TargetLowering()
724     setOperationAction(ISD::SDIVREM, VT, Expand);  in X86TargetLowering()
725     setOperationAction(ISD::UDIVREM, VT, Expand);  in X86TargetLowering()
726     setOperationAction(ISD::CTPOP, VT, Expand);  in X86TargetLowering()
727     setOperationAction(ISD::CTTZ, VT, Expand);  in X86TargetLowering()
728     setOperationAction(ISD::CTLZ, VT, Expand);  in X86TargetLowering()
729     setOperationAction(ISD::ROTL, VT, Expand);  in X86TargetLowering()
730     setOperationAction(ISD::ROTR, VT, Expand);  in X86TargetLowering()
731     setOperationAction(ISD::BSWAP, VT, Expand);  in X86TargetLowering()
732     setOperationAction(ISD::SETCC, VT, Expand);  in X86TargetLowering()
733     setOperationAction(ISD::FP_TO_UINT, VT, Expand);  in X86TargetLowering()
734     setOperationAction(ISD::FP_TO_SINT, VT, Expand);  in X86TargetLowering()
735     setOperationAction(ISD::UINT_TO_FP, VT, Expand);  in X86TargetLowering()
736     setOperationAction(ISD::SINT_TO_FP, VT, Expand);  in X86TargetLowering()
737     setOperationAction(ISD::SIGN_EXTEND_INREG, VT,Expand);  in X86TargetLowering()
738     setOperationAction(ISD::TRUNCATE, VT, Expand);  in X86TargetLowering()
739     setOperationAction(ISD::SIGN_EXTEND, VT, Expand);  in X86TargetLowering()
740     setOperationAction(ISD::ZERO_EXTEND, VT, Expand);  in X86TargetLowering()
741     setOperationAction(ISD::ANY_EXTEND, VT, Expand);  in X86TargetLowering()
742     setOperationAction(ISD::SELECT_CC, VT, Expand);  in X86TargetLowering()
744       setTruncStoreAction(InnerVT, VT, Expand);  in X86TargetLowering()
746       setLoadExtAction(ISD::SEXTLOAD, InnerVT, VT, Expand);  in X86TargetLowering()
747       setLoadExtAction(ISD::ZEXTLOAD, InnerVT, VT, Expand);  in X86TargetLowering()
753       if (VT.getVectorElementType() == MVT::i1)  in X86TargetLowering()
754         setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);  in X86TargetLowering()
758       if (VT.getVectorElementType() == MVT::f16)  in X86TargetLowering()
759         setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);  in X86TargetLowering()
800     for (auto VT : { MVT::v2i8, MVT::v4i8, MVT::v8i8,  in X86TargetLowering()
802       setOperationAction(ISD::SDIV, VT, Custom);  in X86TargetLowering()
803       setOperationAction(ISD::SREM, VT, Custom);  in X86TargetLowering()
804       setOperationAction(ISD::UDIV, VT, Custom);  in X86TargetLowering()
805       setOperationAction(ISD::UREM, VT, Custom);  in X86TargetLowering()
829     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {  in X86TargetLowering()
830       setOperationAction(ISD::SMAX, VT, VT == MVT::v8i16 ? Legal : Custom);  in X86TargetLowering()
831       setOperationAction(ISD::SMIN, VT, VT == MVT::v8i16 ? Legal : Custom);  in X86TargetLowering()
832       setOperationAction(ISD::UMAX, VT, VT == MVT::v16i8 ? Legal : Custom);  in X86TargetLowering()
833       setOperationAction(ISD::UMIN, VT, VT == MVT::v16i8 ? Legal : Custom);  in X86TargetLowering()
847       for (auto VT : { MVT::v8i8, MVT::v4i8, MVT::v2i8,  in X86TargetLowering()
849         setOperationAction(ISD::UADDSAT, VT, Custom);  in X86TargetLowering()
850         setOperationAction(ISD::SADDSAT, VT, Custom);  in X86TargetLowering()
851         setOperationAction(ISD::USUBSAT, VT, Custom);  in X86TargetLowering()
852         setOperationAction(ISD::SSUBSAT, VT, Custom);  in X86TargetLowering()
867     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {  in X86TargetLowering()
868       setOperationAction(ISD::SETCC,              VT, Custom);  in X86TargetLowering()
869       setOperationAction(ISD::CTPOP,              VT, Custom);  in X86TargetLowering()
870       setOperationAction(ISD::ABS,                VT, Custom);  in X86TargetLowering()
874       setCondCodeAction(ISD::SETLT, VT, Custom);  in X86TargetLowering()
875       setCondCodeAction(ISD::SETLE, VT, Custom);  in X86TargetLowering()
878     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {  in X86TargetLowering()
879       setOperationAction(ISD::SCALAR_TO_VECTOR,   VT, Custom);  in X86TargetLowering()
880       setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);  in X86TargetLowering()
881       setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);  in X86TargetLowering()
882       setOperationAction(ISD::VSELECT,            VT, Custom);  in X86TargetLowering()
883       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);  in X86TargetLowering()
890     for (MVT VT : MVT::integer_vector_valuetypes()) {  in X86TargetLowering()  local
891       setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Custom);  in X86TargetLowering()
892       setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i16, Custom);  in X86TargetLowering()
893       setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i32, Custom);  in X86TargetLowering()
894       setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i8, Custom);  in X86TargetLowering()
895       setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i16, Custom);  in X86TargetLowering()
896       setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8i8, Custom);  in X86TargetLowering()
899     for (auto VT : { MVT::v2f64, MVT::v2i64 }) {  in X86TargetLowering()
900       setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);  in X86TargetLowering()
901       setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);  in X86TargetLowering()
902       setOperationAction(ISD::VSELECT,            VT, Custom);  in X86TargetLowering()
904       if (VT == MVT::v2i64 && !Subtarget.is64Bit())  in X86TargetLowering()
907       setOperationAction(ISD::INSERT_VECTOR_ELT,  VT, Custom);  in X86TargetLowering()
908       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);  in X86TargetLowering()
953     for (MVT VT : MVT::fp_vector_valuetypes())  in X86TargetLowering()  local
954       setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2f32, Legal);  in X86TargetLowering()
993     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {  in X86TargetLowering()
994       setOperationAction(ISD::SRL,              VT, Custom);  in X86TargetLowering()
995       setOperationAction(ISD::SHL,              VT, Custom);  in X86TargetLowering()
996       setOperationAction(ISD::SRA,              VT, Custom);  in X86TargetLowering()
1051     for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {  in X86TargetLowering()
1052       setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Legal);  in X86TargetLowering()
1053       setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Legal);  in X86TargetLowering()
1059       for (MVT VT : MVT::integer_vector_valuetypes()) {  in X86TargetLowering()  local
1060         setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Custom);  in X86TargetLowering()
1061         setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i16, Custom);  in X86TargetLowering()
1062         setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i32, Custom);  in X86TargetLowering()
1084     for (auto VT : { MVT::v16i8, MVT::v8i16,  MVT::v4i32, MVT::v2i64,  in X86TargetLowering()
1086       setOperationAction(ISD::ROTL, VT, Custom);  in X86TargetLowering()
1089     for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 })  in X86TargetLowering()
1090       setOperationAction(ISD::BITREVERSE, VT, Custom);  in X86TargetLowering()
1092     for (auto VT : { MVT::v16i8, MVT::v8i16,  MVT::v4i32, MVT::v2i64,  in X86TargetLowering()
1094       setOperationAction(ISD::BITREVERSE, VT, Custom);  in X86TargetLowering()
1113     for (auto VT : { MVT::v8f32, MVT::v4f64 }) {  in X86TargetLowering()
1114       setOperationAction(ISD::FFLOOR,     VT, Legal);  in X86TargetLowering()
1115       setOperationAction(ISD::FCEIL,      VT, Legal);  in X86TargetLowering()
1116       setOperationAction(ISD::FTRUNC,     VT, Legal);  in X86TargetLowering()
1117       setOperationAction(ISD::FRINT,      VT, Legal);  in X86TargetLowering()
1118       setOperationAction(ISD::FNEARBYINT, VT, Legal);  in X86TargetLowering()
1119       setOperationAction(ISD::FNEG,       VT, Custom);  in X86TargetLowering()
1120       setOperationAction(ISD::FABS,       VT, Custom);  in X86TargetLowering()
1121       setOperationAction(ISD::FCOPYSIGN,  VT, Custom);  in X86TargetLowering()
1136     for (MVT VT : MVT::fp_vector_valuetypes())  in X86TargetLowering()  local
1137       setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4f32, Legal);  in X86TargetLowering()
1141     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {  in X86TargetLowering()
1142       setOperationAction(ISD::SRL, VT, Custom);  in X86TargetLowering()
1143       setOperationAction(ISD::SHL, VT, Custom);  in X86TargetLowering()
1144       setOperationAction(ISD::SRA, VT, Custom);  in X86TargetLowering()
1169     for (auto VT : { MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {  in X86TargetLowering()
1170       setOperationAction(ISD::SIGN_EXTEND,     VT, Custom);  in X86TargetLowering()
1171       setOperationAction(ISD::ZERO_EXTEND,     VT, Custom);  in X86TargetLowering()
1172       setOperationAction(ISD::ANY_EXTEND,      VT, Custom);  in X86TargetLowering()
1180     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {  in X86TargetLowering()
1181       setOperationAction(ISD::SETCC,           VT, Custom);  in X86TargetLowering()
1182       setOperationAction(ISD::CTPOP,           VT, Custom);  in X86TargetLowering()
1183       setOperationAction(ISD::CTLZ,            VT, Custom);  in X86TargetLowering()
1186       setOperationAction(ISD::CTTZ,  VT, HasInt256 ? Expand : Custom);  in X86TargetLowering()
1190       setCondCodeAction(ISD::SETLT, VT, Custom);  in X86TargetLowering()
1191       setCondCodeAction(ISD::SETLE, VT, Custom);  in X86TargetLowering()
1195       for (auto VT : { MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32,  in X86TargetLowering()
1197         setOperationAction(ISD::FMA, VT, Legal);  in X86TargetLowering()
1200     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {  in X86TargetLowering()
1201       setOperationAction(ISD::ADD, VT, HasInt256 ? Legal : Custom);  in X86TargetLowering()
1202       setOperationAction(ISD::SUB, VT, HasInt256 ? Legal : Custom);  in X86TargetLowering()
1232     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32 }) {  in X86TargetLowering()
1233       setOperationAction(ISD::ABS,  VT, HasInt256 ? Legal : Custom);  in X86TargetLowering()
1234       setOperationAction(ISD::SMAX, VT, HasInt256 ? Legal : Custom);  in X86TargetLowering()
1235       setOperationAction(ISD::UMAX, VT, HasInt256 ? Legal : Custom);  in X86TargetLowering()
1236       setOperationAction(ISD::SMIN, VT, HasInt256 ? Legal : Custom);  in X86TargetLowering()
1237       setOperationAction(ISD::UMIN, VT, HasInt256 ? Legal : Custom);  in X86TargetLowering()
1240     for (auto VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64}) {  in X86TargetLowering()
1241       setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);  in X86TargetLowering()
1242       setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);  in X86TargetLowering()
1261     for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,  in X86TargetLowering()
1263       setOperationAction(ISD::MLOAD,  VT, Legal);  in X86TargetLowering()
1264       setOperationAction(ISD::MSTORE, VT, Legal);  in X86TargetLowering()
1269     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,  in X86TargetLowering()
1271       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);  in X86TargetLowering()
1275     for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,  in X86TargetLowering()
1277       setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);  in X86TargetLowering()
1278       setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);  in X86TargetLowering()
1279       setOperationAction(ISD::VSELECT,            VT, Custom);  in X86TargetLowering()
1280       setOperationAction(ISD::INSERT_VECTOR_ELT,  VT, Custom);  in X86TargetLowering()
1281       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);  in X86TargetLowering()
1282       setOperationAction(ISD::SCALAR_TO_VECTOR,   VT, Custom);  in X86TargetLowering()
1283       setOperationAction(ISD::INSERT_SUBVECTOR,   VT, Legal);  in X86TargetLowering()
1284       setOperationAction(ISD::CONCAT_VECTORS,     VT, Custom);  in X86TargetLowering()
1295       for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,  in X86TargetLowering()
1297         setOperationAction(ISD::MGATHER,  VT, Custom);  in X86TargetLowering()
1336     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {  in X86TargetLowering()
1337       setOperationAction(ISD::SIGN_EXTEND, VT, Custom);  in X86TargetLowering()
1338       setOperationAction(ISD::ZERO_EXTEND, VT, Custom);  in X86TargetLowering()
1339       setOperationAction(ISD::ANY_EXTEND,  VT, Custom);  in X86TargetLowering()
1342     for (auto VT : { MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 }) {  in X86TargetLowering()
1343       setOperationAction(ISD::ADD,              VT, Custom);  in X86TargetLowering()
1344       setOperationAction(ISD::SUB,              VT, Custom);  in X86TargetLowering()
1345       setOperationAction(ISD::MUL,              VT, Custom);  in X86TargetLowering()
1346       setOperationAction(ISD::SETCC,            VT, Custom);  in X86TargetLowering()
1347       setOperationAction(ISD::SELECT,           VT, Custom);  in X86TargetLowering()
1348       setOperationAction(ISD::TRUNCATE,         VT, Custom);  in X86TargetLowering()
1349       setOperationAction(ISD::UADDSAT,          VT, Custom);  in X86TargetLowering()
1350       setOperationAction(ISD::SADDSAT,          VT, Custom);  in X86TargetLowering()
1351       setOperationAction(ISD::USUBSAT,          VT, Custom);  in X86TargetLowering()
1352       setOperationAction(ISD::SSUBSAT,          VT, Custom);  in X86TargetLowering()
1354       setOperationAction(ISD::BUILD_VECTOR,     VT, Custom);  in X86TargetLowering()
1355       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);  in X86TargetLowering()
1356       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);  in X86TargetLowering()
1357       setOperationAction(ISD::VECTOR_SHUFFLE,   VT,  Custom);  in X86TargetLowering()
1358       setOperationAction(ISD::VSELECT,          VT,  Expand);  in X86TargetLowering()
1368     for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1 })  in X86TargetLowering()
1369       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);  in X86TargetLowering()
1381     for (MVT VT : MVT::fp_vector_valuetypes())  in X86TargetLowering()  local
1382       setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8f32, Legal);  in X86TargetLowering()
1392     for (MVT VT : { MVT::v16f32, MVT::v8f64 }) {  in X86TargetLowering()
1393       setOperationAction(ISD::FNEG,  VT, Custom);  in X86TargetLowering()
1394       setOperationAction(ISD::FABS,  VT, Custom);  in X86TargetLowering()
1395       setOperationAction(ISD::FMA,   VT, Legal);  in X86TargetLowering()
1396       setOperationAction(ISD::FCOPYSIGN, VT, Custom);  in X86TargetLowering()
1420       for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,  in X86TargetLowering()
1422         setOperationAction(ISD::MLOAD,  VT, Custom);  in X86TargetLowering()
1423         setOperationAction(ISD::MSTORE, VT, Custom);  in X86TargetLowering()
1443     for (auto VT : { MVT::v16f32, MVT::v8f64 }) {  in X86TargetLowering()
1444       setOperationAction(ISD::FFLOOR,           VT, Legal);  in X86TargetLowering()
1445       setOperationAction(ISD::FCEIL,            VT, Legal);  in X86TargetLowering()
1446       setOperationAction(ISD::FTRUNC,           VT, Legal);  in X86TargetLowering()
1447       setOperationAction(ISD::FRINT,            VT, Legal);  in X86TargetLowering()
1448       setOperationAction(ISD::FNEARBYINT,       VT, Legal);  in X86TargetLowering()
1452     for (auto VT : {MVT::v16i32, MVT::v8i64, MVT::v64i8}) {  in X86TargetLowering()
1453       setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);  in X86TargetLowering()
1454       setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);  in X86TargetLowering()
1475     for (auto VT : { MVT::v16i32, MVT::v8i64 }) {  in X86TargetLowering()
1476       setOperationAction(ISD::SMAX,             VT, Legal);  in X86TargetLowering()
1477       setOperationAction(ISD::UMAX,             VT, Legal);  in X86TargetLowering()
1478       setOperationAction(ISD::SMIN,             VT, Legal);  in X86TargetLowering()
1479       setOperationAction(ISD::UMIN,             VT, Legal);  in X86TargetLowering()
1480       setOperationAction(ISD::ABS,              VT, Legal);  in X86TargetLowering()
1481       setOperationAction(ISD::SRL,              VT, Custom);  in X86TargetLowering()
1482       setOperationAction(ISD::SHL,              VT, Custom);  in X86TargetLowering()
1483       setOperationAction(ISD::SRA,              VT, Custom);  in X86TargetLowering()
1484       setOperationAction(ISD::CTPOP,            VT, Custom);  in X86TargetLowering()
1485       setOperationAction(ISD::ROTL,             VT, Custom);  in X86TargetLowering()
1486       setOperationAction(ISD::ROTR,             VT, Custom);  in X86TargetLowering()
1487       setOperationAction(ISD::SETCC,            VT, Custom);  in X86TargetLowering()
1491       setCondCodeAction(ISD::SETLT, VT, Custom);  in X86TargetLowering()
1492       setCondCodeAction(ISD::SETLE, VT, Custom);  in X86TargetLowering()
1506       for (auto VT : { MVT::v16i32, MVT::v8i64} ) {  in X86TargetLowering()
1507         setOperationAction(ISD::CTLZ,            VT, Legal);  in X86TargetLowering()
1512       for (auto VT : { MVT::v16i32, MVT::v8i64 })  in X86TargetLowering()
1513         setOperationAction(ISD::CTPOP, VT, Legal);  in X86TargetLowering()
1519     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,  in X86TargetLowering()
1521       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);  in X86TargetLowering()
1523     for (auto VT : { MVT::v16i32, MVT::v8i64, MVT::v16f32, MVT::v8f64 }) {  in X86TargetLowering()
1524       setOperationAction(ISD::VECTOR_SHUFFLE,      VT, Custom);  in X86TargetLowering()
1525       setOperationAction(ISD::INSERT_VECTOR_ELT,   VT, Custom);  in X86TargetLowering()
1526       setOperationAction(ISD::BUILD_VECTOR,        VT, Custom);  in X86TargetLowering()
1527       setOperationAction(ISD::VSELECT,             VT, Custom);  in X86TargetLowering()
1528       setOperationAction(ISD::EXTRACT_VECTOR_ELT,  VT, Custom);  in X86TargetLowering()
1529       setOperationAction(ISD::SCALAR_TO_VECTOR,    VT, Custom);  in X86TargetLowering()
1530       setOperationAction(ISD::INSERT_SUBVECTOR,    VT, Legal);  in X86TargetLowering()
1531       setOperationAction(ISD::MLOAD,               VT, Legal);  in X86TargetLowering()
1532       setOperationAction(ISD::MSTORE,              VT, Legal);  in X86TargetLowering()
1533       setOperationAction(ISD::MGATHER,             VT, Custom);  in X86TargetLowering()
1534       setOperationAction(ISD::MSCATTER,            VT, Custom);  in X86TargetLowering()
1543       for (auto VT : { MVT::v16i32, MVT::v8i64 }) {  in X86TargetLowering()
1544         setOperationAction(ISD::FSHL, VT, Custom);  in X86TargetLowering()
1545         setOperationAction(ISD::FSHR, VT, Custom);  in X86TargetLowering()
1564     for (auto VT : { MVT::v2i64, MVT::v4i64 }) {  in X86TargetLowering()
1565       setOperationAction(ISD::SMAX, VT, Legal);  in X86TargetLowering()
1566       setOperationAction(ISD::UMAX, VT, Legal);  in X86TargetLowering()
1567       setOperationAction(ISD::SMIN, VT, Legal);  in X86TargetLowering()
1568       setOperationAction(ISD::UMIN, VT, Legal);  in X86TargetLowering()
1569       setOperationAction(ISD::ABS,  VT, Legal);  in X86TargetLowering()
1572     for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {  in X86TargetLowering()
1573       setOperationAction(ISD::ROTL,     VT, Custom);  in X86TargetLowering()
1574       setOperationAction(ISD::ROTR,     VT, Custom);  in X86TargetLowering()
1581     for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,  in X86TargetLowering()
1583       setOperationAction(ISD::MSCATTER, VT, Custom);  in X86TargetLowering()
1586       for (auto VT : { MVT::v2i64, MVT::v4i64 }) {  in X86TargetLowering()
1587         setOperationAction(ISD::SINT_TO_FP,     VT, Legal);  in X86TargetLowering()
1588         setOperationAction(ISD::UINT_TO_FP,     VT, Legal);  in X86TargetLowering()
1589         setOperationAction(ISD::FP_TO_SINT,     VT, Legal);  in X86TargetLowering()
1590         setOperationAction(ISD::FP_TO_UINT,     VT, Legal);  in X86TargetLowering()
1592         setOperationAction(ISD::MUL,            VT, Legal);  in X86TargetLowering()
1597       for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {  in X86TargetLowering()
1598         setOperationAction(ISD::CTLZ,            VT, Legal);  in X86TargetLowering()
1603       for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 })  in X86TargetLowering()
1604         setOperationAction(ISD::CTPOP, VT, Legal);  in X86TargetLowering()
1615     for (auto VT : { MVT::v32i1, MVT::v64i1 }) {  in X86TargetLowering()
1616       setOperationAction(ISD::ADD,                VT, Custom);  in X86TargetLowering()
1617       setOperationAction(ISD::SUB,                VT, Custom);  in X86TargetLowering()
1618       setOperationAction(ISD::MUL,                VT, Custom);  in X86TargetLowering()
1619       setOperationAction(ISD::VSELECT,            VT, Expand);  in X86TargetLowering()
1620       setOperationAction(ISD::UADDSAT,            VT, Custom);  in X86TargetLowering()
1621       setOperationAction(ISD::SADDSAT,            VT, Custom);  in X86TargetLowering()
1622       setOperationAction(ISD::USUBSAT,            VT, Custom);  in X86TargetLowering()
1623       setOperationAction(ISD::SSUBSAT,            VT, Custom);  in X86TargetLowering()
1625       setOperationAction(ISD::TRUNCATE,           VT, Custom);  in X86TargetLowering()
1626       setOperationAction(ISD::SETCC,              VT, Custom);  in X86TargetLowering()
1627       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);  in X86TargetLowering()
1628       setOperationAction(ISD::INSERT_VECTOR_ELT,  VT, Custom);  in X86TargetLowering()
1629       setOperationAction(ISD::SELECT,             VT, Custom);  in X86TargetLowering()
1630       setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);  in X86TargetLowering()
1631       setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);  in X86TargetLowering()
1638     for (auto VT : { MVT::v16i1, MVT::v32i1 })  in X86TargetLowering()
1639       setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);  in X86TargetLowering()
1688     for (auto VT : { MVT::v64i8, MVT::v32i16 }) {  in X86TargetLowering()
1689       setOperationAction(ISD::BUILD_VECTOR, VT, Custom);  in X86TargetLowering()
1690       setOperationAction(ISD::VSELECT,      VT, Custom);  in X86TargetLowering()
1691       setOperationAction(ISD::ABS,          VT, Legal);  in X86TargetLowering()
1692       setOperationAction(ISD::SRL,          VT, Custom);  in X86TargetLowering()
1693       setOperationAction(ISD::SHL,          VT, Custom);  in X86TargetLowering()
1694       setOperationAction(ISD::SRA,          VT, Custom);  in X86TargetLowering()
1695       setOperationAction(ISD::MLOAD,        VT, Legal);  in X86TargetLowering()
1696       setOperationAction(ISD::MSTORE,       VT, Legal);  in X86TargetLowering()
1697       setOperationAction(ISD::CTPOP,        VT, Custom);  in X86TargetLowering()
1698       setOperationAction(ISD::CTLZ,         VT, Custom);  in X86TargetLowering()
1699       setOperationAction(ISD::SMAX,         VT, Legal);  in X86TargetLowering()
1700       setOperationAction(ISD::UMAX,         VT, Legal);  in X86TargetLowering()
1701       setOperationAction(ISD::SMIN,         VT, Legal);  in X86TargetLowering()
1702       setOperationAction(ISD::UMIN,         VT, Legal);  in X86TargetLowering()
1703       setOperationAction(ISD::SETCC,        VT, Custom);  in X86TargetLowering()
1704       setOperationAction(ISD::UADDSAT,      VT, Legal);  in X86TargetLowering()
1705       setOperationAction(ISD::SADDSAT,      VT, Legal);  in X86TargetLowering()
1706       setOperationAction(ISD::USUBSAT,      VT, Legal);  in X86TargetLowering()
1707       setOperationAction(ISD::SSUBSAT,      VT, Legal);  in X86TargetLowering()
1711       setCondCodeAction(ISD::SETLT, VT, Custom);  in X86TargetLowering()
1712       setCondCodeAction(ISD::SETLE, VT, Custom);  in X86TargetLowering()
1720       for (auto VT : { MVT::v64i8, MVT::v32i16 })  in X86TargetLowering()
1721         setOperationAction(ISD::CTPOP, VT, Legal);  in X86TargetLowering()
1731     for (auto VT : { MVT::v32i8, MVT::v16i8, MVT::v16i16, MVT::v8i16 }) {  in X86TargetLowering()
1732       setOperationAction(ISD::MLOAD,  VT, Subtarget.hasVLX() ? Legal : Custom);  in X86TargetLowering()
1733       setOperationAction(ISD::MSTORE, VT, Subtarget.hasVLX() ? Legal : Custom);  in X86TargetLowering()
1741       for (auto VT : { MVT::v16i8, MVT::v32i8, MVT::v8i16, MVT::v16i16 })  in X86TargetLowering()
1742         setOperationAction(ISD::CTPOP, VT, Legal);  in X86TargetLowering()
1777       for (auto VT : { MVT::v8i16,  MVT::v4i32, MVT::v2i64,  in X86TargetLowering()
1779         setOperationAction(ISD::FSHL, VT, Custom);  in X86TargetLowering()
1780         setOperationAction(ISD::FSHR, VT, Custom);  in X86TargetLowering()
1800   for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {  in X86TargetLowering()
1801     if (VT == MVT::i64 && !Subtarget.is64Bit())  in X86TargetLowering()
1804     setOperationAction(ISD::SADDO, VT, Custom);  in X86TargetLowering()
1805     setOperationAction(ISD::UADDO, VT, Custom);  in X86TargetLowering()
1806     setOperationAction(ISD::SSUBO, VT, Custom);  in X86TargetLowering()
1807     setOperationAction(ISD::USUBO, VT, Custom);  in X86TargetLowering()
1808     setOperationAction(ISD::SMULO, VT, Custom);  in X86TargetLowering()
1809     setOperationAction(ISD::UMULO, VT, Custom);  in X86TargetLowering()
1812     setOperationAction(ISD::ADDCARRY, VT, Custom);  in X86TargetLowering()
1813     setOperationAction(ISD::SUBCARRY, VT, Custom);  in X86TargetLowering()
1814     setOperationAction(ISD::SETCCCARRY, VT, Custom);  in X86TargetLowering()
1938 X86TargetLowering::getPreferredVectorAction(MVT VT) const {  in getPreferredVectorAction()
1939   if (VT == MVT::v32i1 && Subtarget.hasAVX512() && !Subtarget.hasBWI())  in getPreferredVectorAction()
1943       VT.getVectorNumElements() != 1 &&  in getPreferredVectorAction()
1944       VT.getVectorElementType() != MVT::i1)  in getPreferredVectorAction()
1947   return TargetLoweringBase::getPreferredVectorAction(VT);  in getPreferredVectorAction()
1952                                                      EVT VT) const {  in getRegisterTypeForCallingConv()
1953   if (VT == MVT::v32i1 && Subtarget.hasAVX512() && !Subtarget.hasBWI())  in getRegisterTypeForCallingConv()
1955   return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);  in getRegisterTypeForCallingConv()
1960                                                           EVT VT) const {  in getNumRegistersForCallingConv()
1961   if (VT == MVT::v32i1 && Subtarget.hasAVX512() && !Subtarget.hasBWI())  in getNumRegistersForCallingConv()
1963   return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);  in getNumRegistersForCallingConv()
1968                                           EVT VT) const {  in getSetCCResultType()
1969   if (!VT.isVector())  in getSetCCResultType()
1973     const unsigned NumElts = VT.getVectorNumElements();  in getSetCCResultType()
1976     EVT LegalVT = VT;  in getSetCCResultType()
1994   return VT.changeVectorElementTypeToInteger();  in getSetCCResultType()
2099 bool X86TargetLowering::isSafeMemOpType(MVT VT) const {  in isSafeMemOpType()
2100   if (VT == MVT::f32)  in isSafeMemOpType()
2102   else if (VT == MVT::f64)  in isSafeMemOpType()
2108 X86TargetLowering::allowsMisalignedMemoryAccesses(EVT VT,  in allowsMisalignedMemoryAccesses()  argument
2113     switch (VT.getSizeInBits()) {  in allowsMisalignedMemoryAccesses()
2213                                            MVT VT) const {  in findRepresentativeClass()
2216   switch (VT.SimpleTy) {  in findRepresentativeClass()
2218     return TargetLowering::findRepresentativeClass(TRI, VT);  in findRepresentativeClass()
2665 EVT X86TargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT,  in getTypeForExtReturn()  argument
2670   if (VT == MVT::i1 || (!Darwin && (VT == MVT::i8 || VT == MVT::i16))) {  in getTypeForExtReturn()
2680   return VT.bitsLT(MinVT) ? MinVT : VT;  in getTypeForExtReturn()
3171                    (Ins.size() == 2 && ((Is64Bit && Ins[1].VT == MVT::i64) ||  in LowerFormalArguments()
3172                                         (!Is64Bit && Ins[1].VT == MVT::i32)));  in LowerFormalArguments()
3459       SDValue RegVal = DAG.getCopyFromReg(Chain, dl, F.VReg, F.VT);  in LowerFormalArguments()
3460       F.VReg = MF.getRegInfo().createVirtualRegister(getRegClassFor(F.VT));  in LowerFormalArguments()
3543   EVT VT = getPointerTy(DAG.getDataLayout());  in EmitTailCallLoadRetAddr()  local
3547   OutRetAddr = DAG.getLoad(VT, dl, Chain, OutRetAddr, MachinePointerInfo());  in EmitTailCallLoadRetAddr()
3572 static SDValue getMOVL(SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1,  in getMOVL()  argument
3574   unsigned NumElems = VT.getVectorNumElements();  in getMOVL()
3579   return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);  in getMOVL()
3886       SDValue Val = DAG.getCopyFromReg(Chain, dl, F.VReg, F.VT);  in LowerCall()
4799     MVT VT  = MVT::getVT(I.getArgOperand(1)->getType());  in getTgtMemIntrinsic()  local
4808     Info.memVT = MVT::getVectorVT(ScalarVT, VT.getVectorNumElements());  in getTgtMemIntrinsic()
4823 bool X86TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {  in isFPImmLegal()
4863 bool X86TargetLowering::convertSelectOfConstantsToMath(EVT VT) const {  in convertSelectOfConstantsToMath()
4866   if (VT.isVector() && Subtarget.hasAVX512())  in convertSelectOfConstantsToMath()
4872 bool X86TargetLowering::decomposeMulByConstant(EVT VT, SDValue C) const {  in decomposeMulByConstant()  argument
4883   if (isOperationLegal(ISD::MUL, VT))  in decomposeMulByConstant()
4969   EVT VT = Y.getValueType();  in hasAndNotCompare()  local
4971   if (VT.isVector())  in hasAndNotCompare()
4978   if (VT != MVT::i32 && VT != MVT::i64)  in hasAndNotCompare()
4985   EVT VT = Y.getValueType();  in hasAndNot()  local
4987   if (!VT.isVector())  in hasAndNot()
4992   if (!Subtarget.hasSSE1() || VT.getSizeInBits() < 128)  in hasAndNot()
4995   if (VT == MVT::v4i32)  in hasAndNot()
5002   EVT VT = Y.getValueType();  in preferShiftsToClearExtremeBits()  local
5005   if (VT.isVector())  in preferShiftsToClearExtremeBits()
5009   if (VT == MVT::i64 && !Subtarget.is64Bit())  in preferShiftsToClearExtremeBits()
5015 bool X86TargetLowering::shouldSplatInsEltVarIndex(EVT VT) const {  in shouldSplatInsEltVarIndex()
5018   return isTypeLegal(VT);  in shouldSplatInsEltVarIndex()
5022   MVT VT = MVT::getIntegerVT(NumBits);  in hasFastEqualityCompare()  local
5023   if (isTypeLegal(VT))  in hasFastEqualityCompare()
5024     return VT;  in hasFastEqualityCompare()
5222 static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG,  in getConstVector()  argument
5228   MVT ConstVecVT = VT;  in getConstVector()
5229   unsigned NumElts = VT.getVectorNumElements();  in getConstVector()
5231   if (!In64BitMode && VT.getVectorElementType() == MVT::i64) {  in getConstVector()
5248     ConstsNode = DAG.getBitcast(VT, ConstsNode);  in getConstVector()
5253                               MVT VT, SelectionDAG &DAG, const SDLoc &dl) {  in getConstVector()  argument
5259   MVT ConstVecVT = VT;  in getConstVector()
5260   unsigned NumElts = VT.getVectorNumElements();  in getConstVector()
5262   if (!In64BitMode && VT.getVectorElementType() == MVT::i64) {  in getConstVector()
5274     assert(V.getBitWidth() == VT.getScalarSizeInBits() && "Unexpected sizes");  in getConstVector()
5290   return DAG.getBitcast(VT, ConstsNode);  in getConstVector()
5294 static SDValue getZeroVector(MVT VT, const X86Subtarget &Subtarget,  in getZeroVector()  argument
5296   assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector() ||  in getZeroVector()
5297           VT.getVectorElementType() == MVT::i1) &&  in getZeroVector()
5304   if (!Subtarget.hasSSE2() && VT.is128BitVector()) {  in getZeroVector()
5306   } else if (VT.getVectorElementType() == MVT::i1) {  in getZeroVector()
5307     assert((Subtarget.hasBWI() || VT.getVectorNumElements() <= 16) &&  in getZeroVector()
5309     Vec = DAG.getConstant(0, dl, VT);  in getZeroVector()
5311     unsigned Num32BitElts = VT.getSizeInBits() / 32;  in getZeroVector()
5314   return DAG.getBitcast(VT, Vec);  in getZeroVector()
5319   EVT VT = Vec.getValueType();  in extractSubVector()  local
5320   EVT ElVT = VT.getVectorElementType();  in extractSubVector()
5321   unsigned Factor = VT.getSizeInBits()/vectorWidth;  in extractSubVector()
5323                                   VT.getVectorNumElements()/Factor);  in extractSubVector()
5370   EVT VT = Vec.getValueType();  in insertSubVector()  local
5371   EVT ElVT = VT.getVectorElementType();  in insertSubVector()
5400 static SDValue widenSubVector(MVT VT, SDValue Vec, bool ZeroNewElements,  in widenSubVector()  argument
5403   assert(Vec.getValueSizeInBits() < VT.getSizeInBits() &&  in widenSubVector()
5404          Vec.getValueType().getScalarType() == VT.getScalarType() &&  in widenSubVector()
5406   SDValue Res = ZeroNewElements ? getZeroVector(VT, Subtarget, DAG, dl)  in widenSubVector()
5407                                 : DAG.getUNDEF(VT);  in widenSubVector()
5408   return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, Res, Vec,  in widenSubVector()
5421                          const SDLoc &DL, EVT VT, ArrayRef<SDValue> Ops,  in SplitOpsAndApply()  argument
5427     if (VT.getSizeInBits() > 512) {  in SplitOpsAndApply()
5428       NumSubs = VT.getSizeInBits() / 512;  in SplitOpsAndApply()
5429       assert((VT.getSizeInBits() % 512) == 0 && "Illegal vector size");  in SplitOpsAndApply()
5432     if (VT.getSizeInBits() > 256) {  in SplitOpsAndApply()
5433       NumSubs = VT.getSizeInBits() / 256;  in SplitOpsAndApply()
5434       assert((VT.getSizeInBits() % 256) == 0 && "Illegal vector size");  in SplitOpsAndApply()
5437     if (VT.getSizeInBits() > 128) {  in SplitOpsAndApply()
5438       NumSubs = VT.getSizeInBits() / 128;  in SplitOpsAndApply()
5439       assert((VT.getSizeInBits() % 128) == 0 && "Illegal vector size");  in SplitOpsAndApply()
5457   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs);  in SplitOpsAndApply()
5609 static SDValue concatSubVectors(SDValue V1, SDValue V2, EVT VT,  in concatSubVectors()  argument
5612   SDValue V = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, dl, VectorWidth);  in concatSubVectors()
5619 static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {  in getOnesVector()  argument
5620   assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) &&  in getOnesVector()
5624   unsigned NumElts = VT.getSizeInBits() / 32;  in getOnesVector()
5626   return DAG.getBitcast(VT, Vec);  in getOnesVector()
5629 static SDValue getExtendInVec(bool Signed, const SDLoc &DL, EVT VT, SDValue In,  in getExtendInVec()  argument
5632   assert(VT.isVector() && InVT.isVector() && "Expected vector VTs.");  in getExtendInVec()
5637     assert(VT.getSizeInBits() == InVT.getSizeInBits() &&  in getExtendInVec()
5639     unsigned Scale = VT.getScalarSizeInBits() / InVT.getScalarSizeInBits();  in getExtendInVec()
5641                           std::max(128U, VT.getSizeInBits() / Scale));  in getExtendInVec()
5645   if (VT.getVectorNumElements() == InVT.getVectorNumElements())  in getExtendInVec()
5647                        DL, VT, In);  in getExtendInVec()
5651                      DL, VT, In);  in getExtendInVec()
5655 static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, MVT VT,  in getUnpackl()  argument
5658   createUnpackShuffleMask(VT, Mask, /* Lo = */ true, /* Unary = */ false);  in getUnpackl()
5659   return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);  in getUnpackl()
5663 static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, MVT VT,  in getUnpackh()  argument
5666   createUnpackShuffleMask(VT, Mask, /* Lo = */ false, /* Unary = */ false);  in getUnpackh()
5667   return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);  in getUnpackh()
5678   MVT VT = V2.getSimpleValueType();  in getShuffleVectorZeroOrUndef()  local
5680     ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT);  in getShuffleVectorZeroOrUndef()
5681   int NumElems = VT.getVectorNumElements();  in getShuffleVectorZeroOrUndef()
5686   return DAG.getVectorShuffle(VT, SDLoc(V2), V1, V2, MaskVec);  in getShuffleVectorZeroOrUndef()
5725   EVT VT = Op.getValueType();  in getTargetConstantBitsFromNode()  local
5726   unsigned SizeInBits = VT.getSizeInBits();  in getTargetConstantBitsFromNode()
5829     unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();  in getTargetConstantBitsFromNode()
5846     unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();  in getTargetConstantBitsFromNode()
5886       EltSizeInBits <= VT.getScalarSizeInBits()) {  in getTargetConstantBitsFromNode()
5906     unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();  in getTargetConstantBitsFromNode()
5921     if (EltSizeInBits != VT.getScalarSizeInBits())  in getTargetConstantBitsFromNode()
5929       unsigned NumSubElts = VT.getVectorNumElements();  in getTargetConstantBitsFromNode()
5943     if (EltSizeInBits != VT.getScalarSizeInBits())  in getTargetConstantBitsFromNode()
6030 static void createPackShuffleMask(MVT VT, SmallVectorImpl<int> &Mask,  in createPackShuffleMask()  argument
6033   unsigned NumElts = VT.getVectorNumElements();  in createPackShuffleMask()
6034   unsigned NumLanes = VT.getSizeInBits() / 128;  in createPackShuffleMask()
6035   unsigned NumEltsPerLane = 128 / VT.getScalarSizeInBits();  in createPackShuffleMask()
6047 static void getPackDemandedElts(EVT VT, const APInt &DemandedElts,  in getPackDemandedElts()  argument
6049   int NumLanes = VT.getSizeInBits() / 128;  in getPackDemandedElts()
6078 static bool getTargetShuffleMask(SDNode *N, MVT VT, bool AllowSentinelZero,  in getTargetShuffleMask()  argument
6081   unsigned NumElems = VT.getVectorNumElements();  in getTargetShuffleMask()
6082   unsigned MaskEltSize = VT.getScalarSizeInBits();  in getTargetShuffleMask()
6094     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6095     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6101     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6102     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6109     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6110     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6116     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6126     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6127     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6137     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6138     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6143     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6144     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6149     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6150     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6155     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6156     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6161     assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");  in getTargetShuffleMask()
6162     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6163     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6172     assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");  in getTargetShuffleMask()
6173     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6180     assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");  in getTargetShuffleMask()
6181     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6189     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6196     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6203     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6210     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6219         N0.getOperand(0).getValueType() == VT &&  in getTargetShuffleMask()
6226     if (N0.getValueType() == VT || !Ops.empty()) {  in getTargetShuffleMask()
6234     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6245     assert(VT.getScalarType() == MVT::i8 && "Byte vector expected");  in getTargetShuffleMask()
6246     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6247     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6257     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6264     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6265     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6269     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6270     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6277     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6278     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6285     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6290     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6295     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6300     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6301     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6317     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6318     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6328     assert(N->getOperand(1).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6341     assert(N->getOperand(0).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6342     assert(N->getOperand(2).getValueType() == VT && "Unexpected value type");  in getTargetShuffleMask()
6397   MVT VT = N.getSimpleValueType();  in setTargetShuffleZeroElements()  local
6398   if (!getTargetShuffleMask(N.getNode(), VT, true, Ops, Mask, IsUnary))  in setTargetShuffleZeroElements()
6407   assert((VT.getSizeInBits() % Mask.size()) == 0 &&  in setTargetShuffleZeroElements()
6409   unsigned EltSizeInBits = VT.getSizeInBits() / Mask.size();  in setTargetShuffleZeroElements()
6446       if (Idx != 0 && !VT.isFloatingPoint())  in setTargetShuffleZeroElements()
6462   assert(VT.getVectorNumElements() == Mask.size() &&  in setTargetShuffleZeroElements()
6482   MVT VT = N.getSimpleValueType();  in getFauxShuffleMask()  local
6483   unsigned NumElts = VT.getVectorNumElements();  in getFauxShuffleMask()
6484   unsigned NumSizeInBits = VT.getSizeInBits();  in getFauxShuffleMask()
6485   unsigned NumBitsPerElt = VT.getScalarSizeInBits();  in getFauxShuffleMask()
6579           SubInput.getOperand(0).getValueType() != VT ||  in getFauxShuffleMask()
6605          N0.getOperand(0).getValueType() == VT) ||  in getFauxShuffleMask()
6696     createPackShuffleMask(VT, Mask, IsUnary);  in getFauxShuffleMask()
6800   EVT VT = V.getValueType();  in getShuffleScalarElt()  local
6808       return DAG.getUNDEF(VT.getVectorElementType());  in getShuffleScalarElt()
6810     unsigned NumElems = VT.getVectorNumElements();  in getShuffleScalarElt()
6845     unsigned NumElems = VT.getVectorNumElements();  in getShuffleScalarElt()
6853                         : DAG.getUNDEF(VT.getVectorElementType());  in getShuffleScalarElt()
6866   MVT VT = Op.getSimpleValueType();  in LowerBuildVectorAsInsert()  local
6867   unsigned NumElts = VT.getVectorNumElements();  in LowerBuildVectorAsInsert()
6868   assert(((VT == MVT::v8i16 && Subtarget.hasSSE2()) ||  in LowerBuildVectorAsInsert()
6869           ((VT == MVT::v16i8 || VT == MVT::v4i32) && Subtarget.hasSSE41())) &&  in LowerBuildVectorAsInsert()
6887         V = getZeroVector(VT, Subtarget, DAG, dl);  in LowerBuildVectorAsInsert()
6893         V = DAG.getBitcast(VT, V);  in LowerBuildVectorAsInsert()
6897     V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, V, Op.getOperand(i),  in LowerBuildVectorAsInsert()
6993     MVT VT = Op.getSimpleValueType();  in LowerBuildVectorv4x32()  local
6994     MVT EltVT = VT.getVectorElementType();  in LowerBuildVectorv4x32()
6999     SDValue NewBV = DAG.getBitcast(MVT::v2f64, DAG.getBuildVector(VT, DL, Ops));  in LowerBuildVectorv4x32()
7001     return DAG.getBitcast(VT, Dup);  in LowerBuildVectorv4x32()
7025     MVT VT = Elt.getOperand(0).getSimpleValueType();  in LowerBuildVectorv4x32()  local
7026     if (!VT.is128BitVector())  in LowerBuildVectorv4x32()
7036   MVT VT = V1.getSimpleValueType();  in LowerBuildVectorv4x32()  local
7059     SDValue VZero = getZeroVector(VT, Subtarget, DAG, SDLoc(Op));  in LowerBuildVectorv4x32()
7060     if (V1.getSimpleValueType() != VT)  in LowerBuildVectorv4x32()
7061       V1 = DAG.getBitcast(VT, V1);  in LowerBuildVectorv4x32()
7062     return DAG.getVectorShuffle(VT, SDLoc(V1), V1, VZero, Mask);  in LowerBuildVectorv4x32()
7102   return DAG.getBitcast(VT, Result);  in LowerBuildVectorv4x32()
7106 static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits,  in getVShift()  argument
7109   assert(VT.is128BitVector() && "Unknown type for VShift");  in getVShift()
7115   return DAG.getBitcast(VT, DAG.getNode(Opc, dl, ShVT, SrcOp, ShiftVal));  in getVShift()
7118 static SDValue LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, const SDLoc &dl,  in LowerAsSplatVectorLoad()  argument
7148     unsigned RequiredAlign = VT.getSizeInBits()/8;  in LowerAsSplatVectorLoad()
7177     unsigned NumElems = VT.getVectorNumElements();  in LowerAsSplatVectorLoad()
7196 static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,  in EltsFromConsecutiveLoads()  argument
7223       if ((NumElems * Elt.getValueSizeInBits()) != VT.getSizeInBits())  in EltsFromConsecutiveLoads()
7233     return DAG.getUNDEF(VT);  in EltsFromConsecutiveLoads()
7237     return VT.isInteger() ? DAG.getConstant(0, DL, VT)  in EltsFromConsecutiveLoads()
7238                           : DAG.getConstantFP(0.0, DL, VT);  in EltsFromConsecutiveLoads()
7272   auto CreateLoad = [&DAG, &DL, &Loads](EVT VT, LoadSDNode *LDBase) {  in EltsFromConsecutiveLoads()  argument
7277         DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(),  in EltsFromConsecutiveLoads()
7294     if (VT.getSizeInBits() != EltVT.getSizeInBits() * NumElems)  in EltsFromConsecutiveLoads()
7297     if (isAfterLegalize && !TLI.isOperationLegal(ISD::LOAD, VT))  in EltsFromConsecutiveLoads()
7303         VT.is256BitVector() && !Subtarget.hasInt256())  in EltsFromConsecutiveLoads()
7307       return CreateLoad(VT, LDBase);  in EltsFromConsecutiveLoads()
7311     if (!isAfterLegalize && NumElems == VT.getVectorNumElements()) {  in EltsFromConsecutiveLoads()
7319       SDValue V = CreateLoad(VT, LDBase);  in EltsFromConsecutiveLoads()
7320       SDValue Z = VT.isInteger() ? DAG.getConstant(0, DL, VT)  in EltsFromConsecutiveLoads()
7321                                  : DAG.getConstantFP(0.0, DL, VT);  in EltsFromConsecutiveLoads()
7322       return DAG.getVectorShuffle(VT, DL, V, Z, ClearMask);  in EltsFromConsecutiveLoads()
7332       ((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()))) {  in EltsFromConsecutiveLoads()
7333     MVT VecSVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(LoadSize)  in EltsFromConsecutiveLoads()
7335     MVT VecVT = MVT::getVectorVT(VecSVT, VT.getSizeInBits() / LoadSize);  in EltsFromConsecutiveLoads()
7346       return DAG.getBitcast(VT, ResNode);  in EltsFromConsecutiveLoads()
7353 static Constant *getConstantVector(MVT VT, const APInt &SplatValue,  in getConstantVector()  argument
7355   unsigned ScalarSize = VT.getScalarSizeInBits();  in getConstantVector()
7362     if (VT.isFloatingPoint()) {  in getConstantVector()
7439   MVT VT = BVOp->getSimpleValueType(0);  in lowerBuildVectorAsBroadcast()  local
7442   assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) &&  in lowerBuildVectorAsBroadcast()
7454   if (Subtarget.hasCDI() && (VT.is512BitVector() || Subtarget.hasVLX())) {  in lowerBuildVectorAsBroadcast()
7455     MVT EltType = VT.getScalarType();  in lowerBuildVectorAsBroadcast()
7456     unsigned NumElts = VT.getVectorNumElements();  in lowerBuildVectorAsBroadcast()
7472         return DAG.getBitcast(VT, Brdcst);  in lowerBuildVectorAsBroadcast()
7477   unsigned NumElts = VT.getVectorNumElements();  in lowerBuildVectorAsBroadcast()
7485         SplatBitSize > VT.getScalarSizeInBits() &&  in lowerBuildVectorAsBroadcast()
7486         SplatBitSize < VT.getSizeInBits()) {  in lowerBuildVectorAsBroadcast()
7504           unsigned Repeat = VT.getSizeInBits() / SplatBitSize;  in lowerBuildVectorAsBroadcast()
7513           return DAG.getBitcast(VT, Brdcst);  in lowerBuildVectorAsBroadcast()
7528           unsigned Repeat = VT.getSizeInBits() / SplatBitSize;  in lowerBuildVectorAsBroadcast()
7537           return DAG.getBitcast(VT, Brdcst);  in lowerBuildVectorAsBroadcast()
7540           MVT CVT = VT.getScalarType();  in lowerBuildVectorAsBroadcast()
7541           Constant *VecC = getConstantVector(VT, SplatValue, SplatBitSize,  in lowerBuildVectorAsBroadcast()
7544           unsigned NumElm = SplatBitSize / VT.getScalarSizeInBits();  in lowerBuildVectorAsBroadcast()
7550           SDValue Brdcst = DAG.getNode(X86ISD::SUBV_BROADCAST, dl, VT, Ld);  in lowerBuildVectorAsBroadcast()
7551           return DAG.getBitcast(VT, Brdcst);  in lowerBuildVectorAsBroadcast()
7577   bool IsGE256 = (VT.getSizeInBits() >= 256);  in lowerBuildVectorAsBroadcast()
7619       return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);  in lowerBuildVectorAsBroadcast()
7628     return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);  in lowerBuildVectorAsBroadcast()
7636     return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);  in lowerBuildVectorAsBroadcast()
7642       return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);  in lowerBuildVectorAsBroadcast()
7685   MVT VT = Op.getSimpleValueType();  in buildFromShuffleMostly()  local
7689   if (!TLI.isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT))  in buildFromShuffleMostly()
7724     if (ExtractedFromVec.getValueType() != VT)  in buildFromShuffleMostly()
7746   VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT);  in buildFromShuffleMostly()
7747   SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, Mask);  in buildFromShuffleMostly()
7750     NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx),  in buildFromShuffleMostly()
7767   MVT VT = MVT::getIntegerVT(std::max((int)Op.getValueSizeInBits(), 8));  in ConvertI1VectorToInteger()  local
7768   return DAG.getConstant(Immediate, dl, VT);  in ConvertI1VectorToInteger()
7774   MVT VT = Op.getSimpleValueType();  in LowerBUILD_VECTORvXi1()  local
7775   assert((VT.getVectorElementType() == MVT::i1) &&  in LowerBUILD_VECTORvXi1()
7786     if (VT == MVT::v64i1 && !Subtarget.is64Bit()) {  in LowerBUILD_VECTORvXi1()
7799     if (Imm.getValueSizeInBits() == VT.getSizeInBits())  in LowerBUILD_VECTORvXi1()
7800       return DAG.getBitcast(VT, Imm);  in LowerBUILD_VECTORvXi1()
7802     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, ExtVec,  in LowerBUILD_VECTORvXi1()
7830     return DAG.getSelect(dl, VT, Op.getOperand(SplatIdx),  in LowerBUILD_VECTORvXi1()
7831                          DAG.getConstant(1, dl, VT),  in LowerBUILD_VECTORvXi1()
7832                          DAG.getConstant(0, dl, VT));  in LowerBUILD_VECTORvXi1()
7838     MVT ImmVT = MVT::getIntegerVT(std::max((int)VT.getSizeInBits(), 8));  in LowerBUILD_VECTORvXi1()
7842     Imm = DAG.getConstant(0, dl, VT);  in LowerBUILD_VECTORvXi1()
7844     Imm = DAG.getUNDEF(VT);  in LowerBUILD_VECTORvXi1()
7845   if (Imm.getValueSizeInBits() == VT.getSizeInBits())  in LowerBUILD_VECTORvXi1()
7846     DstVec = DAG.getBitcast(VT, Imm);  in LowerBUILD_VECTORvXi1()
7849     DstVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, ExtVec,  in LowerBUILD_VECTORvXi1()
7855     DstVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DstVec,  in LowerBUILD_VECTORvXi1()
7886   EVT VT = N->getValueType(0);  in isHorizontalBinOpPart()  local
7887   assert(VT.is256BitVector() && "Only use for matching partial 256-bit h-ops");  in isHorizontalBinOpPart()
7889   assert(VT.isVector() && VT.getVectorNumElements() >= LastIdx &&  in isHorizontalBinOpPart()
7896   V0 = DAG.getUNDEF(VT);  in isHorizontalBinOpPart()
7897   V1 = DAG.getUNDEF(VT);  in isHorizontalBinOpPart()
7936         if (V0.getValueType() != VT)  in isHorizontalBinOpPart()
7942         if (V1.getValueType() != VT)  in isHorizontalBinOpPart()
8000   MVT VT = V0.getSimpleValueType();  in ExpandHorizontalBinOp()  local
8001   assert(VT.is256BitVector() && VT == V1.getSimpleValueType() &&  in ExpandHorizontalBinOp()
8004   unsigned NumElts = VT.getVectorNumElements();  in ExpandHorizontalBinOp()
8029   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI);  in ExpandHorizontalBinOp()
8043   MVT VT = BV->getSimpleValueType(0);  in isAddSubOrSubAdd()  local
8044   if (!Subtarget.hasSSE3() || !VT.isFloatingPoint())  in isAddSubOrSubAdd()
8047   unsigned NumElts = VT.getVectorNumElements();  in isAddSubOrSubAdd()
8048   SDValue InVec0 = DAG.getUNDEF(VT);  in isAddSubOrSubAdd()
8049   SDValue InVec1 = DAG.getUNDEF(VT);  in isAddSubOrSubAdd()
8096       if (InVec0.getSimpleValueType() != VT)  in isAddSubOrSubAdd()
8101       if (InVec1.getSimpleValueType() != VT)  in isAddSubOrSubAdd()
8203   MVT VT = BV->getSimpleValueType(0);  in lowerToAddSubOrFMAddSub()  local
8210     return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);  in lowerToAddSubOrFMAddSub()
8222   if (VT.is512BitVector())  in lowerToAddSubOrFMAddSub()
8225   return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);  in lowerToAddSubOrFMAddSub()
8231   MVT VT = BV->getSimpleValueType(0);  in isHopBuildVector()  local
8233   V0 = DAG.getUNDEF(VT);  in isHopBuildVector()
8234   V1 = DAG.getUNDEF(VT);  in isHopBuildVector()
8239   unsigned NumElts = VT.getVectorNumElements();  in isHopBuildVector()
8241   unsigned Num128BitChunks = VT.is256BitVector() ? 2 : 1;  in isHopBuildVector()
8321   MVT VT = BV->getSimpleValueType(0);  in getHopForBuildVector()  local
8322   unsigned Width = VT.getSizeInBits();  in getHopForBuildVector()
8326     V0 = insertSubVector(DAG.getUNDEF(VT), V0, 0, DAG, SDLoc(BV), Width);  in getHopForBuildVector()
8331     V1 = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, SDLoc(BV), Width);  in getHopForBuildVector()
8333   return DAG.getNode(HOpcode, SDLoc(BV), VT, V0, V1);  in getHopForBuildVector()
8352   MVT VT = BV->getSimpleValueType(0);  in LowerToHorizontalOp()  local
8355   if ((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget.hasSSE3())  in LowerToHorizontalOp()
8359   if ((VT == MVT::v8i16 || VT == MVT::v4i32) && Subtarget.hasSSSE3())  in LowerToHorizontalOp()
8363   if ((VT == MVT::v8f32 || VT == MVT::v4f64) && Subtarget.hasAVX())  in LowerToHorizontalOp()
8367   if ((VT == MVT::v16i16 || VT == MVT::v8i32) && Subtarget.hasAVX2())  in LowerToHorizontalOp()
8372   if (!Subtarget.hasAVX() || !VT.is256BitVector())  in LowerToHorizontalOp()
8376   unsigned NumElts = VT.getVectorNumElements();  in LowerToHorizontalOp()
8390   if (VT == MVT::v8i32 || VT == MVT::v16i16) {  in LowerToHorizontalOp()
8430   if (VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 ||  in LowerToHorizontalOp()
8431       VT == MVT::v16i16) {  in LowerToHorizontalOp()
8472   MVT VT = Op->getSimpleValueType(0);  in lowerBuildVectorToBitOp()  local
8473   unsigned NumElems = VT.getVectorNumElements();  in lowerBuildVectorToBitOp()
8494     if (!TLI.isOperationLegalOrPromote(Opcode, VT))  in lowerBuildVectorToBitOp()
8511   SDValue LHS = DAG.getBuildVector(VT, DL, LHSElts);  in lowerBuildVectorToBitOp()
8512   SDValue RHS = DAG.getBuildVector(VT, DL, RHSElts);  in lowerBuildVectorToBitOp()
8513   return DAG.getNode(Opcode, DL, VT, LHS, RHS);  in lowerBuildVectorToBitOp()
8522   MVT VT = Op.getSimpleValueType();  in materializeVectorConstant()  local
8528     if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32)  in materializeVectorConstant()
8531     return getZeroVector(VT, Subtarget, DAG, DL);  in materializeVectorConstant()
8538     if (VT == MVT::v4i32 || VT == MVT::v16i32 ||  in materializeVectorConstant()
8539         (VT == MVT::v8i32 && Subtarget.hasInt256()))  in materializeVectorConstant()
8542     return getOnesVector(VT, DAG, DL);  in materializeVectorConstant()
8551 static SDValue createVariablePermute(MVT VT, SDValue SrcVec, SDValue IndicesVec,  in createVariablePermute()  argument
8554   MVT ShuffleVT = VT;  in createVariablePermute()
8555   EVT IndicesVT = EVT(VT).changeVectorElementTypeToInteger();  in createVariablePermute()
8556   unsigned NumElts = VT.getVectorNumElements();  in createVariablePermute()
8557   unsigned SizeInBits = VT.getSizeInBits();  in createVariablePermute()
8564                                   NumElts * VT.getScalarSizeInBits());  in createVariablePermute()
8572       VT = MVT::getVectorVT(VT.getScalarType(), Scale * NumElts);  in createVariablePermute()
8573       IndicesVT = EVT(VT).changeVectorElementTypeToInteger();  in createVariablePermute()
8577           createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget), 0,  in createVariablePermute()
8581       SrcVec = widenSubVector(VT, SrcVec, false, Subtarget, DAG, SDLoc(SrcVec));  in createVariablePermute()
8611   switch (VT.SimpleTy) {  in createVariablePermute()
8648           DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {0, 0}),  in createVariablePermute()
8649           DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {1, 1}),  in createVariablePermute()
8662           ISD::CONCAT_VECTORS, DL, VT,  in createVariablePermute()
8668       SDValue LoLo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Lo);  in createVariablePermute()
8669       SDValue HiHi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Hi, Hi);  in createVariablePermute()
8676         EVT VT = Idx.getValueType();  in createVariablePermute()  local
8677         return DAG.getSelectCC(DL, Idx, DAG.getConstant(15, DL, VT),  in createVariablePermute()
8678                                DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[1], Idx),  in createVariablePermute()
8679                                DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[0], Idx),  in createVariablePermute()
8694           VT, createVariablePermute(  in createVariablePermute()
8710         return DAG.getBitcast(VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v8f32,  in createVariablePermute()
8720       return DAG.getBitcast(VT, Res);  in createVariablePermute()
8727         MVT WidenSrcVT = MVT::getVectorVT(VT.getScalarType(), 8);  in createVariablePermute()
8746         return DAG.getBitcast(VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v4f64,  in createVariablePermute()
8756       return DAG.getBitcast(VT, Res);  in createVariablePermute()
8778   assert((VT.getSizeInBits() == ShuffleVT.getSizeInBits()) &&  in createVariablePermute()
8779          (VT.getScalarSizeInBits() % ShuffleVT.getScalarSizeInBits()) == 0 &&  in createVariablePermute()
8782   uint64_t Scale = VT.getScalarSizeInBits() / ShuffleVT.getScalarSizeInBits();  in createVariablePermute()
8793   return DAG.getBitcast(VT, Res);  in createVariablePermute()
8848   MVT VT = V.getSimpleValueType();  in LowerBUILD_VECTORAsVariablePermute()  local
8849   return createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget);  in LowerBUILD_VECTORAsVariablePermute()
8856   MVT VT = Op.getSimpleValueType();  in LowerBUILD_VECTOR()  local
8857   MVT EltVT = VT.getVectorElementType();  in LowerBUILD_VECTOR()
8861   if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512())  in LowerBUILD_VECTOR()
8905     return DAG.getUNDEF(VT);  in LowerBUILD_VECTOR()
8914       (isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT) ||  in LowerBUILD_VECTOR()
8915        isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, VT))) {  in LowerBUILD_VECTOR()
8938     SDValue DAGConstVec = DAG.getConstantPool(CV, VT);  in LowerBUILD_VECTOR()
8949     SDValue Ld = DAG.getLoad(VT, dl, DAG.getEntryNode(), LegalDAGConstVec, MPI);  in LowerBUILD_VECTOR()
8951     unsigned NumEltsInLow128Bits = 128 / VT.getScalarSizeInBits();  in LowerBUILD_VECTOR()
8953       return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ld, VarElt, InsIndex);  in LowerBUILD_VECTOR()
8957     assert(VT.getSizeInBits() > 128 && "Invalid insertion index?");  in LowerBUILD_VECTOR()
8960     unsigned NumElts = VT.getVectorNumElements();  in LowerBUILD_VECTOR()
8963     SDValue S2V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, VarElt);  in LowerBUILD_VECTOR()
8964     return DAG.getVectorShuffle(VT, dl, Ld, S2V, ShuffleMask);  in LowerBUILD_VECTOR()
8978         return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);  in LowerBUILD_VECTOR()
8982         assert((VT.is128BitVector() || VT.is256BitVector() ||  in LowerBUILD_VECTOR()
8983                 VT.is512BitVector()) &&  in LowerBUILD_VECTOR()
8985         Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);  in LowerBUILD_VECTOR()
8994         if (VT.getSizeInBits() >= 256) {  in LowerBUILD_VECTOR()
8995           MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits()/32);  in LowerBUILD_VECTOR()
9007           assert(VT.is128BitVector() && "Expected an SSE value type!");  in LowerBUILD_VECTOR()
9011         return DAG.getBitcast(VT, Item);  in LowerBUILD_VECTOR()
9019       unsigned NumBits = VT.getSizeInBits();  in LowerBUILD_VECTOR()
9020       return getVShift(true, VT,  in LowerBUILD_VECTOR()
9022                                    VT, Op.getOperand(1)),  in LowerBUILD_VECTOR()
9035       Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);  in LowerBUILD_VECTOR()
9050         return LowerAsSplatVectorLoad(Item, VT, dl, DAG);  in LowerBUILD_VECTOR()
9067             EltsFromConsecutiveLoads(VT, Ops, dl, DAG, Subtarget, false))  in LowerBUILD_VECTOR()
9085       MVT WideEltVT = VT.isFloatingPoint() ? MVT::f64 : MVT::i64;  in LowerBUILD_VECTOR()
9092       return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, dl, BcastVT,  in LowerBUILD_VECTOR()
9099   if (VT.getSizeInBits() > 128) {  in LowerBUILD_VECTOR()
9109     return concatSubVectors(Lower, Upper, VT, NumElems, DAG, dl,  in LowerBUILD_VECTOR()
9110                             VT.getSizeInBits() / 2);  in LowerBUILD_VECTOR()
9118       SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT,  in LowerBUILD_VECTOR()
9147         Ops[i] = getZeroVector(VT, Subtarget, DAG, dl);  in LowerBUILD_VECTOR()
9149         Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));  in LowerBUILD_VECTOR()
9159           Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2+1], Ops[i*2]);  in LowerBUILD_VECTOR()
9162           Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2], Ops[i*2+1]);  in LowerBUILD_VECTOR()
9165           Ops[i] = getUnpackl(DAG, dl, VT, Ops[i*2], Ops[i*2+1]);  in LowerBUILD_VECTOR()
9178     return DAG.getVectorShuffle(VT, dl, Ops[0], Ops[1], MaskVec);  in LowerBUILD_VECTOR()
9191       Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0));  in LowerBUILD_VECTOR()
9193       Result = DAG.getUNDEF(VT);  in LowerBUILD_VECTOR()
9197       Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result,  in LowerBUILD_VECTOR()
9209       Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));  in LowerBUILD_VECTOR()
9211       Ops[i] = DAG.getUNDEF(VT);  in LowerBUILD_VECTOR()
9228       Ops[i] = DAG.getVectorShuffle(VT, dl, Ops[2*i], Ops[(2*i)+1], Mask);  in LowerBUILD_VECTOR()
9420   MVT VT = Op.getSimpleValueType();  in LowerCONCAT_VECTORS()  local
9421   if (VT.getVectorElementType() == MVT::i1)  in LowerCONCAT_VECTORS()
9424   assert((VT.is256BitVector() && Op.getNumOperands() == 2) ||  in LowerCONCAT_VECTORS()
9425          (VT.is512BitVector() && (Op.getNumOperands() == 2 ||  in LowerCONCAT_VECTORS()
9467 static bool is128BitLaneCrossingShuffleMask(MVT VT, ArrayRef<int> Mask) {  in is128BitLaneCrossingShuffleMask()  argument
9468   int LaneSize = 128 / VT.getScalarSizeInBits();  in is128BitLaneCrossingShuffleMask()
9487 static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT,  in isRepeatedShuffleMask()  argument
9490   auto LaneSize = LaneSizeInBits / VT.getScalarSizeInBits();  in isRepeatedShuffleMask()
9517 is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask,  in is128BitLaneRepeatedShuffleMask()  argument
9519   return isRepeatedShuffleMask(128, VT, Mask, RepeatedMask);  in is128BitLaneRepeatedShuffleMask()
9523 is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask) {  in is128BitLaneRepeatedShuffleMask()  argument
9525   return isRepeatedShuffleMask(128, VT, Mask, RepeatedMask);  in is128BitLaneRepeatedShuffleMask()
9530 is256BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask,  in is256BitLaneRepeatedShuffleMask()  argument
9532   return isRepeatedShuffleMask(256, VT, Mask, RepeatedMask);  in is256BitLaneRepeatedShuffleMask()
9537 static bool isRepeatedTargetShuffleMask(unsigned LaneSizeInBits, MVT VT,  in isRepeatedTargetShuffleMask()  argument
9540   int LaneSize = LaneSizeInBits / VT.getScalarSizeInBits();  in isRepeatedTargetShuffleMask()
9675 static bool isUnpackWdShuffleMask(ArrayRef<int> Mask, MVT VT) {  in isUnpackWdShuffleMask()  argument
9676   if (VT != MVT::v8i32 && VT != MVT::v8f32)  in isUnpackWdShuffleMask()
9829 static SDValue lowerVectorShuffleWithPSHUFB(const SDLoc &DL, MVT VT,  in lowerVectorShuffleWithPSHUFB()  argument
9836   int LaneSize = 128 / VT.getScalarSizeInBits();  in lowerVectorShuffleWithPSHUFB()
9837   const int NumBytes = VT.getSizeInBits() / 8;  in lowerVectorShuffleWithPSHUFB()
9838   const int NumEltBytes = VT.getScalarSizeInBits() / 8;  in lowerVectorShuffleWithPSHUFB()
9840   assert((Subtarget.hasSSSE3() && VT.is128BitVector()) ||  in lowerVectorShuffleWithPSHUFB()
9841          (Subtarget.hasAVX2() && VT.is256BitVector()) ||  in lowerVectorShuffleWithPSHUFB()
9842          (Subtarget.hasBWI() && VT.is512BitVector()));  in lowerVectorShuffleWithPSHUFB()
9879       VT, DAG.getNode(X86ISD::PSHUFB, DL, I8VT, DAG.getBitcast(I8VT, V),  in lowerVectorShuffleWithPSHUFB()
9888 static SDValue lowerVectorShuffleToEXPAND(const SDLoc &DL, MVT VT,  in lowerVectorShuffleToEXPAND()  argument
9899       MVT::getIntegerVT(std::max((int)VT.getVectorNumElements(), 8));  in lowerVectorShuffleToEXPAND()
9901   unsigned NumElts = VT.getVectorNumElements();  in lowerVectorShuffleToEXPAND()
9906   SDValue ZeroVector = getZeroVector(VT, Subtarget, DAG, DL);  in lowerVectorShuffleToEXPAND()
9908   return DAG.getSelect(DL, VT, VMask,  in lowerVectorShuffleToEXPAND()
9909                        DAG.getNode(X86ISD::EXPAND, DL, VT, ExpandedVector),  in lowerVectorShuffleToEXPAND()
9913 static bool matchVectorShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2,  in matchVectorShuffleWithUNPCK()  argument
9918   int NumElts = VT.getVectorNumElements();  in matchVectorShuffleWithUNPCK()
9934   createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, IsUnary);  in matchVectorShuffleWithUNPCK()
9937     V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2));  in matchVectorShuffleWithUNPCK()
9938     V1 = (Undef1 ? DAG.getUNDEF(VT) : V1);  in matchVectorShuffleWithUNPCK()
9942   createUnpackShuffleMask(VT, Unpckh, /* Lo = */ false, IsUnary);  in matchVectorShuffleWithUNPCK()
9945     V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2));  in matchVectorShuffleWithUNPCK()
9946     V1 = (Undef1 ? DAG.getUNDEF(VT) : V1);  in matchVectorShuffleWithUNPCK()
9953     if ((Subtarget.hasSSE41() || VT == MVT::v2i64 || VT == MVT::v2f64) &&  in matchVectorShuffleWithUNPCK()
9972       V2 = Zero2 ? getZeroVector(VT, Subtarget, DAG, DL) : V1;  in matchVectorShuffleWithUNPCK()
9973       V1 = Zero1 ? getZeroVector(VT, Subtarget, DAG, DL) : V1;  in matchVectorShuffleWithUNPCK()
10000 static SDValue lowerVectorShuffleWithUNPCK(const SDLoc &DL, MVT VT,  in lowerVectorShuffleWithUNPCK()  argument
10004   createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, /* Unary = */ false);  in lowerVectorShuffleWithUNPCK()
10006     return DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2);  in lowerVectorShuffleWithUNPCK()
10009   createUnpackShuffleMask(VT, Unpckh, /* Lo = */ false, /* Unary = */ false);  in lowerVectorShuffleWithUNPCK()
10011     return DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2);  in lowerVectorShuffleWithUNPCK()
10016     return DAG.getNode(X86ISD::UNPCKL, DL, VT, V2, V1);  in lowerVectorShuffleWithUNPCK()
10020     return DAG.getNode(X86ISD::UNPCKH, DL, VT, V2, V1);  in lowerVectorShuffleWithUNPCK()
10065                                            MVT VT, SDValue V1, SDValue V2,  in lowerVectorShuffleWithVPMOV()  argument
10068   if (VT != MVT::v16i8 && VT != MVT::v8i16)  in lowerVectorShuffleWithVPMOV()
10071   if (Mask.size() != VT.getVectorNumElements())  in lowerVectorShuffleWithVPMOV()
10105   if (SrcVT.getVectorElementType() == MVT::i16 && VT == MVT::v16i8 &&  in lowerVectorShuffleWithVPMOV()
10115   return DAG.getNode(X86ISD::VTRUNC, DL, VT, Src);  in lowerVectorShuffleWithVPMOV()
10120 static bool matchVectorShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1,  in matchVectorShuffleWithPACK()  argument
10125   unsigned NumElts = VT.getVectorNumElements();  in matchVectorShuffleWithPACK()
10126   unsigned BitSize = VT.getScalarSizeInBits();  in matchVectorShuffleWithPACK()
10157   createPackShuffleMask(VT, BinaryMask, false);  in matchVectorShuffleWithPACK()
10164   createPackShuffleMask(VT, UnaryMask, true);  in matchVectorShuffleWithPACK()
10172 static SDValue lowerVectorShuffleWithPACK(const SDLoc &DL, MVT VT,  in lowerVectorShuffleWithPACK()  argument
10178   if (matchVectorShuffleWithPACK(VT, PackVT, V1, V2, PackOpcode, Mask, DAG,  in lowerVectorShuffleWithPACK()
10180     return DAG.getNode(PackOpcode, DL, VT, DAG.getBitcast(PackVT, V1),  in lowerVectorShuffleWithPACK()
10190 static SDValue lowerVectorShuffleAsBitMask(const SDLoc &DL, MVT VT, SDValue V1,  in lowerVectorShuffleAsBitMask()  argument
10194   assert(!VT.isFloatingPoint() && "Floating point types are not supported");  in lowerVectorShuffleAsBitMask()
10195   MVT EltVT = VT.getVectorElementType();  in lowerVectorShuffleAsBitMask()
10215   SDValue VMask = DAG.getBuildVector(VT, DL, VMaskOps);  in lowerVectorShuffleAsBitMask()
10216   return DAG.getNode(ISD::AND, DL, VT, V, VMask);  in lowerVectorShuffleAsBitMask()
10224 static SDValue lowerVectorShuffleAsBitBlend(const SDLoc &DL, MVT VT, SDValue V1,  in lowerVectorShuffleAsBitBlend()  argument
10227   assert(VT.isInteger() && "Only supports integer vector types!");  in lowerVectorShuffleAsBitBlend()
10228   MVT EltVT = VT.getVectorElementType();  in lowerVectorShuffleAsBitBlend()
10238   SDValue V1Mask = DAG.getBuildVector(VT, DL, MaskOps);  in lowerVectorShuffleAsBitBlend()
10239   V1 = DAG.getNode(ISD::AND, DL, VT, V1, V1Mask);  in lowerVectorShuffleAsBitBlend()
10240   V2 = DAG.getNode(X86ISD::ANDNP, DL, VT, V1Mask, V2);  in lowerVectorShuffleAsBitBlend()
10241   return DAG.getNode(ISD::OR, DL, VT, V1, V2);  in lowerVectorShuffleAsBitBlend()
10308 static SDValue lowerVectorShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,  in lowerVectorShuffleAsBlend()  argument
10323     V1 = getZeroVector(VT, Subtarget, DAG, DL);  in lowerVectorShuffleAsBlend()
10325     V2 = getZeroVector(VT, Subtarget, DAG, DL);  in lowerVectorShuffleAsBlend()
10327   switch (VT.SimpleTy) {  in lowerVectorShuffleAsBlend()
10332     return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2,  in lowerVectorShuffleAsBlend()
10344       int Scale =  VT.getScalarSizeInBits() / 32;  in lowerVectorShuffleAsBlend()
10346       MVT BlendVT = VT.getSizeInBits() > 128 ? MVT::v8i32 : MVT::v4i32;  in lowerVectorShuffleAsBlend()
10350           VT, DAG.getNode(X86ISD::BLENDI, DL, BlendVT, V1, V2,  in lowerVectorShuffleAsBlend()
10357     int Scale = 8 / VT.getVectorNumElements();  in lowerVectorShuffleAsBlend()
10361     return DAG.getBitcast(VT,  in lowerVectorShuffleAsBlend()
10396     assert((VT.is128BitVector() || Subtarget.hasAVX2()) &&  in lowerVectorShuffleAsBlend()
10401             lowerVectorShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable, DAG))  in lowerVectorShuffleAsBlend()
10406           MVT::getIntegerVT(std::max((int)VT.getVectorNumElements(), 8));  in lowerVectorShuffleAsBlend()
10412     int Scale = VT.getScalarSizeInBits() / 8;  in lowerVectorShuffleAsBlend()
10416     MVT BlendVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);  in lowerVectorShuffleAsBlend()
10449         VT,  in lowerVectorShuffleAsBlend()
10460         MVT::getIntegerVT(std::max((int)VT.getVectorNumElements(), 8));  in lowerVectorShuffleAsBlend()
10474 static SDValue lowerVectorShuffleAsBlendAndPermute(const SDLoc &DL, MVT VT,  in lowerVectorShuffleAsBlendAndPermute()  argument
10500   unsigned EltSize = VT.getScalarSizeInBits();  in lowerVectorShuffleAsBlendAndPermute()
10504   SDValue V = DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask);  in lowerVectorShuffleAsBlendAndPermute()
10505   return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), PermuteMask);  in lowerVectorShuffleAsBlendAndPermute()
10513 static SDValue lowerVectorShuffleAsUNPCKAndPermute(const SDLoc &DL, MVT VT,  in lowerVectorShuffleAsUNPCKAndPermute()  argument
10518   int NumLanes = VT.getSizeInBits() / 128;  in lowerVectorShuffleAsUNPCKAndPermute()
10523   SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)};  in lowerVectorShuffleAsUNPCKAndPermute()
10570   SDValue Unpck = DAG.getNode(UnpckOp, DL, VT, Ops);  in lowerVectorShuffleAsUNPCKAndPermute()
10571   return DAG.getVectorShuffle(VT, DL, Unpck, DAG.getUNDEF(VT), PermuteMask);  in lowerVectorShuffleAsUNPCKAndPermute()
10577     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,  in lowerVectorShuffleAsByteRotateAndPermute()  argument
10579   if ((VT.is128BitVector() && !Subtarget.hasSSSE3()) ||  in lowerVectorShuffleAsByteRotateAndPermute()
10580       (VT.is256BitVector() && !Subtarget.hasAVX2()) ||  in lowerVectorShuffleAsByteRotateAndPermute()
10581       (VT.is512BitVector() && !Subtarget.hasBWI()))  in lowerVectorShuffleAsByteRotateAndPermute()
10585   if (is128BitLaneCrossingShuffleMask(VT, Mask))  in lowerVectorShuffleAsByteRotateAndPermute()
10588   int Scale = VT.getScalarSizeInBits() / 8;  in lowerVectorShuffleAsByteRotateAndPermute()
10589   int NumLanes = VT.getSizeInBits() / 128;  in lowerVectorShuffleAsByteRotateAndPermute()
10590   int NumElts = VT.getVectorNumElements();  in lowerVectorShuffleAsByteRotateAndPermute()
10627   if (VT.getSizeInBits() > 128 && (Blend1 || Blend2))  in lowerVectorShuffleAsByteRotateAndPermute()
10632     MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);  in lowerVectorShuffleAsByteRotateAndPermute()
10634         VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, DAG.getBitcast(ByteVT, Hi),  in lowerVectorShuffleAsByteRotateAndPermute()
10649     return DAG.getVectorShuffle(VT, DL, Rotate, DAG.getUNDEF(VT), PermMask);  in lowerVectorShuffleAsByteRotateAndPermute()
10668     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,  in lowerVectorShuffleAsDecomposedShuffleBlend()  argument
10692             DL, VT, V1, V2, Mask, DAG, true))  in lowerVectorShuffleAsDecomposedShuffleBlend()
10695             lowerVectorShuffleAsUNPCKAndPermute(DL, VT, V1, V2, Mask, DAG))  in lowerVectorShuffleAsDecomposedShuffleBlend()
10698             DL, VT, V1, V2, Mask, Subtarget, DAG))  in lowerVectorShuffleAsDecomposedShuffleBlend()
10702             lowerVectorShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask, DAG))  in lowerVectorShuffleAsDecomposedShuffleBlend()
10706   V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask);  in lowerVectorShuffleAsDecomposedShuffleBlend()
10707   V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);  in lowerVectorShuffleAsDecomposedShuffleBlend()
10708   return DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask);  in lowerVectorShuffleAsDecomposedShuffleBlend()
10799 static int matchVectorShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,  in matchVectorShuffleAsByteRotate()  argument
10807   if (!is128BitLaneRepeatedShuffleMask(VT, Mask, RepeatedMask))  in matchVectorShuffleAsByteRotate()
10821 static SDValue lowerVectorShuffleAsByteRotate(const SDLoc &DL, MVT VT,  in lowerVectorShuffleAsByteRotate()  argument
10829   int ByteRotation = matchVectorShuffleAsByteRotate(VT, Lo, Hi, Mask);  in lowerVectorShuffleAsByteRotate()
10835   MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);  in lowerVectorShuffleAsByteRotate()
10841     assert((!VT.is512BitVector() || Subtarget.hasBWI()) &&  in lowerVectorShuffleAsByteRotate()
10844         VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, Lo, Hi,  in lowerVectorShuffleAsByteRotate()
10848   assert(VT.is128BitVector() &&  in lowerVectorShuffleAsByteRotate()
10863   return DAG.getBitcast(VT,  in lowerVectorShuffleAsByteRotate()
10877 static SDValue lowerVectorShuffleAsRotate(const SDLoc &DL, MVT VT,  in lowerVectorShuffleAsRotate()  argument
10882   assert((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) &&  in lowerVectorShuffleAsRotate()
10886   assert((Subtarget.hasVLX() || (!VT.is128BitVector() && !VT.is256BitVector()))  in lowerVectorShuffleAsRotate()
10894   return DAG.getNode(X86ISD::VALIGN, DL, VT, Lo, Hi,  in lowerVectorShuffleAsRotate()
10984 static SDValue lowerVectorShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1,  in lowerVectorShuffleAsShift()  argument
10990   assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");  in lowerVectorShuffleAsShift()
10998       ShiftVT, Opcode, VT.getScalarSizeInBits(), Mask, 0, Zeroable, Subtarget);  in lowerVectorShuffleAsShift()
11003         matchVectorShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(),  in lowerVectorShuffleAsShift()
11016   return DAG.getBitcast(VT, V);  in lowerVectorShuffleAsShift()
11021 static bool matchVectorShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2,  in matchVectorShuffleAsEXTRQ()  argument
11026   assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");  in matchVectorShuffleAsEXTRQ()
11068   BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f;  in matchVectorShuffleAsEXTRQ()
11069   BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f;  in matchVectorShuffleAsEXTRQ()
11077 static bool matchVectorShuffleAsINSERTQ(MVT VT, SDValue &V1, SDValue &V2,  in matchVectorShuffleAsINSERTQ()  argument
11082   assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");  in matchVectorShuffleAsINSERTQ()
11131       BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f;  in matchVectorShuffleAsINSERTQ()
11132       BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f;  in matchVectorShuffleAsINSERTQ()
11143 static SDValue lowerVectorShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1,  in lowerVectorShuffleWithSSE4A()  argument
11148   if (matchVectorShuffleAsEXTRQ(VT, V1, V2, Mask, BitLen, BitIdx, Zeroable))  in lowerVectorShuffleWithSSE4A()
11149     return DAG.getNode(X86ISD::EXTRQI, DL, VT, V1,  in lowerVectorShuffleWithSSE4A()
11153   if (matchVectorShuffleAsINSERTQ(VT, V1, V2, Mask, BitLen, BitIdx))  in lowerVectorShuffleWithSSE4A()
11154     return DAG.getNode(X86ISD::INSERTQI, DL, VT, V1 ? V1 : DAG.getUNDEF(VT),  in lowerVectorShuffleWithSSE4A()
11155                        V2 ? V2 : DAG.getUNDEF(VT),  in lowerVectorShuffleWithSSE4A()
11172     const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV,  in lowerVectorShuffleAsSpecificZeroOrAnyExtend()  argument
11175   int EltBits = VT.getScalarSizeInBits();  in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11176   int NumElements = VT.getVectorNumElements();  in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11201     return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), ShMask);  in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11209     if (Offset && Scale == 2 && VT.is128BitVector())  in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11215     return DAG.getBitcast(VT, InputV);  in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11218   assert(VT.is128BitVector() && "Only 128-bit vectors can be extended.");  in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11226         VT, DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,  in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11239         VT, DAG.getNode(OddEvenOp, DL, MVT::v8i16,  in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11248     assert(VT.is128BitVector() && "Unexpected vector width!");  in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11252         MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,  in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11258       return DAG.getBitcast(VT, Lo);  in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11262         MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,  in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11265     return DAG.getBitcast(VT,  in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11282         VT, DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV,  in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11293     InputV = DAG.getVectorShuffle(VT, DL, InputV, DAG.getUNDEF(VT), ShMask);  in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11314   return DAG.getBitcast(VT, InputV);  in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11330     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,  in lowerVectorShuffleAsZeroOrAnyExtend()  argument
11333   int Bits = VT.getSizeInBits();  in lowerVectorShuffleAsZeroOrAnyExtend()
11335   int NumElements = VT.getVectorNumElements();  in lowerVectorShuffleAsZeroOrAnyExtend()
11337   assert(VT.getScalarSizeInBits() <= 32 &&  in lowerVectorShuffleAsZeroOrAnyExtend()
11401         DL, VT, Scale, Offset, AnyExt, InputV, Mask, Subtarget, DAG);  in lowerVectorShuffleAsZeroOrAnyExtend()
11438     return DAG.getBitcast(VT, V);  in lowerVectorShuffleAsZeroOrAnyExtend()
11450   MVT VT = V.getSimpleValueType();  in getScalarValueForVectorElement()  local
11451   MVT EltVT = VT.getVectorElementType();  in getScalarValueForVectorElement()
11457   if (!NewVT.isVector() || NewVT.getScalarSizeInBits() != VT.getScalarSizeInBits())  in getScalarValueForVectorElement()
11486     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,  in lowerVectorShuffleAsElementInsertion()  argument
11489   MVT ExtVT = VT;  in lowerVectorShuffleAsElementInsertion()
11490   MVT EltVT = VT.getVectorElementType();  in lowerVectorShuffleAsElementInsertion()
11534     assert(VT == ExtVT && "Cannot change extended type when non-zeroable!");  in lowerVectorShuffleAsElementInsertion()
11535     if (!VT.isFloatingPoint() || V2Index != 0)  in lowerVectorShuffleAsElementInsertion()
11541     if (!VT.is128BitVector())  in lowerVectorShuffleAsElementInsertion()
11552   if (VT.isFloatingPoint() && V2Index != 0)  in lowerVectorShuffleAsElementInsertion()
11556   if (ExtVT != VT)  in lowerVectorShuffleAsElementInsertion()
11557     V2 = DAG.getBitcast(VT, V2);  in lowerVectorShuffleAsElementInsertion()
11564     if (VT.isFloatingPoint() || VT.getVectorNumElements() <= 4) {  in lowerVectorShuffleAsElementInsertion()
11567       V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Shuffle);  in lowerVectorShuffleAsElementInsertion()
11573       V2 = DAG.getBitcast(VT, V2);  in lowerVectorShuffleAsElementInsertion()
11583 static SDValue lowerVectorShuffleAsTruncBroadcast(const SDLoc &DL, MVT VT,  in lowerVectorShuffleAsTruncBroadcast()  argument
11590   EVT EltVT = VT.getVectorElementType();  in lowerVectorShuffleAsTruncBroadcast()
11593   assert(VT.isInteger() && "Unexpected non-integer trunc broadcast!");  in lowerVectorShuffleAsTruncBroadcast()
11628   return DAG.getNode(X86ISD::VBROADCAST, DL, VT,  in lowerVectorShuffleAsTruncBroadcast()
11637 static SDValue lowerVectorShuffleAsBroadcast(const SDLoc &DL, MVT VT,  in lowerVectorShuffleAsBroadcast()  argument
11642   if (!((Subtarget.hasSSE3() && VT == MVT::v2f64) ||  in lowerVectorShuffleAsBroadcast()
11643         (Subtarget.hasAVX() && VT.isFloatingPoint()) ||  in lowerVectorShuffleAsBroadcast()
11644         (Subtarget.hasAVX2() && VT.isInteger())))  in lowerVectorShuffleAsBroadcast()
11650   unsigned Opcode = (VT == MVT::v2f64 && !Subtarget.hasAVX2())  in lowerVectorShuffleAsBroadcast()
11720   if (VT.getScalarSizeInBits() != V.getScalarValueSizeInBits()) {  in lowerVectorShuffleAsBroadcast()
11721     unsigned NumEltBits = VT.getScalarSizeInBits();  in lowerVectorShuffleAsBroadcast()
11732     MVT SrcVT = MVT::getVectorVT(VT.getScalarType(), NumSrcElts);  in lowerVectorShuffleAsBroadcast()
11741   if (V.getOpcode() == ISD::BITCAST && VT.isInteger())  in lowerVectorShuffleAsBroadcast()
11743             DL, VT, V.getOperand(0), BroadcastIdx, Subtarget, DAG))  in lowerVectorShuffleAsBroadcast()
11746   MVT BroadcastVT = VT;  in lowerVectorShuffleAsBroadcast()
11761     if (!Subtarget.is64Bit() && VT.getScalarType() == MVT::i64) {  in lowerVectorShuffleAsBroadcast()
11762       BroadcastVT = MVT::getVectorVT(MVT::f64, VT.getVectorNumElements());  in lowerVectorShuffleAsBroadcast()
11786     if (!VT.is256BitVector() && !VT.is512BitVector())  in lowerVectorShuffleAsBroadcast()
11790     if (VT == MVT::v4f64 || VT == MVT::v4i64)  in lowerVectorShuffleAsBroadcast()
11794     unsigned EltSize = VT.getScalarSizeInBits();  in lowerVectorShuffleAsBroadcast()
11843   return DAG.getBitcast(VT, DAG.getNode(Opcode, DL, BroadcastVT, V));  in lowerVectorShuffleAsBroadcast()
11968     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,  in lowerVectorShuffleAsPermuteAndUnpack()  argument
11970   assert(!VT.isFloatingPoint() &&  in lowerVectorShuffleAsPermuteAndUnpack()
11972   assert(VT.is128BitVector() &&  in lowerVectorShuffleAsPermuteAndUnpack()
12017     V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask);  in lowerVectorShuffleAsPermuteAndUnpack()
12018     V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);  in lowerVectorShuffleAsPermuteAndUnpack()
12027         VT, DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,  in lowerVectorShuffleAsPermuteAndUnpack()
12033   int OrigScalarSize = VT.getScalarSizeInBits();  in lowerVectorShuffleAsPermuteAndUnpack()
12067         VT, DL, DAG.getNode(NumLoInputs == 0 ? X86ISD::UNPCKH : X86ISD::UNPCKL,  in lowerVectorShuffleAsPermuteAndUnpack()
12068                             DL, VT, V1, V2),  in lowerVectorShuffleAsPermuteAndUnpack()
12069         DAG.getUNDEF(VT), PermMask);  in lowerVectorShuffleAsPermuteAndUnpack()
12282 static SDValue lowerVectorShuffleWithSHUFPS(const SDLoc &DL, MVT VT,  in lowerVectorShuffleWithSHUFPS()  argument
12309       V2 = DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1,  in lowerVectorShuffleWithSHUFPS()
12348       V1 = DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,  in lowerVectorShuffleWithSHUFPS()
12360   return DAG.getNode(X86ISD::SHUFP, DL, VT, LowV, HighV,  in lowerVectorShuffleWithSHUFPS()
12589     const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef<int> Mask,  in lowerV8I16GeneralSingleInputVectorShuffle()  argument
12591   assert(VT.getVectorElementType() == MVT::i16 && "Bad input type!");  in lowerV8I16GeneralSingleInputVectorShuffle()
12592   MVT PSHUFDVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2);  in lowerV8I16GeneralSingleInputVectorShuffle()
12601     return DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,  in lowerV8I16GeneralSingleInputVectorShuffle()
12608     return DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,  in lowerV8I16GeneralSingleInputVectorShuffle()
12636     V = DAG.getNode(ShufWOp, DL, VT, V,  in lowerV8I16GeneralSingleInputVectorShuffle()
12641     return DAG.getBitcast(VT, V);  in lowerV8I16GeneralSingleInputVectorShuffle()
12815         VT,  in lowerV8I16GeneralSingleInputVectorShuffle()
12828     return lowerV8I16GeneralSingleInputVectorShuffle(DL, VT, V, Mask, Subtarget,  in lowerV8I16GeneralSingleInputVectorShuffle()
13051     V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,  in lowerV8I16GeneralSingleInputVectorShuffle()
13054     V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,  in lowerV8I16GeneralSingleInputVectorShuffle()
13058         VT,  in lowerV8I16GeneralSingleInputVectorShuffle()
13071     V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,  in lowerV8I16GeneralSingleInputVectorShuffle()
13079     V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,  in lowerV8I16GeneralSingleInputVectorShuffle()
13088     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,  in lowerVectorShuffleAsBlendOfPSHUFBs()  argument
13090   assert(!is128BitLaneCrossingShuffleMask(VT, Mask) &&  in lowerVectorShuffleAsBlendOfPSHUFBs()
13093   int NumBytes = VT.getSizeInBits() / 8;  in lowerVectorShuffleAsBlendOfPSHUFBs()
13135   return DAG.getBitcast(VT, V);  in lowerVectorShuffleAsBlendOfPSHUFBs()
13337 static SDValue lowerVectorShuffleWithPERMV(const SDLoc &DL, MVT VT,  in lowerVectorShuffleWithPERMV()  argument
13340   MVT MaskEltVT = MVT::getIntegerVT(VT.getScalarSizeInBits());  in lowerVectorShuffleWithPERMV()
13341   MVT MaskVecVT = MVT::getVectorVT(MaskEltVT, VT.getVectorNumElements());  in lowerVectorShuffleWithPERMV()
13345     return DAG.getNode(X86ISD::VPERMV, DL, VT, MaskNode, V1);  in lowerVectorShuffleWithPERMV()
13347   return DAG.getNode(X86ISD::VPERMV3, DL, VT, V1, MaskNode, V2);  in lowerVectorShuffleWithPERMV()
13665                                         MVT VT, SDValue V1, SDValue V2,  in lower128BitVectorShuffle()  argument
13669   switch (VT.SimpleTy) {  in lower128BitVectorShuffle()
13693 static SDValue splitAndLowerVectorShuffle(const SDLoc &DL, MVT VT, SDValue V1,  in splitAndLowerVectorShuffle()  argument
13696   assert(VT.getSizeInBits() >= 256 &&  in splitAndLowerVectorShuffle()
13698   assert(V1.getSimpleValueType() == VT && "Bad operand type!");  in splitAndLowerVectorShuffle()
13699   assert(V2.getSimpleValueType() == VT && "Bad operand type!");  in splitAndLowerVectorShuffle()
13704   int NumElements = VT.getVectorNumElements();  in splitAndLowerVectorShuffle()
13706   MVT ScalarVT = VT.getVectorElementType();  in splitAndLowerVectorShuffle()
13808   return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);  in splitAndLowerVectorShuffle()
13819 static SDValue lowerVectorShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT,  in lowerVectorShuffleAsSplitOrBlend()  argument
13848     return lowerVectorShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask,  in lowerVectorShuffleAsSplitOrBlend()
13854   int LaneCount = VT.getSizeInBits() / 128;  in lowerVectorShuffleAsSplitOrBlend()
13863     return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG);  in lowerVectorShuffleAsSplitOrBlend()
13867   return lowerVectorShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask,  in lowerVectorShuffleAsSplitOrBlend()
13880     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,  in lowerVectorShuffleAsLanePermuteAndPermute()  argument
13882   int NumElts = VT.getVectorNumElements();  in lowerVectorShuffleAsLanePermuteAndPermute()
13883   int NumLanes = VT.getSizeInBits() / 128;  in lowerVectorShuffleAsLanePermuteAndPermute()
13930   SDValue LanePermute = DAG.getVectorShuffle(VT, DL, V1, V2, LaneMask);  in lowerVectorShuffleAsLanePermuteAndPermute()
13931   return DAG.getVectorShuffle(VT, DL, LanePermute, DAG.getUNDEF(VT), PermMask);  in lowerVectorShuffleAsLanePermuteAndPermute()
13943 static SDValue lowerVectorShuffleAsLanePermuteAndBlend(const SDLoc &DL, MVT VT,  in lowerVectorShuffleAsLanePermuteAndBlend()  argument
13949   assert(VT.is256BitVector() && "Only for 256-bit vector shuffles!");  in lowerVectorShuffleAsLanePermuteAndBlend()
13962       return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG);  in lowerVectorShuffleAsLanePermuteAndBlend()
13969       return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG);  in lowerVectorShuffleAsLanePermuteAndBlend()
13984   MVT PVT = VT.isFloatingPoint() ? MVT::v4f64 : MVT::v4i64;  in lowerVectorShuffleAsLanePermuteAndBlend()
13988   Flipped = DAG.getBitcast(VT, Flipped);  in lowerVectorShuffleAsLanePermuteAndBlend()
13989   return DAG.getVectorShuffle(VT, DL, V1, Flipped, FlippedBlendMask);  in lowerVectorShuffleAsLanePermuteAndBlend()
13993 static SDValue lowerV2X128VectorShuffle(const SDLoc &DL, MVT VT, SDValue V1,  in lowerV2X128VectorShuffle()  argument
14011     MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2);  in lowerV2X128VectorShuffle()
14014     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,  in lowerV2X128VectorShuffle()
14015                        getZeroVector(VT, Subtarget, DAG, DL), LoV,  in lowerV2X128VectorShuffle()
14024   if (SDValue Blend = lowerVectorShuffleAsBlend(DL, VT, V1, V2, Mask,  in lowerV2X128VectorShuffle()
14039         MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2);  in lowerV2X128VectorShuffle()
14043         return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec,  in lowerV2X128VectorShuffle()
14053       return DAG.getNode(X86ISD::SHUF128, DL, VT, V1, V2,  in lowerV2X128VectorShuffle()
14081     V1 = DAG.getUNDEF(VT);  in lowerV2X128VectorShuffle()
14083     V2 = DAG.getUNDEF(VT);  in lowerV2X128VectorShuffle()
14085   return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2,  in lowerV2X128VectorShuffle()
14099     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,  in lowerVectorShuffleByMerging128BitLanes()  argument
14103   if (is128BitLaneRepeatedShuffleMask(VT, Mask))  in lowerVectorShuffleByMerging128BitLanes()
14107   int LaneSize = 128 / VT.getScalarSizeInBits();  in lowerVectorShuffleByMerging128BitLanes()
14227   SDValue NewV1 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);  in lowerVectorShuffleByMerging128BitLanes()
14244   SDValue NewV2 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);  in lowerVectorShuffleByMerging128BitLanes()
14259   return DAG.getVectorShuffle(VT, DL, NewV1, NewV2, NewMask);  in lowerVectorShuffleByMerging128BitLanes()
14265 static SDValue lowerVectorShuffleWithUndefHalf(const SDLoc &DL, MVT VT,  in lowerVectorShuffleWithUndefHalf()  argument
14270   assert((VT.is256BitVector() || VT.is512BitVector()) &&  in lowerVectorShuffleWithUndefHalf()
14273   unsigned NumElts = VT.getVectorNumElements();  in lowerVectorShuffleWithUndefHalf()
14275   MVT HalfVT = MVT::getVectorVT(VT.getVectorElementType(), HalfNumElts);  in lowerVectorShuffleWithUndefHalf()
14288     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi,  in lowerVectorShuffleWithUndefHalf()
14298     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi,  in lowerVectorShuffleWithUndefHalf()
14357     if (VT == MVT::v4f64 || VT == MVT::v4i64)  in lowerVectorShuffleWithUndefHalf()
14360     if (VT == MVT::v8f32 || VT == MVT::v8i32) {  in lowerVectorShuffleWithUndefHalf()
14368   if (VT.is512BitVector() && !(UndefUpper && NumUpperHalves == 0))  in lowerVectorShuffleWithUndefHalf()
14383   return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V,  in lowerVectorShuffleWithUndefHalf()
14407     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,  in lowerShuffleAsRepeatedMaskAndLanePermute()  argument
14409   int NumElts = VT.getVectorNumElements();  in lowerShuffleAsRepeatedMaskAndLanePermute()
14410   int NumLanes = VT.getSizeInBits() / 128;  in lowerShuffleAsRepeatedMaskAndLanePermute()
14417       if (BroadcastSize <= VT.getScalarSizeInBits())  in lowerShuffleAsRepeatedMaskAndLanePermute()
14419       int NumBroadcastElts = BroadcastSize / VT.getScalarSizeInBits();  in lowerShuffleAsRepeatedMaskAndLanePermute()
14445       SDValue RepeatShuf = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatMask);  in lowerShuffleAsRepeatedMaskAndLanePermute()
14452       return DAG.getVectorShuffle(VT, DL, RepeatShuf, DAG.getUNDEF(VT),  in lowerShuffleAsRepeatedMaskAndLanePermute()
14458   if (!is128BitLaneCrossingShuffleMask(VT, Mask))  in lowerShuffleAsRepeatedMaskAndLanePermute()
14463   if (is128BitLaneRepeatedShuffleMask(VT, Mask, RepeatedShuffleMask))  in lowerShuffleAsRepeatedMaskAndLanePermute()
14468   int SubLaneScale = Subtarget.hasAVX2() && VT.is256BitVector() ? 2 : 1;  in lowerShuffleAsRepeatedMaskAndLanePermute()
14556   SDValue RepeatedShuffle = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatedMask);  in lowerShuffleAsRepeatedMaskAndLanePermute()
14568   return DAG.getVectorShuffle(VT, DL, RepeatedShuffle, DAG.getUNDEF(VT),  in lowerShuffleAsRepeatedMaskAndLanePermute()
14572 static bool matchVectorShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2,  in matchVectorShuffleWithSHUFPD()  argument
14575   int NumElts = VT.getVectorNumElements();  in matchVectorShuffleWithSHUFPD()
14576   assert(VT.getScalarSizeInBits() == 64 &&  in matchVectorShuffleWithSHUFPD()
14609 static SDValue lowerVectorShuffleWithSHUFPD(const SDLoc &DL, MVT VT,  in lowerVectorShuffleWithSHUFPD()  argument
14612   assert((VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v8f64)&&  in lowerVectorShuffleWithSHUFPD()
14616   if (!matchVectorShuffleWithSHUFPD(VT, V1, V2, Immediate, Mask))  in lowerVectorShuffleWithSHUFPD()
14619   return DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,  in lowerVectorShuffleWithSHUFPD()
15233                                         MVT VT, SDValue V1, SDValue V2,  in lower256BitVectorShuffle()  argument
15239   int NumElts = VT.getVectorNumElements();  in lower256BitVectorShuffle()
15244             DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))  in lower256BitVectorShuffle()
15249           lowerVectorShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG))  in lower256BitVectorShuffle()
15258   if (VT.isInteger() && !Subtarget.hasAVX2()) {  in lower256BitVectorShuffle()
15259     int ElementBits = VT.getScalarSizeInBits();  in lower256BitVectorShuffle()
15264               lowerVectorShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable, DAG))  in lower256BitVectorShuffle()
15266       if (SDValue V = lowerVectorShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))  in lower256BitVectorShuffle()
15268       return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG);  in lower256BitVectorShuffle()
15272                                 VT.getVectorNumElements());  in lower256BitVectorShuffle()
15275     return DAG.getBitcast(VT, DAG.getVectorShuffle(FpVT, DL, V1, V2, Mask));  in lower256BitVectorShuffle()
15278   switch (VT.SimpleTy) {  in lower256BitVectorShuffle()
15298 static SDValue lowerV4X128VectorShuffle(const SDLoc &DL, MVT VT,  in lowerV4X128VectorShuffle()  argument
15304   assert(VT.getScalarSizeInBits() == 64 &&  in lowerV4X128VectorShuffle()
15309   assert(VT.is512BitVector() && "Unexpected vector size for 512bit shuffle.");  in lowerV4X128VectorShuffle()
15320     MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);  in lowerV4X128VectorShuffle()
15323     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,  in lowerV4X128VectorShuffle()
15324                        getZeroVector(VT, Subtarget, DAG, DL), LoV,  in lowerV4X128VectorShuffle()
15334     MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 4);  in lowerV4X128VectorShuffle()
15338     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec,  in lowerV4X128VectorShuffle()
15368     MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2);  in lowerV4X128VectorShuffle()
15375   SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)};  in lowerV4X128VectorShuffle()
15395   return DAG.getNode(X86ISD::SHUF128, DL, VT, Ops[0], Ops[1],  in lowerV4X128VectorShuffle()
15769                                         MVT VT, SDValue V1, SDValue V2,  in lower512BitVectorShuffle()  argument
15783             DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))  in lower512BitVectorShuffle()
15788         lowerVectorShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG))  in lower512BitVectorShuffle()
15793           lowerVectorShuffleAsBroadcast(DL, VT, V1, V2, Mask, Subtarget, DAG))  in lower512BitVectorShuffle()
15800   switch (VT.SimpleTy) {  in lower512BitVectorShuffle()
15857                                       MVT VT, SDValue V1, SDValue V2,  in lower1BitVectorShuffle()  argument
15885     return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,  in lower1BitVectorShuffle()
15886                        getZeroVector(VT, Subtarget, DAG, DL),  in lower1BitVectorShuffle()
15898         return DAG.getNode(Opcode, DL, VT, V,  in lower1BitVectorShuffle()
15906   switch (VT.SimpleTy) {  in lower1BitVectorShuffle()
15941   int NumElems = VT.getVectorNumElements();  in lower1BitVectorShuffle()
15944     return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, ExtVT),  in lower1BitVectorShuffle()
15947   return DAG.getNode(ISD::TRUNCATE, DL, VT, Shuffle);  in lower1BitVectorShuffle()
16027   MVT VT = Op.getSimpleValueType();  in lowerVectorShuffle()  local
16028   int NumElements = VT.getVectorNumElements();  in lowerVectorShuffle()
16030   bool Is1BitVector = (VT.getVectorElementType() == MVT::i1);  in lowerVectorShuffle()
16032   assert((VT.getSizeInBits() != 64 || Is1BitVector) &&  in lowerVectorShuffle()
16038     return DAG.getUNDEF(VT);  in lowerVectorShuffle()
16056         return DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);  in lowerVectorShuffle()
16070     return getZeroVector(VT, Subtarget, DAG, DL);  in lowerVectorShuffle()
16089   if (VT.getScalarSizeInBits() < 64 && !Is1BitVector &&  in lowerVectorShuffle()
16096             lowerVectorShuffleAsBroadcast(DL, VT, V1, V2, Mask, Subtarget, DAG))  in lowerVectorShuffle()
16099     MVT NewEltVT = VT.isFloatingPoint()  in lowerVectorShuffle()
16100                        ? MVT::getFloatingPointVT(VT.getScalarSizeInBits() * 2)  in lowerVectorShuffle()
16101                        : MVT::getIntegerVT(VT.getScalarSizeInBits() * 2);  in lowerVectorShuffle()
16126           VT, DAG.getVectorShuffle(NewVT, DL, V1, V2, WidenedMask));  in lowerVectorShuffle()
16135           lowerVectorShuffleWithVPMOV(DL, Mask, VT, V1, V2, DAG, Subtarget))  in lowerVectorShuffle()
16139   if (VT.is128BitVector())  in lowerVectorShuffle()
16140     return lower128BitVectorShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget,  in lowerVectorShuffle()
16143   if (VT.is256BitVector())  in lowerVectorShuffle()
16144     return lower256BitVectorShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget,  in lowerVectorShuffle()
16147   if (VT.is512BitVector())  in lowerVectorShuffle()
16148     return lower512BitVectorShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget,  in lowerVectorShuffle()
16152     return lower1BitVectorShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget,  in lowerVectorShuffle()
16165   MVT VT = Op.getSimpleValueType();  in lowerVSELECTtoVectorShuffle()  local
16171     return DAG.getVectorShuffle(VT, SDLoc(Op), LHS, RHS, Mask);  in lowerVSELECTtoVectorShuffle()
16205   MVT VT = Op.getSimpleValueType();  in LowerVSELECT()  local
16206   unsigned EltSize = VT.getScalarSizeInBits();  in LowerVSELECT()
16207   unsigned NumElts = VT.getVectorNumElements();  in LowerVSELECT()
16212   if (VT.getSizeInBits() == 512) {  in LowerVSELECT()
16219     return DAG.getSelect(dl, VT, Mask, LHS, RHS);  in LowerVSELECT()
16231     return DAG.getNode(ISD::VSELECT, dl, VT, Cond, LHS, RHS);  in LowerVSELECT()
16237   switch (VT.SimpleTy) {  in LowerVSELECT()
16257     return DAG.getBitcast(VT, Select);  in LowerVSELECT()
16263   MVT VT = Op.getSimpleValueType();  in LowerEXTRACT_VECTOR_ELT_SSE4()  local
16269   if (VT.getSizeInBits() == 8) {  in LowerEXTRACT_VECTOR_ELT_SSE4()
16272     return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract);  in LowerEXTRACT_VECTOR_ELT_SSE4()
16275   if (VT == MVT::f32) {  in LowerEXTRACT_VECTOR_ELT_SSE4()
16295   if (VT == MVT::i32 || VT == MVT::i64) {  in LowerEXTRACT_VECTOR_ELT_SSE4()
16418   MVT VT = Op.getSimpleValueType();  in LowerEXTRACT_VECTOR_ELT()  local
16420   if (VT.getSizeInBits() == 16) {  in LowerEXTRACT_VECTOR_ELT()
16432     return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract);  in LowerEXTRACT_VECTOR_ELT()
16442   if (VT.getSizeInBits() == 8 && Op->isOnlyUserOf(Vec.getNode())) {  in LowerEXTRACT_VECTOR_ELT()
16453       return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);  in LowerEXTRACT_VECTOR_ELT()
16464     return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);  in LowerEXTRACT_VECTOR_ELT()
16467   if (VT.getSizeInBits() == 32) {  in LowerEXTRACT_VECTOR_ELT()
16474     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,  in LowerEXTRACT_VECTOR_ELT()
16478   if (VT.getSizeInBits() == 64) {  in LowerEXTRACT_VECTOR_ELT()
16490     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,  in LowerEXTRACT_VECTOR_ELT()
16528   MVT VT = Op.getSimpleValueType();  in LowerINSERT_VECTOR_ELT()  local
16529   MVT EltVT = VT.getVectorElementType();  in LowerINSERT_VECTOR_ELT()
16530   unsigned NumElts = VT.getVectorNumElements();  in LowerINSERT_VECTOR_ELT()
16545   bool IsAllOnesElt = VT.isInteger() && llvm::isAllOnesConstant(N1);  in LowerINSERT_VECTOR_ELT()
16555     SDValue CstVector = IsZeroElt ? getZeroVector(VT, Subtarget, DAG, dl)  in LowerINSERT_VECTOR_ELT()
16556                                   : getOnesVector(VT, DAG, dl);  in LowerINSERT_VECTOR_ELT()
16557     return DAG.getVectorShuffle(VT, dl, N0, CstVector, BlendMask);  in LowerINSERT_VECTOR_ELT()
16562   if (VT.is256BitVector() || VT.is512BitVector()) {  in LowerINSERT_VECTOR_ELT()
16565     if (VT.is256BitVector() && IdxVal == 0) {  in LowerINSERT_VECTOR_ELT()
16571         SDValue N1Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1);  in LowerINSERT_VECTOR_ELT()
16573         return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1Vec, N2);  in LowerINSERT_VECTOR_ELT()
16592   assert(VT.is128BitVector() && "Only 128-bit vector types should be left!");  in LowerINSERT_VECTOR_ELT()
16596   if (VT == MVT::v8i16 || (VT == MVT::v16i8 && Subtarget.hasSSE41())) {  in LowerINSERT_VECTOR_ELT()
16598     if (VT == MVT::v8i16) {  in LowerINSERT_VECTOR_ELT()
16602       assert(VT == MVT::v16i8 && "PINSRB requires v16i8 vector");  in LowerINSERT_VECTOR_ELT()
16611     return DAG.getNode(Opc, dl, VT, N0, N1, N2);  in LowerINSERT_VECTOR_ELT()
16636         return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1, N2);  in LowerINSERT_VECTOR_ELT()
16641       return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, N2);  in LowerINSERT_VECTOR_ELT()
17200   MVT VT = Op.getSimpleValueType();  in LowerShiftParts()  local
17201   unsigned VTBits = VT.getSizeInBits();  in LowerShiftParts()
17212   SDValue Tmp1 = isSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,  in LowerShiftParts()
17214                        : DAG.getConstant(0, dl, VT);  in LowerShiftParts()
17218     Tmp2 = DAG.getNode(ISD::FSHL, dl, VT, ShOpHi, ShOpLo, ShAmt);  in LowerShiftParts()
17219     Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt);  in LowerShiftParts()
17221     Tmp2 = DAG.getNode(ISD::FSHR, dl, VT, ShOpHi, ShOpLo, ShAmt);  in LowerShiftParts()
17222     Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt);  in LowerShiftParts()
17235     Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);  in LowerShiftParts()
17236     Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);  in LowerShiftParts()
17238     Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);  in LowerShiftParts()
17239     Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);  in LowerShiftParts()
17247   MVT VT = Op.getSimpleValueType();  in LowerFunnelShift()  local
17258   if (VT.isVector()) {  in LowerFunnelShift()
17267       return DAG.getNode(IsFSHR ? X86ISD::VSHRD : X86ISD::VSHLD, DL, VT,  in LowerFunnelShift()
17271     return DAG.getNode(IsFSHR ? X86ISD::VSHRDV : X86ISD::VSHLDV, DL, VT,  in LowerFunnelShift()
17275   assert((VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) &&  in LowerFunnelShift()
17287   if (VT == MVT::i16)  in LowerFunnelShift()
17292   return DAG.getNode(SHDOp, DL, VT, Op0, Op1, Amt);  in LowerFunnelShift()
17303   MVT VT = Op.getSimpleValueType();  in LowerI64IntToFP_AVX512DQ()  local
17306        (VT != MVT::f32 && VT != MVT::f64))  in LowerI64IntToFP_AVX512DQ()
17314   MVT VecVT = MVT::getVectorVT(VT, NumElts);  in LowerI64IntToFP_AVX512DQ()
17319   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,  in LowerI64IntToFP_AVX512DQ()
17327   MVT VT = Op.getSimpleValueType();  in LowerSINT_TO_FP()  local
17331     if (SrcVT == MVT::v2i32 && VT == MVT::v2f64) {  in LowerSINT_TO_FP()
17332       return DAG.getNode(X86ISD::CVTSI2P, dl, VT,  in LowerSINT_TO_FP()
17344   if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(VT))  in LowerSINT_TO_FP()
17346   if (SrcVT == MVT::i64 && isScalarFPTypeInSSEReg(VT) && Subtarget.is64Bit())  in LowerSINT_TO_FP()
17353   if (SrcVT == MVT::i64 && isScalarFPTypeInSSEReg(VT) &&  in LowerSINT_TO_FP()
17965   MVT VT = Op->getSimpleValueType(0);  in LowerAVXExtend()  local
17970   assert(VT.isVector() && InVT.isVector() && "Expected vector type");  in LowerAVXExtend()
17971   assert(VT.getVectorNumElements() == VT.getVectorNumElements() &&  in LowerAVXExtend()
17973   assert((VT.getVectorElementType() == MVT::i16 ||  in LowerAVXExtend()
17974           VT.getVectorElementType() == MVT::i32 ||  in LowerAVXExtend()
17975           VT.getVectorElementType() == MVT::i64) &&  in LowerAVXExtend()
17984     if (!ExperimentalVectorWideningLegalization || VT != MVT::v8i64)  in LowerAVXExtend()
17990     return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, dl, VT, In);  in LowerAVXExtend()
18009   MVT HalfVT = MVT::getVectorVT(VT.getVectorElementType(),  in LowerAVXExtend()
18010                                 VT.getVectorNumElements() / 2);  in LowerAVXExtend()
18020   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);  in LowerAVXExtend()
18024 static SDValue SplitAndExtendv16i1(unsigned ExtOpc, MVT VT, SDValue In,  in SplitAndExtendv16i1()  argument
18026   assert((VT == MVT::v16i8 || VT == MVT::v16i16) && "Unexpected VT.");  in SplitAndExtendv16i1()
18034   return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);  in SplitAndExtendv16i1()
18040   MVT VT = Op->getSimpleValueType(0);  in LowerZERO_EXTEND_Mask()  local
18045   unsigned NumElts = VT.getVectorNumElements();  in LowerZERO_EXTEND_Mask()
18049   if (VT.getVectorElementType() != MVT::i8) {  in LowerZERO_EXTEND_Mask()
18050     SDValue Extend = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, In);  in LowerZERO_EXTEND_Mask()
18051     return DAG.getNode(ISD::SRL, DL, VT, Extend,  in LowerZERO_EXTEND_Mask()
18052                        DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT));  in LowerZERO_EXTEND_Mask()
18056   MVT ExtVT = VT;  in LowerZERO_EXTEND_Mask()
18060       return SplitAndExtendv16i1(ISD::ZERO_EXTEND, VT, In, DL, DAG);  in LowerZERO_EXTEND_Mask()
18082   if (VT != ExtVT) {  in LowerZERO_EXTEND_Mask()
18088   if (WideVT != VT)  in LowerZERO_EXTEND_Mask()
18089     SelectedVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SelectedVal,  in LowerZERO_EXTEND_Mask()
18218   MVT VT = Op.getSimpleValueType();  in LowerTruncateVecI1()  local
18222   assert(VT.getVectorElementType() == MVT::i1 && "Unexpected vector type.");  in LowerTruncateVecI1()
18238       return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT),  in LowerTruncateVecI1()
18267       return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);  in LowerTruncateVecI1()
18286     return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT), In, ISD::SETGT);  in LowerTruncateVecI1()
18287   return DAG.getSetCC(DL, VT, In, DAG.getConstant(0, DL, InVT), ISD::SETNE);  in LowerTruncateVecI1()
18292   MVT VT = Op.getSimpleValueType();  in LowerTRUNCATE()  local
18297   assert(VT.getVectorNumElements() == InVT.getVectorNumElements() &&  in LowerTRUNCATE()
18304   if (VT.getVectorElementType() == MVT::i1)  in LowerTRUNCATE()
18318   unsigned NumPackedSignBits = std::min<unsigned>(VT.getScalarSizeInBits(), 16);  in LowerTRUNCATE()
18327             truncateVectorWithPACK(X86ISD::PACKUS, VT, In, DL, DAG, Subtarget))  in LowerTRUNCATE()
18334             truncateVectorWithPACK(X86ISD::PACKSS, VT, In, DL, DAG, Subtarget))  in LowerTRUNCATE()
18337   if ((VT == MVT::v4i32) && (InVT == MVT::v4i64)) {  in LowerTRUNCATE()
18343       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, In,  in LowerTRUNCATE()
18354     return DAG.getVectorShuffle(VT, DL, OpLo, OpHi, ShufMask);  in LowerTRUNCATE()
18357   if ((VT == MVT::v8i16) && (InVT == MVT::v8i32)) {  in LowerTRUNCATE()
18374       return DAG.getBitcast(VT, In);  in LowerTRUNCATE()
18402   if (VT == MVT::v16i8 && InVT == MVT::v16i16) {  in LowerTRUNCATE()
18410     return DAG.getNode(X86ISD::PACKUS, DL, VT, InLo, InHi);  in LowerTRUNCATE()
18414   assert(VT.is128BitVector() && InVT.is256BitVector() && "Unexpected types!");  in LowerTRUNCATE()
18418   unsigned NumElems = VT.getVectorNumElements();  in LowerTRUNCATE()
18419   MVT NVT = MVT::getVectorVT(VT.getVectorElementType(), NumElems * 2);  in LowerTRUNCATE()
18427   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V,  in LowerTRUNCATE()
18433   MVT VT = Op.getSimpleValueType();  in LowerFP_TO_INT()  local
18435   if (VT.isVector()) {  in LowerFP_TO_INT()
18439     if (VT == MVT::v2i1 && Src.getSimpleValueType() == MVT::v2f64) {  in LowerFP_TO_INT()
18459     if (VT == MVT::v2i64 && Src.getSimpleValueType() == MVT::v2f32) {  in LowerFP_TO_INT()
18460       return DAG.getNode(IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI, dl, VT,  in LowerFP_TO_INT()
18468   assert(!VT.isVector());  in LowerFP_TO_INT()
18479     return DAG.getLoad(VT, SDLoc(Op), FIST, StackSlot, MachinePointerInfo());  in LowerFP_TO_INT()
18487   MVT VT = Op.getSimpleValueType();  in LowerFP_EXTEND()  local
18493   return DAG.getNode(X86ISD::VFPEXT, DL, VT,  in LowerFP_EXTEND()
18606   MVT VT = Op.getSimpleValueType();  in LowerFABSorFNEG()  local
18608   bool IsF128 = (VT == MVT::f128);  in LowerFABSorFNEG()
18609   assert((VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 ||  in LowerFABSorFNEG()
18610           VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 ||  in LowerFABSorFNEG()
18611           VT == MVT::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) &&  in LowerFABSorFNEG()
18622   bool IsFakeVector = !VT.isVector() && !IsF128;  in LowerFABSorFNEG()
18623   MVT LogicVT = VT;  in LowerFABSorFNEG()
18625     LogicVT = (VT == MVT::f64) ? MVT::v2f64 : MVT::v4f32;  in LowerFABSorFNEG()
18627   unsigned EltBits = VT.getScalarSizeInBits();  in LowerFABSorFNEG()
18631   const fltSemantics &Sem = SelectionDAG::EVTToAPFloatSemantics(VT);  in LowerFABSorFNEG()
18641   if (VT.isVector() || IsF128)  in LowerFABSorFNEG()
18648   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, LogicNode,  in LowerFABSorFNEG()
18658   MVT VT = Op.getSimpleValueType();  in LowerFCOPYSIGN()  local
18659   if (Sign.getSimpleValueType().bitsLT(VT))  in LowerFCOPYSIGN()
18660     Sign = DAG.getNode(ISD::FP_EXTEND, dl, VT, Sign);  in LowerFCOPYSIGN()
18663   if (Sign.getSimpleValueType().bitsGT(VT))  in LowerFCOPYSIGN()
18664     Sign = DAG.getNode(ISD::FP_ROUND, dl, VT, Sign, DAG.getIntPtrConstant(1, dl));  in LowerFCOPYSIGN()
18668   bool IsF128 = (VT == MVT::f128);  in LowerFCOPYSIGN()
18669   assert((VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 ||  in LowerFCOPYSIGN()
18670           VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 ||  in LowerFCOPYSIGN()
18671           VT == MVT::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) &&  in LowerFCOPYSIGN()
18674   const fltSemantics &Sem = SelectionDAG::EVTToAPFloatSemantics(VT);  in LowerFCOPYSIGN()
18681   bool IsFakeVector = !VT.isVector() && !IsF128;  in LowerFCOPYSIGN()
18682   MVT LogicVT = VT;  in LowerFCOPYSIGN()
18684     LogicVT = (VT == MVT::f64) ? MVT::v2f64 : MVT::v4f32;  in LowerFCOPYSIGN()
18687   unsigned EltSizeInBits = VT.getScalarSizeInBits();  in LowerFCOPYSIGN()
18715   return !IsFakeVector ? Or : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Or,  in LowerFCOPYSIGN()
18722   MVT VT = Op.getSimpleValueType();  in LowerFGETSIGN()  local
18732   Res = DAG.getZExtOrTrunc(Res, dl, VT);  in LowerFGETSIGN()
18733   Res = DAG.getNode(ISD::AND, dl, VT, Res, DAG.getConstant(1, dl, VT));  in LowerFGETSIGN()
18763   EVT VT = MVT::Other;  in LowerVectorAllZeroTest()  local
18793       VT = ExtractedFromVec.getValueType();  in LowerVectorAllZeroTest()
18795       if (!VT.is128BitVector() && !VT.is256BitVector())  in LowerVectorAllZeroTest()
18799           VT != VecInMap.begin()->first.getValueType())  in LowerVectorAllZeroTest()
18807   assert((VT.is128BitVector() || VT.is256BitVector()) &&  in LowerVectorAllZeroTest()
18810   unsigned FullMask = (1U << VT.getVectorNumElements()) - 1U;  in LowerVectorAllZeroTest()
18819   MVT TestVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;  in LowerVectorAllZeroTest()
19029   EVT VT = Op.getValueType();  in isFsqrtCheap()  local
19032   if (DAG.getNodeIfExists(X86ISD::FRSQRT, DAG.getVTList(VT), Op))  in isFsqrtCheap()
19035   if (VT.isVector())  in isFsqrtCheap()
19047   EVT VT = Op.getValueType();  in getSqrtEstimate()  local
19057   if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||  in getSqrtEstimate()
19058       (VT == MVT::v4f32 && Subtarget.hasSSE1() && Reciprocal) ||  in getSqrtEstimate()
19059       (VT == MVT::v4f32 && Subtarget.hasSSE2() && !Reciprocal) ||  in getSqrtEstimate()
19060       (VT == MVT::v8f32 && Subtarget.hasAVX()) ||  in getSqrtEstimate()
19061       (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) {  in getSqrtEstimate()
19067     unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RSQRT14 : X86ISD::FRSQRT;  in getSqrtEstimate()
19068     return DAG.getNode(Opcode, SDLoc(Op), VT, Op);  in getSqrtEstimate()
19078   EVT VT = Op.getValueType();  in getRecipEstimate()  local
19087   if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||  in getRecipEstimate()
19088       (VT == MVT::v4f32 && Subtarget.hasSSE1()) ||  in getRecipEstimate()
19089       (VT == MVT::v8f32 && Subtarget.hasAVX()) ||  in getRecipEstimate()
19090       (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) {  in getRecipEstimate()
19094     if (VT == MVT::f32 && Enabled == ReciprocalEstimate::Unspecified)  in getRecipEstimate()
19101     unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RCP14 : X86ISD::FRCP;  in getRecipEstimate()
19102     return DAG.getNode(Opcode, SDLoc(Op), VT, Op);  in getRecipEstimate()
19246   MVT VT = Op.getSimpleValueType();  in Lower256IntVSETCC()  local
19248   assert(VT.is256BitVector() && Op.getOpcode() == ISD::SETCC &&  in Lower256IntVSETCC()
19251   unsigned NumElems = VT.getVectorNumElements();  in Lower256IntVSETCC()
19266   MVT EltVT = VT.getVectorElementType();  in Lower256IntVSETCC()
19268   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,  in Lower256IntVSETCC()
19278   MVT VT = Op.getSimpleValueType();  in LowerIntVSETCC_AVX512()  local
19281   assert(VT.getVectorElementType() == MVT::i1 &&  in LowerIntVSETCC_AVX512()
19299   return DAG.getSetCC(dl, VT, Op0, Op1, SetCCOpcode);  in LowerIntVSETCC_AVX512()
19310   MVT VT = V.getSimpleValueType();  in decrementVectorConstant()  local
19311   MVT EltVT = VT.getVectorElementType();  in decrementVectorConstant()
19312   unsigned NumElts = VT.getVectorNumElements();  in decrementVectorConstant()
19327   return DAG.getBuildVector(VT, DL, NewVecC);  in decrementVectorConstant()
19334 static SDValue LowerVSETCCWithSUBUS(SDValue Op0, SDValue Op1, MVT VT,  in LowerVSETCCWithSUBUS()  argument
19341   MVT VET = VT.getVectorElementType();  in LowerVSETCCWithSUBUS()
19370   SDValue Result = DAG.getNode(ISD::USUBSAT, dl, VT, Op0, Op1);  in LowerVSETCCWithSUBUS()
19371   return DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result,  in LowerVSETCCWithSUBUS()
19372                      DAG.getConstant(0, dl, VT));  in LowerVSETCCWithSUBUS()
19380   MVT VT = Op.getSimpleValueType();  in LowerVSETCC()  local
19392     if (Subtarget.hasAVX512() && VT.getVectorElementType() == MVT::i1) {  in LowerVSETCC()
19393       assert(VT.getVectorNumElements() <= 16);  in LowerVSETCC()
19400       VT = Op0.getSimpleValueType();  in LowerVSETCC()
19422       SDValue Cmp0 = DAG.getNode(Opc, dl, VT, Op0, Op1,  in LowerVSETCC()
19424       SDValue Cmp1 = DAG.getNode(Opc, dl, VT, Op0, Op1,  in LowerVSETCC()
19426       Cmp = DAG.getNode(CombineOpc, dl, VT, Cmp0, Cmp1);  in LowerVSETCC()
19429       Cmp = DAG.getNode(Opc, dl, VT, Op0, Op1,  in LowerVSETCC()
19445   assert(VT.getVectorNumElements() == VTOp0.getVectorNumElements() &&  in LowerVSETCC()
19455   assert((Subtarget.hasAVX512() || (VT == VTOp0)) &&  in LowerVSETCC()
19459   if (VT.is256BitVector() && !Subtarget.hasInt256())  in LowerVSETCC()
19463   if (VT.getVectorElementType() == MVT::i1) {  in LowerVSETCC()
19472   if (VT.is128BitVector() && Subtarget.hasXOP()) {  in LowerVSETCC()
19493     return DAG.getNode(Opc, dl, VT, Op0, Op1,  in LowerVSETCC()
19505                                         VT.getScalarSizeInBits(), UndefElts,  in LowerVSETCC()
19509           Op1 = DAG.getBitcast(VT, BC0.getOperand(1));  in LowerVSETCC()
19539       TLI.isOperationLegal(ISD::UMIN, VT)) {  in LowerVSETCC()
19547       Op1 = DAG.getConstant(C + 1, dl, VT);  in LowerVSETCC()
19552       Op1 = DAG.getConstant(C - 1, dl, VT);  in LowerVSETCC()
19565     SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);  in LowerVSETCC()
19566     Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result);  in LowerVSETCC()
19570       Result = DAG.getNOT(dl, Result, VT);  in LowerVSETCC()
19576   if (SDValue V = LowerVSETCCWithSUBUS(Op0, Op1, VT, Cond, dl, Subtarget, DAG))  in LowerVSETCC()
19594   if (VT == MVT::v2i64) {  in LowerVSETCC()
19631       return DAG.getBitcast(VT, Result);  in LowerVSETCC()
19654       return DAG.getBitcast(VT, Result);  in LowerVSETCC()
19661     MVT EltVT = VT.getVectorElementType();  in LowerVSETCC()
19663                                  VT);  in LowerVSETCC()
19664     Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SM);  in LowerVSETCC()
19665     Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SM);  in LowerVSETCC()
19668   SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);  in LowerVSETCC()
19672     Result = DAG.getNOT(dl, Result, VT);  in LowerVSETCC()
19691   MVT VT = Op0.getSimpleValueType();  in EmitKORTEST()  local
19692   if (!(Subtarget.hasAVX512() && VT == MVT::v16i1) &&  in EmitKORTEST()
19693       !(Subtarget.hasDQI() && VT == MVT::v8i1) &&  in EmitKORTEST()
19694       !(Subtarget.hasBWI() && (VT == MVT::v32i1 || VT == MVT::v64i1)))  in EmitKORTEST()
19779   MVT VT = Op.getSimpleValueType();  in LowerSETCC()  local
19781   if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG);  in LowerSETCC()
19783   assert(VT == MVT::i8 && "SetCC type must be 8-bit integer");  in LowerSETCC()
19913   MVT VT = Op1.getSimpleValueType();  in LowerSELECT()  local
19920       ((Subtarget.hasSSE2() && VT == MVT::f64) ||  in LowerSELECT()
19921        (Subtarget.hasSSE1() && VT == MVT::f32)) &&  in LowerSELECT()
19922       VT == Cond.getOperand(0).getSimpleValueType() && Cond->hasOneUse()) {  in LowerSELECT()
19930       assert(!VT.isVector() && "Not a scalar type?");  in LowerSELECT()
19931       return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2);  in LowerSELECT()
19935       SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,  in LowerSELECT()
19954         MVT VecVT = VT == MVT::f32 ? MVT::v4f32 : MVT::v2f64;  in LowerSELECT()
19959         MVT VCmpVT = VT == MVT::f32 ? MVT::v4i32 : MVT::v2i64;  in LowerSELECT()
19964         return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,  in LowerSELECT()
19967       SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2);  in LowerSELECT()
19968       SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1);  in LowerSELECT()
19969       return DAG.getNode(X86ISD::FOR, DL, VT, AndN, And);  in LowerSELECT()
19974   if ((VT == MVT::f64 || VT == MVT::f32) && Subtarget.hasAVX512()) {  in LowerSELECT()
19976     return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2);  in LowerSELECT()
19980   if (VT == MVT::v64i1 && !Subtarget.is64Bit()) {  in LowerSELECT()
19988     return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);  in LowerSELECT()
19991   if (VT.isVector() && VT.getVectorElementType() == MVT::i1) {  in LowerSELECT()
20005       if (newSelect.getValueSizeInBits() == VT.getSizeInBits())  in LowerSELECT()
20006         return DAG.getBitcast(VT, newSelect);  in LowerSELECT()
20008       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, ExtVec,  in LowerSELECT()
20093         if (CmpSz > VT.getSizeInBits())  in LowerSELECT()
20094           Neg = DAG.getNode(ISD::TRUNCATE, DL, VT, CmpOp0);  in LowerSELECT()
20095         else if (CmpSz < VT.getSizeInBits())  in LowerSELECT()
20096           Neg = DAG.getNode(ISD::AND, DL, VT,  in LowerSELECT()
20097               DAG.getNode(ISD::ANY_EXTEND, DL, VT, CmpOp0.getOperand(0)),  in LowerSELECT()
20098               DAG.getConstant(1, DL, VT));  in LowerSELECT()
20101         SDValue Mask = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),  in LowerSELECT()
20103         SDValue And = DAG.getNode(ISD::AND, DL, VT, Mask, Src1); // Mask & z  in LowerSELECT()
20104         return DAG.getNode(Op2.getOpcode(), DL, VT, And, Src2);  // And Op y  in LowerSELECT()
20124     MVT VT = Op.getSimpleValueType();  in LowerSELECT()  local
20127     if (VT.isFloatingPoint() && !VT.isVector() &&  in LowerSELECT()
20128         !isScalarFPTypeInSSEReg(VT))  // FPStack?  in LowerSELECT()
20223   MVT VT = Op->getSimpleValueType(0);  in LowerSIGN_EXTEND_Mask()  local
20227   MVT VTElt = VT.getVectorElementType();  in LowerSIGN_EXTEND_Mask()
20230   unsigned NumElts = VT.getVectorNumElements();  in LowerSIGN_EXTEND_Mask()
20233   MVT ExtVT = VT;  in LowerSIGN_EXTEND_Mask()
20237       return SplitAndExtendv16i1(Op.getOpcode(), VT, In, dl, DAG);  in LowerSIGN_EXTEND_Mask()
20264   if (VT != ExtVT) {  in LowerSIGN_EXTEND_Mask()
20270   if (WideVT != VT)  in LowerSIGN_EXTEND_Mask()
20271     V = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, V,  in LowerSIGN_EXTEND_Mask()
20297   MVT VT = Op->getSimpleValueType(0);  in LowerEXTEND_VECTOR_INREG()  local
20300   MVT SVT = VT.getVectorElementType();  in LowerEXTEND_VECTOR_INREG()
20308   if (!(VT.is128BitVector() && Subtarget.hasSSE2()) &&  in LowerEXTEND_VECTOR_INREG()
20309       !(VT.is256BitVector() && Subtarget.hasAVX()) &&  in LowerEXTEND_VECTOR_INREG()
20310       !(VT.is512BitVector() && Subtarget.hasAVX512()))  in LowerEXTEND_VECTOR_INREG()
20315   unsigned NumElts = VT.getVectorNumElements();  in LowerEXTEND_VECTOR_INREG()
20331     assert(VT.getSizeInBits() > 128 && "Unexpected 128-bit vector extension");  in LowerEXTEND_VECTOR_INREG()
20334       return DAG.getNode(Op.getOpcode(), dl, VT, In);  in LowerEXTEND_VECTOR_INREG()
20341     return DAG.getNode(ExtOpc, dl, VT, In);  in LowerEXTEND_VECTOR_INREG()
20346     assert(VT.is256BitVector() && "256-bit vector expected");  in LowerEXTEND_VECTOR_INREG()
20358     return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);  in LowerEXTEND_VECTOR_INREG()
20363   assert(VT.is128BitVector() && InVT.is128BitVector() && "Unexpected VTs");  in LowerEXTEND_VECTOR_INREG()
20372     MVT DestVT = VT == MVT::v2i64 ? MVT::v4i32 : VT;  in LowerEXTEND_VECTOR_INREG()
20394   if (VT == MVT::v2i64) {  in LowerEXTEND_VECTOR_INREG()
20399     SignExt = DAG.getBitcast(VT, SignExt);  in LowerEXTEND_VECTOR_INREG()
20407   MVT VT = Op->getSimpleValueType(0);  in LowerSIGN_EXTEND()  local
20415   assert(VT.isVector() && InVT.isVector() && "Expected vector type");  in LowerSIGN_EXTEND()
20416   assert(VT.getVectorNumElements() == VT.getVectorNumElements() &&  in LowerSIGN_EXTEND()
20418   assert((VT.getVectorElementType() == MVT::i16 ||  in LowerSIGN_EXTEND()
20419           VT.getVectorElementType() == MVT::i32 ||  in LowerSIGN_EXTEND()
20420           VT.getVectorElementType() == MVT::i64) &&  in LowerSIGN_EXTEND()
20429     if (!ExperimentalVectorWideningLegalization || VT != MVT::v8i64)  in LowerSIGN_EXTEND()
20434     return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, VT, In);  in LowerSIGN_EXTEND()
20449   MVT HalfVT = MVT::getVectorVT(VT.getVectorElementType(),  in LowerSIGN_EXTEND()
20450                                 VT.getVectorNumElements() / 2);  in LowerSIGN_EXTEND()
20462   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);  in LowerSIGN_EXTEND()
20992   EVT VT = Node->getValueType(0);  in LowerDYNAMIC_STACKALLOC()  local
21008     SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);  in LowerDYNAMIC_STACKALLOC()
21012     Result = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value  in LowerDYNAMIC_STACKALLOC()
21014       Result = DAG.getNode(ISD::AND, dl, VT, Result,  in LowerDYNAMIC_STACKALLOC()
21015                          DAG.getConstant(-(uint64_t)Align, dl, VT));  in LowerDYNAMIC_STACKALLOC()
21047       SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),  in LowerDYNAMIC_STACKALLOC()
21048                        DAG.getConstant(-(uint64_t)Align, dl, VT));  in LowerDYNAMIC_STACKALLOC()
21222 static SDValue getTargetVShiftByConstNode(unsigned Opc, const SDLoc &dl, MVT VT,  in getTargetVShiftByConstNode()  argument
21225   MVT ElementType = VT.getVectorElementType();  in getTargetVShiftByConstNode()
21229   if (VT != SrcOp.getSimpleValueType())  in getTargetVShiftByConstNode()
21230     SrcOp = DAG.getBitcast(VT, SrcOp);  in getTargetVShiftByConstNode()
21241       return DAG.getConstant(0, dl, VT);  in getTargetVShiftByConstNode()
21294     return DAG.getBuildVector(VT, dl, Elts);  in getTargetVShiftByConstNode()
21297   return DAG.getNode(Opc, dl, VT, SrcOp,  in getTargetVShiftByConstNode()
21303 static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT,  in getTargetVShiftNode()  argument
21312     return getTargetVShiftByConstNode(Opc, dl, VT, SrcOp,  in getTargetVShiftNode()
21364   MVT EltVT = VT.getVectorElementType();  in getTargetVShiftNode()
21368   return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt);  in getTargetVShiftNode()
21416   MVT VT = Op.getSimpleValueType();  in getVectorMaskingNode()  local
21417   MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements());  in getVectorMaskingNode()
21427     PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl);  in getVectorMaskingNode()
21428   return DAG.getNode(OpcodeSelect, dl, VT, VMask, Op, PreservedSrc);  in getVectorMaskingNode()
21447   MVT VT = Op.getSimpleValueType();  in getScalarMaskingNode()  local
21457     return DAG.getNode(ISD::AND, dl, VT, Op, IMask);  in getScalarMaskingNode()
21460     PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl);  in getScalarMaskingNode()
21461   return DAG.getNode(X86ISD::SELECTS, dl, VT, IMask, Op, PreservedSrc);  in getScalarMaskingNode()
21538   MVT VT = Op.getSimpleValueType();  in LowerINTRINSIC_WO_CHAIN()  local
21616       return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src,  in LowerINTRINSIC_WO_CHAIN()
21637       return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src),  in LowerINTRINSIC_WO_CHAIN()
21655                                                     dl, VT, Src1, Src2, Rnd),  in LowerINTRINSIC_WO_CHAIN()
21658         return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1,  in LowerINTRINSIC_WO_CHAIN()
21670                                                   dl, VT, Src1, Src2,  in LowerINTRINSIC_WO_CHAIN()
21674       return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1,  in LowerINTRINSIC_WO_CHAIN()
21688         return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2,  in LowerINTRINSIC_WO_CHAIN()
21695       return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2,  in LowerINTRINSIC_WO_CHAIN()
21719       return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT,Src1,Src2),  in LowerINTRINSIC_WO_CHAIN()
21736       return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT,  in LowerINTRINSIC_WO_CHAIN()
21752                                                   dl, VT, Src1, Src2, Src3, Rnd),  in LowerINTRINSIC_WO_CHAIN()
21755       return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1,  in LowerINTRINSIC_WO_CHAIN()
21779       return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT,  in LowerINTRINSIC_WO_CHAIN()
21788       return DAG.getNode(IntrData->Opc0, dl, VT,Src2, Src1);  in LowerINTRINSIC_WO_CHAIN()
21798       return DAG.getNode(IntrData->Opc0, dl, VT, Op.getOperand(1),  in LowerINTRINSIC_WO_CHAIN()
21820       return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src,  in LowerINTRINSIC_WO_CHAIN()
21955       return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT,  in LowerINTRINSIC_WO_CHAIN()
21969                                          Src1 : getZeroVector(VT, Subtarget, DAG, dl);  in LowerINTRINSIC_WO_CHAIN()
21980         return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT,  in LowerINTRINSIC_WO_CHAIN()
21984         return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT,  in LowerINTRINSIC_WO_CHAIN()
22254     return DAG.getNode(getGlobalWrapperKind(), dl, VT,  in LowerINTRINSIC_WO_CHAIN()
22268     SDValue Result = DAG.getMCSymbol(LSDASym, VT);  in LowerINTRINSIC_WO_CHAIN()
22269     return DAG.getNode(X86ISD::Wrapper, dl, VT, Result);  in LowerINTRINSIC_WO_CHAIN()
22293     return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);  in LowerINTRINSIC_WO_CHAIN()
22327   MVT VT = Op.getSimpleValueType();  in getGatherNode()  local
22335                               VT.getVectorNumElements());  in getGatherNode()
22860   EVT VT = Op.getValueType();  in LowerFRAMEADDR()  local
22876     return DAG.getFrameIndex(FrameAddrIndex, VT);  in LowerFRAMEADDR()
22883   assert(((FrameReg == X86::RBP && VT == MVT::i64) ||  in LowerFRAMEADDR()
22884           (FrameReg == X86::EBP && VT == MVT::i32)) &&  in LowerFRAMEADDR()
22886   SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);  in LowerFRAMEADDR()
22888     FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,  in LowerFRAMEADDR()
22895 unsigned X86TargetLowering::getRegisterByName(const char* RegName, EVT VT,  in getRegisterByName()  argument
23186   MVT VT = Op.getSimpleValueType();  in LowerFLT_ROUNDS_()  local
23226   return DAG.getNode((VT.getSizeInBits() < 16 ?  in LowerFLT_ROUNDS_()
23227                       ISD::TRUNCATE : ISD::ZERO_EXTEND), DL, VT, RetVal);  in LowerFLT_ROUNDS_()
23232   MVT VT = Op.getSimpleValueType();  in LowerVectorIntUnary()  local
23233   unsigned NumElems = VT.getVectorNumElements();  in LowerVectorIntUnary()
23234   unsigned SizeInBits = VT.getSizeInBits();  in LowerVectorIntUnary()
23235   MVT EltVT = VT.getVectorElementType();  in LowerVectorIntUnary()
23246   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,  in LowerVectorIntUnary()
23277   MVT VT = Op.getSimpleValueType();  in LowerVectorCTLZ_AVX512CDI()  local
23278   MVT EltVT = VT.getVectorElementType();  in LowerVectorCTLZ_AVX512CDI()
23279   unsigned NumElems = VT.getVectorNumElements();  in LowerVectorCTLZ_AVX512CDI()
23296   SDValue TruncNode = DAG.getNode(ISD::TRUNCATE, dl, VT, CtlzNode);  in LowerVectorCTLZ_AVX512CDI()
23297   SDValue Delta = DAG.getConstant(32 - EltVT.getSizeInBits(), dl, VT);  in LowerVectorCTLZ_AVX512CDI()
23299   return DAG.getNode(ISD::SUB, dl, VT, TruncNode, Delta);  in LowerVectorCTLZ_AVX512CDI()
23306   MVT VT = Op.getSimpleValueType();  in LowerVectorCTLZInRegLUT()  local
23307   int NumElts = VT.getVectorNumElements();  in LowerVectorCTLZInRegLUT()
23308   int NumBytes = NumElts * (VT.getScalarSizeInBits() / 8);  in LowerVectorCTLZInRegLUT()
23352   while (CurrVT != VT) {  in LowerVectorCTLZInRegLUT()
23388   MVT VT = Op.getSimpleValueType();  in LowerVectorCTLZ()  local
23392       (Subtarget.canExtendTo512DQ() || VT.getVectorElementType() != MVT::i8))  in LowerVectorCTLZ()
23396   if (VT.is256BitVector() && !Subtarget.hasInt256())  in LowerVectorCTLZ()
23400   if (VT.is512BitVector() && !Subtarget.hasBWI())  in LowerVectorCTLZ()
23409   MVT VT = Op.getSimpleValueType();  in LowerCTLZ()  local
23410   MVT OpVT = VT;  in LowerCTLZ()
23411   unsigned NumBits = VT.getSizeInBits();  in LowerCTLZ()
23415   if (VT.isVector())  in LowerCTLZ()
23419   if (VT == MVT::i8) {  in LowerCTLZ()
23444   if (VT == MVT::i8)  in LowerCTLZ()
23451   MVT VT = Op.getSimpleValueType();  in LowerCTTZ()  local
23452   unsigned NumBits = VT.getScalarSizeInBits();  in LowerCTTZ()
23457   if (VT.is256BitVector() && !Subtarget.hasInt256())  in LowerCTTZ()
23460   assert(!VT.isVector() && Op.getOpcode() == ISD::CTTZ &&  in LowerCTTZ()
23464   SDVTList VTs = DAG.getVTList(VT, MVT::i32);  in LowerCTTZ()
23470     DAG.getConstant(NumBits, dl, VT),  in LowerCTTZ()
23474   return DAG.getNode(X86ISD::CMOV, dl, VT, Ops);  in LowerCTTZ()
23480   MVT VT = Op.getSimpleValueType();  in split256IntArith()  local
23482   assert(VT.is256BitVector() && VT.isInteger() &&  in split256IntArith()
23485   unsigned NumElems = VT.getVectorNumElements();  in split256IntArith()
23498   MVT EltVT = VT.getVectorElementType();  in split256IntArith()
23501   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,  in split256IntArith()
23509   MVT VT = Op.getSimpleValueType();  in split512IntArith()  local
23511   assert(VT.is512BitVector() && VT.isInteger() &&  in split512IntArith()
23514   unsigned NumElems = VT.getVectorNumElements();  in split512IntArith()
23527   MVT EltVT = VT.getVectorElementType();  in split512IntArith()
23530   return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,  in split512IntArith()
23537   MVT VT = Op.getSimpleValueType();  in lowerAddSub()  local
23538   if (VT == MVT::i16 || VT == MVT::i32)  in lowerAddSub()
23541   if (VT.getScalarType() == MVT::i1)  in lowerAddSub()
23542     return DAG.getNode(ISD::XOR, SDLoc(Op), VT,  in lowerAddSub()
23552   MVT VT = Op.getSimpleValueType();  in LowerADDSAT_SUBSAT()  local
23553   if (VT.getScalarType() == MVT::i1) {  in LowerADDSAT_SUBSAT()
23559       return DAG.getNode(ISD::OR, dl, VT, Op.getOperand(0), Op.getOperand(1));  in LowerADDSAT_SUBSAT()
23562       return DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0),  in LowerADDSAT_SUBSAT()
23563                          DAG.getNOT(dl, Op.getOperand(1), VT));  in LowerADDSAT_SUBSAT()
23575   MVT VT = Op.getSimpleValueType();  in LowerABS()  local
23576   if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) {  in LowerABS()
23581     SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),  in LowerABS()
23582                               DAG.getConstant(0, DL, VT), N0);  in LowerABS()
23585     return DAG.getNode(X86ISD::CMOV, DL, VT, Ops);  in LowerABS()
23589   if ((VT == MVT::v2i64 || VT == MVT::v4i64) && Subtarget.hasSSE41()) {  in LowerABS()
23593         DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Src);  in LowerABS()
23594     return DAG.getNode(X86ISD::BLENDV, DL, VT, Src, Sub, Src);  in LowerABS()
23597   if (VT.is256BitVector() && !Subtarget.hasInt256()) {  in LowerABS()
23598     assert(VT.isInteger() &&  in LowerABS()
23608   MVT VT = Op.getSimpleValueType();  in LowerMINMAX()  local
23611   if (VT.getScalarType() != MVT::i64 && VT.is256BitVector())  in LowerMINMAX()
23621   if (VT == MVT::v8i16) {  in LowerMINMAX()
23624     SDValue Sign = DAG.getConstant(APInt::getSignedMinValue(16), DL, VT);  in LowerMINMAX()
23625     N0 = DAG.getNode(ISD::XOR, DL, VT, N0, Sign);  in LowerMINMAX()
23626     N1 = DAG.getNode(ISD::XOR, DL, VT, N1, Sign);  in LowerMINMAX()
23628     SDValue Result = DAG.getNode(Opcode, DL, VT, N0, N1);  in LowerMINMAX()
23629     return DAG.getNode(ISD::XOR, DL, VT, Result, Sign);  in LowerMINMAX()
23642   SDValue Cond = DAG.getSetCC(DL, VT, N0, N1, CC);  in LowerMINMAX()
23643   return DAG.getSelect(DL, VT, Cond, N0, N1);  in LowerMINMAX()
23649   MVT VT = Op.getSimpleValueType();  in LowerMUL()  local
23651   if (VT.getScalarType() == MVT::i1)  in LowerMUL()
23652     return DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0), Op.getOperand(1));  in LowerMUL()
23655   if (VT.is256BitVector() && !Subtarget.hasInt256())  in LowerMUL()
23663   if (VT == MVT::v16i8 || VT == MVT::v32i8 || VT == MVT::v64i8) {  in LowerMUL()
23664     unsigned NumElts = VT.getVectorNumElements();  in LowerMUL()
23666     if ((VT == MVT::v16i8 && Subtarget.hasInt256()) ||  in LowerMUL()
23667         (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) {  in LowerMUL()
23668       MVT ExVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements());  in LowerMUL()
23670           ISD::TRUNCATE, dl, VT,  in LowerMUL()
23682     SDValue Undef = DAG.getUNDEF(VT);  in LowerMUL()
23683     SDValue ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Undef));  in LowerMUL()
23684     SDValue AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Undef));  in LowerMUL()
23702       BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Undef));  in LowerMUL()
23703       BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B, Undef));  in LowerMUL()
23711     return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);  in LowerMUL()
23715   if (VT == MVT::v4i32) {  in LowerMUL()
23721     SDValue Aodds = DAG.getVectorShuffle(VT, dl, A, A, UnpackMask);  in LowerMUL()
23722     SDValue Bodds = DAG.getVectorShuffle(VT, dl, B, B, UnpackMask);  in LowerMUL()
23733     Evens = DAG.getBitcast(VT, Evens);  in LowerMUL()
23734     Odds = DAG.getBitcast(VT, Odds);  in LowerMUL()
23739     return DAG.getVectorShuffle(VT, dl, Evens, Odds, ShufMask);  in LowerMUL()
23742   assert((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) &&  in LowerMUL()
23766   SDValue Zero = DAG.getConstant(0, dl, VT);  in LowerMUL()
23771     AloBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, B);  in LowerMUL()
23775     SDValue Bhi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, B, 32, DAG);  in LowerMUL()
23776     AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi);  in LowerMUL()
23781     SDValue Ahi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, A, 32, DAG);  in LowerMUL()
23782     AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B);  in LowerMUL()
23785   SDValue Hi = DAG.getNode(ISD::ADD, dl, VT, AloBhi, AhiBlo);  in LowerMUL()
23786   Hi = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Hi, 32, DAG);  in LowerMUL()
23788   return DAG.getNode(ISD::ADD, dl, VT, AloBlo, Hi);  in LowerMUL()
23794   MVT VT = Op.getSimpleValueType();  in LowerMULH()  local
23796   unsigned NumElts = VT.getVectorNumElements();  in LowerMULH()
23801   if (VT.is256BitVector() && !Subtarget.hasInt256())  in LowerMULH()
23804   if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) {  in LowerMULH()
23805     assert((VT == MVT::v4i32 && Subtarget.hasSSE2()) ||  in LowerMULH()
23806            (VT == MVT::v8i32 && Subtarget.hasInt256()) ||  in LowerMULH()
23807            (VT == MVT::v16i32 && Subtarget.hasAVX512()));  in LowerMULH()
23824     SDValue Odd0 = DAG.getVectorShuffle(VT, dl, A, A,  in LowerMULH()
23827     SDValue Odd1 = DAG.getVectorShuffle(VT, dl, B, B,  in LowerMULH()
23837     SDValue Mul1 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,  in LowerMULH()
23842     SDValue Mul2 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,  in LowerMULH()
23851     SDValue Res = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, ShufMask);  in LowerMULH()
23856       SDValue Zero = DAG.getConstant(0, dl, VT);  in LowerMULH()
23857       SDValue T1 = DAG.getNode(ISD::AND, dl, VT,  in LowerMULH()
23858                                DAG.getSetCC(dl, VT, Zero, A, ISD::SETGT), B);  in LowerMULH()
23859       SDValue T2 = DAG.getNode(ISD::AND, dl, VT,  in LowerMULH()
23860                                DAG.getSetCC(dl, VT, Zero, B, ISD::SETGT), A);  in LowerMULH()
23862       SDValue Fixup = DAG.getNode(ISD::ADD, dl, VT, T1, T2);  in LowerMULH()
23863       Res = DAG.getNode(ISD::SUB, dl, VT, Res, Fixup);  in LowerMULH()
23870   assert((VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) ||  in LowerMULH()
23871          (VT == MVT::v64i8 && Subtarget.hasBWI())) &&  in LowerMULH()
23881   if ((VT == MVT::v16i8 && Subtarget.hasInt256()) ||  in LowerMULH()
23882       (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) {  in LowerMULH()
23888     return DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);  in LowerMULH()
23893   if (VT == MVT::v64i8 && IsSigned)  in LowerMULH()
23897   if (VT == MVT::v32i8 && IsSigned) {  in LowerMULH()
23917     Lo = DAG.getBitcast(VT, Lo);  in LowerMULH()
23918     Hi = DAG.getBitcast(VT, Hi);  in LowerMULH()
23919     return DAG.getVectorShuffle(VT, dl, Lo, Hi,  in LowerMULH()
23943   if (IsSigned && VT == MVT::v16i8 && Subtarget.hasSSE41()) {  in LowerMULH()
23946     AHi = DAG.getVectorShuffle(VT, dl, A, A, PSHUFDMask);  in LowerMULH()
23949     ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), A));  in LowerMULH()
23950     AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), A));  in LowerMULH()
23955     ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A,  in LowerMULH()
23956                                           DAG.getConstant(0, dl, VT)));  in LowerMULH()
23957     AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A,  in LowerMULH()
23958                                           DAG.getConstant(0, dl, VT)));  in LowerMULH()
23985   } else if (IsSigned && VT == MVT::v16i8 && Subtarget.hasSSE41()) {  in LowerMULH()
23988     BHi = DAG.getVectorShuffle(VT, dl, B, B, PSHUFDMask);  in LowerMULH()
23991     BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), B));  in LowerMULH()
23992     BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), B));  in LowerMULH()
23997     BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B,  in LowerMULH()
23998                                           DAG.getConstant(0, dl, VT)));  in LowerMULH()
23999     BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B,  in LowerMULH()
24000                                           DAG.getConstant(0, dl, VT)));  in LowerMULH()
24011   return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);  in LowerMULH()
24016   EVT VT = Op.getValueType();  in LowerWin64_i128OP()  local
24017   assert(VT.isInteger() && VT.getSizeInBits() == 128 &&  in LowerWin64_i128OP()
24067   return DAG.getBitcast(VT, CallInfo.first);  in LowerWin64_i128OP()
24072 static bool SupportedVectorShiftWithImm(MVT VT, const X86Subtarget &Subtarget,  in SupportedVectorShiftWithImm()  argument
24074   if (VT.getScalarSizeInBits() < 16)  in SupportedVectorShiftWithImm()
24077   if (VT.is512BitVector() && Subtarget.hasAVX512() &&  in SupportedVectorShiftWithImm()
24078       (VT.getScalarSizeInBits() > 16 || Subtarget.hasBWI()))  in SupportedVectorShiftWithImm()
24081   bool LShift = (VT.is128BitVector() && Subtarget.hasSSE2()) ||  in SupportedVectorShiftWithImm()
24082                 (VT.is256BitVector() && Subtarget.hasInt256());  in SupportedVectorShiftWithImm()
24085                            (VT != MVT::v2i64 && VT != MVT::v4i64));  in SupportedVectorShiftWithImm()
24092 bool SupportedVectorShiftWithBaseAmnt(MVT VT, const X86Subtarget &Subtarget,  in SupportedVectorShiftWithBaseAmnt()  argument
24094   return SupportedVectorShiftWithImm(VT, Subtarget, Opcode);  in SupportedVectorShiftWithBaseAmnt()
24099 static bool SupportedVectorVarShift(MVT VT, const X86Subtarget &Subtarget,  in SupportedVectorVarShift()  argument
24102   if (!Subtarget.hasInt256() || VT.getScalarSizeInBits() < 16)  in SupportedVectorVarShift()
24106   if (VT.getScalarSizeInBits() == 16 && !Subtarget.hasBWI())  in SupportedVectorVarShift()
24112   bool LShift = VT.is128BitVector() || VT.is256BitVector();  in SupportedVectorVarShift()
24113   bool AShift = LShift &&  VT != MVT::v2i64 && VT != MVT::v4i64;  in SupportedVectorVarShift()
24119   MVT VT = Op.getSimpleValueType();  in LowerScalarImmediateShift()  local
24126     assert((VT == MVT::v2i64 || VT == MVT::v4i64) && "Unexpected SRA type");  in LowerScalarImmediateShift()
24127     MVT ExVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() * 2);  in LowerScalarImmediateShift()
24132       assert((VT != MVT::v4i64 || Subtarget.hasInt256()) &&  in LowerScalarImmediateShift()
24134       return DAG.getNode(X86ISD::PCMPGT, dl, VT, DAG.getConstant(0, dl, VT), R);  in LowerScalarImmediateShift()
24143       if (VT == MVT::v2i64)  in LowerScalarImmediateShift()
24145       if (VT == MVT::v4i64)  in LowerScalarImmediateShift()
24153           getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt, DAG);  in LowerScalarImmediateShift()
24155       if (VT == MVT::v2i64)  in LowerScalarImmediateShift()
24157       if (VT == MVT::v4i64)  in LowerScalarImmediateShift()
24161     return DAG.getBitcast(VT, Ex);  in LowerScalarImmediateShift()
24170   if (SupportedVectorShiftWithImm(VT, Subtarget, Op.getOpcode()))  in LowerScalarImmediateShift()
24171     return getTargetVShiftByConstNode(X86Opc, dl, VT, R, ShiftAmt, DAG);  in LowerScalarImmediateShift()
24174   if (((!Subtarget.hasXOP() && VT == MVT::v2i64) ||  in LowerScalarImmediateShift()
24175        (Subtarget.hasInt256() && VT == MVT::v4i64)) &&  in LowerScalarImmediateShift()
24179   if (VT == MVT::v16i8 || (Subtarget.hasInt256() && VT == MVT::v32i8) ||  in LowerScalarImmediateShift()
24180       VT == MVT::v64i8) {  in LowerScalarImmediateShift()
24181     unsigned NumElts = VT.getVectorNumElements();  in LowerScalarImmediateShift()
24186       return DAG.getNode(ISD::ADD, dl, VT, R, R);  in LowerScalarImmediateShift()
24190       SDValue Zeros = DAG.getConstant(0, dl, VT);  in LowerScalarImmediateShift()
24191       if (VT.is512BitVector()) {  in LowerScalarImmediateShift()
24192         assert(VT == MVT::v64i8 && "Unexpected element type!");  in LowerScalarImmediateShift()
24194         return DAG.getNode(ISD::SIGN_EXTEND, dl, VT, CMP);  in LowerScalarImmediateShift()
24196       return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R);  in LowerScalarImmediateShift()
24200     if (VT == MVT::v16i8 && Subtarget.hasXOP())  in LowerScalarImmediateShift()
24207       SHL = DAG.getBitcast(VT, SHL);  in LowerScalarImmediateShift()
24209       return DAG.getNode(ISD::AND, dl, VT, SHL,  in LowerScalarImmediateShift()
24210                          DAG.getConstant(uint8_t(-1U << ShiftAmt), dl, VT));  in LowerScalarImmediateShift()
24216       SRL = DAG.getBitcast(VT, SRL);  in LowerScalarImmediateShift()
24218       return DAG.getNode(ISD::AND, dl, VT, SRL,  in LowerScalarImmediateShift()
24219                          DAG.getConstant(uint8_t(-1U) >> ShiftAmt, dl, VT));  in LowerScalarImmediateShift()
24223       SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt);  in LowerScalarImmediateShift()
24225       SDValue Mask = DAG.getConstant(128 >> ShiftAmt, dl, VT);  in LowerScalarImmediateShift()
24226       Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask);  in LowerScalarImmediateShift()
24227       Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask);  in LowerScalarImmediateShift()
24240   EVT VT = V.getValueType();  in IsSplatVector()  local
24245     APInt DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements());  in IsSplatVector()
24250         return DAG.getUNDEF(VT);  in IsSplatVector()
24286   MVT VT = Op.getSimpleValueType();  in LowerScalarVariableShift()  local
24295     if (SupportedVectorShiftWithBaseAmnt(VT, Subtarget, Opcode)) {  in LowerScalarVariableShift()
24296       MVT EltVT = VT.getVectorElementType();  in LowerScalarVariableShift()
24303       return getTargetVShiftNode(X86OpcI, dl, VT, R, BaseShAmt, Subtarget, DAG);  in LowerScalarVariableShift()
24307     if (((VT == MVT::v16i8 && !Subtarget.canExtendTo512DQ()) ||  in LowerScalarVariableShift()
24308          (VT == MVT::v32i8 && !Subtarget.canExtendTo512BW()) ||  in LowerScalarVariableShift()
24309          VT == MVT::v64i8) &&  in LowerScalarVariableShift()
24311       unsigned NumElts = VT.getVectorNumElements();  in LowerScalarVariableShift()
24326         BitMask = DAG.getBitcast(VT, BitMask);  in LowerScalarVariableShift()
24327         BitMask = DAG.getVectorShuffle(VT, dl, BitMask, BitMask,  in LowerScalarVariableShift()
24333         Res = DAG.getBitcast(VT, Res);  in LowerScalarVariableShift()
24334         Res = DAG.getNode(ISD::AND, dl, VT, Res, BitMask);  in LowerScalarVariableShift()
24342           SignMask = DAG.getBitcast(VT, SignMask);  in LowerScalarVariableShift()
24343           Res = DAG.getNode(ISD::XOR, dl, VT, Res, SignMask);  in LowerScalarVariableShift()
24344           Res = DAG.getNode(ISD::SUB, dl, VT, Res, SignMask);  in LowerScalarVariableShift()
24352   if (VT == MVT::v2i64 && Amt.getOpcode() == ISD::BITCAST &&  in LowerScalarVariableShift()
24365     if (SupportedVectorShiftWithBaseAmnt(VT, Subtarget, Op.getOpcode()))  in LowerScalarVariableShift()
24366       return DAG.getNode(X86OpcV, dl, VT, R, Op.getOperand(1));  in LowerScalarVariableShift()
24375   MVT VT = Amt.getSimpleValueType();  in convertShiftLeftToScale()  local
24376   if (!(VT == MVT::v8i16 || VT == MVT::v4i32 ||  in convertShiftLeftToScale()
24377         (Subtarget.hasInt256() && VT == MVT::v16i16) ||  in convertShiftLeftToScale()
24378         (!Subtarget.hasAVX512() && VT == MVT::v16i8)))  in convertShiftLeftToScale()
24383     MVT SVT = VT.getVectorElementType();  in convertShiftLeftToScale()
24386     unsigned NumElems = VT.getVectorNumElements();  in convertShiftLeftToScale()
24404     return DAG.getBuildVector(VT, dl, Elts);  in convertShiftLeftToScale()
24409   if (VT == MVT::v4i32) {  in convertShiftLeftToScale()
24410     Amt = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, dl, VT));  in convertShiftLeftToScale()
24411     Amt = DAG.getNode(ISD::ADD, dl, VT, Amt,  in convertShiftLeftToScale()
24412                       DAG.getConstant(0x3f800000U, dl, VT));  in convertShiftLeftToScale()
24414     return DAG.getNode(ISD::FP_TO_SINT, dl, VT, Amt);  in convertShiftLeftToScale()
24418   if (VT == MVT::v8i16 && !Subtarget.hasAVX2()) {  in convertShiftLeftToScale()
24419     SDValue Z = DAG.getConstant(0, dl, VT);  in convertShiftLeftToScale()
24420     SDValue Lo = DAG.getBitcast(MVT::v4i32, getUnpackl(DAG, dl, VT, Amt, Z));  in convertShiftLeftToScale()
24421     SDValue Hi = DAG.getBitcast(MVT::v4i32, getUnpackh(DAG, dl, VT, Amt, Z));  in convertShiftLeftToScale()
24425       return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi);  in convertShiftLeftToScale()
24427     return DAG.getVectorShuffle(VT, dl, DAG.getBitcast(VT, Lo),  in convertShiftLeftToScale()
24428                                         DAG.getBitcast(VT, Hi),  in convertShiftLeftToScale()
24437   MVT VT = Op.getSimpleValueType();  in LowerShift()  local
24441   unsigned EltSizeInBits = VT.getScalarSizeInBits();  in LowerShift()
24448   assert(VT.isVector() && "Custom lowering only for vector shifts!");  in LowerShift()
24457   if (SupportedVectorVarShift(VT, Subtarget, Opc))  in LowerShift()
24462   if (Subtarget.hasXOP() && (VT == MVT::v2i64 || VT == MVT::v4i32 ||  in LowerShift()
24463                              VT == MVT::v8i16 || VT == MVT::v16i8)) {  in LowerShift()
24465       SDValue Zero = DAG.getConstant(0, dl, VT);  in LowerShift()
24466       Amt = DAG.getNode(ISD::SUB, dl, VT, Zero, Amt);  in LowerShift()
24469       return DAG.getNode(X86ISD::VPSHL, dl, VT, R, Amt);  in LowerShift()
24471       return DAG.getNode(X86ISD::VPSHA, dl, VT, R, Amt);  in LowerShift()
24476   if (VT == MVT::v2i64 && Opc != ISD::SRA) {  in LowerShift()
24478     SDValue Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {0, 0});  in LowerShift()
24479     SDValue Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {1, 1});  in LowerShift()
24480     SDValue R0 = DAG.getNode(Opc, dl, VT, R, Amt0);  in LowerShift()
24481     SDValue R1 = DAG.getNode(Opc, dl, VT, R, Amt1);  in LowerShift()
24482     return DAG.getVectorShuffle(VT, dl, R0, R1, {0, 3});  in LowerShift()
24488   if ((VT == MVT::v2i64 || (VT == MVT::v4i64 && Subtarget.hasInt256())) &&  in LowerShift()
24490     SDValue S = DAG.getConstant(APInt::getSignMask(64), dl, VT);  in LowerShift()
24491     SDValue M = DAG.getNode(ISD::SRL, dl, VT, S, Amt);  in LowerShift()
24492     R = DAG.getNode(ISD::SRL, dl, VT, R, Amt);  in LowerShift()
24493     R = DAG.getNode(ISD::XOR, dl, VT, R, M);  in LowerShift()
24494     R = DAG.getNode(ISD::SUB, dl, VT, R, M);  in LowerShift()
24508   if (ConstantAmt && (VT == MVT::v8i16 || VT == MVT::v4i32 ||  in LowerShift()
24509                       (VT == MVT::v16i16 && Subtarget.hasInt256()))) {  in LowerShift()
24511     unsigned NumElts = VT.getVectorNumElements();  in LowerShift()
24534         (VT != MVT::v16i16 ||  in LowerShift()
24535          is128BitLaneRepeatedShuffleMask(VT, ShuffleMask)) &&  in LowerShift()
24536         (VT == MVT::v4i32 || Subtarget.hasSSE41() || Opc != ISD::SHL ||  in LowerShift()
24542         SDValue Shift1 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R,  in LowerShift()
24544         SDValue Shift2 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R,  in LowerShift()
24546         return DAG.getVectorShuffle(VT, dl, Shift1, Shift2, ShuffleMask);  in LowerShift()
24555       return DAG.getNode(ISD::MUL, dl, VT, R, Scale);  in LowerShift()
24560       (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256()))) {  in LowerShift()
24561     SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);  in LowerShift()
24562     SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);  in LowerShift()
24564       SDValue Zero = DAG.getConstant(0, dl, VT);  in LowerShift()
24565       SDValue ZAmt = DAG.getSetCC(dl, VT, Amt, Zero, ISD::SETEQ);  in LowerShift()
24566       SDValue Res = DAG.getNode(ISD::MULHU, dl, VT, R, Scale);  in LowerShift()
24567       return DAG.getSelect(dl, VT, ZAmt, R, Res);  in LowerShift()
24576       (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256())) &&  in LowerShift()
24580     SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);  in LowerShift()
24581     SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);  in LowerShift()
24584           DAG.getSetCC(dl, VT, Amt, DAG.getConstant(0, dl, VT), ISD::SETEQ);  in LowerShift()
24586           DAG.getSetCC(dl, VT, Amt, DAG.getConstant(1, dl, VT), ISD::SETEQ);  in LowerShift()
24588           getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, 1, DAG);  in LowerShift()
24589       SDValue Res = DAG.getNode(ISD::MULHS, dl, VT, R, Scale);  in LowerShift()
24590       Res = DAG.getSelect(dl, VT, Amt0, R, Res);  in LowerShift()
24591       return DAG.getSelect(dl, VT, Amt1, Sra1, Res);  in LowerShift()
24600   if (VT == MVT::v4i32) {  in LowerShift()
24603       Amt0 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {0, 0, 0, 0});  in LowerShift()
24604       Amt1 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {1, 1, 1, 1});  in LowerShift()
24605       Amt2 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {2, 2, 2, 2});  in LowerShift()
24606       Amt3 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {3, 3, 3, 3});  in LowerShift()
24613         SDValue Z = DAG.getConstant(0, dl, VT);  in LowerShift()
24614         Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Z, {0, 4, -1, -1});  in LowerShift()
24615         Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Z, {1, 5, -1, -1});  in LowerShift()
24616         Amt2 = DAG.getVectorShuffle(VT, dl, Amt, Z, {2, 6, -1, -1});  in LowerShift()
24617         Amt3 = DAG.getVectorShuffle(VT, dl, Amt, Z, {3, 7, -1, -1});  in LowerShift()
24634     SDValue R0 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt0));  in LowerShift()
24635     SDValue R1 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt1));  in LowerShift()
24636     SDValue R2 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt2));  in LowerShift()
24637     SDValue R3 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt3));  in LowerShift()
24642       SDValue R02 = DAG.getVectorShuffle(VT, dl, R0, R2, {0, -1, 6, -1});  in LowerShift()
24643       SDValue R13 = DAG.getVectorShuffle(VT, dl, R1, R3, {-1, 1, -1, 7});  in LowerShift()
24644       return DAG.getVectorShuffle(VT, dl, R02, R13, {0, 5, 2, 7});  in LowerShift()
24646     SDValue R01 = DAG.getVectorShuffle(VT, dl, R0, R1, {0, -1, -1, 5});  in LowerShift()
24647     SDValue R23 = DAG.getVectorShuffle(VT, dl, R2, R3, {2, -1, -1, 7});  in LowerShift()
24648     return DAG.getVectorShuffle(VT, dl, R01, R23, {0, 3, 4, 7});  in LowerShift()
24655   if ((Subtarget.hasInt256() && VT == MVT::v8i16) ||  in LowerShift()
24656       (Subtarget.canExtendTo512DQ() && VT == MVT::v16i16) ||  in LowerShift()
24657       (Subtarget.canExtendTo512DQ() && VT == MVT::v16i8) ||  in LowerShift()
24658       (Subtarget.canExtendTo512BW() && VT == MVT::v32i8) ||  in LowerShift()
24659       (Subtarget.hasBWI() && Subtarget.hasVLX() && VT == MVT::v16i8)) {  in LowerShift()
24660     assert((!Subtarget.hasBWI() || VT == MVT::v32i8 || VT == MVT::v16i8) &&  in LowerShift()
24663     MVT ExtVT = MVT::getVectorVT(EvtSVT, VT.getVectorNumElements());  in LowerShift()
24667     return DAG.getNode(ISD::TRUNCATE, dl, VT,  in LowerShift()
24674       (VT == MVT::v16i8 || VT == MVT::v64i8 ||  in LowerShift()
24675        (VT == MVT::v32i8 && Subtarget.hasInt256())) &&  in LowerShift()
24677     int NumElts = VT.getVectorNumElements();  in LowerShift()
24689     if (VT == MVT::v16i8 && Subtarget.hasInt256()) {  in LowerShift()
24694       return DAG.getZExtOrTrunc(R, dl, VT);  in LowerShift()
24709     SDValue LoR = DAG.getBitcast(VT16, getUnpackl(DAG, dl, VT, R, R));  in LowerShift()
24710     SDValue HiR = DAG.getBitcast(VT16, getUnpackh(DAG, dl, VT, R, R));  in LowerShift()
24717     return DAG.getNode(X86ISD::PACKUS, dl, VT, LoR, HiR);  in LowerShift()
24720   if (VT == MVT::v16i8 ||  in LowerShift()
24721       (VT == MVT::v32i8 && Subtarget.hasInt256() && !Subtarget.hasXOP()) ||  in LowerShift()
24722       (VT == MVT::v64i8 && Subtarget.hasBWI())) {  in LowerShift()
24723     MVT ExtVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements() / 2);  in LowerShift()
24726       if (VT.is512BitVector()) {  in LowerShift()
24730         MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements());  in LowerShift()
24731         V0 = DAG.getBitcast(VT, V0);  in LowerShift()
24732         V1 = DAG.getBitcast(VT, V1);  in LowerShift()
24733         Sel = DAG.getBitcast(VT, Sel);  in LowerShift()
24734         Sel = DAG.getSetCC(dl, MaskVT, DAG.getConstant(0, dl, VT), Sel,  in LowerShift()
24736         return DAG.getBitcast(SelVT, DAG.getSelect(dl, VT, Sel, V0, V1));  in LowerShift()
24740         V0 = DAG.getBitcast(VT, V0);  in LowerShift()
24741         V1 = DAG.getBitcast(VT, V1);  in LowerShift()
24742         Sel = DAG.getBitcast(VT, Sel);  in LowerShift()
24743         return DAG.getBitcast(SelVT, DAG.getSelect(dl, VT, Sel, V0, V1));  in LowerShift()
24758     Amt = DAG.getBitcast(VT, Amt);  in LowerShift()
24762       SDValue M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(4, dl, VT));  in LowerShift()
24763       R = SignBitSelect(VT, Amt, M, R);  in LowerShift()
24766       Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);  in LowerShift()
24769       M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(2, dl, VT));  in LowerShift()
24770       R = SignBitSelect(VT, Amt, M, R);  in LowerShift()
24773       Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);  in LowerShift()
24776       M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(1, dl, VT));  in LowerShift()
24777       R = SignBitSelect(VT, Amt, M, R);  in LowerShift()
24785       SDValue ALo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), Amt);  in LowerShift()
24786       SDValue AHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), Amt);  in LowerShift()
24787       SDValue RLo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), R);  in LowerShift()
24788       SDValue RHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), R);  in LowerShift()
24824       return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);  in LowerShift()
24828   if (Subtarget.hasInt256() && !Subtarget.hasXOP() && VT == MVT::v16i16) {  in LowerShift()
24830     SDValue Z = DAG.getConstant(0, dl, VT);  in LowerShift()
24831     SDValue ALo = getUnpackl(DAG, dl, VT, Amt, Z);  in LowerShift()
24832     SDValue AHi = getUnpackh(DAG, dl, VT, Amt, Z);  in LowerShift()
24833     SDValue RLo = getUnpackl(DAG, dl, VT, Z, R);  in LowerShift()
24834     SDValue RHi = getUnpackh(DAG, dl, VT, Z, R);  in LowerShift()
24843     return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi);  in LowerShift()
24846   if (VT == MVT::v8i16) {  in LowerShift()
24856         MVT ExtVT = MVT::getVectorVT(MVT::i8, VT.getVectorNumElements() * 2);  in LowerShift()
24860         return DAG.getBitcast(VT, DAG.getSelect(dl, ExtVT, Sel, V0, V1));  in LowerShift()
24866           getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, Sel, 15, DAG);  in LowerShift()
24867       return DAG.getSelect(dl, VT, C, V0, V1);  in LowerShift()
24875           ISD::OR, dl, VT,  in LowerShift()
24876           getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 4, DAG),  in LowerShift()
24877           getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG));  in LowerShift()
24879       Amt = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG);  in LowerShift()
24883     SDValue M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 8, DAG);  in LowerShift()
24887     Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);  in LowerShift()
24890     M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 4, DAG);  in LowerShift()
24894     Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);  in LowerShift()
24897     M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 2, DAG);  in LowerShift()
24901     Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);  in LowerShift()
24904     M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 1, DAG);  in LowerShift()
24910   if (VT.is256BitVector())  in LowerShift()
24918   MVT VT = Op.getSimpleValueType();  in LowerRotate()  local
24919   assert(VT.isVector() && "Custom lowering only for vector rotates!");  in LowerRotate()
24925   unsigned EltSizeInBits = VT.getScalarSizeInBits();  in LowerRotate()
24926   int NumElts = VT.getVectorNumElements();  in LowerRotate()
24949       return DAG.getNode(Op, DL, VT, R,  in LowerRotate()
24963     if (VT.is256BitVector())  in LowerRotate()
24965     assert(VT.is128BitVector() && "Only rotate 128-bit vectors!");  in LowerRotate()
24970       return DAG.getNode(X86ISD::VROTLI, DL, VT, R,  in LowerRotate()
24979   if (VT.is256BitVector() && !Subtarget.hasAVX2())  in LowerRotate()
24982   assert((VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 ||  in LowerRotate()
24983           ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8) &&  in LowerRotate()
25006         V0 = DAG.getBitcast(VT, V0);  in LowerRotate()
25007         V1 = DAG.getBitcast(VT, V1);  in LowerRotate()
25008         Sel = DAG.getBitcast(VT, Sel);  in LowerRotate()
25009         return DAG.getBitcast(SelVT, DAG.getSelect(DL, VT, Sel, V0, V1));  in LowerRotate()
25024     Amt = DAG.getBitcast(VT, Amt);  in LowerRotate()
25029         ISD::OR, DL, VT,  in LowerRotate()
25030         DAG.getNode(ISD::SHL, DL, VT, R, DAG.getConstant(4, DL, VT)),  in LowerRotate()
25031         DAG.getNode(ISD::SRL, DL, VT, R, DAG.getConstant(4, DL, VT)));  in LowerRotate()
25032     R = SignBitSelect(VT, Amt, M, R);  in LowerRotate()
25035     Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt);  in LowerRotate()
25039         ISD::OR, DL, VT,  in LowerRotate()
25040         DAG.getNode(ISD::SHL, DL, VT, R, DAG.getConstant(2, DL, VT)),  in LowerRotate()
25041         DAG.getNode(ISD::SRL, DL, VT, R, DAG.getConstant(6, DL, VT)));  in LowerRotate()
25042     R = SignBitSelect(VT, Amt, M, R);  in LowerRotate()
25045     Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt);  in LowerRotate()
25049         ISD::OR, DL, VT,  in LowerRotate()
25050         DAG.getNode(ISD::SHL, DL, VT, R, DAG.getConstant(1, DL, VT)),  in LowerRotate()
25051         DAG.getNode(ISD::SRL, DL, VT, R, DAG.getConstant(7, DL, VT)));  in LowerRotate()
25052     return SignBitSelect(VT, Amt, M, R);  in LowerRotate()
25056   Amt = DAG.getNode(ISD::AND, DL, VT, Amt,  in LowerRotate()
25057                     DAG.getConstant(EltSizeInBits - 1, DL, VT));  in LowerRotate()
25060   bool LegalVarShifts = SupportedVectorVarShift(VT, Subtarget, ISD::SHL) &&  in LowerRotate()
25061                         SupportedVectorVarShift(VT, Subtarget, ISD::SRL);  in LowerRotate()
25066     SDValue AmtR = DAG.getConstant(EltSizeInBits, DL, VT);  in LowerRotate()
25067     AmtR = DAG.getNode(ISD::SUB, DL, VT, AmtR, Amt);  in LowerRotate()
25068     SDValue SHL = DAG.getNode(ISD::SHL, DL, VT, R, Amt);  in LowerRotate()
25069     SDValue SRL = DAG.getNode(ISD::SRL, DL, VT, R, AmtR);  in LowerRotate()
25070     return DAG.getNode(ISD::OR, DL, VT, SHL, SRL);  in LowerRotate()
25079     SDValue Lo = DAG.getNode(ISD::MUL, DL, VT, R, Scale);  in LowerRotate()
25080     SDValue Hi = DAG.getNode(ISD::MULHU, DL, VT, R, Scale);  in LowerRotate()
25081     return DAG.getNode(ISD::OR, DL, VT, Lo, Hi);  in LowerRotate()
25087   assert(VT == MVT::v4i32 && "Only v4i32 vector rotate expected");  in LowerRotate()
25089   SDValue R13 = DAG.getVectorShuffle(VT, DL, R, R, OddMask);  in LowerRotate()
25090   SDValue Scale13 = DAG.getVectorShuffle(VT, DL, Scale, Scale, OddMask);  in LowerRotate()
25098   Res02 = DAG.getBitcast(VT, Res02);  in LowerRotate()
25099   Res13 = DAG.getBitcast(VT, Res13);  in LowerRotate()
25101   return DAG.getNode(ISD::OR, DL, VT,  in LowerRotate()
25102                      DAG.getVectorShuffle(VT, DL, Res02, Res13, {0, 4, 2, 6}),  in LowerRotate()
25103                      DAG.getVectorShuffle(VT, DL, Res02, Res13, {1, 5, 3, 7}));  in LowerRotate()
25423 static SDValue LowerHorizontalByteSum(SDValue V, MVT VT,  in LowerHorizontalByteSum()  argument
25428   MVT EltVT = VT.getVectorElementType();  in LowerHorizontalByteSum()
25433   unsigned VecSize = VT.getSizeInBits();  in LowerHorizontalByteSum()
25442     return DAG.getBitcast(VT, V);  in LowerHorizontalByteSum()
25451     SDValue Zeros = DAG.getConstant(0, DL, VT);  in LowerHorizontalByteSum()
25452     SDValue V32 = DAG.getBitcast(VT, V);  in LowerHorizontalByteSum()
25453     SDValue Low = getUnpackl(DAG, DL, VT, V32, Zeros);  in LowerHorizontalByteSum()
25454     SDValue High = getUnpackh(DAG, DL, VT, V32, Zeros);  in LowerHorizontalByteSum()
25470     return DAG.getBitcast(VT, V);  in LowerHorizontalByteSum()
25480   SDValue ShifterV = DAG.getConstant(8, DL, VT);  in LowerHorizontalByteSum()
25481   SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, DAG.getBitcast(VT, V), ShifterV);  in LowerHorizontalByteSum()
25484   return DAG.getNode(ISD::SRL, DL, VT, DAG.getBitcast(VT, V), ShifterV);  in LowerHorizontalByteSum()
25490   MVT VT = Op.getSimpleValueType();  in LowerVectorCTPOPInRegLUT()  local
25491   MVT EltVT = VT.getVectorElementType();  in LowerVectorCTPOPInRegLUT()
25492   int NumElts = VT.getVectorNumElements();  in LowerVectorCTPOPInRegLUT()
25514   SDValue InRegLUT = DAG.getBuildVector(VT, DL, LUTVec);  in LowerVectorCTPOPInRegLUT()
25515   SDValue M0F = DAG.getConstant(0x0F, DL, VT);  in LowerVectorCTPOPInRegLUT()
25518   SDValue FourV = DAG.getConstant(4, DL, VT);  in LowerVectorCTPOPInRegLUT()
25519   SDValue HiNibbles = DAG.getNode(ISD::SRL, DL, VT, Op, FourV);  in LowerVectorCTPOPInRegLUT()
25522   SDValue LoNibbles = DAG.getNode(ISD::AND, DL, VT, Op, M0F);  in LowerVectorCTPOPInRegLUT()
25527   SDValue HiPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, HiNibbles);  in LowerVectorCTPOPInRegLUT()
25528   SDValue LoPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, LoNibbles);  in LowerVectorCTPOPInRegLUT()
25529   return DAG.getNode(ISD::ADD, DL, VT, HiPopCnt, LoPopCnt);  in LowerVectorCTPOPInRegLUT()
25536   MVT VT = Op.getSimpleValueType();  in LowerVectorCTPOP()  local
25537   assert((VT.is512BitVector() || VT.is256BitVector() || VT.is128BitVector()) &&  in LowerVectorCTPOP()
25544     unsigned NumElems = VT.getVectorNumElements();  in LowerVectorCTPOP()
25545     assert((VT.getVectorElementType() == MVT::i8 ||  in LowerVectorCTPOP()
25546             VT.getVectorElementType() == MVT::i16) && "Unexpected type");  in LowerVectorCTPOP()
25551       return DAG.getNode(ISD::TRUNCATE, DL, VT, Op);  in LowerVectorCTPOP()
25556   if (VT.is256BitVector() && !Subtarget.hasInt256())  in LowerVectorCTPOP()
25560   if (VT.is512BitVector() && !Subtarget.hasBWI())  in LowerVectorCTPOP()
25564   if (VT.getScalarType() != MVT::i8) {  in LowerVectorCTPOP()
25565     MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);  in LowerVectorCTPOP()
25568     return LowerHorizontalByteSum(PopCnt8, VT, Subtarget, DAG);  in LowerVectorCTPOP()
25586   MVT VT = Op.getSimpleValueType();  in LowerBITREVERSE_XOP()  local
25592   if (!VT.isVector()) {  in LowerBITREVERSE_XOP()
25593     MVT VecVT = MVT::getVectorVT(VT, 128 / VT.getSizeInBits());  in LowerBITREVERSE_XOP()
25596     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Res,  in LowerBITREVERSE_XOP()
25600   int NumElts = VT.getVectorNumElements();  in LowerBITREVERSE_XOP()
25601   int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;  in LowerBITREVERSE_XOP()
25604   if (VT.is256BitVector())  in LowerBITREVERSE_XOP()
25607   assert(VT.is128BitVector() &&  in LowerBITREVERSE_XOP()
25627   return DAG.getBitcast(VT, Res);  in LowerBITREVERSE_XOP()
25632   MVT VT = Op.getSimpleValueType();  in LowerBITREVERSE()  local
25634   if (Subtarget.hasXOP() && !VT.is512BitVector())  in LowerBITREVERSE()
25642   unsigned NumElts = VT.getVectorNumElements();  in LowerBITREVERSE()
25643   assert(VT.getScalarType() == MVT::i8 &&  in LowerBITREVERSE()
25647   if (VT.is256BitVector() && !Subtarget.hasInt256())  in LowerBITREVERSE()
25653   SDValue NibbleMask = DAG.getConstant(0xF, DL, VT);  in LowerBITREVERSE()
25654   SDValue Lo = DAG.getNode(ISD::AND, DL, VT, In, NibbleMask);  in LowerBITREVERSE()
25655   SDValue Hi = DAG.getNode(ISD::SRL, DL, VT, In, DAG.getConstant(4, DL, VT));  in LowerBITREVERSE()
25674   SDValue LoMask = DAG.getBuildVector(VT, DL, LoMaskElts);  in LowerBITREVERSE()
25675   SDValue HiMask = DAG.getBuildVector(VT, DL, HiMaskElts);  in LowerBITREVERSE()
25676   Lo = DAG.getNode(X86ISD::PSHUFB, DL, VT, LoMask, Lo);  in LowerBITREVERSE()
25677   Hi = DAG.getNode(X86ISD::PSHUFB, DL, VT, HiMask, Hi);  in LowerBITREVERSE()
25678   return DAG.getNode(ISD::OR, DL, VT, Lo, Hi);  in LowerBITREVERSE()
25719   MVT VT = N->getSimpleValueType(0);  in lowerAtomicArith()  local
25730       RHS = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS);  in lowerAtomicArith()
25731       return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, VT, Chain, LHS,  in lowerAtomicArith()
25749   EVT VT = cast<AtomicSDNode>(Node)->getMemoryVT();  in LowerATOMIC_STORE()  local
25758       !DAG.getTargetLoweringInfo().isTypeLegal(VT)) {  in LowerATOMIC_STORE()
25772   MVT VT = N->getSimpleValueType(0);  in LowerADDSUBCARRY()  local
25775   if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))  in LowerADDSUBCARRY()
25778   SDVTList VTs = DAG.getVTList(VT, MVT::i32);  in LowerADDSUBCARRY()
25911   MVT VT = Src.getSimpleValueType();  in LowerMSCATTER()  local
25912   assert(VT.getScalarSizeInBits() >= 32 && "Unsupported scatter op");  in LowerMSCATTER()
25921   if (VT == MVT::v2f32) {  in LowerMSCATTER()
25937   if (VT == MVT::v2i32) {  in LowerMSCATTER()
25972   if (!Subtarget.hasVLX() && !VT.is512BitVector() &&  in LowerMSCATTER()
25975     unsigned Factor = std::min(512/VT.getSizeInBits(),  in LowerMSCATTER()
25977     unsigned NumElts = VT.getVectorNumElements() * Factor;  in LowerMSCATTER()
25979     VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);  in LowerMSCATTER()
25983     Src = ExtendToType(Src, VT, DAG);  in LowerMSCATTER()
26000   MVT VT = Op.getSimpleValueType();  in LowerMLOAD()  local
26001   MVT ScalarVT = VT.getScalarType();  in LowerMLOAD()
26011   assert(Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() &&  in LowerMLOAD()
26021   unsigned NumEltsInWideVec = 512 / VT.getScalarSizeInBits();  in LowerMLOAD()
26038   SDValue Exract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT,  in LowerMLOAD()
26049   MVT VT = DataToStore.getSimpleValueType();  in LowerMSTORE()  local
26050   MVT ScalarVT = VT.getScalarType();  in LowerMSTORE()
26060   assert(Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() &&  in LowerMSTORE()
26070   unsigned NumEltsInWideVec = 512/VT.getScalarSizeInBits();  in LowerMSTORE()
26093   MVT VT = Op.getSimpleValueType();  in LowerMGATHER()  local
26100   assert(VT.getScalarSizeInBits() >= 32 && "Unsupported gather op");  in LowerMGATHER()
26108   MVT OrigVT = VT;  in LowerMGATHER()
26109   if (Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() &&  in LowerMGATHER()
26112     unsigned Factor = std::min(512/VT.getSizeInBits(),  in LowerMGATHER()
26115     unsigned NumElts = VT.getVectorNumElements() * Factor;  in LowerMGATHER()
26117     VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);  in LowerMGATHER()
26121     PassThru = ExtendToType(PassThru, VT, DAG);  in LowerMGATHER()
26129       DAG.getVTList(VT, MaskVT, MVT::Other), Ops, dl, N->getMemoryVT(),  in LowerMGATHER()
26333     EVT VT = N->getValueType(0);  in ReplaceNodeResults()  local
26334     assert(VT.isVector() && "Unexpected VT");  in ReplaceNodeResults()
26335     if (getTypeAction(*DAG.getContext(), VT) == TypePromoteInteger &&  in ReplaceNodeResults()
26336         VT.getVectorNumElements() == 2) {  in ReplaceNodeResults()
26343       Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, VT, Mul));  in ReplaceNodeResults()
26344     } else if (getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&  in ReplaceNodeResults()
26345                VT.getVectorElementType() == MVT::i8) {  in ReplaceNodeResults()
26348       MVT MulVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements());  in ReplaceNodeResults()
26352       Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);  in ReplaceNodeResults()
26353       unsigned NumConcats = 16 / VT.getVectorNumElements();  in ReplaceNodeResults()
26354       SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));  in ReplaceNodeResults()
26371     EVT VT = N->getValueType(0);  in ReplaceNodeResults()  local
26373     assert(VT.getSizeInBits() < 128 && 128 % VT.getSizeInBits() == 0 &&  in ReplaceNodeResults()
26381                                   VT.getVectorElementType(),  in ReplaceNodeResults()
26382                                   NumConcat * VT.getVectorNumElements());  in ReplaceNodeResults()
26391     if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector)  in ReplaceNodeResults()
26392       Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,  in ReplaceNodeResults()
26422     EVT VT = N->getValueType(0);  in ReplaceNodeResults()  local
26423     assert(VT == MVT::v2f32 && "Unexpected type (!= v2f32) on FMIN/FMAX.");  in ReplaceNodeResults()
26424     SDValue UNDEF = DAG.getUNDEF(VT);  in ReplaceNodeResults()
26436     EVT VT = N->getValueType(0);  in ReplaceNodeResults()  local
26437     if (getTypeAction(*DAG.getContext(), VT) == TypeWidenVector) {  in ReplaceNodeResults()
26443         unsigned NumConcats = 128 / VT.getSizeInBits();  in ReplaceNodeResults()
26444         SmallVector<SDValue, 8> Ops0(NumConcats, DAG.getUNDEF(VT));  in ReplaceNodeResults()
26446         EVT ResVT = getTypeToTransformTo(*DAG.getContext(), VT);  in ReplaceNodeResults()
26455     if (VT == MVT::v2i32) {  in ReplaceNodeResults()
26463     if (VT.isVector())  in ReplaceNodeResults()
26475     MVT VT = N->getSimpleValueType(0);  in ReplaceNodeResults()  local
26476     if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector)  in ReplaceNodeResults()
26482     MVT WidenVT = getTypeToTransformTo(*DAG.getContext(), VT).getSimpleVT();  in ReplaceNodeResults()
26492       EVT EltVT = VT.getVectorElementType();  in ReplaceNodeResults()
26497       unsigned MinElts = VT.getVectorNumElements();  in ReplaceNodeResults()
26516       if (InVT == MVT::v4i64 && VT == MVT::v4i8 && isTypeLegal(MVT::v8i64)) {  in ReplaceNodeResults()
26529     EVT VT = N->getValueType(0);  in ReplaceNodeResults()  local
26532     if (!Subtarget.hasSSE41() && VT == MVT::v4i64 &&  in ReplaceNodeResults()
26542                                       VT.getVectorNumElements());  in ReplaceNodeResults()
26561       SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);  in ReplaceNodeResults()
26572     EVT VT = N->getValueType(0);  in ReplaceNodeResults()  local
26575     if (!Subtarget.hasSSE41() && VT == MVT::v4i64 &&  in ReplaceNodeResults()
26596       SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);  in ReplaceNodeResults()
26601     if ((VT == MVT::v16i32 || VT == MVT::v8i64) && InVT.is128BitVector()) {  in ReplaceNodeResults()
26622       SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);  in ReplaceNodeResults()
26630     EVT VT = N->getValueType(0);  in ReplaceNodeResults()  local
26636     if ((VT == MVT::v2i8 || VT == MVT::v2i16) &&  in ReplaceNodeResults()
26637         getTypeAction(*DAG.getContext(), VT) == TypePromoteInteger) {  in ReplaceNodeResults()
26646                         DAG.getValueType(VT.getVectorElementType()));  in ReplaceNodeResults()
26647       Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);  in ReplaceNodeResults()
26652     if (VT.isVector() && VT.getScalarSizeInBits() < 32) {  in ReplaceNodeResults()
26653       if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector)  in ReplaceNodeResults()
26657       unsigned NewEltWidth = std::min(128 / VT.getVectorNumElements(), 32U);  in ReplaceNodeResults()
26659                                        VT.getVectorNumElements());  in ReplaceNodeResults()
26668                           DAG.getValueType(VT.getVectorElementType()));  in ReplaceNodeResults()
26671       Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);  in ReplaceNodeResults()
26674       unsigned NumConcats = 128 / VT.getSizeInBits();  in ReplaceNodeResults()
26675       MVT ConcatVT = MVT::getVectorVT(VT.getSimpleVT().getVectorElementType(),  in ReplaceNodeResults()
26676                                       VT.getVectorNumElements() * NumConcats);  in ReplaceNodeResults()
26677       SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));  in ReplaceNodeResults()
26685     if (VT == MVT::v2i32) {  in ReplaceNodeResults()
26711           getTypeAction(*DAG.getContext(), VT) != TypeWidenVector) {  in ReplaceNodeResults()
26727     if (Subtarget.hasDQI() && VT == MVT::i64 &&  in ReplaceNodeResults()
26742       Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Res, ZeroIdx);  in ReplaceNodeResults()
26754             DAG.getLoad(VT, dl, FIST, StackSlot, MachinePointerInfo()));  in ReplaceNodeResults()
26770     EVT VT = N->getValueType(0);  in ReplaceNodeResults()  local
26771     if (VT != MVT::v2f32)  in ReplaceNodeResults()
26979     EVT VT = N->getValueType(0);  in ReplaceNodeResults()  local
26980     if (VT == MVT::v2f32 && (Subtarget.hasVLX() || !Subtarget.hasAVX512())) {  in ReplaceNodeResults()
27006     if (VT == MVT::v2i32) {  in ReplaceNodeResults()
27030         if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector)  in ReplaceNodeResults()
27037       if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector) {  in ReplaceNodeResults()
27066     MVT VT = N->getSimpleValueType(0);  in ReplaceNodeResults()  local
27067     assert(VT.isVector() && VT.getSizeInBits() == 64 && "Unexpected VT");  in ReplaceNodeResults()
27068     if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector)  in ReplaceNodeResults()
27073     MVT LdVT = Subtarget.is64Bit() && VT.isInteger() ? MVT::i64 : MVT::f64;  in ReplaceNodeResults()
27081     MVT CastVT = MVT::getVectorVT(VT.getVectorElementType(),  in ReplaceNodeResults()
27082                                   VT.getVectorNumElements() * 2);  in ReplaceNodeResults()
27580 X86TargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const {  in isFMAFasterThanFMulAndFAdd()
27584   VT = VT.getScalarType();  in isFMAFasterThanFMulAndFAdd()
27586   if (!VT.isSimple())  in isFMAFasterThanFMulAndFAdd()
27589   switch (VT.getSimpleVT().SimpleTy) {  in isFMAFasterThanFMulAndFAdd()
27609 bool X86TargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {  in isShuffleMaskLegal()
27610   if (!VT.isSimple())  in isShuffleMaskLegal()
27614   if (VT.getSimpleVT().getScalarType() == MVT::i1)  in isShuffleMaskLegal()
27618   if (VT.getSimpleVT().getSizeInBits() == 64)  in isShuffleMaskLegal()
27623   return isTypeLegal(VT.getSimpleVT());  in isShuffleMaskLegal()
27627                                                EVT VT) const {  in isVectorClearMaskLegal()
27631     if (VT == MVT::v32i8 || VT == MVT::v16i16)  in isVectorClearMaskLegal()
27635   return isShuffleMaskLegal(Mask, VT);  in isVectorClearMaskLegal()
30024   EVT VT = Op.getValueType();  in targetShrinkDemandedConstant()  local
30027   if (VT.isVector())  in targetShrinkDemandedConstant()
30030   unsigned Size = VT.getSizeInBits();  in targetShrinkDemandedConstant()
30069   SDValue NewC = TLO.DAG.getConstant(ZeroExtendMask, DL, VT);  in targetShrinkDemandedConstant()
30070   SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);  in targetShrinkDemandedConstant()
30081   EVT VT = Op.getValueType();  in computeKnownBitsForTargetNode()  local
30115       if (ShiftImm->getAPIntValue().uge(VT.getScalarSizeInBits())) {  in computeKnownBitsForTargetNode()
30142     getPackDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS);  in computeKnownBitsForTargetNode()
30184     if (getTargetShuffleMask(Op.getNode(), VT.getSimpleVT(), true, Ops, Mask,  in computeKnownBitsForTargetNode()
30187       unsigned NumElts = VT.getVectorNumElements();  in computeKnownBitsForTargetNode()
30209           if (Ops[OpIdx].getValueType() != VT) {  in computeKnownBitsForTargetNode()
31248   MVT VT = Root.getSimpleValueType();  in combineX86ShufflesConstants()  local
31250   unsigned SizeInBits = VT.getSizeInBits();  in combineX86ShufflesConstants()
31313   if (VT.isFloatingPoint() && (MaskSizeInBits == 32 || MaskSizeInBits == 64))  in combineX86ShufflesConstants()
31322   return DAG.getBitcast(VT, CstOp);  in combineX86ShufflesConstants()
31369   MVT VT = Op.getSimpleValueType();  in combineX86ShufflesRecursively()  local
31370   if (!VT.isVector())  in combineX86ShufflesRecursively()
31375   assert(VT.getSizeInBits() == Root.getSimpleValueType().getSizeInBits() &&  in combineX86ShufflesRecursively()
31574   MVT VT = N.getSimpleValueType();  in getPSHUFShuffleMask()  local
31579       getTargetShuffleMask(N.getNode(), VT, false, Ops, Mask, IsUnary);  in getPSHUFShuffleMask()
31585   if (VT.getSizeInBits() > 128) {  in getPSHUFShuffleMask()
31586     int LaneElts = 128 / VT.getScalarSizeInBits();  in getPSHUFShuffleMask()
31588     for (int i = 1, NumLanes = VT.getSizeInBits() / 128; i < NumLanes; ++i)  in getPSHUFShuffleMask()
31750   MVT VT = N.getSimpleValueType();  in combineTargetShuffle()  local
31756   if (VT.getScalarSizeInBits() == 64 &&  in combineTargetShuffle()
31778       return DAG.getBitcast(VT, Horiz);  in combineTargetShuffle()
31791         VT.getScalarSizeInBits() % BCVT.getScalarSizeInBits() == 0) {  in combineTargetShuffle()
31792       unsigned Scale = VT.getScalarSizeInBits() / BCVT.getScalarSizeInBits();  in combineTargetShuffle()
31800         return DAG.getNode(X86ISD::VBROADCAST, DL, VT,  in combineTargetShuffle()
31828         MVT SVT = VT.getVectorElementType();  in combineTargetShuffle()
31833         SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl);  in combineTargetShuffle()
31834         return DAG.getNode(Opcode, DL, VT, N0, SclVec);  in combineTargetShuffle()
31841     assert(VT == MVT::v4f32 && "INSERTPS ValueType must be MVT::v4f32");  in combineTargetShuffle()
31852       return DAG.getNode(X86ISD::INSERTPS, DL, VT, DAG.getUNDEF(VT), Op1,  in combineTargetShuffle()
31857       return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),  in combineTargetShuffle()
31868         return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),  in combineTargetShuffle()
31875       return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,  in combineTargetShuffle()
31920       return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,  in combineTargetShuffle()
31940     assert(VT.getVectorElementType() == MVT::i16 && "Bad word shuffle type!");  in combineTargetShuffle()
31950       MVT DVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2);  in combineTargetShuffle()
31954       return DAG.getBitcast(VT, V);  in combineTargetShuffle()
31983           V = DAG.getBitcast(VT, D.getOperand(0));  in combineTargetShuffle()
31986                              DL, VT, V, V);  in combineTargetShuffle()
32047   EVT VT = N->getValueType(0);  in isAddSubOrSubAdd()  local
32049   if (!Subtarget.hasSSE3() || !TLI.isTypeLegal(VT) ||  in isAddSubOrSubAdd()
32050       !VT.getSimpleVT().isFloatingPoint())  in isAddSubOrSubAdd()
32112   MVT VT = N->getSimpleValueType(0);  in combineShuffleToFMAddSub()  local
32114   if (!Subtarget.hasAnyFMA() || !TLI.isTypeLegal(VT))  in combineShuffleToFMAddSub()
32140   return DAG.getNode(Opcode, DL, VT, FMAdd.getOperand(0), FMAdd.getOperand(1),  in combineShuffleToFMAddSub()
32157   MVT VT = N->getSimpleValueType(0);  in combineShuffleToAddSubOrFMAddSub()  local
32164     return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);  in combineShuffleToAddSubOrFMAddSub()
32173   if (VT.is512BitVector())  in combineShuffleToAddSubOrFMAddSub()
32176   return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);  in combineShuffleToAddSubOrFMAddSub()
32187   EVT VT = N->getValueType(0);  in combineShuffleOfConcatUndef()  local
32190   if (!VT.is128BitVector() && !VT.is256BitVector())  in combineShuffleOfConcatUndef()
32193   if (VT.getVectorElementType() != MVT::i32 &&  in combineShuffleOfConcatUndef()
32194       VT.getVectorElementType() != MVT::i64 &&  in combineShuffleOfConcatUndef()
32195       VT.getVectorElementType() != MVT::f32 &&  in combineShuffleOfConcatUndef()
32196       VT.getVectorElementType() != MVT::f64)  in combineShuffleOfConcatUndef()
32212   int NumElts = VT.getVectorNumElements();  in combineShuffleOfConcatUndef()
32219   SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, N0.getOperand(0),  in combineShuffleOfConcatUndef()
32221   return DAG.getVectorShuffle(VT, DL, Concat, DAG.getUNDEF(VT), Mask);  in combineShuffleOfConcatUndef()
32270   EVT VT = N->getValueType(0);  in combineShuffle()  local
32274   if (TLI.isTypeLegal(VT)) {  in combineShuffle()
32304     unsigned NumElts = VT.getVectorNumElements();  in combineShuffle()
32308         TLI.isOperationLegal(Opcode, VT)) {  in combineShuffle()
32316         CanFold = VT.isInteger();  in combineShuffle()
32322         CanFold = VT.isFloatingPoint();  in combineShuffle()
32334         SDValue BC00 = DAG.getBitcast(VT, BC0.getOperand(0));  in combineShuffle()
32335         SDValue BC01 = DAG.getBitcast(VT, BC0.getOperand(1));  in combineShuffle()
32336         SDValue NewBinOp = DAG.getNode(BC0.getOpcode(), dl, VT, BC00, BC01);  in combineShuffle()
32337         return DAG.getVectorShuffle(VT, dl, NewBinOp, N1, SVOp->getMask());  in combineShuffle()
32346   for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {  in combineShuffle()
32355   if (Elts.size() == VT.getVectorNumElements())  in combineShuffle()
32357             EltsFromConsecutiveLoads(VT, Elts, dl, DAG, Subtarget, true))  in combineShuffle()
32387     APInt DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements());  in combineShuffle()
32395   if (Subtarget.hasSSE41() && VT == MVT::v4i32 &&  in combineShuffle()
32446   EVT VT = Op.getValueType();  in SimplifyDemandedVectorEltsForTargetNode()  local
32490     getPackDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS);  in SimplifyDemandedVectorEltsForTargetNode()
32508       if(Src.getValueType() != VT)  in SimplifyDemandedVectorEltsForTargetNode()
32509         Src = widenSubVector(VT.getSimpleVT(), Src, false, Subtarget, TLO.DAG,  in SimplifyDemandedVectorEltsForTargetNode()
32532   if (!isTargetShuffle(Opc) || !VT.isSimple())  in SimplifyDemandedVectorEltsForTargetNode()
32539   if (!getTargetShuffleMask(Op.getNode(), VT.getSimpleVT(), true, OpInputs,  in SimplifyDemandedVectorEltsForTargetNode()
32545                    [VT](SDValue V) { return VT != V.getValueType(); }))  in SimplifyDemandedVectorEltsForTargetNode()
32564     return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));  in SimplifyDemandedVectorEltsForTargetNode()
32569         Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op)));  in SimplifyDemandedVectorEltsForTargetNode()
32608   EVT VT = Op.getValueType();  in SimplifyDemandedBitsForTargetNode()  local
32650                 NewOpc, SDLoc(Op), VT, Op0.getOperand(0),  in SimplifyDemandedBitsForTargetNode()
32733             Op, TLO.DAG.getNode(X86ISD::VSRLI, SDLoc(Op), VT, Op0, Op1));  in SimplifyDemandedBitsForTargetNode()
32749       return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));  in SimplifyDemandedBitsForTargetNode()
32890   EVT VT = BitCast.getValueType();  in combineBitcastvxi1()  local
32894   if (!VT.isScalarInteger() || !VecVT.isSimple())  in combineBitcastvxi1()
32991   return DAG.getZExtOrTrunc(V, DL, VT);  in combineBitcastvxi1()
33145   EVT VT = N->getValueType(0);  in combineBitcast()  local
33160     if ((VT == MVT::v4i1 || VT == MVT::v2i1) && SrcVT.isScalarInteger() &&  in combineBitcast()
33165       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, N0,  in combineBitcast()
33171     if ((SrcVT == MVT::v4i1 || SrcVT == MVT::v2i1) && VT.isScalarInteger() &&  in combineBitcast()
33179       return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);  in combineBitcast()
33186   if (VT == MVT::x86mmx) {  in combineBitcast()
33194         return DAG.getNode(X86ISD::MMX_MOVW2D, DL, VT,  in combineBitcast()
33199       return DAG.getBitcast(VT, DAG.getConstantFP(F64, DL, MVT::f64));  in combineBitcast()
33217         return DAG.getNode(X86ISD::MMX_MOVW2D, dl, VT, N00);  in combineBitcast()
33235         return DAG.getNode(X86ISD::MOVDQ2Q, SDLoc(N00), VT,  in combineBitcast()
33244       return DAG.getNode(X86ISD::MOVDQ2Q, DL, VT,  in combineBitcast()
33251   if (Subtarget.hasAVX512() && VT.isScalarInteger() &&  in combineBitcast()
33258       VT.isVector() && VT.getVectorElementType() == MVT::i1 &&  in combineBitcast()
33262       return DAG.getConstant(1, SDLoc(N0), VT);  in combineBitcast()
33264       return DAG.getConstant(0, SDLoc(N0), VT);  in combineBitcast()
33285   if (!((Subtarget.hasSSE1() && VT == MVT::f32) ||  in combineBitcast()
33286         (Subtarget.hasSSE2() && VT == MVT::f64)))  in combineBitcast()
33295       LogicOp0.hasOneUse() && LogicOp0.getOperand(0).getValueType() == VT &&  in combineBitcast()
33297     SDValue CastedOp1 = DAG.getBitcast(VT, LogicOp1);  in combineBitcast()
33298     return DAG.getNode(FPOpcode, DL0, VT, LogicOp0.getOperand(0), CastedOp1);  in combineBitcast()
33302       LogicOp1.hasOneUse() && LogicOp1.getOperand(0).getValueType() == VT &&  in combineBitcast()
33304     SDValue CastedOp0 = DAG.getBitcast(VT, LogicOp0);  in combineBitcast()
33305     return DAG.getNode(FPOpcode, DL0, VT, LogicOp1.getOperand(0), CastedOp0);  in combineBitcast()
33404     MVT VT = MVT::getVectorVT(MVT::i64, Ops[0].getValueSizeInBits() / 64);  in createPSADBW()  local
33405     return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops);  in createPSADBW()
33571   EVT VT = Extract->getOperand(0).getValueType();  in combineBasicSADPattern()  local
33572   if (!VT.isSimple() || !(VT.getVectorElementType().getSizeInBits() > 16))  in combineBasicSADPattern()
33584   if (RegSize / VT.getVectorNumElements() < 8)  in combineBasicSADPattern()
33620   unsigned Stages = Log2_32(VT.getVectorNumElements());  in combineBasicSADPattern()
33656   EVT VT = N->getValueType(0);  in combineExtractWithShuffle()  local
33667       Src.getOperand(0).getValueType() == VT)  in combineExtractWithShuffle()
33711     return DAG.getUNDEF(VT);  in combineExtractWithShuffle()
33714     return VT.isFloatingPoint() ? DAG.getConstantFP(0.0, dl, VT)  in combineExtractWithShuffle()
33715                                 : DAG.getConstant(0, dl, VT);  in combineExtractWithShuffle()
33727     assert(SrcSVT == VT && "Unexpected extraction type");  in combineExtractWithShuffle()
33734     assert(VT.getSizeInBits() >= SrcSVT.getSizeInBits() &&  in combineExtractWithShuffle()
33739     return DAG.getZExtOrTrunc(ExtOp, dl, VT);  in combineExtractWithShuffle()
33769   EVT VT = N->getValueType(0);  in combineExtractVectorElt()  local
33774       VT == MVT::i64 && SrcVT == MVT::v1i64 && isNullConstant(EltIdx)) {  in combineExtractVectorElt()
33779       return DAG.getBitcast(VT, InputVector);  in combineExtractVectorElt()
33784       VT == MVT::i32 && SrcVT == MVT::v2i32 && isNullConstant(EltIdx)) {  in combineExtractVectorElt()
33792   if (VT == MVT::i1 && InputVector.getOpcode() == ISD::BITCAST &&  in combineExtractVectorElt()
33829   EVT VT = LHS.getValueType();  in combineVSelectWithAllOnesOrZeros()  local
33847     return DAG.getSelect(DL, VT, CondNew, RHS, LHS);  in combineVSelectWithAllOnesOrZeros()
33855   if (CondVT.getScalarSizeInBits() != VT.getScalarSizeInBits())  in combineVSelectWithAllOnesOrZeros()
33867       TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT) ==  in combineVSelectWithAllOnesOrZeros()
33890     return DAG.getBitcast(VT, Cond);  in combineVSelectWithAllOnesOrZeros()
33899     return DAG.getBitcast(VT, Or);  in combineVSelectWithAllOnesOrZeros()
33906     return DAG.getBitcast(VT, And);  in combineVSelectWithAllOnesOrZeros()
33915     return DAG.getBitcast(VT, AndN);  in combineVSelectWithAllOnesOrZeros()
33933   EVT VT = N->getValueType(0);  in combineSelectOfTwoConstants()  local
33934   if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))  in combineSelectOfTwoConstants()
33956       ((VT == MVT::i32 || VT == MVT::i64) &&  in combineSelectOfTwoConstants()
33968     SDValue R = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);  in combineSelectOfTwoConstants()
33972       R = DAG.getNode(ISD::MUL, DL, VT, R, DAG.getConstant(AbsDiff, DL, VT));  in combineSelectOfTwoConstants()
33976       R = DAG.getNode(ISD::ADD, DL, VT, R, SDValue(FalseC, 0));  in combineSelectOfTwoConstants()
34015   EVT VT = N->getValueType(0);  in combineVSelectToBLENDV()  local
34016   if (!TLI.isOperationLegalOrCustom(ISD::VSELECT, VT))  in combineVSelectToBLENDV()
34021   if (VT.getVectorElementType() == MVT::i16)  in combineVSelectToBLENDV()
34024   if (VT.is128BitVector() && !Subtarget.hasSSE41())  in combineVSelectToBLENDV()
34027   if (VT == MVT::v32i8 && !Subtarget.hasAVX2())  in combineVSelectToBLENDV()
34030   if (VT.is512BitVector())  in combineVSelectToBLENDV()
34080   EVT VT = LHS.getValueType();  in combineSelect()  local
34089       return DAG.getVectorShuffle(VT, DL, LHS, RHS, Mask);  in combineSelect()
34097   if (Cond.getOpcode() == ISD::SETCC && VT.isFloatingPoint() &&  in combineSelect()
34098       VT != MVT::f80 && VT != MVT::f128 &&  in combineSelect()
34099       (TLI.isTypeLegal(VT) || VT == MVT::v2f32) &&  in combineSelect()
34101        (Subtarget.hasSSE1() && VT.getScalarType() == MVT::f32))) {  in combineSelect()
34252       Cond.getOpcode() == ISD::SETCC && (VT == MVT::f32 || VT == MVT::f64)) {  in combineSelect()
34261       return DAG.getNode(ISD::SELECT, DL, VT, AndNode, RHS, LHS);  in combineSelect()
34275        VT.getVectorNumElements() > 4) &&  in combineSelect()
34276       (VT.getVectorElementType() == MVT::i8 ||  in combineSelect()
34277        VT.getVectorElementType() == MVT::i16)) {  in combineSelect()
34278     Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);  in combineSelect()
34279     return DAG.getNode(N->getOpcode(), DL, VT, Cond, LHS, RHS);  in combineSelect()
34311       return DAG.getSelect(DL, VT, Cond, LHS, RHS);  in combineSelect()
34319       Subtarget.hasSSE2() && VT.getVectorNumElements() >= 2 &&  in combineSelect()
34320       isPowerOf2_32(VT.getVectorNumElements()) &&  in combineSelect()
34321       (VT.getVectorElementType() == MVT::i8 ||  in combineSelect()
34322        VT.getVectorElementType() == MVT::i16)) {  in combineSelect()
34345         return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);  in combineSelect()
34358             OpRHS = DAG.getNode(ISD::SUB, DL, VT,  in combineSelect()
34359                                 DAG.getConstant(0, DL, VT), OpRHS);  in combineSelect()
34360             return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);  in combineSelect()
34374               OpRHS = DAG.getConstant(OpRHSConst->getAPIntValue(), DL, VT);  in combineSelect()
34375               return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);  in combineSelect()
34386       Subtarget.hasSSE2() && VT.getVectorNumElements() >= 2 &&  in combineSelect()
34387       isPowerOf2_32(VT.getVectorNumElements()) &&  in combineSelect()
34388       (VT.getVectorElementType() == MVT::i8 ||  in combineSelect()
34389        VT.getVectorElementType() == MVT::i16)) {  in combineSelect()
34419         return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS);  in combineSelect()
34431           return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS);  in combineSelect()
34437   if (!TLI.isTypeLegal(VT))  in combineSelect()
34447   if (VT == MVT::x86mmx) {  in combineSelect()
34451     return DAG.getBitcast(VT, newSelect);  in combineSelect()
34969       EVT VT = N->getValueType(0);  in combineCMov()  local
34971       SDValue Diff = DAG.getNode(ISD::SUB, DL, VT, Const, Add.getOperand(1));  in combineCMov()
34972       SDValue CMov = DAG.getNode(X86ISD::CMOV, DL, VT, Diff, Add.getOperand(0),  in combineCMov()
34975       return DAG.getNode(ISD::ADD, DL, VT, CMov, Add.getOperand(1));  in combineCMov()
34986   EVT VT = N->getOperand(0).getValueType();  in canReduceVMulWidth()  local
34987   if (VT.getScalarSizeInBits() != 32)  in canReduceVMulWidth()
35069   EVT VT = N->getOperand(0).getValueType();  in reduceVMULWidth()  local
35070   unsigned NumElts = VT.getVectorNumElements();  in reduceVMULWidth()
35089                          DL, VT, MulLo);  in reduceVMULWidth()
35116     return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi);  in reduceVMULWidth()
35149     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,  in reduceVMULWidth()
35164   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,  in reduceVMULWidth()
35169                                  EVT VT, const SDLoc &DL) {  in combineMulSpecial()  argument
35172     SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),  in combineMulSpecial()
35173                                  DAG.getConstant(Mult, DL, VT));  in combineMulSpecial()
35174     Result = DAG.getNode(ISD::SHL, DL, VT, Result,  in combineMulSpecial()
35176     Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result,  in combineMulSpecial()
35182     SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),  in combineMulSpecial()
35183                                  DAG.getConstant(Mul1, DL, VT));  in combineMulSpecial()
35184     Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, Result,  in combineMulSpecial()
35185                          DAG.getConstant(Mul2, DL, VT));  in combineMulSpecial()
35186     Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result,  in combineMulSpecial()
35205     return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0),  in combineMulSpecial()
35230     return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0),  in combineMulSpecial()
35243       SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),  in combineMulSpecial()
35245       SDValue Shift2 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),  in combineMulSpecial()
35247       return DAG.getNode(ISD::ADD, DL, VT, Shift1, Shift2);  in combineMulSpecial()
35264   EVT VT = N->getValueType(0);  in combineMulToPMADDWD()  local
35267   if (!VT.isVector() || VT.getVectorElementType() != MVT::i32)  in combineMulToPMADDWD()
35272   MVT WVT = MVT::getVectorVT(MVT::i16, 2 * VT.getVectorNumElements());  in combineMulToPMADDWD()
35273   if (!((ExperimentalVectorWideningLegalization && VT == MVT::v2i32) ||  in combineMulToPMADDWD()
35297     MVT VT = MVT::getVectorVT(MVT::i32, Ops[0].getValueSizeInBits() / 32);  in combineMulToPMADDWD()  local
35298     return DAG.getNode(X86ISD::VPMADDWD, DL, VT, Ops);  in combineMulToPMADDWD()
35300   return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT,  in combineMulToPMADDWD()
35310   EVT VT = N->getValueType(0);  in combineMulToPMULDQ()  local
35313   if (!VT.isVector() || VT.getVectorElementType() != MVT::i64 ||  in combineMulToPMULDQ()
35314       VT.getVectorNumElements() < 2 ||  in combineMulToPMULDQ()
35315       !isPowerOf2_32(VT.getVectorNumElements()))  in combineMulToPMULDQ()
35329     return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, { N0, N1 },  in combineMulToPMULDQ()
35340     return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, { N0, N1 },  in combineMulToPMULDQ()
35352   EVT VT = N->getValueType(0);  in combineMul()  local
35360   if (DCI.isBeforeLegalize() && VT.isVector())  in combineMul()
35372   if (VT != MVT::i64 && VT != MVT::i32)  in combineMul()
35387     SDValue NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),  in combineMul()
35388                                  DAG.getConstant(AbsMulAmt, DL, VT));  in combineMul()
35390       NewMul = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),  in combineMul()
35425       NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),  in combineMul()
35428       NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),  in combineMul()
35429                            DAG.getConstant(MulAmt1, DL, VT));  in combineMul()
35432       NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul,  in combineMul()
35435       NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul,  in combineMul()
35436                            DAG.getConstant(MulAmt2, DL, VT));  in combineMul()
35440       NewMul = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),  in combineMul()
35443     NewMul = combineMulSpecial(C->getZExtValue(), N, DAG, VT, DL);  in combineMul()
35447            C->getZExtValue() != (VT == MVT::i64 ? UINT64_MAX : UINT32_MAX) &&  in combineMul()
35453           ISD::ADD, DL, VT, N->getOperand(0),  in combineMul()
35454           DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),  in combineMul()
35459         NewMul = DAG.getNode(ISD::SUB, DL, VT,  in combineMul()
35460                              DAG.getConstant(0, DL, VT), NewMul);  in combineMul()
35463       NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),  in combineMul()
35468         NewMul = DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), NewMul);  in combineMul()
35470         NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0));  in combineMul()
35473       NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),  in combineMul()
35476       NewMul = DAG.getNode(ISD::ADD, DL, VT, NewMul, N->getOperand(0));  in combineMul()
35477       NewMul = DAG.getNode(ISD::ADD, DL, VT, NewMul, N->getOperand(0));  in combineMul()
35480       NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),  in combineMul()
35483       NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0));  in combineMul()
35484       NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0));  in combineMul()
35495   EVT VT = N0.getValueType();  in combineShiftLeft()  local
35499   if (VT.isInteger() && !VT.isVector() &&  in combineShiftLeft()
35529       return DAG.getNode(ISD::AND, DL, VT, N00, DAG.getConstant(Mask, DL, VT));  in combineShiftLeft()
35544         return DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, N0);  in combineShiftLeft()
35553   EVT VT = N0.getValueType();  in combineShiftRightArithmetic()  local
35554   unsigned Size = VT.getSizeInBits();  in combineShiftRightArithmetic()
35567   if (VT.isVector() || N1.getOpcode() != ISD::Constant ||  in combineShiftRightArithmetic()
35589         DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, N00, DAG.getValueType(SVT));  in combineShiftRightArithmetic()
35594       return DAG.getNode(ISD::SHL, DL, VT, NN,  in combineShiftRightArithmetic()
35597       return DAG.getNode(ISD::SRA, DL, VT, NN,  in combineShiftRightArithmetic()
35607   EVT VT = N0.getValueType();  in combineShiftRightLogical()  local
35645     SDValue NewMask = DAG.getConstant(NewMaskVal, DL, VT);  in combineShiftRightLogical()
35646     SDValue NewShift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1);  in combineShiftRightLogical()
35647     return DAG.getNode(ISD::AND, DL, VT, NewShift, NewMask);  in combineShiftRightLogical()
35677   EVT VT = N->getValueType(0);  in combineVectorPack()  local
35680   unsigned DstBitsPerElt = VT.getScalarSizeInBits();  in combineVectorPack()
35695     unsigned NumLanes = VT.getSizeInBits() / 128;  in combineVectorPack()
35696     unsigned NumDstElts = VT.getVectorNumElements();  in combineVectorPack()
35740     return getConstVector(Bits, Undefs, VT.getSimpleVT(), DAG, SDLoc(N));  in combineVectorPack()
35746       N0.getOpcode() == ISD::TRUNCATE && N1.isUndef() && VT == MVT::v16i8 &&  in combineVectorPack()
35752         return DAG.getNode(X86ISD::VTRUNC, SDLoc(N), VT, N0.getOperand(0));  in combineVectorPack()
35758       return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Concat);  in combineVectorPack()
35779   EVT VT = N->getValueType(0);  in combineVectorShiftVar()  local
35783     return DAG.getConstant(0, SDLoc(N), VT);  in combineVectorShiftVar()
35787   APInt DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements());  in combineVectorShiftVar()
35803   EVT VT = N->getValueType(0);  in combineVectorShiftImm()  local
35806   unsigned NumBitsPerElt = VT.getScalarSizeInBits();  in combineVectorShiftImm()
35807   assert(VT == N0.getValueType() && (NumBitsPerElt % 8) == 0 &&  in combineVectorShiftImm()
35816       return DAG.getConstant(0, SDLoc(N), VT);  in combineVectorShiftImm()
35827     return DAG.getConstant(0, SDLoc(N), VT);  in combineVectorShiftImm()
35836     return DAG.getNode(X86ISD::VSRAI, SDLoc(N), VT, N0.getOperand(0),  in combineVectorShiftImm()
35854     assert(EltBits.size() == VT.getVectorNumElements() &&  in combineVectorShiftImm()
35864     return getConstVector(EltBits, UndefElts, VT.getSimpleVT(), DAG, SDLoc(N));  in combineVectorShiftImm()
35918     EVT     VT    = CMP00.getValueType();  in combineCompareEqual()  local
35920     if (VT == MVT::f32 || VT == MVT::f64) {  in combineCompareEqual()
36005   MVT VT = N->getSimpleValueType(0);  in combineANDXORWithAllOnesIntoANDNP()  local
36006   if (!VT.is128BitVector() && !VT.is256BitVector() && !VT.is512BitVector())  in combineANDXORWithAllOnesIntoANDNP()
36023   X = DAG.getBitcast(VT, X);  in combineANDXORWithAllOnesIntoANDNP()
36024   Y = DAG.getBitcast(VT, Y);  in combineANDXORWithAllOnesIntoANDNP()
36025   return DAG.getNode(X86ISD::ANDNP, SDLoc(N), VT, X, Y);  in combineANDXORWithAllOnesIntoANDNP()
36036   EVT VT = N->getValueType(0);  in PromoteMaskArithmetic()  local
36037   assert(VT.isVector() && "Expected vector type");  in PromoteMaskArithmetic()
36060   if (N0->getOperand(0).getValueType() != VT)  in PromoteMaskArithmetic()
36065                   N1.getOperand(0).getValueType() == VT;  in PromoteMaskArithmetic()
36072   if (!TLI.isOperationLegalOrPromote(Narrow->getOpcode(), VT))  in PromoteMaskArithmetic()
36080     N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N1);  in PromoteMaskArithmetic()
36083   SDValue Op = DAG.getNode(Narrow->getOpcode(), DL, VT, N0, N1);  in PromoteMaskArithmetic()
36092     return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT,  in PromoteMaskArithmetic()
36113   EVT VT = N->getValueType(0);  in convertIntLogicToFPLogic()  local
36118       ((Subtarget.hasSSE1() && VT == MVT::i32) ||  in convertIntLogicToFPLogic()
36119        (Subtarget.hasSSE2() && VT == MVT::i64))) {  in convertIntLogicToFPLogic()
36126       return DAG.getBitcast(VT, FPLogic);  in convertIntLogicToFPLogic()
36188 static bool hasBZHI(const X86Subtarget &Subtarget, MVT VT) {  in hasBZHI()  argument
36189   if (Subtarget.hasBMI2() && VT.isScalarInteger()) {  in hasBZHI()
36190     switch (VT.getSizeInBits()) {  in hasBZHI()
36211   MVT VT = Node->getSimpleValueType(0);  in combineAndLoadToBZHI()  local
36215   if (!hasBZHI(Subtarget, VT))  in combineAndLoadToBZHI()
36241                   VT.getSizeInBits() ||  in combineAndLoadToBZHI()
36265           SDValue SizeC = DAG.getConstant(VT.getSizeInBits(), dl, MVT::i32);  in combineAndLoadToBZHI()
36276           SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);  in combineAndLoadToBZHI()
36277           SDValue LShr = DAG.getNode(ISD::SRL, dl, VT, AllOnes, Sub);  in combineAndLoadToBZHI()
36279           return DAG.getNode(ISD::AND, dl, VT, Inp, LShr);  in combineAndLoadToBZHI()
36291   EVT VT = N->getValueType(0);  in combineParity()  local
36295   if (VT != MVT::i32 && VT != MVT::i64)  in combineParity()
36299   if (TLI.isTypeLegal(VT) && TLI.isOperationLegal(ISD::CTPOP, VT))  in combineParity()
36318   if (VT == MVT::i64) {  in combineParity()
36320                              DAG.getNode(ISD::SRL, DL, VT, X,  in combineParity()
36329     return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Parity);  in combineParity()
36331   assert(VT == MVT::i32 && "Unexpected VT!");  in combineParity()
36334   SDValue Hi16 = DAG.getNode(ISD::SRL, DL, VT, X,  in combineParity()
36336   X = DAG.getNode(ISD::XOR, DL, VT, X, Hi16);  in combineParity()
36342                            DAG.getNode(ISD::SRL, DL, VT, X,  in combineParity()
36357   EVT VT = N->getValueType(0);  in combineAnd()  local
36360   if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) {  in combineAnd()
36368   if (VT == MVT::i64 && Subtarget.is64Bit() &&  in combineAnd()
36404   if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {  in combineAnd()
36413   if ((VT.getScalarSizeInBits() % 8) == 0 &&  in combineAnd()
36423     if (VT == SrcVecVT.getScalarType() &&  in combineAnd()
36446         return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), VT, Shuffle,  in combineAnd()
36498   EVT VT = N->getValueType(0);  in combineLogicBlendIntoPBLENDV()  local
36499   if (!((VT.is128BitVector() && Subtarget.hasSSE2()) ||  in combineLogicBlendIntoPBLENDV()
36500         (VT.is256BitVector() && Subtarget.hasInt256())))  in combineLogicBlendIntoPBLENDV()
36567       return DAG.getBitcast(VT, Res);  in combineLogicBlendIntoPBLENDV()
36575   MVT BlendVT = VT.is256BitVector() ? MVT::v32i8 : MVT::v16i8;  in combineLogicBlendIntoPBLENDV()
36581   return DAG.getBitcast(VT, Mask);  in combineLogicBlendIntoPBLENDV()
36593   EVT VT = Cmp.getOperand(0).getValueType();  in lowerX86CmpEqZeroToCtlzSrl()  local
36594   unsigned Log2b = Log2_32(VT.getSizeInBits());  in lowerX86CmpEqZeroToCtlzSrl()
36596   SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Cmp->getOperand(0));  in lowerX86CmpEqZeroToCtlzSrl()
36662   EVT VT = OR->getValueType(0);  in combineOrCmpEqZeroToCtlzSrl()  local
36663   SDValue NewLHS = lowerX86CmpEqZeroToCtlzSrl(LHS, VT, DAG);  in combineOrCmpEqZeroToCtlzSrl()
36665   if (NewLHS && (NewRHS = lowerX86CmpEqZeroToCtlzSrl(RHS, VT, DAG)))  in combineOrCmpEqZeroToCtlzSrl()
36666     Ret = DAG.getNode(ISD::OR, SDLoc(OR), VT, NewLHS, NewRHS);  in combineOrCmpEqZeroToCtlzSrl()
36679     EVT VT = OR->getValueType(0);  in combineOrCmpEqZeroToCtlzSrl()  local
36680     SDValue NewRHS = lowerX86CmpEqZeroToCtlzSrl(RHS, VT, DAG);  in combineOrCmpEqZeroToCtlzSrl()
36683     Ret = DAG.getNode(ISD::OR, SDLoc(OR), VT, Ret, NewRHS);  in combineOrCmpEqZeroToCtlzSrl()
36697   EVT VT = N->getValueType(0);  in combineOr()  local
36700   if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) {  in combineOr()
36720   if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {  in combineOr()
36728   if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)  in combineOr()
36733   unsigned Bits = VT.getScalarSizeInBits();  in combineOr()
36805         return DAG.getNode(Opc, DL, VT, Op0, Op1,  in combineOr()
36811       return DAG.getNode(Opc, DL, VT,  in combineOr()
36827           return DAG.getNode(Opc, DL, VT, Op0, Op1.getOperand(0),  in combineOr()
36833           return DAG.getNode(Opc, DL, VT, Op0, Op1.getOperand(0),  in combineOr()
36904   EVT VT = N->getValueType(0);  in foldVectorXorShiftIntoCmp()  local
36905   if (!VT.isSimple())  in foldVectorXorShiftIntoCmp()
36908   switch (VT.getSimpleVT().SimpleTy) {  in foldVectorXorShiftIntoCmp()
36940   return DAG.getNode(X86ISD::PCMPGT, SDLoc(N), VT, Shift.getOperand(0), Ones);  in foldVectorXorShiftIntoCmp()
36979 static SDValue detectUSatPattern(SDValue In, EVT VT, SelectionDAG &DAG,  in detectUSatPattern()  argument
36984   assert(InVT.getScalarSizeInBits() > VT.getScalarSizeInBits() &&  in detectUSatPattern()
36999     if (C2.isMask(VT.getScalarSizeInBits()))  in detectUSatPattern()
37004       if (C1.isNonNegative() && C2.isMask(VT.getScalarSizeInBits()))  in detectUSatPattern()
37009       if (C1.isNonNegative() && C2.isMask(VT.getScalarSizeInBits()) &&  in detectUSatPattern()
37026 static SDValue detectSSatPattern(SDValue In, EVT VT, bool MatchPackUS = false) {  in detectSSatPattern()  argument
37027   unsigned NumDstBits = VT.getScalarSizeInBits();  in detectSSatPattern()
37064 static SDValue detectAVX512SSatPattern(SDValue In, EVT VT,  in detectAVX512SSatPattern()  argument
37069   if (!isSATValidOnAVX512Subtarget(In.getValueType(), VT, Subtarget))  in detectAVX512SSatPattern()
37071   return detectSSatPattern(In, VT);  in detectAVX512SSatPattern()
37079 static SDValue detectAVX512USatPattern(SDValue In, EVT VT, SelectionDAG &DAG,  in detectAVX512USatPattern()  argument
37085   if (!isSATValidOnAVX512Subtarget(In.getValueType(), VT, Subtarget))  in detectAVX512USatPattern()
37087   return detectUSatPattern(In, VT, DAG, DL);  in detectAVX512USatPattern()
37090 static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL,  in combineTruncateWithSat()  argument
37093   EVT SVT = VT.getScalarType();  in combineTruncateWithSat()
37097   if (TLI.isTypeLegal(InVT) && TLI.isTypeLegal(VT) &&  in combineTruncateWithSat()
37098       isSATValidOnAVX512Subtarget(InVT, VT, Subtarget)) {  in combineTruncateWithSat()
37099     if (auto SSatVal = detectSSatPattern(In, VT))  in combineTruncateWithSat()
37100       return DAG.getNode(X86ISD::VTRUNCS, DL, VT, SSatVal);  in combineTruncateWithSat()
37101     if (auto USatVal = detectUSatPattern(In, VT, DAG, DL))  in combineTruncateWithSat()
37102       return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, USatVal);  in combineTruncateWithSat()
37104   if (VT.isVector() && isPowerOf2_32(VT.getVectorNumElements()) &&  in combineTruncateWithSat()
37108     if (auto USatVal = detectSSatPattern(In, VT, true)) {  in combineTruncateWithSat()
37112                                      VT.getVectorNumElements());  in combineTruncateWithSat()
37116           return truncateVectorWithPACK(X86ISD::PACKUS, VT, Mid, DL, DAG,  in combineTruncateWithSat()
37119         return truncateVectorWithPACK(X86ISD::PACKUS, VT, USatVal, DL, DAG,  in combineTruncateWithSat()
37122     if (auto SSatVal = detectSSatPattern(In, VT))  in combineTruncateWithSat()
37123       return truncateVectorWithPACK(X86ISD::PACKSS, VT, SSatVal, DL, DAG,  in combineTruncateWithSat()
37132 static SDValue detectAVGPattern(SDValue In, EVT VT, SelectionDAG &DAG,  in detectAVGPattern()  argument
37135   if (!VT.isVector())  in detectAVGPattern()
37138   unsigned NumElems = VT.getVectorNumElements();  in detectAVGPattern()
37140   EVT ScalarVT = VT.getVectorElementType();  in detectAVGPattern()
37206       Operands[0].getOperand(0).getValueType() == VT) {  in detectAVGPattern()
37211     Operands[1] = DAG.getNode(ISD::TRUNCATE, DL, VT, Operands[1]);  in detectAVGPattern()
37212     return SplitOpsAndApply(DAG, Subtarget, DL, VT,  in detectAVGPattern()
37234           Operands[j].getOperand(0).getValueType() != VT)  in detectAVGPattern()
37238     return SplitOpsAndApply(DAG, Subtarget, DL, VT,  in detectAVGPattern()
37373   EVT VT = ML->getValueType(0);  in reduceMaskedLoadToScalarLoad()  local
37374   EVT EltVT = VT.getVectorElementType();  in reduceMaskedLoadToScalarLoad()
37380   SDValue Insert = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT,  in reduceMaskedLoadToScalarLoad()
37392   EVT VT = ML->getValueType(0);  in combineMaskedLoadConstantMask()  local
37397   unsigned NumElts = VT.getVectorNumElements();  in combineMaskedLoadConstantMask()
37402     SDValue VecLd = DAG.getLoad(VT, DL, ML->getChain(), ML->getBasePtr(),  in combineMaskedLoadConstantMask()
37404     SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), VecLd,  in combineMaskedLoadConstantMask()
37420   SDValue NewML = DAG.getMaskedLoad(VT, DL, ML->getChain(), ML->getBasePtr(),  in combineMaskedLoadConstantMask()
37421                                     ML->getMask(), DAG.getUNDEF(VT),  in combineMaskedLoadConstantMask()
37424   SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), NewML,  in combineMaskedLoadConstantMask()
37452   EVT VT = Mld->getValueType(0);  in combineMaskedLoad()  local
37453   unsigned NumElems = VT.getVectorNumElements();  in combineMaskedLoad()
37457   assert(LdVT != VT && "Cannot extend to the same type");  in combineMaskedLoad()
37458   unsigned ToSz = VT.getScalarSizeInBits();  in combineMaskedLoad()
37465   assert(SizeRatio * NumElems * FromSz == VT.getSizeInBits());  in combineMaskedLoad()
37470   assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());  in combineMaskedLoad()
37489   if (Mask.getValueType() == VT) {  in combineMaskedLoad()
37503     unsigned MaskNumElts = VT.getVectorNumElements();  in combineMaskedLoad()
37518   SDValue NewVec = getExtendInVec(/*Signed*/true, dl, VT, WideLd, DAG);  in combineMaskedLoad()
37539   EVT VT = MS->getValue().getValueType();  in reduceMaskedStoreToScalarStore()  local
37540   EVT EltVT = VT.getVectorElementType();  in reduceMaskedStoreToScalarStore()
37556   EVT VT = Mst->getValue().getValueType();  in combineMaskedStore()  local
37566       APInt DemandedMask(APInt::getSignMask(VT.getScalarSizeInBits()));  in combineMaskedStore()
37579   unsigned NumElems = VT.getVectorNumElements();  in combineMaskedStore()
37583   assert(StVT != VT && "Cannot truncate to the same type");  in combineMaskedStore()
37584   unsigned FromSz = VT.getScalarSizeInBits();  in combineMaskedStore()
37593   if (TLI.isTruncStoreLegal(VT, StVT))  in combineMaskedStore()
37605   assert(SizeRatio * NumElems * ToSz == VT.getSizeInBits());  in combineMaskedStore()
37611   assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());  in combineMaskedStore()
37628   if (Mask.getValueType() == VT) {  in combineMaskedStore()
37641     unsigned MaskNumElts = VT.getVectorNumElements();  in combineMaskedStore()
37660   EVT VT = St->getValue().getValueType();  in combineStore()  local
37667   if (!Subtarget.hasAVX512() && VT == StVT && VT.isVector() &&  in combineStore()
37668       VT.getVectorElementType() == MVT::i1) {  in combineStore()
37670     EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), VT.getVectorNumElements());  in combineStore()
37680   if (VT == MVT::v1i1 && VT == StVT && Subtarget.hasAVX512() &&  in combineStore()
37689   if ((VT == MVT::v2i1 || VT == MVT::v4i1) && VT == StVT &&  in combineStore()
37691     unsigned NumConcats = 8 / VT.getVectorNumElements();  in combineStore()
37692     SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(VT));  in combineStore()
37701   if ((VT == MVT::v8i1 || VT == MVT::v16i1 || VT == MVT::v32i1 ||  in combineStore()
37702        VT == MVT::v64i1) && VT == StVT && TLI.isTypeLegal(VT) &&  in combineStore()
37705     if (VT == MVT::v64i1 && !Subtarget.is64Bit()) {  in combineStore()
37740   if (VT.is256BitVector() && StVT == VT &&  in combineStore()
37741       TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,  in combineStore()
37744     unsigned NumElems = VT.getVectorNumElements();  in combineStore()
37767   if (St->isTruncatingStore() && VT.isVector()) {  in combineStore()
37790     unsigned NumElems = VT.getVectorNumElements();  in combineStore()
37791     assert(StVT != VT && "Cannot truncate to the same type");  in combineStore()
37792     unsigned FromSz = VT.getScalarSizeInBits();  in combineStore()
37799     if (TLI.isTruncStoreLegalOrCustom(VT, StVT))  in combineStore()
37810     assert(SizeRatio * NumElems * ToSz == VT.getSizeInBits());  in combineStore()
37816     assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());  in combineStore()
37847             StoreType, VT.getSizeInBits()/StoreType.getSizeInBits());  in combineStore()
37848     assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits());  in combineStore()
37874   if (VT.getSizeInBits() != 64)  in combineStore()
37881   if ((VT.isVector() ||  in combineStore()
37882        (VT == MVT::i64 && F64IsLegal && !Subtarget.is64Bit())) &&  in combineStore()
37895     if (!VT.isVector() && !Ld->hasNUsesOfValue(1, 0))  in combineStore()
37905                   (!VT.isFloatingPoint() || !F64IsLegal)) ? MVT::i64 : MVT::f64;  in combineStore()
37949   if (VT == MVT::i64 && F64IsLegal && !Subtarget.is64Bit() &&  in combineStore()
37998   MVT VT = LHS.getSimpleValueType();  in isHorizontalBinOp()  local
37999   assert((VT.is128BitVector() || VT.is256BitVector()) &&  in isHorizontalBinOp()
38007   unsigned NumElts = VT.getVectorNumElements();  in isHorizontalBinOp()
38056   unsigned Num128BitChunks = VT.getSizeInBits() / 128;  in isHorizontalBinOp()
38092   EVT VT = N->getValueType(0);  in combineFaddFsub()  local
38100   if (((Subtarget.hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||  in combineFaddFsub()
38101        (Subtarget.hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) &&  in combineFaddFsub()
38104     return DAG.getNode(HorizOpcode, SDLoc(N), VT, LHS, RHS);  in combineFaddFsub()
38122   EVT VT = N->getValueType(0);  in combineTruncatedArithmetic()  local
38125   auto IsFreeTruncation = [VT](SDValue Op) {  in combineTruncatedArithmetic()
38126     unsigned TruncSizeInBits = VT.getScalarSizeInBits();  in combineTruncatedArithmetic()
38145     SDValue Trunc0 = DAG.getNode(ISD::TRUNCATE, DL, VT, N0);  in combineTruncatedArithmetic()
38146     SDValue Trunc1 = DAG.getNode(ISD::TRUNCATE, DL, VT, N1);  in combineTruncatedArithmetic()
38147     return DAG.getNode(Opcode, DL, VT, Trunc0, Trunc1);  in combineTruncatedArithmetic()
38156   if (!VT.isVector())  in combineTruncatedArithmetic()
38168     if (TLI.isOperationLegalOrPromote(Opcode, VT) &&  in combineTruncatedArithmetic()
38177     if (SrcVT.getScalarType() == MVT::i64 && TLI.isOperationLegal(Opcode, VT) &&  in combineTruncatedArithmetic()
38184     if (TLI.isOperationLegal(Opcode, VT) &&  in combineTruncatedArithmetic()
38194     if (TLI.isOperationLegal(Opcode, VT) &&  in combineTruncatedArithmetic()
38316   MVT VT = N->getValueType(0).getSimpleVT();  in combineVectorSignBitsTruncation()  local
38317   MVT SVT = VT.getScalarType();  in combineVectorSignBitsTruncation()
38323   if (!VT.is128BitVector() && !VT.is256BitVector())  in combineVectorSignBitsTruncation()
38338     return truncateVectorWithPACK(X86ISD::PACKUS, VT, In, DL, DAG, Subtarget);  in combineVectorSignBitsTruncation()
38344     return truncateVectorWithPACK(X86ISD::PACKSS, VT, In, DL, DAG, Subtarget);  in combineVectorSignBitsTruncation()
38356 static SDValue combinePMULH(SDValue Src, EVT VT, const SDLoc &DL,  in combinePMULH()  argument
38368   if (!VT.isVector() || VT.getVectorElementType() != MVT::i16 ||  in combinePMULH()
38370        VT.getVectorNumElements() < 8))  in combinePMULH()
38397   if (LHS.getValueType() != VT || RHS.getValueType() != VT)  in combinePMULH()
38401   return DAG.getNode(Opc, DL, VT, LHS, RHS);  in combinePMULH()
38412 static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG,  in detectPMADDUBSW()  argument
38415   if (!VT.isVector() || !Subtarget.hasSSSE3())  in detectPMADDUBSW()
38418   unsigned NumElems = VT.getVectorNumElements();  in detectPMADDUBSW()
38419   EVT ScalarVT = VT.getVectorElementType();  in detectPMADDUBSW()
38423   SDValue SSatVal = detectSSatPattern(In, VT);  in detectPMADDUBSW()
38539   return SplitOpsAndApply(DAG, Subtarget, DL, VT, { ZExtIn, SExtIn },  in detectPMADDUBSW()
38545   EVT VT = N->getValueType(0);  in combineTruncate()  local
38554   if (SDValue Avg = detectAVGPattern(Src, VT, DAG, Subtarget, DL))  in combineTruncate()
38558   if (SDValue PMAdd = detectPMADDUBSW(Src, VT, DAG, Subtarget, DL))  in combineTruncate()
38562   if (SDValue Val = combineTruncateWithSat(Src, VT, DL, DAG, Subtarget))  in combineTruncate()
38566   if (SDValue V = combinePMULH(Src, VT, DL, DAG, Subtarget))  in combineTruncate()
38571   if (Src.getOpcode() == ISD::BITCAST && VT == MVT::i32) {  in combineTruncate()
38598   auto VT = Op->getValueType(0);  in isFNEG()  local
38605       return DAG.getVectorShuffle(VT, SDLoc(SVOp), NegOp0, DAG.getUNDEF(VT),  in isFNEG()
38618       return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(Op), VT, InsVector,  in isFNEG()
38664   EVT VT = Arg.getValueType();  in combineFneg()  local
38665   EVT SVT = VT.getScalarType();  in combineFneg()
38669   if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))  in combineFneg()
38677     SDValue Zero = DAG.getConstantFP(0.0, DL, VT);  in combineFneg()
38678     SDValue NewNode = DAG.getNode(X86ISD::FNMSUB, DL, VT, Arg.getOperand(0),  in combineFneg()
38702     return DAG.getBitcast(OrigVT, DAG.getNode(NewOpcode, DL, VT,  in combineFneg()
38710   MVT VT = N->getSimpleValueType(0);  in lowerX86FPLogicOp()  local
38712   if (!VT.isVector() || !Subtarget.hasSSE2())  in lowerX86FPLogicOp()
38717   unsigned IntBits = VT.getScalarSizeInBits();  in lowerX86FPLogicOp()
38719   MVT IntVT = MVT::getVectorVT(IntSVT, VT.getSizeInBits() / IntBits);  in lowerX86FPLogicOp()
38732   return DAG.getBitcast(VT, IntOp);  in lowerX86FPLogicOp()
38787   EVT VT = N->getValueType(0);  in combineBEXTR()  local
38788   unsigned NumBits = VT.getSizeInBits();  in combineBEXTR()
38799       return DAG.getNode(X86ISD::BEXTR, SDLoc(N), VT, Op0,  in combineBEXTR()
38800                          DAG.getConstant(MaskedVal1, SDLoc(N), VT));  in combineBEXTR()
38836   EVT VT = N->getValueType(0);  in combineFAndFNotToFAndn()  local
38840   if (!((VT == MVT::f32 && Subtarget.hasSSE1()) ||  in combineFAndFNotToFAndn()
38841         (VT == MVT::f64 && Subtarget.hasSSE2()) ||  in combineFAndFNotToFAndn()
38842         (VT == MVT::v4f32 && Subtarget.hasSSE1() && !Subtarget.hasSSE2())))  in combineFAndFNotToFAndn()
38854     return DAG.getNode(X86ISD::FANDN, DL, VT, N0.getOperand(0), N1);  in combineFAndFNotToFAndn()
38858     return DAG.getNode(X86ISD::FANDN, DL, VT, N1.getOperand(0), N0);  in combineFAndFNotToFAndn()
38942   EVT VT = N->getValueType(0);  in combineFMinNumFMaxNum()  local
38943   if (!((Subtarget.hasSSE1() && (VT == MVT::f32 || VT == MVT::v4f32)) ||  in combineFMinNumFMaxNum()
38944         (Subtarget.hasSSE2() && (VT == MVT::f64 || VT == MVT::v2f64)) ||  in combineFMinNumFMaxNum()
38945         (Subtarget.hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))))  in combineFMinNumFMaxNum()
38956     return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags());  in combineFMinNumFMaxNum()
38960   if (!VT.isVector() && DAG.getMachineFunction().getFunction().optForMinSize())  in combineFMinNumFMaxNum()
38964       DAG.getDataLayout(), *DAG.getContext(), VT);  in combineFMinNumFMaxNum()
38985   SDValue MinOrMax = DAG.getNode(MinMaxOp, DL, VT, Op1, Op0);  in combineFMinNumFMaxNum()
38990   return DAG.getSelect(DL, VT, IsOp0Nan, Op1, MinOrMax);  in combineFMinNumFMaxNum()
38995   EVT VT = N->getValueType(0);  in combineX86INT_TO_FP()  local
38999   APInt DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements());  in combineX86INT_TO_FP()
39011   MVT VT = N->getSimpleValueType(0);  in combineAndnp()  local
39019     return DAG.getConstant(0, SDLoc(N), VT);  in combineAndnp()
39022   if (VT.isVector() && N->getOperand(0).getOpcode() == ISD::XOR &&  in combineAndnp()
39024     return DAG.getNode(ISD::AND, SDLoc(N), VT,  in combineAndnp()
39029   if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {  in combineAndnp()
39056   EVT VT = N->getValueType(0);  in combineSextInRegCmov()  local
39084   if (N0.getValueType() != VT) {  in combineSextInRegCmov()
39085     CMovOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, VT, CMovOp0);  in combineSextInRegCmov()
39086     CMovOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, VT, CMovOp1);  in combineSextInRegCmov()
39089   CMovOp0 = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, CMovOp0, N1);  in combineSextInRegCmov()
39090   CMovOp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, CMovOp1, N1);  in combineSextInRegCmov()
39092   return DAG.getNode(X86ISD::CMOV, DL, VT, CMovOp0, CMovOp1,  in combineSextInRegCmov()
39101   EVT VT = N->getValueType(0);  in combineSignExtendInReg()  local
39112   if (VT == MVT::v4i64 && (N0.getOpcode() == ISD::ANY_EXTEND ||  in combineSignExtendInReg()
39143   EVT VT = Ext->getValueType(0);  in promoteExtBeforeAdd()  local
39144   if (VT != MVT::i64)  in promoteExtBeforeAdd()
39185   SDValue NewExt = DAG.getNode(Ext->getOpcode(), SDLoc(Ext), VT, AddOp0);  in promoteExtBeforeAdd()
39186   SDValue NewConstant = DAG.getConstant(AddConstant, SDLoc(Add), VT);  in promoteExtBeforeAdd()
39193   return DAG.getNode(ISD::ADD, SDLoc(Add), VT, NewExt, NewConstant, Flags);  in promoteExtBeforeAdd()
39217   EVT VT = CMovN.getValueType();  in combineToExtendCMOV()  local
39231   if (VT != MVT::i16 && !(ExtendOpcode == ISD::SIGN_EXTEND && VT == MVT::i32))  in combineToExtendCMOV()
39269   EVT VT = N->getValueType(0);  in combineToExtendBoolVectorInReg()  local
39270   EVT SVT = VT.getScalarType();  in combineToExtendBoolVectorInReg()
39276   if (!VT.isVector())  in combineToExtendBoolVectorInReg()
39291   unsigned NumElts = VT.getVectorNumElements();  in combineToExtendBoolVectorInReg()
39305     Vec = DAG.getBitcast(VT, Vec);  in combineToExtendBoolVectorInReg()
39314     Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl);  in combineToExtendBoolVectorInReg()
39317   Vec = DAG.getVectorShuffle(VT, DL, Vec, Vec, ShuffleMask);  in combineToExtendBoolVectorInReg()
39326   SDValue BitMask = DAG.getBuildVector(VT, DL, Bits);  in combineToExtendBoolVectorInReg()
39327   Vec = DAG.getNode(ISD::AND, DL, VT, Vec, BitMask);  in combineToExtendBoolVectorInReg()
39332   Vec = DAG.getSExtOrTrunc(Vec, DL, VT);  in combineToExtendBoolVectorInReg()
39338   return DAG.getNode(ISD::SRL, DL, VT, Vec,  in combineToExtendBoolVectorInReg()
39339                      DAG.getConstant(EltSizeInBits - 1, DL, VT));  in combineToExtendBoolVectorInReg()
39361   EVT VT = N->getValueType(0);  in combineToExtendVectorInReg()  local
39362   EVT SVT = VT.getScalarType();  in combineToExtendVectorInReg()
39383   if (!VT.isVector() || VT.getVectorNumElements() < 2)  in combineToExtendVectorInReg()
39392   if (DAG.getTargetLoweringInfo().isTypeLegal(VT) &&  in combineToExtendVectorInReg()
39410   if (VT.getSizeInBits() < 128 && !(128 % VT.getSizeInBits())) {  in combineToExtendVectorInReg()
39411     unsigned Scale = 128 / VT.getSizeInBits();  in combineToExtendVectorInReg()
39416     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SExt,  in combineToExtendVectorInReg()
39423   if (!Subtarget.hasSSE41() || VT.is128BitVector() ||  in combineToExtendVectorInReg()
39424       (VT.is256BitVector() && Subtarget.hasAVX()) ||  in combineToExtendVectorInReg()
39425       (VT.is512BitVector() && Subtarget.useAVX512Regs())) {  in combineToExtendVectorInReg()
39426     SDValue ExOp = ExtendVecSize(DL, N0, VT.getSizeInBits());  in combineToExtendVectorInReg()
39429     return DAG.getNode(Opcode, DL, VT, ExOp);  in combineToExtendVectorInReg()
39433     unsigned NumVecs = VT.getSizeInBits() / SplitSize;  in combineToExtendVectorInReg()
39449     return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Opnds);  in combineToExtendVectorInReg()
39454   if (!Subtarget.hasAVX() && !(VT.getSizeInBits() % 128))  in combineToExtendVectorInReg()
39459   if (!Subtarget.useAVX512Regs() && !(VT.getSizeInBits() % 256))  in combineToExtendVectorInReg()
39470   EVT VT = N->getValueType(0);  in combineExtSetcc()  local
39474   if (!Subtarget.hasAVX512() || !VT.isVector() || N0->getOpcode() != ISD::SETCC)  in combineExtSetcc()
39478   EVT SVT = VT.getVectorElementType();  in combineExtSetcc()
39484   unsigned Size = VT.getSizeInBits();  in combineExtSetcc()
39500   SDValue Res = DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);  in combineExtSetcc()
39512   EVT VT = N->getValueType(0);  in combineSext()  local
39531     SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));  in combineSext()
39532     return DAG.getNode(ISD::SUB, DL, VT, Zext, DAG.getConstant(1, DL, VT));  in combineSext()
39541   if (VT.isVector())  in combineSext()
39586   EVT VT = N->getValueType(0);  in combineFMA()  local
39589   if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))  in combineFMA()
39592   EVT ScalarVT = VT.getScalarType();  in combineFMA()
39632     return DAG.getNode(NewOpcode, dl, VT, A, B, C, N->getOperand(3));  in combineFMA()
39633   return DAG.getNode(NewOpcode, dl, VT, A, B, C);  in combineFMA()
39640   EVT VT = N->getValueType(0);  in combineFMADDSUB()  local
39656     return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1),  in combineFMADDSUB()
39658   return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1),  in combineFMADDSUB()
39671   EVT VT = N->getValueType(0);  in combineZext()  local
39680       return DAG.getNode(ISD::AND, dl, VT,  in combineZext()
39681                          DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,  in combineZext()
39683                          DAG.getConstant(1, dl, VT));  in combineZext()
39692       return DAG.getNode(ISD::AND, dl, VT,  in combineZext()
39693                          DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,  in combineZext()
39695                          DAG.getConstant(1, dl, VT));  in combineZext()
39712   if (VT.isVector())  in combineZext()
39757   EVT VT = SetCC->getValueType(0);  in combineVectorSizedSetCCEquality()  local
39786       return DAG.getSetCC(DL, VT, DAG.getBitcast(MVT::i16, Cmp),  in combineVectorSizedSetCCEquality()
39796     return DAG.getSetCC(DL, VT, MovMsk, FFFFs, CC);  in combineVectorSizedSetCCEquality()
39807   EVT VT = N->getValueType(0);  in combineSetCC()  local
39817       return DAG.getSetCC(DL, VT, Add, DAG.getConstant(0, DL, OpVT), CC);  in combineSetCC()
39824       return DAG.getSetCC(DL, VT, Add, DAG.getConstant(0, DL, OpVT), CC);  in combineSetCC()
39831   if (VT.isVector() && VT.getVectorElementType() == MVT::i1 &&  in combineSetCC()
39845       assert(VT == LHS.getOperand(0).getValueType() &&  in combineSetCC()
39848         return DAG.getConstant(0, DL, VT);  in combineSetCC()
39850         return DAG.getConstant(1, DL, VT);  in combineSetCC()
39852         return DAG.getNOT(DL, LHS.getOperand(0), VT);  in combineSetCC()
39865   if (Subtarget.hasAVX512() && !Subtarget.hasBWI() && VT.isVector() &&  in combineSetCC()
39866       VT.getVectorElementType() == MVT::i1 &&  in combineSetCC()
39868        VT.getVectorNumElements() > 4) &&  in combineSetCC()
39873     return DAG.getNode(ISD::TRUNCATE, DL, VT, Setcc);  in combineSetCC()
39878   if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32 &&  in combineSetCC()
39889   MVT VT = N->getSimpleValueType(0);  in combineMOVMSK()  local
39893     assert(VT== MVT::i32 && "Unexpected result type");  in combineMOVMSK()
39901     return DAG.getConstant(Imm, SDLoc(N), VT);  in combineMOVMSK()
39913   APInt DemandedMask(APInt::getAllOnesValue(VT.getScalarSizeInBits()));  in combineMOVMSK()
39931       MVT VT = Src.getSimpleValueType();  in combineMOVMSK()  local
39932       unsigned BitWidth = VT.getScalarSizeInBits();  in combineMOVMSK()
39939       if (VT.getVectorElementType() == MVT::i8) {  in combineMOVMSK()
39940         VT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements() / 2);  in combineMOVMSK()
39941         X = DAG.getBitcast(VT, X);  in combineMOVMSK()
39943       SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, X,  in combineMOVMSK()
39944                                 DAG.getConstant(ShAmt, DL, VT));  in combineMOVMSK()
40072   EVT VT = N->getValueType(0);  in combineVectorCompareAndMaskUnaryOp()  local
40073   if (!VT.isVector() || N->getOperand(0)->getOpcode() != ISD::AND ||  in combineVectorCompareAndMaskUnaryOp()
40075       VT.getSizeInBits() != N->getOperand(0).getValueSizeInBits())  in combineVectorCompareAndMaskUnaryOp()
40093     SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));  in combineVectorCompareAndMaskUnaryOp()
40098     SDValue Res = DAG.getBitcast(VT, NewAnd);  in combineVectorCompareAndMaskUnaryOp()
40108   EVT VT = N->getValueType(0);  in combineUIntToFP()  local
40121     return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);  in combineUIntToFP()
40128     return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, Op0);  in combineUIntToFP()
40142   EVT VT = N->getValueType(0);  in combineSIntToFP()  local
40153     return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);  in combineSIntToFP()
40169       return DAG.getNode(ISD::SINT_TO_FP, dl, VT, Trunc);  in combineSIntToFP()
40181     if (VT == MVT::f16 || VT == MVT::f128)  in combineSIntToFP()
40186     if (Subtarget.hasDQI() && VT != MVT::f80)  in combineSIntToFP()
40189     if (!Ld->isVolatile() && !VT.isVector() &&  in combineSIntToFP()
40276   EVT VT = Op.getValueType();  in combineCMP()  local
40284     EVT VT = Op.getValueType();  in combineCMP()  local
40285     unsigned BitWidth = VT.getSizeInBits();  in combineCMP()
40292         Op = DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0),  in combineCMP()
40293                          DAG.getConstant(Mask, dl, VT));  in combineCMP()
40295                            DAG.getConstant(0, dl, VT));  in combineCMP()
40338   SDValue Op0 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(0));  in combineCMP()
40339   SDValue Op1 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(1));  in combineCMP()
40342   SDVTList VTs = DAG.getVTList(VT, MVT::i32);  in combineCMP()
40348                        DAG.getConstant(0, dl, VT));  in combineCMP()
40356     MVT VT = N->getSimpleValueType(0);  in combineSBB()  local
40357     SDVTList VTs = DAG.getVTList(VT, MVT::i32);  in combineSBB()
40378     EVT VT = N->getValueType(0);  in combineADC()  local
40380     SDValue Res1 = DAG.getNode(ISD::AND, DL, VT,  in combineADC()
40381                                DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,  in combineADC()
40385                                DAG.getConstant(1, DL, VT));  in combineADC()
40390     MVT VT = N->getSimpleValueType(0);  in combineADC()  local
40391     SDVTList VTs = DAG.getVTList(VT, MVT::i32);  in combineADC()
40432   EVT VT = N->getValueType(0);  in combineAddOrSubToADCOrSBB()  local
40444       return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,  in combineAddOrSubToADCOrSBB()
40462         return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,  in combineAddOrSubToADCOrSBB()
40473                        DAG.getVTList(VT, MVT::i32), X,  in combineAddOrSubToADCOrSBB()
40474                        DAG.getConstant(0, DL, VT), Y.getOperand(1));  in combineAddOrSubToADCOrSBB()
40493                          DAG.getVTList(VT, MVT::i32), X,  in combineAddOrSubToADCOrSBB()
40494                          DAG.getConstant(0, DL, VT), NewEFLAGS);  in combineAddOrSubToADCOrSBB()
40522       return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,  in combineAddOrSubToADCOrSBB()
40535       return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,  in combineAddOrSubToADCOrSBB()
40545   SDVTList VTs = DAG.getVTList(VT, MVT::i32);  in combineAddOrSubToADCOrSBB()
40551                        DAG.getConstant(-1ULL, DL, VT), Cmp1);  in combineAddOrSubToADCOrSBB()
40556                      DAG.getConstant(0, DL, VT), Cmp1);  in combineAddOrSubToADCOrSBB()
40567   EVT VT = N->getValueType(0);  in combineLoopMAddPattern()  local
40571   if (!VT.isVector() || VT.getVectorNumElements() < 8)  in combineLoopMAddPattern()
40585                                    VT.getVectorNumElements());  in combineLoopMAddPattern()
40587                                 VT.getVectorNumElements() / 2);  in combineLoopMAddPattern()
40592     MVT VT = MVT::getVectorVT(MVT::i32, Ops[0].getValueSizeInBits() / 32);  in combineLoopMAddPattern()  local
40593     return DAG.getNode(X86ISD::VPMADDWD, DL, VT, Ops);  in combineLoopMAddPattern()
40604     return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Madd,  in combineLoopMAddPattern()
40616   return DAG.getNode(ISD::ADD, DL, VT, Op0, Op1);  in combineLoopMAddPattern()
40625   EVT VT = N->getValueType(0);  in combineLoopSADPattern()  local
40631   if (!VT.isVector() || !VT.isSimple() ||  in combineLoopSADPattern()
40632       !(VT.getVectorElementType() == MVT::i32))  in combineLoopSADPattern()
40644   if (VT.getSizeInBits() / 4 > RegSize)  in combineLoopSADPattern()
40667     if (VT.getSizeInBits() >= ResVT.getSizeInBits())  in combineLoopSADPattern()
40670       Sad = DAG.getNode(ISD::TRUNCATE, DL, VT, Sad);  in combineLoopSADPattern()
40672     if (VT.getSizeInBits() > ResVT.getSizeInBits()) {  in combineLoopSADPattern()
40674       SDValue Zero = DAG.getConstant(0, DL, VT);  in combineLoopSADPattern()
40675       Sad = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Zero, Sad,  in combineLoopSADPattern()
40695   return DAG.getNode(ISD::ADD, DL, VT, Op0, Op1);  in combineLoopSADPattern()
40710   EVT VT = N->getValueType(0);  in combineIncDecVector()  local
40711   if (!VT.is128BitVector() && !VT.is256BitVector() && !VT.is512BitVector())  in combineIncDecVector()
40718   SDValue AllOnesVec = getOnesVector(VT, DAG, SDLoc(N));  in combineIncDecVector()
40720   return DAG.getNode(NewOpcode, SDLoc(N), VT, N->getOperand(0), AllOnesVec);  in combineIncDecVector()
40724                             const SDLoc &DL, EVT VT,  in matchPMADDWD()  argument
40744   if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 ||  in matchPMADDWD()
40745       VT.getVectorNumElements() < 4 ||  in matchPMADDWD()
40746       !isPowerOf2_32(VT.getVectorNumElements()))  in matchPMADDWD()
40761   for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; i += 2) {  in matchPMADDWD()
40827   return SplitOpsAndApply(DAG, Subtarget, DL, VT,  in matchPMADDWD()
40838   EVT VT = N->getValueType(0);  in combineAddToSUBUS()  local
40841   if (!VT.isVector() || VT.getVectorNumElements() < 2 ||  in combineAddToSUBUS()
40842       !isPowerOf2_32(VT.getVectorNumElements()) ||  in combineAddToSUBUS()
40843       !(VT.getVectorElementType() == MVT::i8 ||  in combineAddToSUBUS()
40844         VT.getVectorElementType() == MVT::i16))  in combineAddToSUBUS()
40861   return DAG.getNode(ISD::USUBSAT, DL, VT, Op0.getOperand(0),  in combineAddToSUBUS()
40869                               const SDLoc &DL, EVT VT,  in matchPMADDWD_2()  argument
40877   if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 ||  in matchPMADDWD_2()
40878       VT.getVectorNumElements() < 4 ||  in matchPMADDWD_2()
40879       !isPowerOf2_32(VT.getVectorNumElements()))  in matchPMADDWD_2()
40982   return SplitOpsAndApply(DAG, Subtarget, DL, VT, { In0, In1 },  in matchPMADDWD_2()
40995   EVT VT = N->getValueType(0);  in combineAdd()  local
40999   if (SDValue MAdd = matchPMADDWD(DAG, Op0, Op1, SDLoc(N), VT, Subtarget))  in combineAdd()
41001   if (SDValue MAdd = matchPMADDWD_2(DAG, Op0, Op1, SDLoc(N), VT, Subtarget))  in combineAdd()
41005   if ((VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v16i16 ||  in combineAdd()
41006        VT == MVT::v8i32) &&  in combineAdd()
41013     return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, {Op0, Op1},  in combineAdd()
41030   EVT VT = N->getValueType(0);  in combineSubToSubus()  local
41034   if (!(Subtarget.hasSSE2() && (VT == MVT::v16i8 || VT == MVT::v8i16)) &&  in combineSubToSubus()
41035       !(Subtarget.hasSSSE3() && (VT == MVT::v8i32 || VT == MVT::v8i64)) &&  in combineSubToSubus()
41036       !(Subtarget.hasAVX() && (VT == MVT::v32i8 || VT == MVT::v16i16)) &&  in combineSubToSubus()
41037       !(Subtarget.useBWIRegs() && (VT == MVT::v64i8 || VT == MVT::v32i16 ||  in combineSubToSubus()
41038                                    VT == MVT::v16i32 || VT == MVT::v8i64)))  in combineSubToSubus()
41075   if (VT != MVT::v8i32 && VT != MVT::v16i32 && VT != MVT::v8i64)  in combineSubToSubus()
41076     return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT,  in combineSubToSubus()
41085   if ((VT == MVT::v8i64 && NumZeros < 48) || NumZeros < 16)  in combineSubToSubus()
41090   if (VT == MVT::v8i32 || VT == MVT::v8i64)  in combineSubToSubus()
41128       EVT VT = Op0.getValueType();  in combineSub()  local
41129       SDValue NewXor = DAG.getNode(ISD::XOR, SDLoc(Op1), VT,  in combineSub()
41131                                    DAG.getConstant(~XorC, SDLoc(Op1), VT));  in combineSub()
41132       return DAG.getNode(ISD::ADD, SDLoc(N), VT, NewXor,  in combineSub()
41133                          DAG.getConstant(C->getAPIntValue() + 1, SDLoc(N), VT));  in combineSub()
41138   EVT VT = N->getValueType(0);  in combineSub()  local
41139   if ((VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v16i16 ||  in combineSub()
41140        VT == MVT::v8i32) &&  in combineSub()
41147     return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, {Op0, Op1},  in combineSub()
41163   MVT VT = N->getSimpleValueType(0);  in combineVectorCompare()  local
41168       return DAG.getConstant(-1, DL, VT);  in combineVectorCompare()
41170       return DAG.getConstant(0, DL, VT);  in combineVectorCompare()
41351   EVT VT = N->getValueType(0);  in combineExtractSubvector()  local
41369       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT,  in combineExtractSubvector()
41443   EVT VT = N->getValueType(0);  in combineScalarToVector()  local
41450   if (VT == MVT::v1i1 && Src.getOpcode() == ISD::AND && Src.hasOneUse())  in combineScalarToVector()
41457   if (VT == MVT::v1i1 && Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&  in combineScalarToVector()
41462         return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT,  in combineScalarToVector()
41633 bool X86TargetLowering::isTypeDesirableForOp(unsigned Opc, EVT VT) const {  in isTypeDesirableForOp()
41634   if (!isTypeLegal(VT))  in isTypeDesirableForOp()
41638   if (Opc == ISD::SHL && VT.isVector() && VT.getVectorElementType() == MVT::i8)  in isTypeDesirableForOp()
41645   if (Opc == ISD::MUL && VT == MVT::i8)  in isTypeDesirableForOp()
41650   if (VT == MVT::i16) {  in isTypeDesirableForOp()
41691   EVT VT = Op.getValueType();  in IsDesirableToPromoteOp()  local
41692   bool Is8BitMulByConstant = VT == MVT::i8 && Op.getOpcode() == ISD::MUL &&  in IsDesirableToPromoteOp()
41699   if (VT != MVT::i16 && !Is8BitMulByConstant)  in IsDesirableToPromoteOp()
42327                                                 MVT VT) const {  in getRegForInlineAsmConstraint()
42340         switch (VT.SimpleTy) {  in getRegForInlineAsmConstraint()
42357         if (VT == MVT::i32 || VT == MVT::f32)  in getRegForInlineAsmConstraint()
42359         if (VT == MVT::i16)  in getRegForInlineAsmConstraint()
42361         if (VT == MVT::i8 || VT == MVT::i1)  in getRegForInlineAsmConstraint()
42363         if (VT == MVT::i64 || VT == MVT::f64)  in getRegForInlineAsmConstraint()
42370       if (VT == MVT::i32 || VT == MVT::f32)  in getRegForInlineAsmConstraint()
42372       if (VT == MVT::i16)  in getRegForInlineAsmConstraint()
42374       if (VT == MVT::i8 || VT == MVT::i1)  in getRegForInlineAsmConstraint()
42376       if (VT == MVT::i64)  in getRegForInlineAsmConstraint()
42381       if (VT == MVT::i8 || VT == MVT::i1)  in getRegForInlineAsmConstraint()
42383       if (VT == MVT::i16)  in getRegForInlineAsmConstraint()
42385       if (VT == MVT::i32 || VT == MVT::f32 || !Subtarget.is64Bit())  in getRegForInlineAsmConstraint()
42389       if (VT == MVT::i8 || VT == MVT::i1)  in getRegForInlineAsmConstraint()
42391       if (VT == MVT::i16)  in getRegForInlineAsmConstraint()
42393       if (VT == MVT::i32 || !Subtarget.is64Bit())  in getRegForInlineAsmConstraint()
42399       if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT))  in getRegForInlineAsmConstraint()
42401       if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT))  in getRegForInlineAsmConstraint()
42415       switch (VT.SimpleTy) {  in getRegForInlineAsmConstraint()
42464       return getRegForInlineAsmConstraint(TRI, "Y", VT);  in getRegForInlineAsmConstraint()
42475         switch (VT.SimpleTy) {  in getRegForInlineAsmConstraint()
42496   Res = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);  in getRegForInlineAsmConstraint()
42560   if (TRI->isTypeLegalForClass(*Res.second, VT) || VT == MVT::Other)  in getRegForInlineAsmConstraint()
42572     unsigned Size = VT.getSizeInBits();  in getRegForInlineAsmConstraint()
42618     if (VT == MVT::f32 || VT == MVT::i32)  in getRegForInlineAsmConstraint()
42620     else if (VT == MVT::f64 || VT == MVT::i64)  in getRegForInlineAsmConstraint()
42622     else if (TRI->isTypeLegalForClass(X86::VR128RegClass, VT))  in getRegForInlineAsmConstraint()
42624     else if (TRI->isTypeLegalForClass(X86::VR256RegClass, VT))  in getRegForInlineAsmConstraint()
42626     else if (TRI->isTypeLegalForClass(X86::VR512RegClass, VT))  in getRegForInlineAsmConstraint()
42666 bool X86TargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const {  in isIntDivCheap()  argument
42676   return OptSize && !VT.isVector();  in isIntDivCheap()