Lines Matching refs:VT
180 for (MVT VT : MVT::integer_valuetypes()) in X86TargetLowering() local
181 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); in X86TargetLowering()
194 for (auto VT : {MVT::f32, MVT::f64, MVT::f80}) { in X86TargetLowering()
195 setCondCodeAction(ISD::SETOEQ, VT, Expand); in X86TargetLowering()
196 setCondCodeAction(ISD::SETUNE, VT, Expand); in X86TargetLowering()
299 for (MVT VT : { MVT::i8, MVT::i16, MVT::i32 }) { in X86TargetLowering()
300 setOperationAction(ISD::FP_TO_UINT_SAT, VT, Custom); in X86TargetLowering()
301 setOperationAction(ISD::FP_TO_SINT_SAT, VT, Custom); in X86TargetLowering()
335 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { in X86TargetLowering()
336 setOperationAction(ISD::MULHS, VT, Expand); in X86TargetLowering()
337 setOperationAction(ISD::MULHU, VT, Expand); in X86TargetLowering()
338 setOperationAction(ISD::SDIV, VT, Expand); in X86TargetLowering()
339 setOperationAction(ISD::UDIV, VT, Expand); in X86TargetLowering()
340 setOperationAction(ISD::SREM, VT, Expand); in X86TargetLowering()
341 setOperationAction(ISD::UREM, VT, Expand); in X86TargetLowering()
346 for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128, in X86TargetLowering()
348 setOperationAction(ISD::BR_CC, VT, Expand); in X86TargetLowering()
349 setOperationAction(ISD::SELECT_CC, VT, Expand); in X86TargetLowering()
393 for (auto VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) { in X86TargetLowering()
394 if (VT == MVT::i64 && !Subtarget.is64Bit()) in X86TargetLowering()
396 setOperationAction(ISD::CTLZ , VT, Custom); in X86TargetLowering()
397 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom); in X86TargetLowering()
415 for (MVT VT : {MVT::f32, MVT::f64, MVT::f80, MVT::f128}) { in X86TargetLowering()
416 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand); in X86TargetLowering()
417 setLoadExtAction(ISD::EXTLOAD, VT, MVT::bf16, Expand); in X86TargetLowering()
418 setTruncStoreAction(VT, MVT::f16, Expand); in X86TargetLowering()
419 setTruncStoreAction(VT, MVT::bf16, Expand); in X86TargetLowering()
421 setOperationAction(ISD::BF16_TO_FP, VT, Expand); in X86TargetLowering()
422 setOperationAction(ISD::FP_TO_BF16, VT, Custom); in X86TargetLowering()
451 for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) { in X86TargetLowering()
452 setOperationAction(ISD::SELECT, VT, Custom); in X86TargetLowering()
453 setOperationAction(ISD::SETCC, VT, Custom); in X86TargetLowering()
454 setOperationAction(ISD::STRICT_FSETCC, VT, Custom); in X86TargetLowering()
455 setOperationAction(ISD::STRICT_FSETCCS, VT, Custom); in X86TargetLowering()
457 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { in X86TargetLowering()
458 if (VT == MVT::i64 && !Subtarget.is64Bit()) in X86TargetLowering()
460 setOperationAction(ISD::SELECT, VT, Custom); in X86TargetLowering()
461 setOperationAction(ISD::SETCC, VT, Custom); in X86TargetLowering()
478 for (auto VT : { MVT::i32, MVT::i64 }) { in X86TargetLowering()
479 if (VT == MVT::i64 && !Subtarget.is64Bit()) in X86TargetLowering()
481 setOperationAction(ISD::ConstantPool , VT, Custom); in X86TargetLowering()
482 setOperationAction(ISD::JumpTable , VT, Custom); in X86TargetLowering()
483 setOperationAction(ISD::GlobalAddress , VT, Custom); in X86TargetLowering()
484 setOperationAction(ISD::GlobalTLSAddress, VT, Custom); in X86TargetLowering()
485 setOperationAction(ISD::ExternalSymbol , VT, Custom); in X86TargetLowering()
486 setOperationAction(ISD::BlockAddress , VT, Custom); in X86TargetLowering()
490 for (auto VT : { MVT::i32, MVT::i64 }) { in X86TargetLowering()
491 if (VT == MVT::i64 && !Subtarget.is64Bit()) in X86TargetLowering()
493 setOperationAction(ISD::SHL_PARTS, VT, Custom); in X86TargetLowering()
494 setOperationAction(ISD::SRA_PARTS, VT, Custom); in X86TargetLowering()
495 setOperationAction(ISD::SRL_PARTS, VT, Custom); in X86TargetLowering()
504 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { in X86TargetLowering()
505 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Custom); in X86TargetLowering()
506 setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom); in X86TargetLowering()
507 setOperationAction(ISD::ATOMIC_LOAD_ADD, VT, Custom); in X86TargetLowering()
508 setOperationAction(ISD::ATOMIC_LOAD_OR, VT, Custom); in X86TargetLowering()
509 setOperationAction(ISD::ATOMIC_LOAD_XOR, VT, Custom); in X86TargetLowering()
510 setOperationAction(ISD::ATOMIC_LOAD_AND, VT, Custom); in X86TargetLowering()
511 setOperationAction(ISD::ATOMIC_STORE, VT, Custom); in X86TargetLowering()
558 auto setF16Action = [&] (MVT VT, LegalizeAction Action) { in X86TargetLowering() argument
559 setOperationAction(ISD::FABS, VT, Action); in X86TargetLowering()
560 setOperationAction(ISD::FNEG, VT, Action); in X86TargetLowering()
561 setOperationAction(ISD::FCOPYSIGN, VT, Expand); in X86TargetLowering()
562 setOperationAction(ISD::FREM, VT, Action); in X86TargetLowering()
563 setOperationAction(ISD::FMA, VT, Action); in X86TargetLowering()
564 setOperationAction(ISD::FMINNUM, VT, Action); in X86TargetLowering()
565 setOperationAction(ISD::FMAXNUM, VT, Action); in X86TargetLowering()
566 setOperationAction(ISD::FMINIMUM, VT, Action); in X86TargetLowering()
567 setOperationAction(ISD::FMAXIMUM, VT, Action); in X86TargetLowering()
568 setOperationAction(ISD::FSIN, VT, Action); in X86TargetLowering()
569 setOperationAction(ISD::FCOS, VT, Action); in X86TargetLowering()
570 setOperationAction(ISD::FSINCOS, VT, Action); in X86TargetLowering()
571 setOperationAction(ISD::FSQRT, VT, Action); in X86TargetLowering()
572 setOperationAction(ISD::FPOW, VT, Action); in X86TargetLowering()
573 setOperationAction(ISD::FLOG, VT, Action); in X86TargetLowering()
574 setOperationAction(ISD::FLOG2, VT, Action); in X86TargetLowering()
575 setOperationAction(ISD::FLOG10, VT, Action); in X86TargetLowering()
576 setOperationAction(ISD::FEXP, VT, Action); in X86TargetLowering()
577 setOperationAction(ISD::FEXP2, VT, Action); in X86TargetLowering()
578 setOperationAction(ISD::FCEIL, VT, Action); in X86TargetLowering()
579 setOperationAction(ISD::FFLOOR, VT, Action); in X86TargetLowering()
580 setOperationAction(ISD::FNEARBYINT, VT, Action); in X86TargetLowering()
581 setOperationAction(ISD::FRINT, VT, Action); in X86TargetLowering()
582 setOperationAction(ISD::BR_CC, VT, Action); in X86TargetLowering()
583 setOperationAction(ISD::SETCC, VT, Action); in X86TargetLowering()
584 setOperationAction(ISD::SELECT, VT, Custom); in X86TargetLowering()
585 setOperationAction(ISD::SELECT_CC, VT, Action); in X86TargetLowering()
586 setOperationAction(ISD::FROUND, VT, Action); in X86TargetLowering()
587 setOperationAction(ISD::FROUNDEVEN, VT, Action); in X86TargetLowering()
588 setOperationAction(ISD::FTRUNC, VT, Action); in X86TargetLowering()
607 for (auto VT : { MVT::f32, MVT::f64 }) { in X86TargetLowering()
609 setOperationAction(ISD::FABS, VT, Custom); in X86TargetLowering()
612 setOperationAction(ISD::FNEG, VT, Custom); in X86TargetLowering()
615 setOperationAction(ISD::FCOPYSIGN, VT, Custom); in X86TargetLowering()
618 setOperationAction(ISD::FADD, VT, Custom); in X86TargetLowering()
619 setOperationAction(ISD::FSUB, VT, Custom); in X86TargetLowering()
622 setOperationAction(ISD::FSIN , VT, Expand); in X86TargetLowering()
623 setOperationAction(ISD::FCOS , VT, Expand); in X86TargetLowering()
624 setOperationAction(ISD::FSINCOS, VT, Expand); in X86TargetLowering()
712 for (auto VT : { MVT::f32, MVT::f64 }) { in X86TargetLowering()
713 setOperationAction(ISD::UNDEF, VT, Expand); in X86TargetLowering()
714 setOperationAction(ISD::FCOPYSIGN, VT, Expand); in X86TargetLowering()
717 setOperationAction(ISD::FSIN , VT, Expand); in X86TargetLowering()
718 setOperationAction(ISD::FCOS , VT, Expand); in X86TargetLowering()
719 setOperationAction(ISD::FSINCOS, VT, Expand); in X86TargetLowering()
892 for (auto VT : { MVT::v8f16, MVT::v16f16, MVT::v32f16, in X86TargetLowering()
895 setOperationAction(ISD::FSIN, VT, Expand); in X86TargetLowering()
896 setOperationAction(ISD::FSINCOS, VT, Expand); in X86TargetLowering()
897 setOperationAction(ISD::FCOS, VT, Expand); in X86TargetLowering()
898 setOperationAction(ISD::FREM, VT, Expand); in X86TargetLowering()
899 setOperationAction(ISD::FCOPYSIGN, VT, Expand); in X86TargetLowering()
900 setOperationAction(ISD::FPOW, VT, Expand); in X86TargetLowering()
901 setOperationAction(ISD::FLOG, VT, Expand); in X86TargetLowering()
902 setOperationAction(ISD::FLOG2, VT, Expand); in X86TargetLowering()
903 setOperationAction(ISD::FLOG10, VT, Expand); in X86TargetLowering()
904 setOperationAction(ISD::FEXP, VT, Expand); in X86TargetLowering()
905 setOperationAction(ISD::FEXP2, VT, Expand); in X86TargetLowering()
911 for (MVT VT : MVT::fixedlen_vector_valuetypes()) { in X86TargetLowering() local
912 setOperationAction(ISD::SDIV, VT, Expand); in X86TargetLowering()
913 setOperationAction(ISD::UDIV, VT, Expand); in X86TargetLowering()
914 setOperationAction(ISD::SREM, VT, Expand); in X86TargetLowering()
915 setOperationAction(ISD::UREM, VT, Expand); in X86TargetLowering()
916 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT,Expand); in X86TargetLowering()
917 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand); in X86TargetLowering()
918 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT,Expand); in X86TargetLowering()
919 setOperationAction(ISD::INSERT_SUBVECTOR, VT,Expand); in X86TargetLowering()
920 setOperationAction(ISD::FMA, VT, Expand); in X86TargetLowering()
921 setOperationAction(ISD::FFLOOR, VT, Expand); in X86TargetLowering()
922 setOperationAction(ISD::FCEIL, VT, Expand); in X86TargetLowering()
923 setOperationAction(ISD::FTRUNC, VT, Expand); in X86TargetLowering()
924 setOperationAction(ISD::FRINT, VT, Expand); in X86TargetLowering()
925 setOperationAction(ISD::FNEARBYINT, VT, Expand); in X86TargetLowering()
926 setOperationAction(ISD::SMUL_LOHI, VT, Expand); in X86TargetLowering()
927 setOperationAction(ISD::MULHS, VT, Expand); in X86TargetLowering()
928 setOperationAction(ISD::UMUL_LOHI, VT, Expand); in X86TargetLowering()
929 setOperationAction(ISD::MULHU, VT, Expand); in X86TargetLowering()
930 setOperationAction(ISD::SDIVREM, VT, Expand); in X86TargetLowering()
931 setOperationAction(ISD::UDIVREM, VT, Expand); in X86TargetLowering()
932 setOperationAction(ISD::CTPOP, VT, Expand); in X86TargetLowering()
933 setOperationAction(ISD::CTTZ, VT, Expand); in X86TargetLowering()
934 setOperationAction(ISD::CTLZ, VT, Expand); in X86TargetLowering()
935 setOperationAction(ISD::ROTL, VT, Expand); in X86TargetLowering()
936 setOperationAction(ISD::ROTR, VT, Expand); in X86TargetLowering()
937 setOperationAction(ISD::BSWAP, VT, Expand); in X86TargetLowering()
938 setOperationAction(ISD::SETCC, VT, Expand); in X86TargetLowering()
939 setOperationAction(ISD::FP_TO_UINT, VT, Expand); in X86TargetLowering()
940 setOperationAction(ISD::FP_TO_SINT, VT, Expand); in X86TargetLowering()
941 setOperationAction(ISD::UINT_TO_FP, VT, Expand); in X86TargetLowering()
942 setOperationAction(ISD::SINT_TO_FP, VT, Expand); in X86TargetLowering()
943 setOperationAction(ISD::SIGN_EXTEND_INREG, VT,Expand); in X86TargetLowering()
944 setOperationAction(ISD::TRUNCATE, VT, Expand); in X86TargetLowering()
945 setOperationAction(ISD::SIGN_EXTEND, VT, Expand); in X86TargetLowering()
946 setOperationAction(ISD::ZERO_EXTEND, VT, Expand); in X86TargetLowering()
947 setOperationAction(ISD::ANY_EXTEND, VT, Expand); in X86TargetLowering()
948 setOperationAction(ISD::SELECT_CC, VT, Expand); in X86TargetLowering()
950 setTruncStoreAction(InnerVT, VT, Expand); in X86TargetLowering()
952 setLoadExtAction(ISD::SEXTLOAD, InnerVT, VT, Expand); in X86TargetLowering()
953 setLoadExtAction(ISD::ZEXTLOAD, InnerVT, VT, Expand); in X86TargetLowering()
959 if (VT.getVectorElementType() == MVT::i1) in X86TargetLowering()
960 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand); in X86TargetLowering()
964 if (VT.getVectorElementType() == MVT::f16 || in X86TargetLowering()
965 VT.getVectorElementType() == MVT::bf16) in X86TargetLowering()
966 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand); in X86TargetLowering()
1017 for (auto VT : { MVT::v2i8, MVT::v4i8, MVT::v8i8, in X86TargetLowering()
1019 setOperationAction(ISD::SDIV, VT, Custom); in X86TargetLowering()
1020 setOperationAction(ISD::SREM, VT, Custom); in X86TargetLowering()
1021 setOperationAction(ISD::UDIV, VT, Custom); in X86TargetLowering()
1022 setOperationAction(ISD::UREM, VT, Custom); in X86TargetLowering()
1050 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) { in X86TargetLowering()
1051 setOperationAction(ISD::SMAX, VT, VT == MVT::v8i16 ? Legal : Custom); in X86TargetLowering()
1052 setOperationAction(ISD::SMIN, VT, VT == MVT::v8i16 ? Legal : Custom); in X86TargetLowering()
1053 setOperationAction(ISD::UMAX, VT, VT == MVT::v16i8 ? Legal : Custom); in X86TargetLowering()
1054 setOperationAction(ISD::UMIN, VT, VT == MVT::v16i8 ? Legal : Custom); in X86TargetLowering()
1073 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) { in X86TargetLowering()
1074 setOperationAction(ISD::SETCC, VT, Custom); in X86TargetLowering()
1075 setOperationAction(ISD::STRICT_FSETCC, VT, Custom); in X86TargetLowering()
1076 setOperationAction(ISD::STRICT_FSETCCS, VT, Custom); in X86TargetLowering()
1077 setOperationAction(ISD::CTPOP, VT, Custom); in X86TargetLowering()
1078 setOperationAction(ISD::ABS, VT, Custom); in X86TargetLowering()
1082 setCondCodeAction(ISD::SETLT, VT, Custom); in X86TargetLowering()
1083 setCondCodeAction(ISD::SETLE, VT, Custom); in X86TargetLowering()
1086 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) { in X86TargetLowering()
1087 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom); in X86TargetLowering()
1088 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in X86TargetLowering()
1089 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in X86TargetLowering()
1090 setOperationAction(ISD::VSELECT, VT, Custom); in X86TargetLowering()
1091 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1094 for (auto VT : { MVT::v8f16, MVT::v2f64, MVT::v2i64 }) { in X86TargetLowering()
1095 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in X86TargetLowering()
1096 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in X86TargetLowering()
1097 setOperationAction(ISD::VSELECT, VT, Custom); in X86TargetLowering()
1099 if (VT == MVT::v2i64 && !Subtarget.is64Bit()) in X86TargetLowering()
1102 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1103 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1127 for (auto VT : {MVT::v2i8, MVT::v4i8, MVT::v8i8, MVT::v2i16, MVT::v4i16}) { in X86TargetLowering()
1128 setOperationAction(ISD::FP_TO_SINT, VT, Custom); in X86TargetLowering()
1129 setOperationAction(ISD::FP_TO_UINT, VT, Custom); in X86TargetLowering()
1130 setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom); in X86TargetLowering()
1131 setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom); in X86TargetLowering()
1191 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) { in X86TargetLowering()
1192 setOperationAction(ISD::SRL, VT, Custom); in X86TargetLowering()
1193 setOperationAction(ISD::SHL, VT, Custom); in X86TargetLowering()
1194 setOperationAction(ISD::SRA, VT, Custom); in X86TargetLowering()
1195 if (VT == MVT::v2i64) continue; in X86TargetLowering()
1196 setOperationAction(ISD::ROTL, VT, Custom); in X86TargetLowering()
1197 setOperationAction(ISD::ROTR, VT, Custom); in X86TargetLowering()
1198 setOperationAction(ISD::FSHL, VT, Custom); in X86TargetLowering()
1199 setOperationAction(ISD::FSHR, VT, Custom); in X86TargetLowering()
1267 for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64 }) { in X86TargetLowering()
1268 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Legal); in X86TargetLowering()
1269 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Legal); in X86TargetLowering()
1299 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, in X86TargetLowering()
1301 setOperationAction(ISD::ROTL, VT, Custom); in X86TargetLowering()
1302 setOperationAction(ISD::ROTR, VT, Custom); in X86TargetLowering()
1306 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) in X86TargetLowering()
1307 setOperationAction(ISD::BITREVERSE, VT, Custom); in X86TargetLowering()
1309 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, in X86TargetLowering()
1311 setOperationAction(ISD::BITREVERSE, VT, Custom); in X86TargetLowering()
1332 for (auto VT : { MVT::v8f32, MVT::v4f64 }) { in X86TargetLowering()
1333 setOperationAction(ISD::FFLOOR, VT, Legal); in X86TargetLowering()
1334 setOperationAction(ISD::STRICT_FFLOOR, VT, Legal); in X86TargetLowering()
1335 setOperationAction(ISD::FCEIL, VT, Legal); in X86TargetLowering()
1336 setOperationAction(ISD::STRICT_FCEIL, VT, Legal); in X86TargetLowering()
1337 setOperationAction(ISD::FTRUNC, VT, Legal); in X86TargetLowering()
1338 setOperationAction(ISD::STRICT_FTRUNC, VT, Legal); in X86TargetLowering()
1339 setOperationAction(ISD::FRINT, VT, Legal); in X86TargetLowering()
1340 setOperationAction(ISD::STRICT_FRINT, VT, Legal); in X86TargetLowering()
1341 setOperationAction(ISD::FNEARBYINT, VT, Legal); in X86TargetLowering()
1342 setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal); in X86TargetLowering()
1343 setOperationAction(ISD::FROUNDEVEN, VT, Legal); in X86TargetLowering()
1344 setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal); in X86TargetLowering()
1346 setOperationAction(ISD::FROUND, VT, Custom); in X86TargetLowering()
1348 setOperationAction(ISD::FNEG, VT, Custom); in X86TargetLowering()
1349 setOperationAction(ISD::FABS, VT, Custom); in X86TargetLowering()
1350 setOperationAction(ISD::FCOPYSIGN, VT, Custom); in X86TargetLowering()
1387 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) { in X86TargetLowering()
1388 setOperationAction(ISD::SRL, VT, Custom); in X86TargetLowering()
1389 setOperationAction(ISD::SHL, VT, Custom); in X86TargetLowering()
1390 setOperationAction(ISD::SRA, VT, Custom); in X86TargetLowering()
1391 if (VT == MVT::v4i64) continue; in X86TargetLowering()
1392 setOperationAction(ISD::ROTL, VT, Custom); in X86TargetLowering()
1393 setOperationAction(ISD::ROTR, VT, Custom); in X86TargetLowering()
1394 setOperationAction(ISD::FSHL, VT, Custom); in X86TargetLowering()
1395 setOperationAction(ISD::FSHR, VT, Custom); in X86TargetLowering()
1412 for (auto VT : { MVT::v16i16, MVT::v8i32, MVT::v4i64 }) { in X86TargetLowering()
1413 setOperationAction(ISD::SIGN_EXTEND, VT, Custom); in X86TargetLowering()
1414 setOperationAction(ISD::ZERO_EXTEND, VT, Custom); in X86TargetLowering()
1415 setOperationAction(ISD::ANY_EXTEND, VT, Custom); in X86TargetLowering()
1423 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) { in X86TargetLowering()
1424 setOperationAction(ISD::SETCC, VT, Custom); in X86TargetLowering()
1425 setOperationAction(ISD::STRICT_FSETCC, VT, Custom); in X86TargetLowering()
1426 setOperationAction(ISD::STRICT_FSETCCS, VT, Custom); in X86TargetLowering()
1427 setOperationAction(ISD::CTPOP, VT, Custom); in X86TargetLowering()
1428 setOperationAction(ISD::CTLZ, VT, Custom); in X86TargetLowering()
1432 setCondCodeAction(ISD::SETLT, VT, Custom); in X86TargetLowering()
1433 setCondCodeAction(ISD::SETLE, VT, Custom); in X86TargetLowering()
1437 for (auto VT : { MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32, in X86TargetLowering()
1439 setOperationAction(ISD::FMA, VT, Legal); in X86TargetLowering()
1440 setOperationAction(ISD::STRICT_FMA, VT, Legal); in X86TargetLowering()
1444 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) { in X86TargetLowering()
1445 setOperationAction(ISD::ADD, VT, HasInt256 ? Legal : Custom); in X86TargetLowering()
1446 setOperationAction(ISD::SUB, VT, HasInt256 ? Legal : Custom); in X86TargetLowering()
1485 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32 }) { in X86TargetLowering()
1486 setOperationAction(ISD::ABS, VT, HasInt256 ? Legal : Custom); in X86TargetLowering()
1487 setOperationAction(ISD::SMAX, VT, HasInt256 ? Legal : Custom); in X86TargetLowering()
1488 setOperationAction(ISD::UMAX, VT, HasInt256 ? Legal : Custom); in X86TargetLowering()
1489 setOperationAction(ISD::SMIN, VT, HasInt256 ? Legal : Custom); in X86TargetLowering()
1490 setOperationAction(ISD::UMIN, VT, HasInt256 ? Legal : Custom); in X86TargetLowering()
1493 for (auto VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64}) { in X86TargetLowering()
1494 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom); in X86TargetLowering()
1495 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom); in X86TargetLowering()
1515 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64, in X86TargetLowering()
1517 setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom); in X86TargetLowering()
1518 setOperationAction(ISD::MSTORE, VT, Legal); in X86TargetLowering()
1523 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, in X86TargetLowering()
1525 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal); in X86TargetLowering()
1529 for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64, in X86TargetLowering()
1531 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in X86TargetLowering()
1532 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in X86TargetLowering()
1533 setOperationAction(ISD::VSELECT, VT, Custom); in X86TargetLowering()
1534 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1535 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1536 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom); in X86TargetLowering()
1537 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal); in X86TargetLowering()
1538 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in X86TargetLowering()
1539 setOperationAction(ISD::STORE, VT, Custom); in X86TargetLowering()
1554 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64, in X86TargetLowering()
1556 setOperationAction(ISD::MGATHER, VT, Custom); in X86TargetLowering()
1562 for (MVT VT : { MVT::f16, MVT::v2f16, MVT::v4f16, MVT::v8f16 }) { in X86TargetLowering()
1563 setOperationAction(ISD::FP_ROUND, VT, Custom); in X86TargetLowering()
1564 setOperationAction(ISD::STRICT_FP_ROUND, VT, Custom); in X86TargetLowering()
1566 for (MVT VT : { MVT::f32, MVT::v2f32, MVT::v4f32 }) { in X86TargetLowering()
1567 setOperationAction(ISD::FP_EXTEND, VT, Custom); in X86TargetLowering()
1568 setOperationAction(ISD::STRICT_FP_EXTEND, VT, Custom); in X86TargetLowering()
1620 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) { in X86TargetLowering()
1621 setOperationAction(ISD::SIGN_EXTEND, VT, Custom); in X86TargetLowering()
1622 setOperationAction(ISD::ZERO_EXTEND, VT, Custom); in X86TargetLowering()
1623 setOperationAction(ISD::ANY_EXTEND, VT, Custom); in X86TargetLowering()
1626 for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 }) in X86TargetLowering()
1627 setOperationAction(ISD::VSELECT, VT, Expand); in X86TargetLowering()
1629 for (auto VT : { MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 }) { in X86TargetLowering()
1630 setOperationAction(ISD::SETCC, VT, Custom); in X86TargetLowering()
1631 setOperationAction(ISD::STRICT_FSETCC, VT, Custom); in X86TargetLowering()
1632 setOperationAction(ISD::STRICT_FSETCCS, VT, Custom); in X86TargetLowering()
1633 setOperationAction(ISD::SELECT, VT, Custom); in X86TargetLowering()
1634 setOperationAction(ISD::TRUNCATE, VT, Custom); in X86TargetLowering()
1636 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in X86TargetLowering()
1637 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in X86TargetLowering()
1638 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1639 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); in X86TargetLowering()
1640 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1641 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in X86TargetLowering()
1644 for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1 }) in X86TargetLowering()
1645 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); in X86TargetLowering()
1672 for (MVT VT : { MVT::v16f32, MVT::v8f64 }) { in X86TargetLowering()
1673 setOperationAction(ISD::FNEG, VT, Custom); in X86TargetLowering()
1674 setOperationAction(ISD::FABS, VT, Custom); in X86TargetLowering()
1675 setOperationAction(ISD::FMA, VT, Legal); in X86TargetLowering()
1676 setOperationAction(ISD::STRICT_FMA, VT, Legal); in X86TargetLowering()
1677 setOperationAction(ISD::FCOPYSIGN, VT, Custom); in X86TargetLowering()
1680 for (MVT VT : { MVT::v16i1, MVT::v16i8, MVT::v16i16 }) { in X86TargetLowering()
1681 setOperationPromotedToType(ISD::FP_TO_SINT , VT, MVT::v16i32); in X86TargetLowering()
1682 setOperationPromotedToType(ISD::FP_TO_UINT , VT, MVT::v16i32); in X86TargetLowering()
1683 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, VT, MVT::v16i32); in X86TargetLowering()
1684 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, VT, MVT::v16i32); in X86TargetLowering()
1721 for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64, in X86TargetLowering()
1723 setOperationAction(ISD::MLOAD, VT, Custom); in X86TargetLowering()
1724 setOperationAction(ISD::MSTORE, VT, Custom); in X86TargetLowering()
1749 for (auto VT : { MVT::v16f32, MVT::v8f64 }) { in X86TargetLowering()
1750 setOperationAction(ISD::FFLOOR, VT, Legal); in X86TargetLowering()
1751 setOperationAction(ISD::STRICT_FFLOOR, VT, Legal); in X86TargetLowering()
1752 setOperationAction(ISD::FCEIL, VT, Legal); in X86TargetLowering()
1753 setOperationAction(ISD::STRICT_FCEIL, VT, Legal); in X86TargetLowering()
1754 setOperationAction(ISD::FTRUNC, VT, Legal); in X86TargetLowering()
1755 setOperationAction(ISD::STRICT_FTRUNC, VT, Legal); in X86TargetLowering()
1756 setOperationAction(ISD::FRINT, VT, Legal); in X86TargetLowering()
1757 setOperationAction(ISD::STRICT_FRINT, VT, Legal); in X86TargetLowering()
1758 setOperationAction(ISD::FNEARBYINT, VT, Legal); in X86TargetLowering()
1759 setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal); in X86TargetLowering()
1760 setOperationAction(ISD::FROUNDEVEN, VT, Legal); in X86TargetLowering()
1761 setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal); in X86TargetLowering()
1763 setOperationAction(ISD::FROUND, VT, Custom); in X86TargetLowering()
1766 for (auto VT : {MVT::v32i16, MVT::v16i32, MVT::v8i64}) { in X86TargetLowering()
1767 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom); in X86TargetLowering()
1768 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom); in X86TargetLowering()
1795 for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64 }) { in X86TargetLowering()
1796 setOperationAction(ISD::SRL, VT, Custom); in X86TargetLowering()
1797 setOperationAction(ISD::SHL, VT, Custom); in X86TargetLowering()
1798 setOperationAction(ISD::SRA, VT, Custom); in X86TargetLowering()
1799 setOperationAction(ISD::ROTL, VT, Custom); in X86TargetLowering()
1800 setOperationAction(ISD::ROTR, VT, Custom); in X86TargetLowering()
1801 setOperationAction(ISD::SETCC, VT, Custom); in X86TargetLowering()
1805 setCondCodeAction(ISD::SETLT, VT, Custom); in X86TargetLowering()
1806 setCondCodeAction(ISD::SETLE, VT, Custom); in X86TargetLowering()
1808 for (auto VT : { MVT::v16i32, MVT::v8i64 }) { in X86TargetLowering()
1809 setOperationAction(ISD::SMAX, VT, Legal); in X86TargetLowering()
1810 setOperationAction(ISD::UMAX, VT, Legal); in X86TargetLowering()
1811 setOperationAction(ISD::SMIN, VT, Legal); in X86TargetLowering()
1812 setOperationAction(ISD::UMIN, VT, Legal); in X86TargetLowering()
1813 setOperationAction(ISD::ABS, VT, Legal); in X86TargetLowering()
1814 setOperationAction(ISD::CTPOP, VT, Custom); in X86TargetLowering()
1815 setOperationAction(ISD::STRICT_FSETCC, VT, Custom); in X86TargetLowering()
1816 setOperationAction(ISD::STRICT_FSETCCS, VT, Custom); in X86TargetLowering()
1819 for (auto VT : { MVT::v64i8, MVT::v32i16 }) { in X86TargetLowering()
1820 setOperationAction(ISD::ABS, VT, HasBWI ? Legal : Custom); in X86TargetLowering()
1821 setOperationAction(ISD::CTPOP, VT, Subtarget.hasBITALG() ? Legal : Custom); in X86TargetLowering()
1822 setOperationAction(ISD::CTLZ, VT, Custom); in X86TargetLowering()
1823 setOperationAction(ISD::SMAX, VT, HasBWI ? Legal : Custom); in X86TargetLowering()
1824 setOperationAction(ISD::UMAX, VT, HasBWI ? Legal : Custom); in X86TargetLowering()
1825 setOperationAction(ISD::SMIN, VT, HasBWI ? Legal : Custom); in X86TargetLowering()
1826 setOperationAction(ISD::UMIN, VT, HasBWI ? Legal : Custom); in X86TargetLowering()
1827 setOperationAction(ISD::UADDSAT, VT, HasBWI ? Legal : Custom); in X86TargetLowering()
1828 setOperationAction(ISD::SADDSAT, VT, HasBWI ? Legal : Custom); in X86TargetLowering()
1829 setOperationAction(ISD::USUBSAT, VT, HasBWI ? Legal : Custom); in X86TargetLowering()
1830 setOperationAction(ISD::SSUBSAT, VT, HasBWI ? Legal : Custom); in X86TargetLowering()
1850 for (auto VT : { MVT::v16i32, MVT::v8i64} ) { in X86TargetLowering()
1851 setOperationAction(ISD::CTLZ, VT, Legal); in X86TargetLowering()
1856 for (auto VT : { MVT::v16i32, MVT::v8i64 }) in X86TargetLowering()
1857 setOperationAction(ISD::CTPOP, VT, Legal); in X86TargetLowering()
1863 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64, in X86TargetLowering()
1865 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal); in X86TargetLowering()
1867 for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64, in X86TargetLowering()
1869 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in X86TargetLowering()
1870 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal); in X86TargetLowering()
1871 setOperationAction(ISD::SELECT, VT, Custom); in X86TargetLowering()
1872 setOperationAction(ISD::VSELECT, VT, Custom); in X86TargetLowering()
1873 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in X86TargetLowering()
1874 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1875 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in X86TargetLowering()
1876 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom); in X86TargetLowering()
1877 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1889 for (auto VT : { MVT::v16i32, MVT::v8i64, MVT::v16f32, MVT::v8f64 }) { in X86TargetLowering()
1890 setOperationAction(ISD::MLOAD, VT, Legal); in X86TargetLowering()
1891 setOperationAction(ISD::MSTORE, VT, Legal); in X86TargetLowering()
1892 setOperationAction(ISD::MGATHER, VT, Custom); in X86TargetLowering()
1893 setOperationAction(ISD::MSCATTER, VT, Custom); in X86TargetLowering()
1896 for (auto VT : { MVT::v64i8, MVT::v32i16 }) { in X86TargetLowering()
1897 setOperationAction(ISD::MLOAD, VT, Legal); in X86TargetLowering()
1898 setOperationAction(ISD::MSTORE, VT, Legal); in X86TargetLowering()
1906 for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64, in X86TargetLowering()
1909 setOperationAction(ISD::FSHL, VT, Custom); in X86TargetLowering()
1910 setOperationAction(ISD::FSHR, VT, Custom); in X86TargetLowering()
1944 for (auto VT : { MVT::v2i64, MVT::v4i64 }) { in X86TargetLowering()
1945 setOperationAction(ISD::SMAX, VT, Legal); in X86TargetLowering()
1946 setOperationAction(ISD::UMAX, VT, Legal); in X86TargetLowering()
1947 setOperationAction(ISD::SMIN, VT, Legal); in X86TargetLowering()
1948 setOperationAction(ISD::UMIN, VT, Legal); in X86TargetLowering()
1949 setOperationAction(ISD::ABS, VT, Legal); in X86TargetLowering()
1952 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) { in X86TargetLowering()
1953 setOperationAction(ISD::ROTL, VT, Custom); in X86TargetLowering()
1954 setOperationAction(ISD::ROTR, VT, Custom); in X86TargetLowering()
1961 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64, in X86TargetLowering()
1963 setOperationAction(ISD::MSCATTER, VT, Custom); in X86TargetLowering()
1977 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) { in X86TargetLowering()
1978 setOperationAction(ISD::CTLZ, VT, Legal); in X86TargetLowering()
1983 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) in X86TargetLowering()
1984 setOperationAction(ISD::CTPOP, VT, Legal); in X86TargetLowering()
1995 for (auto VT : { MVT::v32i1, MVT::v64i1 }) { in X86TargetLowering()
1996 setOperationAction(ISD::VSELECT, VT, Expand); in X86TargetLowering()
1997 setOperationAction(ISD::TRUNCATE, VT, Custom); in X86TargetLowering()
1998 setOperationAction(ISD::SETCC, VT, Custom); in X86TargetLowering()
1999 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
2000 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
2001 setOperationAction(ISD::SELECT, VT, Custom); in X86TargetLowering()
2002 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in X86TargetLowering()
2003 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in X86TargetLowering()
2004 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in X86TargetLowering()
2005 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); in X86TargetLowering()
2008 for (auto VT : { MVT::v16i1, MVT::v32i1 }) in X86TargetLowering()
2009 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); in X86TargetLowering()
2016 for (auto VT : { MVT::v32i8, MVT::v16i8, MVT::v16i16, MVT::v8i16 }) { in X86TargetLowering()
2017 setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom); in X86TargetLowering()
2018 setOperationAction(ISD::MSTORE, VT, Subtarget.hasVLX() ? Legal : Custom); in X86TargetLowering()
2026 for (auto VT : { MVT::v16i8, MVT::v32i8, MVT::v8i16, MVT::v16i16 }) in X86TargetLowering()
2027 setOperationAction(ISD::CTPOP, VT, Legal); in X86TargetLowering()
2032 auto setGroup = [&] (MVT VT) { in X86TargetLowering() argument
2033 setOperationAction(ISD::FADD, VT, Legal); in X86TargetLowering()
2034 setOperationAction(ISD::STRICT_FADD, VT, Legal); in X86TargetLowering()
2035 setOperationAction(ISD::FSUB, VT, Legal); in X86TargetLowering()
2036 setOperationAction(ISD::STRICT_FSUB, VT, Legal); in X86TargetLowering()
2037 setOperationAction(ISD::FMUL, VT, Legal); in X86TargetLowering()
2038 setOperationAction(ISD::STRICT_FMUL, VT, Legal); in X86TargetLowering()
2039 setOperationAction(ISD::FDIV, VT, Legal); in X86TargetLowering()
2040 setOperationAction(ISD::STRICT_FDIV, VT, Legal); in X86TargetLowering()
2041 setOperationAction(ISD::FSQRT, VT, Legal); in X86TargetLowering()
2042 setOperationAction(ISD::STRICT_FSQRT, VT, Legal); in X86TargetLowering()
2044 setOperationAction(ISD::FFLOOR, VT, Legal); in X86TargetLowering()
2045 setOperationAction(ISD::STRICT_FFLOOR, VT, Legal); in X86TargetLowering()
2046 setOperationAction(ISD::FCEIL, VT, Legal); in X86TargetLowering()
2047 setOperationAction(ISD::STRICT_FCEIL, VT, Legal); in X86TargetLowering()
2048 setOperationAction(ISD::FTRUNC, VT, Legal); in X86TargetLowering()
2049 setOperationAction(ISD::STRICT_FTRUNC, VT, Legal); in X86TargetLowering()
2050 setOperationAction(ISD::FRINT, VT, Legal); in X86TargetLowering()
2051 setOperationAction(ISD::STRICT_FRINT, VT, Legal); in X86TargetLowering()
2052 setOperationAction(ISD::FNEARBYINT, VT, Legal); in X86TargetLowering()
2053 setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal); in X86TargetLowering()
2055 setOperationAction(ISD::LOAD, VT, Legal); in X86TargetLowering()
2056 setOperationAction(ISD::STORE, VT, Legal); in X86TargetLowering()
2058 setOperationAction(ISD::FMA, VT, Legal); in X86TargetLowering()
2059 setOperationAction(ISD::STRICT_FMA, VT, Legal); in X86TargetLowering()
2060 setOperationAction(ISD::VSELECT, VT, Legal); in X86TargetLowering()
2061 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in X86TargetLowering()
2062 setOperationAction(ISD::SELECT, VT, Custom); in X86TargetLowering()
2064 setOperationAction(ISD::FNEG, VT, Custom); in X86TargetLowering()
2065 setOperationAction(ISD::FABS, VT, Custom); in X86TargetLowering()
2066 setOperationAction(ISD::FCOPYSIGN, VT, Custom); in X86TargetLowering()
2067 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
2068 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in X86TargetLowering()
2252 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { in X86TargetLowering()
2253 if (VT == MVT::i64 && !Subtarget.is64Bit()) in X86TargetLowering()
2256 setOperationAction(ISD::SADDO, VT, Custom); in X86TargetLowering()
2257 setOperationAction(ISD::UADDO, VT, Custom); in X86TargetLowering()
2258 setOperationAction(ISD::SSUBO, VT, Custom); in X86TargetLowering()
2259 setOperationAction(ISD::USUBO, VT, Custom); in X86TargetLowering()
2260 setOperationAction(ISD::SMULO, VT, Custom); in X86TargetLowering()
2261 setOperationAction(ISD::UMULO, VT, Custom); in X86TargetLowering()
2264 setOperationAction(ISD::ADDCARRY, VT, Custom); in X86TargetLowering()
2265 setOperationAction(ISD::SUBCARRY, VT, Custom); in X86TargetLowering()
2266 setOperationAction(ISD::SETCCCARRY, VT, Custom); in X86TargetLowering()
2267 setOperationAction(ISD::SADDO_CARRY, VT, Custom); in X86TargetLowering()
2268 setOperationAction(ISD::SSUBO_CARRY, VT, Custom); in X86TargetLowering()
2425 X86TargetLowering::getPreferredVectorAction(MVT VT) const { in getPreferredVectorAction()
2426 if ((VT == MVT::v32i1 || VT == MVT::v64i1) && Subtarget.hasAVX512() && in getPreferredVectorAction()
2430 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 && in getPreferredVectorAction()
2431 !Subtarget.hasF16C() && VT.getVectorElementType() == MVT::f16) in getPreferredVectorAction()
2434 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 && in getPreferredVectorAction()
2435 VT.getVectorElementType() != MVT::i1) in getPreferredVectorAction()
2438 return TargetLoweringBase::getPreferredVectorAction(VT); in getPreferredVectorAction()
2477 EVT VT) const { in getRegisterTypeForCallingConv()
2478 if (VT.isVector()) { in getRegisterTypeForCallingConv()
2479 if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512()) { in getRegisterTypeForCallingConv()
2480 unsigned NumElts = VT.getVectorNumElements(); in getRegisterTypeForCallingConv()
2490 if (VT.getVectorElementType() == MVT::f16 && VT.getVectorNumElements() < 8) in getRegisterTypeForCallingConv()
2495 if ((VT == MVT::f64 || VT == MVT::f80) && !Subtarget.is64Bit() && in getRegisterTypeForCallingConv()
2499 if (VT.isVector() && VT.getVectorElementType() == MVT::bf16) in getRegisterTypeForCallingConv()
2501 VT.changeVectorElementTypeToInteger()); in getRegisterTypeForCallingConv()
2503 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT); in getRegisterTypeForCallingConv()
2508 EVT VT) const { in getNumRegistersForCallingConv()
2509 if (VT.isVector()) { in getNumRegistersForCallingConv()
2510 if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512()) { in getNumRegistersForCallingConv()
2511 unsigned NumElts = VT.getVectorNumElements(); in getNumRegistersForCallingConv()
2521 if (VT.getVectorElementType() == MVT::f16 && VT.getVectorNumElements() < 8) in getNumRegistersForCallingConv()
2528 if (VT == MVT::f64) in getNumRegistersForCallingConv()
2530 if (VT == MVT::f80) in getNumRegistersForCallingConv()
2534 if (VT.isVector() && VT.getVectorElementType() == MVT::bf16) in getNumRegistersForCallingConv()
2536 VT.changeVectorElementTypeToInteger()); in getNumRegistersForCallingConv()
2538 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT); in getNumRegistersForCallingConv()
2542 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, in getVectorTypeBreakdownForCallingConv() argument
2545 if (VT.isVector() && VT.getVectorElementType() == MVT::i1 && in getVectorTypeBreakdownForCallingConv()
2547 (!isPowerOf2_32(VT.getVectorNumElements()) || in getVectorTypeBreakdownForCallingConv()
2548 (VT.getVectorNumElements() == 64 && !Subtarget.hasBWI()) || in getVectorTypeBreakdownForCallingConv()
2549 VT.getVectorNumElements() > 64)) { in getVectorTypeBreakdownForCallingConv()
2552 NumIntermediates = VT.getVectorNumElements(); in getVectorTypeBreakdownForCallingConv()
2557 if (VT == MVT::v64i1 && Subtarget.hasBWI() && !Subtarget.useAVX512Regs() && in getVectorTypeBreakdownForCallingConv()
2565 return TargetLowering::getVectorTypeBreakdownForCallingConv(Context, CC, VT, IntermediateVT, in getVectorTypeBreakdownForCallingConv()
2571 EVT VT) const { in getSetCCResultType()
2572 if (!VT.isVector()) in getSetCCResultType()
2577 EVT LegalVT = VT; in getSetCCResultType()
2583 return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount()); in getSetCCResultType()
2591 return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount()); in getSetCCResultType()
2595 return VT.changeVectorElementTypeToInteger(); in getSetCCResultType()
2693 bool X86TargetLowering::isSafeMemOpType(MVT VT) const { in isSafeMemOpType()
2694 if (VT == MVT::f32) in isSafeMemOpType()
2696 if (VT == MVT::f64) in isSafeMemOpType()
2702 EVT VT, unsigned, Align Alignment, MachineMemOperand::Flags Flags, in allowsMisalignedMemoryAccesses() argument
2705 switch (VT.getSizeInBits()) { in allowsMisalignedMemoryAccesses()
2720 if (!!(Flags & MachineMemOperand::MONonTemporal) && VT.isVector()) { in allowsMisalignedMemoryAccesses()
2849 MVT VT) const { in findRepresentativeClass()
2852 switch (VT.SimpleTy) { in findRepresentativeClass()
2854 return TargetLowering::findRepresentativeClass(TRI, VT); in findRepresentativeClass()
3322 EVT X86TargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT, in getTypeForExtReturn() argument
3327 if (VT == MVT::i1 || (!Darwin && (VT == MVT::i8 || VT == MVT::i16))) { in getTypeForExtReturn()
3337 return VT.bitsLT(MinVT) ? MinVT : VT; in getTypeForExtReturn()
3997 SDValue RegVal = DAG.getCopyFromReg(Chain, DL, FR.VReg, FR.VT); in forwardMustTailParameters()
3999 TargLowering.getRegClassFor(FR.VT)); in forwardMustTailParameters()
4285 EVT VT = getPointerTy(DAG.getDataLayout()); in EmitTailCallLoadRetAddr() local
4289 OutRetAddr = DAG.getLoad(VT, dl, Chain, OutRetAddr, MachinePointerInfo()); in EmitTailCallLoadRetAddr()
4314 static SDValue getMOVL(SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1, in getMOVL() argument
4316 unsigned NumElems = VT.getVectorNumElements(); in getMOVL()
4321 return DAG.getVectorShuffle(VT, dl, V1, V2, Mask); in getMOVL()
4644 SDValue Val = DAG.getCopyFromReg(Chain, dl, F.VReg, F.VT); in LowerCall()
5561 static bool useVPTERNLOG(const X86Subtarget &Subtarget, MVT VT) { in useVPTERNLOG() argument
5563 VT.is512BitVector(); in useVPTERNLOG()
5630 MVT VT = MVT::getVT(I.getArgOperand(1)->getType()); in getTgtMemIntrinsic() local
5639 Info.memVT = MVT::getVectorVT(ScalarVT, VT.getVectorNumElements()); in getTgtMemIntrinsic()
5679 bool X86TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, in isFPImmLegal() argument
5702 EVT VT = Load->getValueType(0); in shouldReduceLoadWidth() local
5703 if ((VT.is256BitVector() || VT.is512BitVector()) && !Load->hasOneUse()) { in shouldReduceLoadWidth()
5742 bool X86TargetLowering::convertSelectOfConstantsToMath(EVT VT) const { in convertSelectOfConstantsToMath()
5745 if (VT.isVector() && Subtarget.hasAVX512()) in convertSelectOfConstantsToMath()
5751 bool X86TargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT, in decomposeMulByConstant() argument
5765 while (getTypeAction(Context, VT) != TypeLegal) in decomposeMulByConstant()
5766 VT = getTypeToTransformTo(Context, VT); in decomposeMulByConstant()
5773 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in decomposeMulByConstant()
5774 if (isOperationLegal(ISD::MUL, VT) && EltSizeInBits <= 32 && in decomposeMulByConstant()
5816 bool X86TargetLowering::shouldFormOverflowOp(unsigned Opcode, EVT VT, in shouldFormOverflowOp() argument
5819 if (VT.isVector()) in shouldFormOverflowOp()
5821 return VT.isSimple() || !isOperationExpand(Opcode, VT); in shouldFormOverflowOp()
5834 bool X86TargetLowering::hasBitPreservingFPLogic(EVT VT) const { in hasBitPreservingFPLogic()
5835 return VT == MVT::f32 || VT == MVT::f64 || VT.isVector(); in hasBitPreservingFPLogic()
5838 bool X86TargetLowering::ShouldShrinkFPConstant(EVT VT) const { in ShouldShrinkFPConstant()
5842 return !Subtarget.hasSSE2() || VT == MVT::f80; in ShouldShrinkFPConstant()
5845 bool X86TargetLowering::isScalarFPTypeInSSEReg(EVT VT) const { in isScalarFPTypeInSSEReg()
5846 return (VT == MVT::f64 && Subtarget.hasSSE2()) || in isScalarFPTypeInSSEReg()
5847 (VT == MVT::f32 && Subtarget.hasSSE1()) || VT == MVT::f16; in isScalarFPTypeInSSEReg()
5896 EVT VT = Y.getValueType(); in hasAndNotCompare() local
5898 if (VT.isVector()) in hasAndNotCompare()
5905 if (VT != MVT::i32 && VT != MVT::i64) in hasAndNotCompare()
5912 EVT VT = Y.getValueType(); in hasAndNot() local
5914 if (!VT.isVector()) in hasAndNot()
5919 if (!Subtarget.hasSSE1() || VT.getSizeInBits() < 128) in hasAndNot()
5922 if (VT == MVT::v4i32) in hasAndNot()
5963 EVT VT = N->getValueType(0); in shouldFoldConstantShiftPairToMask() local
5964 if ((Subtarget.hasFastVectorShiftMasks() && VT.isVector()) || in shouldFoldConstantShiftPairToMask()
5965 (Subtarget.hasFastScalarShiftMasks() && !VT.isVector())) { in shouldFoldConstantShiftPairToMask()
5975 EVT VT = Y.getValueType(); in shouldFoldMaskToVariableShiftPair() local
5978 if (VT.isVector()) in shouldFoldMaskToVariableShiftPair()
5982 if (VT == MVT::i64 && !Subtarget.is64Bit()) in shouldFoldMaskToVariableShiftPair()
5996 bool X86TargetLowering::shouldSplatInsEltVarIndex(EVT VT) const { in shouldSplatInsEltVarIndex()
5999 return isTypeLegal(VT); in shouldSplatInsEltVarIndex()
6003 MVT VT = MVT::getIntegerVT(NumBits); in hasFastEqualityCompare() local
6004 if (isTypeLegal(VT)) in hasFastEqualityCompare()
6005 return VT; in hasFastEqualityCompare()
6257 static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG, in getConstVector() argument
6263 MVT ConstVecVT = VT; in getConstVector()
6264 unsigned NumElts = VT.getVectorNumElements(); in getConstVector()
6266 if (!In64BitMode && VT.getVectorElementType() == MVT::i64) { in getConstVector()
6283 ConstsNode = DAG.getBitcast(VT, ConstsNode); in getConstVector()
6288 MVT VT, SelectionDAG &DAG, const SDLoc &dl) { in getConstVector() argument
6294 MVT ConstVecVT = VT; in getConstVector()
6295 unsigned NumElts = VT.getVectorNumElements(); in getConstVector()
6297 if (!In64BitMode && VT.getVectorElementType() == MVT::i64) { in getConstVector()
6309 assert(V.getBitWidth() == VT.getScalarSizeInBits() && "Unexpected sizes"); in getConstVector()
6325 return DAG.getBitcast(VT, ConstsNode); in getConstVector()
6329 static SDValue getZeroVector(MVT VT, const X86Subtarget &Subtarget, in getZeroVector() argument
6331 assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector() || in getZeroVector()
6332 VT.getVectorElementType() == MVT::i1) && in getZeroVector()
6339 if (!Subtarget.hasSSE2() && VT.is128BitVector()) { in getZeroVector()
6341 } else if (VT.isFloatingPoint()) { in getZeroVector()
6342 Vec = DAG.getConstantFP(+0.0, dl, VT); in getZeroVector()
6343 } else if (VT.getVectorElementType() == MVT::i1) { in getZeroVector()
6344 assert((Subtarget.hasBWI() || VT.getVectorNumElements() <= 16) && in getZeroVector()
6346 Vec = DAG.getConstant(0, dl, VT); in getZeroVector()
6348 unsigned Num32BitElts = VT.getSizeInBits() / 32; in getZeroVector()
6351 return DAG.getBitcast(VT, Vec); in getZeroVector()
6379 EVT VT = Vec.getValueType(); in extractSubVector() local
6380 EVT ElVT = VT.getVectorElementType(); in extractSubVector()
6381 unsigned Factor = VT.getSizeInBits() / vectorWidth; in extractSubVector()
6383 VT.getVectorNumElements() / Factor); in extractSubVector()
6430 EVT VT = Vec.getValueType(); in insertSubVector() local
6431 EVT ElVT = VT.getVectorElementType(); in insertSubVector()
6460 static SDValue widenSubVector(MVT VT, SDValue Vec, bool ZeroNewElements, in widenSubVector() argument
6463 assert(Vec.getValueSizeInBits().getFixedSize() < VT.getFixedSizeInBits() && in widenSubVector()
6464 Vec.getValueType().getScalarType() == VT.getScalarType() && in widenSubVector()
6466 SDValue Res = ZeroNewElements ? getZeroVector(VT, Subtarget, DAG, dl) in widenSubVector()
6467 : DAG.getUNDEF(VT); in widenSubVector()
6468 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, Res, Vec, in widenSubVector()
6482 MVT VT = MVT::getVectorVT(SVT, WideNumElts); in widenSubVector() local
6483 return widenSubVector(VT, Vec, ZeroNewElements, Subtarget, DAG, dl); in widenSubVector()
6502 EVT VT = Src.getValueType(); in collectConcatOps() local
6506 if (VT.getSizeInBits() == (SubVT.getSizeInBits() * 2)) { in collectConcatOps()
6513 if (Idx == (VT.getVectorNumElements() / 2)) { in collectConcatOps()
6543 EVT VT = Op.getValueType(); in splitVector() local
6544 unsigned NumElems = VT.getVectorNumElements(); in splitVector()
6545 unsigned SizeInBits = VT.getSizeInBits(); in splitVector()
6562 EVT VT = Op.getValueType(); in splitVectorOp() local
6578 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in splitVectorOp()
6579 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, in splitVectorOp()
6589 EVT VT = Op.getValueType(); in splitVectorIntUnary() local
6590 (void)VT; in splitVectorIntUnary()
6593 (VT.is256BitVector() || VT.is512BitVector()) && "Unsupported VT!"); in splitVectorIntUnary()
6595 VT.getVectorNumElements() && in splitVectorIntUnary()
6604 EVT VT = Op.getValueType(); in splitVectorIntBinary() local
6605 (void)VT; in splitVectorIntBinary()
6606 assert(Op.getOperand(0).getValueType() == VT && in splitVectorIntBinary()
6607 Op.getOperand(1).getValueType() == VT && "Unexpected VTs!"); in splitVectorIntBinary()
6608 assert((VT.is256BitVector() || VT.is512BitVector()) && "Unsupported VT!"); in splitVectorIntBinary()
6621 const SDLoc &DL, EVT VT, ArrayRef<SDValue> Ops, in SplitOpsAndApply() argument
6627 if (VT.getSizeInBits() > 512) { in SplitOpsAndApply()
6628 NumSubs = VT.getSizeInBits() / 512; in SplitOpsAndApply()
6629 assert((VT.getSizeInBits() % 512) == 0 && "Illegal vector size"); in SplitOpsAndApply()
6632 if (VT.getSizeInBits() > 256) { in SplitOpsAndApply()
6633 NumSubs = VT.getSizeInBits() / 256; in SplitOpsAndApply()
6634 assert((VT.getSizeInBits() % 256) == 0 && "Illegal vector size"); in SplitOpsAndApply()
6637 if (VT.getSizeInBits() > 128) { in SplitOpsAndApply()
6638 NumSubs = VT.getSizeInBits() / 128; in SplitOpsAndApply()
6639 assert((VT.getSizeInBits() % 128) == 0 && "Illegal vector size"); in SplitOpsAndApply()
6657 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs); in SplitOpsAndApply()
6662 static SDValue getAVX512Node(unsigned Opcode, const SDLoc &DL, MVT VT, in getAVX512Node() argument
6666 MVT SVT = VT.getScalarType(); in getAVX512Node()
6692 bool Widen = !(Subtarget.hasVLX() || VT.is512BitVector()); in getAVX512Node()
6694 MVT DstVT = VT; in getAVX512Node()
6705 assert(OpVT == VT && "Vector type mismatch"); in getAVX512Node()
6721 Res = extractSubVector(Res, 0, DAG, DL, VT.getSizeInBits()); in getAVX512Node()
6900 EVT VT = EVT::getVectorVT(*DAG.getContext(), SubSVT, 2 * SubNumElts); in concatSubVectors() local
6901 SDValue V = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, dl, SubVectorWidth); in concatSubVectors()
6908 static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) { in getOnesVector() argument
6909 assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) && in getOnesVector()
6913 unsigned NumElts = VT.getSizeInBits() / 32; in getOnesVector()
6915 return DAG.getBitcast(VT, Vec); in getOnesVector()
6950 static SDValue getEXTEND_VECTOR_INREG(unsigned Opcode, const SDLoc &DL, EVT VT, in getEXTEND_VECTOR_INREG() argument
6953 assert(VT.isVector() && InVT.isVector() && "Expected vector VTs."); in getEXTEND_VECTOR_INREG()
6961 assert(VT.getSizeInBits() == InVT.getSizeInBits() && in getEXTEND_VECTOR_INREG()
6963 unsigned Scale = VT.getScalarSizeInBits() / InVT.getScalarSizeInBits(); in getEXTEND_VECTOR_INREG()
6965 std::max(128U, (unsigned)VT.getSizeInBits() / Scale)); in getEXTEND_VECTOR_INREG()
6969 if (VT.getVectorNumElements() != InVT.getVectorNumElements()) in getEXTEND_VECTOR_INREG()
6972 return DAG.getNode(Opcode, DL, VT, In); in getEXTEND_VECTOR_INREG()
7003 void llvm::createUnpackShuffleMask(EVT VT, SmallVectorImpl<int> &Mask, in createUnpackShuffleMask() argument
7005 assert(VT.getScalarType().isSimple() && (VT.getSizeInBits() % 128) == 0 && in createUnpackShuffleMask()
7008 int NumElts = VT.getVectorNumElements(); in createUnpackShuffleMask()
7009 int NumEltsInLane = 128 / VT.getScalarSizeInBits(); in createUnpackShuffleMask()
7023 void llvm::createSplat2ShuffleMask(MVT VT, SmallVectorImpl<int> &Mask, in createSplat2ShuffleMask() argument
7026 int NumElts = VT.getVectorNumElements(); in createSplat2ShuffleMask()
7035 static SDValue getVectorShuffle(SelectionDAG &DAG, EVT VT, const SDLoc &dl, in getVectorShuffle() argument
7039 SmallVector<SDValue> Ops(Mask.size(), DAG.getUNDEF(VT.getScalarType())); in getVectorShuffle()
7049 return DAG.getBuildVector(VT, dl, Ops); in getVectorShuffle()
7052 return DAG.getVectorShuffle(VT, dl, V1, V2, Mask); in getVectorShuffle()
7056 static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, EVT VT, in getUnpackl() argument
7059 createUnpackShuffleMask(VT, Mask, /* Lo = */ true, /* Unary = */ false); in getUnpackl()
7060 return getVectorShuffle(DAG, VT, dl, V1, V2, Mask); in getUnpackl()
7064 static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, EVT VT, in getUnpackh() argument
7067 createUnpackShuffleMask(VT, Mask, /* Lo = */ false, /* Unary = */ false); in getUnpackh()
7068 return getVectorShuffle(DAG, VT, dl, V1, V2, Mask); in getUnpackh()
7075 const SDLoc &dl, MVT VT, SDValue LHS, SDValue RHS, in getPack() argument
7078 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in getPack()
7081 VT.getSizeInBits() == OpVT.getSizeInBits() && in getPack()
7091 int NumElts = VT.getVectorNumElements(); in getPack()
7098 return DAG.getVectorShuffle(VT, dl, DAG.getBitcast(VT, LHS), in getPack()
7099 DAG.getBitcast(VT, RHS), PackMask); in getPack()
7107 return DAG.getNode(X86ISD::PACKUS, dl, VT, LHS, RHS); in getPack()
7111 return DAG.getNode(X86ISD::PACKSS, dl, VT, LHS, RHS); in getPack()
7125 return DAG.getNode(X86ISD::PACKUS, dl, VT, LHS, RHS); in getPack()
7134 return DAG.getNode(X86ISD::PACKSS, dl, VT, LHS, RHS); in getPack()
7145 MVT VT = V2.getSimpleValueType(); in getShuffleVectorZeroOrUndef() local
7147 ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT); in getShuffleVectorZeroOrUndef()
7148 int NumElems = VT.getVectorNumElements(); in getShuffleVectorZeroOrUndef()
7153 return DAG.getVectorShuffle(VT, SDLoc(V2), V1, V2, MaskVec); in getShuffleVectorZeroOrUndef()
7195 EVT VT = Op.getValueType(); in getTargetConstantBitsFromNode() local
7196 unsigned SizeInBits = VT.getSizeInBits(); in getTargetConstantBitsFromNode()
7300 unsigned SrcEltSizeInBits = VT.getScalarSizeInBits(); in getTargetConstantBitsFromNode()
7332 EltSizeInBits <= VT.getScalarSizeInBits()) { in getTargetConstantBitsFromNode()
7334 if (MemIntr->getMemoryVT().getScalarSizeInBits() != VT.getScalarSizeInBits()) in getTargetConstantBitsFromNode()
7389 unsigned SrcEltSizeInBits = VT.getScalarSizeInBits(); in getTargetConstantBitsFromNode()
7404 unsigned SrcEltSizeInBits = VT.getScalarSizeInBits(); in getTargetConstantBitsFromNode()
7428 if (EltSizeInBits != VT.getScalarSizeInBits()) in getTargetConstantBitsFromNode()
7436 unsigned NumSubElts = VT.getVectorNumElements(); in getTargetConstantBitsFromNode()
7450 if (EltSizeInBits != VT.getScalarSizeInBits()) in getTargetConstantBitsFromNode()
7543 static void createPackShuffleMask(MVT VT, SmallVectorImpl<int> &Mask, in createPackShuffleMask() argument
7546 unsigned NumElts = VT.getVectorNumElements(); in createPackShuffleMask()
7547 unsigned NumLanes = VT.getSizeInBits() / 128; in createPackShuffleMask()
7548 unsigned NumEltsPerLane = 128 / VT.getScalarSizeInBits(); in createPackShuffleMask()
7565 static void getPackDemandedElts(EVT VT, const APInt &DemandedElts, in getPackDemandedElts() argument
7567 int NumLanes = VT.getSizeInBits() / 128; in getPackDemandedElts()
7590 static void getHorizDemandedElts(EVT VT, const APInt &DemandedElts, in getHorizDemandedElts() argument
7592 int NumLanes = VT.getSizeInBits() / 128; in getHorizDemandedElts()
7624 static bool getTargetShuffleMask(SDNode *N, MVT VT, bool AllowSentinelZero, in getTargetShuffleMask() argument
7627 unsigned NumElems = VT.getVectorNumElements(); in getTargetShuffleMask()
7628 unsigned MaskEltSize = VT.getScalarSizeInBits(); in getTargetShuffleMask()
7640 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7641 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7647 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7648 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7654 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7655 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7661 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7671 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7672 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7682 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7683 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7688 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7689 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7694 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7695 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7700 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7701 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7706 assert((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) && in getTargetShuffleMask()
7708 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7709 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7717 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected"); in getTargetShuffleMask()
7718 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7719 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7727 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected"); in getTargetShuffleMask()
7728 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7734 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected"); in getTargetShuffleMask()
7735 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7742 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7748 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7754 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7760 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7768 if (N->getOperand(0).getValueType() == VT) { in getTargetShuffleMask()
7775 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7786 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected"); in getTargetShuffleMask()
7787 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7788 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7798 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7806 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7807 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7811 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7812 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7818 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7819 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7825 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7830 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7835 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7840 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7841 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7857 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7858 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7868 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7881 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7882 assert(N->getOperand(2).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
7926 static bool getTargetShuffleMask(SDNode *N, MVT VT, bool AllowSentinelZero, in getTargetShuffleMask() argument
7930 return getTargetShuffleMask(N, VT, AllowSentinelZero, Ops, Mask, IsUnary); in getTargetShuffleMask()
8031 MVT VT = N.getSimpleValueType(); in getTargetShuffleAndZeroables() local
8032 if (!getTargetShuffleMask(N.getNode(), VT, true, Ops, Mask, IsUnary)) in getTargetShuffleAndZeroables()
8043 assert((VT.getSizeInBits() % Size) == 0 && in getTargetShuffleAndZeroables()
8045 unsigned EltSizeInBits = VT.getSizeInBits() / Size; in getTargetShuffleAndZeroables()
8088 if (Idx != 0 && !VT.isFloatingPoint()) in getTargetShuffleAndZeroables()
8118 assert(VT.getVectorNumElements() == (unsigned)Size && in getTargetShuffleAndZeroables()
8202 MVT VT = N.getSimpleValueType(); in getFauxShuffleMask() local
8203 unsigned NumElts = VT.getVectorNumElements(); in getFauxShuffleMask()
8204 unsigned NumSizeInBits = VT.getSizeInBits(); in getFauxShuffleMask()
8205 unsigned NumBitsPerElt = VT.getScalarSizeInBits(); in getFauxShuffleMask()
8296 Sub.getOperand(0).getValueType() == VT) { in getFauxShuffleMask()
8440 getPackDemandedElts(VT, DemandedElts, EltsLHS, EltsRHS); in getFauxShuffleMask()
8479 createPackShuffleMask(VT, Mask, IsUnary); in getFauxShuffleMask()
8569 VT.getScalarType()) in getFauxShuffleMask()
8655 EVT VT = Op.getValueType(); in getTargetShuffleInputs() local
8656 if (!VT.isSimple() || !VT.isVector()) in getTargetShuffleInputs()
8676 EVT VT = Op.getValueType(); in getTargetShuffleInputs() local
8677 if (!VT.isSimple() || !VT.isVector()) in getTargetShuffleInputs()
8688 static SDValue getBROADCAST_LOAD(unsigned Opcode, const SDLoc &DL, EVT VT, in getBROADCAST_LOAD() argument
8701 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in getBROADCAST_LOAD()
8718 EVT VT = Op.getValueType(); in getShuffleScalarElt() local
8720 unsigned NumElems = VT.getVectorNumElements(); in getShuffleScalarElt()
8727 return DAG.getUNDEF(VT.getVectorElementType()); in getShuffleScalarElt()
8735 MVT ShufVT = VT.getSimpleVT(); in getShuffleScalarElt()
8806 : DAG.getUNDEF(VT.getVectorElementType()); in getShuffleScalarElt()
8819 MVT VT = Op.getSimpleValueType(); in LowerBuildVectorAsInsert() local
8820 unsigned NumElts = VT.getVectorNumElements(); in LowerBuildVectorAsInsert()
8821 assert(((VT == MVT::v8i16 && Subtarget.hasSSE2()) || in LowerBuildVectorAsInsert()
8822 ((VT == MVT::v16i8 || VT == MVT::v4i32) && Subtarget.hasSSE41())) && in LowerBuildVectorAsInsert()
8840 V = getZeroVector(VT, Subtarget, DAG, dl); in LowerBuildVectorAsInsert()
8845 V = DAG.getBitcast(VT, V); in LowerBuildVectorAsInsert()
8849 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, V, Op.getOperand(i), in LowerBuildVectorAsInsert()
8947 MVT VT = Op.getSimpleValueType(); in LowerBuildVectorv4x32() local
8948 MVT EltVT = VT.getVectorElementType(); in LowerBuildVectorv4x32()
8953 SDValue NewBV = DAG.getBitcast(MVT::v2f64, DAG.getBuildVector(VT, DL, Ops)); in LowerBuildVectorv4x32()
8955 return DAG.getBitcast(VT, Dup); in LowerBuildVectorv4x32()
8980 MVT VT = Elt.getOperand(0).getSimpleValueType(); in LowerBuildVectorv4x32() local
8981 if (!VT.is128BitVector()) in LowerBuildVectorv4x32()
8991 MVT VT = V1.getSimpleValueType(); in LowerBuildVectorv4x32() local
9015 ? DAG.getUNDEF(VT) in LowerBuildVectorv4x32()
9016 : getZeroVector(VT, Subtarget, DAG, SDLoc(Op)); in LowerBuildVectorv4x32()
9017 if (V1.getSimpleValueType() != VT) in LowerBuildVectorv4x32()
9018 V1 = DAG.getBitcast(VT, V1); in LowerBuildVectorv4x32()
9019 return DAG.getVectorShuffle(VT, SDLoc(V1), V1, VZeroOrUndef, Mask); in LowerBuildVectorv4x32()
9059 return DAG.getBitcast(VT, Result); in LowerBuildVectorv4x32()
9063 static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits, in getVShift() argument
9066 assert(VT.is128BitVector() && "Unknown type for VShift"); in getVShift()
9072 return DAG.getBitcast(VT, DAG.getNode(Opc, dl, ShVT, SrcOp, ShiftVal)); in getVShift()
9075 static SDValue LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, const SDLoc &dl, in LowerAsSplatVectorLoad() argument
9105 Align RequiredAlign(VT.getSizeInBits() / 8); in LowerAsSplatVectorLoad()
9135 unsigned NumElems = VT.getVectorNumElements(); in LowerAsSplatVectorLoad()
9197 static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts, in EltsFromConsecutiveLoads() argument
9201 if ((VT.getScalarSizeInBits() % 8) != 0) in EltsFromConsecutiveLoads()
9232 if ((NumElems * EltSizeInBits) != VT.getSizeInBits()) in EltsFromConsecutiveLoads()
9250 return DAG.getUNDEF(VT); in EltsFromConsecutiveLoads()
9252 return VT.isInteger() ? DAG.getConstant(0, DL, VT) in EltsFromConsecutiveLoads()
9253 : DAG.getConstantFP(0.0, DL, VT); in EltsFromConsecutiveLoads()
9304 auto CreateLoad = [&DAG, &DL, &Loads](EVT VT, LoadSDNode *LDBase) { in EltsFromConsecutiveLoads() argument
9309 DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(), in EltsFromConsecutiveLoads()
9320 VT.getSizeInBits() / 8, *DAG.getContext(), DAG.getDataLayout()); in EltsFromConsecutiveLoads()
9330 if (IsAfterLegalize && !TLI.isOperationLegal(ISD::LOAD, VT)) in EltsFromConsecutiveLoads()
9336 VT.is256BitVector() && !Subtarget.hasInt256()) in EltsFromConsecutiveLoads()
9340 return DAG.getBitcast(VT, Elts[FirstLoadedElt]); in EltsFromConsecutiveLoads()
9343 return CreateLoad(VT, LDBase); in EltsFromConsecutiveLoads()
9347 if (!IsAfterLegalize && VT.isVector()) { in EltsFromConsecutiveLoads()
9348 unsigned NumMaskElts = VT.getVectorNumElements(); in EltsFromConsecutiveLoads()
9359 SDValue V = CreateLoad(VT, LDBase); in EltsFromConsecutiveLoads()
9360 SDValue Z = VT.isInteger() ? DAG.getConstant(0, DL, VT) in EltsFromConsecutiveLoads()
9361 : DAG.getConstantFP(0.0, DL, VT); in EltsFromConsecutiveLoads()
9362 return DAG.getVectorShuffle(VT, DL, V, Z, ClearMask); in EltsFromConsecutiveLoads()
9368 if (VT.is256BitVector() || VT.is512BitVector()) { in EltsFromConsecutiveLoads()
9372 EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), HalfNumElems); in EltsFromConsecutiveLoads()
9377 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), in EltsFromConsecutiveLoads()
9386 ((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()))) { in EltsFromConsecutiveLoads()
9387 MVT VecSVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(LoadSizeInBits) in EltsFromConsecutiveLoads()
9389 MVT VecVT = MVT::getVectorVT(VecSVT, VT.getSizeInBits() / LoadSizeInBits); in EltsFromConsecutiveLoads()
9392 if (!Subtarget.hasSSE2() && VT == MVT::v4f32) in EltsFromConsecutiveLoads()
9403 return DAG.getBitcast(VT, ResNode); in EltsFromConsecutiveLoads()
9410 (VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector())) { in EltsFromConsecutiveLoads()
9441 VT.isInteger() && (RepeatSize != 64 || TLI.isTypeLegal(MVT::i64)) in EltsFromConsecutiveLoads()
9449 VT.getSizeInBits() / ScalarSize); in EltsFromConsecutiveLoads()
9455 while (Broadcast.getValueSizeInBits() < VT.getSizeInBits()) in EltsFromConsecutiveLoads()
9467 return DAG.getBitcast(VT, Broadcast); in EltsFromConsecutiveLoads()
9479 static SDValue combineToConsecutiveLoads(EVT VT, SDValue Op, const SDLoc &DL, in combineToConsecutiveLoads() argument
9484 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) { in combineToConsecutiveLoads()
9491 assert(Elts.size() == VT.getVectorNumElements()); in combineToConsecutiveLoads()
9492 return EltsFromConsecutiveLoads(VT, Elts, DL, DAG, Subtarget, in combineToConsecutiveLoads()
9496 static Constant *getConstantVector(MVT VT, const APInt &SplatValue, in getConstantVector() argument
9498 unsigned ScalarSize = VT.getScalarSizeInBits(); in getConstantVector()
9505 if (VT.isFloatingPoint()) { in getConstantVector()
9560 MVT VT = BVOp->getSimpleValueType(0); in lowerBuildVectorAsBroadcast() local
9561 unsigned NumElts = VT.getVectorNumElements(); in lowerBuildVectorAsBroadcast()
9564 assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) && in lowerBuildVectorAsBroadcast()
9599 MVT EltType = MVT::getIntegerVT(VT.getScalarSizeInBits() * SeqLen); in lowerBuildVectorAsBroadcast()
9603 if (!VT.is512BitVector() && !Subtarget.hasVLX()) { in lowerBuildVectorAsBroadcast()
9604 unsigned Scale = 512 / VT.getSizeInBits(); in lowerBuildVectorAsBroadcast()
9608 if (BcstVT.getSizeInBits() != VT.getSizeInBits()) in lowerBuildVectorAsBroadcast()
9609 Bcst = extractSubVector(Bcst, 0, DAG, dl, VT.getSizeInBits()); in lowerBuildVectorAsBroadcast()
9610 return DAG.getBitcast(VT, Bcst); in lowerBuildVectorAsBroadcast()
9622 SplatBitSize > VT.getScalarSizeInBits() && in lowerBuildVectorAsBroadcast()
9623 SplatBitSize < VT.getSizeInBits()) { in lowerBuildVectorAsBroadcast()
9641 unsigned Repeat = VT.getSizeInBits() / SplatBitSize; in lowerBuildVectorAsBroadcast()
9652 return DAG.getBitcast(VT, Brdcst); in lowerBuildVectorAsBroadcast()
9656 Constant *VecC = getConstantVector(VT, SplatValue, SplatBitSize, in lowerBuildVectorAsBroadcast()
9659 unsigned NumElm = SplatBitSize / VT.getScalarSizeInBits(); in lowerBuildVectorAsBroadcast()
9660 MVT VVT = MVT::getVectorVT(VT.getScalarType(), NumElm); in lowerBuildVectorAsBroadcast()
9662 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in lowerBuildVectorAsBroadcast()
9698 bool IsGE256 = (VT.getSizeInBits() >= 256); in lowerBuildVectorAsBroadcast()
9738 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in lowerBuildVectorAsBroadcast()
9750 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld); in lowerBuildVectorAsBroadcast()
9763 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in lowerBuildVectorAsBroadcast()
9777 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in lowerBuildVectorAsBroadcast()
9787 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld); in lowerBuildVectorAsBroadcast()
9829 MVT VT = Op.getSimpleValueType(); in buildFromShuffleMostly() local
9833 if (!TLI.isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT)) in buildFromShuffleMostly()
9868 if (ExtractedFromVec.getValueType() != VT) in buildFromShuffleMostly()
9890 VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT); in buildFromShuffleMostly()
9891 SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, Mask); in buildFromShuffleMostly()
9894 NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx), in buildFromShuffleMostly()
9904 MVT VT = Op.getSimpleValueType(); in LowerBUILD_VECTORvXi1() local
9905 assert((VT.getVectorElementType() == MVT::i1) && in LowerBUILD_VECTORvXi1()
9947 if (VT == MVT::v64i1 && !Subtarget.is64Bit()) { in LowerBUILD_VECTORvXi1()
9954 MVT ImmVT = MVT::getIntegerVT(std::max((unsigned)VT.getSizeInBits(), 8U)); in LowerBUILD_VECTORvXi1()
9958 MVT VecVT = VT.getSizeInBits() >= 8 ? VT : MVT::v8i1; in LowerBUILD_VECTORvXi1()
9960 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Select, in LowerBUILD_VECTORvXi1()
9968 if (VT == MVT::v64i1 && !Subtarget.is64Bit()) { in LowerBUILD_VECTORvXi1()
9975 MVT ImmVT = MVT::getIntegerVT(std::max((unsigned)VT.getSizeInBits(), 8U)); in LowerBUILD_VECTORvXi1()
9977 MVT VecVT = VT.getSizeInBits() >= 8 ? VT : MVT::v8i1; in LowerBUILD_VECTORvXi1()
9979 DstVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, DstVec, in LowerBUILD_VECTORvXi1()
9983 DstVec = DAG.getUNDEF(VT); in LowerBUILD_VECTORvXi1()
9987 DstVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DstVec, in LowerBUILD_VECTORvXi1()
10031 EVT VT = N->getValueType(0); in isHorizontalBinOpPart() local
10032 assert(VT.is256BitVector() && "Only use for matching partial 256-bit h-ops"); in isHorizontalBinOpPart()
10034 assert(VT.isVector() && VT.getVectorNumElements() >= LastIdx && in isHorizontalBinOpPart()
10041 V0 = DAG.getUNDEF(VT); in isHorizontalBinOpPart()
10042 V1 = DAG.getUNDEF(VT); in isHorizontalBinOpPart()
10081 if (V0.getValueType() != VT) in isHorizontalBinOpPart()
10087 if (V1.getValueType() != VT) in isHorizontalBinOpPart()
10145 MVT VT = V0.getSimpleValueType(); in ExpandHorizontalBinOp() local
10146 assert(VT.is256BitVector() && VT == V1.getSimpleValueType() && in ExpandHorizontalBinOp()
10149 unsigned NumElts = VT.getVectorNumElements(); in ExpandHorizontalBinOp()
10174 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI); in ExpandHorizontalBinOp()
10188 MVT VT = BV->getSimpleValueType(0); in isAddSubOrSubAdd() local
10189 if (!Subtarget.hasSSE3() || !VT.isFloatingPoint()) in isAddSubOrSubAdd()
10192 unsigned NumElts = VT.getVectorNumElements(); in isAddSubOrSubAdd()
10193 SDValue InVec0 = DAG.getUNDEF(VT); in isAddSubOrSubAdd()
10194 SDValue InVec1 = DAG.getUNDEF(VT); in isAddSubOrSubAdd()
10240 if (InVec0.getSimpleValueType() != VT) in isAddSubOrSubAdd()
10245 if (InVec1.getSimpleValueType() != VT) in isAddSubOrSubAdd()
10347 MVT VT = BV->getSimpleValueType(0); in lowerToAddSubOrFMAddSub() local
10354 return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2); in lowerToAddSubOrFMAddSub()
10363 if (VT.is512BitVector()) { in lowerToAddSubOrFMAddSub()
10365 for (int I = 0, E = VT.getVectorNumElements(); I != E; I += 2) { in lowerToAddSubOrFMAddSub()
10369 SDValue Sub = DAG.getNode(ISD::FSUB, DL, VT, Opnd0, Opnd1); in lowerToAddSubOrFMAddSub()
10370 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, Opnd0, Opnd1); in lowerToAddSubOrFMAddSub()
10371 return DAG.getVectorShuffle(VT, DL, Sub, Add, Mask); in lowerToAddSubOrFMAddSub()
10374 return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1); in lowerToAddSubOrFMAddSub()
10380 MVT VT = BV->getSimpleValueType(0); in isHopBuildVector() local
10382 V0 = DAG.getUNDEF(VT); in isHopBuildVector()
10383 V1 = DAG.getUNDEF(VT); in isHopBuildVector()
10388 unsigned NumElts = VT.getVectorNumElements(); in isHopBuildVector()
10390 unsigned Num128BitChunks = VT.is256BitVector() ? 2 : 1; in isHopBuildVector()
10470 MVT VT = BV->getSimpleValueType(0); in getHopForBuildVector() local
10471 unsigned Width = VT.getSizeInBits(); in getHopForBuildVector()
10475 V0 = insertSubVector(DAG.getUNDEF(VT), V0, 0, DAG, SDLoc(BV), Width); in getHopForBuildVector()
10480 V1 = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, SDLoc(BV), Width); in getHopForBuildVector()
10482 unsigned NumElts = VT.getVectorNumElements(); in getHopForBuildVector()
10490 if (VT.is256BitVector() && DemandedElts.lshr(HalfNumElts) == 0) { in getHopForBuildVector()
10491 MVT HalfVT = VT.getHalfNumVectorElementsVT(); in getHopForBuildVector()
10495 return insertSubVector(DAG.getUNDEF(VT), Half, 0, DAG, SDLoc(BV), 256); in getHopForBuildVector()
10498 return DAG.getNode(HOpcode, SDLoc(BV), VT, V0, V1); in getHopForBuildVector()
10514 MVT VT = BV->getSimpleValueType(0); in LowerToHorizontalOp() local
10515 if (((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget.hasSSE3()) || in LowerToHorizontalOp()
10516 ((VT == MVT::v8i16 || VT == MVT::v4i32) && Subtarget.hasSSSE3()) || in LowerToHorizontalOp()
10517 ((VT == MVT::v8f32 || VT == MVT::v4f64) && Subtarget.hasAVX()) || in LowerToHorizontalOp()
10518 ((VT == MVT::v16i16 || VT == MVT::v8i32) && Subtarget.hasAVX2())) { in LowerToHorizontalOp()
10526 if (!Subtarget.hasAVX() || !VT.is256BitVector()) in LowerToHorizontalOp()
10530 unsigned NumElts = VT.getVectorNumElements(); in LowerToHorizontalOp()
10544 if (VT == MVT::v8i32 || VT == MVT::v16i16) { in LowerToHorizontalOp()
10584 if (VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 || in LowerToHorizontalOp()
10585 VT == MVT::v16i16) { in LowerToHorizontalOp()
10630 MVT VT = Op->getSimpleValueType(0); in lowerBuildVectorToBitOp() local
10631 unsigned NumElems = VT.getVectorNumElements(); in lowerBuildVectorToBitOp()
10658 if (!TLI.isOperationLegalOrPromote(Opcode, VT)) in lowerBuildVectorToBitOp()
10673 if (RHS.getValueSizeInBits() != VT.getScalarSizeInBits()) { in lowerBuildVectorToBitOp()
10676 RHS = DAG.getZExtOrTrunc(RHS, DL, VT.getScalarType()); in lowerBuildVectorToBitOp()
10689 SDValue LHS = DAG.getBuildVector(VT, DL, LHSElts); in lowerBuildVectorToBitOp()
10690 SDValue RHS = DAG.getBuildVector(VT, DL, RHSElts); in lowerBuildVectorToBitOp()
10691 SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS); in lowerBuildVectorToBitOp()
10707 MVT VT = Op.getSimpleValueType(); in materializeVectorConstant() local
10717 if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) in materializeVectorConstant()
10720 return getOnesVector(VT, DAG, DL); in materializeVectorConstant()
10729 static SDValue createVariablePermute(MVT VT, SDValue SrcVec, SDValue IndicesVec, in createVariablePermute() argument
10732 MVT ShuffleVT = VT; in createVariablePermute()
10733 EVT IndicesVT = EVT(VT).changeVectorElementTypeToInteger(); in createVariablePermute()
10734 unsigned NumElts = VT.getVectorNumElements(); in createVariablePermute()
10735 unsigned SizeInBits = VT.getSizeInBits(); in createVariablePermute()
10744 NumElts * VT.getScalarSizeInBits()); in createVariablePermute()
10760 VT = MVT::getVectorVT(VT.getScalarType(), Scale * NumElts); in createVariablePermute()
10761 IndicesVT = EVT(VT).changeVectorElementTypeToInteger(); in createVariablePermute()
10765 createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget); in createVariablePermute()
10771 SrcVec = widenSubVector(VT, SrcVec, false, Subtarget, DAG, SDLoc(SrcVec)); in createVariablePermute()
10801 switch (VT.SimpleTy) { in createVariablePermute()
10838 DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {0, 0}), in createVariablePermute()
10839 DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {1, 1}), in createVariablePermute()
10852 ISD::CONCAT_VECTORS, DL, VT, in createVariablePermute()
10858 SDValue LoLo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Lo); in createVariablePermute()
10859 SDValue HiHi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Hi, Hi); in createVariablePermute()
10866 EVT VT = Idx.getValueType(); in createVariablePermute() local
10867 return DAG.getSelectCC(DL, Idx, DAG.getConstant(15, DL, VT), in createVariablePermute()
10868 DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[1], Idx), in createVariablePermute()
10869 DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[0], Idx), in createVariablePermute()
10884 VT, createVariablePermute( in createVariablePermute()
10901 VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v8f32, LoLo, HiHi, in createVariablePermute()
10910 return DAG.getBitcast(VT, Res); in createVariablePermute()
10917 MVT WidenSrcVT = MVT::getVectorVT(VT.getScalarType(), 8); in createVariablePermute()
10937 VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v4f64, LoLo, HiHi, in createVariablePermute()
10946 return DAG.getBitcast(VT, Res); in createVariablePermute()
10968 assert((VT.getSizeInBits() == ShuffleVT.getSizeInBits()) && in createVariablePermute()
10969 (VT.getScalarSizeInBits() % ShuffleVT.getScalarSizeInBits()) == 0 && in createVariablePermute()
10972 uint64_t Scale = VT.getScalarSizeInBits() / ShuffleVT.getScalarSizeInBits(); in createVariablePermute()
10983 return DAG.getBitcast(VT, Res); in createVariablePermute()
11038 MVT VT = V.getSimpleValueType(); in LowerBUILD_VECTORAsVariablePermute() local
11039 return createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget); in LowerBUILD_VECTORAsVariablePermute()
11046 MVT VT = Op.getSimpleValueType(); in LowerBUILD_VECTOR() local
11047 MVT EltVT = VT.getVectorElementType(); in LowerBUILD_VECTOR()
11051 if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512()) in LowerBUILD_VECTOR()
11085 return DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
11092 if ((VT.is256BitVector() || VT.is512BitVector()) && in LowerBUILD_VECTOR()
11098 if (VT.is512BitVector() && in LowerBUILD_VECTOR()
11105 return widenSubVector(VT, NewBV, !UndefUpper, Subtarget, DAG, dl); in LowerBUILD_VECTOR()
11128 (isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT) || in LowerBUILD_VECTOR()
11129 isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, VT))) { in LowerBUILD_VECTOR()
11152 SDValue DAGConstVec = DAG.getConstantPool(CV, VT); in LowerBUILD_VECTOR()
11163 SDValue Ld = DAG.getLoad(VT, dl, DAG.getEntryNode(), LegalDAGConstVec, MPI); in LowerBUILD_VECTOR()
11165 unsigned NumEltsInLow128Bits = 128 / VT.getScalarSizeInBits(); in LowerBUILD_VECTOR()
11167 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ld, VarElt, InsIndex); in LowerBUILD_VECTOR()
11171 assert(VT.getSizeInBits() > 128 && "Invalid insertion index?"); in LowerBUILD_VECTOR()
11174 unsigned NumElts = VT.getVectorNumElements(); in LowerBUILD_VECTOR()
11177 SDValue S2V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, VarElt); in LowerBUILD_VECTOR()
11178 return DAG.getVectorShuffle(VT, dl, Ld, S2V, ShuffleMask); in LowerBUILD_VECTOR()
11192 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); in LowerBUILD_VECTOR()
11197 assert((VT.is128BitVector() || VT.is256BitVector() || in LowerBUILD_VECTOR()
11198 VT.is512BitVector()) && in LowerBUILD_VECTOR()
11200 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); in LowerBUILD_VECTOR()
11210 MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32); in LowerBUILD_VECTOR()
11213 return DAG.getBitcast(VT, Item); in LowerBUILD_VECTOR()
11221 unsigned NumBits = VT.getSizeInBits(); in LowerBUILD_VECTOR()
11222 return getVShift(true, VT, in LowerBUILD_VECTOR()
11224 VT, Op.getOperand(1)), in LowerBUILD_VECTOR()
11237 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); in LowerBUILD_VECTOR()
11252 return LowerAsSplatVectorLoad(Item, VT, dl, DAG); in LowerBUILD_VECTOR()
11269 EltsFromConsecutiveLoads(VT, Ops, dl, DAG, Subtarget, false)) in LowerBUILD_VECTOR()
11287 MVT WideEltVT = VT.isFloatingPoint() ? MVT::f64 : MVT::i64; in LowerBUILD_VECTOR()
11294 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, dl, BcastVT, in LowerBUILD_VECTOR()
11301 if (VT.getSizeInBits() > 128) { in LowerBUILD_VECTOR()
11319 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, in LowerBUILD_VECTOR()
11348 Ops[i] = getZeroVector(VT, Subtarget, DAG, dl); in LowerBUILD_VECTOR()
11350 Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); in LowerBUILD_VECTOR()
11360 Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2+1], Ops[i*2]); in LowerBUILD_VECTOR()
11363 Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2], Ops[i*2+1]); in LowerBUILD_VECTOR()
11366 Ops[i] = getUnpackl(DAG, dl, VT, Ops[i*2], Ops[i*2+1]); in LowerBUILD_VECTOR()
11379 return DAG.getVectorShuffle(VT, dl, Ops[0], Ops[1], MaskVec); in LowerBUILD_VECTOR()
11392 Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0)); in LowerBUILD_VECTOR()
11394 Result = DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
11398 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result, in LowerBUILD_VECTOR()
11410 Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); in LowerBUILD_VECTOR()
11412 Ops[i] = DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
11429 Ops[i] = DAG.getVectorShuffle(VT, dl, Ops[2*i], Ops[(2*i)+1], Mask); in LowerBUILD_VECTOR()
11577 MVT VT = Op.getSimpleValueType(); in LowerCONCAT_VECTORS() local
11578 if (VT.getVectorElementType() == MVT::i1) in LowerCONCAT_VECTORS()
11581 assert((VT.is256BitVector() && Op.getNumOperands() == 2) || in LowerCONCAT_VECTORS()
11582 (VT.is512BitVector() && (Op.getNumOperands() == 2 || in LowerCONCAT_VECTORS()
11640 static bool is128BitLaneCrossingShuffleMask(MVT VT, ArrayRef<int> Mask) { in is128BitLaneCrossingShuffleMask() argument
11641 return isLaneCrossingShuffleMask(128, VT.getScalarSizeInBits(), Mask); in is128BitLaneCrossingShuffleMask()
11684 static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT, in isRepeatedShuffleMask() argument
11687 auto LaneSize = LaneSizeInBits / VT.getScalarSizeInBits(); in isRepeatedShuffleMask()
11714 is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask, in is128BitLaneRepeatedShuffleMask() argument
11716 return isRepeatedShuffleMask(128, VT, Mask, RepeatedMask); in is128BitLaneRepeatedShuffleMask()
11720 is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask) { in is128BitLaneRepeatedShuffleMask() argument
11722 return isRepeatedShuffleMask(128, VT, Mask, RepeatedMask); in is128BitLaneRepeatedShuffleMask()
11727 is256BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask, in is256BitLaneRepeatedShuffleMask() argument
11729 return isRepeatedShuffleMask(256, VT, Mask, RepeatedMask); in is256BitLaneRepeatedShuffleMask()
11771 static bool isRepeatedTargetShuffleMask(unsigned LaneSizeInBits, MVT VT, in isRepeatedTargetShuffleMask() argument
11774 return isRepeatedTargetShuffleMask(LaneSizeInBits, VT.getScalarSizeInBits(), in isRepeatedTargetShuffleMask()
11811 MVT VT = Op.getSimpleValueType(); in IsElementEquivalent() local
11812 int NumElts = VT.getVectorNumElements(); in IsElementEquivalent()
11814 int NumLanes = VT.getSizeInBits() / 128; in IsElementEquivalent()
11872 static bool isTargetShuffleEquivalent(MVT VT, ArrayRef<int> Mask, in isTargetShuffleEquivalent() argument
11889 if (V1 && V1.getValueSizeInBits() != VT.getSizeInBits()) in isTargetShuffleEquivalent()
11891 if (V2 && V2.getValueSizeInBits() != VT.getSizeInBits()) in isTargetShuffleEquivalent()
11931 static bool isUnpackWdShuffleMask(ArrayRef<int> Mask, MVT VT, in isUnpackWdShuffleMask() argument
11933 if (VT != MVT::v8i32 && VT != MVT::v8f32) in isUnpackWdShuffleMask()
11942 bool IsUnpackwdMask = (isTargetShuffleEquivalent(VT, Mask, Unpcklwd, DAG) || in isUnpackWdShuffleMask()
11943 isTargetShuffleEquivalent(VT, Mask, Unpckhwd, DAG)); in isUnpackWdShuffleMask()
11951 MVT VT = MVT::getVectorVT(EltVT, Mask.size()); in is128BitUnpackShuffleMask() local
11960 createUnpackShuffleMask(VT, UnpackMask, (i >> 1) % 2, i % 2); in is128BitUnpackShuffleMask()
11961 if (isTargetShuffleEquivalent(VT, Mask, UnpackMask, DAG) || in is128BitUnpackShuffleMask()
11962 isTargetShuffleEquivalent(VT, CommutedMask, UnpackMask, DAG)) in is128BitUnpackShuffleMask()
12052 static SDValue lowerShuffleWithPSHUFB(const SDLoc &DL, MVT VT, in lowerShuffleWithPSHUFB() argument
12058 int LaneSize = 128 / VT.getScalarSizeInBits(); in lowerShuffleWithPSHUFB()
12059 const int NumBytes = VT.getSizeInBits() / 8; in lowerShuffleWithPSHUFB()
12060 const int NumEltBytes = VT.getScalarSizeInBits() / 8; in lowerShuffleWithPSHUFB()
12062 assert((Subtarget.hasSSSE3() && VT.is128BitVector()) || in lowerShuffleWithPSHUFB()
12063 (Subtarget.hasAVX2() && VT.is256BitVector()) || in lowerShuffleWithPSHUFB()
12064 (Subtarget.hasBWI() && VT.is512BitVector())); in lowerShuffleWithPSHUFB()
12101 VT, DAG.getNode(X86ISD::PSHUFB, DL, I8VT, DAG.getBitcast(I8VT, V), in lowerShuffleWithPSHUFB()
12110 static SDValue lowerShuffleToEXPAND(const SDLoc &DL, MVT VT, in lowerShuffleToEXPAND() argument
12121 MVT::getIntegerVT(std::max((int)VT.getVectorNumElements(), 8)); in lowerShuffleToEXPAND()
12123 unsigned NumElts = VT.getVectorNumElements(); in lowerShuffleToEXPAND()
12128 SDValue ZeroVector = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleToEXPAND()
12130 return DAG.getNode(X86ISD::EXPAND, DL, VT, ExpandedVector, ZeroVector, VMask); in lowerShuffleToEXPAND()
12133 static bool matchShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2, in matchShuffleWithUNPCK() argument
12138 int NumElts = VT.getVectorNumElements(); in matchShuffleWithUNPCK()
12154 createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, IsUnary); in matchShuffleWithUNPCK()
12155 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckl, DAG, V1, in matchShuffleWithUNPCK()
12158 V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2)); in matchShuffleWithUNPCK()
12159 V1 = (Undef1 ? DAG.getUNDEF(VT) : V1); in matchShuffleWithUNPCK()
12163 createUnpackShuffleMask(VT, Unpckh, /* Lo = */ false, IsUnary); in matchShuffleWithUNPCK()
12164 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckh, DAG, V1, in matchShuffleWithUNPCK()
12167 V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2)); in matchShuffleWithUNPCK()
12168 V1 = (Undef1 ? DAG.getUNDEF(VT) : V1); in matchShuffleWithUNPCK()
12175 if ((Subtarget.hasSSE41() || VT == MVT::v2i64 || VT == MVT::v2f64) && in matchShuffleWithUNPCK()
12194 V2 = Zero2 ? getZeroVector(VT, Subtarget, DAG, DL) : V1; in matchShuffleWithUNPCK()
12195 V1 = Zero1 ? getZeroVector(VT, Subtarget, DAG, DL) : V1; in matchShuffleWithUNPCK()
12203 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckl, DAG)) { in matchShuffleWithUNPCK()
12210 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckh, DAG)) { in matchShuffleWithUNPCK()
12222 static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT, in lowerShuffleWithUNPCK() argument
12226 createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, /* Unary = */ false); in lowerShuffleWithUNPCK()
12228 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2); in lowerShuffleWithUNPCK()
12231 createUnpackShuffleMask(VT, Unpckh, /* Lo = */ false, /* Unary = */ false); in lowerShuffleWithUNPCK()
12233 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2); in lowerShuffleWithUNPCK()
12238 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V2, V1); in lowerShuffleWithUNPCK()
12242 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V2, V1); in lowerShuffleWithUNPCK()
12249 static SDValue lowerShuffleWithUNPCK256(const SDLoc &DL, MVT VT, in lowerShuffleWithUNPCK256() argument
12253 createSplat2ShuffleMask(VT, Unpckl, /* Lo */ true); in lowerShuffleWithUNPCK256()
12254 createSplat2ShuffleMask(VT, Unpckh, /* Lo */ false); in lowerShuffleWithUNPCK256()
12269 V1 = DAG.getBitcast(VT, V1); in lowerShuffleWithUNPCK256()
12270 return DAG.getNode(UnpackOpcode, DL, VT, V1, V1); in lowerShuffleWithUNPCK256()
12275 static bool matchShuffleAsVTRUNC(MVT &SrcVT, MVT &DstVT, MVT VT, in matchShuffleAsVTRUNC() argument
12278 if (!VT.is512BitVector() && !Subtarget.hasVLX()) in matchShuffleAsVTRUNC()
12282 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in matchShuffleAsVTRUNC()
12374 static SDValue lowerShuffleWithVPMOV(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleWithVPMOV() argument
12379 assert((VT == MVT::v16i8 || VT == MVT::v8i16) && "Unexpected VTRUNC type"); in lowerShuffleWithVPMOV()
12383 unsigned NumElts = VT.getVectorNumElements(); in lowerShuffleWithVPMOV()
12384 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in lowerShuffleWithVPMOV()
12405 if (SrcVT.getVectorElementType() == MVT::i16 && VT == MVT::v16i8 && in lowerShuffleWithVPMOV()
12410 return getAVX512TruncNode(DL, VT, Src, Subtarget, DAG, !UndefUppers); in lowerShuffleWithVPMOV()
12417 static SDValue lowerShuffleAsVTRUNC(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsVTRUNC() argument
12422 assert((VT.is128BitVector() || VT.is256BitVector()) && in lowerShuffleAsVTRUNC()
12427 unsigned NumElts = VT.getVectorNumElements(); in lowerShuffleAsVTRUNC()
12428 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in lowerShuffleAsVTRUNC()
12454 MVT ConcatVT = MVT::getVectorVT(VT.getScalarType(), NumElts * 2); in lowerShuffleAsVTRUNC()
12460 return getAVX512TruncNode(DL, VT, Src, Subtarget, DAG, !UndefUppers); in lowerShuffleAsVTRUNC()
12542 static bool matchShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2, in matchShuffleWithPACK() argument
12547 unsigned NumElts = VT.getVectorNumElements(); in matchShuffleWithPACK()
12548 unsigned BitSize = VT.getScalarSizeInBits(); in matchShuffleWithPACK()
12597 createPackShuffleMask(VT, BinaryMask, false, NumStages); in matchShuffleWithPACK()
12598 if (isTargetShuffleEquivalent(VT, TargetMask, BinaryMask, DAG, V1, V2)) in matchShuffleWithPACK()
12604 createPackShuffleMask(VT, UnaryMask, true, NumStages); in matchShuffleWithPACK()
12605 if (isTargetShuffleEquivalent(VT, TargetMask, UnaryMask, DAG, V1)) in matchShuffleWithPACK()
12613 static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, ArrayRef<int> Mask, in lowerShuffleWithPACK() argument
12618 unsigned SizeBits = VT.getSizeInBits(); in lowerShuffleWithPACK()
12619 unsigned EltBits = VT.getScalarSizeInBits(); in lowerShuffleWithPACK()
12621 if (!matchShuffleWithPACK(VT, PackVT, V1, V2, PackOpcode, Mask, DAG, in lowerShuffleWithPACK()
12653 assert(Res && Res.getValueType() == VT && in lowerShuffleWithPACK()
12662 static SDValue lowerShuffleAsBitMask(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsBitMask() argument
12667 MVT MaskVT = VT; in lowerShuffleAsBitMask()
12668 MVT EltVT = VT.getVectorElementType(); in lowerShuffleAsBitMask()
12676 MVT LogicVT = VT; in lowerShuffleAsBitMask()
12710 return DAG.getBitcast(VT, And); in lowerShuffleAsBitMask()
12718 static SDValue lowerShuffleAsBitBlend(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsBitBlend() argument
12721 assert(VT.isInteger() && "Only supports integer vector types!"); in lowerShuffleAsBitBlend()
12722 MVT EltVT = VT.getVectorElementType(); in lowerShuffleAsBitBlend()
12732 SDValue V1Mask = DAG.getBuildVector(VT, DL, MaskOps); in lowerShuffleAsBitBlend()
12733 V1 = DAG.getNode(ISD::AND, DL, VT, V1, V1Mask); in lowerShuffleAsBitBlend()
12734 V2 = DAG.getNode(X86ISD::ANDNP, DL, VT, V1Mask, V2); in lowerShuffleAsBitBlend()
12735 return DAG.getNode(ISD::OR, DL, VT, V1, V2); in lowerShuffleAsBitBlend()
12806 static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsBlend() argument
12820 V1 = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleAsBlend()
12822 V2 = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleAsBlend()
12824 unsigned NumElts = VT.getVectorNumElements(); in lowerShuffleAsBlend()
12826 switch (VT.SimpleTy) { in lowerShuffleAsBlend()
12841 return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2, in lowerShuffleAsBlend()
12879 if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable, in lowerShuffleAsBlend()
12892 lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG)) in lowerShuffleAsBlend()
12896 int Scale = VT.getScalarSizeInBits() / 8; in lowerShuffleAsBlend()
12900 MVT BlendVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8); in lowerShuffleAsBlend()
12933 VT, in lowerShuffleAsBlend()
12946 if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable, in lowerShuffleAsBlend()
12967 static SDValue lowerShuffleAsBlendAndPermute(const SDLoc &DL, MVT VT, in lowerShuffleAsBlendAndPermute() argument
12993 unsigned EltSize = VT.getScalarSizeInBits(); in lowerShuffleAsBlendAndPermute()
12997 SDValue V = DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask); in lowerShuffleAsBlendAndPermute()
12998 return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), PermuteMask); in lowerShuffleAsBlendAndPermute()
13006 static SDValue lowerShuffleAsUNPCKAndPermute(const SDLoc &DL, MVT VT, in lowerShuffleAsUNPCKAndPermute() argument
13011 int NumLanes = VT.getSizeInBits() / 128; in lowerShuffleAsUNPCKAndPermute()
13016 SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)}; in lowerShuffleAsUNPCKAndPermute()
13063 SDValue Unpck = DAG.getNode(UnpckOp, DL, VT, Ops); in lowerShuffleAsUNPCKAndPermute()
13064 return DAG.getVectorShuffle(VT, DL, Unpck, DAG.getUNDEF(VT), PermuteMask); in lowerShuffleAsUNPCKAndPermute()
13070 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsByteRotateAndPermute() argument
13072 if ((VT.is128BitVector() && !Subtarget.hasSSSE3()) || in lowerShuffleAsByteRotateAndPermute()
13073 (VT.is256BitVector() && !Subtarget.hasAVX2()) || in lowerShuffleAsByteRotateAndPermute()
13074 (VT.is512BitVector() && !Subtarget.hasBWI())) in lowerShuffleAsByteRotateAndPermute()
13078 if (is128BitLaneCrossingShuffleMask(VT, Mask)) in lowerShuffleAsByteRotateAndPermute()
13081 int Scale = VT.getScalarSizeInBits() / 8; in lowerShuffleAsByteRotateAndPermute()
13082 int NumLanes = VT.getSizeInBits() / 128; in lowerShuffleAsByteRotateAndPermute()
13083 int NumElts = VT.getVectorNumElements(); in lowerShuffleAsByteRotateAndPermute()
13120 if (VT.getSizeInBits() > 128 && (Blend1 || Blend2)) in lowerShuffleAsByteRotateAndPermute()
13125 MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8); in lowerShuffleAsByteRotateAndPermute()
13127 VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, DAG.getBitcast(ByteVT, Hi), in lowerShuffleAsByteRotateAndPermute()
13142 return DAG.getVectorShuffle(VT, DL, Rotate, DAG.getUNDEF(VT), PermMask); in lowerShuffleAsByteRotateAndPermute()
13169 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsDecomposedShuffleMerge() argument
13172 int NumLanes = VT.getSizeInBits() / 128; in lowerShuffleAsDecomposedShuffleMerge()
13197 auto canonicalizeBroadcastableInput = [DL, VT, &Subtarget, in lowerShuffleAsDecomposedShuffleMerge()
13208 Input = DAG.getNode(X86ISD::VBROADCAST, DL, VT, Input); in lowerShuffleAsDecomposedShuffleMerge()
13233 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask, in lowerShuffleAsDecomposedShuffleMerge()
13236 if (SDValue UnpackPerm = lowerShuffleAsUNPCKAndPermute(DL, VT, V1, V2, Mask, in lowerShuffleAsDecomposedShuffleMerge()
13240 DL, VT, V1, V2, Mask, Subtarget, DAG)) in lowerShuffleAsDecomposedShuffleMerge()
13243 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask, in lowerShuffleAsDecomposedShuffleMerge()
13252 if (IsAlternating && VT.getScalarSizeInBits() < 32) { in lowerShuffleAsDecomposedShuffleMerge()
13269 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask); in lowerShuffleAsDecomposedShuffleMerge()
13270 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask); in lowerShuffleAsDecomposedShuffleMerge()
13271 return DAG.getVectorShuffle(VT, DL, V1, V2, FinalMask); in lowerShuffleAsDecomposedShuffleMerge()
13323 static SDValue lowerShuffleAsBitRotate(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsBitRotate() argument
13330 (VT.is128BitVector() && Subtarget.hasXOP()) || Subtarget.hasAVX512(); in lowerShuffleAsBitRotate()
13335 int RotateAmt = matchShuffleAsBitRotate(RotateVT, VT.getScalarSizeInBits(), in lowerShuffleAsBitRotate()
13355 return DAG.getBitcast(VT, Rot); in lowerShuffleAsBitRotate()
13361 return DAG.getBitcast(VT, Rot); in lowerShuffleAsBitRotate()
13452 static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2, in matchShuffleAsByteRotate() argument
13460 if (!is128BitLaneRepeatedShuffleMask(VT, Mask, RepeatedMask)) in matchShuffleAsByteRotate()
13474 static SDValue lowerShuffleAsByteRotate(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsByteRotate() argument
13481 int ByteRotation = matchShuffleAsByteRotate(VT, Lo, Hi, Mask); in lowerShuffleAsByteRotate()
13487 MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8); in lowerShuffleAsByteRotate()
13493 assert((!VT.is512BitVector() || Subtarget.hasBWI()) && in lowerShuffleAsByteRotate()
13496 VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, Lo, Hi, in lowerShuffleAsByteRotate()
13500 assert(VT.is128BitVector() && in lowerShuffleAsByteRotate()
13517 return DAG.getBitcast(VT, in lowerShuffleAsByteRotate()
13531 static SDValue lowerShuffleAsVALIGN(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsVALIGN() argument
13535 assert((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) && in lowerShuffleAsVALIGN()
13539 assert((Subtarget.hasVLX() || (!VT.is128BitVector() && !VT.is256BitVector())) in lowerShuffleAsVALIGN()
13547 return DAG.getNode(X86ISD::VALIGN, DL, VT, Lo, Hi, in lowerShuffleAsVALIGN()
13552 static SDValue lowerShuffleAsByteShiftMask(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsByteShiftMask() argument
13558 assert(VT.is128BitVector() && "Only 128-bit vectors supported"); in lowerShuffleAsByteShiftMask()
13572 unsigned Scale = VT.getScalarSizeInBits() / 8; in lowerShuffleAsByteShiftMask()
13613 return DAG.getBitcast(VT, Res); in lowerShuffleAsByteShiftMask()
13701 static SDValue lowerShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsShift() argument
13707 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size"); in lowerShuffleAsShift()
13714 int ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(), in lowerShuffleAsShift()
13719 ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(), in lowerShuffleAsShift()
13732 return DAG.getBitcast(VT, V); in lowerShuffleAsShift()
13737 static bool matchShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2, in matchShuffleAsEXTRQ() argument
13742 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size"); in matchShuffleAsEXTRQ()
13784 BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f; in matchShuffleAsEXTRQ()
13785 BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f; in matchShuffleAsEXTRQ()
13793 static bool matchShuffleAsINSERTQ(MVT VT, SDValue &V1, SDValue &V2, in matchShuffleAsINSERTQ() argument
13798 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size"); in matchShuffleAsINSERTQ()
13847 BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f; in matchShuffleAsINSERTQ()
13848 BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f; in matchShuffleAsINSERTQ()
13859 static SDValue lowerShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleWithSSE4A() argument
13863 if (matchShuffleAsEXTRQ(VT, V1, V2, Mask, BitLen, BitIdx, Zeroable)) in lowerShuffleWithSSE4A()
13864 return DAG.getNode(X86ISD::EXTRQI, DL, VT, V1, in lowerShuffleWithSSE4A()
13868 if (matchShuffleAsINSERTQ(VT, V1, V2, Mask, BitLen, BitIdx)) in lowerShuffleWithSSE4A()
13869 return DAG.getNode(X86ISD::INSERTQI, DL, VT, V1 ? V1 : DAG.getUNDEF(VT), in lowerShuffleWithSSE4A()
13870 V2 ? V2 : DAG.getUNDEF(VT), in lowerShuffleWithSSE4A()
13887 const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV, in lowerShuffleAsSpecificZeroOrAnyExtend() argument
13890 int EltBits = VT.getScalarSizeInBits(); in lowerShuffleAsSpecificZeroOrAnyExtend()
13891 int NumElements = VT.getVectorNumElements(); in lowerShuffleAsSpecificZeroOrAnyExtend()
13916 return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), ShMask); in lowerShuffleAsSpecificZeroOrAnyExtend()
13924 if (Offset && Scale == 2 && VT.is128BitVector()) in lowerShuffleAsSpecificZeroOrAnyExtend()
13931 return DAG.getBitcast(VT, InputV); in lowerShuffleAsSpecificZeroOrAnyExtend()
13934 assert(VT.is128BitVector() && "Only 128-bit vectors can be extended."); in lowerShuffleAsSpecificZeroOrAnyExtend()
13942 VT, DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, in lowerShuffleAsSpecificZeroOrAnyExtend()
13955 VT, DAG.getNode(OddEvenOp, DL, MVT::v8i16, in lowerShuffleAsSpecificZeroOrAnyExtend()
13964 assert(VT.is128BitVector() && "Unexpected vector width!"); in lowerShuffleAsSpecificZeroOrAnyExtend()
13968 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV, in lowerShuffleAsSpecificZeroOrAnyExtend()
13973 return DAG.getBitcast(VT, Lo); in lowerShuffleAsSpecificZeroOrAnyExtend()
13977 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV, in lowerShuffleAsSpecificZeroOrAnyExtend()
13980 return DAG.getBitcast(VT, in lowerShuffleAsSpecificZeroOrAnyExtend()
14001 VT, DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV, in lowerShuffleAsSpecificZeroOrAnyExtend()
14012 InputV = DAG.getVectorShuffle(VT, DL, InputV, DAG.getUNDEF(VT), ShMask); in lowerShuffleAsSpecificZeroOrAnyExtend()
14033 return DAG.getBitcast(VT, InputV); in lowerShuffleAsSpecificZeroOrAnyExtend()
14049 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsZeroOrAnyExtend() argument
14052 int Bits = VT.getSizeInBits(); in lowerShuffleAsZeroOrAnyExtend()
14054 int NumElements = VT.getVectorNumElements(); in lowerShuffleAsZeroOrAnyExtend()
14056 assert(VT.getScalarSizeInBits() <= 32 && in lowerShuffleAsZeroOrAnyExtend()
14119 return lowerShuffleAsSpecificZeroOrAnyExtend(DL, VT, Scale, Offset, AnyExt, in lowerShuffleAsZeroOrAnyExtend()
14157 return DAG.getBitcast(VT, V); in lowerShuffleAsZeroOrAnyExtend()
14169 MVT VT = V.getSimpleValueType(); in getScalarValueForVectorElement() local
14170 MVT EltVT = VT.getVectorElementType(); in getScalarValueForVectorElement()
14176 if (!NewVT.isVector() || NewVT.getScalarSizeInBits() != VT.getScalarSizeInBits()) in getScalarValueForVectorElement()
14201 static bool isSoftFP16(T VT, const X86Subtarget &Subtarget) { in isSoftFP16() argument
14202 return VT.getScalarType() == MVT::f16 && !Subtarget.hasFP16(); in isSoftFP16()
14206 bool X86TargetLowering::isSoftFP16(T VT) const { in isSoftFP16()
14207 return ::isSoftFP16(VT, Subtarget); in isSoftFP16()
14215 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsElementInsertion() argument
14218 MVT ExtVT = VT; in lowerShuffleAsElementInsertion()
14219 MVT EltVT = VT.getVectorElementType(); in lowerShuffleAsElementInsertion()
14266 assert(VT == ExtVT && "Cannot change extended type when non-zeroable!"); in lowerShuffleAsElementInsertion()
14267 if (!VT.isFloatingPoint() || V2Index != 0) in lowerShuffleAsElementInsertion()
14273 if (!VT.is128BitVector()) in lowerShuffleAsElementInsertion()
14290 if (VT.isFloatingPoint() && V2Index != 0) in lowerShuffleAsElementInsertion()
14294 if (ExtVT != VT) in lowerShuffleAsElementInsertion()
14295 V2 = DAG.getBitcast(VT, V2); in lowerShuffleAsElementInsertion()
14302 if (VT.isFloatingPoint() || VT.getVectorNumElements() <= 4) { in lowerShuffleAsElementInsertion()
14305 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Shuffle); in lowerShuffleAsElementInsertion()
14311 V2 = DAG.getBitcast(VT, V2); in lowerShuffleAsElementInsertion()
14321 static SDValue lowerShuffleAsTruncBroadcast(const SDLoc &DL, MVT VT, SDValue V0, in lowerShuffleAsTruncBroadcast() argument
14328 MVT EltVT = VT.getVectorElementType(); in lowerShuffleAsTruncBroadcast()
14331 assert(VT.isInteger() && "Unexpected non-integer trunc broadcast!"); in lowerShuffleAsTruncBroadcast()
14366 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, in lowerShuffleAsTruncBroadcast()
14398 MVT VT = N0.getSimpleValueType(); in lowerShuffleOfExtractsAsVperm() local
14399 assert((VT.is128BitVector() && in lowerShuffleOfExtractsAsVperm()
14400 (VT.getScalarSizeInBits() == 32 || VT.getScalarSizeInBits() == 64)) && in lowerShuffleOfExtractsAsVperm()
14417 unsigned NumElts = VT.getVectorNumElements(); in lowerShuffleOfExtractsAsVperm()
14440 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuf, in lowerShuffleOfExtractsAsVperm()
14449 static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsBroadcast() argument
14453 if (!((Subtarget.hasSSE3() && VT == MVT::v2f64) || in lowerShuffleAsBroadcast()
14454 (Subtarget.hasAVX() && VT.isFloatingPoint()) || in lowerShuffleAsBroadcast()
14455 (Subtarget.hasAVX2() && VT.isInteger()))) in lowerShuffleAsBroadcast()
14460 unsigned NumEltBits = VT.getScalarSizeInBits(); in lowerShuffleAsBroadcast()
14461 unsigned Opcode = (VT == MVT::v2f64 && !Subtarget.hasAVX2()) in lowerShuffleAsBroadcast()
14531 if (BitCastSrc && VT.isInteger()) in lowerShuffleAsBroadcast()
14533 DL, VT, V, BroadcastIdx, Subtarget, DAG)) in lowerShuffleAsBroadcast()
14554 MVT SVT = VT.getScalarType(); in lowerShuffleAsBroadcast()
14564 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in lowerShuffleAsBroadcast()
14571 return DAG.getBitcast(VT, V); in lowerShuffleAsBroadcast()
14585 if (!VT.is256BitVector() && !VT.is512BitVector()) in lowerShuffleAsBroadcast()
14589 if (VT == MVT::v4f64 || VT == MVT::v4i64) in lowerShuffleAsBroadcast()
14609 return DAG.getBitcast(VT, V); in lowerShuffleAsBroadcast()
14619 VT.getVectorNumElements()); in lowerShuffleAsBroadcast()
14620 return DAG.getBitcast(VT, DAG.getNode(Opcode, DL, BroadcastVT, V)); in lowerShuffleAsBroadcast()
14632 MVT CastVT = MVT::getVectorVT(VT.getVectorElementType(), NumSrcElts); in lowerShuffleAsBroadcast()
14633 return DAG.getNode(Opcode, DL, VT, DAG.getBitcast(CastVT, V)); in lowerShuffleAsBroadcast()
14756 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsPermuteAndUnpack() argument
14758 assert(!VT.isFloatingPoint() && in lowerShuffleAsPermuteAndUnpack()
14760 assert(VT.is128BitVector() && in lowerShuffleAsPermuteAndUnpack()
14805 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask); in lowerShuffleAsPermuteAndUnpack()
14806 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask); in lowerShuffleAsPermuteAndUnpack()
14815 VT, DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL, in lowerShuffleAsPermuteAndUnpack()
14821 int OrigScalarSize = VT.getScalarSizeInBits(); in lowerShuffleAsPermuteAndUnpack()
14855 VT, DL, DAG.getNode(NumLoInputs == 0 ? X86ISD::UNPCKH : X86ISD::UNPCKL, in lowerShuffleAsPermuteAndUnpack()
14856 DL, VT, V1, V2), in lowerShuffleAsPermuteAndUnpack()
14857 DAG.getUNDEF(VT), PermMask); in lowerShuffleAsPermuteAndUnpack()
15053 static SDValue lowerShuffleWithSHUFPS(const SDLoc &DL, MVT VT, in lowerShuffleWithSHUFPS() argument
15079 V2 = DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1, in lowerShuffleWithSHUFPS()
15118 V1 = DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2, in lowerShuffleWithSHUFPS()
15134 return lowerShuffleWithSHUFPS(DL, VT, NewMask, V2, V1, DAG); in lowerShuffleWithSHUFPS()
15136 return DAG.getNode(X86ISD::SHUFP, DL, VT, LowV, HighV, in lowerShuffleWithSHUFPS()
15370 const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef<int> Mask, in lowerV8I16GeneralSingleInputShuffle() argument
15372 assert(VT.getVectorElementType() == MVT::i16 && "Bad input type!"); in lowerV8I16GeneralSingleInputShuffle()
15373 MVT PSHUFDVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2); in lowerV8I16GeneralSingleInputShuffle()
15382 return DAG.getNode(X86ISD::PSHUFLW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
15389 return DAG.getNode(X86ISD::PSHUFHW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
15415 V = DAG.getNode(ShufWOp, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
15420 return DAG.getBitcast(VT, V); in lowerV8I16GeneralSingleInputShuffle()
15592 VT, in lowerV8I16GeneralSingleInputShuffle()
15605 return lowerV8I16GeneralSingleInputShuffle(DL, VT, V, Mask, Subtarget, DAG); in lowerV8I16GeneralSingleInputShuffle()
15827 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
15830 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
15834 VT, in lowerV8I16GeneralSingleInputShuffle()
15847 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
15855 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
15864 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsBlendOfPSHUFBs() argument
15866 assert(!is128BitLaneCrossingShuffleMask(VT, Mask) && in lowerShuffleAsBlendOfPSHUFBs()
15869 int NumBytes = VT.getSizeInBits() / 8; in lowerShuffleAsBlendOfPSHUFBs()
15911 return DAG.getBitcast(VT, V); in lowerShuffleAsBlendOfPSHUFBs()
16147 static SDValue lowerShuffleWithPERMV(const SDLoc &DL, MVT VT, in lowerShuffleWithPERMV() argument
16151 MVT MaskVT = VT.changeTypeToInteger(); in lowerShuffleWithPERMV()
16153 MVT ShuffleVT = VT; in lowerShuffleWithPERMV()
16154 if (!VT.is512BitVector() && !Subtarget.hasVLX()) { in lowerShuffleWithPERMV()
16160 int NumElts = VT.getVectorNumElements(); in lowerShuffleWithPERMV()
16161 unsigned Scale = 512 / VT.getSizeInBits(); in lowerShuffleWithPERMV()
16178 if (VT != ShuffleVT) in lowerShuffleWithPERMV()
16179 Result = extractSubVector(Result, 0, DAG, DL, VT.getSizeInBits()); in lowerShuffleWithPERMV()
16547 MVT VT, SDValue V1, SDValue V2, in lower128BitShuffle() argument
16551 switch (VT.SimpleTy) { in lower128BitShuffle()
16577 static SDValue splitAndLowerShuffle(const SDLoc &DL, MVT VT, SDValue V1, in splitAndLowerShuffle() argument
16580 assert(VT.getSizeInBits() >= 256 && in splitAndLowerShuffle()
16582 assert(V1.getSimpleValueType() == VT && "Bad operand type!"); in splitAndLowerShuffle()
16583 assert(V2.getSimpleValueType() == VT && "Bad operand type!"); in splitAndLowerShuffle()
16588 int NumElements = VT.getVectorNumElements(); in splitAndLowerShuffle()
16590 MVT ScalarVT = VT.getVectorElementType(); in splitAndLowerShuffle()
16668 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); in splitAndLowerShuffle()
16679 static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsSplitOrBlend() argument
16707 return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Subtarget, in lowerShuffleAsSplitOrBlend()
16713 int LaneCount = VT.getSizeInBits() / 128; in lowerShuffleAsSplitOrBlend()
16722 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG); in lowerShuffleAsSplitOrBlend()
16726 return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Subtarget, in lowerShuffleAsSplitOrBlend()
16732 static SDValue lowerShuffleAsLanePermuteAndSHUFP(const SDLoc &DL, MVT VT, in lowerShuffleAsLanePermuteAndSHUFP() argument
16736 assert(VT == MVT::v4f64 && "Only for v4f64 shuffles"); in lowerShuffleAsLanePermuteAndSHUFP()
16754 SDValue LHS = DAG.getVectorShuffle(VT, DL, V1, V2, LHSMask); in lowerShuffleAsLanePermuteAndSHUFP()
16755 SDValue RHS = DAG.getVectorShuffle(VT, DL, V1, V2, RHSMask); in lowerShuffleAsLanePermuteAndSHUFP()
16756 return DAG.getNode(X86ISD::SHUFP, DL, VT, LHS, RHS, in lowerShuffleAsLanePermuteAndSHUFP()
16769 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsLanePermuteAndPermute() argument
16771 int NumElts = VT.getVectorNumElements(); in lowerShuffleAsLanePermuteAndPermute()
16772 int NumLanes = VT.getSizeInBits() / 128; in lowerShuffleAsLanePermuteAndPermute()
16839 SDValue CrossLane = DAG.getVectorShuffle(VT, DL, V1, V2, CrossLaneMask); in lowerShuffleAsLanePermuteAndPermute()
16840 return DAG.getVectorShuffle(VT, DL, CrossLane, DAG.getUNDEF(VT), in lowerShuffleAsLanePermuteAndPermute()
16872 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsLanePermuteAndShuffle() argument
16875 assert(VT.is256BitVector() && "Only for 256-bit vector shuffles!"); in lowerShuffleAsLanePermuteAndShuffle()
16882 if (VT == MVT::v4f64 && in lowerShuffleAsLanePermuteAndShuffle()
16884 return lowerShuffleAsLanePermuteAndSHUFP(DL, VT, V1, V2, Mask, DAG); in lowerShuffleAsLanePermuteAndShuffle()
16916 assert(!is128BitLaneCrossingShuffleMask(VT, InLaneMask) && in lowerShuffleAsLanePermuteAndShuffle()
16921 if (!AllLanes && !is128BitLaneRepeatedShuffleMask(VT, InLaneMask)) in lowerShuffleAsLanePermuteAndShuffle()
16922 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG); in lowerShuffleAsLanePermuteAndShuffle()
16925 MVT PVT = VT.isFloatingPoint() ? MVT::v4f64 : MVT::v4i64; in lowerShuffleAsLanePermuteAndShuffle()
16929 Flipped = DAG.getBitcast(VT, Flipped); in lowerShuffleAsLanePermuteAndShuffle()
16930 return DAG.getVectorShuffle(VT, DL, V1, Flipped, InLaneMask); in lowerShuffleAsLanePermuteAndShuffle()
16934 static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1, in lowerV2X128Shuffle() argument
16945 MVT MemVT = VT.getHalfNumVectorElementsVT(); in lowerV2X128Shuffle()
16949 VT, MemVT, Ld, Ofs, DAG)) in lowerV2X128Shuffle()
16969 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2); in lowerV2X128Shuffle()
16972 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in lowerV2X128Shuffle()
16973 getZeroVector(VT, Subtarget, DAG, DL), LoV, in lowerV2X128Shuffle()
16982 if (SDValue Blend = lowerShuffleAsBlend(DL, VT, V1, V2, Mask, Zeroable, in lowerV2X128Shuffle()
16997 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2); in lowerV2X128Shuffle()
17001 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec, in lowerV2X128Shuffle()
17011 return DAG.getNode(X86ISD::SHUF128, DL, VT, V1, V2, in lowerV2X128Shuffle()
17039 V1 = DAG.getUNDEF(VT); in lowerV2X128Shuffle()
17041 V2 = DAG.getUNDEF(VT); in lowerV2X128Shuffle()
17043 return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2, in lowerV2X128Shuffle()
17055 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsLanePermuteAndRepeatedMask() argument
17059 if (is128BitLaneRepeatedShuffleMask(VT, Mask)) in lowerShuffleAsLanePermuteAndRepeatedMask()
17063 int NumLanes = VT.getSizeInBits() / 128; in lowerShuffleAsLanePermuteAndRepeatedMask()
17064 int NumLaneElts = 128 / VT.getScalarSizeInBits(); in lowerShuffleAsLanePermuteAndRepeatedMask()
17181 SDValue NewV1 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask); in lowerShuffleAsLanePermuteAndRepeatedMask()
17198 SDValue NewV2 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask); in lowerShuffleAsLanePermuteAndRepeatedMask()
17213 return DAG.getVectorShuffle(VT, DL, NewV1, NewV2, NewMask); in lowerShuffleAsLanePermuteAndRepeatedMask()
17281 MVT VT = V1.getSimpleValueType(); in getShuffleHalfVectors() local
17282 MVT HalfVT = VT.getHalfNumVectorElementsVT(); in getShuffleHalfVectors()
17303 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Op0, Op1); in getShuffleHalfVectors()
17307 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, in getShuffleHalfVectors()
17314 static SDValue lowerShuffleWithUndefHalf(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleWithUndefHalf() argument
17318 assert((VT.is256BitVector() || VT.is512BitVector()) && in lowerShuffleWithUndefHalf()
17330 MVT HalfVT = VT.getHalfNumVectorElementsVT(); in lowerShuffleWithUndefHalf()
17336 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi, in lowerShuffleWithUndefHalf()
17346 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi, in lowerShuffleWithUndefHalf()
17366 unsigned EltWidth = VT.getVectorElementType().getSizeInBits(); in lowerShuffleWithUndefHalf()
17391 if (Subtarget.hasAVX512() && VT.is512BitVector()) in lowerShuffleWithUndefHalf()
17410 if (Subtarget.hasAVX512() && VT.is512BitVector()) in lowerShuffleWithUndefHalf()
17441 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsRepeatedMaskAndLanePermute() argument
17443 int NumElts = VT.getVectorNumElements(); in lowerShuffleAsRepeatedMaskAndLanePermute()
17444 int NumLanes = VT.getSizeInBits() / 128; in lowerShuffleAsRepeatedMaskAndLanePermute()
17451 if (BroadcastSize <= VT.getScalarSizeInBits()) in lowerShuffleAsRepeatedMaskAndLanePermute()
17453 int NumBroadcastElts = BroadcastSize / VT.getScalarSizeInBits(); in lowerShuffleAsRepeatedMaskAndLanePermute()
17479 SDValue RepeatShuf = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatMask); in lowerShuffleAsRepeatedMaskAndLanePermute()
17486 return DAG.getVectorShuffle(VT, DL, RepeatShuf, DAG.getUNDEF(VT), in lowerShuffleAsRepeatedMaskAndLanePermute()
17492 if (!is128BitLaneCrossingShuffleMask(VT, Mask)) in lowerShuffleAsRepeatedMaskAndLanePermute()
17496 if (is128BitLaneRepeatedShuffleMask(VT, Mask)) in lowerShuffleAsRepeatedMaskAndLanePermute()
17590 DAG.getVectorShuffle(VT, DL, V1, V2, RepeatedMask); in lowerShuffleAsRepeatedMaskAndLanePermute()
17602 return DAG.getVectorShuffle(VT, DL, RepeatedShuffle, DAG.getUNDEF(VT), in lowerShuffleAsRepeatedMaskAndLanePermute()
17611 if (Subtarget.hasAVX2() && VT.is256BitVector()) { in lowerShuffleAsRepeatedMaskAndLanePermute()
17615 (!OnlyLowestElts && V2.isUndef() && VT == MVT::v32i8) ? 4 : 2; in lowerShuffleAsRepeatedMaskAndLanePermute()
17617 if (Subtarget.hasBWI() && VT == MVT::v64i8) in lowerShuffleAsRepeatedMaskAndLanePermute()
17627 static bool matchShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2, in matchShuffleWithSHUFPD() argument
17631 int NumElts = VT.getVectorNumElements(); in matchShuffleWithSHUFPD()
17632 assert(VT.getScalarSizeInBits() == 64 && in matchShuffleWithSHUFPD()
17672 static SDValue lowerShuffleWithSHUFPD(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleWithSHUFPD() argument
17677 assert((VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v8f64) && in lowerShuffleWithSHUFPD()
17682 if (!matchShuffleWithSHUFPD(VT, V1, V2, ForceV1Zero, ForceV2Zero, Immediate, in lowerShuffleWithSHUFPD()
17688 V1 = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleWithSHUFPD()
17690 V2 = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleWithSHUFPD()
17692 return DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2, in lowerShuffleWithSHUFPD()
17699 static SDValue lowerShuffleAsVTRUNCAndUnpack(const SDLoc &DL, MVT VT, in lowerShuffleAsVTRUNCAndUnpack() argument
17704 assert(VT == MVT::v32i8 && "Unexpected type!"); in lowerShuffleAsVTRUNCAndUnpack()
18404 static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT, in lower256BitShuffle() argument
18410 int NumElts = VT.getVectorNumElements(); in lower256BitShuffle()
18415 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lower256BitShuffle()
18420 lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lower256BitShuffle()
18429 if (VT.isInteger() && !Subtarget.hasAVX2()) { in lower256BitShuffle()
18430 int ElementBits = VT.getScalarSizeInBits(); in lower256BitShuffle()
18434 if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable, in lower256BitShuffle()
18437 if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG)) in lower256BitShuffle()
18439 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG); in lower256BitShuffle()
18443 VT.getVectorNumElements()); in lower256BitShuffle()
18446 return DAG.getBitcast(VT, DAG.getVectorShuffle(FpVT, DL, V1, V2, Mask)); in lower256BitShuffle()
18449 if (VT == MVT::v16f16) { in lower256BitShuffle()
18456 switch (VT.SimpleTy) { in lower256BitShuffle()
18476 static SDValue lowerV4X128Shuffle(const SDLoc &DL, MVT VT, ArrayRef<int> Mask, in lowerV4X128Shuffle() argument
18480 assert(VT.getScalarSizeInBits() == 64 && in lowerV4X128Shuffle()
18485 assert(VT.is512BitVector() && "Unexpected vector size for 512bit shuffle."); in lowerV4X128Shuffle()
18497 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), NumElts); in lowerV4X128Shuffle()
18500 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in lowerV4X128Shuffle()
18501 getZeroVector(VT, Subtarget, DAG, DL), LoV, in lowerV4X128Shuffle()
18510 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 4); in lowerV4X128Shuffle()
18514 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec, in lowerV4X128Shuffle()
18542 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2); in lowerV4X128Shuffle()
18559 SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)}; in lowerV4X128Shuffle()
18579 return DAG.getNode(X86ISD::SHUF128, DL, VT, Ops[0], Ops[1], in lowerV4X128Shuffle()
19000 MVT VT, SDValue V1, SDValue V2, in lower512BitShuffle() argument
19014 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lower512BitShuffle()
19019 lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lower512BitShuffle()
19023 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, Mask, in lower512BitShuffle()
19027 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) { in lower512BitShuffle()
19030 if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable, in lower512BitShuffle()
19033 if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG)) in lower512BitShuffle()
19036 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG); in lower512BitShuffle()
19039 if (VT == MVT::v32f16) { in lower512BitShuffle()
19050 switch (VT.SimpleTy) { in lower512BitShuffle()
19070 MVT VT, SDValue V1, SDValue V2, in lower1BitShuffleAsKSHIFTR() argument
19100 MVT WideVT = VT; in lower1BitShuffleAsKSHIFTR()
19108 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in lower1BitShuffleAsKSHIFTR()
19150 MVT VT, SDValue V1, SDValue V2, in lower1BitShuffle() argument
19187 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in lower1BitShuffle()
19188 DAG.getConstant(0, DL, VT), in lower1BitShuffle()
19193 if (SDValue Shift = lower1BitShuffleAsKSHIFTR(DL, Mask, VT, V1, V2, Subtarget, in lower1BitShuffle()
19203 MVT WideVT = VT; in lower1BitShuffle()
19210 if (Opcode == X86ISD::KSHIFTR && WideVT != VT) { in lower1BitShuffle()
19221 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in lower1BitShuffle()
19236 DL, VT, DAG.getVectorShuffle(OpVT, DL, Op0, DAG.getUNDEF(OpVT), Mask), in lower1BitShuffle()
19241 switch (VT.SimpleTy) { in lower1BitShuffle()
19280 int NumElems = VT.getVectorNumElements(); in lower1BitShuffle()
19283 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, ExtVT), in lower1BitShuffle()
19286 return DAG.getNode(ISD::TRUNCATE, DL, VT, Shuffle); in lower1BitShuffle()
19372 MVT VT = Op.getSimpleValueType(); in lowerVECTOR_SHUFFLE() local
19373 int NumElements = VT.getVectorNumElements(); in lowerVECTOR_SHUFFLE()
19375 bool Is1BitVector = (VT.getVectorElementType() == MVT::i1); in lowerVECTOR_SHUFFLE()
19377 assert((VT.getSizeInBits() != 64 || Is1BitVector) && in lowerVECTOR_SHUFFLE()
19383 return DAG.getUNDEF(VT); in lowerVECTOR_SHUFFLE()
19400 return DAG.getVectorShuffle(VT, DL, V1, V2, NewMask); in lowerVECTOR_SHUFFLE()
19418 return getZeroVector(VT, Subtarget, DAG, DL); in lowerVECTOR_SHUFFLE()
19427 if (VT.getScalarSizeInBits() < 64 && !Is1BitVector && in lowerVECTOR_SHUFFLE()
19433 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, OrigMask, in lowerVECTOR_SHUFFLE()
19437 MVT NewEltVT = VT.isFloatingPoint() in lowerVECTOR_SHUFFLE()
19438 ? MVT::getFloatingPointVT(VT.getScalarSizeInBits() * 2) in lowerVECTOR_SHUFFLE()
19439 : MVT::getIntegerVT(VT.getScalarSizeInBits() * 2); in lowerVECTOR_SHUFFLE()
19464 VT, DAG.getVectorShuffle(NewVT, DL, V1, V2, WidenedMask)); in lowerVECTOR_SHUFFLE()
19474 Ops, Mask, VT.getSizeInBits(), DL, DAG, Subtarget)) in lowerVECTOR_SHUFFLE()
19475 return DAG.getBitcast(VT, HOp); in lowerVECTOR_SHUFFLE()
19477 V1 = DAG.getBitcast(VT, Ops[0]); in lowerVECTOR_SHUFFLE()
19478 V2 = DAG.getBitcast(VT, Ops[1]); in lowerVECTOR_SHUFFLE()
19490 if (VT.is128BitVector()) in lowerVECTOR_SHUFFLE()
19491 return lower128BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG); in lowerVECTOR_SHUFFLE()
19493 if (VT.is256BitVector()) in lowerVECTOR_SHUFFLE()
19494 return lower256BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG); in lowerVECTOR_SHUFFLE()
19496 if (VT.is512BitVector()) in lowerVECTOR_SHUFFLE()
19497 return lower512BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG); in lowerVECTOR_SHUFFLE()
19500 return lower1BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG); in lowerVECTOR_SHUFFLE()
19512 MVT VT = Op.getSimpleValueType(); in lowerVSELECTtoVectorShuffle() local
19519 return DAG.getVectorShuffle(VT, SDLoc(Op), LHS, RHS, Mask); in lowerVSELECTtoVectorShuffle()
19531 MVT VT = Op.getSimpleValueType(); in LowerVSELECT() local
19532 if (isSoftFP16(VT)) { in LowerVSELECT()
19533 MVT NVT = VT.changeVectorElementTypeToInteger(); in LowerVSELECT()
19534 return DAG.getBitcast(VT, DAG.getNode(ISD::VSELECT, dl, NVT, Cond, in LowerVSELECT()
19562 unsigned EltSize = VT.getScalarSizeInBits(); in LowerVSELECT()
19563 unsigned NumElts = VT.getVectorNumElements(); in LowerVSELECT()
19566 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) in LowerVSELECT()
19572 if (VT.getSizeInBits() == 512) { in LowerVSELECT()
19579 return DAG.getSelect(dl, VT, Mask, LHS, RHS); in LowerVSELECT()
19591 return DAG.getNode(ISD::VSELECT, dl, VT, Cond, LHS, RHS); in LowerVSELECT()
19597 switch (VT.SimpleTy) { in LowerVSELECT()
19617 return DAG.getBitcast(VT, Select); in LowerVSELECT()
19623 MVT VT = Op.getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT_SSE4() local
19632 if (VT.getSizeInBits() == 8) { in LowerEXTRACT_VECTOR_ELT_SSE4()
19644 return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract); in LowerEXTRACT_VECTOR_ELT_SSE4()
19647 if (VT == MVT::f32) { in LowerEXTRACT_VECTOR_ELT_SSE4()
19665 if (VT == MVT::i32 || VT == MVT::i64) in LowerEXTRACT_VECTOR_ELT_SSE4()
19787 MVT VT = Op.getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT() local
19789 if (VT == MVT::i16) { in LowerEXTRACT_VECTOR_ELT()
19804 return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract); in LowerEXTRACT_VECTOR_ELT()
19814 if (VT.getSizeInBits() == 8 && Op->isOnlyUserOf(Vec.getNode())) { in LowerEXTRACT_VECTOR_ELT()
19825 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in LowerEXTRACT_VECTOR_ELT()
19836 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in LowerEXTRACT_VECTOR_ELT()
19839 if (VT == MVT::f16 || VT.getSizeInBits() == 32) { in LowerEXTRACT_VECTOR_ELT()
19847 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec, in LowerEXTRACT_VECTOR_ELT()
19851 if (VT.getSizeInBits() == 64) { in LowerEXTRACT_VECTOR_ELT()
19863 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec, in LowerEXTRACT_VECTOR_ELT()
19899 MVT VT = Op.getSimpleValueType(); in LowerINSERT_VECTOR_ELT() local
19900 MVT EltVT = VT.getVectorElementType(); in LowerINSERT_VECTOR_ELT()
19901 unsigned NumElts = VT.getVectorNumElements(); in LowerINSERT_VECTOR_ELT()
19919 (Subtarget.hasSSE41() && VT.isFloatingPoint()))) in LowerINSERT_VECTOR_ELT()
19929 SDValue EltSplat = DAG.getSplatBuildVector(VT, dl, N1); in LowerINSERT_VECTOR_ELT()
19946 bool IsAllOnesElt = VT.isInteger() && llvm::isAllOnesConstant(N1); in LowerINSERT_VECTOR_ELT()
19952 ((VT == MVT::v16i8 && !Subtarget.hasSSE41()) || in LowerINSERT_VECTOR_ELT()
19953 ((VT == MVT::v32i8 || VT == MVT::v16i16) && !Subtarget.hasInt256()))) { in LowerINSERT_VECTOR_ELT()
19954 SDValue ZeroCst = DAG.getConstant(0, dl, VT.getScalarType()); in LowerINSERT_VECTOR_ELT()
19955 SDValue OnesCst = DAG.getAllOnesConstant(dl, VT.getScalarType()); in LowerINSERT_VECTOR_ELT()
19958 SDValue CstVector = DAG.getBuildVector(VT, dl, CstVectorElts); in LowerINSERT_VECTOR_ELT()
19959 return DAG.getNode(ISD::OR, dl, VT, N0, CstVector); in LowerINSERT_VECTOR_ELT()
19964 (EltSizeInBits >= 16 || (IsZeroElt && !VT.is128BitVector()))) { in LowerINSERT_VECTOR_ELT()
19968 SDValue CstVector = IsZeroElt ? getZeroVector(VT, Subtarget, DAG, dl) in LowerINSERT_VECTOR_ELT()
19969 : getOnesVector(VT, DAG, dl); in LowerINSERT_VECTOR_ELT()
19970 return DAG.getVectorShuffle(VT, dl, N0, CstVector, BlendMask); in LowerINSERT_VECTOR_ELT()
19976 if (VT.is256BitVector() || VT.is512BitVector()) { in LowerINSERT_VECTOR_ELT()
19979 if (VT.is256BitVector() && IdxVal == 0) { in LowerINSERT_VECTOR_ELT()
19985 SDValue N1Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1); in LowerINSERT_VECTOR_ELT()
19986 return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1Vec, in LowerINSERT_VECTOR_ELT()
20002 SDValue N1SplatVec = DAG.getSplatBuildVector(VT, dl, N1); in LowerINSERT_VECTOR_ELT()
20006 return DAG.getVectorShuffle(VT, dl, N0, N1SplatVec, BlendMask); in LowerINSERT_VECTOR_ELT()
20022 assert(VT.is128BitVector() && "Only 128-bit vector types should be left!"); in LowerINSERT_VECTOR_ELT()
20028 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1); in LowerINSERT_VECTOR_ELT()
20036 MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32); in LowerINSERT_VECTOR_ELT()
20039 return DAG.getBitcast(VT, N1); in LowerINSERT_VECTOR_ELT()
20045 if (VT == MVT::v8i16 || (VT == MVT::v16i8 && Subtarget.hasSSE41())) { in LowerINSERT_VECTOR_ELT()
20047 if (VT == MVT::v8i16) { in LowerINSERT_VECTOR_ELT()
20051 assert(VT == MVT::v16i8 && "PINSRB requires v16i8 vector"); in LowerINSERT_VECTOR_ELT()
20059 return DAG.getNode(Opc, dl, VT, N0, N1, N2); in LowerINSERT_VECTOR_ELT()
20083 return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1, in LowerINSERT_VECTOR_ELT()
20088 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, in LowerINSERT_VECTOR_ELT()
20679 MVT VT = Op.getSimpleValueType(); in LowerI64IntToFP_AVX512DQ() local
20682 (VT != MVT::f32 && VT != MVT::f64)) in LowerI64IntToFP_AVX512DQ()
20690 MVT VecVT = MVT::getVectorVT(VT, NumElts); in LowerI64IntToFP_AVX512DQ()
20698 SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec, in LowerI64IntToFP_AVX512DQ()
20705 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec, in LowerI64IntToFP_AVX512DQ()
20720 MVT VT = Op.getSimpleValueType(); in LowerI64IntToFP16() local
20722 if (SrcVT != MVT::i64 || Subtarget.is64Bit() || VT != MVT::f16) in LowerI64IntToFP16()
20735 SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec, in LowerI64IntToFP16()
20742 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec, in LowerI64IntToFP16()
20817 MVT VT = CastToFP.getSimpleValueType(); in lowerFPToIntToFP() local
20818 if (CastToInt.getOpcode() != ISD::FP_TO_SINT || VT.isVector()) in lowerFPToIntToFP()
20829 if (!Subtarget.hasSSE2() || (VT != MVT::f32 && VT != MVT::f64) || in lowerFPToIntToFP()
20835 unsigned VTSize = VT.getSizeInBits(); in lowerFPToIntToFP()
20838 MVT VecVT = MVT::getVectorVT(VT, 128 / VTSize); in lowerFPToIntToFP()
20857 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, VCastToFP, ZeroIdx); in lowerFPToIntToFP()
20864 MVT VT = Op->getSimpleValueType(0); in lowerINT_TO_FP_vXi64() local
20875 assert((VT == MVT::v4f32 || VT == MVT::v2f64 || VT == MVT::v4f64) && in lowerINT_TO_FP_vXi64()
20877 MVT WideVT = VT == MVT::v4f32 ? MVT::v8f32 : MVT::v8f64; in lowerINT_TO_FP_vXi64()
20894 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in lowerINT_TO_FP_vXi64()
20904 if (VT != MVT::v4f32 || IsSigned) in lowerINT_TO_FP_vXi64()
20928 SDValue SignCvt = DAG.getBuildVector(VT, DL, SignCvts); in lowerINT_TO_FP_vXi64()
20953 MVT VT = Op.getSimpleValueType(); in promoteXINT_TO_FP() local
20954 MVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32; in promoteXINT_TO_FP()
20960 ISD::STRICT_FP_ROUND, dl, {VT, MVT::Other}, in promoteXINT_TO_FP()
20964 return DAG.getNode(ISD::FP_ROUND, dl, VT, in promoteXINT_TO_FP()
20968 static bool isLegalConversion(MVT VT, bool IsSigned, in isLegalConversion() argument
20970 if (VT == MVT::v4i32 && Subtarget.hasSSE2() && IsSigned) in isLegalConversion()
20972 if (VT == MVT::v8i32 && Subtarget.hasAVX() && IsSigned) in isLegalConversion()
20974 if (Subtarget.hasVLX() && (VT == MVT::v4i32 || VT == MVT::v8i32)) in isLegalConversion()
20977 if (VT == MVT::v16i32) in isLegalConversion()
20979 if (VT == MVT::v8i64 && Subtarget.hasDQI()) in isLegalConversion()
20983 (VT == MVT::v2i64 || VT == MVT::v4i64)) in isLegalConversion()
20995 MVT VT = Op.getSimpleValueType(); in LowerSINT_TO_FP() local
20998 if (isSoftFP16(VT)) in LowerSINT_TO_FP()
21013 if (SrcVT == MVT::v2i32 && VT == MVT::v2f64) { in LowerSINT_TO_FP()
21018 X86ISD::STRICT_CVTSI2P, dl, {VT, MVT::Other}, in LowerSINT_TO_FP()
21021 return DAG.getNode(X86ISD::CVTSI2P, dl, VT, in LowerSINT_TO_FP()
21034 bool UseSSEReg = isScalarFPTypeInSSEReg(VT); in LowerSINT_TO_FP()
21049 if (SrcVT == MVT::i16 && (UseSSEReg || VT == MVT::f128)) { in LowerSINT_TO_FP()
21052 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in LowerSINT_TO_FP()
21055 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, Ext); in LowerSINT_TO_FP()
21058 if (VT == MVT::f128) in LowerSINT_TO_FP()
21078 BuildFILD(VT, SrcVT, dl, Chain, StackSlot, MPI, Alignment, DAG); in LowerSINT_TO_FP()
21321 MVT VT = Op->getSimpleValueType(0); in lowerUINT_TO_FP_vXi32() local
21324 if (VT == MVT::v8f64) in lowerUINT_TO_FP_vXi32()
21327 assert((VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v4f64) && in lowerUINT_TO_FP_vXi32()
21329 MVT WideVT = VT == MVT::v4f64 ? MVT::v8f64 : MVT::v16f32; in lowerUINT_TO_FP_vXi32()
21330 MVT WideIntVT = VT == MVT::v4f64 ? MVT::v8i32 : MVT::v16i32; in lowerUINT_TO_FP_vXi32()
21346 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in lowerUINT_TO_FP_vXi32()
21794 MVT VT = Op.getSimpleValueType(); in LowerAVXExtend() local
21800 assert(VT.isVector() && InVT.isVector() && "Expected vector type"); in LowerAVXExtend()
21803 assert(VT.getVectorNumElements() == InVT.getVectorNumElements() && in LowerAVXExtend()
21805 assert((VT.getVectorElementType() == MVT::i16 || in LowerAVXExtend()
21806 VT.getVectorElementType() == MVT::i32 || in LowerAVXExtend()
21807 VT.getVectorElementType() == MVT::i64) && in LowerAVXExtend()
21816 if (VT == MVT::v32i16 && !Subtarget.hasBWI()) { in LowerAVXExtend()
21836 MVT HalfVT = VT.getHalfNumVectorElementsVT(); in LowerAVXExtend()
21843 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpLo); in LowerAVXExtend()
21851 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi); in LowerAVXExtend()
21855 static SDValue SplitAndExtendv16i1(unsigned ExtOpc, MVT VT, SDValue In, in SplitAndExtendv16i1() argument
21857 assert((VT == MVT::v16i8 || VT == MVT::v16i16) && "Unexpected VT."); in SplitAndExtendv16i1()
21865 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in SplitAndExtendv16i1()
21871 MVT VT = Op->getSimpleValueType(0); in LowerZERO_EXTEND_Mask() local
21876 unsigned NumElts = VT.getVectorNumElements(); in LowerZERO_EXTEND_Mask()
21880 if (VT.getVectorElementType() != MVT::i8) { in LowerZERO_EXTEND_Mask()
21881 SDValue Extend = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, In); in LowerZERO_EXTEND_Mask()
21882 return DAG.getNode(ISD::SRL, DL, VT, Extend, in LowerZERO_EXTEND_Mask()
21883 DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT)); in LowerZERO_EXTEND_Mask()
21887 MVT ExtVT = VT; in LowerZERO_EXTEND_Mask()
21891 return SplitAndExtendv16i1(ISD::ZERO_EXTEND, VT, In, DL, DAG); in LowerZERO_EXTEND_Mask()
21913 if (VT != ExtVT) { in LowerZERO_EXTEND_Mask()
21919 if (WideVT != VT) in LowerZERO_EXTEND_Mask()
21920 SelectedVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SelectedVal, in LowerZERO_EXTEND_Mask()
22051 MVT VT = Op.getSimpleValueType(); in LowerTruncateVecI1() local
22055 assert(VT.getVectorElementType() == MVT::i1 && "Unexpected vector type."); in LowerTruncateVecI1()
22071 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT), in LowerTruncateVecI1()
22105 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); in LowerTruncateVecI1()
22124 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT), In, ISD::SETGT); in LowerTruncateVecI1()
22125 return DAG.getSetCC(DL, VT, In, DAG.getConstant(0, DL, InVT), ISD::SETNE); in LowerTruncateVecI1()
22130 MVT VT = Op.getSimpleValueType(); in LowerTRUNCATE() local
22135 assert(VT.getVectorNumElements() == InVT.getVectorNumElements() && in LowerTRUNCATE()
22142 VT.is128BitVector()) { in LowerTRUNCATE()
22152 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in LowerTRUNCATE()
22156 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); in LowerTRUNCATE()
22163 if (VT.getVectorElementType() == MVT::i1) in LowerTRUNCATE()
22169 assert(VT == MVT::v32i8 && "Unexpected VT!"); in LowerTRUNCATE()
22182 unsigned NumPackedSignBits = std::min<unsigned>(VT.getScalarSizeInBits(), 16); in LowerTRUNCATE()
22191 truncateVectorWithPACK(X86ISD::PACKUS, VT, In, DL, DAG, Subtarget)) in LowerTRUNCATE()
22198 truncateVectorWithPACK(X86ISD::PACKSS, VT, In, DL, DAG, Subtarget)) in LowerTRUNCATE()
22202 assert(VT.is128BitVector() && InVT.is256BitVector() && "Unexpected types!"); in LowerTRUNCATE()
22204 if ((VT == MVT::v4i32) && (InVT == MVT::v4i64)) { in LowerTRUNCATE()
22210 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, In, in LowerTRUNCATE()
22219 return DAG.getVectorShuffle(VT, DL, DAG.getBitcast(MVT::v4i32, OpLo), in LowerTRUNCATE()
22223 if ((VT == MVT::v8i16) && (InVT == MVT::v8i32)) { in LowerTRUNCATE()
22265 if (VT == MVT::v16i8 && InVT == MVT::v16i16) { in LowerTRUNCATE()
22273 return DAG.getNode(X86ISD::PACKUS, DL, VT, InLo, InHi); in LowerTRUNCATE()
22281 static SDValue expandFP_TO_UINT_SSE(MVT VT, SDValue Src, const SDLoc &dl, in expandFP_TO_UINT_SSE() argument
22285 unsigned DstBits = VT.getScalarSizeInBits(); in expandFP_TO_UINT_SSE()
22290 SDValue Small = DAG.getNode(X86ISD::CVTTP2SI, dl, VT, Src); in expandFP_TO_UINT_SSE()
22292 DAG.getNode(X86ISD::CVTTP2SI, dl, VT, in expandFP_TO_UINT_SSE()
22305 if (VT == MVT::v8i32 && !Subtarget.hasAVX2()) { in expandFP_TO_UINT_SSE()
22306 SDValue Overflow = DAG.getNode(ISD::OR, dl, VT, Small, Big); in expandFP_TO_UINT_SSE()
22307 return DAG.getNode(X86ISD::BLENDV, dl, VT, Small, Overflow, Small); in expandFP_TO_UINT_SSE()
22311 DAG.getNode(X86ISD::VSRAI, dl, VT, Small, in expandFP_TO_UINT_SSE()
22313 return DAG.getNode(ISD::OR, dl, VT, Small, in expandFP_TO_UINT_SSE()
22314 DAG.getNode(ISD::AND, dl, VT, Big, IsOverflown)); in expandFP_TO_UINT_SSE()
22321 MVT VT = Op->getSimpleValueType(0); in LowerFP_TO_INT() local
22329 MVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32; in LowerFP_TO_INT()
22331 return DAG.getNode(Op.getOpcode(), dl, {VT, MVT::Other}, in LowerFP_TO_INT()
22334 return DAG.getNode(Op.getOpcode(), dl, VT, in LowerFP_TO_INT()
22336 } else if (isTypeLegal(SrcVT) && isLegalConversion(VT, IsSigned, Subtarget)) { in LowerFP_TO_INT()
22340 if (VT.isVector()) { in LowerFP_TO_INT()
22341 if (VT == MVT::v2i1 && SrcVT == MVT::v2f64) { in LowerFP_TO_INT()
22380 if (VT == MVT::v8i16 || VT == MVT::v16i16 || VT == MVT::v32i16) in LowerFP_TO_INT()
22383 MVT ResVT = VT; in LowerFP_TO_INT()
22384 MVT EleVT = VT.getVectorElementType(); in LowerFP_TO_INT()
22412 if (ResVT != VT) in LowerFP_TO_INT()
22413 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res, in LowerFP_TO_INT()
22421 if (VT == MVT::v8i16 && (SrcVT == MVT::v8f32 || SrcVT == MVT::v8f64)) { in LowerFP_TO_INT()
22441 if (VT == MVT::v8i32 && SrcVT == MVT::v8f64) { in LowerFP_TO_INT()
22448 if ((VT == MVT::v4i32 || VT == MVT::v8i32) && in LowerFP_TO_INT()
22471 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res, in LowerFP_TO_INT()
22480 if ((VT == MVT::v2i64 || VT == MVT::v4i64) && in LowerFP_TO_INT()
22501 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res, in LowerFP_TO_INT()
22509 if (VT == MVT::v2i64 && SrcVT == MVT::v2f32) { in LowerFP_TO_INT()
22533 return DAG.getNode(Opc, dl, {VT, MVT::Other}, {Op->getOperand(0), Tmp}); in LowerFP_TO_INT()
22536 return DAG.getNode(Opc, dl, VT, Tmp); in LowerFP_TO_INT()
22541 if ((VT == MVT::v4i32 && SrcVT == MVT::v4f32) || in LowerFP_TO_INT()
22542 (VT == MVT::v4i32 && SrcVT == MVT::v4f64) || in LowerFP_TO_INT()
22543 (VT == MVT::v8i32 && SrcVT == MVT::v8f32)) { in LowerFP_TO_INT()
22545 return expandFP_TO_UINT_SSE(VT, Src, dl, DAG, Subtarget); in LowerFP_TO_INT()
22551 assert(!VT.isVector()); in LowerFP_TO_INT()
22562 if (!IsStrict && ((VT == MVT::i32 && !Subtarget.is64Bit()) || in LowerFP_TO_INT()
22563 (VT == MVT::i64 && Subtarget.is64Bit()))) { in LowerFP_TO_INT()
22564 unsigned DstBits = VT.getScalarSizeInBits(); in LowerFP_TO_INT()
22567 DAG.getConstant(UIntLimit, dl, VT)); in LowerFP_TO_INT()
22574 DAG.getNode(X86ISD::CVTTS2SI, dl, VT, in LowerFP_TO_INT()
22577 X86ISD::CVTTS2SI, dl, VT, in LowerFP_TO_INT()
22588 ISD::SRA, dl, VT, Small, DAG.getConstant(DstBits - 1, dl, MVT::i8)); in LowerFP_TO_INT()
22589 return DAG.getNode(ISD::OR, dl, VT, Small, in LowerFP_TO_INT()
22590 DAG.getNode(ISD::AND, dl, VT, Big, IsOverflown)); in LowerFP_TO_INT()
22594 if (VT == MVT::i64) in LowerFP_TO_INT()
22597 assert(VT == MVT::i32 && "Unexpected VT!"); in LowerFP_TO_INT()
22610 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in LowerFP_TO_INT()
22625 if (VT == MVT::i16 && (UseSSEReg || SrcVT == MVT::f128)) { in LowerFP_TO_INT()
22634 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in LowerFP_TO_INT()
22648 LC = RTLIB::getFPTOSINT(SrcVT, VT); in LowerFP_TO_INT()
22650 LC = RTLIB::getFPTOUINT(SrcVT, VT); in LowerFP_TO_INT()
22653 std::pair<SDValue, SDValue> Tmp = makeLibCall(DAG, LC, VT, Src, CallOptions, in LowerFP_TO_INT()
22885 MVT VT = Op.getSimpleValueType(); in LowerFP_EXTEND() local
22890 if (VT == MVT::f128 || (SVT == MVT::f16 && VT == MVT::f80)) in LowerFP_EXTEND()
22897 if (VT != MVT::f32) { in LowerFP_EXTEND()
22900 ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, in LowerFP_EXTEND()
22904 return DAG.getNode(ISD::FP_EXTEND, DL, VT, in LowerFP_EXTEND()
22942 return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other}, in LowerFP_EXTEND()
22944 return DAG.getNode(X86ISD::VFPEXT, DL, VT, Res); in LowerFP_EXTEND()
22945 } else if (VT == MVT::v4f64 || VT == MVT::v8f64) { in LowerFP_EXTEND()
22954 return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other}, in LowerFP_EXTEND()
22956 return DAG.getNode(X86ISD::VFPEXT, DL, VT, Res); in LowerFP_EXTEND()
22965 MVT VT = Op.getSimpleValueType(); in LowerFP_ROUND() local
22968 if (SVT == MVT::f128 || (VT == MVT::f16 && SVT == MVT::f80)) in LowerFP_ROUND()
22971 if (VT.getScalarType() == MVT::f16 && !Subtarget.hasFP16()) { in LowerFP_ROUND()
22975 if (VT.isVector()) in LowerFP_ROUND()
23167 MVT VT = Op.getSimpleValueType(); in LowerFROUND() local
23170 const fltSemantics &Sem = SelectionDAG::EVTToAPFloatSemantics(VT); in LowerFROUND()
23176 SDValue Adder = DAG.getNode(ISD::FCOPYSIGN, dl, VT, in LowerFROUND()
23177 DAG.getConstantFP(Point5Pred, dl, VT), N0); in LowerFROUND()
23178 N0 = DAG.getNode(ISD::FADD, dl, VT, N0, Adder); in LowerFROUND()
23181 return DAG.getNode(ISD::FTRUNC, dl, VT, N0); in LowerFROUND()
23200 MVT VT = Op.getSimpleValueType(); in LowerFABSorFNEG() local
23202 bool IsF128 = (VT == MVT::f128); in LowerFABSorFNEG()
23203 assert(VT.isFloatingPoint() && VT != MVT::f80 && in LowerFABSorFNEG()
23204 DAG.getTargetLoweringInfo().isTypeLegal(VT) && in LowerFABSorFNEG()
23215 bool IsFakeVector = !VT.isVector() && !IsF128; in LowerFABSorFNEG()
23216 MVT LogicVT = VT; in LowerFABSorFNEG()
23218 LogicVT = (VT == MVT::f64) ? MVT::v2f64 in LowerFABSorFNEG()
23219 : (VT == MVT::f32) ? MVT::v4f32 in LowerFABSorFNEG()
23222 unsigned EltBits = VT.getScalarSizeInBits(); in LowerFABSorFNEG()
23226 const fltSemantics &Sem = SelectionDAG::EVTToAPFloatSemantics(VT); in LowerFABSorFNEG()
23236 if (VT.isVector() || IsF128) in LowerFABSorFNEG()
23243 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, LogicNode, in LowerFABSorFNEG()
23253 MVT VT = Op.getSimpleValueType(); in LowerFCOPYSIGN() local
23254 if (Sign.getSimpleValueType().bitsLT(VT)) in LowerFCOPYSIGN()
23255 Sign = DAG.getNode(ISD::FP_EXTEND, dl, VT, Sign); in LowerFCOPYSIGN()
23258 if (Sign.getSimpleValueType().bitsGT(VT)) in LowerFCOPYSIGN()
23260 DAG.getNode(ISD::FP_ROUND, dl, VT, Sign, DAG.getIntPtrConstant(0, dl)); in LowerFCOPYSIGN()
23264 bool IsF128 = (VT == MVT::f128); in LowerFCOPYSIGN()
23265 assert(VT.isFloatingPoint() && VT != MVT::f80 && in LowerFCOPYSIGN()
23266 DAG.getTargetLoweringInfo().isTypeLegal(VT) && in LowerFCOPYSIGN()
23269 const fltSemantics &Sem = SelectionDAG::EVTToAPFloatSemantics(VT); in LowerFCOPYSIGN()
23276 bool IsFakeVector = !VT.isVector() && !IsF128; in LowerFCOPYSIGN()
23277 MVT LogicVT = VT; in LowerFCOPYSIGN()
23279 LogicVT = (VT == MVT::f64) ? MVT::v2f64 in LowerFCOPYSIGN()
23280 : (VT == MVT::f32) ? MVT::v4f32 in LowerFCOPYSIGN()
23284 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in LowerFCOPYSIGN()
23312 return !IsFakeVector ? Or : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Or, in LowerFCOPYSIGN()
23319 MVT VT = Op.getSimpleValueType(); in LowerFGETSIGN() local
23329 Res = DAG.getZExtOrTrunc(Res, dl, VT); in LowerFGETSIGN()
23330 Res = DAG.getNode(ISD::AND, dl, VT, Res, DAG.getConstant(1, dl, VT)); in LowerFGETSIGN()
23391 EVT VT = MVT::Other; in matchScalarReduction() local
23423 VT = Src.getValueType(); in matchScalarReduction()
23425 if (!SrcOpMap.empty() && VT != SrcOpMap.begin()->first.getValueType()) in matchScalarReduction()
23427 unsigned NumElts = VT.getVectorNumElements(); in matchScalarReduction()
23459 EVT VT = V.getValueType(); in LowerVectorAllZero() local
23460 unsigned ScalarSize = VT.getScalarSizeInBits(); in LowerVectorAllZero()
23478 if (VT.getSizeInBits() < 128) { in LowerVectorAllZero()
23479 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); in LowerVectorAllZero()
23488 if (!isPowerOf2_32(VT.getSizeInBits())) in LowerVectorAllZero()
23493 while (VT.getSizeInBits() > TestSize) { in LowerVectorAllZero()
23495 VT = Split.first.getValueType(); in LowerVectorAllZero()
23496 V = DAG.getNode(ISD::OR, DL, VT, Split.first, Split.second); in LowerVectorAllZero()
23501 MVT TestVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64; in LowerVectorAllZero()
23508 if (!Mask.isAllOnes() && VT.getScalarSizeInBits() > 32) in LowerVectorAllZero()
23552 EVT VT = VecIns[0].getValueType(); in MatchVectorAllZeroTest() local
23554 [VT](SDValue V) { return VT == V.getValueType(); }) && in MatchVectorAllZeroTest()
23558 if (VT.getSizeInBits() < 128 || !isPowerOf2_32(VT.getSizeInBits())) in MatchVectorAllZeroTest()
23568 VecIns.push_back(DAG.getNode(ISD::OR, DL, VT, LHS, RHS)); in MatchVectorAllZeroTest()
23814 EVT VT = Op.getValueType(); in isFsqrtCheap() local
23817 if (VT.getScalarType() == MVT::f16) in isFsqrtCheap()
23821 if (DAG.doesNodeExist(X86ISD::FRSQRT, DAG.getVTList(VT), Op)) in isFsqrtCheap()
23824 if (VT.isVector()) in isFsqrtCheap()
23837 EVT VT = Op.getValueType(); in getSqrtEstimate() local
23847 if ((VT == MVT::f32 && Subtarget.hasSSE1()) || in getSqrtEstimate()
23848 (VT == MVT::v4f32 && Subtarget.hasSSE1() && Reciprocal) || in getSqrtEstimate()
23849 (VT == MVT::v4f32 && Subtarget.hasSSE2() && !Reciprocal) || in getSqrtEstimate()
23850 (VT == MVT::v8f32 && Subtarget.hasAVX()) || in getSqrtEstimate()
23851 (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) { in getSqrtEstimate()
23857 unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RSQRT14 : X86ISD::FRSQRT; in getSqrtEstimate()
23858 SDValue Estimate = DAG.getNode(Opcode, DL, VT, Op); in getSqrtEstimate()
23860 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Op, Estimate); in getSqrtEstimate()
23864 if (VT.getScalarType() == MVT::f16 && isTypeLegal(VT) && in getSqrtEstimate()
23870 if (VT == MVT::f16) { in getSqrtEstimate()
23878 return DAG.getNode(X86ISD::RSQRT14, DL, VT, Op); in getSqrtEstimate()
23889 EVT VT = Op.getValueType(); in getRecipEstimate() local
23898 if ((VT == MVT::f32 && Subtarget.hasSSE1()) || in getRecipEstimate()
23899 (VT == MVT::v4f32 && Subtarget.hasSSE1()) || in getRecipEstimate()
23900 (VT == MVT::v8f32 && Subtarget.hasAVX()) || in getRecipEstimate()
23901 (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) { in getRecipEstimate()
23905 if (VT == MVT::f32 && Enabled == ReciprocalEstimate::Unspecified) in getRecipEstimate()
23912 unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RCP14 : X86ISD::FRCP; in getRecipEstimate()
23913 return DAG.getNode(Opcode, DL, VT, Op); in getRecipEstimate()
23916 if (VT.getScalarType() == MVT::f16 && isTypeLegal(VT) && in getRecipEstimate()
23921 if (VT == MVT::f16) { in getRecipEstimate()
23929 return DAG.getNode(X86ISD::RCP14, DL, VT, Op); in getRecipEstimate()
23961 EVT VT = N->getValueType(0); in BuildSDIVPow2() local
23963 if (VT != MVT::i16 && VT != MVT::i32 && in BuildSDIVPow2()
23964 !(Subtarget.is64Bit() && VT == MVT::i64)) in BuildSDIVPow2()
23975 SDValue Zero = DAG.getConstant(0, DL, VT); in BuildSDIVPow2()
23976 APInt Lg2Mask = APInt::getLowBitsSet(VT.getSizeInBits(), Lg2); in BuildSDIVPow2()
23977 SDValue Pow2MinusOne = DAG.getConstant(Lg2Mask, DL, VT); in BuildSDIVPow2()
23981 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne); in BuildSDIVPow2()
23982 SDValue CMov = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0); in BuildSDIVPow2()
23990 DAG.getNode(ISD::SRA, DL, VT, CMov, DAG.getConstant(Lg2, DL, MVT::i8)); in BuildSDIVPow2()
23998 return DAG.getNode(ISD::SUB, DL, VT, Zero, SRA); in BuildSDIVPow2()
24138 static SDValue splitIntVSETCC(EVT VT, SDValue LHS, SDValue RHS, in splitIntVSETCC() argument
24141 assert(VT.isInteger() && VT == LHS.getValueType() && in splitIntVSETCC()
24142 VT == RHS.getValueType() && "Unsupported VTs!"); in splitIntVSETCC()
24156 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in splitIntVSETCC()
24157 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, in splitIntVSETCC()
24167 MVT VT = Op.getSimpleValueType(); in LowerIntVSETCC_AVX512() local
24170 assert(VT.getVectorElementType() == MVT::i1 && in LowerIntVSETCC_AVX512()
24181 return DAG.getSetCC(dl, VT, Op0, Op1, SetCCOpcode); in LowerIntVSETCC_AVX512()
24193 MVT VT = V.getSimpleValueType(); in incDecVectorConstant() local
24194 MVT EltVT = VT.getVectorElementType(); in incDecVectorConstant()
24195 unsigned NumElts = VT.getVectorNumElements(); in incDecVectorConstant()
24211 return DAG.getBuildVector(VT, DL, NewVecC); in incDecVectorConstant()
24218 static SDValue LowerVSETCCWithSUBUS(SDValue Op0, SDValue Op1, MVT VT, in LowerVSETCCWithSUBUS() argument
24225 MVT VET = VT.getVectorElementType(); in LowerVSETCCWithSUBUS()
24266 SDValue Result = DAG.getNode(ISD::USUBSAT, dl, VT, Op0, Op1); in LowerVSETCCWithSUBUS()
24267 return DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result, in LowerVSETCCWithSUBUS()
24268 DAG.getConstant(0, dl, VT)); in LowerVSETCCWithSUBUS()
24278 MVT VT = Op->getSimpleValueType(0); in LowerVSETCC() local
24297 if (Subtarget.hasAVX512() && VT.getVectorElementType() == MVT::i1 && in LowerVSETCC()
24301 unsigned Num = VT.getVectorNumElements(); in LowerVSETCC()
24310 VT = Op0.getSimpleValueType(); in LowerVSETCC()
24330 Opc, dl, {VT, MVT::Other}, in LowerVSETCC()
24360 Opc, dl, {VT, MVT::Other}, in LowerVSETCC()
24363 Opc, dl, {VT, MVT::Other}, in LowerVSETCC()
24369 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC0, dl, MVT::i8)); in LowerVSETCC()
24371 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8)); in LowerVSETCC()
24373 Cmp = DAG.getNode(CombineOpc, dl, VT, Cmp0, Cmp1); in LowerVSETCC()
24377 Opc, dl, {VT, MVT::Other}, in LowerVSETCC()
24382 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)); in LowerVSETCC()
24390 Opc, dl, {VT, MVT::Other}, in LowerVSETCC()
24395 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)); in LowerVSETCC()
24398 if (VT.getFixedSizeInBits() > in LowerVSETCC()
24402 EVT CastVT = EVT(VT).changeVectorElementTypeToInteger(); in LowerVSETCC()
24425 assert(VT.getVectorNumElements() == VTOp0.getVectorNumElements() && in LowerVSETCC()
24430 assert((Subtarget.hasAVX512() || (VT == VTOp0)) && in LowerVSETCC()
24434 if (VT.getVectorElementType() == MVT::i1) { in LowerVSETCC()
24443 if (VT.is128BitVector() && Subtarget.hasXOP()) { in LowerVSETCC()
24464 return DAG.getNode(Opc, dl, VT, Op0, Op1, in LowerVSETCC()
24476 VT.getScalarSizeInBits(), UndefElts, in LowerVSETCC()
24480 Op1 = DAG.getBitcast(VT, BC0.getOperand(1)); in LowerVSETCC()
24491 unsigned BitWidth = VT.getScalarSizeInBits(); in LowerVSETCC()
24495 Result = DAG.getNode(ISD::SHL, dl, VT, Result, in LowerVSETCC()
24496 DAG.getConstant(ShiftAmt, dl, VT)); in LowerVSETCC()
24497 Result = DAG.getNode(ISD::SRA, dl, VT, Result, in LowerVSETCC()
24498 DAG.getConstant(BitWidth - 1, dl, VT)); in LowerVSETCC()
24504 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerVSETCC()
24505 return splitIntVSETCC(VT, Op0, Op1, Cond, DAG, dl); in LowerVSETCC()
24509 if (VT.is512BitVector()) in LowerVSETCC()
24510 return splitIntVSETCC(VT, Op0, Op1, Cond, DAG, dl); in LowerVSETCC()
24539 TLI.isOperationLegal(ISD::UMIN, VT)) { in LowerVSETCC()
24566 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1); in LowerVSETCC()
24567 Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result); in LowerVSETCC()
24571 Result = DAG.getNOT(dl, Result, VT); in LowerVSETCC()
24579 LowerVSETCCWithSUBUS(Op0, Op1, VT, Cond, dl, Subtarget, DAG)) in LowerVSETCC()
24597 if (VT == MVT::v2i64) { in LowerVSETCC()
24611 return DAG.getBitcast(VT, Result); in LowerVSETCC()
24622 return DAG.getBitcast(VT, Result); in LowerVSETCC()
24656 return DAG.getBitcast(VT, Result); in LowerVSETCC()
24679 return DAG.getBitcast(VT, Result); in LowerVSETCC()
24686 MVT EltVT = VT.getVectorElementType(); in LowerVSETCC()
24688 VT); in LowerVSETCC()
24689 Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SM); in LowerVSETCC()
24690 Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SM); in LowerVSETCC()
24693 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1); in LowerVSETCC()
24697 Result = DAG.getNOT(dl, Result, VT); in LowerVSETCC()
24716 MVT VT = Op0.getSimpleValueType(); in EmitAVX512Test() local
24717 if (!(Subtarget.hasAVX512() && VT == MVT::v16i1) && in EmitAVX512Test()
24718 !(Subtarget.hasDQI() && VT == MVT::v8i1) && in EmitAVX512Test()
24719 !(Subtarget.hasBWI() && (VT == MVT::v32i1 || VT == MVT::v64i1))) in EmitAVX512Test()
24733 if (Subtarget.hasDQI() && (VT == MVT::v8i1 || VT == MVT::v16i1)) in EmitAVX512Test()
24735 if (Subtarget.hasBWI() && (VT == MVT::v32i1 || VT == MVT::v64i1)) in EmitAVX512Test()
24837 MVT VT = Op->getSimpleValueType(0); in LowerSETCC() local
24839 if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG); in LowerSETCC()
24841 assert(VT == MVT::i8 && "SetCC type must be 8-bit integer"); in LowerSETCC()
25039 MVT VT = Op1.getSimpleValueType(); in LowerSELECT() local
25042 if (isSoftFP16(VT)) { in LowerSELECT()
25043 MVT NVT = VT.changeTypeToInteger(); in LowerSELECT()
25044 return DAG.getBitcast(VT, DAG.getNode(ISD::SELECT, DL, NVT, Cond, in LowerSELECT()
25052 if (Cond.getOpcode() == ISD::SETCC && isScalarFPTypeInSSEReg(VT) && in LowerSELECT()
25053 VT == Cond.getOperand(0).getSimpleValueType() && Cond->hasOneUse()) { in LowerSELECT()
25064 assert(!VT.isVector() && "Not a scalar type?"); in LowerSELECT()
25065 return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2); in LowerSELECT()
25069 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1, in LowerSELECT()
25088 MVT VecVT = VT == MVT::f32 ? MVT::v4f32 : MVT::v2f64; in LowerSELECT()
25093 MVT VCmpVT = VT == MVT::f32 ? MVT::v4i32 : MVT::v2i64; in LowerSELECT()
25098 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in LowerSELECT()
25101 SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2); in LowerSELECT()
25102 SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1); in LowerSELECT()
25103 return DAG.getNode(X86ISD::FOR, DL, VT, AndN, And); in LowerSELECT()
25108 if (isScalarFPTypeInSSEReg(VT) && Subtarget.hasAVX512()) { in LowerSELECT()
25110 return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2); in LowerSELECT()
25149 if (Subtarget.canUseCMOV() && (VT == MVT::i32 || VT == MVT::i64) && in LowerSELECT()
25173 SDValue SBB = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in LowerSELECT()
25176 return DAG.getNode(ISD::OR, DL, VT, SBB, Y); in LowerSELECT()
25200 if (CmpSz > VT.getSizeInBits()) in LowerSELECT()
25201 Neg = DAG.getNode(ISD::TRUNCATE, DL, VT, CmpOp0); in LowerSELECT()
25202 else if (CmpSz < VT.getSizeInBits()) in LowerSELECT()
25203 Neg = DAG.getNode(ISD::AND, DL, VT, in LowerSELECT()
25204 DAG.getNode(ISD::ANY_EXTEND, DL, VT, CmpOp0.getOperand(0)), in LowerSELECT()
25205 DAG.getConstant(1, DL, VT)); in LowerSELECT()
25208 SDValue Mask = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in LowerSELECT()
25210 SDValue And = DAG.getNode(ISD::AND, DL, VT, Mask, Src1); // Mask & z in LowerSELECT()
25211 return DAG.getNode(Op2.getOpcode(), DL, VT, And, Src2); // And Op y in LowerSELECT()
25213 } else if ((VT == MVT::i32 || VT == MVT::i64) && isNullConstant(Op2) && in LowerSELECT()
25223 unsigned ShCt = VT.getSizeInBits() - 1; in LowerSELECT()
25224 SDValue ShiftAmt = DAG.getConstant(ShCt, DL, VT); in LowerSELECT()
25225 SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, Op1, ShiftAmt); in LowerSELECT()
25227 Shift = DAG.getNOT(DL, Shift, VT); in LowerSELECT()
25228 return DAG.getNode(ISD::AND, DL, VT, Shift, Op1); in LowerSELECT()
25247 if (VT.isFloatingPoint() && !VT.isVector() && in LowerSELECT()
25248 !isScalarFPTypeInSSEReg(VT) && Subtarget.canUseCMOV()) // FPStack? in LowerSELECT()
25348 MVT VT = Op->getSimpleValueType(0); in LowerSIGN_EXTEND_Mask() local
25352 MVT VTElt = VT.getVectorElementType(); in LowerSIGN_EXTEND_Mask()
25355 unsigned NumElts = VT.getVectorNumElements(); in LowerSIGN_EXTEND_Mask()
25358 MVT ExtVT = VT; in LowerSIGN_EXTEND_Mask()
25362 return SplitAndExtendv16i1(Op.getOpcode(), VT, In, dl, DAG); in LowerSIGN_EXTEND_Mask()
25389 if (VT != ExtVT) { in LowerSIGN_EXTEND_Mask()
25395 if (WideVT != VT) in LowerSIGN_EXTEND_Mask()
25396 V = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, V, in LowerSIGN_EXTEND_Mask()
25422 MVT VT = Op->getSimpleValueType(0); in LowerEXTEND_VECTOR_INREG() local
25425 MVT SVT = VT.getVectorElementType(); in LowerEXTEND_VECTOR_INREG()
25433 if (!(VT.is128BitVector() && Subtarget.hasSSE2()) && in LowerEXTEND_VECTOR_INREG()
25434 !(VT.is256BitVector() && Subtarget.hasAVX()) && in LowerEXTEND_VECTOR_INREG()
25435 !(VT.is512BitVector() && Subtarget.hasAVX512())) in LowerEXTEND_VECTOR_INREG()
25440 unsigned NumElts = VT.getVectorNumElements(); in LowerEXTEND_VECTOR_INREG()
25456 assert(VT.getSizeInBits() > 128 && "Unexpected 128-bit vector extension"); in LowerEXTEND_VECTOR_INREG()
25459 return DAG.getNode(Op.getOpcode(), dl, VT, In); in LowerEXTEND_VECTOR_INREG()
25466 return DAG.getNode(ExtOpc, dl, VT, In); in LowerEXTEND_VECTOR_INREG()
25471 assert(VT.is256BitVector() && "256-bit vector expected"); in LowerEXTEND_VECTOR_INREG()
25472 MVT HalfVT = VT.getHalfNumVectorElementsVT(); in LowerEXTEND_VECTOR_INREG()
25483 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in LowerEXTEND_VECTOR_INREG()
25488 assert(VT.is128BitVector() && InVT.is128BitVector() && "Unexpected VTs"); in LowerEXTEND_VECTOR_INREG()
25497 MVT DestVT = VT == MVT::v2i64 ? MVT::v4i32 : VT; in LowerEXTEND_VECTOR_INREG()
25519 if (VT == MVT::v2i64) { in LowerEXTEND_VECTOR_INREG()
25524 SignExt = DAG.getBitcast(VT, SignExt); in LowerEXTEND_VECTOR_INREG()
25532 MVT VT = Op->getSimpleValueType(0); in LowerSIGN_EXTEND() local
25540 assert(VT.isVector() && InVT.isVector() && "Expected vector type"); in LowerSIGN_EXTEND()
25541 assert(VT.getVectorNumElements() == InVT.getVectorNumElements() && in LowerSIGN_EXTEND()
25543 assert((VT.getVectorElementType() == MVT::i16 || in LowerSIGN_EXTEND()
25544 VT.getVectorElementType() == MVT::i32 || in LowerSIGN_EXTEND()
25545 VT.getVectorElementType() == MVT::i64) && in LowerSIGN_EXTEND()
25552 if (VT == MVT::v32i16 && !Subtarget.hasBWI()) { in LowerSIGN_EXTEND()
25568 MVT HalfVT = VT.getHalfNumVectorElementsVT(); in LowerSIGN_EXTEND()
25579 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi); in LowerSIGN_EXTEND()
25925 EVT VT = Node->getValueType(0); in LowerDYNAMIC_STACKALLOC() local
25952 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); in LowerDYNAMIC_STACKALLOC()
25954 Result = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value in LowerDYNAMIC_STACKALLOC()
25958 DAG.getNode(ISD::AND, dl, VT, Result, in LowerDYNAMIC_STACKALLOC()
25959 DAG.getConstant(~(Alignment->value() - 1ULL), dl, VT)); in LowerDYNAMIC_STACKALLOC()
25991 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0), in LowerDYNAMIC_STACKALLOC()
25992 DAG.getConstant(~(Alignment->value() - 1ULL), dl, VT)); in LowerDYNAMIC_STACKALLOC()
26166 static SDValue getTargetVShiftByConstNode(unsigned Opc, const SDLoc &dl, MVT VT, in getTargetVShiftByConstNode() argument
26169 MVT ElementType = VT.getVectorElementType(); in getTargetVShiftByConstNode()
26173 if (VT != SrcOp.getSimpleValueType()) in getTargetVShiftByConstNode()
26174 SrcOp = DAG.getBitcast(VT, SrcOp); in getTargetVShiftByConstNode()
26185 return DAG.getConstant(0, dl, VT); in getTargetVShiftByConstNode()
26208 SDValue Amt = DAG.getConstant(ShiftAmt, dl, VT); in getTargetVShiftByConstNode()
26209 if (SDValue C = DAG.FoldConstantArithmetic(ShiftOpc, dl, VT, {SrcOp, Amt})) in getTargetVShiftByConstNode()
26213 return DAG.getNode(Opc, dl, VT, SrcOp, in getTargetVShiftByConstNode()
26218 static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT, in getTargetVShiftNode() argument
26306 MVT EltVT = VT.getVectorElementType(); in getTargetVShiftNode()
26310 return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt); in getTargetVShiftNode()
26358 MVT VT = Op.getSimpleValueType(); in getVectorMaskingNode() local
26359 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements()); in getVectorMaskingNode()
26369 PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl); in getVectorMaskingNode()
26370 return DAG.getNode(OpcodeSelect, dl, VT, VMask, Op, PreservedSrc); in getVectorMaskingNode()
26389 MVT VT = Op.getSimpleValueType(); in getScalarMaskingNode() local
26399 return DAG.getNode(ISD::AND, dl, VT, Op, IMask); in getScalarMaskingNode()
26402 PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl); in getScalarMaskingNode()
26403 return DAG.getNode(X86ISD::SELECTS, dl, VT, IMask, Op, PreservedSrc); in getScalarMaskingNode()
26507 MVT VT = Op.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN() local
26643 DAG.getNode(IntrData->Opc0, dl, VT, Src), Mask, PassThru, in LowerINTRINSIC_WO_CHAIN()
26660 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src), Mask, PassThru, in LowerINTRINSIC_WO_CHAIN()
26679 DAG.getNode(IntrWithRoundingModeOpcode, dl, VT, Src1, Src2, in LowerINTRINSIC_WO_CHAIN()
26685 return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1, in LowerINTRINSIC_WO_CHAIN()
26701 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, in LowerINTRINSIC_WO_CHAIN()
26715 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2); in LowerINTRINSIC_WO_CHAIN()
26717 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2, in LowerINTRINSIC_WO_CHAIN()
26738 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2), in LowerINTRINSIC_WO_CHAIN()
26751 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2, in LowerINTRINSIC_WO_CHAIN()
26757 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2); in LowerINTRINSIC_WO_CHAIN()
26775 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2), in LowerINTRINSIC_WO_CHAIN()
26793 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2, Src3), in LowerINTRINSIC_WO_CHAIN()
26811 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2, Src3), in LowerINTRINSIC_WO_CHAIN()
26823 return DAG.getNode(IntrData->Opc0, dl, VT, Src3, Src2, Src1); in LowerINTRINSIC_WO_CHAIN()
26830 return DAG.getNode(IntrData->Opc0, dl, VT,Src2, Src1); in LowerINTRINSIC_WO_CHAIN()
26838 MVT VT = Op.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN() local
26842 PassThru = getZeroVector(VT, Subtarget, DAG, dl); in LowerINTRINSIC_WO_CHAIN()
26852 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2, Src3, in LowerINTRINSIC_WO_CHAIN()
26858 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2, Src3); in LowerINTRINSIC_WO_CHAIN()
27012 PassThru = DAG.getConstant(0, dl, VT); in LowerINTRINSIC_WO_CHAIN()
27014 return DAG.getNode(IntrData->Opc0, dl, VT, DataToCompress, PassThru, in LowerINTRINSIC_WO_CHAIN()
27026 : getZeroVector(VT, Subtarget, DAG, dl); in LowerINTRINSIC_WO_CHAIN()
27037 SDValue FixupImm = DAG.getNode(Opc, dl, VT, Src1, Src2, Src3, Imm); in LowerINTRINSIC_WO_CHAIN()
27335 return DAG.getNode(getGlobalWrapperKind(), dl, VT, in LowerINTRINSIC_WO_CHAIN()
27349 SDValue Result = DAG.getMCSymbol(LSDASym, VT); in LowerINTRINSIC_WO_CHAIN()
27350 return DAG.getNode(X86ISD::Wrapper, dl, VT, Result); in LowerINTRINSIC_WO_CHAIN()
27379 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT); in LowerINTRINSIC_WO_CHAIN()
27515 MVT VT = Op.getSimpleValueType(); in getGatherNode() local
27525 VT.getVectorNumElements()); in getGatherNode()
27974 MVT VT = Op.getSimpleValueType(); in LowerINTRINSIC_W_CHAIN() local
27981 SDValue Size = DAG.getConstant(VT.getScalarSizeInBits(), DL, MVT::i32); in LowerINTRINSIC_W_CHAIN()
27985 {Chain, Op1, Op2, Size}, VT, MMO); in LowerINTRINSIC_W_CHAIN()
27987 Res = DAG.getZExtOrTrunc(getSETCC(X86::COND_B, Res, DL, DAG), DL, VT); in LowerINTRINSIC_W_CHAIN()
27990 Res = DAG.getNode(ISD::SHL, DL, VT, Res, in LowerINTRINSIC_W_CHAIN()
27991 DAG.getShiftAmountConstant(Imm, VT, DL)); in LowerINTRINSIC_W_CHAIN()
28183 EVT VT = Op.getValueType(); in LowerFRAMEADDR() local
28199 return DAG.getFrameIndex(FrameAddrIndex, VT); in LowerFRAMEADDR()
28206 assert(((FrameReg == X86::RBP && VT == MVT::i64) || in LowerFRAMEADDR()
28207 (FrameReg == X86::EBP && VT == MVT::i32)) && in LowerFRAMEADDR()
28209 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT); in LowerFRAMEADDR()
28211 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr, in LowerFRAMEADDR()
28218 Register X86TargetLowering::getRegisterByName(const char* RegName, LLT VT, in getRegisterByName() argument
28507 MVT VT = Op.getSimpleValueType(); in LowerFLT_ROUNDS_() local
28541 RetVal = DAG.getZExtOrTrunc(RetVal, DL, VT); in LowerFLT_ROUNDS_()
28668 MVT VT = Op.getSimpleValueType(); in LowerVectorCTLZ_AVX512CDI() local
28669 MVT EltVT = VT.getVectorElementType(); in LowerVectorCTLZ_AVX512CDI()
28670 unsigned NumElems = VT.getVectorNumElements(); in LowerVectorCTLZ_AVX512CDI()
28687 SDValue TruncNode = DAG.getNode(ISD::TRUNCATE, dl, VT, CtlzNode); in LowerVectorCTLZ_AVX512CDI()
28688 SDValue Delta = DAG.getConstant(32 - EltVT.getSizeInBits(), dl, VT); in LowerVectorCTLZ_AVX512CDI()
28690 return DAG.getNode(ISD::SUB, dl, VT, TruncNode, Delta); in LowerVectorCTLZ_AVX512CDI()
28697 MVT VT = Op.getSimpleValueType(); in LowerVectorCTLZInRegLUT() local
28698 int NumElts = VT.getVectorNumElements(); in LowerVectorCTLZInRegLUT()
28699 int NumBytes = NumElts * (VT.getScalarSizeInBits() / 8); in LowerVectorCTLZInRegLUT()
28743 while (CurrVT != VT) { in LowerVectorCTLZInRegLUT()
28779 MVT VT = Op.getSimpleValueType(); in LowerVectorCTLZ() local
28783 (Subtarget.canExtendTo512DQ() || VT.getVectorElementType() != MVT::i8)) in LowerVectorCTLZ()
28787 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerVectorCTLZ()
28791 if (VT.is512BitVector() && !Subtarget.hasBWI()) in LowerVectorCTLZ()
28800 MVT VT = Op.getSimpleValueType(); in LowerCTLZ() local
28801 MVT OpVT = VT; in LowerCTLZ()
28802 unsigned NumBits = VT.getSizeInBits(); in LowerCTLZ()
28806 if (VT.isVector()) in LowerCTLZ()
28810 if (VT == MVT::i8) { in LowerCTLZ()
28832 if (VT == MVT::i8) in LowerCTLZ()
28839 MVT VT = Op.getSimpleValueType(); in LowerCTTZ() local
28840 unsigned NumBits = VT.getScalarSizeInBits(); in LowerCTTZ()
28844 assert(!VT.isVector() && Op.getOpcode() == ISD::CTTZ && in LowerCTTZ()
28848 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerCTTZ()
28852 SDValue Ops[] = {Op, DAG.getConstant(NumBits, dl, VT), in LowerCTTZ()
28855 return DAG.getNode(X86ISD::CMOV, dl, VT, Ops); in LowerCTTZ()
28860 MVT VT = Op.getSimpleValueType(); in lowerAddSub() local
28861 if (VT == MVT::i16 || VT == MVT::i32) in lowerAddSub()
28864 if (VT == MVT::v32i16 || VT == MVT::v64i8) in lowerAddSub()
28875 MVT VT = Op.getSimpleValueType(); in LowerADDSAT_SUBSAT() local
28880 if (VT == MVT::v32i16 || VT == MVT::v64i8 || in LowerADDSAT_SUBSAT()
28881 (VT.is256BitVector() && !Subtarget.hasInt256())) { in LowerADDSAT_SUBSAT()
28890 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in LowerADDSAT_SUBSAT()
28892 unsigned BitWidth = VT.getScalarSizeInBits(); in LowerADDSAT_SUBSAT()
28894 if (!TLI.isOperationLegal(ISD::UMAX, VT) || useVPTERNLOG(Subtarget, VT)) { in LowerADDSAT_SUBSAT()
28902 SDValue SignMask = DAG.getConstant(C->getAPIntValue(), DL, VT); in LowerADDSAT_SUBSAT()
28903 SDValue ShiftAmt = DAG.getConstant(BitWidth - 1, DL, VT); in LowerADDSAT_SUBSAT()
28904 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, X, SignMask); in LowerADDSAT_SUBSAT()
28905 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShiftAmt); in LowerADDSAT_SUBSAT()
28906 return DAG.getNode(ISD::AND, DL, VT, Xor, Sra); in LowerADDSAT_SUBSAT()
28909 if (!TLI.isOperationLegal(ISD::UMAX, VT)) { in LowerADDSAT_SUBSAT()
28911 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, X, Y); in LowerADDSAT_SUBSAT()
28914 if (SetCCResultType == VT && in LowerADDSAT_SUBSAT()
28915 DAG.ComputeNumSignBits(Cmp) == VT.getScalarSizeInBits()) in LowerADDSAT_SUBSAT()
28916 return DAG.getNode(ISD::AND, DL, VT, Cmp, Sub); in LowerADDSAT_SUBSAT()
28917 return DAG.getSelect(DL, VT, Cmp, Sub, DAG.getConstant(0, DL, VT)); in LowerADDSAT_SUBSAT()
28922 (!VT.isVector() || VT == MVT::v2i64)) { in LowerADDSAT_SUBSAT()
28925 SDValue Zero = DAG.getConstant(0, DL, VT); in LowerADDSAT_SUBSAT()
28928 DAG.getVTList(VT, SetCCResultType), X, Y); in LowerADDSAT_SUBSAT()
28931 SDValue SatMin = DAG.getConstant(MinVal, DL, VT); in LowerADDSAT_SUBSAT()
28932 SDValue SatMax = DAG.getConstant(MaxVal, DL, VT); in LowerADDSAT_SUBSAT()
28935 Result = DAG.getSelect(DL, VT, SumNeg, SatMax, SatMin); in LowerADDSAT_SUBSAT()
28936 return DAG.getSelect(DL, VT, Overflow, Result, SumDiff); in LowerADDSAT_SUBSAT()
28945 MVT VT = Op.getSimpleValueType(); in LowerABS() local
28946 if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) { in LowerABS()
28951 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32), in LowerABS()
28952 DAG.getConstant(0, DL, VT), N0); in LowerABS()
28955 return DAG.getNode(X86ISD::CMOV, DL, VT, Ops); in LowerABS()
28959 if ((VT == MVT::v2i64 || VT == MVT::v4i64) && Subtarget.hasSSE41()) { in LowerABS()
28963 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Src); in LowerABS()
28964 return DAG.getNode(X86ISD::BLENDV, DL, VT, Src, Sub, Src); in LowerABS()
28967 if (VT.is256BitVector() && !Subtarget.hasInt256()) { in LowerABS()
28968 assert(VT.isInteger() && in LowerABS()
28973 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) in LowerABS()
28982 MVT VT = Op.getSimpleValueType(); in LowerAVG() local
28985 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerAVG()
28988 if (VT == MVT::v32i16 || VT == MVT::v64i8) in LowerAVG()
28997 MVT VT = Op.getSimpleValueType(); in LowerMINMAX() local
29000 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerMINMAX()
29003 if (VT == MVT::v32i16 || VT == MVT::v64i8) in LowerMINMAX()
29013 MVT VT = Op.getSimpleValueType(); in LowerMUL() local
29016 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerMUL()
29019 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) in LowerMUL()
29027 if (VT == MVT::v16i8 || VT == MVT::v32i8 || VT == MVT::v64i8) { in LowerMUL()
29028 unsigned NumElts = VT.getVectorNumElements(); in LowerMUL()
29030 if ((VT == MVT::v16i8 && Subtarget.hasInt256()) || in LowerMUL()
29031 (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) { in LowerMUL()
29032 MVT ExVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements()); in LowerMUL()
29034 ISD::TRUNCATE, dl, VT, in LowerMUL()
29046 SDValue Undef = DAG.getUNDEF(VT); in LowerMUL()
29047 SDValue ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Undef)); in LowerMUL()
29048 SDValue AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Undef)); in LowerMUL()
29066 BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Undef)); in LowerMUL()
29067 BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B, Undef)); in LowerMUL()
29073 return getPack(DAG, Subtarget, dl, VT, RLo, RHi); in LowerMUL()
29077 if (VT == MVT::v4i32) { in LowerMUL()
29083 SDValue Aodds = DAG.getVectorShuffle(VT, dl, A, A, UnpackMask); in LowerMUL()
29084 SDValue Bodds = DAG.getVectorShuffle(VT, dl, B, B, UnpackMask); in LowerMUL()
29095 Evens = DAG.getBitcast(VT, Evens); in LowerMUL()
29096 Odds = DAG.getBitcast(VT, Odds); in LowerMUL()
29101 return DAG.getVectorShuffle(VT, dl, Evens, Odds, ShufMask); in LowerMUL()
29104 assert((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) && in LowerMUL()
29128 SDValue Zero = DAG.getConstant(0, dl, VT); in LowerMUL()
29133 AloBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, B); in LowerMUL()
29137 SDValue Bhi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, B, 32, DAG); in LowerMUL()
29138 AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi); in LowerMUL()
29143 SDValue Ahi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, A, 32, DAG); in LowerMUL()
29144 AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B); in LowerMUL()
29147 SDValue Hi = DAG.getNode(ISD::ADD, dl, VT, AloBhi, AhiBlo); in LowerMUL()
29148 Hi = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Hi, 32, DAG); in LowerMUL()
29150 return DAG.getNode(ISD::ADD, dl, VT, AloBlo, Hi); in LowerMUL()
29154 MVT VT, bool IsSigned, in LowervXi8MulWithUNPCK() argument
29158 unsigned NumElts = VT.getVectorNumElements(); in LowervXi8MulWithUNPCK()
29173 SDValue Zero = DAG.getConstant(0, dl, VT); in LowervXi8MulWithUNPCK()
29177 ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, A)); in LowervXi8MulWithUNPCK()
29178 AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, A)); in LowervXi8MulWithUNPCK()
29180 ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Zero)); in LowervXi8MulWithUNPCK()
29181 AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Zero)); in LowervXi8MulWithUNPCK()
29213 BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, B)); in LowervXi8MulWithUNPCK()
29214 BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, B)); in LowervXi8MulWithUNPCK()
29216 BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Zero)); in LowervXi8MulWithUNPCK()
29217 BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B, Zero)); in LowervXi8MulWithUNPCK()
29227 *Low = getPack(DAG, Subtarget, dl, VT, RLo, RHi); in LowervXi8MulWithUNPCK()
29229 return getPack(DAG, Subtarget, dl, VT, RLo, RHi, /*PackHiHalf*/ true); in LowervXi8MulWithUNPCK()
29235 MVT VT = Op.getSimpleValueType(); in LowerMULH() local
29237 unsigned NumElts = VT.getVectorNumElements(); in LowerMULH()
29242 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerMULH()
29245 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) in LowerMULH()
29248 if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) { in LowerMULH()
29249 assert((VT == MVT::v4i32 && Subtarget.hasSSE2()) || in LowerMULH()
29250 (VT == MVT::v8i32 && Subtarget.hasInt256()) || in LowerMULH()
29251 (VT == MVT::v16i32 && Subtarget.hasAVX512())); in LowerMULH()
29268 SDValue Odd0 = DAG.getVectorShuffle(VT, dl, A, A, in LowerMULH()
29271 SDValue Odd1 = DAG.getVectorShuffle(VT, dl, B, B, in LowerMULH()
29281 SDValue Mul1 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT, in LowerMULH()
29286 SDValue Mul2 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT, in LowerMULH()
29295 SDValue Res = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, ShufMask); in LowerMULH()
29300 SDValue Zero = DAG.getConstant(0, dl, VT); in LowerMULH()
29301 SDValue T1 = DAG.getNode(ISD::AND, dl, VT, in LowerMULH()
29302 DAG.getSetCC(dl, VT, Zero, A, ISD::SETGT), B); in LowerMULH()
29303 SDValue T2 = DAG.getNode(ISD::AND, dl, VT, in LowerMULH()
29304 DAG.getSetCC(dl, VT, Zero, B, ISD::SETGT), A); in LowerMULH()
29306 SDValue Fixup = DAG.getNode(ISD::ADD, dl, VT, T1, T2); in LowerMULH()
29307 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Fixup); in LowerMULH()
29314 assert((VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) || in LowerMULH()
29315 (VT == MVT::v64i8 && Subtarget.hasBWI())) && in LowerMULH()
29324 if ((VT == MVT::v16i8 && Subtarget.hasInt256()) || in LowerMULH()
29325 (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) { in LowerMULH()
29332 return DAG.getNode(ISD::TRUNCATE, dl, VT, Mul); in LowerMULH()
29335 return LowervXi8MulWithUNPCK(A, B, dl, VT, IsSigned, Subtarget, DAG); in LowerMULH()
29341 MVT VT = Op.getSimpleValueType(); in LowerMULO() local
29344 if (!VT.isVector()) in LowerMULO()
29353 if ((VT == MVT::v32i8 && !Subtarget.hasInt256()) || in LowerMULO()
29354 (VT == MVT::v64i8 && !Subtarget.hasBWI())) { in LowerMULO()
29373 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in LowerMULO()
29382 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in LowerMULO()
29384 if ((VT == MVT::v16i8 && Subtarget.hasInt256()) || in LowerMULO()
29385 (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) { in LowerMULO()
29386 unsigned NumElts = VT.getVectorNumElements(); in LowerMULO()
29393 SDValue Low = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul); in LowerMULO()
29417 High = DAG.getNode(ISD::TRUNCATE, dl, VT, High); in LowerMULO()
29419 DAG.getNode(ISD::SRA, dl, VT, Low, DAG.getConstant(7, dl, VT)); in LowerMULO()
29436 High = DAG.getNode(ISD::TRUNCATE, dl, VT, High); in LowerMULO()
29451 LowervXi8MulWithUNPCK(A, B, dl, VT, IsSigned, Subtarget, DAG, &Low); in LowerMULO()
29457 DAG.getNode(ISD::SRA, dl, VT, Low, DAG.getConstant(7, dl, VT)); in LowerMULO()
29462 DAG.getSetCC(dl, SetccVT, High, DAG.getConstant(0, dl, VT), ISD::SETNE); in LowerMULO()
29472 EVT VT = Op.getValueType(); in LowerWin64_i128OP() local
29473 assert(VT.isInteger() && VT.getSizeInBits() == 128 && in LowerWin64_i128OP()
29524 return DAG.getBitcast(VT, CallInfo.first); in LowerWin64_i128OP()
29531 EVT VT = Op.getValueType(); in LowerWin64_FP_TO_INT128() local
29537 assert(VT.isInteger() && VT.getSizeInBits() == 128 && in LowerWin64_FP_TO_INT128()
29543 LC = RTLIB::getFPTOSINT(ArgVT, VT); in LowerWin64_FP_TO_INT128()
29545 LC = RTLIB::getFPTOUINT(ArgVT, VT); in LowerWin64_FP_TO_INT128()
29557 Result = DAG.getBitcast(VT, Result); in LowerWin64_FP_TO_INT128()
29564 EVT VT = Op.getValueType(); in LowerWin64_INT128_TO_FP() local
29576 LC = RTLIB::getSINTTOFP(ArgVT, VT); in LowerWin64_INT128_TO_FP()
29578 LC = RTLIB::getUINTTOFP(ArgVT, VT); in LowerWin64_INT128_TO_FP()
29594 makeLibCall(DAG, LC, VT, StackPtr, CallOptions, dl, Chain); in LowerWin64_INT128_TO_FP()
29600 static bool supportedVectorShiftWithImm(MVT VT, const X86Subtarget &Subtarget, in supportedVectorShiftWithImm() argument
29602 if (!(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector())) in supportedVectorShiftWithImm()
29605 if (VT.getScalarSizeInBits() < 16) in supportedVectorShiftWithImm()
29608 if (VT.is512BitVector() && Subtarget.useAVX512Regs() && in supportedVectorShiftWithImm()
29609 (VT.getScalarSizeInBits() > 16 || Subtarget.hasBWI())) in supportedVectorShiftWithImm()
29612 bool LShift = (VT.is128BitVector() && Subtarget.hasSSE2()) || in supportedVectorShiftWithImm()
29613 (VT.is256BitVector() && Subtarget.hasInt256()); in supportedVectorShiftWithImm()
29616 (VT != MVT::v2i64 && VT != MVT::v4i64)); in supportedVectorShiftWithImm()
29623 bool supportedVectorShiftWithBaseAmnt(MVT VT, const X86Subtarget &Subtarget, in supportedVectorShiftWithBaseAmnt() argument
29625 return supportedVectorShiftWithImm(VT, Subtarget, Opcode); in supportedVectorShiftWithBaseAmnt()
29630 static bool supportedVectorVarShift(MVT VT, const X86Subtarget &Subtarget, in supportedVectorVarShift() argument
29632 if (!(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector())) in supportedVectorVarShift()
29635 if (!Subtarget.hasInt256() || VT.getScalarSizeInBits() < 16) in supportedVectorVarShift()
29639 if (VT.getScalarSizeInBits() == 16 && !Subtarget.hasBWI()) in supportedVectorVarShift()
29643 (Subtarget.useAVX512Regs() || !VT.is512BitVector())) in supportedVectorVarShift()
29646 bool LShift = VT.is128BitVector() || VT.is256BitVector(); in supportedVectorVarShift()
29647 bool AShift = LShift && VT != MVT::v2i64 && VT != MVT::v4i64; in supportedVectorVarShift()
29653 MVT VT = Op.getSimpleValueType(); in LowerShiftByScalarImmediate() local
29660 assert((VT == MVT::v2i64 || VT == MVT::v4i64) && "Unexpected SRA type"); in LowerShiftByScalarImmediate()
29661 MVT ExVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() * 2); in LowerShiftByScalarImmediate()
29666 assert((VT != MVT::v4i64 || Subtarget.hasInt256()) && in LowerShiftByScalarImmediate()
29668 return DAG.getNode(X86ISD::PCMPGT, dl, VT, DAG.getConstant(0, dl, VT), R); in LowerShiftByScalarImmediate()
29677 if (VT == MVT::v2i64) in LowerShiftByScalarImmediate()
29679 if (VT == MVT::v4i64) in LowerShiftByScalarImmediate()
29687 getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt, DAG); in LowerShiftByScalarImmediate()
29689 if (VT == MVT::v2i64) in LowerShiftByScalarImmediate()
29691 if (VT == MVT::v4i64) in LowerShiftByScalarImmediate()
29695 return DAG.getBitcast(VT, Ex); in LowerShiftByScalarImmediate()
29704 if (APIntShiftAmt.uge(VT.getScalarSizeInBits())) in LowerShiftByScalarImmediate()
29705 return DAG.getUNDEF(VT); in LowerShiftByScalarImmediate()
29709 if (supportedVectorShiftWithImm(VT, Subtarget, Op.getOpcode())) in LowerShiftByScalarImmediate()
29710 return getTargetVShiftByConstNode(X86Opc, dl, VT, R, ShiftAmt, DAG); in LowerShiftByScalarImmediate()
29713 if (((!Subtarget.hasXOP() && VT == MVT::v2i64) || in LowerShiftByScalarImmediate()
29714 (Subtarget.hasInt256() && VT == MVT::v4i64)) && in LowerShiftByScalarImmediate()
29718 if (VT == MVT::v16i8 || (Subtarget.hasInt256() && VT == MVT::v32i8) || in LowerShiftByScalarImmediate()
29719 (Subtarget.hasBWI() && VT == MVT::v64i8)) { in LowerShiftByScalarImmediate()
29720 unsigned NumElts = VT.getVectorNumElements(); in LowerShiftByScalarImmediate()
29730 R = DAG.getNode(ISD::FREEZE, dl, VT, R); in LowerShiftByScalarImmediate()
29731 return DAG.getNode(ISD::ADD, dl, VT, R, R); in LowerShiftByScalarImmediate()
29736 SDValue Zeros = DAG.getConstant(0, dl, VT); in LowerShiftByScalarImmediate()
29737 if (VT.is512BitVector()) { in LowerShiftByScalarImmediate()
29738 assert(VT == MVT::v64i8 && "Unexpected element type!"); in LowerShiftByScalarImmediate()
29740 return DAG.getNode(ISD::SIGN_EXTEND, dl, VT, CMP); in LowerShiftByScalarImmediate()
29742 return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R); in LowerShiftByScalarImmediate()
29746 if (VT == MVT::v16i8 && Subtarget.hasXOP()) in LowerShiftByScalarImmediate()
29753 SHL = DAG.getBitcast(VT, SHL); in LowerShiftByScalarImmediate()
29756 return DAG.getNode(ISD::AND, dl, VT, SHL, DAG.getConstant(Mask, dl, VT)); in LowerShiftByScalarImmediate()
29762 SRL = DAG.getBitcast(VT, SRL); in LowerShiftByScalarImmediate()
29765 return DAG.getNode(ISD::AND, dl, VT, SRL, DAG.getConstant(Mask, dl, VT)); in LowerShiftByScalarImmediate()
29769 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt); in LowerShiftByScalarImmediate()
29771 SDValue Mask = DAG.getConstant(128 >> ShiftAmt, dl, VT); in LowerShiftByScalarImmediate()
29772 Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask); in LowerShiftByScalarImmediate()
29773 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask); in LowerShiftByScalarImmediate()
29784 MVT VT = Op.getSimpleValueType(); in LowerShiftByScalarVariable() local
29793 if (supportedVectorShiftWithBaseAmnt(VT, Subtarget, Opcode)) in LowerShiftByScalarVariable()
29794 return getTargetVShiftNode(X86OpcI, dl, VT, R, BaseShAmt, BaseShAmtIdx, in LowerShiftByScalarVariable()
29798 if (((VT == MVT::v16i8 && !Subtarget.canExtendTo512DQ()) || in LowerShiftByScalarVariable()
29799 (VT == MVT::v32i8 && !Subtarget.canExtendTo512BW()) || in LowerShiftByScalarVariable()
29800 VT == MVT::v64i8) && in LowerShiftByScalarVariable()
29802 unsigned NumElts = VT.getVectorNumElements(); in LowerShiftByScalarVariable()
29816 BitMask = DAG.getBitcast(VT, BitMask); in LowerShiftByScalarVariable()
29817 BitMask = DAG.getVectorShuffle(VT, dl, BitMask, BitMask, in LowerShiftByScalarVariable()
29823 Res = DAG.getBitcast(VT, Res); in LowerShiftByScalarVariable()
29824 Res = DAG.getNode(ISD::AND, dl, VT, Res, BitMask); in LowerShiftByScalarVariable()
29833 SignMask = DAG.getBitcast(VT, SignMask); in LowerShiftByScalarVariable()
29834 Res = DAG.getNode(ISD::XOR, dl, VT, Res, SignMask); in LowerShiftByScalarVariable()
29835 Res = DAG.getNode(ISD::SUB, dl, VT, Res, SignMask); in LowerShiftByScalarVariable()
29849 MVT VT = Amt.getSimpleValueType(); in convertShiftLeftToScale() local
29850 if (!(VT == MVT::v8i16 || VT == MVT::v4i32 || in convertShiftLeftToScale()
29851 (Subtarget.hasInt256() && VT == MVT::v16i16) || in convertShiftLeftToScale()
29852 (Subtarget.hasAVX512() && VT == MVT::v32i16) || in convertShiftLeftToScale()
29853 (!Subtarget.hasAVX512() && VT == MVT::v16i8) || in convertShiftLeftToScale()
29854 (Subtarget.hasInt256() && VT == MVT::v32i8) || in convertShiftLeftToScale()
29855 (Subtarget.hasBWI() && VT == MVT::v64i8))) in convertShiftLeftToScale()
29858 MVT SVT = VT.getVectorElementType(); in convertShiftLeftToScale()
29860 unsigned NumElems = VT.getVectorNumElements(); in convertShiftLeftToScale()
29873 return DAG.getBuildVector(VT, dl, Elts); in convertShiftLeftToScale()
29878 if (VT == MVT::v4i32) { in convertShiftLeftToScale()
29879 Amt = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, dl, VT)); in convertShiftLeftToScale()
29880 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, in convertShiftLeftToScale()
29881 DAG.getConstant(0x3f800000U, dl, VT)); in convertShiftLeftToScale()
29883 return DAG.getNode(ISD::FP_TO_SINT, dl, VT, Amt); in convertShiftLeftToScale()
29887 if (VT == MVT::v8i16 && !Subtarget.hasAVX2()) { in convertShiftLeftToScale()
29888 SDValue Z = DAG.getConstant(0, dl, VT); in convertShiftLeftToScale()
29889 SDValue Lo = DAG.getBitcast(MVT::v4i32, getUnpackl(DAG, dl, VT, Amt, Z)); in convertShiftLeftToScale()
29890 SDValue Hi = DAG.getBitcast(MVT::v4i32, getUnpackh(DAG, dl, VT, Amt, Z)); in convertShiftLeftToScale()
29894 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi); in convertShiftLeftToScale()
29895 return getPack(DAG, Subtarget, dl, VT, Lo, Hi); in convertShiftLeftToScale()
29903 MVT VT = Op.getSimpleValueType(); in LowerShift() local
29907 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in LowerShift()
29914 assert(VT.isVector() && "Custom lowering only for vector shifts!"); in LowerShift()
29923 if (supportedVectorVarShift(VT, Subtarget, Opc)) in LowerShift()
29929 if (((VT == MVT::v2i64 && !Subtarget.hasXOP()) || in LowerShift()
29930 (VT == MVT::v4i64 && Subtarget.hasInt256())) && in LowerShift()
29932 SDValue S = DAG.getConstant(APInt::getSignMask(64), dl, VT); in LowerShift()
29933 SDValue M = DAG.getNode(ISD::SRL, dl, VT, S, Amt); in LowerShift()
29934 R = DAG.getNode(ISD::SRL, dl, VT, R, Amt); in LowerShift()
29935 R = DAG.getNode(ISD::XOR, dl, VT, R, M); in LowerShift()
29936 R = DAG.getNode(ISD::SUB, dl, VT, R, M); in LowerShift()
29942 if (Subtarget.hasXOP() && (VT == MVT::v2i64 || VT == MVT::v4i32 || in LowerShift()
29943 VT == MVT::v8i16 || VT == MVT::v16i8)) { in LowerShift()
29945 SDValue Zero = DAG.getConstant(0, dl, VT); in LowerShift()
29946 Amt = DAG.getNode(ISD::SUB, dl, VT, Zero, Amt); in LowerShift()
29949 return DAG.getNode(X86ISD::VPSHL, dl, VT, R, Amt); in LowerShift()
29951 return DAG.getNode(X86ISD::VPSHA, dl, VT, R, Amt); in LowerShift()
29956 if (VT == MVT::v2i64 && Opc != ISD::SRA) { in LowerShift()
29958 SDValue Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {0, 0}); in LowerShift()
29959 SDValue Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {1, 1}); in LowerShift()
29960 SDValue R0 = DAG.getNode(Opc, dl, VT, R, Amt0); in LowerShift()
29961 SDValue R1 = DAG.getNode(Opc, dl, VT, R, Amt1); in LowerShift()
29962 return DAG.getVectorShuffle(VT, dl, R0, R1, {0, 3}); in LowerShift()
29975 if (ConstantAmt && (VT == MVT::v8i16 || VT == MVT::v4i32 || in LowerShift()
29976 (VT == MVT::v16i16 && Subtarget.hasInt256()))) { in LowerShift()
29978 unsigned NumElts = VT.getVectorNumElements(); in LowerShift()
30001 (VT != MVT::v16i16 || in LowerShift()
30002 is128BitLaneRepeatedShuffleMask(VT, ShuffleMask)) && in LowerShift()
30003 (VT == MVT::v4i32 || Subtarget.hasSSE41() || Opc != ISD::SHL || in LowerShift()
30009 SDValue Shift1 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, in LowerShift()
30011 SDValue Shift2 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, in LowerShift()
30013 return DAG.getVectorShuffle(VT, dl, Shift1, Shift2, ShuffleMask); in LowerShift()
30021 if (Opc == ISD::SHL && !(VT == MVT::v32i8 && (Subtarget.hasXOP() || in LowerShift()
30024 return DAG.getNode(ISD::MUL, dl, VT, R, Scale); in LowerShift()
30029 (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256()))) { in LowerShift()
30030 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT); in LowerShift()
30031 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt); in LowerShift()
30033 SDValue Zero = DAG.getConstant(0, dl, VT); in LowerShift()
30034 SDValue ZAmt = DAG.getSetCC(dl, VT, Amt, Zero, ISD::SETEQ); in LowerShift()
30035 SDValue Res = DAG.getNode(ISD::MULHU, dl, VT, R, Scale); in LowerShift()
30036 return DAG.getSelect(dl, VT, ZAmt, R, Res); in LowerShift()
30045 (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256())) && in LowerShift()
30049 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT); in LowerShift()
30050 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt); in LowerShift()
30053 DAG.getSetCC(dl, VT, Amt, DAG.getConstant(0, dl, VT), ISD::SETEQ); in LowerShift()
30055 DAG.getSetCC(dl, VT, Amt, DAG.getConstant(1, dl, VT), ISD::SETEQ); in LowerShift()
30057 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, 1, DAG); in LowerShift()
30058 SDValue Res = DAG.getNode(ISD::MULHS, dl, VT, R, Scale); in LowerShift()
30059 Res = DAG.getSelect(dl, VT, Amt0, R, Res); in LowerShift()
30060 return DAG.getSelect(dl, VT, Amt1, Sra1, Res); in LowerShift()
30069 if (VT == MVT::v4i32) { in LowerShift()
30072 Amt0 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {0, 0, 0, 0}); in LowerShift()
30073 Amt1 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {1, 1, 1, 1}); in LowerShift()
30074 Amt2 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {2, 2, 2, 2}); in LowerShift()
30075 Amt3 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {3, 3, 3, 3}); in LowerShift()
30082 SDValue Z = DAG.getConstant(0, dl, VT); in LowerShift()
30083 Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Z, {0, 4, -1, -1}); in LowerShift()
30084 Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Z, {1, 5, -1, -1}); in LowerShift()
30085 Amt2 = DAG.getVectorShuffle(VT, dl, Amt, Z, {2, 6, -1, -1}); in LowerShift()
30086 Amt3 = DAG.getVectorShuffle(VT, dl, Amt, Z, {3, 7, -1, -1}); in LowerShift()
30101 SDValue R0 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt0)); in LowerShift()
30102 SDValue R1 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt1)); in LowerShift()
30103 SDValue R2 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt2)); in LowerShift()
30104 SDValue R3 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt3)); in LowerShift()
30109 SDValue R02 = DAG.getVectorShuffle(VT, dl, R0, R2, {0, -1, 6, -1}); in LowerShift()
30110 SDValue R13 = DAG.getVectorShuffle(VT, dl, R1, R3, {-1, 1, -1, 7}); in LowerShift()
30111 return DAG.getVectorShuffle(VT, dl, R02, R13, {0, 5, 2, 7}); in LowerShift()
30113 SDValue R01 = DAG.getVectorShuffle(VT, dl, R0, R1, {0, -1, -1, 5}); in LowerShift()
30114 SDValue R23 = DAG.getVectorShuffle(VT, dl, R2, R3, {2, -1, -1, 7}); in LowerShift()
30115 return DAG.getVectorShuffle(VT, dl, R01, R23, {0, 3, 4, 7}); in LowerShift()
30122 if ((Subtarget.hasInt256() && VT == MVT::v8i16) || in LowerShift()
30123 (Subtarget.canExtendTo512DQ() && VT == MVT::v16i16) || in LowerShift()
30124 (Subtarget.canExtendTo512DQ() && VT == MVT::v16i8) || in LowerShift()
30125 (Subtarget.canExtendTo512BW() && VT == MVT::v32i8) || in LowerShift()
30126 (Subtarget.hasBWI() && Subtarget.hasVLX() && VT == MVT::v16i8)) { in LowerShift()
30127 assert((!Subtarget.hasBWI() || VT == MVT::v32i8 || VT == MVT::v16i8) && in LowerShift()
30130 MVT ExtVT = MVT::getVectorVT(EvtSVT, VT.getVectorNumElements()); in LowerShift()
30134 return DAG.getNode(ISD::TRUNCATE, dl, VT, in LowerShift()
30141 (VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) || in LowerShift()
30142 (VT == MVT::v64i8 && Subtarget.hasBWI())) && in LowerShift()
30144 int NumElts = VT.getVectorNumElements(); in LowerShift()
30156 if (VT == MVT::v16i8 && Subtarget.hasInt256()) { in LowerShift()
30161 return DAG.getZExtOrTrunc(R, dl, VT); in LowerShift()
30176 SDValue LoR = DAG.getBitcast(VT16, getUnpackl(DAG, dl, VT, R, R)); in LowerShift()
30177 SDValue HiR = DAG.getBitcast(VT16, getUnpackh(DAG, dl, VT, R, R)); in LowerShift()
30184 return DAG.getNode(X86ISD::PACKUS, dl, VT, LoR, HiR); in LowerShift()
30187 if (VT == MVT::v16i8 || in LowerShift()
30188 (VT == MVT::v32i8 && Subtarget.hasInt256() && !Subtarget.hasXOP()) || in LowerShift()
30189 (VT == MVT::v64i8 && Subtarget.hasBWI())) { in LowerShift()
30190 MVT ExtVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements() / 2); in LowerShift()
30193 if (VT.is512BitVector()) { in LowerShift()
30197 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements()); in LowerShift()
30198 V0 = DAG.getBitcast(VT, V0); in LowerShift()
30199 V1 = DAG.getBitcast(VT, V1); in LowerShift()
30200 Sel = DAG.getBitcast(VT, Sel); in LowerShift()
30201 Sel = DAG.getSetCC(dl, MaskVT, DAG.getConstant(0, dl, VT), Sel, in LowerShift()
30203 return DAG.getBitcast(SelVT, DAG.getSelect(dl, VT, Sel, V0, V1)); in LowerShift()
30207 V0 = DAG.getBitcast(VT, V0); in LowerShift()
30208 V1 = DAG.getBitcast(VT, V1); in LowerShift()
30209 Sel = DAG.getBitcast(VT, Sel); in LowerShift()
30211 DAG.getNode(X86ISD::BLENDV, dl, VT, Sel, V0, V1)); in LowerShift()
30226 Amt = DAG.getBitcast(VT, Amt); in LowerShift()
30230 SDValue M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(4, dl, VT)); in LowerShift()
30231 R = SignBitSelect(VT, Amt, M, R); in LowerShift()
30234 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
30237 M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(2, dl, VT)); in LowerShift()
30238 R = SignBitSelect(VT, Amt, M, R); in LowerShift()
30241 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
30244 M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(1, dl, VT)); in LowerShift()
30245 R = SignBitSelect(VT, Amt, M, R); in LowerShift()
30253 SDValue ALo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), Amt); in LowerShift()
30254 SDValue AHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), Amt); in LowerShift()
30255 SDValue RLo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), R); in LowerShift()
30256 SDValue RHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), R); in LowerShift()
30292 return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi); in LowerShift()
30296 if (Subtarget.hasInt256() && !Subtarget.hasXOP() && VT == MVT::v16i16) { in LowerShift()
30298 SDValue Z = DAG.getConstant(0, dl, VT); in LowerShift()
30299 SDValue ALo = getUnpackl(DAG, dl, VT, Amt, Z); in LowerShift()
30300 SDValue AHi = getUnpackh(DAG, dl, VT, Amt, Z); in LowerShift()
30301 SDValue RLo = getUnpackl(DAG, dl, VT, Z, R); in LowerShift()
30302 SDValue RHi = getUnpackh(DAG, dl, VT, Z, R); in LowerShift()
30311 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi); in LowerShift()
30314 if (VT == MVT::v8i16) { in LowerShift()
30324 MVT ExtVT = MVT::getVectorVT(MVT::i8, VT.getVectorNumElements() * 2); in LowerShift()
30329 VT, DAG.getNode(X86ISD::BLENDV, dl, ExtVT, Sel, V0, V1)); in LowerShift()
30335 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, Sel, 15, DAG); in LowerShift()
30336 return DAG.getSelect(dl, VT, C, V0, V1); in LowerShift()
30344 ISD::OR, dl, VT, in LowerShift()
30345 getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 4, DAG), in LowerShift()
30346 getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG)); in LowerShift()
30348 Amt = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG); in LowerShift()
30352 SDValue M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 8, DAG); in LowerShift()
30356 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
30359 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 4, DAG); in LowerShift()
30363 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
30366 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 2, DAG); in LowerShift()
30370 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
30373 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 1, DAG); in LowerShift()
30379 if (VT.is256BitVector()) in LowerShift()
30382 if (VT == MVT::v32i16 || VT == MVT::v64i8) in LowerShift()
30390 MVT VT = Op.getSimpleValueType(); in LowerFunnelShift() local
30398 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in LowerFunnelShift()
30401 if (VT.isVector()) { in LowerFunnelShift()
30412 return getAVX512Node(IsFSHR ? X86ISD::VSHRD : X86ISD::VSHLD, DL, VT, in LowerFunnelShift()
30415 return getAVX512Node(IsFSHR ? X86ISD::VSHRDV : X86ISD::VSHLDV, DL, VT, in LowerFunnelShift()
30418 assert((VT == MVT::v16i8 || VT == MVT::v32i8 || VT == MVT::v64i8 || in LowerFunnelShift()
30419 VT == MVT::v8i16 || VT == MVT::v16i16 || VT == MVT::v32i16 || in LowerFunnelShift()
30420 VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) && in LowerFunnelShift()
30428 SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT); in LowerFunnelShift()
30429 SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); in LowerFunnelShift()
30437 unsigned NumElts = VT.getVectorNumElements(); in LowerFunnelShift()
30443 if ((VT.is256BitVector() && ((Subtarget.hasXOP() && EltSizeInBits < 16) || in LowerFunnelShift()
30445 (VT.is512BitVector() && !Subtarget.useBWIRegs() && in LowerFunnelShift()
30448 Op = DAG.getNode(Op.getOpcode(), DL, VT, Op0, Op1, AmtMod); in LowerFunnelShift()
30460 SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0)); in LowerFunnelShift()
30461 SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, Op1, Op0)); in LowerFunnelShift()
30466 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, !IsFSHR); in LowerFunnelShift()
30475 if (supportedVectorVarShift(VT, Subtarget, ShiftOpc) || Subtarget.hasXOP()) in LowerFunnelShift()
30493 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res); in LowerFunnelShift()
30499 SDValue Z = DAG.getConstant(0, DL, VT); in LowerFunnelShift()
30500 SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0)); in LowerFunnelShift()
30501 SDValue RHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, Op1, Op0)); in LowerFunnelShift()
30502 SDValue ALo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, AmtMod, Z)); in LowerFunnelShift()
30503 SDValue AHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, AmtMod, Z)); in LowerFunnelShift()
30506 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, !IsFSHR); in LowerFunnelShift()
30513 (VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) && in LowerFunnelShift()
30522 if ((VT == MVT::i8 || (ExpandFunnel && VT == MVT::i16)) && in LowerFunnelShift()
30537 return DAG.getZExtOrTrunc(Res, DL, VT); in LowerFunnelShift()
30540 if (VT == MVT::i8 || ExpandFunnel) in LowerFunnelShift()
30544 if (VT == MVT::i16) { in LowerFunnelShift()
30548 return DAG.getNode(FSHOp, DL, VT, Op0, Op1, Amt); in LowerFunnelShift()
30556 MVT VT = Op.getSimpleValueType(); in LowerRotate() local
30557 assert(VT.isVector() && "Custom lowering only for vector rotates!"); in LowerRotate()
30563 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in LowerRotate()
30564 int NumElts = VT.getVectorNumElements(); in LowerRotate()
30581 return DAG.getNode(RotOpc, DL, VT, R, in LowerRotate()
30592 return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt); in LowerRotate()
30595 SDValue Z = DAG.getConstant(0, DL, VT); in LowerRotate()
30600 if (SDValue NegAmt = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {Z, Amt})) in LowerRotate()
30601 return DAG.getNode(ISD::ROTL, DL, VT, R, NegAmt); in LowerRotate()
30605 return DAG.getNode(ISD::ROTL, DL, VT, R, in LowerRotate()
30606 DAG.getNode(ISD::SUB, DL, VT, Z, Amt)); in LowerRotate()
30610 if (VT.is256BitVector() && (Subtarget.hasXOP() || !Subtarget.hasAVX2())) in LowerRotate()
30618 assert(VT.is128BitVector() && "Only rotate 128-bit vectors!"); in LowerRotate()
30623 return DAG.getNode(X86ISD::VROTLI, DL, VT, R, in LowerRotate()
30636 if (VT.is512BitVector() && !Subtarget.useBWIRegs()) in LowerRotate()
30640 (VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 || in LowerRotate()
30641 ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8) && in LowerRotate()
30643 ((VT == MVT::v32i16 || VT == MVT::v64i8) && Subtarget.useBWIRegs())) && in LowerRotate()
30649 SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT); in LowerRotate()
30650 SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); in LowerRotate()
30660 return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt); in LowerRotate()
30663 SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R)); in LowerRotate()
30664 SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R)); in LowerRotate()
30669 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL); in LowerRotate()
30699 return DAG.getNode(ISD::TRUNCATE, DL, VT, R); in LowerRotate()
30707 SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R)); in LowerRotate()
30708 SDValue RHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R)); in LowerRotate()
30709 SDValue ALo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, AmtMod, Z)); in LowerRotate()
30710 SDValue AHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, AmtMod, Z)); in LowerRotate()
30713 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL); in LowerRotate()
30715 assert((VT == MVT::v16i8 || VT == MVT::v32i8) && "Unsupported vXi8 type"); in LowerRotate()
30722 V0 = DAG.getBitcast(VT, V0); in LowerRotate()
30723 V1 = DAG.getBitcast(VT, V1); in LowerRotate()
30724 Sel = DAG.getBitcast(VT, Sel); in LowerRotate()
30726 DAG.getNode(X86ISD::BLENDV, DL, VT, Sel, V0, V1)); in LowerRotate()
30737 if (!IsROTL && !useVPTERNLOG(Subtarget, VT)) { in LowerRotate()
30738 Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt); in LowerRotate()
30750 Amt = DAG.getBitcast(VT, Amt); in LowerRotate()
30755 ISD::OR, DL, VT, in LowerRotate()
30756 DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(4, DL, VT)), in LowerRotate()
30757 DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(4, DL, VT))); in LowerRotate()
30758 R = SignBitSelect(VT, Amt, M, R); in LowerRotate()
30761 Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt); in LowerRotate()
30765 ISD::OR, DL, VT, in LowerRotate()
30766 DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(2, DL, VT)), in LowerRotate()
30767 DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(6, DL, VT))); in LowerRotate()
30768 R = SignBitSelect(VT, Amt, M, R); in LowerRotate()
30771 Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt); in LowerRotate()
30775 ISD::OR, DL, VT, in LowerRotate()
30776 DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(1, DL, VT)), in LowerRotate()
30777 DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(7, DL, VT))); in LowerRotate()
30778 return SignBitSelect(VT, Amt, M, R); in LowerRotate()
30783 bool LegalVarShifts = supportedVectorVarShift(VT, Subtarget, ISD::SHL) && in LowerRotate()
30784 supportedVectorVarShift(VT, Subtarget, ISD::SRL); in LowerRotate()
30789 Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); in LowerRotate()
30790 SDValue AmtR = DAG.getConstant(EltSizeInBits, DL, VT); in LowerRotate()
30791 AmtR = DAG.getNode(ISD::SUB, DL, VT, AmtR, Amt); in LowerRotate()
30792 SDValue SHL = DAG.getNode(IsROTL ? ISD::SHL : ISD::SRL, DL, VT, R, Amt); in LowerRotate()
30793 SDValue SRL = DAG.getNode(IsROTL ? ISD::SRL : ISD::SHL, DL, VT, R, AmtR); in LowerRotate()
30794 return DAG.getNode(ISD::OR, DL, VT, SHL, SRL); in LowerRotate()
30799 Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt); in LowerRotate()
30804 Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); in LowerRotate()
30816 SDValue Lo = DAG.getNode(ISD::MUL, DL, VT, R, Scale); in LowerRotate()
30817 SDValue Hi = DAG.getNode(ISD::MULHU, DL, VT, R, Scale); in LowerRotate()
30818 return DAG.getNode(ISD::OR, DL, VT, Lo, Hi); in LowerRotate()
30824 assert(VT == MVT::v4i32 && "Only v4i32 vector rotate expected"); in LowerRotate()
30826 SDValue R13 = DAG.getVectorShuffle(VT, DL, R, R, OddMask); in LowerRotate()
30827 SDValue Scale13 = DAG.getVectorShuffle(VT, DL, Scale, Scale, OddMask); in LowerRotate()
30835 Res02 = DAG.getBitcast(VT, Res02); in LowerRotate()
30836 Res13 = DAG.getBitcast(VT, Res13); in LowerRotate()
30838 return DAG.getNode(ISD::OR, DL, VT, in LowerRotate()
30839 DAG.getVectorShuffle(VT, DL, Res02, Res13, {0, 4, 2, 6}), in LowerRotate()
30840 DAG.getVectorShuffle(VT, DL, Res02, Res13, {1, 5, 3, 7})); in LowerRotate()
31295 static SDValue LowerHorizontalByteSum(SDValue V, MVT VT, in LowerHorizontalByteSum() argument
31300 MVT EltVT = VT.getVectorElementType(); in LowerHorizontalByteSum()
31305 unsigned VecSize = VT.getSizeInBits(); in LowerHorizontalByteSum()
31314 return DAG.getBitcast(VT, V); in LowerHorizontalByteSum()
31323 SDValue Zeros = DAG.getConstant(0, DL, VT); in LowerHorizontalByteSum()
31324 SDValue V32 = DAG.getBitcast(VT, V); in LowerHorizontalByteSum()
31325 SDValue Low = getUnpackl(DAG, DL, VT, V32, Zeros); in LowerHorizontalByteSum()
31326 SDValue High = getUnpackh(DAG, DL, VT, V32, Zeros); in LowerHorizontalByteSum()
31342 return DAG.getBitcast(VT, V); in LowerHorizontalByteSum()
31352 SDValue ShifterV = DAG.getConstant(8, DL, VT); in LowerHorizontalByteSum()
31353 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, DAG.getBitcast(VT, V), ShifterV); in LowerHorizontalByteSum()
31356 return DAG.getNode(ISD::SRL, DL, VT, DAG.getBitcast(VT, V), ShifterV); in LowerHorizontalByteSum()
31362 MVT VT = Op.getSimpleValueType(); in LowerVectorCTPOPInRegLUT() local
31363 MVT EltVT = VT.getVectorElementType(); in LowerVectorCTPOPInRegLUT()
31364 int NumElts = VT.getVectorNumElements(); in LowerVectorCTPOPInRegLUT()
31386 SDValue InRegLUT = DAG.getBuildVector(VT, DL, LUTVec); in LowerVectorCTPOPInRegLUT()
31387 SDValue M0F = DAG.getConstant(0x0F, DL, VT); in LowerVectorCTPOPInRegLUT()
31390 SDValue FourV = DAG.getConstant(4, DL, VT); in LowerVectorCTPOPInRegLUT()
31391 SDValue HiNibbles = DAG.getNode(ISD::SRL, DL, VT, Op, FourV); in LowerVectorCTPOPInRegLUT()
31394 SDValue LoNibbles = DAG.getNode(ISD::AND, DL, VT, Op, M0F); in LowerVectorCTPOPInRegLUT()
31399 SDValue HiPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, HiNibbles); in LowerVectorCTPOPInRegLUT()
31400 SDValue LoPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, LoNibbles); in LowerVectorCTPOPInRegLUT()
31401 return DAG.getNode(ISD::ADD, DL, VT, HiPopCnt, LoPopCnt); in LowerVectorCTPOPInRegLUT()
31408 MVT VT = Op.getSimpleValueType(); in LowerVectorCTPOP() local
31409 assert((VT.is512BitVector() || VT.is256BitVector() || VT.is128BitVector()) && in LowerVectorCTPOP()
31416 unsigned NumElems = VT.getVectorNumElements(); in LowerVectorCTPOP()
31417 assert((VT.getVectorElementType() == MVT::i8 || in LowerVectorCTPOP()
31418 VT.getVectorElementType() == MVT::i16) && "Unexpected type"); in LowerVectorCTPOP()
31423 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op); in LowerVectorCTPOP()
31428 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerVectorCTPOP()
31432 if (VT.is512BitVector() && !Subtarget.hasBWI()) in LowerVectorCTPOP()
31436 if (VT.getScalarType() != MVT::i8) { in LowerVectorCTPOP()
31437 MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8); in LowerVectorCTPOP()
31440 return LowerHorizontalByteSum(PopCnt8, VT, Subtarget, DAG); in LowerVectorCTPOP()
31458 MVT VT = Op.getSimpleValueType(); in LowerBITREVERSE_XOP() local
31464 if (!VT.isVector()) { in LowerBITREVERSE_XOP()
31465 MVT VecVT = MVT::getVectorVT(VT, 128 / VT.getSizeInBits()); in LowerBITREVERSE_XOP()
31468 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Res, in LowerBITREVERSE_XOP()
31472 int NumElts = VT.getVectorNumElements(); in LowerBITREVERSE_XOP()
31473 int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8; in LowerBITREVERSE_XOP()
31476 if (VT.is256BitVector()) in LowerBITREVERSE_XOP()
31479 assert(VT.is128BitVector() && in LowerBITREVERSE_XOP()
31499 return DAG.getBitcast(VT, Res); in LowerBITREVERSE_XOP()
31504 MVT VT = Op.getSimpleValueType(); in LowerBITREVERSE() local
31506 if (Subtarget.hasXOP() && !VT.is512BitVector()) in LowerBITREVERSE()
31514 assert(VT.getScalarType() == MVT::i8 && in LowerBITREVERSE()
31518 if (VT == MVT::v64i8 && !Subtarget.hasBWI()) in LowerBITREVERSE()
31522 if (VT == MVT::v32i8 && !Subtarget.hasInt256()) in LowerBITREVERSE()
31525 unsigned NumElts = VT.getVectorNumElements(); in LowerBITREVERSE()
31531 Matrix = DAG.getBitcast(VT, Matrix); in LowerBITREVERSE()
31532 return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, In, Matrix, in LowerBITREVERSE()
31539 SDValue NibbleMask = DAG.getConstant(0xF, DL, VT); in LowerBITREVERSE()
31540 SDValue Lo = DAG.getNode(ISD::AND, DL, VT, In, NibbleMask); in LowerBITREVERSE()
31541 SDValue Hi = DAG.getNode(ISD::SRL, DL, VT, In, DAG.getConstant(4, DL, VT)); in LowerBITREVERSE()
31560 SDValue LoMask = DAG.getBuildVector(VT, DL, LoMaskElts); in LowerBITREVERSE()
31561 SDValue HiMask = DAG.getBuildVector(VT, DL, HiMaskElts); in LowerBITREVERSE()
31562 Lo = DAG.getNode(X86ISD::PSHUFB, DL, VT, LoMask, Lo); in LowerBITREVERSE()
31563 Hi = DAG.getNode(X86ISD::PSHUFB, DL, VT, HiMask, Hi); in LowerBITREVERSE()
31564 return DAG.getNode(ISD::OR, DL, VT, Lo, Hi); in LowerBITREVERSE()
31571 MVT VT = Op.getSimpleValueType(); in LowerPARITY() local
31574 if (VT == MVT::i8 || in LowerPARITY()
31575 DAG.MaskedValueIsZero(X, APInt::getBitsSetFrom(VT.getSizeInBits(), 8))) { in LowerPARITY()
31582 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Setnp); in LowerPARITY()
31589 if (VT == MVT::i64) { in LowerPARITY()
31598 if (VT != MVT::i16) { in LowerPARITY()
31620 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Setnp); in LowerPARITY()
31662 MVT VT = N->getSimpleValueType(0); in lowerAtomicArith() local
31672 RHS = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS); in lowerAtomicArith()
31673 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, VT, Chain, LHS, in lowerAtomicArith()
31701 DAG.getUNDEF(VT), NewChain); in lowerAtomicArith()
31708 DAG.getUNDEF(VT), NewChain); in lowerAtomicArith()
31716 DAG.getUNDEF(VT), LockOp.getValue(1)); in lowerAtomicArith()
31723 EVT VT = Node->getMemoryVT(); in LowerATOMIC_STORE() local
31727 bool IsTypeLegal = DAG.getTargetLoweringInfo().isTypeLegal(VT); in LowerATOMIC_STORE()
31734 if (VT == MVT::i64 && !IsTypeLegal) { in LowerATOMIC_STORE()
31799 MVT VT = N->getSimpleValueType(0); in LowerADDSUBCARRY() local
31803 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in LowerADDSUBCARRY()
31806 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerADDSUBCARRY()
31941 MVT VT = Src.getSimpleValueType(); in LowerMSCATTER() local
31942 assert(VT.getScalarSizeInBits() >= 32 && "Unsupported scatter op"); in LowerMSCATTER()
31951 if (VT == MVT::v2f32 || VT == MVT::v2i32) { in LowerMSCATTER()
31956 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); in LowerMSCATTER()
31957 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, Src, DAG.getUNDEF(VT)); in LowerMSCATTER()
31975 if (!Subtarget.hasVLX() && !VT.is512BitVector() && in LowerMSCATTER()
31978 unsigned Factor = std::min(512/VT.getSizeInBits(), in LowerMSCATTER()
31980 unsigned NumElts = VT.getVectorNumElements() * Factor; in LowerMSCATTER()
31982 VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts); in LowerMSCATTER()
31986 Src = ExtendToType(Src, VT, DAG); in LowerMSCATTER()
32001 MVT VT = Op.getSimpleValueType(); in LowerMLOAD() local
32002 MVT ScalarVT = VT.getScalarType(); in LowerMLOAD()
32015 VT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask, in LowerMLOAD()
32016 getZeroVector(VT, Subtarget, DAG, dl), N->getMemoryVT(), in LowerMLOAD()
32020 SDValue Select = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru); in LowerMLOAD()
32030 assert(Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() && in LowerMLOAD()
32040 unsigned NumEltsInWideVec = 512 / VT.getScalarSizeInBits(); in LowerMLOAD()
32057 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, NewLoad.getValue(0), in LowerMLOAD()
32067 MVT VT = DataToStore.getSimpleValueType(); in LowerMSTORE() local
32068 MVT ScalarVT = VT.getScalarType(); in LowerMSTORE()
32078 assert(Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() && in LowerMSTORE()
32088 unsigned NumEltsInWideVec = 512/VT.getScalarSizeInBits(); in LowerMSTORE()
32112 MVT VT = Op.getSimpleValueType(); in LowerMGATHER() local
32118 assert(VT.getScalarSizeInBits() >= 32 && "Unsupported gather op"); in LowerMGATHER()
32126 MVT OrigVT = VT; in LowerMGATHER()
32127 if (Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() && in LowerMGATHER()
32130 unsigned Factor = std::min(512/VT.getSizeInBits(), in LowerMGATHER()
32133 unsigned NumElts = VT.getVectorNumElements() * Factor; in LowerMGATHER()
32135 VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts); in LowerMGATHER()
32139 PassThru = ExtendToType(PassThru, VT, DAG); in LowerMGATHER()
32146 PassThru = getZeroVector(VT, Subtarget, DAG, dl); in LowerMGATHER()
32151 X86ISD::MGATHER, dl, DAG.getVTList(VT, MVT::Other), Ops, N->getMemoryVT(), in LowerMGATHER()
32201 EVT VT = Op.getValueType(); in LowerCVTPS2PH() local
32205 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in LowerCVTPS2PH()
32209 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in LowerCVTPS2PH()
32374 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
32378 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in ReplaceNodeResults()
32381 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in ReplaceNodeResults()
32386 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
32390 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in ReplaceNodeResults()
32397 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in ReplaceNodeResults()
32426 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
32427 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
32428 VT.getVectorElementType() == MVT::i8 && "Unexpected VT!"); in ReplaceNodeResults()
32431 MVT MulVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements()); in ReplaceNodeResults()
32435 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in ReplaceNodeResults()
32436 unsigned NumConcats = 16 / VT.getVectorNumElements(); in ReplaceNodeResults()
32437 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT)); in ReplaceNodeResults()
32445 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
32446 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
32447 VT == MVT::v2i32 && "Unexpected VT!"); in ReplaceNodeResults()
32457 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Hi, in ReplaceNodeResults()
32461 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in ReplaceNodeResults()
32466 HiCmp = DAG.getNode(ISD::SRA, dl, VT, Res, DAG.getConstant(31, dl, VT)); in ReplaceNodeResults()
32469 HiCmp = DAG.getConstant(0, dl, VT); in ReplaceNodeResults()
32475 DAG.getUNDEF(VT)); in ReplaceNodeResults()
32484 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
32486 assert(VT.getSizeInBits() < 128 && 128 % VT.getSizeInBits() == 0 && in ReplaceNodeResults()
32488 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
32496 VT.getVectorElementType(), in ReplaceNodeResults()
32497 NumConcat * VT.getVectorNumElements()); in ReplaceNodeResults()
32514 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
32515 assert(VT == MVT::v2f32 && "Unexpected type (!= v2f32) on FMIN/FMAX."); in ReplaceNodeResults()
32516 SDValue UNDEF = DAG.getUNDEF(VT); in ReplaceNodeResults()
32528 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
32529 if (VT.isVector()) { in ReplaceNodeResults()
32530 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
32537 unsigned NumConcats = 128 / VT.getSizeInBits(); in ReplaceNodeResults()
32538 SmallVector<SDValue, 8> Ops0(NumConcats, DAG.getUNDEF(VT)); in ReplaceNodeResults()
32540 EVT ResVT = getTypeToTransformTo(*DAG.getContext(), VT); in ReplaceNodeResults()
32554 MVT VT = N->getSimpleValueType(0); in ReplaceNodeResults() local
32555 if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector) in ReplaceNodeResults()
32561 MVT WidenVT = getTypeToTransformTo(*DAG.getContext(), VT).getSimpleVT(); in ReplaceNodeResults()
32571 EVT EltVT = VT.getVectorElementType(); in ReplaceNodeResults()
32576 unsigned MinElts = VT.getVectorNumElements(); in ReplaceNodeResults()
32595 if (InVT == MVT::v4i64 && VT == MVT::v4i8 && isTypeLegal(MVT::v8i64)) { in ReplaceNodeResults()
32602 if (Subtarget.hasVLX() && InVT == MVT::v8i64 && VT == MVT::v8i8 && in ReplaceNodeResults()
32629 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
32632 if (!Subtarget.hasSSE41() && VT == MVT::v4i64 && in ReplaceNodeResults()
32656 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in ReplaceNodeResults()
32661 if (VT == MVT::v16i32 || VT == MVT::v8i64) { in ReplaceNodeResults()
32694 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in ReplaceNodeResults()
32706 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
32713 EVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32; in ReplaceNodeResults()
32716 DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other}, in ReplaceNodeResults()
32721 Res = DAG.getNode(N->getOpcode(), dl, VT, in ReplaceNodeResults()
32731 if (VT.isVector() && Subtarget.hasFP16() && in ReplaceNodeResults()
32733 EVT EleVT = VT.getVectorElementType(); in ReplaceNodeResults()
32775 if (VT.isVector() && VT.getScalarSizeInBits() < 32) { in ReplaceNodeResults()
32776 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
32780 unsigned NewEltWidth = std::min(128 / VT.getVectorNumElements(), 32U); in ReplaceNodeResults()
32782 VT.getVectorNumElements()); in ReplaceNodeResults()
32800 DAG.getValueType(VT.getVectorElementType())); in ReplaceNodeResults()
32807 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in ReplaceNodeResults()
32810 unsigned NumConcats = 128 / VT.getSizeInBits(); in ReplaceNodeResults()
32811 MVT ConcatVT = MVT::getVectorVT(VT.getSimpleVT().getVectorElementType(), in ReplaceNodeResults()
32812 VT.getVectorNumElements() * NumConcats); in ReplaceNodeResults()
32813 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT)); in ReplaceNodeResults()
32823 if (VT == MVT::v2i32) { in ReplaceNodeResults()
32827 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
32888 assert(!VT.isVector() && "Vectors should have been handled above!"); in ReplaceNodeResults()
32890 if ((Subtarget.hasDQI() && VT == MVT::i64 && in ReplaceNodeResults()
32919 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Res, ZeroIdx); in ReplaceNodeResults()
32926 if (VT == MVT::i128 && Subtarget.isTargetWin64()) { in ReplaceNodeResults()
32956 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
32958 if (VT.getVectorElementType() == MVT::f16 && Subtarget.hasFP16() && in ReplaceNodeResults()
32963 if (VT == MVT::v2f16 && Src.getValueType() == MVT::v2i32) in ReplaceNodeResults()
32980 if (VT != MVT::v2f32) in ReplaceNodeResults()
33085 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
33087 if (VT == MVT::v2f16 && Src.getValueType() == MVT::v2f32) { in ReplaceNodeResults()
33092 if (!Subtarget.hasFP16() && VT.getVectorElementType() == MVT::f16) { in ReplaceNodeResults()
33110 EVT NewVT = VT.getVectorElementType() == MVT::f16 ? MVT::v8f16 : MVT::v4f32; in ReplaceNodeResults()
33358 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
33359 if ((VT == MVT::v2f32 || VT == MVT::v2i32) && in ReplaceNodeResults()
33365 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
33367 EVT WideVT = getTypeToTransformTo(*DAG.getContext(), VT); in ReplaceNodeResults()
33372 DAG.getUNDEF(VT)); in ReplaceNodeResults()
33395 MVT VT = N->getSimpleValueType(0); in ReplaceNodeResults() local
33396 assert(VT.isVector() && VT.getSizeInBits() == 64 && "Unexpected VT"); in ReplaceNodeResults()
33397 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
33403 MVT LdVT = Subtarget.is64Bit() && VT.isInteger() ? MVT::i64 : MVT::f64; in ReplaceNodeResults()
33410 EVT WideVT = getTypeToTransformTo(*DAG.getContext(), VT); in ReplaceNodeResults()
34106 EVT VT) const { in isFMAFasterThanFMulAndFAdd()
34110 VT = VT.getScalarType(); in isFMAFasterThanFMulAndFAdd()
34112 if (!VT.isSimple()) in isFMAFasterThanFMulAndFAdd()
34115 switch (VT.getSimpleVT().SimpleTy) { in isFMAFasterThanFMulAndFAdd()
34134 EVT VT) const { in shouldFoldSelectWithIdentityConstant()
34139 if (!Subtarget.hasVLX() && !VT.is512BitVector()) in shouldFoldSelectWithIdentityConstant()
34141 if (!VT.isVector()) in shouldFoldSelectWithIdentityConstant()
34151 bool X86TargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const { in isShuffleMaskLegal()
34152 if (!VT.isSimple()) in isShuffleMaskLegal()
34156 if (VT.getSimpleVT().getScalarType() == MVT::i1) in isShuffleMaskLegal()
34160 if (VT.getSimpleVT().getSizeInBits() == 64) in isShuffleMaskLegal()
34165 return isTypeLegal(VT.getSimpleVT()); in isShuffleMaskLegal()
34169 EVT VT) const { in isVectorClearMaskLegal()
34173 if (VT == MVT::v32i8 || VT == MVT::v16i16) in isVectorClearMaskLegal()
34177 return isShuffleMaskLegal(Mask, VT); in isVectorClearMaskLegal()
36671 EVT VT = Op.getValueType(); in targetShrinkDemandedConstant() local
36673 unsigned EltSize = VT.getScalarSizeInBits(); in targetShrinkDemandedConstant()
36675 if (VT.isVector()) { in targetShrinkDemandedConstant()
36695 if (EltSize > ActiveBits && EltSize > 1 && isTypeLegal(VT) && in targetShrinkDemandedConstant()
36700 VT.getVectorNumElements()); in targetShrinkDemandedConstant()
36702 TLO.DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(Op), VT, in targetShrinkDemandedConstant()
36705 TLO.DAG.getNode(Opcode, SDLoc(Op), VT, Op.getOperand(0), NewC); in targetShrinkDemandedConstant()
36753 SDValue NewC = TLO.DAG.getConstant(ZeroExtendMask, DL, VT); in targetShrinkDemandedConstant()
36754 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC); in targetShrinkDemandedConstant()
36766 EVT VT = Op.getValueType(); in computeKnownBitsForTargetNode() local
36800 if (ShAmt >= VT.getScalarSizeInBits()) { in computeKnownBitsForTargetNode()
36825 getPackDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS); in computeKnownBitsForTargetNode()
36881 assert(VT.getScalarType() == MVT::i64 && in computeKnownBitsForTargetNode()
37032 if (getTargetShuffleMask(Op.getNode(), VT.getSimpleVT(), true, Ops, Mask)) { in computeKnownBitsForTargetNode()
37034 unsigned NumElts = VT.getVectorNumElements(); in computeKnownBitsForTargetNode()
37057 if (Ops[OpIdx].getValueType() != VT) { in computeKnownBitsForTargetNode()
37080 EVT VT = Op.getValueType(); in ComputeNumSignBitsForTargetNode() local
37081 unsigned VTBits = VT.getScalarSizeInBits(); in ComputeNumSignBitsForTargetNode()
37148 if (VT == MVT::f32 || VT == MVT::f64 || in ComputeNumSignBitsForTargetNode()
37149 ((VT == MVT::v4f32 || VT == MVT::v2f64) && DemandedElts == 1)) in ComputeNumSignBitsForTargetNode()
37183 if (getTargetShuffleMask(Op.getNode(), VT.getSimpleVT(), true, Ops, Mask)) { in ComputeNumSignBitsForTargetNode()
37185 unsigned NumElts = VT.getVectorNumElements(); in ComputeNumSignBitsForTargetNode()
37205 if (Ops[OpIdx].getValueType() != VT) { in ComputeNumSignBitsForTargetNode()
37236 static SDValue narrowLoadToVZLoad(LoadSDNode *LN, MVT MemVT, MVT VT, in narrowLoadToVZLoad() argument
37242 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in narrowLoadToVZLoad()
37893 auto CanonicalizeShuffleInput = [&](MVT VT, SDValue Op) { in combineX86ShuffleChain() argument
37894 if (VT.getSizeInBits() > Op.getValueSizeInBits()) in combineX86ShuffleChain()
37895 Op = widenSubVector(Op, false, Subtarget, DAG, DL, VT.getSizeInBits()); in combineX86ShuffleChain()
37896 else if (VT.getSizeInBits() < Op.getValueSizeInBits()) in combineX86ShuffleChain()
37897 Op = extractSubVector(Op, 0, DAG, DL, VT.getSizeInBits()); in combineX86ShuffleChain()
37898 return DAG.getBitcast(VT, Op); in combineX86ShuffleChain()
38923 MVT VT = Root.getSimpleValueType(); in combineX86ShufflesConstants() local
38925 unsigned SizeInBits = VT.getSizeInBits(); in combineX86ShufflesConstants()
38994 if (VT.isFloatingPoint() && (MaskSizeInBits == 32 || MaskSizeInBits == 64)) in combineX86ShufflesConstants()
39004 return DAG.getBitcast(VT, CstOp); in combineX86ShufflesConstants()
39066 EVT VT = Op.getValueType(); in combineX86ShufflesRecursively() local
39067 if (!VT.isVector() || !VT.isSimple()) in combineX86ShufflesRecursively()
39071 if (VT.getVectorElementType() == MVT::f16) in combineX86ShufflesRecursively()
39074 assert((RootSizeInBits % VT.getSizeInBits()) == 0 && in combineX86ShufflesRecursively()
39082 APInt OpDemandedElts = APInt::getAllOnes(VT.getVectorNumElements()); in combineX86ShufflesRecursively()
39088 if (llvm::any_of(OpInputs, [VT](SDValue OpInput) { in combineX86ShufflesRecursively()
39089 return OpInput.getValueSizeInBits() > VT.getSizeInBits(); in combineX86ShufflesRecursively()
39097 unsigned NumElts = VT.getVectorNumElements(); in combineX86ShufflesRecursively()
39108 if (RootSizeInBits > VT.getSizeInBits()) { in combineX86ShufflesRecursively()
39109 unsigned NumSubVecs = RootSizeInBits / VT.getSizeInBits(); in combineX86ShufflesRecursively()
39436 MVT VT = N.getSimpleValueType(); in getPSHUFShuffleMask() local
39440 getTargetShuffleMask(N.getNode(), VT, false, Ops, Mask); in getPSHUFShuffleMask()
39446 if (VT.getSizeInBits() > 128) { in getPSHUFShuffleMask()
39447 int LaneElts = 128 / VT.getScalarSizeInBits(); in getPSHUFShuffleMask()
39449 for (int i = 1, NumLanes = VT.getSizeInBits() / 128; i < NumLanes; ++i) in getPSHUFShuffleMask()
39608 static SDValue combineCommutableSHUFP(SDValue N, MVT VT, const SDLoc &DL, in combineCommutableSHUFP() argument
39611 if (VT != MVT::v4f32 && VT != MVT::v8f32 && VT != MVT::v16f32) in combineCommutableSHUFP()
39615 auto commuteSHUFP = [&VT, &DL, &DAG](SDValue Parent, SDValue V) { in combineCommutableSHUFP()
39626 return DAG.getNode(X86ISD::SHUFP, DL, VT, N1, N0, in combineCommutableSHUFP()
39634 return DAG.getNode(X86ISD::VPERMILPI, DL, VT, NewSHUFP, in combineCommutableSHUFP()
39644 return DAG.getNode(X86ISD::SHUFP, DL, VT, NewSHUFP, NewSHUFP, in combineCommutableSHUFP()
39647 return DAG.getNode(X86ISD::SHUFP, DL, VT, NewSHUFP, N1, in combineCommutableSHUFP()
39650 return DAG.getNode(X86ISD::SHUFP, DL, VT, N0, NewSHUFP, in combineCommutableSHUFP()
39800 MVT VT = V.getSimpleValueType(); in canonicalizeLaneShuffleWithRepeatedOps() local
39818 return DAG.getBitcast(VT, Res); in canonicalizeLaneShuffleWithRepeatedOps()
39838 return DAG.getBitcast(VT, Res); in canonicalizeLaneShuffleWithRepeatedOps()
39851 MVT VT = N.getSimpleValueType(); in combineTargetShuffle() local
39855 if (SDValue R = combineCommutableSHUFP(N, VT, DL, DAG)) in combineTargetShuffle()
39863 if (VT == MVT::v2f64 && Src.hasOneUse() && in combineTargetShuffle()
39886 VT.getScalarSizeInBits() % BCVT.getScalarSizeInBits() == 0) { in combineTargetShuffle()
39887 unsigned Scale = VT.getScalarSizeInBits() / BCVT.getScalarSizeInBits(); in combineTargetShuffle()
39897 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, in combineTargetShuffle()
39909 VT.getVectorNumElements()); in combineTargetShuffle()
39910 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, DL, NewVT, BC)); in combineTargetShuffle()
39915 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, in combineTargetShuffle()
39920 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0)); in combineTargetShuffle()
39927 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0)); in combineTargetShuffle()
39935 VT.getFixedSizeInBits()) { in combineTargetShuffle()
39937 VT.getSizeInBits()); in combineTargetShuffle()
39942 if (!SrcVT.isVector() && (Src.hasOneUse() || VT.isFloatingPoint()) && in combineTargetShuffle()
39945 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
39977 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
39995 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
40020 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
40040 if (LN->getMemoryVT().getSizeInBits() == VT.getScalarSizeInBits()) { in combineTargetShuffle()
40041 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
40060 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
40083 narrowLoadToVZLoad(LN, VT.getVectorElementType(), VT, DAG)) { in combineTargetShuffle()
40096 if (VT.getScalarSizeInBits() == LN->getMemoryVT().getSizeInBits()) { in combineTargetShuffle()
40097 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
40119 MVT VecVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() * 2); in combineTargetShuffle()
40122 return DAG.getBitcast(VT, Movl); in combineTargetShuffle()
40134 unsigned NumElts = VT.getVectorNumElements(); in combineTargetShuffle()
40145 return DAG.getLoad(VT, DL, DAG.getEntryNode(), CP, MPI, Alignment, in combineTargetShuffle()
40160 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), in combineTargetShuffle()
40162 VT.getScalarSizeInBits()); in combineTargetShuffle()
40165 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in combineTargetShuffle()
40166 getZeroVector(VT, Subtarget, DAG, DL), Movl, in combineTargetShuffle()
40182 if ((VT.getScalarSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 && in combineTargetShuffle()
40185 unsigned Size = VT.getVectorNumElements(); in combineTargetShuffle()
40186 unsigned Scale = VT.getScalarSizeInBits() / SrcVT.getScalarSizeInBits(); in combineTargetShuffle()
40189 VT, DAG.getNode(X86ISD::BLENDI, DL, SrcVT, N0.getOperand(0), in combineTargetShuffle()
40200 if (VT == MVT::v4f32) { in combineTargetShuffle()
40204 if (getTargetShuffleMask(N.getNode(), VT, false, Ops, Mask) && in combineTargetShuffle()
40217 Ops[i] = DAG.getBitcast(VT, SubOps[0]); in combineTargetShuffle()
40226 return DAG.getNode(X86ISD::SHUFP, DL, VT, Ops); in combineTargetShuffle()
40236 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in combineTargetShuffle()
40242 return DAG.getBitcast(VT, Res); in combineTargetShuffle()
40254 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT, in combineTargetShuffle()
40285 MVT SubVT = VT.getHalfNumVectorElementsVT(); in combineTargetShuffle()
40288 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, SubLo, SubHi); in combineTargetShuffle()
40317 MVT SVT = VT.getVectorElementType(); in combineTargetShuffle()
40322 SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl); in combineTargetShuffle()
40323 return DAG.getNode(Opcode, DL, VT, N0, SclVec); in combineTargetShuffle()
40330 assert(VT == MVT::v4f32 && "INSERTPS ValueType must be MVT::v4f32"); in combineTargetShuffle()
40340 return DAG.getNode(X86ISD::INSERTPS, DL, VT, DAG.getUNDEF(VT), Op1, in combineTargetShuffle()
40345 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT), in combineTargetShuffle()
40357 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT), in combineTargetShuffle()
40365 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1, in combineTargetShuffle()
40412 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1, in combineTargetShuffle()
40425 SDValue Insert = DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, in combineTargetShuffle()
40426 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, in combineTargetShuffle()
40451 assert(VT.getVectorElementType() == MVT::i16 && "Bad word shuffle type!"); in combineTargetShuffle()
40461 MVT DVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2); in combineTargetShuffle()
40465 return DAG.getBitcast(VT, V); in combineTargetShuffle()
40494 V = DAG.getBitcast(VT, D.getOperand(0)); in combineTargetShuffle()
40497 DL, VT, V, V); in combineTargetShuffle()
40558 EVT VT = N->getValueType(0); in isAddSubOrSubAdd() local
40560 if (!Subtarget.hasSSE3() || !TLI.isTypeLegal(VT) || in isAddSubOrSubAdd()
40561 !VT.getSimpleVT().isFloatingPoint()) in isAddSubOrSubAdd()
40623 MVT VT = N->getSimpleValueType(0); in combineShuffleToFMAddSub() local
40625 if (!Subtarget.hasAnyFMA() || !TLI.isTypeLegal(VT)) in combineShuffleToFMAddSub()
40651 return DAG.getNode(Opcode, DL, VT, FMAdd.getOperand(0), FMAdd.getOperand(1), in combineShuffleToFMAddSub()
40668 MVT VT = N->getSimpleValueType(0); in combineShuffleToAddSubOrFMAddSub() local
40675 return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2); in combineShuffleToAddSubOrFMAddSub()
40684 if (VT.is512BitVector()) in combineShuffleToAddSubOrFMAddSub()
40690 if (VT.getVectorElementType() == MVT::f16) in combineShuffleToAddSubOrFMAddSub()
40693 return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1); in combineShuffleToAddSubOrFMAddSub()
40704 EVT VT = N->getValueType(0); in combineShuffleOfConcatUndef() local
40707 if (!VT.is128BitVector() && !VT.is256BitVector()) in combineShuffleOfConcatUndef()
40710 if (VT.getVectorElementType() != MVT::i32 && in combineShuffleOfConcatUndef()
40711 VT.getVectorElementType() != MVT::i64 && in combineShuffleOfConcatUndef()
40712 VT.getVectorElementType() != MVT::f32 && in combineShuffleOfConcatUndef()
40713 VT.getVectorElementType() != MVT::f64) in combineShuffleOfConcatUndef()
40729 int NumElts = VT.getVectorNumElements(); in combineShuffleOfConcatUndef()
40736 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, N0.getOperand(0), in combineShuffleOfConcatUndef()
40738 return DAG.getVectorShuffle(VT, DL, Concat, DAG.getUNDEF(VT), Mask); in combineShuffleOfConcatUndef()
40747 MVT VT = Shuf->getSimpleValueType(0); in narrowShuffle() local
40748 if (!VT.is256BitVector() && !VT.is512BitVector()) in narrowShuffle()
40784 EVT VT = N->getValueType(0); in combineShuffle() local
40786 if (TLI.isTypeLegal(VT)) in combineShuffle()
40792 VT, SDValue(N, 0), dl, DAG, Subtarget, /*IsAfterLegalize*/ true)) in combineShuffle()
40819 APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements()); in combineShuffle()
40907 EVT VT = Op.getValueType(); in SimplifyDemandedVectorEltsForTargetNode() local
40957 assert(VT.getScalarType() == MVT::i64 && in SimplifyDemandedVectorEltsForTargetNode()
40974 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewLHS, NewRHS)); in SimplifyDemandedVectorEltsForTargetNode()
41016 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op))); in SimplifyDemandedVectorEltsForTargetNode()
41023 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc, Op.getOperand(1))); in SimplifyDemandedVectorEltsForTargetNode()
41042 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op))); in SimplifyDemandedVectorEltsForTargetNode()
41076 Op, TLO.DAG.getNode(NewOpc, dl, VT, Src.getOperand(0), NewSA)); in SimplifyDemandedVectorEltsForTargetNode()
41115 Op, TLO.DAG.getNode(NewOpc, dl, VT, Src.getOperand(0), NewSA)); in SimplifyDemandedVectorEltsForTargetNode()
41137 int NumElts = VT.getVectorNumElements(); in SimplifyDemandedVectorEltsForTargetNode()
41138 int EltSizeInBits = VT.getScalarSizeInBits(); in SimplifyDemandedVectorEltsForTargetNode()
41183 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewLHS, NewRHS)); in SimplifyDemandedVectorEltsForTargetNode()
41205 getPackDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS); in SimplifyDemandedVectorEltsForTargetNode()
41229 TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewN0, NewN1)); in SimplifyDemandedVectorEltsForTargetNode()
41242 getHorizDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS); in SimplifyDemandedVectorEltsForTargetNode()
41266 TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewN0, NewN1)); in SimplifyDemandedVectorEltsForTargetNode()
41322 if (Src.getValueType() != VT) in SimplifyDemandedVectorEltsForTargetNode()
41323 Src = widenSubVector(VT.getSimpleVT(), Src, false, Subtarget, TLO.DAG, in SimplifyDemandedVectorEltsForTargetNode()
41336 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc)); in SimplifyDemandedVectorEltsForTargetNode()
41362 if ((VT.is256BitVector() || VT.is512BitVector()) && in SimplifyDemandedVectorEltsForTargetNode()
41364 unsigned SizeInBits = VT.getSizeInBits(); in SimplifyDemandedVectorEltsForTargetNode()
41368 if (VT.is512BitVector() && DemandedElts.lshr(NumElts / 4) == 0) in SimplifyDemandedVectorEltsForTargetNode()
41378 EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(), in SimplifyDemandedVectorEltsForTargetNode()
41379 ExtSizeInBits / VT.getScalarSizeInBits()); in SimplifyDemandedVectorEltsForTargetNode()
41381 return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Bcst, 0, in SimplifyDemandedVectorEltsForTargetNode()
41387 EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(), in SimplifyDemandedVectorEltsForTargetNode()
41388 ExtSizeInBits / VT.getScalarSizeInBits()); in SimplifyDemandedVectorEltsForTargetNode()
41396 return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Bcst, 0, in SimplifyDemandedVectorEltsForTargetNode()
41410 return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Ld, 0, in SimplifyDemandedVectorEltsForTargetNode()
41414 EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(), in SimplifyDemandedVectorEltsForTargetNode()
41415 ExtSizeInBits / VT.getScalarSizeInBits()); in SimplifyDemandedVectorEltsForTargetNode()
41419 insertSubVector(TLO.DAG.getUNDEF(VT), BcstLd, 0, in SimplifyDemandedVectorEltsForTargetNode()
41440 SDValue UndefVec = TLO.DAG.getUNDEF(VT); in SimplifyDemandedVectorEltsForTargetNode()
41448 if (VT == MVT::v4f64 || VT == MVT::v4i64) { in SimplifyDemandedVectorEltsForTargetNode()
41454 SDValue UndefVec = TLO.DAG.getUNDEF(VT); in SimplifyDemandedVectorEltsForTargetNode()
41467 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, DL)); in SimplifyDemandedVectorEltsForTargetNode()
41472 SDValue UndefVec = TLO.DAG.getUNDEF(VT); in SimplifyDemandedVectorEltsForTargetNode()
41510 MVT ExtVT = VT.getSimpleVT(); in SimplifyDemandedVectorEltsForTargetNode()
41514 SDValue UndefVec = TLO.DAG.getUNDEF(VT); in SimplifyDemandedVectorEltsForTargetNode()
41538 llvm::any_of(OpInputs, [VT](SDValue V) { in SimplifyDemandedVectorEltsForTargetNode()
41539 return VT.getSizeInBits() != V.getValueSizeInBits() || in SimplifyDemandedVectorEltsForTargetNode()
41555 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); in SimplifyDemandedVectorEltsForTargetNode()
41560 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op))); in SimplifyDemandedVectorEltsForTargetNode()
41564 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, OpInputs[Src])); in SimplifyDemandedVectorEltsForTargetNode()
41569 if (OpInputs[Src].getValueType() != VT) in SimplifyDemandedVectorEltsForTargetNode()
41619 EVT VT = Op.getValueType(); in SimplifyDemandedBitsForTargetNode() local
41670 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, DemandedLHS, DemandedRHS)); in SimplifyDemandedBitsForTargetNode()
41696 NewOpc, SDLoc(Op), VT, Op0.getOperand(0), in SimplifyDemandedBitsForTargetNode()
41783 Op, TLO.DAG.getNode(X86ISD::VSRLI, SDLoc(Op), VT, Op0, Op1)); in SimplifyDemandedBitsForTargetNode()
41807 return TLO.CombineTo(Op, TLO.DAG.getNode(X86ISD::BLENDV, SDLoc(Op), VT, in SimplifyDemandedBitsForTargetNode()
41827 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT)); in SimplifyDemandedBitsForTargetNode()
41843 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, V, Op.getOperand(1))); in SimplifyDemandedBitsForTargetNode()
41887 getPackDemandedElts(VT, OriginalDemandedElts, DemandedLHS, DemandedRHS); in SimplifyDemandedBitsForTargetNode()
41906 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, Op0, Op1)); in SimplifyDemandedBitsForTargetNode()
41928 MVT NewVT = MVT::getVectorVT(NewSrcVT, VT.getVectorNumElements() * 2); in SimplifyDemandedBitsForTargetNode()
41931 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, NewBcst)); in SimplifyDemandedBitsForTargetNode()
41950 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT)); in SimplifyDemandedBitsForTargetNode()
41956 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc)); in SimplifyDemandedBitsForTargetNode()
41984 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc)); in SimplifyDemandedBitsForTargetNode()
42000 Op, TLO.DAG.getNode(X86ISD::BEXTR, DL, VT, Op0, in SimplifyDemandedBitsForTargetNode()
42001 TLO.DAG.getConstant(MaskedVal1, DL, VT))); in SimplifyDemandedBitsForTargetNode()
42032 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT)); in SimplifyDemandedBitsForTargetNode()
42076 EVT VT = Op.getValueType(); in SimplifyMultipleUseDemandedBitsForTargetNode() local
42126 llvm::all_of(ShuffleOps, [VT](SDValue V) { in SimplifyMultipleUseDemandedBitsForTargetNode()
42127 return VT.getSizeInBits() == V.getValueSizeInBits(); in SimplifyMultipleUseDemandedBitsForTargetNode()
42131 return DAG.getUNDEF(VT); in SimplifyMultipleUseDemandedBitsForTargetNode()
42133 return getZeroVector(VT.getSimpleVT(), Subtarget, DAG, SDLoc(Op)); in SimplifyMultipleUseDemandedBitsForTargetNode()
42155 return DAG.getBitcast(VT, ShuffleOps[IdentityOp.countTrailingZeros()]); in SimplifyMultipleUseDemandedBitsForTargetNode()
42270 static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src, in combineBitcastvxi1() argument
42283 return DAG.getZExtOrTrunc(V, DL, VT); in combineBitcastvxi1()
42396 return DAG.getBitcast(VT, V); in combineBitcastvxi1()
42553 static SDValue combineBitcastToBoolVector(EVT VT, SDValue V, const SDLoc &DL, in combineBitcastToBoolVector() argument
42564 return DAG.getBitcast(VT, Src); in combineBitcastToBoolVector()
42575 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, N0, in combineBitcastToBoolVector()
42588 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in combineBitcastToBoolVector()
42589 Opc == ISD::ANY_EXTEND ? DAG.getUNDEF(VT) in combineBitcastToBoolVector()
42590 : DAG.getConstant(0, DL, VT), in combineBitcastToBoolVector()
42598 if (SDValue N0 = combineBitcastToBoolVector(VT, Src0, DL, DAG, Subtarget)) in combineBitcastToBoolVector()
42599 if (SDValue N1 = combineBitcastToBoolVector(VT, Src1, DL, DAG, Subtarget)) in combineBitcastToBoolVector()
42600 return DAG.getNode(Opc, DL, VT, N0, N1); in combineBitcastToBoolVector()
42606 if ((VT == MVT::v8i1 && !Subtarget.hasDQI()) || in combineBitcastToBoolVector()
42607 ((VT == MVT::v32i1 || VT == MVT::v64i1) && !Subtarget.hasBWI())) in combineBitcastToBoolVector()
42611 if (SDValue N0 = combineBitcastToBoolVector(VT, Src0, DL, DAG, Subtarget)) in combineBitcastToBoolVector()
42613 X86ISD::KSHIFTL, DL, VT, N0, in combineBitcastToBoolVector()
42625 EVT VT = N->getValueType(0); in combineBitcast() local
42637 if (SDValue V = combineBitcastvxi1(DAG, VT, N0, dl, Subtarget)) in combineBitcast()
42642 if ((VT == MVT::v4i1 || VT == MVT::v2i1) && SrcVT.isScalarInteger() && in combineBitcast()
42646 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, N0, in combineBitcast()
42652 if ((SrcVT == MVT::v4i1 || SrcVT == MVT::v2i1) && VT.isScalarInteger() && in combineBitcast()
42670 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0); in combineBitcast()
42679 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0); in combineBitcast()
42684 if (VT.isVector() && VT.getScalarType() == MVT::i1 && in combineBitcast()
42685 SrcVT.isScalarInteger() && TLI.isTypeLegal(VT)) { in combineBitcast()
42687 combineBitcastToBoolVector(VT, N0, SDLoc(N), DAG, Subtarget)) in combineBitcast()
42697 if (VT == MVT::i8 && SrcVT == MVT::v8i1 && Subtarget.hasAVX512() && in combineBitcast()
42701 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, in combineBitcast()
42708 VT.isFloatingPoint() != SrcVT.isFloatingPoint() && VT.isVector()) { in combineBitcast()
42714 MVT MemVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(MemSize) in combineBitcast()
42716 MVT LoadVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(SrcVTSize) in combineBitcast()
42726 return DAG.getBitcast(VT, ResNode); in combineBitcast()
42733 if (VT == MVT::x86mmx) { in combineBitcast()
42741 return DAG.getNode(X86ISD::MMX_MOVW2D, DL, VT, in combineBitcast()
42746 return DAG.getBitcast(VT, DAG.getConstantFP(F64, DL, MVT::f64)); in combineBitcast()
42764 return DAG.getNode(X86ISD::MMX_MOVW2D, dl, VT, N00); in combineBitcast()
42782 return DAG.getNode(X86ISD::MOVDQ2Q, SDLoc(N00), VT, in combineBitcast()
42791 return DAG.getNode(X86ISD::MOVDQ2Q, DL, VT, in combineBitcast()
42798 if (Subtarget.hasAVX512() && VT.isScalarInteger() && in combineBitcast()
42805 VT.isVector() && VT.getVectorElementType() == MVT::i1 && in combineBitcast()
42809 return DAG.getConstant(1, SDLoc(N0), VT); in combineBitcast()
42811 return DAG.getConstant(0, SDLoc(N0), VT); in combineBitcast()
42818 VT.isVector() && VT.getVectorElementType() == MVT::i1 && in combineBitcast()
42819 isPowerOf2_32(VT.getVectorNumElements())) { in combineBitcast()
42820 unsigned NumElts = VT.getVectorNumElements(); in combineBitcast()
42842 if (EVT(CmpVT) == VT) in combineBitcast()
42850 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Ops); in combineBitcast()
42874 if (!((Subtarget.hasSSE1() && VT == MVT::f32) || in combineBitcast()
42875 (Subtarget.hasSSE2() && VT == MVT::f64) || in combineBitcast()
42876 (Subtarget.hasFP16() && VT == MVT::f16) || in combineBitcast()
42877 (Subtarget.hasSSE2() && VT.isInteger() && VT.isVector() && in combineBitcast()
42878 TLI.isTypeLegal(VT)))) in combineBitcast()
42888 LogicOp0.getOperand(0).getValueType() == VT && in combineBitcast()
42890 SDValue CastedOp1 = DAG.getBitcast(VT, LogicOp1); in combineBitcast()
42891 unsigned Opcode = VT.isFloatingPoint() ? FPOpcode : N0.getOpcode(); in combineBitcast()
42892 return DAG.getNode(Opcode, DL0, VT, LogicOp0.getOperand(0), CastedOp1); in combineBitcast()
42897 LogicOp1.getOperand(0).getValueType() == VT && in combineBitcast()
42899 SDValue CastedOp0 = DAG.getBitcast(VT, LogicOp0); in combineBitcast()
42900 unsigned Opcode = VT.isFloatingPoint() ? FPOpcode : N0.getOpcode(); in combineBitcast()
42901 return DAG.getNode(Opcode, DL0, VT, LogicOp1.getOperand(0), CastedOp0); in combineBitcast()
42993 MVT VT = MVT::getVectorVT(MVT::i32, Ops[0].getValueSizeInBits() / 32); in createVPDPBUSD() local
42994 return DAG.getNode(X86ISD::VPDPBUSD, DL, VT, Ops); in createVPDPBUSD()
43025 MVT VT = MVT::getVectorVT(MVT::i64, Ops[0].getValueSizeInBits() / 64); in createPSADBW() local
43026 return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops); in createPSADBW()
43259 EVT VT = Extract->getOperand(0).getValueType(); in combineVPDPBUSDPattern() local
43260 if (!isPowerOf2_32(VT.getVectorNumElements())) in combineVPDPBUSDPattern()
43293 unsigned Stages = Log2_32(VT.getVectorNumElements()); in combineVPDPBUSDPattern()
43331 EVT VT = Extract->getOperand(0).getValueType(); in combineBasicSADPattern() local
43332 if (!isPowerOf2_32(VT.getVectorNumElements())) in combineBasicSADPattern()
43368 unsigned Stages = Log2_32(VT.getVectorNumElements()); in combineBasicSADPattern()
43405 EVT VT = N->getValueType(0); in combineExtractWithShuffle() local
43425 if (SrcOpVT.isScalarInteger() && VT.isInteger() && in combineExtractWithShuffle()
43432 SrcOp = DAG.getZExtOrTrunc(SrcOp, dl, VT); in combineExtractWithShuffle()
43444 VT.getSizeInBits() == SrcBCWidth && SrcEltBits == SrcBCWidth) { in combineExtractWithShuffle()
43445 SDValue Load = DAG.getLoad(VT, dl, MemIntr->getChain(), in combineExtractWithShuffle()
43457 if (SrcBC.getOpcode() == ISD::SCALAR_TO_VECTOR && VT.isInteger() && in combineExtractWithShuffle()
43472 Scl = DAG.getZExtOrTrunc(Scl, dl, VT); in combineExtractWithShuffle()
43484 return DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(ExtractVT, Src), in combineExtractWithShuffle()
43578 return DAG.getUNDEF(VT); in combineExtractWithShuffle()
43581 return VT.isFloatingPoint() ? DAG.getConstantFP(0.0, dl, VT) in combineExtractWithShuffle()
43582 : DAG.getConstant(0, dl, VT); in combineExtractWithShuffle()
43587 return DAG.getZExtOrTrunc(V, dl, VT); in combineExtractWithShuffle()
43599 EVT VT = ExtElt->getValueType(0); in scalarizeExtEltFP() local
43604 if (!Vec.hasOneUse() || !isNullConstant(Index) || VecVT.getScalarType() != VT) in scalarizeExtEltFP()
43609 if (Vec.getOpcode() == ISD::SETCC && VT == MVT::i1) { in scalarizeExtEltFP()
43620 return DAG.getNode(Vec.getOpcode(), DL, VT, Ext0, Ext1, Vec.getOperand(2)); in scalarizeExtEltFP()
43623 if (!(VT == MVT::f16 && Subtarget.hasFP16()) && VT != MVT::f32 && in scalarizeExtEltFP()
43624 VT != MVT::f64) in scalarizeExtEltFP()
43642 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in scalarizeExtEltFP()
43644 SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in scalarizeExtEltFP()
43646 return DAG.getNode(ISD::SELECT, DL, VT, Ext0, Ext1, Ext2); in scalarizeExtEltFP()
43683 ExtOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op, Index)); in scalarizeExtEltFP()
43684 return DAG.getNode(Vec.getOpcode(), DL, VT, ExtOps); in scalarizeExtEltFP()
43712 EVT VT = ExtElt->getValueType(0); in combineArithReduction() local
43714 if (VecVT.getScalarType() != VT) in combineArithReduction()
43741 if (VT != MVT::i8 || NumElts < 4 || !isPowerOf2_32(NumElts)) in combineArithReduction()
43770 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
43779 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
43787 if (VT == MVT::i8) { in combineArithReduction()
43803 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
43822 MVT VT = MVT::getVectorVT(MVT::i64, Ops[0].getValueSizeInBits() / 64); in combineArithReduction() local
43824 return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops[0], Zero); in combineArithReduction()
43843 VecVT = MVT::getVectorVT(VT.getSimpleVT(), 128 / VT.getSizeInBits()); in combineArithReduction()
43845 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
43875 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
43893 EVT VT = N->getValueType(0); in combineExtractVectorElt() local
43899 return IsPextr ? DAG.getConstant(0, dl, VT) : DAG.getUNDEF(VT); in combineExtractVectorElt()
43902 if (CIdx && VT.isInteger()) { in combineExtractVectorElt()
43910 return IsPextr ? DAG.getConstant(0, dl, VT) : DAG.getUNDEF(VT); in combineExtractVectorElt()
43911 return DAG.getConstant(EltBits[Idx].zext(VT.getScalarSizeInBits()), dl, in combineExtractVectorElt()
43912 VT); in combineExtractVectorElt()
43919 APInt::getAllOnes(VT.getSizeInBits()), DCI)) in combineExtractVectorElt()
43930 return DAG.getZExtOrTrunc(Scl, dl, VT); in combineExtractVectorElt()
43941 VT == MVT::i64 && SrcVT == MVT::v1i64 && isNullConstant(EltIdx)) { in combineExtractVectorElt()
43946 return DAG.getBitcast(VT, InputVector); in combineExtractVectorElt()
43951 VT == MVT::i32 && SrcVT == MVT::v2i32 && isNullConstant(EltIdx)) { in combineExtractVectorElt()
44041 if (LoadVec && CIdx && ISD::isNormalLoad(LoadVec) && VT.isInteger() && in combineExtractVectorElt()
44042 SrcVT.getVectorElementType() == VT && DCI.isAfterLegalizeDAG() && in combineExtractVectorElt()
44047 unsigned PtrOff = VT.getSizeInBits() * CIdx->getZExtValue() / 8; in combineExtractVectorElt()
44051 DAG.getLoad(VT, dl, LoadVec->getChain(), NewPtr, MPI, Alignment, in combineExtractVectorElt()
44063 unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N0, SelectionDAG &DAG, in combineToExtendBoolVectorInReg() argument
44073 EVT SVT = VT.getScalarType(); in combineToExtendBoolVectorInReg()
44079 if (!VT.isVector()) in combineToExtendBoolVectorInReg()
44093 unsigned NumElts = VT.getVectorNumElements(); in combineToExtendBoolVectorInReg()
44106 Vec = DAG.getBitcast(VT, Vec); in combineToExtendBoolVectorInReg()
44110 Vec = DAG.getVectorShuffle(VT, DL, Vec, Vec, ShuffleMask); in combineToExtendBoolVectorInReg()
44124 Vec = DAG.getBitcast(VT, Vec); in combineToExtendBoolVectorInReg()
44130 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl); in combineToExtendBoolVectorInReg()
44132 Vec = DAG.getVectorShuffle(VT, DL, Vec, Vec, ShuffleMask); in combineToExtendBoolVectorInReg()
44142 SDValue BitMask = DAG.getBuildVector(VT, DL, Bits); in combineToExtendBoolVectorInReg()
44143 Vec = DAG.getNode(ISD::AND, DL, VT, Vec, BitMask); in combineToExtendBoolVectorInReg()
44146 EVT CCVT = VT.changeVectorElementType(MVT::i1); in combineToExtendBoolVectorInReg()
44148 Vec = DAG.getSExtOrTrunc(Vec, DL, VT); in combineToExtendBoolVectorInReg()
44154 return DAG.getNode(ISD::SRL, DL, VT, Vec, in combineToExtendBoolVectorInReg()
44155 DAG.getConstant(EltSizeInBits - 1, DL, VT)); in combineToExtendBoolVectorInReg()
44168 EVT VT = LHS.getValueType(); in combineVSelectWithAllOnesOrZeros() local
44187 if (VT.isFloatingPoint()) in combineVSelectWithAllOnesOrZeros()
44188 return DAG.getConstantFP(0.0, DL, VT); in combineVSelectWithAllOnesOrZeros()
44189 return DAG.getConstant(0, DL, VT); in combineVSelectWithAllOnesOrZeros()
44197 if (CondVT.getScalarSizeInBits() != VT.getScalarSizeInBits()) in combineVSelectWithAllOnesOrZeros()
44207 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT) == in combineVSelectWithAllOnesOrZeros()
44229 return DAG.getBitcast(VT, Cond); in combineVSelectWithAllOnesOrZeros()
44238 return DAG.getBitcast(VT, Or); in combineVSelectWithAllOnesOrZeros()
44245 return DAG.getBitcast(VT, And); in combineVSelectWithAllOnesOrZeros()
44258 return DAG.getBitcast(VT, AndN); in combineVSelectWithAllOnesOrZeros()
44275 EVT VT = N->getValueType(0); in narrowVectorSelect() local
44276 if (!VT.is256BitVector()) in narrowVectorSelect()
44293 return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, { Cond, TVal, FVal }, in narrowVectorSelect()
44309 EVT VT = N->getValueType(0); in combineSelectOfTwoConstants() local
44310 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in combineSelectOfTwoConstants()
44341 ((VT == MVT::i32 || VT == MVT::i64) && in combineSelectOfTwoConstants()
44353 SDValue R = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond); in combineSelectOfTwoConstants()
44357 R = DAG.getNode(ISD::MUL, DL, VT, R, DAG.getConstant(AbsDiff, DL, VT)); in combineSelectOfTwoConstants()
44361 R = DAG.getNode(ISD::ADD, DL, VT, R, SDValue(FalseC, 0)); in combineSelectOfTwoConstants()
44385 EVT VT = N->getValueType(0); in combineVSelectToBLENDV() local
44397 if (!TLI.isOperationLegalOrCustom(ISD::VSELECT, VT)) in combineVSelectToBLENDV()
44402 if (VT.getVectorElementType() == MVT::i16) in combineVSelectToBLENDV()
44405 if (VT.is128BitVector() && !Subtarget.hasSSE41()) in combineVSelectToBLENDV()
44408 if (VT == MVT::v32i8 && !Subtarget.hasAVX2()) in combineVSelectToBLENDV()
44411 if (VT.is512BitVector()) in combineVSelectToBLENDV()
44482 EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc &DL, in combineLogicBlendIntoConditionalNegate() argument
44524 return DAG.getBitcast(VT, Res); in combineLogicBlendIntoConditionalNegate()
44541 EVT VT = LHS.getValueType(); in combineSelect() local
44550 CondVT.getScalarSizeInBits() == VT.getScalarSizeInBits() && in combineSelect()
44553 if (SDValue V = combineLogicBlendIntoConditionalNegate(VT, Cond, RHS, LHS, in combineSelect()
44563 return DAG.getVectorShuffle(VT, DL, LHS, RHS, Mask); in combineSelect()
44572 MVT SimpleVT = VT.getSimpleVT(); in combineSelect()
44578 int NumElts = VT.getVectorNumElements(); in combineSelect()
44590 LHS = DAG.getNode(X86ISD::PSHUFB, DL, VT, LHS.getOperand(0), in combineSelect()
44592 RHS = DAG.getNode(X86ISD::PSHUFB, DL, VT, RHS.getOperand(0), in combineSelect()
44594 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS); in combineSelect()
44603 if (Cond.getOpcode() == ISD::SETCC && VT.isFloatingPoint() && in combineSelect()
44604 VT != MVT::f80 && VT != MVT::f128 && !isSoftFP16(VT, Subtarget) && in combineSelect()
44605 (TLI.isTypeLegal(VT) || VT == MVT::v2f32) && in combineSelect()
44607 (Subtarget.hasSSE1() && VT.getScalarType() == MVT::f32))) { in combineSelect()
44757 Cond.getOpcode() == ISD::SETCC && (VT == MVT::f32 || VT == MVT::f64)) { in combineSelect()
44766 return DAG.getNode(ISD::SELECT, DL, VT, AndNode, RHS, LHS); in combineSelect()
44779 (VT.getVectorElementType() == MVT::i8 || in combineSelect()
44780 VT.getVectorElementType() == MVT::i16)) { in combineSelect()
44781 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond); in combineSelect()
44782 return DAG.getNode(N->getOpcode(), DL, VT, Cond, LHS, RHS); in combineSelect()
44809 VT.getSizeInBits()); in combineSelect()
44811 VT.getSizeInBits()); in combineSelect()
44816 return extractSubVector(Res, 0, DAG, DL, VT.getSizeInBits()); in combineSelect()
44854 return DAG.getSelect(DL, VT, Cond, LHS, RHS); in combineSelect()
44859 return DAG.getSelect(DL, VT, Cond, LHS, RHS); in combineSelect()
44888 return DAG.getSelect(DL, VT, Cond, LHS, RHS.getOperand(2)); in combineSelect()
44904 return DAG.getSelect(DL, VT, CondNew, RHS, LHS); in combineSelect()
44911 TLI.isTypeLegal(VT.getScalarType())) { in combineSelect()
44912 EVT ExtCondVT = VT.changeVectorElementTypeToInteger(); in combineSelect()
44916 return DAG.getSelect(DL, VT, ExtCond, LHS, RHS); in combineSelect()
44921 if (!TLI.isTypeLegal(VT) || isSoftFP16(VT, Subtarget)) in combineSelect()
44936 return DAG.getNode(N->getOpcode(), DL, VT, in combineSelect()
44943 return DAG.getNode(N->getOpcode(), DL, VT, Cond, RHS, LHS); in combineSelect()
44952 if (N->getOpcode() == ISD::SELECT && VT.isVector() && in combineSelect()
44953 VT.getVectorElementType() == MVT::i1 && in combineSelect()
44954 (DCI.isBeforeLegalize() || (VT != MVT::v64i1 || Subtarget.is64Bit()))) { in combineSelect()
44955 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getVectorNumElements()); in combineSelect()
44976 return DAG.getBitcast(VT, Select); in combineSelect()
44989 Cond.getOperand(0).getValueType() == VT) { in combineSelect()
44997 return DAG.getSelect(DL, VT, NotCond, RHS, LHS); in combineSelect()
45003 unsigned EltBitWidth = VT.getScalarSizeInBits(); in combineSelect()
45005 TLI.isTypeLegal(VT) && ((Subtarget.hasAVX() && EltBitWidth == 32) || in combineSelect()
45015 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) { in combineSelect()
45021 SDValue ShlAmt = getConstVector(ShlVals, VT.getSimpleVT(), DAG, DL); in combineSelect()
45022 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, And.getOperand(0), ShlAmt); in combineSelect()
45025 return DAG.getSelect(DL, VT, NewCond, RHS, LHS); in combineSelect()
45400 EVT VT = EFLAGS.getValueType(); in combinePTESTCC() local
45437 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
45446 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
45458 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
45466 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
45478 assert(VT == MVT::i32 && "Expected i32 EFLAGS comparison result"); in combinePTESTCC()
45503 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, Op1, Op1); in combinePTESTCC()
45507 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, Op0, Op0); in combinePTESTCC()
45520 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
45970 EVT VT = N->getValueType(0); in combineCMov() local
45972 SDValue Diff = DAG.getNode(ISD::SUB, DL, VT, Const, Add.getOperand(1)); in combineCMov()
45974 DAG.getNode(X86ISD::CMOV, DL, VT, Diff, Add.getOperand(0), in combineCMov()
45976 return DAG.getNode(ISD::ADD, DL, VT, CMov, Add.getOperand(1)); in combineCMov()
45987 EVT VT = N->getOperand(0).getValueType(); in canReduceVMulWidth() local
45988 if (VT.getScalarSizeInBits() != 32) in canReduceVMulWidth()
46070 EVT VT = N->getOperand(0).getValueType(); in reduceVMULWidth() local
46071 unsigned NumElts = VT.getVectorNumElements(); in reduceVMULWidth()
46087 DL, VT, MulLo); in reduceVMULWidth()
46115 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi); in reduceVMULWidth()
46119 EVT VT, const SDLoc &DL) { in combineMulSpecial() argument
46122 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMulSpecial()
46123 DAG.getConstant(Mult, DL, VT)); in combineMulSpecial()
46124 Result = DAG.getNode(ISD::SHL, DL, VT, Result, in combineMulSpecial()
46126 Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result, in combineMulSpecial()
46132 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMulSpecial()
46133 DAG.getConstant(Mul1, DL, VT)); in combineMulSpecial()
46134 Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, Result, in combineMulSpecial()
46135 DAG.getConstant(Mul2, DL, VT)); in combineMulSpecial()
46136 Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result, in combineMulSpecial()
46155 return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), in combineMulSpecial()
46180 return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), in combineMulSpecial()
46193 SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMulSpecial()
46195 SDValue Shift2 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMulSpecial()
46197 return DAG.getNode(ISD::ADD, DL, VT, Shift1, Shift2); in combineMulSpecial()
46215 EVT VT = N->getValueType(0); in combineMulToPMADDWD() local
46218 if (!VT.isVector() || VT.getVectorElementType() != MVT::i32) in combineMulToPMADDWD()
46223 unsigned NumElts = VT.getVectorNumElements(); in combineMulToPMADDWD()
46269 return DAG.getNode(ISD::AND, SDLoc(N), VT, Op, in combineMulToPMADDWD()
46270 DAG.getConstant(0xFFFF, SDLoc(N), VT)); in combineMulToPMADDWD()
46274 if (Src.getScalarValueSizeInBits() == 16 && VT.getSizeInBits() <= 128) in combineMulToPMADDWD()
46275 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, Src); in combineMulToPMADDWD()
46279 EVT ExtVT = VT.changeVectorElementType(MVT::i16); in combineMulToPMADDWD()
46281 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, Src); in combineMulToPMADDWD()
46289 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(N), VT, Src); in combineMulToPMADDWD()
46294 return DAG.getNode(X86ISD::VSRLI, SDLoc(N), VT, Op.getOperand(0), in combineMulToPMADDWD()
46315 return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, {N0, N1}, in combineMulToPMADDWD()
46324 EVT VT = N->getValueType(0); in combineMulToPMULDQ() local
46327 if (!VT.isVector() || VT.getVectorElementType() != MVT::i64 || in combineMulToPMULDQ()
46328 VT.getVectorNumElements() < 2 || in combineMulToPMULDQ()
46329 !isPowerOf2_32(VT.getVectorNumElements())) in combineMulToPMULDQ()
46343 return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, { N0, N1 }, in combineMulToPMULDQ()
46354 return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, { N0, N1 }, in combineMulToPMULDQ()
46364 EVT VT = N->getValueType(0); in combineMul() local
46372 if (DCI.isBeforeLegalize() && VT.isVector()) in combineMul()
46387 if (VT != MVT::i64 && VT != MVT::i32) in combineMul()
46402 SDValue NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMul()
46403 DAG.getConstant(AbsMulAmt, DL, VT)); in combineMul()
46405 NewMul = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in combineMul()
46440 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
46443 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMul()
46444 DAG.getConstant(MulAmt1, DL, VT)); in combineMul()
46447 NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul, in combineMul()
46450 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul, in combineMul()
46451 DAG.getConstant(MulAmt2, DL, VT)); in combineMul()
46455 NewMul = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in combineMul()
46458 NewMul = combineMulSpecial(C->getZExtValue(), N, DAG, VT, DL); in combineMul()
46462 C->getZExtValue() != (VT == MVT::i64 ? UINT64_MAX : UINT32_MAX) && in combineMul()
46468 ISD::ADD, DL, VT, N->getOperand(0), in combineMul()
46469 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
46474 NewMul = DAG.getNode(ISD::SUB, DL, VT, in combineMul()
46475 DAG.getConstant(0, DL, VT), NewMul); in combineMul()
46478 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
46483 NewMul = DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), NewMul); in combineMul()
46485 NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0)); in combineMul()
46488 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
46491 NewMul = DAG.getNode(ISD::ADD, DL, VT, NewMul, N->getOperand(0)); in combineMul()
46492 NewMul = DAG.getNode(ISD::ADD, DL, VT, NewMul, N->getOperand(0)); in combineMul()
46495 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
46498 NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0)); in combineMul()
46499 NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0)); in combineMul()
46528 EVT VT = N->getValueType(0); in combineShiftToPMULH() local
46529 if (!VT.isVector() || VT.getVectorElementType().getSizeInBits() < 32) in combineShiftToPMULH()
46559 return DAG.getNode(ExtOpc, DL, VT, Mulh); in combineShiftToPMULH()
46566 EVT VT = N0.getValueType(); in combineShiftLeft() local
46570 if (VT.isInteger() && !VT.isVector() && in combineShiftLeft()
46600 return DAG.getNode(ISD::AND, DL, VT, N00, DAG.getConstant(Mask, DL, VT)); in combineShiftLeft()
46615 return DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, N0); in combineShiftLeft()
46625 EVT VT = N0.getValueType(); in combineShiftRightArithmetic() local
46626 unsigned Size = VT.getSizeInBits(); in combineShiftRightArithmetic()
46642 if (VT.isVector() || N1.getOpcode() != ISD::Constant || in combineShiftRightArithmetic()
46664 DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, N00, DAG.getValueType(SVT)); in combineShiftRightArithmetic()
46669 return DAG.getNode(ISD::SHL, DL, VT, NN, in combineShiftRightArithmetic()
46671 return DAG.getNode(ISD::SRA, DL, VT, NN, in combineShiftRightArithmetic()
46682 EVT VT = N0.getValueType(); in combineShiftRightLogical() local
46723 SDValue NewMask = DAG.getConstant(NewMaskVal, DL, VT); in combineShiftRightLogical()
46724 SDValue NewShift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1); in combineShiftRightLogical()
46725 return DAG.getNode(ISD::AND, DL, VT, NewShift, NewMask); in combineShiftRightLogical()
46736 EVT VT = N->getValueType(0); in combineHorizOpWithShuffle() local
46751 if (VT.is128BitVector() && SrcVT.getScalarSizeInBits() <= 32) { in combineHorizOpWithShuffle()
46764 MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f32 : MVT::v4i32; in combineHorizOpWithShuffle()
46768 SDValue Res = DAG.getNode(Opcode, DL, VT, Lo, Hi); in combineHorizOpWithShuffle()
46771 return DAG.getBitcast(VT, Res); in combineHorizOpWithShuffle()
46778 if (VT.is128BitVector() && SrcVT.getScalarSizeInBits() <= 32) { in combineHorizOpWithShuffle()
46825 MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f32 : MVT::v4i32; in combineHorizOpWithShuffle()
46826 SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS); in combineHorizOpWithShuffle()
46829 return DAG.getBitcast(VT, Res); in combineHorizOpWithShuffle()
46835 if (VT.is256BitVector() && Subtarget.hasInt256()) { in combineHorizOpWithShuffle()
46861 MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f64 : MVT::v4i64; in combineHorizOpWithShuffle()
46862 SDValue Res = DAG.getNode(Opcode, DL, VT, DAG.getBitcast(SrcVT, Op00), in combineHorizOpWithShuffle()
46866 return DAG.getBitcast(VT, Res); in combineHorizOpWithShuffle()
46881 EVT VT = N->getValueType(0); in combineVectorPack() local
46884 unsigned NumDstElts = VT.getVectorNumElements(); in combineVectorPack()
46885 unsigned DstBitsPerElt = VT.getScalarSizeInBits(); in combineVectorPack()
46900 unsigned NumLanes = VT.getSizeInBits() / 128; in combineVectorPack()
46944 return getConstVector(Bits, Undefs, VT.getSimpleVT(), DAG, SDLoc(N)); in combineVectorPack()
46954 N0.getOpcode() == ISD::TRUNCATE && N1.isUndef() && VT == MVT::v16i8 && in combineVectorPack()
46960 return DAG.getNode(X86ISD::VTRUNC, SDLoc(N), VT, N0.getOperand(0)); in combineVectorPack()
46966 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Concat); in combineVectorPack()
46971 if (VT.is128BitVector()) { in combineVectorPack()
46988 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Src0, Src1); in combineVectorPack()
46996 return getEXTEND_VECTOR_INREG(ExtOpc, SDLoc(N), VT, N0.getOperand(0), in combineVectorPack()
47016 MVT VT = N->getSimpleValueType(0); in combineVectorHADDSUB() local
47035 MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32); in combineVectorHADDSUB()
47043 return DAG.getNode(N->getOpcode(), DL, VT, DAG.getBitcast(VT, NewLHS), in combineVectorHADDSUB()
47044 DAG.getBitcast(VT, NewRHS)); in combineVectorHADDSUB()
47062 EVT VT = N->getValueType(0); in combineVectorShiftVar() local
47068 return DAG.getConstant(0, SDLoc(N), VT); in combineVectorShiftVar()
47075 return getTargetVShiftByConstNode(X86Opc, SDLoc(N), VT.getSimpleVT(), N0, in combineVectorShiftVar()
47080 APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements()); in combineVectorShiftVar()
47095 EVT VT = N->getValueType(0); in combineVectorShiftImm() local
47097 unsigned NumBitsPerElt = VT.getScalarSizeInBits(); in combineVectorShiftImm()
47098 assert(VT == N0.getValueType() && (NumBitsPerElt % 8) == 0 && in combineVectorShiftImm()
47105 return DAG.getConstant(0, SDLoc(N), VT); in combineVectorShiftImm()
47112 return DAG.getConstant(0, SDLoc(N), VT); in combineVectorShiftImm()
47124 return DAG.getConstant(0, SDLoc(N), VT); in combineVectorShiftImm()
47130 return DAG.getConstant(-1, SDLoc(N), VT); in combineVectorShiftImm()
47140 return DAG.getConstant(0, SDLoc(N), VT); in combineVectorShiftImm()
47143 return DAG.getNode(Opcode, SDLoc(N), VT, N0.getOperand(0), in combineVectorShiftImm()
47159 assert(EltBits.size() == VT.getVectorNumElements() && in combineVectorShiftImm()
47177 return getConstVector(EltBits, UndefElts, VT.getSimpleVT(), DAG, SDLoc(N)); in combineVectorShiftImm()
47191 EVT VT = N->getValueType(0); in combineVectorInsert() local
47192 assert(((N->getOpcode() == X86ISD::PINSRB && VT == MVT::v16i8) || in combineVectorInsert()
47193 (N->getOpcode() == X86ISD::PINSRW && VT == MVT::v8i16) || in combineVectorInsert()
47198 unsigned NumBitsPerElt = VT.getScalarSizeInBits(); in combineVectorInsert()
47206 if (VT.isSimple() && DCI.isAfterLegalizeDAG()) { in combineVectorInsert()
47238 EVT VT = CMP00.getValueType(); in combineCompareEqual() local
47240 if (VT == MVT::f32 || VT == MVT::f64 || in combineCompareEqual()
47241 (VT == MVT::f16 && Subtarget.hasFP16())) { in combineCompareEqual()
47328 MVT VT = N->getSimpleValueType(0); in combineAndNotIntoANDNP() local
47329 if (!VT.is128BitVector() && !VT.is256BitVector() && !VT.is512BitVector()) in combineAndNotIntoANDNP()
47336 auto GetNot = [&VT, &DAG](SDValue V) { in combineAndNotIntoANDNP()
47347 return DAG.getNode(X86ISD::VBROADCAST, SDLoc(V), VT, in combineAndNotIntoANDNP()
47362 X = DAG.getBitcast(VT, X); in combineAndNotIntoANDNP()
47363 Y = DAG.getBitcast(VT, Y); in combineAndNotIntoANDNP()
47364 return DAG.getNode(X86ISD::ANDNP, SDLoc(N), VT, X, Y); in combineAndNotIntoANDNP()
47376 static SDValue PromoteMaskArithmetic(SDNode *N, EVT VT, SelectionDAG &DAG, in PromoteMaskArithmetic() argument
47391 if (!TLI.isOperationLegalOrPromote(N->getOpcode(), VT)) in PromoteMaskArithmetic()
47394 if (SDValue NN0 = PromoteMaskArithmetic(N0.getNode(), VT, DAG, Depth + 1)) in PromoteMaskArithmetic()
47402 if (N0.getOperand(0).getValueType() != VT) in PromoteMaskArithmetic()
47408 if (SDValue NN1 = PromoteMaskArithmetic(N1.getNode(), VT, DAG, Depth + 1)) in PromoteMaskArithmetic()
47413 N1.getOperand(0).getValueType() == VT; in PromoteMaskArithmetic()
47420 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N1); in PromoteMaskArithmetic()
47423 return DAG.getNode(N->getOpcode(), DL, VT, N0, N1); in PromoteMaskArithmetic()
47434 EVT VT = N->getValueType(0); in PromoteMaskArithmetic() local
47435 assert(VT.isVector() && "Expected vector type"); in PromoteMaskArithmetic()
47446 SDValue Op = PromoteMaskArithmetic(Narrow.getNode(), VT, DAG, 0); in PromoteMaskArithmetic()
47456 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, in PromoteMaskArithmetic()
47478 EVT VT = N->getValueType(0); in convertIntLogicToFPLogic() local
47501 return DAG.getBitcast(VT, FPLogic); in convertIntLogicToFPLogic()
47504 if (VT != MVT::i1 || N0.getOpcode() != ISD::SETCC || !N0.hasOneUse() || in convertIntLogicToFPLogic()
47534 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Logic, ZeroIndex); in convertIntLogicToFPLogic()
47581 EVT VT = N->getValueType(0); in combineBitOpWithShift() local
47607 return DAG.getBitcast(VT, Shift); in combineBitOpWithShift()
47621 EVT VT = Op0.getValueType(); in combineAndMaskToShift() local
47622 if (VT != Op1.getValueType() || !VT.isSimple() || !VT.isInteger()) in combineAndMaskToShift()
47634 if (N->getValueType(0) == VT && in combineAndMaskToShift()
47635 supportedVectorShiftWithImm(VT.getSimpleVT(), Subtarget, ISD::SRA)) { in combineAndMaskToShift()
47649 getTargetVShiftByConstNode(X86ISD::VSRAI, DL, VT.getSimpleVT(), X, in combineAndMaskToShift()
47650 VT.getScalarSizeInBits() - 1, DAG); in combineAndMaskToShift()
47651 return DAG.getNode(X86ISD::ANDNP, DL, VT, Sra, Y); in combineAndMaskToShift()
47664 if (!supportedVectorShiftWithImm(VT.getSimpleVT(), Subtarget, ISD::SRL)) in combineAndMaskToShift()
47667 unsigned EltBitWidth = VT.getScalarSizeInBits(); in combineAndMaskToShift()
47674 SDValue Shift = DAG.getNode(X86ISD::VSRLI, DL, VT, Op0, ShAmt); in combineAndMaskToShift()
47698 static bool hasBZHI(const X86Subtarget &Subtarget, MVT VT) { in hasBZHI() argument
47699 if (Subtarget.hasBMI2() && VT.isScalarInteger()) { in hasBZHI()
47700 switch (VT.getSizeInBits()) { in hasBZHI()
47721 MVT VT = Node->getSimpleValueType(0); in combineAndLoadToBZHI() local
47725 if (!hasBZHI(Subtarget, VT)) in combineAndLoadToBZHI()
47751 VT.getSizeInBits() || in combineAndLoadToBZHI()
47774 SDValue SizeC = DAG.getConstant(VT.getSizeInBits(), dl, MVT::i32); in combineAndLoadToBZHI()
47785 SDValue AllOnes = DAG.getAllOnesConstant(dl, VT); in combineAndLoadToBZHI()
47786 SDValue LShr = DAG.getNode(ISD::SRL, dl, VT, AllOnes, Sub); in combineAndLoadToBZHI()
47788 return DAG.getNode(ISD::AND, dl, VT, Inp, LShr); in combineAndLoadToBZHI()
47805 EVT VT = N->getValueType(0); in combineScalarAndWithMaskSetcc() local
47813 assert(!VT.isVector() && "Expected scalar VT!"); in combineScalarAndWithMaskSetcc()
47857 return DAG.getBitcast(VT, Concat); in combineScalarAndWithMaskSetcc()
47865 EVT VT = N->getValueType(0); in combineAnd() local
47870 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) { in combineAnd()
47878 if (VT == MVT::i64 && Subtarget.is64Bit() && !isa<ConstantSDNode>(N1)) { in combineAnd()
47891 if (VT == MVT::i1) { in combineAnd()
47941 if (VT.isVector() && getTargetConstantFromNode(N1)) { in combineAnd()
47945 DAG.ComputeNumSignBits(N1) == VT.getScalarSizeInBits() && in combineAnd()
47947 SDValue MaskMul = DAG.getNode(ISD::AND, dl, VT, N0.getOperand(1), N1); in combineAnd()
47948 return DAG.getNode(Opc0, dl, VT, N0.getOperand(0), MaskMul); in combineAnd()
47979 if (!(Subtarget.hasBMI2() && !ContainsNOT && VT.getSizeInBits() >= 32)) in combineAnd()
47981 return DAG.getZExtOrTrunc(getSETCC(X86CC, BT, dl, DAG), dl, VT); in combineAnd()
47985 if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) { in combineAnd()
47996 int NumElts = VT.getVectorNumElements(); in combineAnd()
47997 int EltSizeInBits = VT.getScalarSizeInBits(); in combineAnd()
48035 return DAG.getNode(ISD::AND, dl, VT, NewN0 ? NewN0 : N0, in combineAnd()
48040 if ((VT.getScalarSizeInBits() % 8) == 0 && in combineAnd()
48050 if (VT == SrcVecVT.getScalarType() && N0->isOnlyUserOf(SrcVec.getNode()) && in combineAnd()
48073 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Shuffle, in combineAnd()
48086 MVT VT = N->getSimpleValueType(0); in canonicalizeBitSelect() local
48087 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in canonicalizeBitSelect()
48088 if (!VT.isVector() || (EltSizeInBits % 8) != 0) in canonicalizeBitSelect()
48098 if (!(Subtarget.hasXOP() || useVPTERNLOG(Subtarget, VT) || in canonicalizeBitSelect()
48122 if (useVPTERNLOG(Subtarget, VT)) { in canonicalizeBitSelect()
48127 MVT::getVectorVT(OpSVT, VT.getSizeInBits() / OpSVT.getSizeInBits()); in canonicalizeBitSelect()
48134 return DAG.getBitcast(VT, Res); in canonicalizeBitSelect()
48139 DAG.getNode(X86ISD::ANDNP, DL, VT, DAG.getBitcast(VT, N0.getOperand(1)), in canonicalizeBitSelect()
48140 DAG.getBitcast(VT, N1.getOperand(0))); in canonicalizeBitSelect()
48141 return DAG.getNode(ISD::OR, DL, VT, X, Y); in canonicalizeBitSelect()
48188 EVT VT = N->getValueType(0); in combineLogicBlendIntoPBLENDV() local
48189 if (!((VT.is128BitVector() && Subtarget.hasSSE2()) || in combineLogicBlendIntoPBLENDV()
48190 (VT.is256BitVector() && Subtarget.hasInt256()))) in combineLogicBlendIntoPBLENDV()
48212 if (SDValue Res = combineLogicBlendIntoConditionalNegate(VT, Mask, X, Y, DL, in combineLogicBlendIntoPBLENDV()
48224 MVT BlendVT = VT.is256BitVector() ? MVT::v32i8 : MVT::v16i8; in combineLogicBlendIntoPBLENDV()
48230 return DAG.getBitcast(VT, Mask); in combineLogicBlendIntoPBLENDV()
48241 EVT VT = Cmp.getOperand(0).getValueType(); in lowerX86CmpEqZeroToCtlzSrl() local
48242 unsigned Log2b = Log2_32(VT.getSizeInBits()); in lowerX86CmpEqZeroToCtlzSrl()
48244 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Cmp->getOperand(0)); in lowerX86CmpEqZeroToCtlzSrl()
48348 EVT VT = And1_L->getValueType(0); in foldMaskedMergeImpl() local
48349 SDValue Freeze_And0_R = DAG.getNode(ISD::FREEZE, SDLoc(), VT, And0_R); in foldMaskedMergeImpl()
48350 SDValue Xor0 = DAG.getNode(ISD::XOR, DL, VT, And1_R, Freeze_And0_R); in foldMaskedMergeImpl()
48351 SDValue And = DAG.getNode(ISD::AND, DL, VT, Xor0, NotOp); in foldMaskedMergeImpl()
48352 SDValue Xor1 = DAG.getNode(ISD::XOR, DL, VT, And, Freeze_And0_R); in foldMaskedMergeImpl()
48393 EVT VT = N->getValueType(0); in combineOr() local
48398 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) { in combineOr()
48407 if (VT == MVT::i1) { in combineOr()
48454 unsigned NumElts = VT.getVectorNumElements(); in combineOr()
48461 ISD::CONCAT_VECTORS, dl, VT, in combineOr()
48469 ISD::CONCAT_VECTORS, dl, VT, in combineOr()
48475 if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) { in combineOr()
48486 int NumElts = VT.getVectorNumElements(); in combineOr()
48487 int EltSizeInBits = VT.getScalarSizeInBits(); in combineOr()
48506 if (!Subtarget.hasBMI() && VT.isScalarInteger() && VT != MVT::i1) in combineOr()
48574 EVT VT = N->getValueType(0); in foldVectorXorShiftIntoCmp() local
48575 if (!VT.isSimple()) in foldVectorXorShiftIntoCmp()
48578 switch (VT.getSimpleVT().SimpleTy) { in foldVectorXorShiftIntoCmp()
48607 return DAG.getSetCC(SDLoc(N), VT, Shift.getOperand(0), Ones, ISD::SETGT); in foldVectorXorShiftIntoCmp()
48626 static SDValue detectUSatPattern(SDValue In, EVT VT, SelectionDAG &DAG, in detectUSatPattern() argument
48631 assert(InVT.getScalarSizeInBits() > VT.getScalarSizeInBits() && in detectUSatPattern()
48646 if (C2.isMask(VT.getScalarSizeInBits())) in detectUSatPattern()
48651 if (C1.isNonNegative() && C2.isMask(VT.getScalarSizeInBits())) in detectUSatPattern()
48656 if (C1.isNonNegative() && C2.isMask(VT.getScalarSizeInBits()) && in detectUSatPattern()
48673 static SDValue detectSSatPattern(SDValue In, EVT VT, bool MatchPackUS = false) { in detectSSatPattern() argument
48674 unsigned NumDstBits = VT.getScalarSizeInBits(); in detectSSatPattern()
48707 static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL, in combineTruncateWithSat() argument
48710 if (!Subtarget.hasSSE2() || !VT.isVector()) in combineTruncateWithSat()
48713 EVT SVT = VT.getVectorElementType(); in combineTruncateWithSat()
48722 InVT == MVT::v16i32 && VT == MVT::v16i8) { in combineTruncateWithSat()
48723 if (SDValue USatVal = detectSSatPattern(In, VT, true)) { in combineTruncateWithSat()
48728 return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, Mid); in combineTruncateWithSat()
48742 !(!Subtarget.useAVX512Regs() && VT.getSizeInBits() >= 256); in combineTruncateWithSat()
48744 if (isPowerOf2_32(VT.getVectorNumElements()) && !PreferAVX512 && in combineTruncateWithSat()
48745 VT.getSizeInBits() >= 64 && in combineTruncateWithSat()
48748 if (SDValue USatVal = detectSSatPattern(In, VT, true)) { in combineTruncateWithSat()
48753 EVT MidVT = VT.changeVectorElementType(MVT::i16); in combineTruncateWithSat()
48757 SDValue V = truncateVectorWithPACK(X86ISD::PACKUS, VT, Mid, DL, DAG, in combineTruncateWithSat()
48762 return truncateVectorWithPACK(X86ISD::PACKUS, VT, USatVal, DL, DAG, in combineTruncateWithSat()
48765 if (SDValue SSatVal = detectSSatPattern(In, VT)) in combineTruncateWithSat()
48766 return truncateVectorWithPACK(X86ISD::PACKSS, VT, SSatVal, DL, DAG, in combineTruncateWithSat()
48776 if (SDValue SSatVal = detectSSatPattern(In, VT)) { in combineTruncateWithSat()
48779 } else if (SDValue USatVal = detectUSatPattern(In, VT, DAG, DL)) { in combineTruncateWithSat()
48784 unsigned ResElts = VT.getVectorNumElements(); in combineTruncateWithSat()
48801 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in combineTruncateWithSat()
48812 static SDValue detectAVGPattern(SDValue In, EVT VT, SelectionDAG &DAG, in detectAVGPattern() argument
48815 if (!VT.isVector()) in detectAVGPattern()
48818 unsigned NumElems = VT.getVectorNumElements(); in detectAVGPattern()
48820 EVT ScalarVT = VT.getVectorElementType(); in detectAVGPattern()
48879 if (Op.getValueType() != VT) in detectAVGPattern()
48880 Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op); in detectAVGPattern()
48898 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in detectAVGPattern()
48924 if (V.getValueType() != VT || ISD::OR != V.getOpcode() || in detectAVGPattern()
48949 if (Operands[j].getValueType() != VT) in detectAVGPattern()
49147 EVT VT = ML->getValueType(0); in reduceMaskedLoadToScalarLoad() local
49148 EVT EltVT = VT.getVectorElementType(); in reduceMaskedLoadToScalarLoad()
49150 EVT CastVT = VT; in reduceMaskedLoadToScalarLoad()
49153 CastVT = VT.changeVectorElementType(EltVT); in reduceMaskedLoadToScalarLoad()
49166 Insert = DAG.getBitcast(VT, Insert); in reduceMaskedLoadToScalarLoad()
49178 EVT VT = ML->getValueType(0); in combineMaskedLoadConstantMask() local
49183 unsigned NumElts = VT.getVectorNumElements(); in combineMaskedLoadConstantMask()
49188 SDValue VecLd = DAG.getLoad(VT, DL, ML->getChain(), ML->getBasePtr(), in combineMaskedLoadConstantMask()
49190 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), VecLd, in combineMaskedLoadConstantMask()
49210 VT, DL, ML->getChain(), ML->getBasePtr(), ML->getOffset(), ML->getMask(), in combineMaskedLoadConstantMask()
49211 DAG.getUNDEF(VT), ML->getMemoryVT(), ML->getMemOperand(), in combineMaskedLoadConstantMask()
49213 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), NewML, in combineMaskedLoadConstantMask()
49243 EVT VT = Mld->getValueType(0); in combineMaskedLoad() local
49245 APInt DemandedBits(APInt::getSignMask(VT.getScalarSizeInBits())); in combineMaskedLoad()
49254 VT, SDLoc(N), Mld->getChain(), Mld->getBasePtr(), Mld->getOffset(), in combineMaskedLoad()
49282 EVT VT = Value.getValueType(); in reduceMaskedStoreToScalarStore() local
49283 EVT EltVT = VT.getVectorElementType(); in reduceMaskedStoreToScalarStore()
49286 EVT CastVT = VT.changeVectorElementType(EltVT); in reduceMaskedStoreToScalarStore()
49305 EVT VT = Mst->getValue().getValueType(); in combineMaskedStore() local
49319 APInt DemandedBits(APInt::getSignMask(VT.getScalarSizeInBits())); in combineMaskedStore()
49353 EVT VT = StoredVal.getValueType(); in combineStore() local
49357 if (!Subtarget.hasAVX512() && VT == StVT && VT.isVector() && in combineStore()
49358 VT.getVectorElementType() == MVT::i1) { in combineStore()
49360 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), VT.getVectorNumElements()); in combineStore()
49370 if (VT == MVT::v1i1 && VT == StVT && Subtarget.hasAVX512() && in combineStore()
49383 if ((VT == MVT::v1i1 || VT == MVT::v2i1 || VT == MVT::v4i1) && VT == StVT && in combineStore()
49385 unsigned NumConcats = 8 / VT.getVectorNumElements(); in combineStore()
49387 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, dl, VT)); in combineStore()
49396 if ((VT == MVT::v8i1 || VT == MVT::v16i1 || VT == MVT::v32i1 || in combineStore()
49397 VT == MVT::v64i1) && VT == StVT && TLI.isTypeLegal(VT) && in combineStore()
49400 if (!DCI.isBeforeLegalize() && VT == MVT::v64i1 && !Subtarget.is64Bit()) { in combineStore()
49432 if (VT.is256BitVector() && StVT == VT && in combineStore()
49433 TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, in combineStore()
49436 unsigned NumElems = VT.getVectorNumElements(); in combineStore()
49444 if (St->isNonTemporal() && StVT == VT && in combineStore()
49445 St->getAlign().value() < VT.getStoreSize()) { in combineStore()
49448 if (VT.is256BitVector() || VT.is512BitVector()) { in combineStore()
49449 unsigned NumElems = VT.getVectorNumElements(); in combineStore()
49457 if (VT.is128BitVector() && Subtarget.hasSSE2()) { in combineStore()
49467 if (!St->isTruncatingStore() && VT == MVT::v16i8 && !Subtarget.hasBWI() && in combineStore()
49483 TLI.isTruncStoreLegal(StoredVal.getOperand(0).getValueType(), VT)) { in combineStore()
49487 VT, St->getMemOperand(), DAG); in combineStore()
49511 if (NumTruncBits == VT.getSizeInBits() && in combineStore()
49523 if (St->isTruncatingStore() && VT.isVector()) { in combineStore()
49534 if (TLI.isTruncStoreLegal(VT, StVT)) { in combineStore()
49569 if (VT.getSizeInBits() != 64) in combineStore()
49576 if ((VT == MVT::i64 && F64IsLegal && !Subtarget.is64Bit()) && in combineStore()
49607 if (VT == MVT::i64 && F64IsLegal && !Subtarget.is64Bit() && in combineStore()
49630 MVT VT = StoredVal.getSimpleValueType(); in combineVEXTRACT_STORE() local
49634 unsigned StElts = MemVT.getSizeInBits() / VT.getScalarSizeInBits(); in combineVEXTRACT_STORE()
49635 APInt DemandedElts = APInt::getLowBitsSet(VT.getVectorNumElements(), StElts); in combineVEXTRACT_STORE()
49677 MVT VT = LHS.getSimpleValueType(); in isHorizontalBinOp() local
49678 assert((VT.is128BitVector() || VT.is256BitVector()) && in isHorizontalBinOp()
49680 unsigned NumElts = VT.getVectorNumElements(); in isHorizontalBinOp()
49776 unsigned Num128BitChunks = VT.getSizeInBits() / 128; in isHorizontalBinOp()
49820 if (!IsIdentityPostShuffle && !Subtarget.hasAVX2() && VT.isFloatingPoint() && in isHorizontalBinOp()
49821 isMultiLaneShuffleMask(128, VT.getScalarSizeInBits(), PostShuffleMask)) in isHorizontalBinOp()
49827 return User->getOpcode() == HOpcode && User->getValueType(0) == VT; in isHorizontalBinOp()
49830 return User->getOpcode() == HOpcode && User->getValueType(0) == VT; in isHorizontalBinOp()
49842 LHS = DAG.getBitcast(VT, NewLHS); in isHorizontalBinOp()
49843 RHS = DAG.getBitcast(VT, NewRHS); in isHorizontalBinOp()
49850 EVT VT = N->getValueType(0); in combineToHorizontalAddSub() local
49858 if ((Subtarget.hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) || in combineToHorizontalAddSub()
49859 (Subtarget.hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) { in combineToHorizontalAddSub()
49865 SDValue HorizBinOp = DAG.getNode(HorizOpcode, SDLoc(N), VT, LHS, RHS); in combineToHorizontalAddSub()
49867 HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp, in combineToHorizontalAddSub()
49868 DAG.getUNDEF(VT), PostShuffleMask); in combineToHorizontalAddSub()
49875 if (Subtarget.hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32 || in combineToHorizontalAddSub()
49876 VT == MVT::v16i16 || VT == MVT::v8i32)) { in combineToHorizontalAddSub()
49886 SDValue HorizBinOp = SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, in combineToHorizontalAddSub()
49889 HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp, in combineToHorizontalAddSub()
49890 DAG.getUNDEF(VT), PostShuffleMask); in combineToHorizontalAddSub()
49915 EVT VT = N->getValueType(0); in combineFMulcFCMulc() local
49952 SDValue I2F = DAG.getBitcast(VT, LHS.getOperand(0).getOperand(0)); in combineFMulcFCMulc()
49953 SDValue FCMulC = DAG.getNode(CombineOpcode, SDLoc(N), VT, RHS, I2F); in combineFMulcFCMulc()
49954 r = DAG.getBitcast(VT, FCMulC); in combineFMulcFCMulc()
50004 EVT VT = N->getValueType(0); in combineFaddCFmul() local
50005 if (VT != MVT::v8f16 && VT != MVT::v16f16 && VT != MVT::v32f16) in combineFaddCFmul()
50046 MVT CVT = MVT::getVectorVT(MVT::f32, VT.getVectorNumElements() / 2); in combineFaddCFmul()
50053 return DAG.getBitcast(VT, CFmul); in combineFaddCFmul()
50081 EVT VT = N->getValueType(0); in combineTruncatedArithmetic() local
50084 auto IsFreeTruncation = [VT](SDValue Op) { in combineTruncatedArithmetic()
50085 unsigned TruncSizeInBits = VT.getScalarSizeInBits(); in combineTruncatedArithmetic()
50104 SDValue Trunc0 = DAG.getNode(ISD::TRUNCATE, DL, VT, N0); in combineTruncatedArithmetic()
50105 SDValue Trunc1 = DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in combineTruncatedArithmetic()
50106 return DAG.getNode(SrcOpcode, DL, VT, Trunc0, Trunc1); in combineTruncatedArithmetic()
50115 if (!VT.isVector()) in combineTruncatedArithmetic()
50126 TLI.isOperationLegal(SrcOpcode, VT) && in combineTruncatedArithmetic()
50137 if (TLI.isOperationLegal(SrcOpcode, VT) && in combineTruncatedArithmetic()
50243 MVT VT = N->getValueType(0).getSimpleVT(); in combineVectorSignBitsTruncation() local
50244 MVT SVT = VT.getScalarType(); in combineVectorSignBitsTruncation()
50250 if (!isPowerOf2_32(VT.getVectorNumElements())) in combineVectorSignBitsTruncation()
50258 if (SVT == MVT::i32 && VT.getSizeInBits() < 128) in combineVectorSignBitsTruncation()
50264 !(!Subtarget.useAVX512Regs() && VT.is256BitVector() && in combineVectorSignBitsTruncation()
50269 if (VT.getSizeInBits() > 128 || in combineVectorSignBitsTruncation()
50282 return truncateVectorWithPACK(X86ISD::PACKUS, VT, In, DL, DAG, Subtarget); in combineVectorSignBitsTruncation()
50296 return truncateVectorWithPACK(X86ISD::PACKSS, VT, In, DL, DAG, Subtarget); in combineVectorSignBitsTruncation()
50303 In, APInt::getAllOnes(VT.getVectorNumElements()))) { in combineVectorSignBitsTruncation()
50306 return truncateVectorWithPACK(X86ISD::PACKSS, VT, NewIn, DL, DAG, in combineVectorSignBitsTruncation()
50321 static SDValue combinePMULH(SDValue Src, EVT VT, const SDLoc &DL, in combinePMULH() argument
50333 if (!VT.isVector() || VT.getVectorElementType() != MVT::i16) in combinePMULH()
50380 !(Subtarget.hasAVX512() && !Subtarget.hasBWI() && VT.is256BitVector()) && in combinePMULH()
50386 return DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getBitcast(InVT, Res)); in combinePMULH()
50390 LHS = DAG.getNode(ISD::TRUNCATE, DL, VT, LHS); in combinePMULH()
50391 RHS = DAG.getNode(ISD::TRUNCATE, DL, VT, RHS); in combinePMULH()
50394 return DAG.getNode(Opc, DL, VT, LHS, RHS); in combinePMULH()
50405 static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG, in detectPMADDUBSW() argument
50408 if (!VT.isVector() || !Subtarget.hasSSSE3()) in detectPMADDUBSW()
50411 unsigned NumElems = VT.getVectorNumElements(); in detectPMADDUBSW()
50412 EVT ScalarVT = VT.getVectorElementType(); in detectPMADDUBSW()
50416 SDValue SSatVal = detectSSatPattern(In, VT); in detectPMADDUBSW()
50532 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { ZExtIn, SExtIn }, in detectPMADDUBSW()
50538 EVT VT = N->getValueType(0); in combineTruncate() local
50547 if (SDValue Avg = detectAVGPattern(Src, VT, DAG, Subtarget, DL)) in combineTruncate()
50551 if (SDValue PMAdd = detectPMADDUBSW(Src, VT, DAG, Subtarget, DL)) in combineTruncate()
50555 if (SDValue Val = combineTruncateWithSat(Src, VT, DL, DAG, Subtarget)) in combineTruncate()
50559 if (SDValue V = combinePMULH(Src, VT, DL, DAG, Subtarget)) in combineTruncate()
50564 if (Src.getOpcode() == ISD::BITCAST && VT == MVT::i32) { in combineTruncate()
50579 EVT VT = N->getValueType(0); in combineVTRUNC() local
50583 if (SDValue SSatVal = detectSSatPattern(In, VT)) in combineVTRUNC()
50584 return DAG.getNode(X86ISD::VTRUNCS, DL, VT, SSatVal); in combineVTRUNC()
50585 if (SDValue USatVal = detectUSatPattern(In, VT, DAG, DL)) in combineVTRUNC()
50586 return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, USatVal); in combineVTRUNC()
50589 APInt DemandedMask(APInt::getAllOnes(VT.getScalarSizeInBits())); in combineVTRUNC()
50616 EVT VT = Op->getValueType(0); in isFNEG() local
50619 if (VT.getScalarSizeInBits() != ScalarSize) in isFNEG()
50630 if (NegOp0.getValueType() == VT) // FIXME: Can we do better? in isFNEG()
50631 return DAG.getVectorShuffle(VT, SDLoc(Op), NegOp0, DAG.getUNDEF(VT), in isFNEG()
50643 if (NegInsVal.getValueType() == VT.getVectorElementType()) // FIXME in isFNEG()
50644 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(Op), VT, InsVector, in isFNEG()
50754 EVT VT = Arg.getValueType(); in combineFneg() local
50755 EVT SVT = VT.getScalarType(); in combineFneg()
50759 if (!TLI.isTypeLegal(VT)) in combineFneg()
50767 SDValue Zero = DAG.getConstantFP(0.0, DL, VT); in combineFneg()
50768 SDValue NewNode = DAG.getNode(X86ISD::FNMSUB, DL, VT, Arg.getOperand(0), in combineFneg()
50793 EVT VT = Op.getValueType(); in getNegatedExpression() local
50794 EVT SVT = VT.getScalarType(); in getNegatedExpression()
50806 if (!Op.hasOneUse() || !Subtarget.hasAnyFMA() || !isTypeLegal(VT) || in getNegatedExpression()
50808 !isOperationLegal(ISD::FMA, VT)) in getNegatedExpression()
50835 return DAG.getNode(NewOpc, SDLoc(Op), VT, NewOps); in getNegatedExpression()
50841 return DAG.getNode(Opc, SDLoc(Op), VT, NegOp0); in getNegatedExpression()
50851 MVT VT = N->getSimpleValueType(0); in lowerX86FPLogicOp() local
50853 if (!VT.isVector() || !Subtarget.hasSSE2()) in lowerX86FPLogicOp()
50858 unsigned IntBits = VT.getScalarSizeInBits(); in lowerX86FPLogicOp()
50860 MVT IntVT = MVT::getVectorVT(IntSVT, VT.getSizeInBits() / IntBits); in lowerX86FPLogicOp()
50873 return DAG.getBitcast(VT, IntOp); in lowerX86FPLogicOp()
50897 EVT VT = N->getValueType(0); in combineXor() local
50900 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) { in combineXor()
50934 return DAG.getBitcast(VT, DAG.getNOT(SDLoc(N), N0.getOperand(0), in combineXor()
50940 if (ISD::isBuildVectorAllOnes(N1.getNode()) && VT.isVector() && in combineXor()
50941 VT.getVectorElementType() == MVT::i1 && in combineXor()
50945 ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0.getOperand(0), in combineXor()
50960 SDValue LHS = DAG.getZExtOrTrunc(TruncExtSrc.getOperand(0), DL, VT); in combineXor()
50961 SDValue RHS = DAG.getZExtOrTrunc(TruncExtSrc.getOperand(1), DL, VT); in combineXor()
50962 return DAG.getNode(ISD::XOR, DL, VT, LHS, in combineXor()
50963 DAG.getNode(ISD::XOR, DL, VT, RHS, N1)); in combineXor()
50973 EVT VT = N->getValueType(0); in combineBEXTR() local
50974 unsigned NumBits = VT.getSizeInBits(); in combineBEXTR()
51012 EVT VT = N->getValueType(0); in combineFAndFNotToFAndn() local
51016 if (!((VT == MVT::f32 && Subtarget.hasSSE1()) || in combineFAndFNotToFAndn()
51017 (VT == MVT::f64 && Subtarget.hasSSE2()) || in combineFAndFNotToFAndn()
51018 (VT == MVT::v4f32 && Subtarget.hasSSE1() && !Subtarget.hasSSE2()))) in combineFAndFNotToFAndn()
51030 return DAG.getNode(X86ISD::FANDN, DL, VT, N0.getOperand(0), N1); in combineFAndFNotToFAndn()
51034 return DAG.getNode(X86ISD::FANDN, DL, VT, N1.getOperand(0), N0); in combineFAndFNotToFAndn()
51119 EVT VT = N->getValueType(0); in combineFMinNumFMaxNum() local
51120 if (!((Subtarget.hasSSE1() && VT == MVT::f32) || in combineFMinNumFMaxNum()
51121 (Subtarget.hasSSE2() && VT == MVT::f64) || in combineFMinNumFMaxNum()
51122 (Subtarget.hasFP16() && VT == MVT::f16) || in combineFMinNumFMaxNum()
51123 (VT.isVector() && TLI.isTypeLegal(VT)))) in combineFMinNumFMaxNum()
51134 return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags()); in combineFMinNumFMaxNum()
51139 return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags()); in combineFMinNumFMaxNum()
51141 return DAG.getNode(MinMaxOp, DL, VT, Op1, Op0, N->getFlags()); in combineFMinNumFMaxNum()
51145 if (!VT.isVector() && DAG.getMachineFunction().getFunction().hasMinSize()) in combineFMinNumFMaxNum()
51149 VT); in combineFMinNumFMaxNum()
51170 SDValue MinOrMax = DAG.getNode(MinMaxOp, DL, VT, Op1, Op0); in combineFMinNumFMaxNum()
51175 return DAG.getSelect(DL, VT, IsOp0Nan, Op1, MinOrMax); in combineFMinNumFMaxNum()
51180 EVT VT = N->getValueType(0); in combineX86INT_TO_FP() local
51183 APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements()); in combineX86INT_TO_FP()
51190 if (VT.getVectorNumElements() < InVT.getVectorNumElements() && in combineX86INT_TO_FP()
51194 unsigned NumBits = InVT.getScalarSizeInBits() * VT.getVectorNumElements(); in combineX86INT_TO_FP()
51199 SDValue Convert = DAG.getNode(N->getOpcode(), dl, VT, in combineX86INT_TO_FP()
51214 EVT VT = N->getValueType(0); in combineCVTP2I_CVTTP2I() local
51219 if (VT.getVectorNumElements() < InVT.getVectorNumElements() && in combineCVTP2I_CVTTP2I()
51223 unsigned NumBits = InVT.getScalarSizeInBits() * VT.getVectorNumElements(); in combineCVTP2I_CVTTP2I()
51230 DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other}, in combineCVTP2I_CVTTP2I()
51235 DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(InVT, VZLoad)); in combineCVTP2I_CVTTP2I()
51253 MVT VT = N->getSimpleValueType(0); in combineAndnp() local
51254 int NumElts = VT.getVectorNumElements(); in combineAndnp()
51255 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in combineAndnp()
51260 return DAG.getConstant(0, SDLoc(N), VT); in combineAndnp()
51268 return DAG.getConstant(0, SDLoc(N), VT); in combineAndnp()
51272 return DAG.getNode(ISD::AND, SDLoc(N), VT, DAG.getBitcast(VT, Not), N1); in combineAndnp()
51284 return getConstVector(ResultBits, ResultUndefs, VT, DAG, DL); in combineAndnp()
51291 if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) { in combineAndnp()
51464 EVT VT = N->getValueType(0); in combineSignExtendInReg() local
51475 if (VT == MVT::v4i64 && (N0.getOpcode() == ISD::ANY_EXTEND || in combineSignExtendInReg()
51488 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT, Promote, N1); in combineSignExtendInReg()
51511 EVT VT = Ext->getValueType(0); in promoteExtBeforeAdd() local
51512 if (VT != MVT::i64) in promoteExtBeforeAdd()
51553 SDValue NewExt = DAG.getNode(Ext->getOpcode(), SDLoc(Ext), VT, AddOp0); in promoteExtBeforeAdd()
51554 SDValue NewConstant = DAG.getConstant(AddConstant, SDLoc(Add), VT); in promoteExtBeforeAdd()
51561 return DAG.getNode(ISD::ADD, SDLoc(Add), VT, NewExt, NewConstant, Flags); in promoteExtBeforeAdd()
51585 EVT VT = CMovN.getValueType(); in combineToExtendCMOV() local
51599 if (VT != MVT::i16 && !(ExtendOpcode == ISD::SIGN_EXTEND && VT == MVT::i32)) in combineToExtendCMOV()
51626 EVT VT = N->getValueType(0); in combineExtSetcc() local
51630 if (!Subtarget.hasAVX512() || !VT.isVector() || N0.getOpcode() != ISD::SETCC) in combineExtSetcc()
51634 EVT SVT = VT.getVectorElementType(); in combineExtSetcc()
51643 unsigned Size = VT.getSizeInBits(); in combineExtSetcc()
51659 SDValue Res = DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC); in combineExtSetcc()
51671 EVT VT = N->getValueType(0); in combineSext() local
51677 SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, N0->getOperand(0), in combineSext()
51700 if (SDValue V = combineToExtendBoolVectorInReg(N->getOpcode(), DL, VT, N0, in combineSext()
51704 if (VT.isVector()) { in combineSext()
51709 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0)); in combineSext()
51722 EVT VT = N->getValueType(0); in combineFMA() local
51727 if (!TLI.isTypeLegal(VT)) in combineFMA()
51738 TLI.isOperationExpand(ISD::FMA, VT)) { in combineFMA()
51739 SDValue Fmul = DAG.getNode(ISD::FMUL, dl, VT, A, B, Flags); in combineFMA()
51740 return DAG.getNode(ISD::FADD, dl, VT, Fmul, C, Flags); in combineFMA()
51743 EVT ScalarVT = VT.getScalarType(); in combineFMA()
51789 return DAG.getNode(NewOpcode, dl, {VT, MVT::Other}, in combineFMA()
51793 return DAG.getNode(NewOpcode, dl, VT, A, B, C, N->getOperand(3)); in combineFMA()
51794 return DAG.getNode(NewOpcode, dl, VT, A, B, C); in combineFMA()
51803 EVT VT = N->getValueType(0); in combineFMADDSUB() local
51817 return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1), in combineFMADDSUB()
51819 return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1), in combineFMADDSUB()
51828 EVT VT = N->getValueType(0); in combineZext() local
51834 SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, dl, VT, N0->getOperand(0), in combineZext()
51855 if (SDValue V = combineToExtendBoolVectorInReg(N->getOpcode(), dl, VT, N0, in combineZext()
51859 if (VT.isVector()) in combineZext()
51871 VT.getScalarSizeInBits() == N0.getOperand(0).getScalarValueSizeInBits()) { in combineZext()
51958 EVT VT = SetCC->getValueType(0); in combineVectorSizedSetCCEquality() local
52050 return DAG.getSetCC(DL, VT, DAG.getBitcast(KRegVT, Cmp), in combineVectorSizedSetCCEquality()
52059 return DAG.getNode(ISD::TRUNCATE, DL, VT, X86SetCC.getValue(0)); in combineVectorSizedSetCCEquality()
52068 return DAG.getSetCC(DL, VT, MovMsk, FFFFs, CC); in combineVectorSizedSetCCEquality()
52080 EVT VT = N->getValueType(0); in combineSetCC() local
52088 if (VT == MVT::i1 && isNullConstant(RHS)) { in combineSetCC()
52092 return DAG.getNode(ISD::TRUNCATE, DL, VT, in combineSetCC()
52111 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC); in combineSetCC()
52113 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC); in combineSetCC()
52129 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC); in combineSetCC()
52131 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC); in combineSetCC()
52146 return DAG.getSetCC(DL, VT, LHS.getOperand(0), in combineSetCC()
52152 if (VT.isVector() && VT.getVectorElementType() == MVT::i1 && in combineSetCC()
52171 assert(VT == Op0.getOperand(0).getValueType() && in combineSetCC()
52174 return DAG.getConstant(0, DL, VT); in combineSetCC()
52176 return DAG.getConstant(1, DL, VT); in combineSetCC()
52178 return DAG.getNOT(DL, Op0.getOperand(0), VT); in combineSetCC()
52191 if (Subtarget.hasAVX512() && !Subtarget.hasBWI() && VT.isVector() && in combineSetCC()
52192 VT.getVectorElementType() == MVT::i1 && in combineSetCC()
52196 return DAG.getNode(ISD::TRUNCATE, DL, VT, Setcc); in combineSetCC()
52201 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32 && in combineSetCC()
52212 return DAG.getSetCC(DL, VT, LHS, SDValue(FNeg, 0), CC); in combineSetCC()
52223 MVT VT = N->getSimpleValueType(0); in combineMOVMSK() local
52224 unsigned NumBits = VT.getScalarSizeInBits(); in combineMOVMSK()
52227 assert(VT == MVT::i32 && NumElts <= NumBits && "Unexpected MOVMSK types"); in combineMOVMSK()
52238 return DAG.getConstant(Imm, SDLoc(N), VT); in combineMOVMSK()
52245 return DAG.getNode(X86ISD::MOVMSK, SDLoc(N), VT, Src.getOperand(0)); in combineMOVMSK()
52253 return DAG.getNode(ISD::XOR, DL, VT, in combineMOVMSK()
52254 DAG.getNode(X86ISD::MOVMSK, DL, VT, NotSrc), in combineMOVMSK()
52255 DAG.getConstant(NotMask, DL, VT)); in combineMOVMSK()
52264 return DAG.getNode(ISD::XOR, DL, VT, in combineMOVMSK()
52265 DAG.getNode(X86ISD::MOVMSK, DL, VT, Src.getOperand(0)), in combineMOVMSK()
52266 DAG.getConstant(NotMask, DL, VT)); in combineMOVMSK()
52289 return DAG.getNode(X86ISD::MOVMSK, DL, VT, LHS); in combineMOVMSK()
52535 EVT VT = N->getValueType(0); in combineVectorCompareAndMaskUnaryOp() local
52537 unsigned NumEltBits = VT.getScalarSizeInBits(); in combineVectorCompareAndMaskUnaryOp()
52539 if (!VT.isVector() || Op0.getOpcode() != ISD::AND || in combineVectorCompareAndMaskUnaryOp()
52541 VT.getSizeInBits() != Op0.getValueSizeInBits()) in combineVectorCompareAndMaskUnaryOp()
52560 SourceConst = DAG.getNode(N->getOpcode(), DL, {VT, MVT::Other}, in combineVectorCompareAndMaskUnaryOp()
52563 SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0)); in combineVectorCompareAndMaskUnaryOp()
52568 SDValue Res = DAG.getBitcast(VT, NewAnd); in combineVectorCompareAndMaskUnaryOp()
52616 EVT VT = N->getValueType(0); in combineUIntToFP() local
52622 if (InVT.isVector() && VT.getVectorElementType() == MVT::f16) { in combineUIntToFP()
52634 return DAG.getNode(ISD::STRICT_UINT_TO_FP, dl, {VT, MVT::Other}, in combineUIntToFP()
52636 return DAG.getNode(ISD::UINT_TO_FP, dl, VT, P); in combineUIntToFP()
52643 VT.getScalarType() != MVT::f16) { in combineUIntToFP()
52650 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in combineUIntToFP()
52652 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P); in combineUIntToFP()
52660 return DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(N), {VT, MVT::Other}, in combineUIntToFP()
52662 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, Op0); in combineUIntToFP()
52679 EVT VT = N->getValueType(0); in combineSIntToFP() local
52685 if (InVT.isVector() && VT.getVectorElementType() == MVT::f16) { in combineSIntToFP()
52697 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in combineSIntToFP()
52699 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P); in combineSIntToFP()
52706 VT.getScalarType() != MVT::f16) { in combineSIntToFP()
52711 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in combineSIntToFP()
52713 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P); in combineSIntToFP()
52730 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in combineSIntToFP()
52732 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, Trunc); in combineSIntToFP()
52741 return DAG.getNode(X86ISD::STRICT_CVTSI2P, dl, {VT, MVT::Other}, in combineSIntToFP()
52743 return DAG.getNode(X86ISD::CVTSI2P, dl, VT, Shuf); in combineSIntToFP()
52754 if (VT == MVT::f16 || VT == MVT::f128) in combineSIntToFP()
52759 if (Subtarget.hasDQI() && VT != MVT::f80) in combineSIntToFP()
52762 if (Ld->isSimple() && !VT.isVector() && ISD::isNormalLoad(Op0.getNode()) && in combineSIntToFP()
52766 VT, InVT, SDLoc(N), Ld->getChain(), Ld->getBasePtr(), in combineSIntToFP()
52846 static SDValue combineAddOrSubToADCOrSBB(bool IsSub, const SDLoc &DL, EVT VT, in combineAddOrSubToADCOrSBB() argument
52850 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in combineAddOrSubToADCOrSBB()
52879 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineAddOrSubToADCOrSBB()
52896 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineAddOrSubToADCOrSBB()
52907 DAG.getVTList(VT, MVT::i32), X, in combineAddOrSubToADCOrSBB()
52908 DAG.getConstant(0, DL, VT), EFLAGS); in combineAddOrSubToADCOrSBB()
52929 DAG.getVTList(VT, MVT::i32), X, in combineAddOrSubToADCOrSBB()
52930 DAG.getConstant(0, DL, VT), NewEFLAGS); in combineAddOrSubToADCOrSBB()
52938 DAG.getVTList(VT, MVT::i32), X, in combineAddOrSubToADCOrSBB()
52939 DAG.getConstant(-1, DL, VT), EFLAGS); in combineAddOrSubToADCOrSBB()
52959 DAG.getVTList(VT, MVT::i32), X, in combineAddOrSubToADCOrSBB()
52960 DAG.getConstant(-1, DL, VT), NewEFLAGS); in combineAddOrSubToADCOrSBB()
52987 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineAddOrSubToADCOrSBB()
53001 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineAddOrSubToADCOrSBB()
53013 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in combineAddOrSubToADCOrSBB()
53019 DAG.getConstant(-1ULL, DL, VT), Cmp1.getValue(1)); in combineAddOrSubToADCOrSBB()
53024 DAG.getConstant(0, DL, VT), Cmp1.getValue(1)); in combineAddOrSubToADCOrSBB()
53034 EVT VT = N->getValueType(0); in combineAddOrSubToADCOrSBB() local
53037 if (SDValue ADCOrSBB = combineAddOrSubToADCOrSBB(IsSub, DL, VT, X, Y, DAG)) in combineAddOrSubToADCOrSBB()
53041 if (SDValue ADCOrSBB = combineAddOrSubToADCOrSBB(IsSub, DL, VT, Y, X, DAG)) { in combineAddOrSubToADCOrSBB()
53044 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), ADCOrSBB); in combineAddOrSubToADCOrSBB()
53062 EVT VT = Op.getValueType(); in combineCMP() local
53070 unsigned BitWidth = VT.getSizeInBits(); in combineCMP()
53078 Op = DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0), in combineCMP()
53079 DAG.getConstant(Mask, dl, VT)); in combineCMP()
53081 DAG.getConstant(0, dl, VT)); in combineCMP()
53108 APInt::getBitsSetFrom(OpVT.getSizeInBits(), VT.getSizeInBits()); in combineCMP()
53146 SDValue Op0 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(0)); in combineCMP()
53147 SDValue Op1 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(1)); in combineCMP()
53150 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in combineCMP()
53156 DAG.getConstant(0, dl, VT)); in combineCMP()
53170 MVT VT = LHS.getSimpleValueType(); in combineX86AddSub() local
53176 SDValue Res = DAG.getNode(GenericOpc, DL, VT, LHS, RHS); in combineX86AddSub()
53187 Op = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op); in combineX86AddSub()
53196 return combineAddOrSubToADCOrSBB(IsSub, DL, VT, LHS, RHS, DAG, in combineX86AddSub()
53206 MVT VT = N->getSimpleValueType(0); in combineSBB() local
53207 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in combineSBB()
53243 EVT VT = N->getValueType(0); in combineADC() local
53246 ISD::AND, DL, VT, in combineADC()
53247 DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineADC()
53249 DAG.getConstant(1, DL, VT)); in combineADC()
53265 MVT VT = N->getSimpleValueType(0); in combineADC() local
53266 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in combineADC()
53281 const SDLoc &DL, EVT VT, in matchPMADDWD() argument
53301 if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 || in matchPMADDWD()
53302 VT.getVectorNumElements() < 4 || in matchPMADDWD()
53303 !isPowerOf2_32(VT.getVectorNumElements())) in matchPMADDWD()
53318 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; i += 2) { in matchPMADDWD()
53370 VT.getVectorNumElements() * 2); in matchPMADDWD()
53382 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { N0, N1 }, PMADDBuilder); in matchPMADDWD()
53389 const SDLoc &DL, EVT VT, in matchPMADDWD_2() argument
53397 if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 || in matchPMADDWD_2()
53398 VT.getVectorNumElements() < 4 || in matchPMADDWD_2()
53399 !isPowerOf2_32(VT.getVectorNumElements())) in matchPMADDWD_2()
53483 if (In0.getValueSizeInBits() < VT.getSizeInBits() || in matchPMADDWD_2()
53484 In1.getValueSizeInBits() < VT.getSizeInBits()) in matchPMADDWD_2()
53511 VT.getVectorNumElements() * 2); in matchPMADDWD_2()
53520 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { In0, In1 }, in matchPMADDWD_2()
53529 const SDLoc &DL, EVT VT) { in combineAddOfPMADDWD() argument
53534 if (VT.getSizeInBits() > 128) in combineAddOfPMADDWD()
53537 unsigned NumElts = VT.getVectorNumElements(); in combineAddOfPMADDWD()
53565 return DAG.getNode(X86ISD::VPMADDWD, DL, VT, LHS, RHS); in combineAddOfPMADDWD()
53603 EVT VT = N->getValueType(0); in pushAddIntoCmovOfConsts() local
53623 FalseOp = DAG.getNode(ISD::ADD, DL, VT, X, FalseOp); in pushAddIntoCmovOfConsts()
53624 TrueOp = DAG.getNode(ISD::ADD, DL, VT, X, TrueOp); in pushAddIntoCmovOfConsts()
53625 Cmov = DAG.getNode(X86ISD::CMOV, DL, VT, FalseOp, TrueOp, in pushAddIntoCmovOfConsts()
53627 return DAG.getNode(ISD::ADD, DL, VT, Cmov, Y); in pushAddIntoCmovOfConsts()
53631 FalseOp = DAG.getNode(ISD::ADD, DL, VT, OtherOp, FalseOp); in pushAddIntoCmovOfConsts()
53632 TrueOp = DAG.getNode(ISD::ADD, DL, VT, OtherOp, TrueOp); in pushAddIntoCmovOfConsts()
53633 return DAG.getNode(X86ISD::CMOV, DL, VT, FalseOp, TrueOp, Cmov.getOperand(2), in pushAddIntoCmovOfConsts()
53640 EVT VT = N->getValueType(0); in combineAdd() local
53648 if (SDValue MAdd = matchPMADDWD(DAG, Op0, Op1, DL, VT, Subtarget)) in combineAdd()
53650 if (SDValue MAdd = matchPMADDWD_2(DAG, Op0, Op1, DL, VT, Subtarget)) in combineAdd()
53652 if (SDValue MAdd = combineAddOfPMADDWD(DAG, Op0, Op1, DL, VT)) in combineAdd()
53664 if (VT.isVector()) { in combineAdd()
53669 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op0.getOperand(0)); in combineAdd()
53670 return DAG.getNode(ISD::SUB, DL, VT, Op1, SExt); in combineAdd()
53676 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op1.getOperand(0)); in combineAdd()
53677 return DAG.getNode(ISD::SUB, DL, VT, Op0, SExt); in combineAdd()
53726 MVT VT = N->getSimpleValueType(0); in combineSubABS() local
53727 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, VT, TrueOp, FalseOp, in combineSubABS()
53730 return DAG.getNode(ISD::ADD, DL, VT, N0, Cmov); in combineSubABS()
53756 EVT VT = Op0.getValueType(); in combineSub() local
53757 SDValue NewXor = DAG.getNode(ISD::XOR, SDLoc(Op1), VT, Op1.getOperand(0), in combineSub()
53758 DAG.getNOT(SDLoc(Op1), Op1.getOperand(1), VT)); in combineSub()
53760 DAG.getNode(ISD::ADD, DL, VT, Op0, DAG.getConstant(1, DL, VT)); in combineSub()
53761 return DAG.getNode(ISD::ADD, DL, VT, NewXor, NewAdd); in combineSub()
53795 MVT VT = N->getSimpleValueType(0); in combineVectorCompare() local
53800 return DAG.getConstant(-1, DL, VT); in combineVectorCompare()
53802 return DAG.getConstant(0, DL, VT); in combineVectorCompare()
53811 static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, in combineConcatVectorOps() argument
53816 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in combineConcatVectorOps()
53819 return DAG.getUNDEF(VT); in combineConcatVectorOps()
53824 return getZeroVector(VT, Subtarget, DAG, DL); in combineConcatVectorOps()
53831 (VT.is256BitVector() || (VT.is512BitVector() && Subtarget.hasAVX512()))) { in combineConcatVectorOps()
53834 return DAG.getNode(Op0.getOpcode(), DL, VT, Op0.getOperand(0)); in combineConcatVectorOps()
53847 getBROADCAST_LOAD(Opc, DL, VT, Mem->getMemoryVT(), Mem, 0, DAG)) { in combineConcatVectorOps()
53856 if (Op0.getOpcode() == X86ISD::MOVDDUP && VT == MVT::v4f64 && in combineConcatVectorOps()
53859 VT.getScalarType(), Subtarget))) in combineConcatVectorOps()
53860 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, in combineConcatVectorOps()
53870 Op0.getOperand(0).getValueType() == VT.getScalarType()) in combineConcatVectorOps()
53871 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Op0.getOperand(0)); in combineConcatVectorOps()
53876 Op0.getOperand(0).getValueType() == VT) { in combineConcatVectorOps()
53886 if (VT.is256BitVector() && Ops.size() == 2) { in combineConcatVectorOps()
53898 return DAG.getNode(X86ISD::VPERM2X128, DL, VT, in combineConcatVectorOps()
53899 DAG.getBitcast(VT, Src0.getOperand(0)), in combineConcatVectorOps()
53900 DAG.getBitcast(VT, Src1.getOperand(0)), in combineConcatVectorOps()
53912 auto ConcatSubOperand = [&](MVT VT, ArrayRef<SDValue> SubOps, unsigned I) { in combineConcatVectorOps() argument
53916 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs); in combineConcatVectorOps()
53918 auto IsConcatFree = [](MVT VT, ArrayRef<SDValue> SubOps, unsigned Op) { in combineConcatVectorOps() argument
53923 Sub.getOperand(0).getValueType() != VT || in combineConcatVectorOps()
53933 if (!IsSplat && VT == MVT::v4f64 && llvm::all_of(Ops, [](SDValue Op) { in combineConcatVectorOps()
53936 return DAG.getNode(X86ISD::MOVDDUP, DL, VT, in combineConcatVectorOps()
53937 ConcatSubOperand(VT, Ops, 0)); in combineConcatVectorOps()
53944 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
53945 ConcatSubOperand(VT, Ops, 0)); in combineConcatVectorOps()
53950 if (!IsSplat && VT.getScalarType() == MVT::f32 && in combineConcatVectorOps()
53954 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
53955 ConcatSubOperand(VT, Ops, 0), in combineConcatVectorOps()
53956 ConcatSubOperand(VT, Ops, 1), Op0.getOperand(2)); in combineConcatVectorOps()
53963 if (!IsSplat && NumOps == 2 && VT.is256BitVector() && in combineConcatVectorOps()
53965 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
53966 ConcatSubOperand(VT, Ops, 0), Op0.getOperand(1)); in combineConcatVectorOps()
53970 if (!IsSplat && NumOps == 2 && (VT == MVT::v8f32 || VT == MVT::v8i32) && in combineConcatVectorOps()
53972 SDValue Res = DAG.getBitcast(MVT::v8f32, ConcatSubOperand(VT, Ops, 0)); in combineConcatVectorOps()
53975 return DAG.getBitcast(VT, Res); in combineConcatVectorOps()
53977 if (!IsSplat && NumOps == 2 && VT == MVT::v4f64) { in combineConcatVectorOps()
53981 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
53982 ConcatSubOperand(VT, Ops, 0), in combineConcatVectorOps()
53987 if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) || in combineConcatVectorOps()
53988 (VT.is512BitVector() && Subtarget.useBWIRegs()))) { in combineConcatVectorOps()
53989 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
53990 ConcatSubOperand(VT, Ops, 0), in combineConcatVectorOps()
53991 ConcatSubOperand(VT, Ops, 1)); in combineConcatVectorOps()
53995 if (!IsSplat && NumOps == 2 && VT.is512BitVector()) { in combineConcatVectorOps()
54018 MVT IntMaskSVT = MVT::getIntegerVT(VT.getScalarSizeInBits()); in combineConcatVectorOps()
54021 return DAG.getNode(X86ISD::VPERMV3, DL, VT, Src0, Mask, Src1); in combineConcatVectorOps()
54029 if (VT == MVT::v4i64 && !Subtarget.hasInt256() && in combineConcatVectorOps()
54033 SDValue Res = DAG.getBitcast(MVT::v8i32, ConcatSubOperand(VT, Ops, 0)); in combineConcatVectorOps()
54042 return DAG.getBitcast(VT, Res); in combineConcatVectorOps()
54049 if (((VT.is256BitVector() && Subtarget.hasInt256()) || in combineConcatVectorOps()
54050 (VT.is512BitVector() && Subtarget.useAVX512Regs() && in combineConcatVectorOps()
54055 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
54056 ConcatSubOperand(VT, Ops, 0), Op0.getOperand(1)); in combineConcatVectorOps()
54062 if (VT.is512BitVector() && Subtarget.useAVX512Regs() && in combineConcatVectorOps()
54066 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
54067 ConcatSubOperand(VT, Ops, 0), Op0.getOperand(1)); in combineConcatVectorOps()
54075 if (!IsSplat && VT.is512BitVector()) { in combineConcatVectorOps()
54079 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
54090 if (!IsSplat && VT.is256BitVector() && in combineConcatVectorOps()
54091 (VT.isFloatingPoint() || Subtarget.hasInt256())) { in combineConcatVectorOps()
54095 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
54102 ((VT.is256BitVector() && Subtarget.hasInt256()) || in combineConcatVectorOps()
54103 (VT.is512BitVector() && Subtarget.useBWIRegs())) && in combineConcatVectorOps()
54107 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
54108 ConcatSubOperand(VT, Ops, 0), in combineConcatVectorOps()
54109 ConcatSubOperand(VT, Ops, 1), Op0.getOperand(2)); in combineConcatVectorOps()
54114 if (!IsSplat && VT.is256BitVector() && Ops.size() == 2 && in combineConcatVectorOps()
54115 (VT.getScalarSizeInBits() >= 32 || Subtarget.hasInt256()) && in combineConcatVectorOps()
54116 IsConcatFree(VT, Ops, 1) && IsConcatFree(VT, Ops, 2)) { in combineConcatVectorOps()
54119 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
54121 ConcatSubOperand(VT, Ops, 1), in combineConcatVectorOps()
54122 ConcatSubOperand(VT, Ops, 2)); in combineConcatVectorOps()
54133 if (TLI->allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, in combineConcatVectorOps()
54137 EltsFromConsecutiveLoads(VT, Ops, DL, DAG, Subtarget, false)) in combineConcatVectorOps()
54145 APInt UndefElts = APInt::getNullValue(VT.getVectorNumElements()); in combineConcatVectorOps()
54155 if (EltBits.size() == VT.getVectorNumElements()) in combineConcatVectorOps()
54156 return getConstVector(EltBits, UndefElts, VT, DAG, DL); in combineConcatVectorOps()
54165 EVT VT = N->getValueType(0); in combineCONCAT_VECTORS() local
54170 if (VT.getVectorElementType() == MVT::i1) in combineCONCAT_VECTORS()
54173 if (Subtarget.hasAVX() && TLI.isTypeLegal(VT) && TLI.isTypeLegal(SrcVT)) { in combineCONCAT_VECTORS()
54175 if (SDValue R = combineConcatVectorOps(SDLoc(N), VT.getSimpleVT(), Ops, DAG, in combineCONCAT_VECTORS()
54334 MVT VT = Ext->getSimpleValueType(0); in narrowExtractedVectorSelect() local
54335 if (!VT.is128BitVector()) in narrowExtractedVectorSelect()
54365 unsigned NarrowingFactor = WideVT.getSizeInBits() / VT.getSizeInBits(); in narrowExtractedVectorSelect()
54373 return DAG.getBitcast(VT, NarrowSel); in narrowExtractedVectorSelect()
54392 MVT VT = N->getSimpleValueType(0); in combineEXTRACT_SUBVECTOR() local
54397 unsigned SizeInBits = VT.getSizeInBits(); in combineEXTRACT_SUBVECTOR()
54399 unsigned NumSubElts = VT.getVectorNumElements(); in combineEXTRACT_SUBVECTOR()
54416 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT, in combineEXTRACT_SUBVECTOR()
54428 return getZeroVector(VT, Subtarget, DAG, SDLoc(N)); in combineEXTRACT_SUBVECTOR()
54431 if (VT.getScalarType() == MVT::i1) in combineEXTRACT_SUBVECTOR()
54432 return DAG.getConstant(1, SDLoc(N), VT); in combineEXTRACT_SUBVECTOR()
54433 return getOnesVector(VT, DAG, SDLoc(N)); in combineEXTRACT_SUBVECTOR()
54437 return DAG.getBuildVector(VT, SDLoc(N), in combineEXTRACT_SUBVECTOR()
54444 if (VT.getVectorElementType() != MVT::i1 && in combineEXTRACT_SUBVECTOR()
54449 SDValue NewExt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, in combineEXTRACT_SUBVECTOR()
54452 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, NewExt, in combineEXTRACT_SUBVECTOR()
54467 cast<MemIntrinsicSDNode>(InVec)->getMemoryVT() == VT) in combineEXTRACT_SUBVECTOR()
54481 return DAG.getUNDEF(VT); in combineEXTRACT_SUBVECTOR()
54483 return getZeroVector(VT, Subtarget, DAG, SDLoc(N)); in combineEXTRACT_SUBVECTOR()
54498 if (IdxVal == 0 && VT == MVT::v2f64 && InVecVT == MVT::v4f64) { in combineEXTRACT_SUBVECTOR()
54502 return DAG.getNode(X86ISD::CVTSI2P, SDLoc(N), VT, InVec.getOperand(0)); in combineEXTRACT_SUBVECTOR()
54507 return DAG.getNode(X86ISD::CVTUI2P, SDLoc(N), VT, InVec.getOperand(0)); in combineEXTRACT_SUBVECTOR()
54512 return DAG.getNode(X86ISD::VFPEXT, SDLoc(N), VT, InVec.getOperand(0)); in combineEXTRACT_SUBVECTOR()
54529 return DAG.getNode(ExtOp, DL, VT, Ext); in combineEXTRACT_SUBVECTOR()
54539 return DAG.getNode(InOpcode, DL, VT, Ext0, Ext1, Ext2); in combineEXTRACT_SUBVECTOR()
54542 (VT.is128BitVector() || VT.is256BitVector())) { in combineEXTRACT_SUBVECTOR()
54547 return DAG.getNode(InOpcode, DL, VT, Ext); in combineEXTRACT_SUBVECTOR()
54550 (VT.is128BitVector() || VT.is256BitVector())) { in combineEXTRACT_SUBVECTOR()
54554 return DAG.getNode(InOpcode, DL, VT, Ext0); in combineEXTRACT_SUBVECTOR()
54566 return DAG.getNode(InOpcode, DL, VT, Ext, InVec.getOperand(1)); in combineEXTRACT_SUBVECTOR()
54573 EVT VT = N->getValueType(0); in combineScalarToVector() local
54581 if (VT == MVT::v1i1 && Src.getOpcode() == ISD::AND && Src.hasOneUse()) in combineScalarToVector()
54588 if (VT == MVT::v1i1 && Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in combineScalarToVector()
54593 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src.getOperand(0), in combineScalarToVector()
54598 if (VT == MVT::v2i64 || VT == MVT::v2f64) { in combineScalarToVector()
54613 VT, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32, in combineScalarToVector()
54618 if (VT == MVT::v2i64 && Src.getOpcode() == ISD::BITCAST && in combineScalarToVector()
54620 return DAG.getNode(X86ISD::MOVQ2DQ, DL, VT, Src.getOperand(0)); in combineScalarToVector()
54624 if (VT.getScalarType() == Src.getValueType()) in combineScalarToVector()
54628 unsigned SizeInBits = VT.getFixedSizeInBits(); in combineScalarToVector()
54698 EVT VT = N->getValueType(0); in combineVPMADD() local
54706 return DAG.getConstant(0, SDLoc(N), VT); in combineVPMADD()
54709 APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements()); in combineVPMADD()
54719 EVT VT = N->getValueType(0); in combineEXTEND_VECTOR_INREG() local
54735 EVT MemVT = VT.changeVectorElementType(SVT); in combineEXTEND_VECTOR_INREG()
54736 if (TLI.isLoadExtLegal(Ext, VT, MemVT)) { in combineEXTEND_VECTOR_INREG()
54738 Ext, DL, VT, Ld->getChain(), Ld->getBasePtr(), Ld->getPointerInfo(), in combineEXTEND_VECTOR_INREG()
54748 return DAG.getNode(Opcode, DL, VT, In.getOperand(0)); in combineEXTEND_VECTOR_INREG()
54757 return DAG.getNode(Opcode, DL, VT, In.getOperand(0).getOperand(0)); in combineEXTEND_VECTOR_INREG()
54763 In.getValueSizeInBits() == VT.getSizeInBits()) { in combineEXTEND_VECTOR_INREG()
54764 unsigned NumElts = VT.getVectorNumElements(); in combineEXTEND_VECTOR_INREG()
54765 unsigned Scale = VT.getScalarSizeInBits() / In.getScalarValueSizeInBits(); in combineEXTEND_VECTOR_INREG()
54770 return DAG.getBitcast(VT, DAG.getBuildVector(In.getValueType(), DL, Elts)); in combineEXTEND_VECTOR_INREG()
54778 if (TLI.isTypeLegal(VT) && TLI.isTypeLegal(In.getValueType())) in combineEXTEND_VECTOR_INREG()
54788 EVT VT = N->getValueType(0); in combineKSHIFT() local
54791 return DAG.getConstant(0, SDLoc(N), VT); in combineKSHIFT()
54794 APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements()); in combineKSHIFT()
54835 EVT VT = N->getValueType(0); in combineFP_EXTEND() local
54842 if (VT.getVectorElementType() != MVT::f32 && in combineFP_EXTEND()
54843 VT.getVectorElementType() != MVT::f64) in combineFP_EXTEND()
54846 unsigned NumElts = VT.getVectorNumElements(); in combineFP_EXTEND()
54886 if (Cvt.getValueType() != VT) { in combineFP_EXTEND()
54887 Cvt = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {VT, MVT::Other}, in combineFP_EXTEND()
54895 return DAG.getNode(ISD::FP_EXTEND, dl, VT, Cvt); in combineFP_EXTEND()
54915 EVT VT = N->getSimpleValueType(0); in combineBROADCAST_LOAD() local
54927 User->getValueSizeInBits(0).getFixedSize() > VT.getFixedSizeInBits()) { in combineBROADCAST_LOAD()
54929 VT.getSizeInBits()); in combineBROADCAST_LOAD()
54930 Extract = DAG.getBitcast(VT, Extract); in combineBROADCAST_LOAD()
54946 EVT VT = N->getValueType(0); in combineFP_ROUND() local
54950 if (!VT.isVector() || VT.getVectorElementType() != MVT::f16 || in combineFP_ROUND()
54954 unsigned NumElts = VT.getVectorNumElements(); in combineFP_ROUND()
54980 EVT IntVT = VT.changeVectorElementTypeToInteger(); in combineFP_ROUND()
54985 Cvt = DAG.getBitcast(VT, Cvt); in combineFP_ROUND()
55209 bool X86TargetLowering::isTypeDesirableForOp(unsigned Opc, EVT VT) const { in isTypeDesirableForOp()
55210 if (!isTypeLegal(VT)) in isTypeDesirableForOp()
55214 if (Opc == ISD::SHL && VT.isVector() && VT.getVectorElementType() == MVT::i8) in isTypeDesirableForOp()
55225 if ((Opc == ISD::MUL || Opc == ISD::SHL) && VT == MVT::i8) in isTypeDesirableForOp()
55230 if (VT == MVT::i16) { in isTypeDesirableForOp()
55272 EVT VT = Op.getValueType(); in IsDesirableToPromoteOp() local
55273 bool Is8BitMulByConstant = VT == MVT::i8 && Op.getOpcode() == ISD::MUL && in IsDesirableToPromoteOp()
55280 if (VT != MVT::i16 && !Is8BitMulByConstant) in IsDesirableToPromoteOp()
55933 MVT VT) const { in getRegForInlineAsmConstraint()
55953 if (VT == MVT::i1) in getRegForInlineAsmConstraint()
55955 if (VT == MVT::i8) in getRegForInlineAsmConstraint()
55957 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
55961 if (VT == MVT::i32) in getRegForInlineAsmConstraint()
55963 if (VT == MVT::i64) in getRegForInlineAsmConstraint()
55969 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
55971 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
55973 if (VT == MVT::i32 || VT == MVT::f32) in getRegForInlineAsmConstraint()
55975 if (VT != MVT::f80 && !VT.isVector()) in getRegForInlineAsmConstraint()
55982 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
55984 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
55986 if (VT == MVT::i32 || VT == MVT::f32 || in getRegForInlineAsmConstraint()
55987 (!VT.isVector() && !Subtarget.is64Bit())) in getRegForInlineAsmConstraint()
55989 if (VT != MVT::f80 && !VT.isVector()) in getRegForInlineAsmConstraint()
55994 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
55996 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
55998 if (VT == MVT::i32 || VT == MVT::f32 || in getRegForInlineAsmConstraint()
55999 (!VT.isVector() && !Subtarget.is64Bit())) in getRegForInlineAsmConstraint()
56001 if (VT != MVT::f80 && !VT.isVector()) in getRegForInlineAsmConstraint()
56005 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
56007 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
56009 if (VT == MVT::i32 || VT == MVT::f32 || in getRegForInlineAsmConstraint()
56010 (!VT.isVector() && !Subtarget.is64Bit())) in getRegForInlineAsmConstraint()
56012 if (VT != MVT::f80 && !VT.isVector()) in getRegForInlineAsmConstraint()
56018 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT)) in getRegForInlineAsmConstraint()
56020 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT)) in getRegForInlineAsmConstraint()
56022 if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f80) in getRegForInlineAsmConstraint()
56033 switch (VT.SimpleTy) { in getRegForInlineAsmConstraint()
56112 return getRegForInlineAsmConstraint(TRI, "x", VT); in getRegForInlineAsmConstraint()
56118 switch (VT.SimpleTy) { in getRegForInlineAsmConstraint()
56175 if (VT == MVT::i1) in getRegForInlineAsmConstraint()
56177 if (VT == MVT::i8) in getRegForInlineAsmConstraint()
56179 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
56183 if (VT == MVT::i32) in getRegForInlineAsmConstraint()
56185 if (VT == MVT::i64) in getRegForInlineAsmConstraint()
56198 Res = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); in getRegForInlineAsmConstraint()
56204 if (VT == MVT::Other || VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f80) { in getRegForInlineAsmConstraint()
56231 VT == MVT::Other) in getRegForInlineAsmConstraint()
56260 if (TRI->isTypeLegalForClass(*Res.second, VT) || VT == MVT::Other) in getRegForInlineAsmConstraint()
56272 unsigned Size = VT.getSizeInBits(); in getRegForInlineAsmConstraint()
56318 if (VT == MVT::f16) in getRegForInlineAsmConstraint()
56320 else if (VT == MVT::f32 || VT == MVT::i32) in getRegForInlineAsmConstraint()
56322 else if (VT == MVT::f64 || VT == MVT::i64) in getRegForInlineAsmConstraint()
56324 else if (TRI->isTypeLegalForClass(X86::VR128XRegClass, VT)) in getRegForInlineAsmConstraint()
56326 else if (TRI->isTypeLegalForClass(X86::VR256XRegClass, VT)) in getRegForInlineAsmConstraint()
56328 else if (TRI->isTypeLegalForClass(X86::VR512RegClass, VT)) in getRegForInlineAsmConstraint()
56336 if (VT == MVT::i1) in getRegForInlineAsmConstraint()
56338 else if (VT == MVT::i8) in getRegForInlineAsmConstraint()
56340 else if (VT == MVT::i16) in getRegForInlineAsmConstraint()
56342 else if (VT == MVT::i32) in getRegForInlineAsmConstraint()
56344 else if (VT == MVT::i64) in getRegForInlineAsmConstraint()
56385 bool X86TargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const { in isIntDivCheap() argument
56394 return OptSize && !VT.isVector(); in isIntDivCheap()