Lines Matching refs:MVT
105 MVT PtrVT = MVT::getIntegerVT(TM.getPointerSizeInBits(0)); in X86TargetLowering()
163 addRegisterClass(MVT::i8, &X86::GR8RegClass); in X86TargetLowering()
164 addRegisterClass(MVT::i16, &X86::GR16RegClass); in X86TargetLowering()
165 addRegisterClass(MVT::i32, &X86::GR32RegClass); in X86TargetLowering()
167 addRegisterClass(MVT::i64, &X86::GR64RegClass); in X86TargetLowering()
169 for (MVT VT : MVT::integer_valuetypes()) in X86TargetLowering()
170 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); in X86TargetLowering()
173 setTruncStoreAction(MVT::i64, MVT::i32, Expand); in X86TargetLowering()
174 setTruncStoreAction(MVT::i64, MVT::i16, Expand); in X86TargetLowering()
175 setTruncStoreAction(MVT::i64, MVT::i8 , Expand); in X86TargetLowering()
176 setTruncStoreAction(MVT::i32, MVT::i16, Expand); in X86TargetLowering()
177 setTruncStoreAction(MVT::i32, MVT::i8 , Expand); in X86TargetLowering()
178 setTruncStoreAction(MVT::i16, MVT::i8, Expand); in X86TargetLowering()
180 setTruncStoreAction(MVT::f64, MVT::f32, Expand); in X86TargetLowering()
183 setCondCodeAction(ISD::SETOEQ, MVT::f32, Expand); in X86TargetLowering()
184 setCondCodeAction(ISD::SETOEQ, MVT::f64, Expand); in X86TargetLowering()
185 setCondCodeAction(ISD::SETOEQ, MVT::f80, Expand); in X86TargetLowering()
186 setCondCodeAction(ISD::SETUNE, MVT::f32, Expand); in X86TargetLowering()
187 setCondCodeAction(ISD::SETUNE, MVT::f64, Expand); in X86TargetLowering()
188 setCondCodeAction(ISD::SETUNE, MVT::f80, Expand); in X86TargetLowering()
192 setOperationAction(ISD::ABS , MVT::i16 , Custom); in X86TargetLowering()
193 setOperationAction(ISD::ABS , MVT::i32 , Custom); in X86TargetLowering()
195 setOperationAction(ISD::ABS , MVT::i64 , Custom); in X86TargetLowering()
200 setOperationAction(ShiftOp , MVT::i16 , Custom); in X86TargetLowering()
201 setOperationAction(ShiftOp , MVT::i32 , Custom); in X86TargetLowering()
203 setOperationAction(ShiftOp , MVT::i64 , Custom); in X86TargetLowering()
208 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote); in X86TargetLowering()
209 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote); in X86TargetLowering()
210 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote); in X86TargetLowering()
215 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Custom); in X86TargetLowering()
217 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); in X86TargetLowering()
218 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom); in X86TargetLowering()
222 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom); in X86TargetLowering()
225 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Custom); in X86TargetLowering()
227 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Expand); in X86TargetLowering()
232 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote); in X86TargetLowering()
233 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote); in X86TargetLowering()
238 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); in X86TargetLowering()
240 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); in X86TargetLowering()
242 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom); in X86TargetLowering()
243 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); in X86TargetLowering()
246 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); in X86TargetLowering()
247 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Expand); in X86TargetLowering()
252 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote); in X86TargetLowering()
253 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote); in X86TargetLowering()
258 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom); in X86TargetLowering()
259 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom); in X86TargetLowering()
262 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); in X86TargetLowering()
264 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); in X86TargetLowering()
266 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom); in X86TargetLowering()
267 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); in X86TargetLowering()
270 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); in X86TargetLowering()
271 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Expand); in X86TargetLowering()
272 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Expand); in X86TargetLowering()
277 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote); in X86TargetLowering()
278 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote); in X86TargetLowering()
279 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote); in X86TargetLowering()
284 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Custom); in X86TargetLowering()
285 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Custom); in X86TargetLowering()
287 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); in X86TargetLowering()
288 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand); in X86TargetLowering()
296 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand); in X86TargetLowering()
301 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Custom); in X86TargetLowering()
303 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Custom); in X86TargetLowering()
308 setOperationAction(ISD::BITCAST , MVT::f32 , Expand); in X86TargetLowering()
309 setOperationAction(ISD::BITCAST , MVT::i32 , Expand); in X86TargetLowering()
311 setOperationAction(ISD::BITCAST , MVT::f64 , Expand); in X86TargetLowering()
313 setOperationAction(ISD::BITCAST , MVT::i64 , Expand); in X86TargetLowering()
316 setOperationAction(ISD::BITCAST , MVT::i64 , Custom); in X86TargetLowering()
328 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { in X86TargetLowering()
337 setOperationAction(ISD::BR_JT , MVT::Other, Expand); in X86TargetLowering()
338 setOperationAction(ISD::BRCOND , MVT::Other, Custom); in X86TargetLowering()
339 for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128, in X86TargetLowering()
340 MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { in X86TargetLowering()
345 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); in X86TargetLowering()
346 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal); in X86TargetLowering()
347 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal); in X86TargetLowering()
348 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand); in X86TargetLowering()
349 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand); in X86TargetLowering()
351 setOperationAction(ISD::FREM , MVT::f32 , Expand); in X86TargetLowering()
352 setOperationAction(ISD::FREM , MVT::f64 , Expand); in X86TargetLowering()
353 setOperationAction(ISD::FREM , MVT::f80 , Expand); in X86TargetLowering()
354 setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom); in X86TargetLowering()
358 setOperationPromotedToType(ISD::CTTZ , MVT::i8 , MVT::i32); in X86TargetLowering()
359 setOperationPromotedToType(ISD::CTTZ_ZERO_UNDEF, MVT::i8 , MVT::i32); in X86TargetLowering()
361 setOperationAction(ISD::CTTZ , MVT::i16 , Custom); in X86TargetLowering()
362 setOperationAction(ISD::CTTZ , MVT::i32 , Custom); in X86TargetLowering()
363 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16 , Legal); in X86TargetLowering()
364 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32 , Legal); in X86TargetLowering()
366 setOperationAction(ISD::CTTZ , MVT::i64 , Custom); in X86TargetLowering()
367 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Legal); in X86TargetLowering()
374 setOperationPromotedToType(ISD::CTLZ , MVT::i8 , MVT::i32); in X86TargetLowering()
375 setOperationPromotedToType(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , MVT::i32); in X86TargetLowering()
377 setOperationAction(ISD::CTLZ , MVT::i8 , Custom); in X86TargetLowering()
378 setOperationAction(ISD::CTLZ , MVT::i16 , Custom); in X86TargetLowering()
379 setOperationAction(ISD::CTLZ , MVT::i32 , Custom); in X86TargetLowering()
380 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , Custom); in X86TargetLowering()
381 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16 , Custom); in X86TargetLowering()
382 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32 , Custom); in X86TargetLowering()
384 setOperationAction(ISD::CTLZ , MVT::i64 , Custom); in X86TargetLowering()
385 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Custom); in X86TargetLowering()
393 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand); in X86TargetLowering()
394 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand); in X86TargetLowering()
398 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); in X86TargetLowering()
399 setOperationAction(ISD::FP16_TO_FP, MVT::f80, Expand); in X86TargetLowering()
400 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand); in X86TargetLowering()
401 setOperationAction(ISD::FP_TO_FP16, MVT::f80, Expand); in X86TargetLowering()
403 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand); in X86TargetLowering()
404 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand); in X86TargetLowering()
405 setLoadExtAction(ISD::EXTLOAD, MVT::f80, MVT::f16, Expand); in X86TargetLowering()
406 setTruncStoreAction(MVT::f32, MVT::f16, Expand); in X86TargetLowering()
407 setTruncStoreAction(MVT::f64, MVT::f16, Expand); in X86TargetLowering()
408 setTruncStoreAction(MVT::f80, MVT::f16, Expand); in X86TargetLowering()
411 setOperationPromotedToType(ISD::CTPOP, MVT::i8, MVT::i32); in X86TargetLowering()
413 setOperationAction(ISD::CTPOP , MVT::i8 , Expand); in X86TargetLowering()
414 setOperationAction(ISD::CTPOP , MVT::i16 , Expand); in X86TargetLowering()
415 setOperationAction(ISD::CTPOP , MVT::i32 , Expand); in X86TargetLowering()
417 setOperationAction(ISD::CTPOP , MVT::i64 , Expand); in X86TargetLowering()
420 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); in X86TargetLowering()
423 setOperationAction(ISD::BSWAP , MVT::i16 , Expand); in X86TargetLowering()
426 setOperationAction(ISD::SELECT , MVT::i1 , Promote); in X86TargetLowering()
428 for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) { in X86TargetLowering()
432 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { in X86TargetLowering()
433 if (VT == MVT::i64 && !Subtarget.is64Bit()) in X86TargetLowering()
440 setOperationAction(ISD::SELECT, MVT::x86mmx, Custom); in X86TargetLowering()
441 setOperationAction(ISD::SELECT_CC, MVT::x86mmx, Expand); in X86TargetLowering()
443 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom); in X86TargetLowering()
446 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom); in X86TargetLowering()
447 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom); in X86TargetLowering()
448 setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom); in X86TargetLowering()
453 for (auto VT : { MVT::i32, MVT::i64 }) { in X86TargetLowering()
454 if (VT == MVT::i64 && !Subtarget.is64Bit()) in X86TargetLowering()
465 for (auto VT : { MVT::i32, MVT::i64 }) { in X86TargetLowering()
466 if (VT == MVT::i64 && !Subtarget.is64Bit()) in X86TargetLowering()
474 setOperationAction(ISD::PREFETCH , MVT::Other, Legal); in X86TargetLowering()
476 setOperationAction(ISD::ATOMIC_FENCE , MVT::Other, Custom); in X86TargetLowering()
479 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { in X86TargetLowering()
490 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom); in X86TargetLowering()
497 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand); in X86TargetLowering()
500 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom); in X86TargetLowering()
501 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom); in X86TargetLowering()
503 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom); in X86TargetLowering()
504 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom); in X86TargetLowering()
506 setOperationAction(ISD::TRAP, MVT::Other, Legal); in X86TargetLowering()
507 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal); in X86TargetLowering()
510 setOperationAction(ISD::VASTART , MVT::Other, Custom); in X86TargetLowering()
511 setOperationAction(ISD::VAEND , MVT::Other, Expand); in X86TargetLowering()
513 setOperationAction(ISD::VAARG, MVT::Other, Is64Bit ? Custom : Expand); in X86TargetLowering()
514 setOperationAction(ISD::VACOPY, MVT::Other, Is64Bit ? Custom : Expand); in X86TargetLowering()
516 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); in X86TargetLowering()
517 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); in X86TargetLowering()
522 setOperationAction(ISD::GC_TRANSITION_START, MVT::Other, Custom); in X86TargetLowering()
523 setOperationAction(ISD::GC_TRANSITION_END, MVT::Other, Custom); in X86TargetLowering()
528 addRegisterClass(MVT::f32, Subtarget.hasAVX512() ? &X86::FR32XRegClass in X86TargetLowering()
530 addRegisterClass(MVT::f64, Subtarget.hasAVX512() ? &X86::FR64XRegClass in X86TargetLowering()
533 for (auto VT : { MVT::f32, MVT::f64 }) { in X86TargetLowering()
554 setOperationAction(ISD::FGETSIGN, MVT::i64, Custom); in X86TargetLowering()
555 setOperationAction(ISD::FGETSIGN, MVT::i32, Custom); in X86TargetLowering()
560 addRegisterClass(MVT::f32, &X86::FR32RegClass); in X86TargetLowering()
562 addRegisterClass(MVT::f64, &X86::RFP64RegClass); in X86TargetLowering()
565 setOperationAction(ISD::FABS , MVT::f32, Custom); in X86TargetLowering()
568 setOperationAction(ISD::FNEG , MVT::f32, Custom); in X86TargetLowering()
571 setOperationAction(ISD::UNDEF, MVT::f64, Expand); in X86TargetLowering()
575 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); in X86TargetLowering()
576 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); in X86TargetLowering()
579 setOperationAction(ISD::FSIN , MVT::f32, Expand); in X86TargetLowering()
580 setOperationAction(ISD::FCOS , MVT::f32, Expand); in X86TargetLowering()
581 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); in X86TargetLowering()
585 setOperationAction(ISD::FSIN, MVT::f64, Expand); in X86TargetLowering()
586 setOperationAction(ISD::FCOS, MVT::f64, Expand); in X86TargetLowering()
587 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); in X86TargetLowering()
592 addRegisterClass(MVT::f64, &X86::RFP64RegClass); in X86TargetLowering()
593 addRegisterClass(MVT::f32, &X86::RFP32RegClass); in X86TargetLowering()
595 for (auto VT : { MVT::f32, MVT::f64 }) { in X86TargetLowering()
607 if (isTypeLegal(MVT::f32)) { in X86TargetLowering()
608 if (UseX87 && (getRegClassFor(MVT::f32) == &X86::RFP32RegClass)) { in X86TargetLowering()
617 if (isTypeLegal(MVT::f64)) { in X86TargetLowering()
618 if (UseX87 && getRegClassFor(MVT::f64) == &X86::RFP64RegClass) { in X86TargetLowering()
628 setOperationAction(ISD::FMA, MVT::f64, Expand); in X86TargetLowering()
629 setOperationAction(ISD::FMA, MVT::f32, Expand); in X86TargetLowering()
634 addRegisterClass(MVT::f128, Subtarget.hasVLX() ? &X86::VR128XRegClass in X86TargetLowering()
636 ValueTypeActions.setTypeAction(MVT::f128, TypeSoftenFloat); in X86TargetLowering()
637 setOperationAction(ISD::FABS , MVT::f128, Custom); in X86TargetLowering()
638 setOperationAction(ISD::FNEG , MVT::f128, Custom); in X86TargetLowering()
639 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Custom); in X86TargetLowering()
642 addRegisterClass(MVT::f80, &X86::RFP80RegClass); in X86TargetLowering()
643 setOperationAction(ISD::UNDEF, MVT::f80, Expand); in X86TargetLowering()
644 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand); in X86TargetLowering()
661 setOperationAction(ISD::FSIN , MVT::f80, Expand); in X86TargetLowering()
662 setOperationAction(ISD::FCOS , MVT::f80, Expand); in X86TargetLowering()
663 setOperationAction(ISD::FSINCOS, MVT::f80, Expand); in X86TargetLowering()
665 setOperationAction(ISD::FFLOOR, MVT::f80, Expand); in X86TargetLowering()
666 setOperationAction(ISD::FCEIL, MVT::f80, Expand); in X86TargetLowering()
667 setOperationAction(ISD::FTRUNC, MVT::f80, Expand); in X86TargetLowering()
668 setOperationAction(ISD::FRINT, MVT::f80, Expand); in X86TargetLowering()
669 setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand); in X86TargetLowering()
670 setOperationAction(ISD::FMA, MVT::f80, Expand); in X86TargetLowering()
674 setOperationAction(ISD::FPOW , MVT::f32 , Expand); in X86TargetLowering()
675 setOperationAction(ISD::FPOW , MVT::f64 , Expand); in X86TargetLowering()
676 setOperationAction(ISD::FPOW , MVT::f80 , Expand); in X86TargetLowering()
678 setOperationAction(ISD::FLOG, MVT::f80, Expand); in X86TargetLowering()
679 setOperationAction(ISD::FLOG2, MVT::f80, Expand); in X86TargetLowering()
680 setOperationAction(ISD::FLOG10, MVT::f80, Expand); in X86TargetLowering()
681 setOperationAction(ISD::FEXP, MVT::f80, Expand); in X86TargetLowering()
682 setOperationAction(ISD::FEXP2, MVT::f80, Expand); in X86TargetLowering()
683 setOperationAction(ISD::FMINNUM, MVT::f80, Expand); in X86TargetLowering()
684 setOperationAction(ISD::FMAXNUM, MVT::f80, Expand); in X86TargetLowering()
687 for (auto VT : { MVT::v4f32, MVT::v8f32, MVT::v16f32, in X86TargetLowering()
688 MVT::v2f64, MVT::v4f64, MVT::v8f64 }) { in X86TargetLowering()
705 for (MVT VT : MVT::vector_valuetypes()) { in X86TargetLowering()
743 for (MVT InnerVT : MVT::vector_valuetypes()) { in X86TargetLowering()
753 if (VT.getVectorElementType() == MVT::i1) in X86TargetLowering()
758 if (VT.getVectorElementType() == MVT::f16) in X86TargetLowering()
766 addRegisterClass(MVT::x86mmx, &X86::VR64RegClass); in X86TargetLowering()
771 addRegisterClass(MVT::v4f32, Subtarget.hasVLX() ? &X86::VR128XRegClass in X86TargetLowering()
774 setOperationAction(ISD::FNEG, MVT::v4f32, Custom); in X86TargetLowering()
775 setOperationAction(ISD::FABS, MVT::v4f32, Custom); in X86TargetLowering()
776 setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Custom); in X86TargetLowering()
777 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); in X86TargetLowering()
778 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom); in X86TargetLowering()
779 setOperationAction(ISD::VSELECT, MVT::v4f32, Custom); in X86TargetLowering()
780 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); in X86TargetLowering()
781 setOperationAction(ISD::SELECT, MVT::v4f32, Custom); in X86TargetLowering()
782 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom); in X86TargetLowering()
786 addRegisterClass(MVT::v2f64, Subtarget.hasVLX() ? &X86::VR128XRegClass in X86TargetLowering()
791 addRegisterClass(MVT::v16i8, Subtarget.hasVLX() ? &X86::VR128XRegClass in X86TargetLowering()
793 addRegisterClass(MVT::v8i16, Subtarget.hasVLX() ? &X86::VR128XRegClass in X86TargetLowering()
795 addRegisterClass(MVT::v4i32, Subtarget.hasVLX() ? &X86::VR128XRegClass in X86TargetLowering()
797 addRegisterClass(MVT::v2i64, Subtarget.hasVLX() ? &X86::VR128XRegClass in X86TargetLowering()
800 for (auto VT : { MVT::v2i8, MVT::v4i8, MVT::v8i8, in X86TargetLowering()
801 MVT::v2i16, MVT::v4i16, MVT::v2i32 }) { in X86TargetLowering()
808 setOperationAction(ISD::MUL, MVT::v2i8, Custom); in X86TargetLowering()
809 setOperationAction(ISD::MUL, MVT::v2i16, Custom); in X86TargetLowering()
810 setOperationAction(ISD::MUL, MVT::v2i32, Custom); in X86TargetLowering()
811 setOperationAction(ISD::MUL, MVT::v4i8, Custom); in X86TargetLowering()
812 setOperationAction(ISD::MUL, MVT::v4i16, Custom); in X86TargetLowering()
813 setOperationAction(ISD::MUL, MVT::v8i8, Custom); in X86TargetLowering()
815 setOperationAction(ISD::MUL, MVT::v16i8, Custom); in X86TargetLowering()
816 setOperationAction(ISD::MUL, MVT::v4i32, Custom); in X86TargetLowering()
817 setOperationAction(ISD::MUL, MVT::v2i64, Custom); in X86TargetLowering()
818 setOperationAction(ISD::MULHU, MVT::v4i32, Custom); in X86TargetLowering()
819 setOperationAction(ISD::MULHS, MVT::v4i32, Custom); in X86TargetLowering()
820 setOperationAction(ISD::MULHU, MVT::v16i8, Custom); in X86TargetLowering()
821 setOperationAction(ISD::MULHS, MVT::v16i8, Custom); in X86TargetLowering()
822 setOperationAction(ISD::MULHU, MVT::v8i16, Legal); in X86TargetLowering()
823 setOperationAction(ISD::MULHS, MVT::v8i16, Legal); in X86TargetLowering()
824 setOperationAction(ISD::MUL, MVT::v8i16, Legal); in X86TargetLowering()
825 setOperationAction(ISD::FNEG, MVT::v2f64, Custom); in X86TargetLowering()
826 setOperationAction(ISD::FABS, MVT::v2f64, Custom); in X86TargetLowering()
827 setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Custom); in X86TargetLowering()
829 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) { in X86TargetLowering()
830 setOperationAction(ISD::SMAX, VT, VT == MVT::v8i16 ? Legal : Custom); in X86TargetLowering()
831 setOperationAction(ISD::SMIN, VT, VT == MVT::v8i16 ? Legal : Custom); in X86TargetLowering()
832 setOperationAction(ISD::UMAX, VT, VT == MVT::v16i8 ? Legal : Custom); in X86TargetLowering()
833 setOperationAction(ISD::UMIN, VT, VT == MVT::v16i8 ? Legal : Custom); in X86TargetLowering()
836 setOperationAction(ISD::UADDSAT, MVT::v16i8, Legal); in X86TargetLowering()
837 setOperationAction(ISD::SADDSAT, MVT::v16i8, Legal); in X86TargetLowering()
838 setOperationAction(ISD::USUBSAT, MVT::v16i8, Legal); in X86TargetLowering()
839 setOperationAction(ISD::SSUBSAT, MVT::v16i8, Legal); in X86TargetLowering()
840 setOperationAction(ISD::UADDSAT, MVT::v8i16, Legal); in X86TargetLowering()
841 setOperationAction(ISD::SADDSAT, MVT::v8i16, Legal); in X86TargetLowering()
842 setOperationAction(ISD::USUBSAT, MVT::v8i16, Legal); in X86TargetLowering()
843 setOperationAction(ISD::SSUBSAT, MVT::v8i16, Legal); in X86TargetLowering()
847 for (auto VT : { MVT::v8i8, MVT::v4i8, MVT::v2i8, in X86TargetLowering()
848 MVT::v4i16, MVT::v2i16 }) { in X86TargetLowering()
856 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); in X86TargetLowering()
857 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); in X86TargetLowering()
858 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); in X86TargetLowering()
865 setOperationAction(ISD::SETCC, MVT::v2i32, Custom); in X86TargetLowering()
867 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) { in X86TargetLowering()
878 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) { in X86TargetLowering()
890 for (MVT VT : MVT::integer_vector_valuetypes()) { in X86TargetLowering()
891 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Custom); in X86TargetLowering()
892 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i16, Custom); in X86TargetLowering()
893 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i32, Custom); in X86TargetLowering()
894 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i8, Custom); in X86TargetLowering()
895 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i16, Custom); in X86TargetLowering()
896 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8i8, Custom); in X86TargetLowering()
899 for (auto VT : { MVT::v2f64, MVT::v2i64 }) { in X86TargetLowering()
904 if (VT == MVT::v2i64 && !Subtarget.is64Bit()) in X86TargetLowering()
912 setOperationAction(ISD::SELECT, MVT::v2f64, Custom); in X86TargetLowering()
913 setOperationAction(ISD::SELECT, MVT::v2i64, Custom); in X86TargetLowering()
914 setOperationAction(ISD::SELECT, MVT::v4i32, Custom); in X86TargetLowering()
915 setOperationAction(ISD::SELECT, MVT::v8i16, Custom); in X86TargetLowering()
916 setOperationAction(ISD::SELECT, MVT::v16i8, Custom); in X86TargetLowering()
918 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal); in X86TargetLowering()
919 setOperationAction(ISD::FP_TO_SINT, MVT::v2i32, Custom); in X86TargetLowering()
920 setOperationAction(ISD::FP_TO_SINT, MVT::v2i16, Custom); in X86TargetLowering()
923 setOperationAction(ISD::FP_TO_SINT, MVT::v2i8, Custom); in X86TargetLowering()
924 setOperationAction(ISD::FP_TO_SINT, MVT::v4i8, Custom); in X86TargetLowering()
925 setOperationAction(ISD::FP_TO_SINT, MVT::v8i8, Custom); in X86TargetLowering()
926 setOperationAction(ISD::FP_TO_SINT, MVT::v2i16, Custom); in X86TargetLowering()
927 setOperationAction(ISD::FP_TO_SINT, MVT::v4i16, Custom); in X86TargetLowering()
928 setOperationAction(ISD::FP_TO_UINT, MVT::v2i8, Custom); in X86TargetLowering()
929 setOperationAction(ISD::FP_TO_UINT, MVT::v4i8, Custom); in X86TargetLowering()
930 setOperationAction(ISD::FP_TO_UINT, MVT::v8i8, Custom); in X86TargetLowering()
931 setOperationAction(ISD::FP_TO_UINT, MVT::v2i16, Custom); in X86TargetLowering()
932 setOperationAction(ISD::FP_TO_UINT, MVT::v4i16, Custom); in X86TargetLowering()
940 setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Custom); in X86TargetLowering()
942 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal); in X86TargetLowering()
943 setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom); in X86TargetLowering()
945 setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom); in X86TargetLowering()
948 setOperationAction(ISD::UINT_TO_FP, MVT::v2f32, Custom); in X86TargetLowering()
950 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom); in X86TargetLowering()
951 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Custom); in X86TargetLowering()
953 for (MVT VT : MVT::fp_vector_valuetypes()) in X86TargetLowering()
954 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2f32, Legal); in X86TargetLowering()
959 setOperationAction(ISD::LOAD, MVT::v2f32, Custom); in X86TargetLowering()
960 setOperationAction(ISD::LOAD, MVT::v2i32, Custom); in X86TargetLowering()
961 setOperationAction(ISD::LOAD, MVT::v4i16, Custom); in X86TargetLowering()
962 setOperationAction(ISD::LOAD, MVT::v8i8, Custom); in X86TargetLowering()
963 setOperationAction(ISD::STORE, MVT::v2f32, Custom); in X86TargetLowering()
964 setOperationAction(ISD::STORE, MVT::v2i32, Custom); in X86TargetLowering()
965 setOperationAction(ISD::STORE, MVT::v4i16, Custom); in X86TargetLowering()
966 setOperationAction(ISD::STORE, MVT::v8i8, Custom); in X86TargetLowering()
968 setOperationAction(ISD::BITCAST, MVT::v2i32, Custom); in X86TargetLowering()
969 setOperationAction(ISD::BITCAST, MVT::v4i16, Custom); in X86TargetLowering()
970 setOperationAction(ISD::BITCAST, MVT::v8i8, Custom); in X86TargetLowering()
972 setOperationAction(ISD::BITCAST, MVT::v16i1, Custom); in X86TargetLowering()
974 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v2i64, Custom); in X86TargetLowering()
975 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i32, Custom); in X86TargetLowering()
976 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v8i16, Custom); in X86TargetLowering()
979 setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom); in X86TargetLowering()
981 setOperationAction(ISD::TRUNCATE, MVT::v2i8, Custom); in X86TargetLowering()
982 setOperationAction(ISD::TRUNCATE, MVT::v2i16, Custom); in X86TargetLowering()
983 setOperationAction(ISD::TRUNCATE, MVT::v2i32, Custom); in X86TargetLowering()
984 setOperationAction(ISD::TRUNCATE, MVT::v4i8, Custom); in X86TargetLowering()
985 setOperationAction(ISD::TRUNCATE, MVT::v4i16, Custom); in X86TargetLowering()
986 setOperationAction(ISD::TRUNCATE, MVT::v8i8, Custom); in X86TargetLowering()
988 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i64, Custom); in X86TargetLowering()
993 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) { in X86TargetLowering()
999 setOperationAction(ISD::ROTL, MVT::v4i32, Custom); in X86TargetLowering()
1000 setOperationAction(ISD::ROTL, MVT::v8i16, Custom); in X86TargetLowering()
1004 setOperationAction(ISD::ROTL, MVT::v16i8, Custom); in X86TargetLowering()
1008 setOperationAction(ISD::ABS, MVT::v16i8, Legal); in X86TargetLowering()
1009 setOperationAction(ISD::ABS, MVT::v8i16, Legal); in X86TargetLowering()
1010 setOperationAction(ISD::ABS, MVT::v4i32, Legal); in X86TargetLowering()
1011 setOperationAction(ISD::BITREVERSE, MVT::v16i8, Custom); in X86TargetLowering()
1012 setOperationAction(ISD::CTLZ, MVT::v16i8, Custom); in X86TargetLowering()
1013 setOperationAction(ISD::CTLZ, MVT::v8i16, Custom); in X86TargetLowering()
1014 setOperationAction(ISD::CTLZ, MVT::v4i32, Custom); in X86TargetLowering()
1015 setOperationAction(ISD::CTLZ, MVT::v2i64, Custom); in X86TargetLowering()
1018 setOperationAction(ISD::ADD, MVT::i16, Custom); in X86TargetLowering()
1019 setOperationAction(ISD::ADD, MVT::i32, Custom); in X86TargetLowering()
1020 setOperationAction(ISD::SUB, MVT::i16, Custom); in X86TargetLowering()
1021 setOperationAction(ISD::SUB, MVT::i32, Custom); in X86TargetLowering()
1025 for (MVT RoundedTy : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) { in X86TargetLowering()
1033 setOperationAction(ISD::SMAX, MVT::v16i8, Legal); in X86TargetLowering()
1034 setOperationAction(ISD::SMAX, MVT::v4i32, Legal); in X86TargetLowering()
1035 setOperationAction(ISD::UMAX, MVT::v8i16, Legal); in X86TargetLowering()
1036 setOperationAction(ISD::UMAX, MVT::v4i32, Legal); in X86TargetLowering()
1037 setOperationAction(ISD::SMIN, MVT::v16i8, Legal); in X86TargetLowering()
1038 setOperationAction(ISD::SMIN, MVT::v4i32, Legal); in X86TargetLowering()
1039 setOperationAction(ISD::UMIN, MVT::v8i16, Legal); in X86TargetLowering()
1040 setOperationAction(ISD::UMIN, MVT::v4i32, Legal); in X86TargetLowering()
1043 setOperationAction(ISD::MUL, MVT::v4i32, Legal); in X86TargetLowering()
1047 setOperationAction(ISD::VSELECT, MVT::v16i8, Legal); in X86TargetLowering()
1051 for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64 }) { in X86TargetLowering()
1059 for (MVT VT : MVT::integer_vector_valuetypes()) { in X86TargetLowering()
1060 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Custom); in X86TargetLowering()
1061 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i16, Custom); in X86TargetLowering()
1062 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i32, Custom); in X86TargetLowering()
1068 setLoadExtAction(LoadExtOp, MVT::v8i16, MVT::v8i8, Legal); in X86TargetLowering()
1069 setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i8, Legal); in X86TargetLowering()
1071 setLoadExtAction(LoadExtOp, MVT::v2i32, MVT::v2i8, Legal); in X86TargetLowering()
1072 setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i8, Legal); in X86TargetLowering()
1073 setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i16, Legal); in X86TargetLowering()
1074 setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i16, Legal); in X86TargetLowering()
1075 setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i32, Legal); in X86TargetLowering()
1080 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); in X86TargetLowering()
1084 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, in X86TargetLowering()
1085 MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) in X86TargetLowering()
1089 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) in X86TargetLowering()
1092 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, in X86TargetLowering()
1093 MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) in X86TargetLowering()
1100 addRegisterClass(MVT::v32i8, Subtarget.hasVLX() ? &X86::VR256XRegClass in X86TargetLowering()
1102 addRegisterClass(MVT::v16i16, Subtarget.hasVLX() ? &X86::VR256XRegClass in X86TargetLowering()
1104 addRegisterClass(MVT::v8i32, Subtarget.hasVLX() ? &X86::VR256XRegClass in X86TargetLowering()
1106 addRegisterClass(MVT::v8f32, Subtarget.hasVLX() ? &X86::VR256XRegClass in X86TargetLowering()
1108 addRegisterClass(MVT::v4i64, Subtarget.hasVLX() ? &X86::VR256XRegClass in X86TargetLowering()
1110 addRegisterClass(MVT::v4f64, Subtarget.hasVLX() ? &X86::VR256XRegClass in X86TargetLowering()
1113 for (auto VT : { MVT::v8f32, MVT::v4f64 }) { in X86TargetLowering()
1126 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i16, MVT::v8i32); in X86TargetLowering()
1127 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i16, MVT::v8i32); in X86TargetLowering()
1128 setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Legal); in X86TargetLowering()
1130 setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Legal); in X86TargetLowering()
1131 setOperationAction(ISD::FP_ROUND, MVT::v4f32, Legal); in X86TargetLowering()
1134 setOperationAction(ISD::BITCAST, MVT::v32i1, Custom); in X86TargetLowering()
1136 for (MVT VT : MVT::fp_vector_valuetypes()) in X86TargetLowering()
1137 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4f32, Legal); in X86TargetLowering()
1141 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) { in X86TargetLowering()
1149 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom); in X86TargetLowering()
1150 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom); in X86TargetLowering()
1151 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom); in X86TargetLowering()
1152 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom); in X86TargetLowering()
1155 setOperationAction(ISD::ROTL, MVT::v8i32, Custom); in X86TargetLowering()
1156 setOperationAction(ISD::ROTL, MVT::v16i16, Custom); in X86TargetLowering()
1160 setOperationAction(ISD::ROTL, MVT::v32i8, Custom); in X86TargetLowering()
1162 setOperationAction(ISD::SELECT, MVT::v4f64, Custom); in X86TargetLowering()
1163 setOperationAction(ISD::SELECT, MVT::v4i64, Custom); in X86TargetLowering()
1164 setOperationAction(ISD::SELECT, MVT::v8i32, Custom); in X86TargetLowering()
1165 setOperationAction(ISD::SELECT, MVT::v16i16, Custom); in X86TargetLowering()
1166 setOperationAction(ISD::SELECT, MVT::v32i8, Custom); in X86TargetLowering()
1167 setOperationAction(ISD::SELECT, MVT::v8f32, Custom); in X86TargetLowering()
1169 for (auto VT : { MVT::v16i16, MVT::v8i32, MVT::v4i64 }) { in X86TargetLowering()
1175 setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom); in X86TargetLowering()
1176 setOperationAction(ISD::TRUNCATE, MVT::v8i16, Custom); in X86TargetLowering()
1177 setOperationAction(ISD::TRUNCATE, MVT::v4i32, Custom); in X86TargetLowering()
1178 setOperationAction(ISD::BITREVERSE, MVT::v32i8, Custom); in X86TargetLowering()
1180 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) { in X86TargetLowering()
1195 for (auto VT : { MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32, in X86TargetLowering()
1196 MVT::v2f64, MVT::v4f64 }) in X86TargetLowering()
1200 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) { in X86TargetLowering()
1205 setOperationAction(ISD::MUL, MVT::v4i64, Custom); in X86TargetLowering()
1206 setOperationAction(ISD::MUL, MVT::v8i32, HasInt256 ? Legal : Custom); in X86TargetLowering()
1207 setOperationAction(ISD::MUL, MVT::v16i16, HasInt256 ? Legal : Custom); in X86TargetLowering()
1208 setOperationAction(ISD::MUL, MVT::v32i8, Custom); in X86TargetLowering()
1210 setOperationAction(ISD::MULHU, MVT::v8i32, Custom); in X86TargetLowering()
1211 setOperationAction(ISD::MULHS, MVT::v8i32, Custom); in X86TargetLowering()
1212 setOperationAction(ISD::MULHU, MVT::v16i16, HasInt256 ? Legal : Custom); in X86TargetLowering()
1213 setOperationAction(ISD::MULHS, MVT::v16i16, HasInt256 ? Legal : Custom); in X86TargetLowering()
1214 setOperationAction(ISD::MULHU, MVT::v32i8, Custom); in X86TargetLowering()
1215 setOperationAction(ISD::MULHS, MVT::v32i8, Custom); in X86TargetLowering()
1217 setOperationAction(ISD::ABS, MVT::v4i64, Custom); in X86TargetLowering()
1218 setOperationAction(ISD::SMAX, MVT::v4i64, Custom); in X86TargetLowering()
1219 setOperationAction(ISD::UMAX, MVT::v4i64, Custom); in X86TargetLowering()
1220 setOperationAction(ISD::SMIN, MVT::v4i64, Custom); in X86TargetLowering()
1221 setOperationAction(ISD::UMIN, MVT::v4i64, Custom); in X86TargetLowering()
1223 setOperationAction(ISD::UADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom); in X86TargetLowering()
1224 setOperationAction(ISD::SADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom); in X86TargetLowering()
1225 setOperationAction(ISD::USUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom); in X86TargetLowering()
1226 setOperationAction(ISD::SSUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom); in X86TargetLowering()
1227 setOperationAction(ISD::UADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom); in X86TargetLowering()
1228 setOperationAction(ISD::SADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom); in X86TargetLowering()
1229 setOperationAction(ISD::USUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom); in X86TargetLowering()
1230 setOperationAction(ISD::SSUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom); in X86TargetLowering()
1232 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32 }) { in X86TargetLowering()
1240 for (auto VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64}) { in X86TargetLowering()
1248 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Custom); in X86TargetLowering()
1252 setLoadExtAction(LoadExtOp, MVT::v16i16, MVT::v16i8, Legal); in X86TargetLowering()
1253 setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i8, Legal); in X86TargetLowering()
1254 setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i8, Legal); in X86TargetLowering()
1255 setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i16, Legal); in X86TargetLowering()
1256 setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i16, Legal); in X86TargetLowering()
1257 setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i32, Legal); in X86TargetLowering()
1261 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64, in X86TargetLowering()
1262 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 }) { in X86TargetLowering()
1269 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, in X86TargetLowering()
1270 MVT::v4f32, MVT::v2f64 }) { in X86TargetLowering()
1275 for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64, in X86TargetLowering()
1276 MVT::v8f32, MVT::v4f64 }) { in X86TargetLowering()
1288 setOperationAction(ISD::VSELECT, MVT::v32i8, Legal); in X86TargetLowering()
1292 setOperationAction(ISD::MGATHER, MVT::v2f32, Custom); in X86TargetLowering()
1293 setOperationAction(ISD::MGATHER, MVT::v2i32, Custom); in X86TargetLowering()
1295 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64, in X86TargetLowering()
1296 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 }) in X86TargetLowering()
1305 addRegisterClass(MVT::v1i1, &X86::VK1RegClass); in X86TargetLowering()
1306 addRegisterClass(MVT::v2i1, &X86::VK2RegClass); in X86TargetLowering()
1307 addRegisterClass(MVT::v4i1, &X86::VK4RegClass); in X86TargetLowering()
1308 addRegisterClass(MVT::v8i1, &X86::VK8RegClass); in X86TargetLowering()
1309 addRegisterClass(MVT::v16i1, &X86::VK16RegClass); in X86TargetLowering()
1311 setOperationAction(ISD::SELECT, MVT::v1i1, Custom); in X86TargetLowering()
1312 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v1i1, Custom); in X86TargetLowering()
1313 setOperationAction(ISD::BUILD_VECTOR, MVT::v1i1, Custom); in X86TargetLowering()
1315 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i1, MVT::v8i32); in X86TargetLowering()
1316 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i1, MVT::v8i32); in X86TargetLowering()
1317 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v4i1, MVT::v4i32); in X86TargetLowering()
1318 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v4i1, MVT::v4i32); in X86TargetLowering()
1319 setOperationAction(ISD::FP_TO_SINT, MVT::v2i1, Custom); in X86TargetLowering()
1320 setOperationAction(ISD::FP_TO_UINT, MVT::v2i1, Custom); in X86TargetLowering()
1324 setOperationAction(ISD::LOAD, MVT::v1i1, Custom); in X86TargetLowering()
1325 setOperationAction(ISD::LOAD, MVT::v2i1, Custom); in X86TargetLowering()
1326 setOperationAction(ISD::LOAD, MVT::v4i1, Custom); in X86TargetLowering()
1327 setOperationAction(ISD::LOAD, MVT::v8i1, Custom); in X86TargetLowering()
1329 setOperationAction(ISD::STORE, MVT::v1i1, Custom); in X86TargetLowering()
1330 setOperationAction(ISD::STORE, MVT::v2i1, Custom); in X86TargetLowering()
1331 setOperationAction(ISD::STORE, MVT::v4i1, Custom); in X86TargetLowering()
1332 setOperationAction(ISD::STORE, MVT::v8i1, Custom); in X86TargetLowering()
1336 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) { in X86TargetLowering()
1342 for (auto VT : { MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 }) { in X86TargetLowering()
1361 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i1, Custom); in X86TargetLowering()
1362 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i1, Custom); in X86TargetLowering()
1363 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i1, Custom); in X86TargetLowering()
1364 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v2i1, Custom); in X86TargetLowering()
1365 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v4i1, Custom); in X86TargetLowering()
1366 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v8i1, Custom); in X86TargetLowering()
1367 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v16i1, Custom); in X86TargetLowering()
1368 for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1 }) in X86TargetLowering()
1376 addRegisterClass(MVT::v16i32, &X86::VR512RegClass); in X86TargetLowering()
1377 addRegisterClass(MVT::v16f32, &X86::VR512RegClass); in X86TargetLowering()
1378 addRegisterClass(MVT::v8i64, &X86::VR512RegClass); in X86TargetLowering()
1379 addRegisterClass(MVT::v8f64, &X86::VR512RegClass); in X86TargetLowering()
1381 for (MVT VT : MVT::fp_vector_valuetypes()) in X86TargetLowering()
1382 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8f32, Legal); in X86TargetLowering()
1385 setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i8, Legal); in X86TargetLowering()
1386 setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i16, Legal); in X86TargetLowering()
1387 setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i8, Legal); in X86TargetLowering()
1388 setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i16, Legal); in X86TargetLowering()
1389 setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i32, Legal); in X86TargetLowering()
1392 for (MVT VT : { MVT::v16f32, MVT::v8f64 }) { in X86TargetLowering()
1399 setOperationAction(ISD::FP_TO_SINT, MVT::v16i32, Legal); in X86TargetLowering()
1400 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v16i16, MVT::v16i32); in X86TargetLowering()
1401 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v16i8, MVT::v16i32); in X86TargetLowering()
1402 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v16i1, MVT::v16i32); in X86TargetLowering()
1403 setOperationAction(ISD::FP_TO_UINT, MVT::v16i32, Legal); in X86TargetLowering()
1404 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v16i1, MVT::v16i32); in X86TargetLowering()
1405 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v16i8, MVT::v16i32); in X86TargetLowering()
1406 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v16i16, MVT::v16i32); in X86TargetLowering()
1407 setOperationAction(ISD::SINT_TO_FP, MVT::v16i32, Legal); in X86TargetLowering()
1408 setOperationAction(ISD::UINT_TO_FP, MVT::v16i32, Legal); in X86TargetLowering()
1410 setTruncStoreAction(MVT::v8i64, MVT::v8i8, Legal); in X86TargetLowering()
1411 setTruncStoreAction(MVT::v8i64, MVT::v8i16, Legal); in X86TargetLowering()
1412 setTruncStoreAction(MVT::v8i64, MVT::v8i32, Legal); in X86TargetLowering()
1413 setTruncStoreAction(MVT::v16i32, MVT::v16i8, Legal); in X86TargetLowering()
1414 setTruncStoreAction(MVT::v16i32, MVT::v16i16, Legal); in X86TargetLowering()
1420 for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64, in X86TargetLowering()
1421 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64}) { in X86TargetLowering()
1427 setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom); in X86TargetLowering()
1428 setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom); in X86TargetLowering()
1429 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom); in X86TargetLowering()
1430 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom); in X86TargetLowering()
1431 setOperationAction(ISD::ANY_EXTEND, MVT::v16i32, Custom); in X86TargetLowering()
1432 setOperationAction(ISD::ANY_EXTEND, MVT::v8i64, Custom); in X86TargetLowering()
1433 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom); in X86TargetLowering()
1434 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom); in X86TargetLowering()
1438 setOperationAction(ISD::ANY_EXTEND, MVT::v8i8, Custom); in X86TargetLowering()
1439 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i8, Custom); in X86TargetLowering()
1440 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i8, Custom); in X86TargetLowering()
1443 for (auto VT : { MVT::v16f32, MVT::v8f64 }) { in X86TargetLowering()
1452 for (auto VT : {MVT::v16i32, MVT::v8i64, MVT::v64i8}) { in X86TargetLowering()
1457 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f64, Custom); in X86TargetLowering()
1458 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i64, Custom); in X86TargetLowering()
1459 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16f32, Custom); in X86TargetLowering()
1460 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i32, Custom); in X86TargetLowering()
1462 setOperationAction(ISD::MUL, MVT::v8i64, Custom); in X86TargetLowering()
1463 setOperationAction(ISD::MUL, MVT::v16i32, Legal); in X86TargetLowering()
1465 setOperationAction(ISD::MULHU, MVT::v16i32, Custom); in X86TargetLowering()
1466 setOperationAction(ISD::MULHS, MVT::v16i32, Custom); in X86TargetLowering()
1468 setOperationAction(ISD::SELECT, MVT::v8f64, Custom); in X86TargetLowering()
1469 setOperationAction(ISD::SELECT, MVT::v8i64, Custom); in X86TargetLowering()
1470 setOperationAction(ISD::SELECT, MVT::v16i32, Custom); in X86TargetLowering()
1471 setOperationAction(ISD::SELECT, MVT::v32i16, Custom); in X86TargetLowering()
1472 setOperationAction(ISD::SELECT, MVT::v64i8, Custom); in X86TargetLowering()
1473 setOperationAction(ISD::SELECT, MVT::v16f32, Custom); in X86TargetLowering()
1475 for (auto VT : { MVT::v16i32, MVT::v8i64 }) { in X86TargetLowering()
1496 setOperationAction(ISD::SINT_TO_FP, MVT::v8i64, Legal); in X86TargetLowering()
1497 setOperationAction(ISD::UINT_TO_FP, MVT::v8i64, Legal); in X86TargetLowering()
1498 setOperationAction(ISD::FP_TO_SINT, MVT::v8i64, Legal); in X86TargetLowering()
1499 setOperationAction(ISD::FP_TO_UINT, MVT::v8i64, Legal); in X86TargetLowering()
1501 setOperationAction(ISD::MUL, MVT::v8i64, Legal); in X86TargetLowering()
1506 for (auto VT : { MVT::v16i32, MVT::v8i64} ) { in X86TargetLowering()
1512 for (auto VT : { MVT::v16i32, MVT::v8i64 }) in X86TargetLowering()
1519 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64, in X86TargetLowering()
1520 MVT::v8f32, MVT::v4f64 }) in X86TargetLowering()
1523 for (auto VT : { MVT::v16i32, MVT::v8i64, MVT::v16f32, MVT::v8f64 }) { in X86TargetLowering()
1538 setOperationAction(ISD::BITCAST, MVT::v32i16, Custom); in X86TargetLowering()
1539 setOperationAction(ISD::BITCAST, MVT::v64i8, Custom); in X86TargetLowering()
1543 for (auto VT : { MVT::v16i32, MVT::v8i64 }) { in X86TargetLowering()
1558 setOperationAction(ISD::FP_TO_UINT, MVT::v8i32, Legal); in X86TargetLowering()
1559 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal); in X86TargetLowering()
1560 setOperationAction(ISD::FP_TO_UINT, MVT::v2i32, Custom); in X86TargetLowering()
1561 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Legal); in X86TargetLowering()
1562 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal); in X86TargetLowering()
1564 for (auto VT : { MVT::v2i64, MVT::v4i64 }) { in X86TargetLowering()
1572 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) { in X86TargetLowering()
1578 setOperationAction(ISD::MSCATTER, MVT::v2f32, Custom); in X86TargetLowering()
1579 setOperationAction(ISD::MSCATTER, MVT::v2i32, Custom); in X86TargetLowering()
1581 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64, in X86TargetLowering()
1582 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 }) in X86TargetLowering()
1586 for (auto VT : { MVT::v2i64, MVT::v4i64 }) { in X86TargetLowering()
1597 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) { in X86TargetLowering()
1603 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) in X86TargetLowering()
1612 addRegisterClass(MVT::v32i1, &X86::VK32RegClass); in X86TargetLowering()
1613 addRegisterClass(MVT::v64i1, &X86::VK64RegClass); in X86TargetLowering()
1615 for (auto VT : { MVT::v32i1, MVT::v64i1 }) { in X86TargetLowering()
1634 setOperationAction(ISD::CONCAT_VECTORS, MVT::v32i1, Custom); in X86TargetLowering()
1635 setOperationAction(ISD::CONCAT_VECTORS, MVT::v64i1, Custom); in X86TargetLowering()
1636 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32i1, Custom); in X86TargetLowering()
1637 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v64i1, Custom); in X86TargetLowering()
1638 for (auto VT : { MVT::v16i1, MVT::v32i1 }) in X86TargetLowering()
1642 setOperationAction(ISD::SIGN_EXTEND, MVT::v32i8, Custom); in X86TargetLowering()
1643 setOperationAction(ISD::ZERO_EXTEND, MVT::v32i8, Custom); in X86TargetLowering()
1644 setOperationAction(ISD::ANY_EXTEND, MVT::v32i8, Custom); in X86TargetLowering()
1651 addRegisterClass(MVT::v32i16, &X86::VR512RegClass); in X86TargetLowering()
1652 addRegisterClass(MVT::v64i8, &X86::VR512RegClass); in X86TargetLowering()
1655 setOperationAction(ISD::SIGN_EXTEND, MVT::v64i8, Custom); in X86TargetLowering()
1656 setOperationAction(ISD::ZERO_EXTEND, MVT::v64i8, Custom); in X86TargetLowering()
1657 setOperationAction(ISD::ANY_EXTEND, MVT::v64i8, Custom); in X86TargetLowering()
1659 setOperationAction(ISD::MUL, MVT::v32i16, Legal); in X86TargetLowering()
1660 setOperationAction(ISD::MUL, MVT::v64i8, Custom); in X86TargetLowering()
1661 setOperationAction(ISD::MULHS, MVT::v32i16, Legal); in X86TargetLowering()
1662 setOperationAction(ISD::MULHU, MVT::v32i16, Legal); in X86TargetLowering()
1663 setOperationAction(ISD::MULHS, MVT::v64i8, Custom); in X86TargetLowering()
1664 setOperationAction(ISD::MULHU, MVT::v64i8, Custom); in X86TargetLowering()
1665 setOperationAction(ISD::CONCAT_VECTORS, MVT::v32i16, Custom); in X86TargetLowering()
1666 setOperationAction(ISD::CONCAT_VECTORS, MVT::v64i8, Custom); in X86TargetLowering()
1667 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32i16, Legal); in X86TargetLowering()
1668 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v64i8, Legal); in X86TargetLowering()
1669 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v32i16, Custom); in X86TargetLowering()
1670 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v64i8, Custom); in X86TargetLowering()
1671 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v32i16, Custom); in X86TargetLowering()
1672 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v64i8, Custom); in X86TargetLowering()
1673 setOperationAction(ISD::SIGN_EXTEND, MVT::v32i16, Custom); in X86TargetLowering()
1674 setOperationAction(ISD::ZERO_EXTEND, MVT::v32i16, Custom); in X86TargetLowering()
1675 setOperationAction(ISD::ANY_EXTEND, MVT::v32i16, Custom); in X86TargetLowering()
1676 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v32i16, Custom); in X86TargetLowering()
1677 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v64i8, Custom); in X86TargetLowering()
1678 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v32i16, Custom); in X86TargetLowering()
1679 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v64i8, Custom); in X86TargetLowering()
1680 setOperationAction(ISD::TRUNCATE, MVT::v32i8, Custom); in X86TargetLowering()
1681 setOperationAction(ISD::BITREVERSE, MVT::v64i8, Custom); in X86TargetLowering()
1683 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v32i16, Custom); in X86TargetLowering()
1684 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, MVT::v32i16, Custom); in X86TargetLowering()
1686 setTruncStoreAction(MVT::v32i16, MVT::v32i8, Legal); in X86TargetLowering()
1688 for (auto VT : { MVT::v64i8, MVT::v32i16 }) { in X86TargetLowering()
1716 setLoadExtAction(ExtType, MVT::v32i16, MVT::v32i8, Legal); in X86TargetLowering()
1720 for (auto VT : { MVT::v64i8, MVT::v32i16 }) in X86TargetLowering()
1725 setOperationAction(ISD::FSHL, MVT::v32i16, Custom); in X86TargetLowering()
1726 setOperationAction(ISD::FSHR, MVT::v32i16, Custom); in X86TargetLowering()
1731 for (auto VT : { MVT::v32i8, MVT::v16i8, MVT::v16i16, MVT::v8i16 }) { in X86TargetLowering()
1741 for (auto VT : { MVT::v16i8, MVT::v32i8, MVT::v8i16, MVT::v16i16 }) in X86TargetLowering()
1747 setTruncStoreAction(MVT::v4i64, MVT::v4i8, Legal); in X86TargetLowering()
1748 setTruncStoreAction(MVT::v4i64, MVT::v4i16, Legal); in X86TargetLowering()
1749 setTruncStoreAction(MVT::v4i64, MVT::v4i32, Legal); in X86TargetLowering()
1750 setTruncStoreAction(MVT::v8i32, MVT::v8i8, Legal); in X86TargetLowering()
1751 setTruncStoreAction(MVT::v8i32, MVT::v8i16, Legal); in X86TargetLowering()
1753 setTruncStoreAction(MVT::v2i64, MVT::v2i8, Legal); in X86TargetLowering()
1754 setTruncStoreAction(MVT::v2i64, MVT::v2i16, Legal); in X86TargetLowering()
1755 setTruncStoreAction(MVT::v2i64, MVT::v2i32, Legal); in X86TargetLowering()
1756 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Legal); in X86TargetLowering()
1757 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal); in X86TargetLowering()
1762 setOperationAction(ISD::SINT_TO_FP, MVT::v2f32, Custom); in X86TargetLowering()
1763 assert(isOperationCustom(ISD::UINT_TO_FP, MVT::v2f32) && in X86TargetLowering()
1766 setOperationAction(ISD::FP_TO_SINT, MVT::v2f32, Custom); in X86TargetLowering()
1767 setOperationAction(ISD::FP_TO_UINT, MVT::v2f32, Custom); in X86TargetLowering()
1771 setTruncStoreAction(MVT::v16i16, MVT::v16i8, Legal); in X86TargetLowering()
1772 setTruncStoreAction(MVT::v8i16, MVT::v8i8, Legal); in X86TargetLowering()
1777 for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64, in X86TargetLowering()
1778 MVT::v16i16, MVT::v8i32, MVT::v4i64 }) { in X86TargetLowering()
1786 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); in X86TargetLowering()
1787 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); in X86TargetLowering()
1788 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); in X86TargetLowering()
1790 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom); in X86TargetLowering()
1791 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom); in X86TargetLowering()
1800 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { in X86TargetLowering()
1801 if (VT == MVT::i64 && !Subtarget.is64Bit()) in X86TargetLowering()
1828 setOperationAction(ISD::FSINCOS, MVT::f64, Custom); in X86TargetLowering()
1829 setOperationAction(ISD::FSINCOS, MVT::f32, Custom); in X86TargetLowering()
1833 setOperationAction(ISD::SDIV, MVT::i128, Custom); in X86TargetLowering()
1834 setOperationAction(ISD::UDIV, MVT::i128, Custom); in X86TargetLowering()
1835 setOperationAction(ISD::SREM, MVT::i128, Custom); in X86TargetLowering()
1836 setOperationAction(ISD::UREM, MVT::i128, Custom); in X86TargetLowering()
1837 setOperationAction(ISD::SDIVREM, MVT::i128, Custom); in X86TargetLowering()
1838 setOperationAction(ISD::UDIVREM, MVT::i128, Custom); in X86TargetLowering()
1850 if (isOperationExpand(Op, MVT::f32)) in X86TargetLowering()
1851 setOperationAction(Op, MVT::f32, Promote); in X86TargetLowering()
1938 X86TargetLowering::getPreferredVectorAction(MVT VT) const { in getPreferredVectorAction()
1939 if (VT == MVT::v32i1 && Subtarget.hasAVX512() && !Subtarget.hasBWI()) in getPreferredVectorAction()
1944 VT.getVectorElementType() != MVT::i1) in getPreferredVectorAction()
1950 MVT X86TargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context, in getRegisterTypeForCallingConv()
1953 if (VT == MVT::v32i1 && Subtarget.hasAVX512() && !Subtarget.hasBWI()) in getRegisterTypeForCallingConv()
1954 return MVT::v32i8; in getRegisterTypeForCallingConv()
1961 if (VT == MVT::v32i1 && Subtarget.hasAVX512() && !Subtarget.hasBWI()) in getNumRegistersForCallingConv()
1970 return MVT::i8; in getSetCCResultType()
1982 return EVT::getVectorVT(Context, MVT::i1, NumElts); in getSetCCResultType()
1988 MVT EltVT = LegalVT.getSimpleVT().getVectorElementType(); in getSetCCResultType()
1990 return EVT::getVectorVT(Context, MVT::i1, NumElts); in getSetCCResultType()
2072 return MVT::v32i8; in getOptimalMemOpType()
2075 return MVT::v16i8; in getOptimalMemOpType()
2079 return MVT::v4f32; in getOptimalMemOpType()
2088 return MVT::f64; in getOptimalMemOpType()
2095 return MVT::i64; in getOptimalMemOpType()
2096 return MVT::i32; in getOptimalMemOpType()
2099 bool X86TargetLowering::isSafeMemOpType(MVT VT) const { in isSafeMemOpType()
2100 if (VT == MVT::f32) in isSafeMemOpType()
2102 else if (VT == MVT::f64) in isSafeMemOpType()
2213 MVT VT) const { in findRepresentativeClass()
2219 case MVT::i8: case MVT::i16: case MVT::i32: case MVT::i64: in findRepresentativeClass()
2222 case MVT::x86mmx: in findRepresentativeClass()
2225 case MVT::f32: case MVT::f64: in findRepresentativeClass()
2226 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64: in findRepresentativeClass()
2227 case MVT::v4f32: case MVT::v2f64: in findRepresentativeClass()
2228 case MVT::v32i8: case MVT::v16i16: case MVT::v8i32: case MVT::v4i64: in findRepresentativeClass()
2229 case MVT::v8f32: case MVT::v4f64: in findRepresentativeClass()
2230 case MVT::v64i8: case MVT::v32i16: case MVT::v16i32: case MVT::v8i64: in findRepresentativeClass()
2231 case MVT::v16f32: case MVT::v8f64: in findRepresentativeClass()
2371 if (ValVT == MVT::v1i1) in lowerMasksToReg()
2375 if ((ValVT == MVT::v8i1 && (ValLoc == MVT::i8 || ValLoc == MVT::i32)) || in lowerMasksToReg()
2376 (ValVT == MVT::v16i1 && (ValLoc == MVT::i16 || ValLoc == MVT::i32))) { in lowerMasksToReg()
2380 EVT TempValLoc = ValVT == MVT::v8i1 ? MVT::i8 : MVT::i16; in lowerMasksToReg()
2382 if (ValLoc == MVT::i32) in lowerMasksToReg()
2387 if ((ValVT == MVT::v32i1 && ValLoc == MVT::i32) || in lowerMasksToReg()
2388 (ValVT == MVT::v64i1 && ValLoc == MVT::i64)) { in lowerMasksToReg()
2404 assert(Arg.getValueType() == MVT::i64 && "Expecting 64 bit value"); in Passv64i1ArgInRegs()
2409 Arg = DAG.getBitcast(MVT::i64, Arg); in Passv64i1ArgInRegs()
2413 Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, Dl, MVT::i32, Arg, in Passv64i1ArgInRegs()
2414 DAG.getConstant(0, Dl, MVT::i32)); in Passv64i1ArgInRegs()
2415 Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, Dl, MVT::i32, Arg, in Passv64i1ArgInRegs()
2416 DAG.getConstant(1, Dl, MVT::i32)); in Passv64i1ArgInRegs()
2450 MVT::i32)); in LowerReturn()
2471 if (ValVT.isVector() && ValVT.getVectorElementType() == MVT::i1) in LowerReturn()
2484 if ((ValVT == MVT::f32 || ValVT == MVT::f64 || in LowerReturn()
2489 } else if (ValVT == MVT::f64 && in LowerReturn()
2505 ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy); in LowerReturn()
2514 if (ValVT == MVT::x86mmx) { in LowerReturn()
2516 ValToCopy = DAG.getBitcast(MVT::i64, ValToCopy); in LowerReturn()
2517 ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, in LowerReturn()
2522 ValToCopy = DAG.getBitcast(MVT::v4f32, ValToCopy); in LowerReturn()
2530 assert(VA.getValVT() == MVT::v64i1 && in LowerReturn()
2610 RetOps.push_back(DAG.getRegister(*I, MVT::i64)); in LowerReturn()
2625 return DAG.getNode(opcode, dl, MVT::Other, RetOps); in LowerReturn()
2637 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue) in isUsedByReturnOnly()
2653 UI->getOperand(UI->getNumOperands()-1).getValueType() != MVT::Glue) in isUsedByReturnOnly()
2667 MVT ReturnMVT = MVT::i32; in getTypeForExtReturn()
2670 if (VT == MVT::i1 || (!Darwin && (VT == MVT::i8 || VT == MVT::i16))) { in getTypeForExtReturn()
2676 ReturnMVT = MVT::i8; in getTypeForExtReturn()
2698 assert(VA.getValVT() == MVT::v64i1 && in getv64i1Argument()
2717 ArgValueLo = DAG.getCopyFromReg(Root, Dl, Reg, MVT::i32); in getv64i1Argument()
2719 ArgValueHi = DAG.getCopyFromReg(Root, Dl, Reg, MVT::i32); in getv64i1Argument()
2724 DAG.getCopyFromReg(Root, Dl, VA.getLocReg(), MVT::i32, *InFlag); in getv64i1Argument()
2727 DAG.getCopyFromReg(Root, Dl, NextVA.getLocReg(), MVT::i32, *InFlag); in getv64i1Argument()
2732 Lo = DAG.getBitcast(MVT::v32i1, ArgValueLo); in getv64i1Argument()
2735 Hi = DAG.getBitcast(MVT::v32i1, ArgValueHi); in getv64i1Argument()
2738 return DAG.getNode(ISD::CONCAT_VECTORS, Dl, MVT::v64i1, Lo, Hi); in getv64i1Argument()
2749 if (ValVT == MVT::v1i1) in lowerRegToMasks()
2750 return DAG.getNode(ISD::SCALAR_TO_VECTOR, Dl, MVT::v1i1, ValReturned); in lowerRegToMasks()
2752 if (ValVT == MVT::v64i1) { in lowerRegToMasks()
2754 assert(ValLoc == MVT::i64 && "Expecting only i64 locations"); in lowerRegToMasks()
2757 MVT maskLen; in lowerRegToMasks()
2759 case MVT::v8i1: in lowerRegToMasks()
2760 maskLen = MVT::i8; in lowerRegToMasks()
2762 case MVT::v16i1: in lowerRegToMasks()
2763 maskLen = MVT::i16; in lowerRegToMasks()
2765 case MVT::v32i1: in lowerRegToMasks()
2766 maskLen = MVT::i32; in lowerRegToMasks()
2809 if ((CopyVT == MVT::f32 || CopyVT == MVT::f64 || CopyVT == MVT::f128) && in LowerCallResult()
2822 CopyVT = MVT::f80; in LowerCallResult()
2828 assert(VA.getValVT() == MVT::v64i1 && in LowerCallResult()
2844 if (VA.isExtInLoc() && (VA.getValVT().getScalarType() == MVT::i1)) { in LowerCallResult()
2846 ((VA.getLocVT() == MVT::i64) || (VA.getLocVT() == MVT::i32) || in LowerCallResult()
2847 (VA.getLocVT() == MVT::i16) || (VA.getLocVT() == MVT::i8))) { in LowerCallResult()
2910 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32); in CreateCopyOfByValArgument()
2975 MVT PtrVT = getPointerTy(DAG.getDataLayout()); in LowerMemArgument()
2981 VA.isExtInLoc() && VA.getValVT().getScalarType() == MVT::i1 && in LowerMemArgument()
3171 (Ins.size() == 2 && ((Is64Bit && Ins[1].VT == MVT::i64) || in LowerFormalArguments()
3172 (!Is64Bit && Ins[1].VT == MVT::i32))); in LowerFormalArguments()
3208 VA.getValVT() == MVT::v64i1 && in LowerFormalArguments()
3217 if (RegVT == MVT::i8) in LowerFormalArguments()
3219 else if (RegVT == MVT::i16) in LowerFormalArguments()
3221 else if (RegVT == MVT::i32) in LowerFormalArguments()
3223 else if (Is64Bit && RegVT == MVT::i64) in LowerFormalArguments()
3225 else if (RegVT == MVT::f32) in LowerFormalArguments()
3227 else if (RegVT == MVT::f64) in LowerFormalArguments()
3229 else if (RegVT == MVT::f80) in LowerFormalArguments()
3231 else if (RegVT == MVT::f128) in LowerFormalArguments()
3239 else if (RegVT == MVT::x86mmx) in LowerFormalArguments()
3241 else if (RegVT == MVT::v1i1) in LowerFormalArguments()
3243 else if (RegVT == MVT::v8i1) in LowerFormalArguments()
3245 else if (RegVT == MVT::v16i1) in LowerFormalArguments()
3247 else if (RegVT == MVT::v32i1) in LowerFormalArguments()
3249 else if (RegVT == MVT::v64i1) in LowerFormalArguments()
3272 if (RegVT.isVector() && VA.getValVT().getScalarType() != MVT::i1) in LowerFormalArguments()
3275 VA.getValVT().getScalarType() == MVT::i1 && in LowerFormalArguments()
3276 ((VA.getLocVT() == MVT::i64) || (VA.getLocVT() == MVT::i32) || in LowerFormalArguments()
3277 (VA.getLocVT() == MVT::i16) || (VA.getLocVT() == MVT::i8))) { in LowerFormalArguments()
3310 MVT PtrTy = getPointerTy(DAG.getDataLayout()); in LowerFormalArguments()
3315 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain); in LowerFormalArguments()
3358 DAG.getCopyFromReg(Chain, dl, GPR, MVT::i64)); in LowerFormalArguments()
3362 ALVal = DAG.getCopyFromReg(Chain, dl, AL, MVT::i8); in LowerFormalArguments()
3366 DAG.getCopyFromReg(Chain, dl, XMMReg, MVT::v4f32)); in LowerFormalArguments()
3418 MVT::Other, SaveXMMOps)); in LowerFormalArguments()
3422 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); in LowerFormalArguments()
3427 MVT VecVT = MVT::Other; in LowerFormalArguments()
3432 VecVT = MVT::v16f32; in LowerFormalArguments()
3434 VecVT = MVT::v8f32; in LowerFormalArguments()
3436 VecVT = MVT::v4f32; in LowerFormalArguments()
3439 SmallVector<MVT, 2> RegParmTypes; in LowerFormalArguments()
3440 MVT IntVT = Is64Bit ? MVT::i64 : MVT::i32; in LowerFormalArguments()
3442 if (VecVT != MVT::Other) in LowerFormalArguments()
3453 Forwards.push_back(ForwardedRegister(ALVReg, X86::AL, MVT::i8)); in LowerFormalArguments()
3572 static SDValue getMOVL(SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1, in getMOVL()
3759 Arg.getValueType().getVectorElementType() == MVT::i1) in LowerCall()
3763 Arg = DAG.getBitcast(MVT::i64, Arg); in LowerCall()
3764 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg); in LowerCall()
3765 Arg = getMOVL(DAG, dl, MVT::v2i64, DAG.getUNDEF(MVT::v2i64), Arg); in LowerCall()
3801 assert(VA.getValVT() == MVT::v64i1 && in LowerCall()
3832 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); in LowerCall()
3880 MVT::i8))); in LowerCall()
3955 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2); in LowerCall()
4019 Callee->getValueType(0) == MVT::i32) { in LowerCall()
4021 Callee = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Callee); in LowerCall()
4025 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerCall()
4039 Ops.push_back(DAG.getConstant(FPDiff, dl, MVT::i32)); in LowerCall()
4799 MVT VT = MVT::getVT(I.getArgOperand(1)->getType()); in getTgtMemIntrinsic()
4800 MVT ScalarVT = MVT::INVALID_SIMPLE_VALUE_TYPE; in getTgtMemIntrinsic()
4802 ScalarVT = MVT::i8; in getTgtMemIntrinsic()
4804 ScalarVT = MVT::i16; in getTgtMemIntrinsic()
4806 ScalarVT = MVT::i32; in getTgtMemIntrinsic()
4808 Info.memVT = MVT::getVectorVT(ScalarVT, VT.getVectorNumElements()); in getTgtMemIntrinsic()
4894 return !IsSigned && FpVT == MVT::f80 && Subtarget.hasCMov(); in shouldUseStrictFP_TO_INT()
4904 if (ResVT.getVectorElementType() == MVT::i1) in isExtractSubvectorCheap()
4936 BitcastVT.getVectorElementType() == MVT::i1) in isLoadBitCastBeneficial()
4939 if (!Subtarget.hasDQI() && BitcastVT == MVT::v8i1 && LoadVT == MVT::i8) in isLoadBitCastBeneficial()
4978 if (VT != MVT::i32 && VT != MVT::i64) in hasAndNotCompare()
4995 if (VT == MVT::v4i32) in hasAndNot()
5009 if (VT == MVT::i64 && !Subtarget.is64Bit()) in preferShiftsToClearExtremeBits()
5021 MVT X86TargetLowering::hasFastEqualityCompare(unsigned NumBits) const { in hasFastEqualityCompare()
5022 MVT VT = MVT::getIntegerVT(NumBits); in hasFastEqualityCompare()
5027 if (NumBits == 128 && isTypeLegal(MVT::v16i8)) in hasFastEqualityCompare()
5028 return MVT::v16i8; in hasFastEqualityCompare()
5031 if (NumBits == 256 && isTypeLegal(MVT::v32i8)) in hasFastEqualityCompare()
5032 return MVT::v32i8; in hasFastEqualityCompare()
5038 return MVT::INVALID_SIMPLE_VALUE_TYPE; in hasFastEqualityCompare()
5222 static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG, in getConstVector()
5228 MVT ConstVecVT = VT; in getConstVector()
5230 bool In64BitMode = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64); in getConstVector()
5231 if (!In64BitMode && VT.getVectorElementType() == MVT::i64) { in getConstVector()
5232 ConstVecVT = MVT::getVectorVT(MVT::i32, NumElts * 2); in getConstVector()
5236 MVT EltVT = ConstVecVT.getVectorElementType(); in getConstVector()
5253 MVT VT, SelectionDAG &DAG, const SDLoc &dl) { in getConstVector()
5259 MVT ConstVecVT = VT; in getConstVector()
5261 bool In64BitMode = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64); in getConstVector()
5262 if (!In64BitMode && VT.getVectorElementType() == MVT::i64) { in getConstVector()
5263 ConstVecVT = MVT::getVectorVT(MVT::i32, NumElts * 2); in getConstVector()
5267 MVT EltVT = ConstVecVT.getVectorElementType(); in getConstVector()
5278 } else if (EltVT == MVT::f32) { in getConstVector()
5281 } else if (EltVT == MVT::f64) { in getConstVector()
5294 static SDValue getZeroVector(MVT VT, const X86Subtarget &Subtarget, in getZeroVector()
5297 VT.getVectorElementType() == MVT::i1) && in getZeroVector()
5305 Vec = DAG.getConstantFP(+0.0, dl, MVT::v4f32); in getZeroVector()
5306 } else if (VT.getVectorElementType() == MVT::i1) { in getZeroVector()
5312 Vec = DAG.getConstant(0, dl, MVT::getVectorVT(MVT::i32, Num32BitElts)); in getZeroVector()
5400 static SDValue widenSubVector(MVT VT, SDValue Vec, bool ZeroNewElements, in widenSubVector()
5493 MVT OpVT = Op.getSimpleValueType(); in insert1BitVector()
5499 MVT WideOpVT = OpVT; in insert1BitVector()
5501 WideOpVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1; in insert1BitVector()
5513 MVT SubVecVT = SubVec.getSimpleValueType(); in insert1BitVector()
5524 SDValue ShiftBits = DAG.getConstant(SubVecNumElems, dl, MVT::i8); in insert1BitVector()
5543 DAG.getConstant(IdxVal, dl, MVT::i8)); in insert1BitVector()
5553 DAG.getConstant(ShiftLeft, dl, MVT::i8)); in insert1BitVector()
5556 DAG.getConstant(ShiftRight, dl, MVT::i8)); in insert1BitVector()
5563 DAG.getConstant(IdxVal, dl, MVT::i8)); in insert1BitVector()
5576 SDValue ShiftBits = DAG.getConstant(NumElems - IdxVal, dl, MVT::i8); in insert1BitVector()
5592 DAG.getConstant(IdxVal, dl, MVT::i8)); in insert1BitVector()
5598 DAG.getConstant(ShiftLeft, dl, MVT::i8)); in insert1BitVector()
5602 DAG.getConstant(ShiftRight, dl, MVT::i8)); in insert1BitVector()
5625 SDValue Vec = DAG.getConstant(Ones, dl, MVT::getVectorVT(MVT::i32, NumElts)); in getOnesVector()
5655 static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, MVT VT, in getUnpackl()
5663 static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, MVT VT, in getUnpackh()
5678 MVT VT = V2.getSimpleValueType(); in getShuffleVectorZeroOrUndef()
6030 static void createPackShuffleMask(MVT VT, SmallVectorImpl<int> &Mask, in createPackShuffleMask()
6078 static bool getTargetShuffleMask(SDNode *N, MVT VT, bool AllowSentinelZero, in getTargetShuffleMask()
6161 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected"); in getTargetShuffleMask()
6172 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected"); in getTargetShuffleMask()
6180 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected"); in getTargetShuffleMask()
6245 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected"); in getTargetShuffleMask()
6397 MVT VT = N.getSimpleValueType(); in setTargetShuffleZeroElements()
6482 MVT VT = N.getSimpleValueType(); in getFauxShuffleMask()
6607 N0.getOperand(0).getValueType() == MVT::v8i16) || in getFauxShuffleMask()
6609 N0.getOperand(0).getValueType() == MVT::v16i8)) { in getFauxShuffleMask()
6735 MVT SrcVT = Src.getSimpleValueType(); in getFauxShuffleMask()
6818 MVT ShufVT = V.getSimpleValueType(); in getShuffleScalarElt()
6819 MVT ShufSVT = ShufVT.getVectorElementType(); in getShuffleScalarElt()
6866 MVT VT = Op.getSimpleValueType(); in LowerBuildVectorAsInsert()
6868 assert(((VT == MVT::v8i16 && Subtarget.hasSSE2()) || in LowerBuildVectorAsInsert()
6869 ((VT == MVT::v16i8 || VT == MVT::v4i32) && Subtarget.hasSSE41())) && in LowerBuildVectorAsInsert()
6890 V = DAG.getAnyExtOrTrunc(Op.getOperand(i), dl, MVT::i32); in LowerBuildVectorAsInsert()
6891 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, V); in LowerBuildVectorAsInsert()
6892 V = DAG.getNode(X86ISD::VZEXT_MOVL, dl, MVT::v4i32, V); in LowerBuildVectorAsInsert()
6926 V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl); in LowerBuildVectorv16i8()
6928 V = DAG.getUNDEF(MVT::v8i16); in LowerBuildVectorv16i8()
6939 DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i - 1)); in LowerBuildVectorv16i8()
6942 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i)); in LowerBuildVectorv16i8()
6943 ThisElt = DAG.getNode(ISD::SHL, dl, MVT::i16, ThisElt, in LowerBuildVectorv16i8()
6944 DAG.getConstant(8, dl, MVT::i8)); in LowerBuildVectorv16i8()
6946 ThisElt = DAG.getNode(ISD::OR, dl, MVT::i16, ThisElt, LastElt); in LowerBuildVectorv16i8()
6952 V = NumZero ? DAG.getZExtOrTrunc(ThisElt, dl, MVT::i32) in LowerBuildVectorv16i8()
6953 : DAG.getAnyExtOrTrunc(ThisElt, dl, MVT::i32); in LowerBuildVectorv16i8()
6954 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, V); in LowerBuildVectorv16i8()
6955 V = DAG.getNode(X86ISD::VZEXT_MOVL, dl, MVT::v4i32, V); in LowerBuildVectorv16i8()
6956 V = DAG.getBitcast(MVT::v8i16, V); in LowerBuildVectorv16i8()
6958 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, ThisElt, in LowerBuildVectorv16i8()
6965 return DAG.getBitcast(MVT::v16i8, V); in LowerBuildVectorv16i8()
6993 MVT VT = Op.getSimpleValueType(); in LowerBuildVectorv4x32()
6994 MVT EltVT = VT.getVectorElementType(); in LowerBuildVectorv4x32()
6999 SDValue NewBV = DAG.getBitcast(MVT::v2f64, DAG.getBuildVector(VT, DL, Ops)); in LowerBuildVectorv4x32()
7000 SDValue Dup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, NewBV); in LowerBuildVectorv4x32()
7025 MVT VT = Elt.getOperand(0).getSimpleValueType(); in LowerBuildVectorv4x32()
7036 MVT VT = V1.getSimpleValueType(); in LowerBuildVectorv4x32()
7089 if (V1.getSimpleValueType() != MVT::v4f32) in LowerBuildVectorv4x32()
7090 V1 = DAG.getBitcast(MVT::v4f32, V1); in LowerBuildVectorv4x32()
7091 if (V2.getSimpleValueType() != MVT::v4f32) in LowerBuildVectorv4x32()
7092 V2 = DAG.getBitcast(MVT::v4f32, V2); in LowerBuildVectorv4x32()
7100 SDValue Result = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2, in LowerBuildVectorv4x32()
7110 MVT ShVT = MVT::v16i8; in getVShift()
7114 SDValue ShiftVal = DAG.getConstant(NumBits/8, dl, MVT::i8); in getVShift()
7118 static SDValue LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, const SDLoc &dl, in LowerAsSplatVectorLoad()
7129 if (PVT != MVT::i32 && PVT != MVT::f32) in LowerAsSplatVectorLoad()
7333 MVT VecSVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(LoadSize) in EltsFromConsecutiveLoads()
7334 : MVT::getIntegerVT(LoadSize); in EltsFromConsecutiveLoads()
7335 MVT VecVT = MVT::getVectorVT(VecSVT, VT.getSizeInBits() / LoadSize); in EltsFromConsecutiveLoads()
7337 SDVTList Tys = DAG.getVTList(VecVT, MVT::Other); in EltsFromConsecutiveLoads()
7353 static Constant *getConstantVector(MVT VT, const APInt &SplatValue, in getConstantVector()
7392 unsigned &NumElt, MVT &EltType) { in isSplatZeroExtended()
7418 EltType = MVT::getIntegerVT(ExtVTSize); in isSplatZeroExtended()
7439 MVT VT = BVOp->getSimpleValueType(0); in lowerBuildVectorAsBroadcast()
7455 MVT EltType = VT.getScalarType(); in lowerBuildVectorAsBroadcast()
7466 MVT MaskVT = BOperand.getSimpleValueType(); in lowerBuildVectorAsBroadcast()
7467 if ((EltType == MVT::i64 && MaskVT == MVT::v8i1) || // for broadcastmb2q in lowerBuildVectorAsBroadcast()
7468 (EltType == MVT::i32 && MaskVT == MVT::v16i1)) { // for broadcastmw2d in lowerBuildVectorAsBroadcast()
7471 MVT::getVectorVT(EltType, NumElts), BOperand); in lowerBuildVectorAsBroadcast()
7494 MVT PVT = TLI.getPointerTy(DAG.getDataLayout()); in lowerBuildVectorAsBroadcast()
7500 MVT CVT = MVT::getIntegerVT(SplatBitSize); in lowerBuildVectorAsBroadcast()
7512 MVT::getVectorVT(CVT, Repeat), Ld); in lowerBuildVectorAsBroadcast()
7519 MVT CVT = MVT::getFloatingPointVT(SplatBitSize); in lowerBuildVectorAsBroadcast()
7536 MVT::getVectorVT(CVT, Repeat), Ld); in lowerBuildVectorAsBroadcast()
7540 MVT CVT = VT.getScalarType(); in lowerBuildVectorAsBroadcast()
7547 MVT::getVectorVT(CVT, NumElm), dl, DAG.getEntryNode(), VCP, in lowerBuildVectorAsBroadcast()
7672 MVT ShuffleVecVT = ShuffleVec.getSimpleValueType(); in getUnderlyingExtractedFromVec()
7685 MVT VT = Op.getSimpleValueType(); in buildFromShuffleMostly()
7767 MVT VT = MVT::getIntegerVT(std::max((int)Op.getValueSizeInBits(), 8)); in ConvertI1VectorToInteger()
7774 MVT VT = Op.getSimpleValueType(); in LowerBUILD_VECTORvXi1()
7775 assert((VT.getVectorElementType() == MVT::i1) && in LowerBUILD_VECTORvXi1()
7786 if (VT == MVT::v64i1 && !Subtarget.is64Bit()) { in LowerBUILD_VECTORvXi1()
7789 DAG.getBuildVector(MVT::v32i1, dl, Op.getNode()->ops().slice(0, 32)); in LowerBUILD_VECTORvXi1()
7791 DAG.getBuildVector(MVT::v32i1, dl, Op.getNode()->ops().slice(32, 32)); in LowerBUILD_VECTORvXi1()
7796 return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lower, Upper); in LowerBUILD_VECTORvXi1()
7801 SDValue ExtVec = DAG.getBitcast(MVT::v8i1, Imm); in LowerBUILD_VECTORvXi1()
7838 MVT ImmVT = MVT::getIntegerVT(std::max((int)VT.getSizeInBits(), 8)); in LowerBUILD_VECTORvXi1()
7848 SDValue ExtVec = DAG.getBitcast(MVT::v8i1, Imm); in LowerBUILD_VECTORvXi1()
8000 MVT VT = V0.getSimpleValueType(); in ExpandHorizontalBinOp()
8009 MVT NewVT = V0_LO.getSimpleValueType(); in ExpandHorizontalBinOp()
8043 MVT VT = BV->getSimpleValueType(0); in isAddSubOrSubAdd()
8203 MVT VT = BV->getSimpleValueType(0); in lowerToAddSubOrFMAddSub()
8231 MVT VT = BV->getSimpleValueType(0); in isHopBuildVector()
8321 MVT VT = BV->getSimpleValueType(0); in getHopForBuildVector()
8352 MVT VT = BV->getSimpleValueType(0); in LowerToHorizontalOp()
8355 if ((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget.hasSSE3()) in LowerToHorizontalOp()
8359 if ((VT == MVT::v8i16 || VT == MVT::v4i32) && Subtarget.hasSSSE3()) in LowerToHorizontalOp()
8363 if ((VT == MVT::v8f32 || VT == MVT::v4f64) && Subtarget.hasAVX()) in LowerToHorizontalOp()
8367 if ((VT == MVT::v16i16 || VT == MVT::v8i32) && Subtarget.hasAVX2()) in LowerToHorizontalOp()
8390 if (VT == MVT::v8i32 || VT == MVT::v16i16) { in LowerToHorizontalOp()
8430 if (VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 || in LowerToHorizontalOp()
8431 VT == MVT::v16i16) { in LowerToHorizontalOp()
8472 MVT VT = Op->getSimpleValueType(0); in lowerBuildVectorToBitOp()
8522 MVT VT = Op.getSimpleValueType(); in materializeVectorConstant()
8528 if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) in materializeVectorConstant()
8538 if (VT == MVT::v4i32 || VT == MVT::v16i32 || in materializeVectorConstant()
8539 (VT == MVT::v8i32 && Subtarget.hasInt256())) in materializeVectorConstant()
8551 static SDValue createVariablePermute(MVT VT, SDValue SrcVec, SDValue IndicesVec, in createVariablePermute()
8554 MVT ShuffleVT = VT; in createVariablePermute()
8572 VT = MVT::getVectorVT(VT.getScalarType(), Scale * NumElts); in createVariablePermute()
8614 case MVT::v16i8: in createVariablePermute()
8618 case MVT::v8i16: in createVariablePermute()
8623 ShuffleVT = MVT::v16i8; in createVariablePermute()
8626 case MVT::v4f32: in createVariablePermute()
8627 case MVT::v4i32: in createVariablePermute()
8630 ShuffleVT = MVT::v4f32; in createVariablePermute()
8633 ShuffleVT = MVT::v16i8; in createVariablePermute()
8636 case MVT::v2f64: in createVariablePermute()
8637 case MVT::v2i64: in createVariablePermute()
8642 ShuffleVT = MVT::v2f64; in createVariablePermute()
8653 case MVT::v32i8: in createVariablePermute()
8663 DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, LoIdx), in createVariablePermute()
8664 DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, HiIdx)); in createVariablePermute()
8683 return SplitOpsAndApply(DAG, Subtarget, DL, MVT::v32i8, Ops, in createVariablePermute()
8687 case MVT::v16i16: in createVariablePermute()
8695 MVT::v32i8, DAG.getBitcast(MVT::v32i8, SrcVec), in createVariablePermute()
8696 DAG.getBitcast(MVT::v32i8, IndicesVec), DL, DAG, Subtarget)); in createVariablePermute()
8699 case MVT::v8f32: in createVariablePermute()
8700 case MVT::v8i32: in createVariablePermute()
8704 SrcVec = DAG.getBitcast(MVT::v8f32, SrcVec); in createVariablePermute()
8705 SDValue LoLo = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec, in createVariablePermute()
8707 SDValue HiHi = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec, in createVariablePermute()
8710 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v8f32, in createVariablePermute()
8712 DAG.getConstant(0, DL, MVT::i8))); in createVariablePermute()
8716 DL, IndicesVec, DAG.getConstant(3, DL, MVT::v8i32), in createVariablePermute()
8717 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, HiHi, IndicesVec), in createVariablePermute()
8718 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, LoLo, IndicesVec), in createVariablePermute()
8723 case MVT::v4i64: in createVariablePermute()
8724 case MVT::v4f64: in createVariablePermute()
8727 MVT WidenSrcVT = MVT::getVectorVT(VT.getScalarType(), 8); in createVariablePermute()
8730 IndicesVec = widenSubVector(MVT::v8i64, IndicesVec, false, Subtarget, in createVariablePermute()
8738 SrcVec = DAG.getBitcast(MVT::v4f64, SrcVec); in createVariablePermute()
8740 DAG.getVectorShuffle(MVT::v4f64, DL, SrcVec, SrcVec, {0, 1, 0, 1}); in createVariablePermute()
8742 DAG.getVectorShuffle(MVT::v4f64, DL, SrcVec, SrcVec, {2, 3, 2, 3}); in createVariablePermute()
8746 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v4f64, in createVariablePermute()
8748 DAG.getConstant(0, DL, MVT::i8))); in createVariablePermute()
8752 DL, IndicesVec, DAG.getConstant(2, DL, MVT::v4i64), in createVariablePermute()
8753 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, HiHi, IndicesVec), in createVariablePermute()
8754 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, LoLo, IndicesVec), in createVariablePermute()
8759 case MVT::v64i8: in createVariablePermute()
8763 case MVT::v32i16: in createVariablePermute()
8767 case MVT::v16f32: in createVariablePermute()
8768 case MVT::v16i32: in createVariablePermute()
8769 case MVT::v8f64: in createVariablePermute()
8770 case MVT::v8i64: in createVariablePermute()
8848 MVT VT = V.getSimpleValueType(); in LowerBUILD_VECTORAsVariablePermute()
8856 MVT VT = Op.getSimpleValueType(); in LowerBUILD_VECTOR()
8857 MVT EltVT = VT.getVectorElementType(); in LowerBUILD_VECTOR()
8861 if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512()) in LowerBUILD_VECTOR()
8980 if (EltVT == MVT::i32 || EltVT == MVT::f32 || EltVT == MVT::f64 || in LowerBUILD_VECTOR()
8981 (EltVT == MVT::i64 && Subtarget.is64Bit())) { in LowerBUILD_VECTOR()
8992 if (EltVT == MVT::i16 || EltVT == MVT::i8) { in LowerBUILD_VECTOR()
8993 Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item); in LowerBUILD_VECTOR()
8995 MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits()/32); in LowerBUILD_VECTOR()
9002 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Item); in LowerBUILD_VECTOR()
9008 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Item); in LowerBUILD_VECTOR()
9085 MVT WideEltVT = VT.isFloatingPoint() ? MVT::f64 : MVT::i64; in LowerBUILD_VECTOR()
9086 MVT NarrowVT = MVT::getVectorVT(EltVT, 4); in LowerBUILD_VECTOR()
9088 SDValue NewBV = DAG.getBitcast(MVT::getVectorVT(WideEltVT, 2), in LowerBUILD_VECTOR()
9091 MVT BcastVT = MVT::getVectorVT(WideEltVT, NumElems/2); in LowerBUILD_VECTOR()
9100 MVT HVT = MVT::getVectorVT(EltVT, NumElems/2); in LowerBUILD_VECTOR()
9239 MVT ResVT = Op.getSimpleValueType(); in LowerAVXCONCAT_VECTORS()
9263 MVT HalfVT = MVT::getVectorVT(ResVT.getVectorElementType(), in LowerAVXCONCAT_VECTORS()
9277 MVT SubVT = Op.getOperand(0).getSimpleValueType(); in LowerAVXCONCAT_VECTORS()
9311 Op.getSimpleValueType().getVectorElementType() == MVT::i1 && in isTypePromotionOfi1ZeroUpBits()
9350 MVT ResVT = Op.getSimpleValueType(); in LowerCONCAT_VECTORSvXi1()
9394 MVT HalfVT = MVT::getVectorVT(ResVT.getVectorElementType(), in LowerCONCAT_VECTORSvXi1()
9420 MVT VT = Op.getSimpleValueType(); in LowerCONCAT_VECTORS()
9421 if (VT.getVectorElementType() == MVT::i1) in LowerCONCAT_VECTORS()
9467 static bool is128BitLaneCrossingShuffleMask(MVT VT, ArrayRef<int> Mask) { in is128BitLaneCrossingShuffleMask()
9487 static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT, in isRepeatedShuffleMask()
9517 is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask, in is128BitLaneRepeatedShuffleMask()
9523 is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask) { in is128BitLaneRepeatedShuffleMask()
9530 is256BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask, in is256BitLaneRepeatedShuffleMask()
9537 static bool isRepeatedTargetShuffleMask(unsigned LaneSizeInBits, MVT VT, in isRepeatedTargetShuffleMask()
9675 static bool isUnpackWdShuffleMask(ArrayRef<int> Mask, MVT VT) { in isUnpackWdShuffleMask()
9676 if (VT != MVT::v8i32 && VT != MVT::v8f32) in isUnpackWdShuffleMask()
9680 createUnpackShuffleMask(MVT::v8i16, Unpcklwd, /* Lo = */ true, in isUnpackWdShuffleMask()
9683 createUnpackShuffleMask(MVT::v8i16, Unpckhwd, /* Lo = */ false, in isUnpackWdShuffleMask()
9715 return DAG.getConstant(getV4X86ShuffleImm(Mask), DL, MVT::i8); in getV4X86ShuffleImm8ForMask()
9829 static SDValue lowerVectorShuffleWithPSHUFB(const SDLoc &DL, MVT VT, in lowerVectorShuffleWithPSHUFB()
9846 SDValue ZeroMask = DAG.getConstant(0x80, DL, MVT::i8); in lowerVectorShuffleWithPSHUFB()
9852 PSHUFBMask[i] = DAG.getUNDEF(MVT::i8); in lowerVectorShuffleWithPSHUFB()
9873 PSHUFBMask[i] = DAG.getConstant(M, DL, MVT::i8); in lowerVectorShuffleWithPSHUFB()
9877 MVT I8VT = MVT::getVectorVT(MVT::i8, NumBytes); in lowerVectorShuffleWithPSHUFB()
9883 static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
9888 static SDValue lowerVectorShuffleToEXPAND(const SDLoc &DL, MVT VT, in lowerVectorShuffleToEXPAND()
9898 MVT IntegerType = in lowerVectorShuffleToEXPAND()
9899 MVT::getIntegerVT(std::max((int)VT.getVectorNumElements(), 8)); in lowerVectorShuffleToEXPAND()
9904 SDValue VMask = getMaskNode(MaskNode, MVT::getVectorVT(MVT::i1, NumElts), in lowerVectorShuffleToEXPAND()
9913 static bool matchVectorShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2, in matchVectorShuffleWithUNPCK()
9953 if ((Subtarget.hasSSE41() || VT == MVT::v2i64 || VT == MVT::v2f64) && in matchVectorShuffleWithUNPCK()
10000 static SDValue lowerVectorShuffleWithUNPCK(const SDLoc &DL, MVT VT, in lowerVectorShuffleWithUNPCK()
10065 MVT VT, SDValue V1, SDValue V2, in lowerVectorShuffleWithVPMOV()
10068 if (VT != MVT::v16i8 && VT != MVT::v8i16) in lowerVectorShuffleWithVPMOV()
10096 MVT SrcVT = Src.getSimpleValueType(); in lowerVectorShuffleWithVPMOV()
10105 if (SrcVT.getVectorElementType() == MVT::i16 && VT == MVT::v16i8 && in lowerVectorShuffleWithVPMOV()
10120 static bool matchVectorShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1, in matchVectorShuffleWithPACK()
10127 MVT PackSVT = MVT::getIntegerVT(BitSize * 2); in matchVectorShuffleWithPACK()
10128 MVT PackVT = MVT::getVectorVT(PackSVT, NumElts / 2); in matchVectorShuffleWithPACK()
10133 if (Subtarget.hasSSE41() || PackSVT == MVT::i16) { in matchVectorShuffleWithPACK()
10172 static SDValue lowerVectorShuffleWithPACK(const SDLoc &DL, MVT VT, in lowerVectorShuffleWithPACK()
10176 MVT PackVT; in lowerVectorShuffleWithPACK()
10190 static SDValue lowerVectorShuffleAsBitMask(const SDLoc &DL, MVT VT, SDValue V1, in lowerVectorShuffleAsBitMask()
10195 MVT EltVT = VT.getVectorElementType(); in lowerVectorShuffleAsBitMask()
10224 static SDValue lowerVectorShuffleAsBitBlend(const SDLoc &DL, MVT VT, SDValue V1, in lowerVectorShuffleAsBitBlend()
10228 MVT EltVT = VT.getVectorElementType(); in lowerVectorShuffleAsBitBlend()
10308 static SDValue lowerVectorShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1, in lowerVectorShuffleAsBlend()
10328 case MVT::v2f64: in lowerVectorShuffleAsBlend()
10329 case MVT::v4f32: in lowerVectorShuffleAsBlend()
10330 case MVT::v4f64: in lowerVectorShuffleAsBlend()
10331 case MVT::v8f32: in lowerVectorShuffleAsBlend()
10333 DAG.getConstant(BlendMask, DL, MVT::i8)); in lowerVectorShuffleAsBlend()
10334 case MVT::v4i64: in lowerVectorShuffleAsBlend()
10335 case MVT::v8i32: in lowerVectorShuffleAsBlend()
10338 case MVT::v2i64: in lowerVectorShuffleAsBlend()
10339 case MVT::v4i32: in lowerVectorShuffleAsBlend()
10346 MVT BlendVT = VT.getSizeInBits() > 128 ? MVT::v8i32 : MVT::v4i32; in lowerVectorShuffleAsBlend()
10351 DAG.getConstant(BlendMask, DL, MVT::i8))); in lowerVectorShuffleAsBlend()
10354 case MVT::v8i16: { in lowerVectorShuffleAsBlend()
10359 V1 = DAG.getBitcast(MVT::v8i16, V1); in lowerVectorShuffleAsBlend()
10360 V2 = DAG.getBitcast(MVT::v8i16, V2); in lowerVectorShuffleAsBlend()
10362 DAG.getNode(X86ISD::BLENDI, DL, MVT::v8i16, V1, V2, in lowerVectorShuffleAsBlend()
10363 DAG.getConstant(BlendMask, DL, MVT::i8))); in lowerVectorShuffleAsBlend()
10365 case MVT::v16i16: { in lowerVectorShuffleAsBlend()
10368 if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) { in lowerVectorShuffleAsBlend()
10375 return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2, in lowerVectorShuffleAsBlend()
10376 DAG.getConstant(BlendMask, DL, MVT::i8)); in lowerVectorShuffleAsBlend()
10384 SDValue Lo = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2, in lowerVectorShuffleAsBlend()
10385 DAG.getConstant(LoMask, DL, MVT::i8)); in lowerVectorShuffleAsBlend()
10386 SDValue Hi = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2, in lowerVectorShuffleAsBlend()
10387 DAG.getConstant(HiMask, DL, MVT::i8)); in lowerVectorShuffleAsBlend()
10389 MVT::v16i16, DL, Lo, Hi, in lowerVectorShuffleAsBlend()
10394 case MVT::v16i8: in lowerVectorShuffleAsBlend()
10395 case MVT::v32i8: { in lowerVectorShuffleAsBlend()
10405 MVT IntegerType = in lowerVectorShuffleAsBlend()
10406 MVT::getIntegerVT(std::max((int)VT.getVectorNumElements(), 8)); in lowerVectorShuffleAsBlend()
10416 MVT BlendVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8); in lowerVectorShuffleAsBlend()
10442 Mask[i] < 0 ? DAG.getUNDEF(MVT::i8) in lowerVectorShuffleAsBlend()
10444 MVT::i8)); in lowerVectorShuffleAsBlend()
10453 case MVT::v16f32: in lowerVectorShuffleAsBlend()
10454 case MVT::v8f64: in lowerVectorShuffleAsBlend()
10455 case MVT::v8i64: in lowerVectorShuffleAsBlend()
10456 case MVT::v16i32: in lowerVectorShuffleAsBlend()
10457 case MVT::v32i16: in lowerVectorShuffleAsBlend()
10458 case MVT::v64i8: { in lowerVectorShuffleAsBlend()
10459 MVT IntegerType = in lowerVectorShuffleAsBlend()
10460 MVT::getIntegerVT(std::max((int)VT.getVectorNumElements(), 8)); in lowerVectorShuffleAsBlend()
10474 static SDValue lowerVectorShuffleAsBlendAndPermute(const SDLoc &DL, MVT VT, in lowerVectorShuffleAsBlendAndPermute()
10513 static SDValue lowerVectorShuffleAsUNPCKAndPermute(const SDLoc &DL, MVT VT, in lowerVectorShuffleAsUNPCKAndPermute()
10577 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerVectorShuffleAsByteRotateAndPermute()
10632 MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8); in lowerVectorShuffleAsByteRotateAndPermute()
10636 DAG.getConstant(Scale * RotAmt, DL, MVT::i8))); in lowerVectorShuffleAsByteRotateAndPermute()
10668 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerVectorShuffleAsDecomposedShuffleBlend()
10799 static int matchVectorShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2, in matchVectorShuffleAsByteRotate()
10821 static SDValue lowerVectorShuffleAsByteRotate(const SDLoc &DL, MVT VT, in lowerVectorShuffleAsByteRotate()
10835 MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8); in lowerVectorShuffleAsByteRotate()
10845 DAG.getConstant(ByteRotation, DL, MVT::i8))); in lowerVectorShuffleAsByteRotate()
10852 assert(ByteVT == MVT::v16i8 && in lowerVectorShuffleAsByteRotate()
10859 SDValue LoShift = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Lo, in lowerVectorShuffleAsByteRotate()
10860 DAG.getConstant(LoByteShift, DL, MVT::i8)); in lowerVectorShuffleAsByteRotate()
10861 SDValue HiShift = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Hi, in lowerVectorShuffleAsByteRotate()
10862 DAG.getConstant(HiByteShift, DL, MVT::i8)); in lowerVectorShuffleAsByteRotate()
10864 DAG.getNode(ISD::OR, DL, MVT::v16i8, LoShift, HiShift)); in lowerVectorShuffleAsByteRotate()
10877 static SDValue lowerVectorShuffleAsRotate(const SDLoc &DL, MVT VT, in lowerVectorShuffleAsRotate()
10882 assert((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) && in lowerVectorShuffleAsRotate()
10895 DAG.getConstant(Rotation, DL, MVT::i8)); in lowerVectorShuffleAsRotate()
10921 static int matchVectorShuffleAsShift(MVT &ShiftVT, unsigned &Opcode, in matchVectorShuffleAsShift()
10958 MVT ShiftSVT = MVT::getIntegerVT(ScalarSizeInBits * Scale); in matchVectorShuffleAsShift()
10959 ShiftVT = ByteShift ? MVT::getVectorVT(MVT::i8, SizeInBits / 8) in matchVectorShuffleAsShift()
10960 : MVT::getVectorVT(ShiftSVT, Size / Scale); in matchVectorShuffleAsShift()
10984 static SDValue lowerVectorShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1, in lowerVectorShuffleAsShift()
10992 MVT ShiftVT; in lowerVectorShuffleAsShift()
11015 DAG.getConstant(ShiftAmt, DL, MVT::i8)); in lowerVectorShuffleAsShift()
11021 static bool matchVectorShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2, in matchVectorShuffleAsEXTRQ()
11077 static bool matchVectorShuffleAsINSERTQ(MVT VT, SDValue &V1, SDValue &V2, in matchVectorShuffleAsINSERTQ()
11143 static SDValue lowerVectorShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1, in lowerVectorShuffleWithSSE4A()
11150 DAG.getConstant(BitLen, DL, MVT::i8), in lowerVectorShuffleWithSSE4A()
11151 DAG.getConstant(BitIdx, DL, MVT::i8)); in lowerVectorShuffleWithSSE4A()
11156 DAG.getConstant(BitLen, DL, MVT::i8), in lowerVectorShuffleWithSSE4A()
11157 DAG.getConstant(BitIdx, DL, MVT::i8)); in lowerVectorShuffleWithSSE4A()
11172 const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11211 MVT ExtVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Scale), in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11226 VT, DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11227 DAG.getBitcast(MVT::v4i32, InputV), in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11233 InputV = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11234 DAG.getBitcast(MVT::v4i32, InputV), in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11239 VT, DAG.getNode(OddEvenOp, DL, MVT::v8i16, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11240 DAG.getBitcast(MVT::v8i16, InputV), in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11252 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11253 DAG.getConstant(EltBits, DL, MVT::i8), in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11254 DAG.getConstant(LoIdx, DL, MVT::i8))); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11262 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11263 DAG.getConstant(EltBits, DL, MVT::i8), in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11264 DAG.getConstant(HiIdx, DL, MVT::i8))); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11266 DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, Lo, Hi)); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11278 (i % Scale == 0 && SafeOffset(Idx)) ? Idx : 0x80, DL, MVT::i8); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11280 InputV = DAG.getBitcast(MVT::v16i8, InputV); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11282 VT, DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11283 DAG.getBuildVector(MVT::v16i8, DL, PSHUFBMask))); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11305 MVT InputVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits), NumElements); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
11330 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerVectorShuffleAsZeroOrAnyExtend()
11436 V = DAG.getBitcast(MVT::v2i64, V); in lowerVectorShuffleAsZeroOrAnyExtend()
11437 V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v2i64, V); in lowerVectorShuffleAsZeroOrAnyExtend()
11450 MVT VT = V.getSimpleValueType(); in getScalarValueForVectorElement()
11451 MVT EltVT = VT.getVectorElementType(); in getScalarValueForVectorElement()
11456 MVT NewVT = V.getSimpleValueType(); in getScalarValueForVectorElement()
11486 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerVectorShuffleAsElementInsertion()
11489 MVT ExtVT = VT; in lowerVectorShuffleAsElementInsertion()
11490 MVT EltVT = VT.getVectorElementType(); in lowerVectorShuffleAsElementInsertion()
11512 if (EltVT == MVT::i8 || EltVT == MVT::i16) { in lowerVectorShuffleAsElementInsertion()
11519 ExtVT = MVT::getVectorVT(MVT::i32, ExtVT.getSizeInBits() / 32); in lowerVectorShuffleAsElementInsertion()
11520 V2S = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, V2S); in lowerVectorShuffleAsElementInsertion()
11523 } else if (Mask[V2Index] != (int)Mask.size() || EltVT == MVT::i8 || in lowerVectorShuffleAsElementInsertion()
11524 EltVT == MVT::i16) { in lowerVectorShuffleAsElementInsertion()
11545 assert((EltVT == MVT::f32 || EltVT == MVT::f64) && in lowerVectorShuffleAsElementInsertion()
11547 return DAG.getNode(EltVT == MVT::f32 ? X86ISD::MOVSS : X86ISD::MOVSD, DL, in lowerVectorShuffleAsElementInsertion()
11569 V2 = DAG.getBitcast(MVT::v16i8, V2); in lowerVectorShuffleAsElementInsertion()
11571 X86ISD::VSHLDQ, DL, MVT::v16i8, V2, in lowerVectorShuffleAsElementInsertion()
11572 DAG.getConstant(V2Index * EltVT.getSizeInBits() / 8, DL, MVT::i8)); in lowerVectorShuffleAsElementInsertion()
11583 static SDValue lowerVectorShuffleAsTruncBroadcast(const SDLoc &DL, MVT VT, in lowerVectorShuffleAsTruncBroadcast()
11626 DAG.getConstant(OffsetIdx * EltSize, DL, MVT::i8)); in lowerVectorShuffleAsTruncBroadcast()
11637 static SDValue lowerVectorShuffleAsBroadcast(const SDLoc &DL, MVT VT, in lowerVectorShuffleAsBroadcast()
11642 if (!((Subtarget.hasSSE3() && VT == MVT::v2f64) || in lowerVectorShuffleAsBroadcast()
11650 unsigned Opcode = (VT == MVT::v2f64 && !Subtarget.hasAVX2()) in lowerVectorShuffleAsBroadcast()
11732 MVT SrcVT = MVT::getVectorVT(VT.getScalarType(), NumSrcElts); in lowerVectorShuffleAsBroadcast()
11746 MVT BroadcastVT = VT; in lowerVectorShuffleAsBroadcast()
11761 if (!Subtarget.is64Bit() && VT.getScalarType() == MVT::i64) { in lowerVectorShuffleAsBroadcast()
11762 BroadcastVT = MVT::getVectorVT(MVT::f64, VT.getVectorNumElements()); in lowerVectorShuffleAsBroadcast()
11790 if (VT == MVT::v4f64 || VT == MVT::v4i64) in lowerVectorShuffleAsBroadcast()
11809 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, in lowerVectorShuffleAsBroadcast()
11810 DAG.getBitcast(MVT::f64, V)); in lowerVectorShuffleAsBroadcast()
11813 MVT SrcVT = V.getSimpleValueType(); in lowerVectorShuffleAsBroadcast()
11819 SrcVT = MVT::getVectorVT(BroadcastVT.getScalarType(), NumSrcElts); in lowerVectorShuffleAsBroadcast()
11827 if (!Subtarget.is64Bit() && SrcVT == MVT::i64) { in lowerVectorShuffleAsBroadcast()
11828 V = DAG.getBitcast(MVT::f64, V); in lowerVectorShuffleAsBroadcast()
11830 BroadcastVT = MVT::getVectorVT(MVT::f64, NumBroadcastElts); in lowerVectorShuffleAsBroadcast()
11837 MVT ExtVT = MVT::getVectorVT(SrcVT.getScalarType(), in lowerVectorShuffleAsBroadcast()
11917 VA = DAG.getUNDEF(MVT::v4f32); in matchVectorShuffleAsInsertPS()
11945 assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); in lowerVectorShuffleAsInsertPS()
11946 assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); in lowerVectorShuffleAsInsertPS()
11954 return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2, in lowerVectorShuffleAsInsertPS()
11955 DAG.getConstant(InsertPSMask, DL, MVT::i8)); in lowerVectorShuffleAsInsertPS()
11968 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerVectorShuffleAsPermuteAndUnpack()
12021 MVT UnpackVT = MVT::getVectorVT(MVT::getIntegerVT(ScalarSize), Size / Scale); in lowerVectorShuffleAsPermuteAndUnpack()
12087 assert(V1.getSimpleValueType() == MVT::v2f64 && "Bad operand type!"); in lowerV2F64VectorShuffle()
12088 assert(V2.getSimpleValueType() == MVT::v2f64 && "Bad operand type!"); in lowerV2F64VectorShuffle()
12094 DL, MVT::v2f64, V1, V2, Mask, Subtarget, DAG)) in lowerV2F64VectorShuffle()
12104 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1, in lowerV2F64VectorShuffle()
12105 DAG.getConstant(SHUFPDMask, DL, MVT::i8)); in lowerV2F64VectorShuffle()
12109 X86ISD::SHUFP, DL, MVT::v2f64, in lowerV2F64VectorShuffle()
12110 Mask[0] == SM_SentinelUndef ? DAG.getUNDEF(MVT::v2f64) : V1, in lowerV2F64VectorShuffle()
12111 Mask[1] == SM_SentinelUndef ? DAG.getUNDEF(MVT::v2f64) : V1, in lowerV2F64VectorShuffle()
12112 DAG.getConstant(SHUFPDMask, DL, MVT::i8)); in lowerV2F64VectorShuffle()
12122 DL, MVT::v2f64, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV2F64VectorShuffle()
12129 DL, MVT::v2f64, V2, V1, InverseMask, Zeroable, Subtarget, DAG)) in lowerV2F64VectorShuffle()
12140 X86ISD::MOVSD, DL, MVT::v2f64, V2, in lowerV2F64VectorShuffle()
12141 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V1S)); in lowerV2F64VectorShuffle()
12144 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask, in lowerV2F64VectorShuffle()
12150 lowerVectorShuffleWithUNPCK(DL, MVT::v2f64, Mask, V1, V2, DAG)) in lowerV2F64VectorShuffle()
12154 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v2f64, V1, V2, in lowerV2F64VectorShuffle()
12155 DAG.getConstant(SHUFPDMask, DL, MVT::i8)); in lowerV2F64VectorShuffle()
12169 assert(V1.getSimpleValueType() == MVT::v2i64 && "Bad operand type!"); in lowerV2I64VectorShuffle()
12170 assert(V2.getSimpleValueType() == MVT::v2i64 && "Bad operand type!"); in lowerV2I64VectorShuffle()
12176 DL, MVT::v2i64, V1, V2, Mask, Subtarget, DAG)) in lowerV2I64VectorShuffle()
12182 V1 = DAG.getBitcast(MVT::v4i32, V1); in lowerV2I64VectorShuffle()
12187 MVT::v2i64, in lowerV2I64VectorShuffle()
12188 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1, in lowerV2I64VectorShuffle()
12197 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v2i64, V1, V2, Mask, in lowerV2I64VectorShuffle()
12204 DL, MVT::v2i64, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV2I64VectorShuffle()
12210 DL, MVT::v2i64, V2, V1, InverseMask, Zeroable, Subtarget, DAG)) in lowerV2I64VectorShuffle()
12217 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v2i64, V1, V2, Mask, in lowerV2I64VectorShuffle()
12223 lowerVectorShuffleWithUNPCK(DL, MVT::v2i64, Mask, V1, V2, DAG)) in lowerV2I64VectorShuffle()
12230 if (SDValue Rotate = lowerVectorShuffleAsRotate(DL, MVT::v2i64, V1, V2, in lowerV2I64VectorShuffle()
12235 DL, MVT::v2i64, V1, V2, Mask, Subtarget, DAG)) in lowerV2I64VectorShuffle()
12242 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v2i64, V1, V2, in lowerV2I64VectorShuffle()
12249 V1 = DAG.getBitcast(MVT::v2f64, V1); in lowerV2I64VectorShuffle()
12250 V2 = DAG.getBitcast(MVT::v2f64, V2); in lowerV2I64VectorShuffle()
12251 return DAG.getBitcast(MVT::v2i64, in lowerV2I64VectorShuffle()
12252 DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask)); in lowerV2I64VectorShuffle()
12282 static SDValue lowerVectorShuffleWithSHUFPS(const SDLoc &DL, MVT VT, in lowerVectorShuffleWithSHUFPS()
12374 assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); in lowerV4F32VectorShuffle()
12375 assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); in lowerV4F32VectorShuffle()
12383 DL, MVT::v4f32, V1, V2, Mask, Subtarget, DAG)) in lowerV4F32VectorShuffle()
12389 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v4f32, V1); in lowerV4F32VectorShuffle()
12391 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v4f32, V1); in lowerV4F32VectorShuffle()
12397 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f32, V1, in lowerV4F32VectorShuffle()
12405 return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V1); in lowerV4F32VectorShuffle()
12407 return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V1, V1); in lowerV4F32VectorShuffle()
12412 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1, in lowerV4F32VectorShuffle()
12423 DL, MVT::v4f32, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV4F32VectorShuffle()
12427 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4f32, V1, V2, Mask, in lowerV4F32VectorShuffle()
12438 DL, MVT::v4f32, V1, V2, Mask, DAG)) in lowerV4F32VectorShuffle()
12446 return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V2); in lowerV4F32VectorShuffle()
12448 return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V2, V1); in lowerV4F32VectorShuffle()
12453 lowerVectorShuffleWithUNPCK(DL, MVT::v4f32, Mask, V1, V2, DAG)) in lowerV4F32VectorShuffle()
12457 return lowerVectorShuffleWithSHUFPS(DL, MVT::v4f32, Mask, V1, V2, DAG); in lowerV4F32VectorShuffle()
12469 assert(V1.getSimpleValueType() == MVT::v4i32 && "Bad operand type!"); in lowerV4I32VectorShuffle()
12470 assert(V2.getSimpleValueType() == MVT::v4i32 && "Bad operand type!"); in lowerV4I32VectorShuffle()
12477 DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV4I32VectorShuffle()
12485 DL, MVT::v4i32, V1, V2, Mask, Subtarget, DAG)) in lowerV4I32VectorShuffle()
12500 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1, in lowerV4I32VectorShuffle()
12505 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask, in lowerV4I32VectorShuffle()
12512 DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV4I32VectorShuffle()
12519 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4i32, V1, V2, Mask, in lowerV4I32VectorShuffle()
12523 if (SDValue Masked = lowerVectorShuffleAsBitMask(DL, MVT::v4i32, V1, V2, Mask, in lowerV4I32VectorShuffle()
12529 lowerVectorShuffleWithUNPCK(DL, MVT::v4i32, Mask, V1, V2, DAG)) in lowerV4I32VectorShuffle()
12536 if (SDValue Rotate = lowerVectorShuffleAsRotate(DL, MVT::v4i32, V1, V2, in lowerV4I32VectorShuffle()
12541 DL, MVT::v4i32, V1, V2, Mask, Subtarget, DAG)) in lowerV4I32VectorShuffle()
12552 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4i32, V1, V2, in lowerV4I32VectorShuffle()
12557 DL, MVT::v4i32, V1, V2, Mask, Subtarget, DAG)) in lowerV4I32VectorShuffle()
12566 SDValue CastV1 = DAG.getBitcast(MVT::v4f32, V1); in lowerV4I32VectorShuffle()
12567 SDValue CastV2 = DAG.getBitcast(MVT::v4f32, V2); in lowerV4I32VectorShuffle()
12568 SDValue ShufPS = DAG.getVectorShuffle(MVT::v4f32, DL, CastV1, CastV2, Mask); in lowerV4I32VectorShuffle()
12569 return DAG.getBitcast(MVT::v4i32, ShufPS); in lowerV4I32VectorShuffle()
12589 const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef<int> Mask, in lowerV8I16GeneralSingleInputVectorShuffle()
12591 assert(VT.getVectorElementType() == MVT::i16 && "Bad input type!"); in lowerV8I16GeneralSingleInputVectorShuffle()
12592 MVT PSHUFDVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2); in lowerV8I16GeneralSingleInputVectorShuffle()
12790 MVT::getVectorVT(MVT::i16, V.getValueSizeInBits() / 16), V, in lowerV8I16GeneralSingleInputVectorShuffle()
13088 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerVectorShuffleAsBlendOfPSHUFBs()
13097 SmallVector<SDValue, 64> V1Mask(NumBytes, DAG.getUNDEF(MVT::i8)); in lowerVectorShuffleAsBlendOfPSHUFBs()
13098 SmallVector<SDValue, 64> V2Mask(NumBytes, DAG.getUNDEF(MVT::i8)); in lowerVectorShuffleAsBlendOfPSHUFBs()
13113 V1Mask[i] = DAG.getConstant(V1Idx, DL, MVT::i8); in lowerVectorShuffleAsBlendOfPSHUFBs()
13114 V2Mask[i] = DAG.getConstant(V2Idx, DL, MVT::i8); in lowerVectorShuffleAsBlendOfPSHUFBs()
13119 MVT ShufVT = MVT::getVectorVT(MVT::i8, NumBytes); in lowerVectorShuffleAsBlendOfPSHUFBs()
13155 assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad operand type!"); in lowerV8I16VectorShuffle()
13156 assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad operand type!"); in lowerV8I16VectorShuffle()
13162 DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV8I16VectorShuffle()
13170 DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG)) in lowerV8I16VectorShuffle()
13174 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v8i16, V1, V1, Mask, in lowerV8I16VectorShuffle()
13180 lowerVectorShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG)) in lowerV8I16VectorShuffle()
13184 if (SDValue V = lowerVectorShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, in lowerV8I16VectorShuffle()
13189 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(DL, MVT::v8i16, V1, V1, in lowerV8I16VectorShuffle()
13195 return lowerV8I16GeneralSingleInputVectorShuffle(DL, MVT::v8i16, V1, in lowerV8I16VectorShuffle()
13205 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v8i16, V1, V2, Mask, in lowerV8I16VectorShuffle()
13211 if (SDValue V = lowerVectorShuffleWithSSE4A(DL, MVT::v8i16, V1, V2, Mask, in lowerV8I16VectorShuffle()
13218 DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV8I16VectorShuffle()
13225 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i16, V1, V2, Mask, in lowerV8I16VectorShuffle()
13229 if (SDValue Masked = lowerVectorShuffleAsBitMask(DL, MVT::v8i16, V1, V2, Mask, in lowerV8I16VectorShuffle()
13235 lowerVectorShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG)) in lowerV8I16VectorShuffle()
13239 if (SDValue V = lowerVectorShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG, in lowerV8I16VectorShuffle()
13245 DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG)) in lowerV8I16VectorShuffle()
13249 lowerVectorShuffleAsBitBlend(DL, MVT::v8i16, V1, V2, Mask, DAG)) in lowerV8I16VectorShuffle()
13254 DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG)) in lowerV8I16VectorShuffle()
13261 return lowerVectorShuffleAsBlendOfPSHUFBs(DL, MVT::v8i16, V1, V2, Mask, in lowerV8I16VectorShuffle()
13267 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8i16, V1, V2, in lowerV8I16VectorShuffle()
13337 static SDValue lowerVectorShuffleWithPERMV(const SDLoc &DL, MVT VT, in lowerVectorShuffleWithPERMV()
13340 MVT MaskEltVT = MVT::getIntegerVT(VT.getScalarSizeInBits()); in lowerVectorShuffleWithPERMV()
13341 MVT MaskVecVT = MVT::getVectorVT(MaskEltVT, VT.getVectorNumElements()); in lowerVectorShuffleWithPERMV()
13362 assert(V1.getSimpleValueType() == MVT::v16i8 && "Bad operand type!"); in lowerV16I8VectorShuffle()
13363 assert(V2.getSimpleValueType() == MVT::v16i8 && "Bad operand type!"); in lowerV16I8VectorShuffle()
13367 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v16i8, V1, V2, Mask, in lowerV16I8VectorShuffle()
13373 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG)) in lowerV16I8VectorShuffle()
13377 if (SDValue V = lowerVectorShuffleWithPACK(DL, MVT::v16i8, Mask, V1, V2, DAG, in lowerV16I8VectorShuffle()
13383 DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV16I8VectorShuffle()
13388 if (SDValue V = lowerVectorShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask, in lowerV16I8VectorShuffle()
13398 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG)) in lowerV16I8VectorShuffle()
13402 lowerVectorShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG)) in lowerV16I8VectorShuffle()
13467 MVT::v16i8, in lowerV16I8VectorShuffle()
13468 DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1), in lowerV16I8VectorShuffle()
13469 DAG.getUNDEF(MVT::v8i16), PreDupI16Shuffle)); in lowerV16I8VectorShuffle()
13473 MVT::v16i8, V1, V1); in lowerV16I8VectorShuffle()
13487 MVT::v16i8, in lowerV16I8VectorShuffle()
13488 DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1), in lowerV16I8VectorShuffle()
13489 DAG.getUNDEF(MVT::v8i16), PostDupI16Shuffle)); in lowerV16I8VectorShuffle()
13495 if (SDValue Masked = lowerVectorShuffleAsBitMask(DL, MVT::v16i8, V1, V2, Mask, in lowerV16I8VectorShuffle()
13501 lowerVectorShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG)) in lowerV16I8VectorShuffle()
13522 DL, MVT::v16i8, V1, V2, Mask, Zeroable, DAG, V1InUse, V2InUse); in lowerV16I8VectorShuffle()
13530 DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV16I8VectorShuffle()
13542 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG)) in lowerV16I8VectorShuffle()
13547 return lowerVectorShuffleWithPERMV(DL, MVT::v16i8, Mask, V1, V2, DAG); in lowerV16I8VectorShuffle()
13552 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG)) in lowerV16I8VectorShuffle()
13562 DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV16I8VectorShuffle()
13566 lowerVectorShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG)) in lowerV16I8VectorShuffle()
13587 MVT MaskVTs[] = { MVT::v8i16, MVT::v4i32, MVT::v2i64 }; in lowerV16I8VectorShuffle()
13589 MVT::v16i8, DAG.getConstant(0xFF, DL, MaskVTs[NumEvenDrops - 1])); in lowerV16I8VectorShuffle()
13590 V1 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V1, ByteClearMask); in lowerV16I8VectorShuffle()
13592 V2 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V2, ByteClearMask); in lowerV16I8VectorShuffle()
13595 V1 = DAG.getBitcast(MVT::v8i16, V1); in lowerV16I8VectorShuffle()
13596 V2 = IsSingleInput ? V1 : DAG.getBitcast(MVT::v8i16, V2); in lowerV16I8VectorShuffle()
13597 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1, V2); in lowerV16I8VectorShuffle()
13599 Result = DAG.getBitcast(MVT::v8i16, Result); in lowerV16I8VectorShuffle()
13600 Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result); in lowerV16I8VectorShuffle()
13608 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v16i8, V1, V2, in lowerV16I8VectorShuffle()
13629 VLoHalf = DAG.getBitcast(MVT::v8i16, V); in lowerV16I8VectorShuffle()
13630 VLoHalf = DAG.getNode(ISD::AND, DL, MVT::v8i16, VLoHalf, in lowerV16I8VectorShuffle()
13631 DAG.getConstant(0x00FF, DL, MVT::v8i16)); in lowerV16I8VectorShuffle()
13634 VHiHalf = DAG.getUNDEF(MVT::v8i16); in lowerV16I8VectorShuffle()
13646 SDValue Zero = getZeroVector(MVT::v16i8, Subtarget, DAG, DL); in lowerV16I8VectorShuffle()
13649 MVT::v8i16, DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero)); in lowerV16I8VectorShuffle()
13651 MVT::v8i16, DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero)); in lowerV16I8VectorShuffle()
13654 SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask); in lowerV16I8VectorShuffle()
13655 SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask); in lowerV16I8VectorShuffle()
13657 return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV); in lowerV16I8VectorShuffle()
13665 MVT VT, SDValue V1, SDValue V2, in lower128BitVectorShuffle()
13670 case MVT::v2i64: in lower128BitVectorShuffle()
13672 case MVT::v2f64: in lower128BitVectorShuffle()
13674 case MVT::v4i32: in lower128BitVectorShuffle()
13676 case MVT::v4f32: in lower128BitVectorShuffle()
13678 case MVT::v8i16: in lower128BitVectorShuffle()
13680 case MVT::v16i8: in lower128BitVectorShuffle()
13693 static SDValue splitAndLowerVectorShuffle(const SDLoc &DL, MVT VT, SDValue V1, in splitAndLowerVectorShuffle()
13706 MVT ScalarVT = VT.getVectorElementType(); in splitAndLowerVectorShuffle()
13707 MVT SplitVT = MVT::getVectorVT(ScalarVT, NumElements / 2); in splitAndLowerVectorShuffle()
13714 MVT OrigVT = V.getSimpleValueType(); in splitAndLowerVectorShuffle()
13717 MVT OrigScalarVT = OrigVT.getVectorElementType(); in splitAndLowerVectorShuffle()
13718 MVT OrigSplitVT = MVT::getVectorVT(OrigScalarVT, OrigNumElements / 2); in splitAndLowerVectorShuffle()
13819 static SDValue lowerVectorShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, in lowerVectorShuffleAsSplitOrBlend()
13880 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerVectorShuffleAsLanePermuteAndPermute()
13943 static SDValue lowerVectorShuffleAsLanePermuteAndBlend(const SDLoc &DL, MVT VT, in lowerVectorShuffleAsLanePermuteAndBlend()
13984 MVT PVT = VT.isFloatingPoint() ? MVT::v4f64 : MVT::v4i64; in lowerVectorShuffleAsLanePermuteAndBlend()
13993 static SDValue lowerV2X128VectorShuffle(const SDLoc &DL, MVT VT, SDValue V1, in lowerV2X128VectorShuffle()
14011 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2); in lowerV2X128VectorShuffle()
14039 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2); in lowerV2X128VectorShuffle()
14054 DAG.getConstant(PermMask, DL, MVT::i8)); in lowerV2X128VectorShuffle()
14086 DAG.getConstant(PermMask, DL, MVT::i8)); in lowerV2X128VectorShuffle()
14099 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerVectorShuffleByMerging128BitLanes()
14265 static SDValue lowerVectorShuffleWithUndefHalf(const SDLoc &DL, MVT VT, in lowerVectorShuffleWithUndefHalf()
14275 MVT HalfVT = MVT::getVectorVT(VT.getVectorElementType(), HalfNumElts); in lowerVectorShuffleWithUndefHalf()
14357 if (VT == MVT::v4f64 || VT == MVT::v4i64) in lowerVectorShuffleWithUndefHalf()
14360 if (VT == MVT::v8f32 || VT == MVT::v8i32) { in lowerVectorShuffleWithUndefHalf()
14407 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsRepeatedMaskAndLanePermute()
14572 static bool matchVectorShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2, in matchVectorShuffleWithSHUFPD()
14609 static SDValue lowerVectorShuffleWithSHUFPD(const SDLoc &DL, MVT VT, in lowerVectorShuffleWithSHUFPD()
14612 assert((VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v8f64)&& in lowerVectorShuffleWithSHUFPD()
14620 DAG.getConstant(Immediate, DL, MVT::i8)); in lowerVectorShuffleWithSHUFPD()
14632 assert(V1.getSimpleValueType() == MVT::v4f64 && "Bad operand type!"); in lowerV4F64VectorShuffle()
14633 assert(V2.getSimpleValueType() == MVT::v4f64 && "Bad operand type!"); in lowerV4F64VectorShuffle()
14636 if (SDValue V = lowerV2X128VectorShuffle(DL, MVT::v4f64, V1, V2, Mask, in lowerV4F64VectorShuffle()
14643 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG)) in lowerV4F64VectorShuffle()
14648 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v4f64, V1); in lowerV4F64VectorShuffle()
14650 if (!is128BitLaneCrossingShuffleMask(MVT::v4f64, Mask)) { in lowerV4F64VectorShuffle()
14655 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1, in lowerV4F64VectorShuffle()
14656 DAG.getConstant(VPERMILPMask, DL, MVT::i8)); in lowerV4F64VectorShuffle()
14661 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4f64, V1, in lowerV4F64VectorShuffle()
14667 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG)) in lowerV4F64VectorShuffle()
14672 DL, MVT::v4f64, V1, V2, Mask, DAG, Subtarget)) in lowerV4F64VectorShuffle()
14676 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v4f64, V1, V2, Mask, in lowerV4F64VectorShuffle()
14682 lowerVectorShuffleWithUNPCK(DL, MVT::v4f64, Mask, V1, V2, DAG)) in lowerV4F64VectorShuffle()
14685 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask, in lowerV4F64VectorShuffle()
14691 lowerVectorShuffleWithSHUFPD(DL, MVT::v4f64, Mask, V1, V2, DAG)) in lowerV4F64VectorShuffle()
14697 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG)) in lowerV4F64VectorShuffle()
14707 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG)) in lowerV4F64VectorShuffle()
14712 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v4f64, Zeroable, Mask, in lowerV4F64VectorShuffle()
14719 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4f64, V1, V2, in lowerV4F64VectorShuffle()
14723 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask, in lowerV4F64VectorShuffle()
14736 assert(V1.getSimpleValueType() == MVT::v4i64 && "Bad operand type!"); in lowerV4I64VectorShuffle()
14737 assert(V2.getSimpleValueType() == MVT::v4i64 && "Bad operand type!"); in lowerV4I64VectorShuffle()
14741 if (SDValue V = lowerV2X128VectorShuffle(DL, MVT::v4i64, V1, V2, Mask, in lowerV4I64VectorShuffle()
14745 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4i64, V1, V2, Mask, in lowerV4I64VectorShuffle()
14750 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v4i64, V1, V2, in lowerV4I64VectorShuffle()
14758 if (is128BitLaneRepeatedShuffleMask(MVT::v4i64, Mask, RepeatedMask)) { in lowerV4I64VectorShuffle()
14762 MVT::v4i64, in lowerV4I64VectorShuffle()
14763 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, in lowerV4I64VectorShuffle()
14764 DAG.getBitcast(MVT::v8i32, V1), in lowerV4I64VectorShuffle()
14770 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64, V1, in lowerV4I64VectorShuffle()
14775 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask, in lowerV4I64VectorShuffle()
14781 if (SDValue Rotate = lowerVectorShuffleAsRotate(DL, MVT::v4i64, V1, V2, in lowerV4I64VectorShuffle()
14785 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v4i64, Zeroable, Mask, in lowerV4I64VectorShuffle()
14791 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(DL, MVT::v4i64, V1, V2, in lowerV4I64VectorShuffle()
14797 lowerVectorShuffleWithUNPCK(DL, MVT::v4i64, Mask, V1, V2, DAG)) in lowerV4I64VectorShuffle()
14803 DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG)) in lowerV4I64VectorShuffle()
14813 DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG)) in lowerV4I64VectorShuffle()
14817 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4i64, V1, V2, in lowerV4I64VectorShuffle()
14830 assert(V1.getSimpleValueType() == MVT::v8f32 && "Bad operand type!"); in lowerV8F32VectorShuffle()
14831 assert(V2.getSimpleValueType() == MVT::v8f32 && "Bad operand type!"); in lowerV8F32VectorShuffle()
14834 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask, in lowerV8F32VectorShuffle()
14839 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v8f32, V1, V2, in lowerV8F32VectorShuffle()
14846 if (is128BitLaneRepeatedShuffleMask(MVT::v8f32, Mask, RepeatedMask)) { in lowerV8F32VectorShuffle()
14852 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v8f32, V1); in lowerV8F32VectorShuffle()
14854 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v8f32, V1); in lowerV8F32VectorShuffle()
14857 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, V1, in lowerV8F32VectorShuffle()
14862 lowerVectorShuffleWithUNPCK(DL, MVT::v8f32, Mask, V1, V2, DAG)) in lowerV8F32VectorShuffle()
14867 return lowerVectorShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask, V1, V2, DAG); in lowerV8F32VectorShuffle()
14873 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG)) in lowerV8F32VectorShuffle()
14879 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true); in lowerV8F32VectorShuffle()
14880 if (!is128BitLaneCrossingShuffleMask(MVT::v8f32, Mask)) in lowerV8F32VectorShuffle()
14881 return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, V1, VPermMask); in lowerV8F32VectorShuffle()
14884 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8f32, VPermMask, V1); in lowerV8F32VectorShuffle()
14887 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v8f32, V1, V2, Mask, in lowerV8F32VectorShuffle()
14894 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG)) in lowerV8F32VectorShuffle()
14898 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v8f32, Zeroable, Mask, in lowerV8F32VectorShuffle()
14905 if (!Subtarget.hasAVX512() && isUnpackWdShuffleMask(Mask, MVT::v8f32)) in lowerV8F32VectorShuffle()
14906 if (SDValue V = lowerVectorShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, in lowerV8F32VectorShuffle()
14913 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8f32, V1, V2, in lowerV8F32VectorShuffle()
14917 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, in lowerV8F32VectorShuffle()
14930 assert(V1.getSimpleValueType() == MVT::v8i32 && "Bad operand type!"); in lowerV8I32VectorShuffle()
14931 assert(V2.getSimpleValueType() == MVT::v8i32 && "Bad operand type!"); in lowerV8I32VectorShuffle()
14939 DL, MVT::v8i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV8I32VectorShuffle()
14945 if (isUnpackWdShuffleMask(Mask, MVT::v8i32) && !V2.isUndef() && in lowerV8I32VectorShuffle()
14947 if (SDValue V = lowerVectorShuffleAsSplitOrBlend(DL, MVT::v8i32, V1, V2, in lowerV8I32VectorShuffle()
14951 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask, in lowerV8I32VectorShuffle()
14956 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v8i32, V1, V2, in lowerV8I32VectorShuffle()
14965 is128BitLaneRepeatedShuffleMask(MVT::v8i32, Mask, RepeatedMask); in lowerV8I32VectorShuffle()
14969 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, V1, in lowerV8I32VectorShuffle()
14974 lowerVectorShuffleWithUNPCK(DL, MVT::v8i32, Mask, V1, V2, DAG)) in lowerV8I32VectorShuffle()
14979 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask, in lowerV8I32VectorShuffle()
14985 if (SDValue Rotate = lowerVectorShuffleAsRotate(DL, MVT::v8i32, V1, V2, in lowerV8I32VectorShuffle()
14989 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v8i32, Zeroable, Mask, in lowerV8I32VectorShuffle()
14996 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG)) in lowerV8I32VectorShuffle()
15002 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG)) in lowerV8I32VectorShuffle()
15008 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true); in lowerV8I32VectorShuffle()
15009 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8i32, VPermMask, V1); in lowerV8I32VectorShuffle()
15016 SDValue CastV1 = DAG.getBitcast(MVT::v8f32, V1); in lowerV8I32VectorShuffle()
15017 SDValue CastV2 = DAG.getBitcast(MVT::v8f32, V2); in lowerV8I32VectorShuffle()
15018 SDValue ShufPS = lowerVectorShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask, in lowerV8I32VectorShuffle()
15020 return DAG.getBitcast(MVT::v8i32, ShufPS); in lowerV8I32VectorShuffle()
15026 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG)) in lowerV8I32VectorShuffle()
15030 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8i32, V1, V2, in lowerV8I32VectorShuffle()
15043 assert(V1.getSimpleValueType() == MVT::v16i16 && "Bad operand type!"); in lowerV16I16VectorShuffle()
15044 assert(V2.getSimpleValueType() == MVT::v16i16 && "Bad operand type!"); in lowerV16I16VectorShuffle()
15052 DL, MVT::v16i16, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV16I16VectorShuffle()
15056 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v16i16, V1, V2, in lowerV16I16VectorShuffle()
15060 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16i16, V1, V2, Mask, in lowerV16I16VectorShuffle()
15066 lowerVectorShuffleWithUNPCK(DL, MVT::v16i16, Mask, V1, V2, DAG)) in lowerV16I16VectorShuffle()
15070 if (SDValue V = lowerVectorShuffleWithPACK(DL, MVT::v16i16, Mask, V1, V2, DAG, in lowerV16I16VectorShuffle()
15075 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v16i16, V1, V2, Mask, in lowerV16I16VectorShuffle()
15081 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG)) in lowerV16I16VectorShuffle()
15087 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG)) in lowerV16I16VectorShuffle()
15093 if (is128BitLaneCrossingShuffleMask(MVT::v16i16, Mask)) { in lowerV16I16VectorShuffle()
15095 DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget)) in lowerV16I16VectorShuffle()
15098 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v16i16, V1, V2, in lowerV16I16VectorShuffle()
15103 if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) { in lowerV16I16VectorShuffle()
15108 DL, MVT::v16i16, V1, RepeatedMask, Subtarget, DAG); in lowerV16I16VectorShuffle()
15113 DL, MVT::v16i16, Mask, V1, V2, Zeroable, Subtarget, DAG)) in lowerV16I16VectorShuffle()
15118 return lowerVectorShuffleWithPERMV(DL, MVT::v16i16, Mask, V1, V2, DAG); in lowerV16I16VectorShuffle()
15123 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG)) in lowerV16I16VectorShuffle()
15128 DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget)) in lowerV16I16VectorShuffle()
15132 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask, in lowerV16I16VectorShuffle()
15145 assert(V1.getSimpleValueType() == MVT::v32i8 && "Bad operand type!"); in lowerV32I8VectorShuffle()
15146 assert(V2.getSimpleValueType() == MVT::v32i8 && "Bad operand type!"); in lowerV32I8VectorShuffle()
15154 DL, MVT::v32i8, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV32I8VectorShuffle()
15158 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v32i8, V1, V2, in lowerV32I8VectorShuffle()
15162 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask, in lowerV32I8VectorShuffle()
15168 lowerVectorShuffleWithUNPCK(DL, MVT::v32i8, Mask, V1, V2, DAG)) in lowerV32I8VectorShuffle()
15172 if (SDValue V = lowerVectorShuffleWithPACK(DL, MVT::v32i8, Mask, V1, V2, DAG, in lowerV32I8VectorShuffle()
15177 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v32i8, V1, V2, Mask, in lowerV32I8VectorShuffle()
15183 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG)) in lowerV32I8VectorShuffle()
15189 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG)) in lowerV32I8VectorShuffle()
15194 if (V2.isUndef() && is128BitLaneCrossingShuffleMask(MVT::v32i8, Mask)) { in lowerV32I8VectorShuffle()
15196 DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget)) in lowerV32I8VectorShuffle()
15199 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v32i8, V1, V2, Mask, in lowerV32I8VectorShuffle()
15204 DL, MVT::v32i8, Mask, V1, V2, Zeroable, Subtarget, DAG)) in lowerV32I8VectorShuffle()
15209 return lowerVectorShuffleWithPERMV(DL, MVT::v32i8, Mask, V1, V2, DAG); in lowerV32I8VectorShuffle()
15214 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG)) in lowerV32I8VectorShuffle()
15219 DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget)) in lowerV32I8VectorShuffle()
15223 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask, in lowerV32I8VectorShuffle()
15233 MVT VT, SDValue V1, SDValue V2, in lower256BitVectorShuffle()
15271 MVT FpVT = MVT::getVectorVT(MVT::getFloatingPointVT(ElementBits), in lower256BitVectorShuffle()
15279 case MVT::v4f64: in lower256BitVectorShuffle()
15281 case MVT::v4i64: in lower256BitVectorShuffle()
15283 case MVT::v8f32: in lower256BitVectorShuffle()
15285 case MVT::v8i32: in lower256BitVectorShuffle()
15287 case MVT::v16i16: in lower256BitVectorShuffle()
15289 case MVT::v32i8: in lower256BitVectorShuffle()
15298 static SDValue lowerV4X128VectorShuffle(const SDLoc &DL, MVT VT, in lowerV4X128VectorShuffle()
15320 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), NumElts); in lowerV4X128VectorShuffle()
15334 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 4); in lowerV4X128VectorShuffle()
15368 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2); in lowerV4X128VectorShuffle()
15396 DAG.getConstant(PermMask, DL, MVT::i8)); in lowerV4X128VectorShuffle()
15405 assert(V1.getSimpleValueType() == MVT::v8f64 && "Bad operand type!"); in lowerV8F64VectorShuffle()
15406 assert(V2.getSimpleValueType() == MVT::v8f64 && "Bad operand type!"); in lowerV8F64VectorShuffle()
15412 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v8f64, V1); in lowerV8F64VectorShuffle()
15414 if (!is128BitLaneCrossingShuffleMask(MVT::v8f64, Mask)) { in lowerV8F64VectorShuffle()
15421 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f64, V1, in lowerV8F64VectorShuffle()
15422 DAG.getConstant(VPERMILPMask, DL, MVT::i8)); in lowerV8F64VectorShuffle()
15426 if (is256BitLaneRepeatedShuffleMask(MVT::v8f64, Mask, RepeatedMask)) in lowerV8F64VectorShuffle()
15427 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8f64, V1, in lowerV8F64VectorShuffle()
15432 lowerV4X128VectorShuffle(DL, MVT::v8f64, Mask, Zeroable, V1, V2, in lowerV8F64VectorShuffle()
15437 lowerVectorShuffleWithUNPCK(DL, MVT::v8f64, Mask, V1, V2, DAG)) in lowerV8F64VectorShuffle()
15442 lowerVectorShuffleWithSHUFPD(DL, MVT::v8f64, Mask, V1, V2, DAG)) in lowerV8F64VectorShuffle()
15445 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v8f64, Zeroable, Mask, V1, in lowerV8F64VectorShuffle()
15449 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8f64, V1, V2, Mask, in lowerV8F64VectorShuffle()
15453 return lowerVectorShuffleWithPERMV(DL, MVT::v8f64, Mask, V1, V2, DAG); in lowerV8F64VectorShuffle()
15462 assert(V1.getSimpleValueType() == MVT::v16f32 && "Bad operand type!"); in lowerV16F32VectorShuffle()
15463 assert(V2.getSimpleValueType() == MVT::v16f32 && "Bad operand type!"); in lowerV16F32VectorShuffle()
15469 if (is128BitLaneRepeatedShuffleMask(MVT::v16f32, Mask, RepeatedMask)) { in lowerV16F32VectorShuffle()
15474 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v16f32, V1); in lowerV16F32VectorShuffle()
15476 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v16f32, V1); in lowerV16F32VectorShuffle()
15479 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v16f32, V1, in lowerV16F32VectorShuffle()
15484 lowerVectorShuffleWithUNPCK(DL, MVT::v16f32, Mask, V1, V2, DAG)) in lowerV16F32VectorShuffle()
15487 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask, in lowerV16F32VectorShuffle()
15492 return lowerVectorShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask, V1, V2, DAG); in lowerV16F32VectorShuffle()
15498 !is128BitLaneCrossingShuffleMask(MVT::v16f32, Mask)) { in lowerV16F32VectorShuffle()
15499 SDValue VPermMask = getConstVector(Mask, MVT::v16i32, DAG, DL, true); in lowerV16F32VectorShuffle()
15500 return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v16f32, V1, VPermMask); in lowerV16F32VectorShuffle()
15504 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v16f32, Zeroable, Mask, in lowerV16F32VectorShuffle()
15508 return lowerVectorShuffleWithPERMV(DL, MVT::v16f32, Mask, V1, V2, DAG); in lowerV16F32VectorShuffle()
15517 assert(V1.getSimpleValueType() == MVT::v8i64 && "Bad operand type!"); in lowerV8I64VectorShuffle()
15518 assert(V2.getSimpleValueType() == MVT::v8i64 && "Bad operand type!"); in lowerV8I64VectorShuffle()
15526 if (is128BitLaneRepeatedShuffleMask(MVT::v8i64, Mask, Repeated128Mask)) { in lowerV8I64VectorShuffle()
15530 MVT::v8i64, in lowerV8I64VectorShuffle()
15531 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32, in lowerV8I64VectorShuffle()
15532 DAG.getBitcast(MVT::v16i32, V1), in lowerV8I64VectorShuffle()
15537 if (is256BitLaneRepeatedShuffleMask(MVT::v8i64, Mask, Repeated256Mask)) in lowerV8I64VectorShuffle()
15538 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8i64, V1, in lowerV8I64VectorShuffle()
15543 lowerV4X128VectorShuffle(DL, MVT::v8i64, Mask, Zeroable, in lowerV8I64VectorShuffle()
15548 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v8i64, V1, V2, Mask, in lowerV8I64VectorShuffle()
15553 if (SDValue Rotate = lowerVectorShuffleAsRotate(DL, MVT::v8i64, V1, V2, in lowerV8I64VectorShuffle()
15558 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(DL, MVT::v8i64, V1, V2, in lowerV8I64VectorShuffle()
15563 lowerVectorShuffleWithUNPCK(DL, MVT::v8i64, Mask, V1, V2, DAG)) in lowerV8I64VectorShuffle()
15566 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v8i64, Zeroable, Mask, V1, in lowerV8I64VectorShuffle()
15570 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i64, V1, V2, Mask, in lowerV8I64VectorShuffle()
15574 return lowerVectorShuffleWithPERMV(DL, MVT::v8i64, Mask, V1, V2, DAG); in lowerV8I64VectorShuffle()
15583 assert(V1.getSimpleValueType() == MVT::v16i32 && "Bad operand type!"); in lowerV16I32VectorShuffle()
15584 assert(V2.getSimpleValueType() == MVT::v16i32 && "Bad operand type!"); in lowerV16I32VectorShuffle()
15591 DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV16I32VectorShuffle()
15599 is128BitLaneRepeatedShuffleMask(MVT::v16i32, Mask, RepeatedMask); in lowerV16I32VectorShuffle()
15603 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32, V1, in lowerV16I32VectorShuffle()
15608 lowerVectorShuffleWithUNPCK(DL, MVT::v16i32, Mask, V1, V2, DAG)) in lowerV16I32VectorShuffle()
15613 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v16i32, V1, V2, Mask, in lowerV16I32VectorShuffle()
15618 if (SDValue Rotate = lowerVectorShuffleAsRotate(DL, MVT::v16i32, V1, V2, in lowerV16I32VectorShuffle()
15625 DL, MVT::v16i32, V1, V2, Mask, Subtarget, DAG)) in lowerV16I32VectorShuffle()
15631 SDValue CastV1 = DAG.getBitcast(MVT::v16f32, V1); in lowerV16I32VectorShuffle()
15632 SDValue CastV2 = DAG.getBitcast(MVT::v16f32, V2); in lowerV16I32VectorShuffle()
15633 SDValue ShufPS = lowerVectorShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask, in lowerV16I32VectorShuffle()
15635 return DAG.getBitcast(MVT::v16i32, ShufPS); in lowerV16I32VectorShuffle()
15638 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v16i32, Zeroable, Mask, in lowerV16I32VectorShuffle()
15642 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16i32, V1, V2, Mask, in lowerV16I32VectorShuffle()
15645 return lowerVectorShuffleWithPERMV(DL, MVT::v16i32, Mask, V1, V2, DAG); in lowerV16I32VectorShuffle()
15654 assert(V1.getSimpleValueType() == MVT::v32i16 && "Bad operand type!"); in lowerV32I16VectorShuffle()
15655 assert(V2.getSimpleValueType() == MVT::v32i16 && "Bad operand type!"); in lowerV32I16VectorShuffle()
15663 DL, MVT::v32i16, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV32I16VectorShuffle()
15668 lowerVectorShuffleWithUNPCK(DL, MVT::v32i16, Mask, V1, V2, DAG)) in lowerV32I16VectorShuffle()
15672 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v32i16, V1, V2, Mask, in lowerV32I16VectorShuffle()
15678 DL, MVT::v32i16, V1, V2, Mask, Subtarget, DAG)) in lowerV32I16VectorShuffle()
15683 if (is128BitLaneRepeatedShuffleMask(MVT::v32i16, Mask, RepeatedMask)) { in lowerV32I16VectorShuffle()
15688 DL, MVT::v32i16, V1, RepeatedMask, Subtarget, DAG); in lowerV32I16VectorShuffle()
15692 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v32i16, V1, V2, Mask, in lowerV32I16VectorShuffle()
15697 DL, MVT::v32i16, Mask, V1, V2, Zeroable, Subtarget, DAG)) in lowerV32I16VectorShuffle()
15700 return lowerVectorShuffleWithPERMV(DL, MVT::v32i16, Mask, V1, V2, DAG); in lowerV32I16VectorShuffle()
15709 assert(V1.getSimpleValueType() == MVT::v64i8 && "Bad operand type!"); in lowerV64I8VectorShuffle()
15710 assert(V2.getSimpleValueType() == MVT::v64i8 && "Bad operand type!"); in lowerV64I8VectorShuffle()
15718 DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV64I8VectorShuffle()
15723 lowerVectorShuffleWithUNPCK(DL, MVT::v64i8, Mask, V1, V2, DAG)) in lowerV64I8VectorShuffle()
15727 if (SDValue V = lowerVectorShuffleWithPACK(DL, MVT::v64i8, Mask, V1, V2, DAG, in lowerV64I8VectorShuffle()
15732 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v64i8, V1, V2, Mask, in lowerV64I8VectorShuffle()
15738 DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG)) in lowerV64I8VectorShuffle()
15742 DL, MVT::v64i8, Mask, V1, V2, Zeroable, Subtarget, DAG)) in lowerV64I8VectorShuffle()
15747 return lowerVectorShuffleWithPERMV(DL, MVT::v64i8, Mask, V1, V2, DAG); in lowerV64I8VectorShuffle()
15752 DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG)) in lowerV64I8VectorShuffle()
15755 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v64i8, V1, V2, Mask, in lowerV64I8VectorShuffle()
15760 return splitAndLowerVectorShuffle(DL, MVT::v64i8, V1, V2, Mask, DAG); in lowerV64I8VectorShuffle()
15769 MVT VT, SDValue V1, SDValue V2, in lower512BitVectorShuffle()
15801 case MVT::v8f64: in lower512BitVectorShuffle()
15803 case MVT::v16f32: in lower512BitVectorShuffle()
15805 case MVT::v8i64: in lower512BitVectorShuffle()
15807 case MVT::v16i32: in lower512BitVectorShuffle()
15809 case MVT::v32i16: in lower512BitVectorShuffle()
15811 case MVT::v64i8: in lower512BitVectorShuffle()
15857 MVT VT, SDValue V1, SDValue V2, in lower1BitVectorShuffle()
15882 MVT ExtractVT = MVT::getVectorVT(MVT::i1, SubvecElts); in lower1BitVectorShuffle()
15899 DAG.getConstant(ShiftAmt, DL, MVT::i8)); in lower1BitVectorShuffle()
15905 MVT ExtVT; in lower1BitVectorShuffle()
15909 case MVT::v2i1: in lower1BitVectorShuffle()
15910 ExtVT = MVT::v2i64; in lower1BitVectorShuffle()
15912 case MVT::v4i1: in lower1BitVectorShuffle()
15913 ExtVT = MVT::v4i32; in lower1BitVectorShuffle()
15915 case MVT::v8i1: in lower1BitVectorShuffle()
15918 ExtVT = Subtarget.hasVLX() ? MVT::v8i32 : MVT::v8i64; in lower1BitVectorShuffle()
15920 case MVT::v16i1: in lower1BitVectorShuffle()
15923 ExtVT = Subtarget.canExtendTo512DQ() ? MVT::v16i32 : MVT::v16i16; in lower1BitVectorShuffle()
15925 case MVT::v32i1: in lower1BitVectorShuffle()
15929 ExtVT = Subtarget.canExtendTo512BW() ? MVT::v32i16 : MVT::v32i8; in lower1BitVectorShuffle()
15931 case MVT::v64i1: in lower1BitVectorShuffle()
15932 ExtVT = MVT::v64i8; in lower1BitVectorShuffle()
16027 MVT VT = Op.getSimpleValueType(); in lowerVectorShuffle()
16030 bool Is1BitVector = (VT.getVectorElementType() == MVT::i1); in lowerVectorShuffle()
16099 MVT NewEltVT = VT.isFloatingPoint() in lowerVectorShuffle()
16100 ? MVT::getFloatingPointVT(VT.getScalarSizeInBits() * 2) in lowerVectorShuffle()
16101 : MVT::getIntegerVT(VT.getScalarSizeInBits() * 2); in lowerVectorShuffle()
16103 MVT NewVT = MVT::getVectorVT(NewEltVT, NewNumElts); in lowerVectorShuffle()
16165 MVT VT = Op.getSimpleValueType(); in lowerVSELECTtoVectorShuffle()
16195 MVT CondVT = Cond.getSimpleValueType(); in LowerVSELECT()
16205 MVT VT = Op.getSimpleValueType(); in LowerVSELECT()
16214 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts); in LowerVSELECT()
16228 MVT NewCondSVT = MVT::getIntegerVT(EltSize); in LowerVSELECT()
16229 MVT NewCondVT = MVT::getVectorVT(NewCondSVT, NumElts); in LowerVSELECT()
16242 case MVT::v32i8: in LowerVSELECT()
16249 case MVT::v8i16: in LowerVSELECT()
16250 case MVT::v16i16: { in LowerVSELECT()
16252 MVT CastVT = MVT::getVectorVT(MVT::i8, NumElts * 2); in LowerVSELECT()
16263 MVT VT = Op.getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT_SSE4()
16270 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
16275 if (VT == MVT::f32) { in LowerEXTRACT_VECTOR_ELT_SSE4()
16287 User->getValueType(0) != MVT::i32)) in LowerEXTRACT_VECTOR_ELT_SSE4()
16289 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
16290 DAG.getBitcast(MVT::v4i32, Op.getOperand(0)), in LowerEXTRACT_VECTOR_ELT_SSE4()
16292 return DAG.getBitcast(MVT::f32, Extract); in LowerEXTRACT_VECTOR_ELT_SSE4()
16295 if (VT == MVT::i32 || VT == MVT::i64) { in LowerEXTRACT_VECTOR_ELT_SSE4()
16310 MVT VecVT = Vec.getSimpleValueType(); in ExtractBitFromMaskVector()
16312 MVT EltVT = Op.getSimpleValueType(); in ExtractBitFromMaskVector()
16323 MVT ExtEltVT = (NumElts <= 8) ? MVT::getIntegerVT(128 / NumElts) : MVT::i8; in ExtractBitFromMaskVector()
16324 MVT ExtVecVT = MVT::getVectorVT(ExtEltVT, NumElts); in ExtractBitFromMaskVector()
16336 MVT WideVecVT = VecVT; in ExtractBitFromMaskVector()
16338 WideVecVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1; in ExtractBitFromMaskVector()
16346 DAG.getConstant(IdxVal, dl, MVT::i8)); in ExtractBitFromMaskVector()
16357 MVT VecVT = Vec.getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT()
16360 if (VecVT.getVectorElementType() == MVT::i1) in LowerEXTRACT_VECTOR_ELT()
16404 MVT EltVT = VecVT.getVectorElementType(); in LowerEXTRACT_VECTOR_ELT()
16413 DAG.getConstant(IdxVal, dl, MVT::i32)); in LowerEXTRACT_VECTOR_ELT()
16418 MVT VT = Op.getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT()
16425 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, in LowerEXTRACT_VECTOR_ELT()
16426 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT()
16427 DAG.getBitcast(MVT::v4i32, Vec), Idx)); in LowerEXTRACT_VECTOR_ELT()
16430 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT()
16446 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT()
16447 DAG.getBitcast(MVT::v4i32, Vec), in LowerEXTRACT_VECTOR_ELT()
16451 Res = DAG.getNode(ISD::SRL, dl, MVT::i32, Res, in LowerEXTRACT_VECTOR_ELT()
16452 DAG.getConstant(ShiftVal, dl, MVT::i8)); in LowerEXTRACT_VECTOR_ELT()
16457 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, in LowerEXTRACT_VECTOR_ELT()
16458 DAG.getBitcast(MVT::v8i16, Vec), in LowerEXTRACT_VECTOR_ELT()
16462 Res = DAG.getNode(ISD::SRL, dl, MVT::i16, Res, in LowerEXTRACT_VECTOR_ELT()
16463 DAG.getConstant(ShiftVal, dl, MVT::i8)); in LowerEXTRACT_VECTOR_ELT()
16505 MVT VecVT = Vec.getSimpleValueType(); in InsertBitToMaskVector()
16511 MVT ExtEltVT = (NumElts <= 8) ? MVT::getIntegerVT(128 / NumElts) : MVT::i8; in InsertBitToMaskVector()
16512 MVT ExtVecVT = MVT::getVectorVT(ExtEltVT, NumElts); in InsertBitToMaskVector()
16520 SDValue EltInVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i1, Elt); in InsertBitToMaskVector()
16528 MVT VT = Op.getSimpleValueType(); in LowerINSERT_VECTOR_ELT()
16529 MVT EltVT = VT.getVectorElementType(); in LowerINSERT_VECTOR_ELT()
16532 if (EltVT == MVT::i1) in LowerINSERT_VECTOR_ELT()
16569 if ((Subtarget.hasAVX() && (EltVT == MVT::f64 || EltVT == MVT::f32)) || in LowerINSERT_VECTOR_ELT()
16570 (Subtarget.hasAVX2() && EltVT == MVT::i32)) { in LowerINSERT_VECTOR_ELT()
16587 DAG.getConstant(IdxIn128, dl, MVT::i32)); in LowerINSERT_VECTOR_ELT()
16596 if (VT == MVT::v8i16 || (VT == MVT::v16i8 && Subtarget.hasSSE41())) { in LowerINSERT_VECTOR_ELT()
16598 if (VT == MVT::v8i16) { in LowerINSERT_VECTOR_ELT()
16602 assert(VT == MVT::v16i8 && "PINSRB requires v16i8 vector"); in LowerINSERT_VECTOR_ELT()
16607 if (N1.getValueType() != MVT::i32) in LowerINSERT_VECTOR_ELT()
16608 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); in LowerINSERT_VECTOR_ELT()
16609 if (N2.getValueType() != MVT::i32) in LowerINSERT_VECTOR_ELT()
16615 if (EltVT == MVT::f32) { in LowerINSERT_VECTOR_ELT()
16635 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1); in LowerINSERT_VECTOR_ELT()
16640 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1); in LowerINSERT_VECTOR_ELT()
16645 if (EltVT == MVT::i32 || EltVT == MVT::i64) in LowerINSERT_VECTOR_ELT()
16655 MVT OpVT = Op.getSimpleValueType(); in LowerSCALAR_TO_VECTOR()
16667 MVT VT128 = MVT::getVectorVT(OpVT.getVectorElementType(), in LowerSCALAR_TO_VECTOR()
16678 if (OpVT == MVT::v4i32) in LowerSCALAR_TO_VECTOR()
16681 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0)); in LowerSCALAR_TO_VECTOR()
16683 OpVT, DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, AnyExt)); in LowerSCALAR_TO_VECTOR()
16691 assert(Op.getSimpleValueType().getVectorElementType() == MVT::i1); in LowerINSERT_SUBVECTOR()
16698 assert(Op.getSimpleValueType().getVectorElementType() == MVT::i1 && in LowerEXTRACT_SUBVECTOR()
16712 MVT VecVT = Vec.getSimpleValueType(); in LowerEXTRACT_SUBVECTOR()
16716 MVT WideVecVT = VecVT; in LowerEXTRACT_SUBVECTOR()
16718 WideVecVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1; in LowerEXTRACT_SUBVECTOR()
16726 DAG.getConstant(IdxVal, dl, MVT::i8)); in LowerEXTRACT_SUBVECTOR()
16906 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in GetTLSADDR()
17106 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerGlobalTLSAddress()
17167 MachinePointerInfo(), MVT::i32); in LowerGlobalTLSAddress()
17173 DAG.getConstant(Log2_64_Ceil(DL.getPointerSize()), dl, MVT::i8); in LowerGlobalTLSAddress()
17200 MVT VT = Op.getSimpleValueType(); in LowerShiftParts()
17210 SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt, in LowerShiftParts()
17211 DAG.getConstant(VTBits - 1, dl, MVT::i8)); in LowerShiftParts()
17213 DAG.getConstant(VTBits - 1, dl, MVT::i8)) in LowerShiftParts()
17228 SDValue AndNode = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt, in LowerShiftParts()
17229 DAG.getConstant(VTBits, dl, MVT::i8)); in LowerShiftParts()
17230 SDValue Cond = DAG.getSetCC(dl, MVT::i8, AndNode, in LowerShiftParts()
17231 DAG.getConstant(0, dl, MVT::i8), ISD::SETNE); in LowerShiftParts()
17247 MVT VT = Op.getSimpleValueType(); in LowerFunnelShift()
17268 Op0, Op1, DAG.getConstant(ShiftAmt, DL, MVT::i8)); in LowerFunnelShift()
17275 assert((VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) && in LowerFunnelShift()
17287 if (VT == MVT::i16) in LowerFunnelShift()
17302 MVT SrcVT = Src.getSimpleValueType(); in LowerI64IntToFP_AVX512DQ()
17303 MVT VT = Op.getSimpleValueType(); in LowerI64IntToFP_AVX512DQ()
17305 if (!Subtarget.hasDQI() || SrcVT != MVT::i64 || Subtarget.is64Bit() || in LowerI64IntToFP_AVX512DQ()
17306 (VT != MVT::f32 && VT != MVT::f64)) in LowerI64IntToFP_AVX512DQ()
17313 MVT VecInVT = MVT::getVectorVT(MVT::i64, NumElts); in LowerI64IntToFP_AVX512DQ()
17314 MVT VecVT = MVT::getVectorVT(VT, NumElts); in LowerI64IntToFP_AVX512DQ()
17326 MVT SrcVT = Src.getSimpleValueType(); in LowerSINT_TO_FP()
17327 MVT VT = Op.getSimpleValueType(); in LowerSINT_TO_FP()
17331 if (SrcVT == MVT::v2i32 && VT == MVT::v2f64) { in LowerSINT_TO_FP()
17333 DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src, in LowerSINT_TO_FP()
17339 assert(SrcVT <= MVT::i64 && SrcVT >= MVT::i16 && in LowerSINT_TO_FP()
17344 if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(VT)) in LowerSINT_TO_FP()
17346 if (SrcVT == MVT::i64 && isScalarFPTypeInSSEReg(VT) && Subtarget.is64Bit()) in LowerSINT_TO_FP()
17353 if (SrcVT == MVT::i64 && isScalarFPTypeInSSEReg(VT) && in LowerSINT_TO_FP()
17358 ValueToStore = DAG.getBitcast(MVT::f64, ValueToStore); in LowerSINT_TO_FP()
17379 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Glue); in BuildFILD()
17381 Tys = DAG.getVTList(Op.getValueType(), MVT::Other); in BuildFILD()
17413 Tys = DAG.getVTList(MVT::Other); in BuildFILD()
17467 SDValue XR1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, in LowerUINT_TO_FP_i64()
17470 DAG.getLoad(MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0, in LowerUINT_TO_FP_i64()
17474 getUnpackl(DAG, dl, MVT::v4i32, DAG.getBitcast(MVT::v4i32, XR1), CLod0); in LowerUINT_TO_FP_i64()
17477 DAG.getLoad(MVT::v2f64, dl, CLod0.getValue(1), CPIdx1, in LowerUINT_TO_FP_i64()
17480 SDValue XR2F = DAG.getBitcast(MVT::v2f64, Unpck1); in LowerUINT_TO_FP_i64()
17482 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1); in LowerUINT_TO_FP_i64()
17487 Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub); in LowerUINT_TO_FP_i64()
17489 SDValue Shuffle = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, Sub, {1,-1}); in LowerUINT_TO_FP_i64()
17490 Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64, Shuffle, Sub); in LowerUINT_TO_FP_i64()
17493 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result, in LowerUINT_TO_FP_i64()
17503 MVT::f64); in LowerUINT_TO_FP_i32()
17506 SDValue Load = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, in LowerUINT_TO_FP_i32()
17512 Load = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in LowerUINT_TO_FP_i32()
17513 DAG.getBitcast(MVT::v2f64, Load), in LowerUINT_TO_FP_i32()
17518 ISD::OR, dl, MVT::v2i64, in LowerUINT_TO_FP_i32()
17519 DAG.getBitcast(MVT::v2i64, in LowerUINT_TO_FP_i32()
17520 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, Load)), in LowerUINT_TO_FP_i32()
17521 DAG.getBitcast(MVT::v2i64, in LowerUINT_TO_FP_i32()
17522 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, Bias))); in LowerUINT_TO_FP_i32()
17524 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in LowerUINT_TO_FP_i32()
17525 DAG.getBitcast(MVT::v2f64, Or), DAG.getIntPtrConstant(0, dl)); in LowerUINT_TO_FP_i32()
17529 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias); in LowerUINT_TO_FP_i32()
17538 if (Op.getSimpleValueType() != MVT::v2f64) in lowerUINT_TO_FP_v2i32()
17542 assert(N0.getSimpleValueType() == MVT::v2i32 && "Unexpected input type"); in lowerUINT_TO_FP_v2i32()
17545 N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0, in lowerUINT_TO_FP_v2i32()
17546 DAG.getUNDEF(MVT::v2i32)); in lowerUINT_TO_FP_v2i32()
17549 return DAG.getNode(X86ISD::CVTUI2P, DL, MVT::v2f64, N0); in lowerUINT_TO_FP_v2i32()
17553 SDValue HalfWord = DAG.getConstant(16, DL, MVT::v4i32); in lowerUINT_TO_FP_v2i32()
17554 SDValue HalfWordMask = DAG.getConstant(0x0000FFFF, DL, MVT::v4i32); in lowerUINT_TO_FP_v2i32()
17557 SDValue TWOHW = DAG.getConstantFP(1 << 16, DL, MVT::v2f64); in lowerUINT_TO_FP_v2i32()
17560 SDValue HI = DAG.getNode(ISD::SRL, DL, MVT::v4i32, N0, HalfWord); in lowerUINT_TO_FP_v2i32()
17561 SDValue LO = DAG.getNode(ISD::AND, DL, MVT::v4i32, N0, HalfWordMask); in lowerUINT_TO_FP_v2i32()
17563 SDValue fHI = DAG.getNode(X86ISD::CVTSI2P, DL, MVT::v2f64, HI); in lowerUINT_TO_FP_v2i32()
17564 fHI = DAG.getNode(ISD::FMUL, DL, MVT::v2f64, fHI, TWOHW); in lowerUINT_TO_FP_v2i32()
17565 SDValue fLO = DAG.getNode(X86ISD::CVTSI2P, DL, MVT::v2f64, LO); in lowerUINT_TO_FP_v2i32()
17568 return DAG.getNode(ISD::FADD, DL, MVT::v2f64, fHI, fLO); in lowerUINT_TO_FP_v2i32()
17596 MVT VecIntVT = V.getSimpleValueType(); in lowerUINT_TO_FP_vXi32()
17597 bool Is128 = VecIntVT == MVT::v4i32; in lowerUINT_TO_FP_vXi32()
17598 MVT VecFloatVT = Is128 ? MVT::v4f32 : MVT::v8f32; in lowerUINT_TO_FP_vXi32()
17604 assert((VecIntVT == MVT::v4i32 || VecIntVT == MVT::v8i32) && in lowerUINT_TO_FP_vXi32()
17625 MVT VecI16VT = Is128 ? MVT::v8i16 : MVT::v16i16; in lowerUINT_TO_FP_vXi32()
17632 VecCstLowBitcast, DAG.getConstant(0xaa, DL, MVT::i32)); in lowerUINT_TO_FP_vXi32()
17640 VecCstHighBitcast, DAG.getConstant(0xaa, DL, MVT::i32)); in lowerUINT_TO_FP_vXi32()
17668 MVT SrcVT = N0.getSimpleValueType(); in lowerUINT_TO_FP_vec()
17674 case MVT::v2i32: in lowerUINT_TO_FP_vec()
17676 case MVT::v4i32: in lowerUINT_TO_FP_vec()
17677 case MVT::v8i32: in lowerUINT_TO_FP_vec()
17692 MVT SrcVT = N0.getSimpleValueType(); in LowerUINT_TO_FP()
17693 MVT DstVT = Op.getSimpleValueType(); in LowerUINT_TO_FP()
17696 (SrcVT == MVT::i32 || (SrcVT == MVT::i64 && Subtarget.is64Bit()))) { in LowerUINT_TO_FP()
17705 if (SrcVT == MVT::i64 && DstVT == MVT::f64 && X86ScalarSSEf64) in LowerUINT_TO_FP()
17707 if (SrcVT == MVT::i32 && X86ScalarSSEf64) in LowerUINT_TO_FP()
17709 if (Subtarget.is64Bit() && SrcVT == MVT::i64 && DstVT == MVT::f32) in LowerUINT_TO_FP()
17713 SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64); in LowerUINT_TO_FP()
17714 if (SrcVT == MVT::i32) { in LowerUINT_TO_FP()
17718 SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, dl, MVT::i32), in LowerUINT_TO_FP()
17720 SDValue Fild = BuildFILD(Op, MVT::i64, Store2, StackSlot, DAG); in LowerUINT_TO_FP()
17724 assert(SrcVT == MVT::i64 && "Unexpected type in UINT_TO_FP"); in LowerUINT_TO_FP()
17730 ValueToStore = DAG.getBitcast(MVT::f64, ValueToStore); in LowerUINT_TO_FP()
17743 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other); in LowerUINT_TO_FP()
17744 SDValue Ops[] = { Store, StackSlot, DAG.getValueType(MVT::i64) }; in LowerUINT_TO_FP()
17746 MVT::i64, MMO); in LowerUINT_TO_FP()
17752 dl, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i64), in LowerUINT_TO_FP()
17753 Op.getOperand(0), DAG.getConstant(0, dl, MVT::i64), ISD::SETLT); in LowerUINT_TO_FP()
17768 ISD::EXTLOAD, dl, MVT::f80, DAG.getEntryNode(), FudgePtr, in LowerUINT_TO_FP()
17769 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32, in LowerUINT_TO_FP()
17773 SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::f80, Fild, Fudge); in LowerUINT_TO_FP()
17796 if (TheVT != MVT::f32 && TheVT != MVT::f64 && TheVT != MVT::f80) { in FP_TO_INTHelper()
17806 DstTy == MVT::i64 && in FP_TO_INTHelper()
17810 if (!IsSigned && DstTy != MVT::i64 && !Subtarget.hasAVX512()) { in FP_TO_INTHelper()
17813 assert(DstTy == MVT::i32 && "Unexpected FP_TO_UINT"); in FP_TO_INTHelper()
17814 DstTy = MVT::i64; in FP_TO_INTHelper()
17817 assert(DstTy.getSimpleVT() <= MVT::i64 && in FP_TO_INTHelper()
17818 DstTy.getSimpleVT() >= MVT::i16 && in FP_TO_INTHelper()
17822 if (DstTy == MVT::i32 && in FP_TO_INTHelper()
17826 DstTy == MVT::i64 && in FP_TO_INTHelper()
17840 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break; in FP_TO_INTHelper()
17841 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break; in FP_TO_INTHelper()
17842 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break; in FP_TO_INTHelper()
17869 if (TheVT == MVT::f64) in FP_TO_INTHelper()
17873 else if (TheVT == MVT::f80) in FP_TO_INTHelper()
17886 Adjust = DAG.getSelect(DL, MVT::i32, Cmp, in FP_TO_INTHelper()
17887 DAG.getConstant(0, DL, MVT::i32), in FP_TO_INTHelper()
17888 DAG.getConstant(0x80000000, DL, MVT::i32)); in FP_TO_INTHelper()
17899 assert(DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!"); in FP_TO_INTHelper()
17902 SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other); in FP_TO_INTHelper()
17926 SDValue FIST = DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::Other), in FP_TO_INTHelper()
17930 DAG.getLoad(MVT::i32, DL, FIST, StackSlot, MachinePointerInfo()); in FP_TO_INTHelper()
17934 DAG.getLoad(MVT::i32, DL, FIST, HighAddr, MachinePointerInfo()); in FP_TO_INTHelper()
17935 High32 = DAG.getNode(ISD::XOR, DL, MVT::i32, High32, Adjust); in FP_TO_INTHelper()
17940 Low32 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Low32); in FP_TO_INTHelper()
17941 High32 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, High32); in FP_TO_INTHelper()
17942 High32 = DAG.getNode(ISD::SHL, DL, MVT::i64, High32, in FP_TO_INTHelper()
17943 DAG.getConstant(32, DL, MVT::i8)); in FP_TO_INTHelper()
17944 SDValue Result = DAG.getNode(ISD::OR, DL, MVT::i64, High32, Low32); in FP_TO_INTHelper()
17951 ? DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, ResultOps) in FP_TO_INTHelper()
17957 SDValue FIST = DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::Other), in FP_TO_INTHelper()
17965 MVT VT = Op->getSimpleValueType(0); in LowerAVXExtend()
17967 MVT InVT = In.getSimpleValueType(); in LowerAVXExtend()
17973 assert((VT.getVectorElementType() == MVT::i16 || in LowerAVXExtend()
17974 VT.getVectorElementType() == MVT::i32 || in LowerAVXExtend()
17975 VT.getVectorElementType() == MVT::i64) && in LowerAVXExtend()
17977 assert((InVT.getVectorElementType() == MVT::i8 || in LowerAVXExtend()
17978 InVT.getVectorElementType() == MVT::i16 || in LowerAVXExtend()
17979 InVT.getVectorElementType() == MVT::i32) && in LowerAVXExtend()
17983 if (InVT == MVT::v8i8) { in LowerAVXExtend()
17984 if (!ExperimentalVectorWideningLegalization || VT != MVT::v8i64) in LowerAVXExtend()
17988 MVT::v16i8, In, DAG.getUNDEF(MVT::v8i8)); in LowerAVXExtend()
18009 MVT HalfVT = MVT::getVectorVT(VT.getVectorElementType(), in LowerAVXExtend()
18024 static SDValue SplitAndExtendv16i1(unsigned ExtOpc, MVT VT, SDValue In, in SplitAndExtendv16i1()
18026 assert((VT == MVT::v16i8 || VT == MVT::v16i16) && "Unexpected VT."); in SplitAndExtendv16i1()
18027 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In, in SplitAndExtendv16i1()
18029 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In, in SplitAndExtendv16i1()
18031 Lo = DAG.getNode(ExtOpc, dl, MVT::v8i16, Lo); in SplitAndExtendv16i1()
18032 Hi = DAG.getNode(ExtOpc, dl, MVT::v8i16, Hi); in SplitAndExtendv16i1()
18033 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i16, Lo, Hi); in SplitAndExtendv16i1()
18040 MVT VT = Op->getSimpleValueType(0); in LowerZERO_EXTEND_Mask()
18042 MVT InVT = In.getSimpleValueType(); in LowerZERO_EXTEND_Mask()
18043 assert(InVT.getVectorElementType() == MVT::i1 && "Unexpected input type!"); in LowerZERO_EXTEND_Mask()
18049 if (VT.getVectorElementType() != MVT::i8) { in LowerZERO_EXTEND_Mask()
18056 MVT ExtVT = VT; in LowerZERO_EXTEND_Mask()
18062 ExtVT = MVT::getVectorVT(MVT::i32, NumElts); in LowerZERO_EXTEND_Mask()
18066 MVT WideVT = ExtVT; in LowerZERO_EXTEND_Mask()
18069 InVT = MVT::getVectorVT(MVT::i1, NumElts); in LowerZERO_EXTEND_Mask()
18072 WideVT = MVT::getVectorVT(ExtVT.getVectorElementType(), in LowerZERO_EXTEND_Mask()
18083 WideVT = MVT::getVectorVT(MVT::i8, NumElts); in LowerZERO_EXTEND_Mask()
18098 MVT SVT = In.getSimpleValueType(); in LowerZERO_EXTEND()
18100 if (SVT.getVectorElementType() == MVT::i1) in LowerZERO_EXTEND()
18148 EVT InVT = MVT::i16, OutVT = MVT::i8; in truncateVectorWithPACK()
18151 InVT = MVT::i32; in truncateVectorWithPACK()
18152 OutVT = MVT::i16; in truncateVectorWithPACK()
18191 Res = DAG.getBitcast(MVT::v4i64, Res); in truncateVectorWithPACK()
18192 Res = DAG.getVectorShuffle(MVT::v4i64, DL, Res, Res, {0, 2, 1, 3}); in truncateVectorWithPACK()
18218 MVT VT = Op.getSimpleValueType(); in LowerTruncateVecI1()
18220 MVT InVT = In.getSimpleValueType(); in LowerTruncateVecI1()
18222 assert(VT.getVectorElementType() == MVT::i1 && "Unexpected vector type."); in LowerTruncateVecI1()
18232 MVT ExtVT = MVT::getVectorVT(MVT::i16, InVT.getSizeInBits()/16); in LowerTruncateVecI1()
18256 if (InVT == MVT::v16i8) { in LowerTruncateVecI1()
18258 InVT = MVT::v16i16; in LowerTruncateVecI1()
18265 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Lo); in LowerTruncateVecI1()
18266 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Hi); in LowerTruncateVecI1()
18272 MVT EltVT = Subtarget.hasVLX() ? MVT::i32 : MVT::getIntegerVT(512/NumElts); in LowerTruncateVecI1()
18273 MVT ExtVT = MVT::getVectorVT(EltVT, NumElts); in LowerTruncateVecI1()
18292 MVT VT = Op.getSimpleValueType(); in LowerTRUNCATE()
18294 MVT InVT = In.getSimpleValueType(); in LowerTRUNCATE()
18304 if (VT.getVectorElementType() == MVT::i1) in LowerTRUNCATE()
18313 if (InVT != MVT::v16i16 || Subtarget.hasBWI() || in LowerTRUNCATE()
18337 if ((VT == MVT::v4i32) && (InVT == MVT::v4i64)) { in LowerTRUNCATE()
18341 In = DAG.getBitcast(MVT::v8i32, In); in LowerTRUNCATE()
18342 In = DAG.getVectorShuffle(MVT::v8i32, DL, In, In, ShufMask); in LowerTRUNCATE()
18347 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
18349 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
18351 OpLo = DAG.getBitcast(MVT::v4i32, OpLo); in LowerTRUNCATE()
18352 OpHi = DAG.getBitcast(MVT::v4i32, OpHi); in LowerTRUNCATE()
18357 if ((VT == MVT::v8i16) && (InVT == MVT::v8i32)) { in LowerTRUNCATE()
18360 In = DAG.getBitcast(MVT::v32i8, In); in LowerTRUNCATE()
18367 In = DAG.getVectorShuffle(MVT::v32i8, DL, In, In, ShufMask1); in LowerTRUNCATE()
18368 In = DAG.getBitcast(MVT::v4i64, In); in LowerTRUNCATE()
18371 In = DAG.getVectorShuffle(MVT::v4i64, DL, In, In, ShufMask2); in LowerTRUNCATE()
18372 In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
18377 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In, in LowerTRUNCATE()
18380 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In, in LowerTRUNCATE()
18383 OpLo = DAG.getBitcast(MVT::v16i8, OpLo); in LowerTRUNCATE()
18384 OpHi = DAG.getBitcast(MVT::v16i8, OpHi); in LowerTRUNCATE()
18390 OpLo = DAG.getVectorShuffle(MVT::v16i8, DL, OpLo, OpLo, ShufMask1); in LowerTRUNCATE()
18391 OpHi = DAG.getVectorShuffle(MVT::v16i8, DL, OpHi, OpHi, ShufMask1); in LowerTRUNCATE()
18393 OpLo = DAG.getBitcast(MVT::v4i32, OpLo); in LowerTRUNCATE()
18394 OpHi = DAG.getBitcast(MVT::v4i32, OpHi); in LowerTRUNCATE()
18398 SDValue res = DAG.getVectorShuffle(MVT::v4i32, DL, OpLo, OpHi, ShufMask2); in LowerTRUNCATE()
18399 return DAG.getBitcast(MVT::v8i16, res); in LowerTRUNCATE()
18402 if (VT == MVT::v16i8 && InVT == MVT::v16i16) { in LowerTRUNCATE()
18406 SDValue InLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i16, In, in LowerTRUNCATE()
18408 SDValue InHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i16, In, in LowerTRUNCATE()
18419 MVT NVT = MVT::getVectorVT(VT.getVectorElementType(), NumElems * 2); in LowerTRUNCATE()
18433 MVT VT = Op.getSimpleValueType(); in LowerFP_TO_INT()
18439 if (VT == MVT::v2i1 && Src.getSimpleValueType() == MVT::v2f64) { in LowerFP_TO_INT()
18440 MVT ResVT = MVT::v4i32; in LowerFP_TO_INT()
18441 MVT TruncVT = MVT::v4i1; in LowerFP_TO_INT()
18445 ResVT = MVT::v8i32; in LowerFP_TO_INT()
18446 TruncVT = MVT::v8i1; in LowerFP_TO_INT()
18448 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8f64, in LowerFP_TO_INT()
18449 DAG.getUNDEF(MVT::v8f64), in LowerFP_TO_INT()
18454 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i1, Res, in LowerFP_TO_INT()
18459 if (VT == MVT::v2i64 && Src.getSimpleValueType() == MVT::v2f32) { in LowerFP_TO_INT()
18461 DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src, in LowerFP_TO_INT()
18462 DAG.getUNDEF(MVT::v2f32))); in LowerFP_TO_INT()
18487 MVT VT = Op.getSimpleValueType(); in LowerFP_EXTEND()
18489 MVT SVT = In.getSimpleValueType(); in LowerFP_EXTEND()
18491 assert(SVT == MVT::v2f32 && "Only customize MVT::v2f32 type legalization!"); in LowerFP_EXTEND()
18494 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32, in LowerFP_EXTEND()
18585 assert((Op.getValueType() == MVT::f32 || Op.getValueType() == MVT::f64) && in lowerFaddFsub()
18606 MVT VT = Op.getSimpleValueType(); in LowerFABSorFNEG()
18608 bool IsF128 = (VT == MVT::f128); in LowerFABSorFNEG()
18609 assert((VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 || in LowerFABSorFNEG()
18610 VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 || in LowerFABSorFNEG()
18611 VT == MVT::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) && in LowerFABSorFNEG()
18623 MVT LogicVT = VT; in LowerFABSorFNEG()
18625 LogicVT = (VT == MVT::f64) ? MVT::v2f64 : MVT::v4f32; in LowerFABSorFNEG()
18658 MVT VT = Op.getSimpleValueType(); in LowerFCOPYSIGN()
18668 bool IsF128 = (VT == MVT::f128); in LowerFCOPYSIGN()
18669 assert((VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 || in LowerFCOPYSIGN()
18670 VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 || in LowerFCOPYSIGN()
18671 VT == MVT::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) && in LowerFCOPYSIGN()
18682 MVT LogicVT = VT; in LowerFCOPYSIGN()
18684 LogicVT = (VT == MVT::f64) ? MVT::v2f64 : MVT::v4f32; in LowerFCOPYSIGN()
18722 MVT VT = Op.getSimpleValueType(); in LowerFGETSIGN()
18724 MVT OpVT = N0.getSimpleValueType(); in LowerFGETSIGN()
18725 assert((OpVT == MVT::f32 || OpVT == MVT::f64) && in LowerFGETSIGN()
18729 MVT VecVT = (OpVT == MVT::f32 ? MVT::v4f32 : MVT::v2f64); in LowerFGETSIGN()
18731 Res = DAG.getNode(X86ISD::MOVMSK, dl, MVT::i32, Res); in LowerFGETSIGN()
18740 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in getSETCC()
18741 DAG.getConstant(Cond, dl, MVT::i8), EFLAGS); in getSETCC()
18763 EVT VT = MVT::Other; in LowerVectorAllZeroTest()
18819 MVT TestVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64; in LowerVectorAllZeroTest()
18835 DL, MVT::i8); in LowerVectorAllZeroTest()
18836 return DAG.getNode(X86ISD::PTEST, DL, MVT::i32, in LowerVectorAllZeroTest()
18899 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in EmitTest()
18957 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in EmitTest()
18960 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); in EmitTest()
18975 if ((Op0.getValueType() == MVT::i8 || Op0.getValueType() == MVT::i16 || in EmitCmp()
18976 Op0.getValueType() == MVT::i32 || Op0.getValueType() == MVT::i64)) { in EmitCmp()
18979 if (Op0.getValueType() == MVT::i16 && in EmitCmp()
18988 Op0 = DAG.getNode(ExtendOp, dl, MVT::i32, Op0); in EmitCmp()
18989 Op1 = DAG.getNode(ExtendOp, dl, MVT::i32, Op1); in EmitCmp()
18992 SDVTList VTs = DAG.getVTList(Op0.getValueType(), MVT::i32); in EmitCmp()
18997 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op0, Op1); in EmitCmp()
19016 SDValue TruncFPSW = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Cmp); in ConvertCmpIfNecessary()
19017 SDValue FNStSW = DAG.getNode(X86ISD::FNSTSW16r, dl, MVT::i16, TruncFPSW); in ConvertCmpIfNecessary()
19018 SDValue Srl = DAG.getNode(ISD::SRL, dl, MVT::i16, FNStSW, in ConvertCmpIfNecessary()
19019 DAG.getConstant(8, dl, MVT::i8)); in ConvertCmpIfNecessary()
19020 SDValue TruncSrl = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Srl); in ConvertCmpIfNecessary()
19024 return DAG.getNode(X86ISD::SAHF, dl, MVT::i32, TruncSrl); in ConvertCmpIfNecessary()
19057 if ((VT == MVT::f32 && Subtarget.hasSSE1()) || in getSqrtEstimate()
19058 (VT == MVT::v4f32 && Subtarget.hasSSE1() && Reciprocal) || in getSqrtEstimate()
19059 (VT == MVT::v4f32 && Subtarget.hasSSE2() && !Reciprocal) || in getSqrtEstimate()
19060 (VT == MVT::v8f32 && Subtarget.hasAVX()) || in getSqrtEstimate()
19061 (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) { in getSqrtEstimate()
19067 unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RSQRT14 : X86ISD::FRSQRT; in getSqrtEstimate()
19087 if ((VT == MVT::f32 && Subtarget.hasSSE1()) || in getRecipEstimate()
19088 (VT == MVT::v4f32 && Subtarget.hasSSE1()) || in getRecipEstimate()
19089 (VT == MVT::v8f32 && Subtarget.hasAVX()) || in getRecipEstimate()
19090 (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) { in getRecipEstimate()
19094 if (VT == MVT::f32 && Enabled == ReciprocalEstimate::Unspecified) in getRecipEstimate()
19101 unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RCP14 : X86ISD::FRCP; in getRecipEstimate()
19177 if (Src.getValueType() == MVT::i8 || Src.getValueType() == MVT::i16) in LowerAndToBT()
19178 Src = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Src); in LowerAndToBT()
19184 if (Src.getValueType() == MVT::i64 && in LowerAndToBT()
19186 Src = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Src); in LowerAndToBT()
19194 dl, MVT::i8); in LowerAndToBT()
19195 return DAG.getNode(X86ISD::BT, dl, MVT::i32, Src, BitNo); in LowerAndToBT()
19246 MVT VT = Op.getSimpleValueType(); in Lower256IntVSETCC()
19266 MVT EltVT = VT.getVectorElementType(); in Lower256IntVSETCC()
19267 MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); in Lower256IntVSETCC()
19278 MVT VT = Op.getSimpleValueType(); in LowerIntVSETCC_AVX512()
19281 assert(VT.getVectorElementType() == MVT::i1 && in LowerIntVSETCC_AVX512()
19310 MVT VT = V.getSimpleValueType(); in decrementVectorConstant()
19311 MVT EltVT = VT.getVectorElementType(); in decrementVectorConstant()
19334 static SDValue LowerVSETCCWithSUBUS(SDValue Op0, SDValue Op1, MVT VT, in LowerVSETCCWithSUBUS()
19341 MVT VET = VT.getVectorElementType(); in LowerVSETCCWithSUBUS()
19342 if (VET != MVT::i8 && VET != MVT::i16) in LowerVSETCCWithSUBUS()
19380 MVT VT = Op.getSimpleValueType(); in LowerVSETCC()
19387 MVT EltVT = Op0.getSimpleValueType().getVectorElementType(); in LowerVSETCC()
19388 assert(EltVT == MVT::f32 || EltVT == MVT::f64); in LowerVSETCC()
19392 if (Subtarget.hasAVX512() && VT.getVectorElementType() == MVT::i1) { in LowerVSETCC()
19423 DAG.getConstant(CC0, dl, MVT::i8)); in LowerVSETCC()
19425 DAG.getConstant(CC1, dl, MVT::i8)); in LowerVSETCC()
19430 DAG.getConstant(SSECC, dl, MVT::i8)); in LowerVSETCC()
19442 MVT VTOp0 = Op0.getSimpleValueType(); in LowerVSETCC()
19450 if (VTOp0 == MVT::v2i32) in LowerVSETCC()
19463 if (VT.getVectorElementType() == MVT::i1) { in LowerVSETCC()
19494 DAG.getConstant(CmpMode, dl, MVT::i8)); in LowerVSETCC()
19594 if (VT == MVT::v2i64) { in LowerVSETCC()
19603 SB = DAG.getConstant(0x8000000080000000ULL, dl, MVT::v2i64); in LowerVSETCC()
19605 SB = DAG.getConstant(0x0000000080000000ULL, dl, MVT::v2i64); in LowerVSETCC()
19607 Op0 = DAG.getNode(ISD::XOR, dl, MVT::v2i64, Op0, SB); in LowerVSETCC()
19608 Op1 = DAG.getNode(ISD::XOR, dl, MVT::v2i64, Op1, SB); in LowerVSETCC()
19611 Op0 = DAG.getBitcast(MVT::v4i32, Op0); in LowerVSETCC()
19612 Op1 = DAG.getBitcast(MVT::v4i32, Op1); in LowerVSETCC()
19615 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
19616 SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
19621 SDValue EQHi = DAG.getVectorShuffle(MVT::v4i32, dl, EQ, EQ, MaskHi); in LowerVSETCC()
19622 SDValue GTLo = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo); in LowerVSETCC()
19623 SDValue GTHi = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi); in LowerVSETCC()
19625 SDValue Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, EQHi, GTLo); in LowerVSETCC()
19626 Result = DAG.getNode(ISD::OR, dl, MVT::v4i32, Result, GTHi); in LowerVSETCC()
19629 Result = DAG.getNOT(dl, Result, MVT::v4i32); in LowerVSETCC()
19640 Op0 = DAG.getBitcast(MVT::v4i32, Op0); in LowerVSETCC()
19641 Op1 = DAG.getBitcast(MVT::v4i32, Op1); in LowerVSETCC()
19644 SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
19648 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Result, Result, Mask); in LowerVSETCC()
19649 Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, Result, Shuf); in LowerVSETCC()
19652 Result = DAG.getNOT(dl, Result, MVT::v4i32); in LowerVSETCC()
19661 MVT EltVT = VT.getVectorElementType(); in LowerVSETCC()
19691 MVT VT = Op0.getSimpleValueType(); in EmitKORTEST()
19692 if (!(Subtarget.hasAVX512() && VT == MVT::v16i1) && in EmitKORTEST()
19693 !(Subtarget.hasDQI() && VT == MVT::v8i1) && in EmitKORTEST()
19694 !(Subtarget.hasBWI() && (VT == MVT::v32i1 || VT == MVT::v64i1))) in EmitKORTEST()
19714 X86CC = DAG.getConstant(X86Cond, dl, MVT::i8); in EmitKORTEST()
19715 return DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS); in EmitKORTEST()
19759 X86CC = DAG.getConstant(CCode, dl, MVT::i8); in emitFlagsForSetcc()
19773 X86CC = DAG.getConstant(CondCode, dl, MVT::i8); in emitFlagsForSetcc()
19779 MVT VT = Op.getSimpleValueType(); in LowerSETCC()
19783 assert(VT == MVT::i8 && "SetCC type must be 8-bit integer"); in LowerSETCC()
19794 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS); in LowerSETCC()
19810 Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32), in LowerSETCCCARRY()
19813 SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32); in LowerSETCCCARRY()
19860 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); in getX86XALUOOp()
19913 MVT VT = Op1.getSimpleValueType(); in LowerSELECT()
19920 ((Subtarget.hasSSE2() && VT == MVT::f64) || in LowerSELECT()
19921 (Subtarget.hasSSE1() && VT == MVT::f32)) && in LowerSELECT()
19928 SDValue Cmp = DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CondOp0, in LowerSELECT()
19929 CondOp1, DAG.getConstant(SSECC, DL, MVT::i8)); in LowerSELECT()
19936 DAG.getConstant(SSECC, DL, MVT::i8)); in LowerSELECT()
19954 MVT VecVT = VT == MVT::f32 ? MVT::v4f32 : MVT::v2f64; in LowerSELECT()
19959 MVT VCmpVT = VT == MVT::f32 ? MVT::v4i32 : MVT::v2i64; in LowerSELECT()
19974 if ((VT == MVT::f64 || VT == MVT::f32) && Subtarget.hasAVX512()) { in LowerSELECT()
19975 SDValue Cmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, Cond); in LowerSELECT()
19980 if (VT == MVT::v64i1 && !Subtarget.is64Bit()) { in LowerSELECT()
19986 SDValue Lo = DAG.getSelect(DL, MVT::v32i1, Cond, Op1Lo, Op2Lo); in LowerSELECT()
19987 SDValue Hi = DAG.getSelect(DL, MVT::v32i1, Cond, Op1Hi, Op2Hi); in LowerSELECT()
19991 if (VT.isVector() && VT.getVectorElementType() == MVT::i1) { in LowerSELECT()
20007 SDValue ExtVec = DAG.getBitcast(MVT::v8i1, newSelect); in LowerSELECT()
20048 SDValue Cmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, Zero, CmpOp0); in LowerSELECT()
20049 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); in LowerSELECT()
20054 Cmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, in LowerSELECT()
20058 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); in LowerSELECT()
20124 MVT VT = Op.getSimpleValueType(); in LowerSELECT()
20143 CC = DAG.getConstant(X86Cond, DL, MVT::i8); in LowerSELECT()
20165 CC = DAG.getConstant(X86::COND_NE, DL, MVT::i8); in LowerSELECT()
20182 DAG.getConstant(X86::COND_B, DL, MVT::i8), in LowerSELECT()
20193 if (Op.getValueType() == MVT::i8 && in LowerSELECT()
20206 if (Op.getValueType() == MVT::i16 && !MayFoldLoad(Op1) && !MayFoldLoad(Op2)) { in LowerSELECT()
20207 Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1); in LowerSELECT()
20208 Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2); in LowerSELECT()
20210 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, MVT::i32, Ops); in LowerSELECT()
20223 MVT VT = Op->getSimpleValueType(0); in LowerSIGN_EXTEND_Mask()
20225 MVT InVT = In.getSimpleValueType(); in LowerSIGN_EXTEND_Mask()
20226 assert(InVT.getVectorElementType() == MVT::i1 && "Unexpected input type!"); in LowerSIGN_EXTEND_Mask()
20227 MVT VTElt = VT.getVectorElementType(); in LowerSIGN_EXTEND_Mask()
20233 MVT ExtVT = VT; in LowerSIGN_EXTEND_Mask()
20239 ExtVT = MVT::getVectorVT(MVT::i32, NumElts); in LowerSIGN_EXTEND_Mask()
20243 MVT WideVT = ExtVT; in LowerSIGN_EXTEND_Mask()
20246 InVT = MVT::getVectorVT(MVT::i1, NumElts); in LowerSIGN_EXTEND_Mask()
20249 WideVT = MVT::getVectorVT(ExtVT.getVectorElementType(), NumElts); in LowerSIGN_EXTEND_Mask()
20253 MVT WideEltVT = WideVT.getVectorElementType(); in LowerSIGN_EXTEND_Mask()
20265 WideVT = MVT::getVectorVT(VTElt, NumElts); in LowerSIGN_EXTEND_Mask()
20280 MVT InVT = In.getSimpleValueType(); in LowerANY_EXTEND()
20282 if (InVT.getVectorElementType() == MVT::i1) in LowerANY_EXTEND()
20297 MVT VT = Op->getSimpleValueType(0); in LowerEXTEND_VECTOR_INREG()
20298 MVT InVT = In.getSimpleValueType(); in LowerEXTEND_VECTOR_INREG()
20300 MVT SVT = VT.getVectorElementType(); in LowerEXTEND_VECTOR_INREG()
20301 MVT InSVT = InVT.getVectorElementType(); in LowerEXTEND_VECTOR_INREG()
20304 if (SVT != MVT::i64 && SVT != MVT::i32 && SVT != MVT::i16) in LowerEXTEND_VECTOR_INREG()
20306 if (InSVT != MVT::i32 && InSVT != MVT::i16 && InSVT != MVT::i8) in LowerEXTEND_VECTOR_INREG()
20348 MVT HalfVT = MVT::getVectorVT(SVT, HalfNumElts); in LowerEXTEND_VECTOR_INREG()
20371 if (InVT != MVT::v4i32) { in LowerEXTEND_VECTOR_INREG()
20372 MVT DestVT = VT == MVT::v2i64 ? MVT::v4i32 : VT; in LowerEXTEND_VECTOR_INREG()
20391 DAG.getConstant(SignExtShift, dl, MVT::i8)); in LowerEXTEND_VECTOR_INREG()
20394 if (VT == MVT::v2i64) { in LowerEXTEND_VECTOR_INREG()
20395 assert(Curr.getValueType() == MVT::v4i32 && "Unexpected input VT"); in LowerEXTEND_VECTOR_INREG()
20396 SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32); in LowerEXTEND_VECTOR_INREG()
20397 SDValue Sign = DAG.getSetCC(dl, MVT::v4i32, Zero, Curr, ISD::SETGT); in LowerEXTEND_VECTOR_INREG()
20398 SignExt = DAG.getVectorShuffle(MVT::v4i32, dl, SignExt, Sign, {0, 4, 1, 5}); in LowerEXTEND_VECTOR_INREG()
20407 MVT VT = Op->getSimpleValueType(0); in LowerSIGN_EXTEND()
20409 MVT InVT = In.getSimpleValueType(); in LowerSIGN_EXTEND()
20412 if (InVT.getVectorElementType() == MVT::i1) in LowerSIGN_EXTEND()
20418 assert((VT.getVectorElementType() == MVT::i16 || in LowerSIGN_EXTEND()
20419 VT.getVectorElementType() == MVT::i32 || in LowerSIGN_EXTEND()
20420 VT.getVectorElementType() == MVT::i64) && in LowerSIGN_EXTEND()
20422 assert((InVT.getVectorElementType() == MVT::i8 || in LowerSIGN_EXTEND()
20423 InVT.getVectorElementType() == MVT::i16 || in LowerSIGN_EXTEND()
20424 InVT.getVectorElementType() == MVT::i32) && in LowerSIGN_EXTEND()
20428 if (InVT == MVT::v8i8) { in LowerSIGN_EXTEND()
20429 if (!ExperimentalVectorWideningLegalization || VT != MVT::v8i64) in LowerSIGN_EXTEND()
20433 MVT::v16i8, In, DAG.getUNDEF(MVT::v8i8)); in LowerSIGN_EXTEND()
20449 MVT HalfVT = MVT::getVectorVT(VT.getVectorElementType(), in LowerSIGN_EXTEND()
20473 StoredVal.getValueType().getVectorElementType() == MVT::i1) { in LowerStore()
20480 StoredVal = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1, in LowerStore()
20481 DAG.getUNDEF(MVT::v16i1), StoredVal, in LowerStore()
20483 StoredVal = DAG.getBitcast(MVT::i16, StoredVal); in LowerStore()
20484 StoredVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, StoredVal); in LowerStore()
20494 MVT StoreVT = StoredVal.getSimpleValueType(); in LowerStore()
20503 MVT WideVT = MVT::getVectorVT(StoreVT.getVectorElementType(), in LowerStore()
20507 MVT StVT = Subtarget.is64Bit() && StoreVT.isInteger() ? MVT::i64 : MVT::f64; in LowerStore()
20508 MVT CastVT = MVT::getVectorVT(StVT, 2); in LowerStore()
20527 MVT RegVT = Op.getSimpleValueType(); in LowerLoad()
20537 if (RegVT.getVectorElementType() == MVT::i1) { in LowerLoad()
20543 SDValue NewLd = DAG.getLoad(MVT::i8, dl, Ld->getChain(), Ld->getBasePtr(), in LowerLoad()
20550 SDValue Val = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, NewLd); in LowerLoad()
20552 DAG.getBitcast(MVT::v16i1, Val), in LowerLoad()
20622 MVT SclrLoadTy = MVT::i8; in LowerLoad()
20623 for (MVT Tp : MVT::integer_valuetypes()) { in LowerLoad()
20630 if (TLI.isTypeLegal(MVT::f64) && SclrLoadTy.getSizeInBits() < 64 && in LowerLoad()
20632 SclrLoadTy = MVT::f64; in LowerLoad()
20647 if (Ext == ISD::EXTLOAD && !Subtarget.hasBWI() && RegVT == MVT::v8i64 && in LowerLoad()
20648 MemVT == MVT::v8i8) in LowerLoad()
20698 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); in LowerLoad()
20710 if (Ext == ISD::EXTLOAD && !Subtarget.hasBWI() && RegVT == MVT::v8i64 && in LowerLoad()
20711 MemVT == MVT::v8i8) { in LowerLoad()
20831 CC = DAG.getConstant(X86Cond, dl, MVT::i8); in LowerBRCOND()
20862 CC = DAG.getConstant(CCode, dl, MVT::i8); in LowerBRCOND()
20880 CC = DAG.getConstant(CCode, dl, MVT::i8); in LowerBRCOND()
20893 CC = DAG.getConstant(CCode, dl, MVT::i8); in LowerBRCOND()
20916 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32, in LowerBRCOND()
20919 CC = DAG.getConstant(X86::COND_NE, dl, MVT::i8); in LowerBRCOND()
20922 CC = DAG.getConstant(X86::COND_P, dl, MVT::i8); in LowerBRCOND()
20932 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32, in LowerBRCOND()
20935 CC = DAG.getConstant(X86::COND_NE, dl, MVT::i8); in LowerBRCOND()
20938 CC = DAG.getConstant(X86::COND_P, dl, MVT::i8); in LowerBRCOND()
20963 CC = DAG.getConstant(X86Cond, dl, MVT::i8); in LowerBRCOND()
20999 MVT SPTy = getPointerTy(DAG.getDataLayout()); in LowerDYNAMIC_STACKALLOC()
21037 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerDYNAMIC_STACKALLOC()
21089 DAG.getConstant(FuncInfo->getVarArgsGPOffset(), DL, MVT::i32), FIN, in LowerVASTART()
21097 DAG.getConstant(FuncInfo->getVarArgsFPOffset(), DL, MVT::i32), FIN, in LowerVASTART()
21116 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in LowerVASTART()
21143 if (ArgVT == MVT::f80) { in LowerVAARG()
21162 SDValue InstOps[] = {Chain, SrcPtr, DAG.getConstant(ArgSize, dl, MVT::i32), in LowerVAARG()
21163 DAG.getConstant(ArgMode, dl, MVT::i8), in LowerVAARG()
21164 DAG.getConstant(Align, dl, MVT::i32)}; in LowerVAARG()
21165 SDVTList VTs = DAG.getVTList(getPointerTy(DAG.getDataLayout()), MVT::Other); in LowerVAARG()
21168 VTs, InstOps, MVT::i64, in LowerVAARG()
21222 static SDValue getTargetVShiftByConstNode(unsigned Opc, const SDLoc &dl, MVT VT, in getTargetVShiftByConstNode()
21225 MVT ElementType = VT.getVectorElementType(); in getTargetVShiftByConstNode()
21298 DAG.getConstant(ShiftAmt, dl, MVT::i8)); in getTargetVShiftByConstNode()
21303 static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT, in getTargetVShiftNode()
21307 MVT SVT = ShAmt.getSimpleValueType(); in getTargetVShiftNode()
21308 assert((SVT == MVT::i32 || SVT == MVT::i64) && "Unexpected value type!"); in getTargetVShiftNode()
21330 if (SVT == MVT::i64) in getTargetVShiftNode()
21331 ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(ShAmt), MVT::v2i64, ShAmt); in getTargetVShiftNode()
21334 (ShAmt.getOperand(0).getSimpleValueType() == MVT::i16 || in getTargetVShiftNode()
21335 ShAmt.getOperand(0).getSimpleValueType() == MVT::i8)) { in getTargetVShiftNode()
21337 MVT AmtTy = ShAmt.getSimpleValueType() == MVT::i8 ? MVT::v16i8 : MVT::v8i16; in getTargetVShiftNode()
21341 MVT::v2i64, ShAmt); in getTargetVShiftNode()
21344 (128 - AmtTy.getScalarSizeInBits()) / 8, SDLoc(ShAmt), MVT::i8); in getTargetVShiftNode()
21345 ShAmt = DAG.getBitcast(MVT::v16i8, ShAmt); in getTargetVShiftNode()
21346 ShAmt = DAG.getNode(X86ISD::VSHLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt, in getTargetVShiftNode()
21348 ShAmt = DAG.getNode(X86ISD::VSRLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt, in getTargetVShiftNode()
21353 ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(ShAmt), MVT::v4i32, ShAmt); in getTargetVShiftNode()
21355 MVT::v2i64, ShAmt); in getTargetVShiftNode()
21359 ShAmt = DAG.getBuildVector(MVT::v4i32, dl, ShOps); in getTargetVShiftNode()
21364 MVT EltVT = VT.getVectorElementType(); in getTargetVShiftNode()
21365 MVT ShVT = MVT::getVectorVT(EltVT, 128 / EltVT.getSizeInBits()); in getTargetVShiftNode()
21373 static SDValue getMaskNode(SDValue Mask, MVT MaskVT, in getMaskNode()
21384 if (Mask.getSimpleValueType() == MVT::i64 && Subtarget.is32Bit()) { in getMaskNode()
21385 assert(MaskVT == MVT::v64i1 && "Expected v64i1 mask!"); in getMaskNode()
21389 Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Mask, in getMaskNode()
21390 DAG.getConstant(0, dl, MVT::i32)); in getMaskNode()
21391 Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Mask, in getMaskNode()
21392 DAG.getConstant(1, dl, MVT::i32)); in getMaskNode()
21394 Lo = DAG.getBitcast(MVT::v32i1, Lo); in getMaskNode()
21395 Hi = DAG.getBitcast(MVT::v32i1, Hi); in getMaskNode()
21397 return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi); in getMaskNode()
21399 MVT BitcastVT = MVT::getVectorVT(MVT::i1, in getMaskNode()
21416 MVT VT = Op.getSimpleValueType(); in getVectorMaskingNode()
21417 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements()); in getVectorMaskingNode()
21447 MVT VT = Op.getSimpleValueType(); in getScalarMaskingNode()
21450 assert(Mask.getValueType() == MVT::i8 && "Unexpect type"); in getScalarMaskingNode()
21451 SDValue IMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v1i1, in getScalarMaskingNode()
21452 DAG.getBitcast(MVT::v8i1, Mask), in getScalarMaskingNode()
21493 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); in recoverFramePointer()
21538 MVT VT = Op.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN()
21581 Src3 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Src3); in LowerINTRINSIC_WO_CHAIN()
21612 DAG.getConstant(X86::STATIC_ROUNDING::CUR_DIRECTION, dl, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
21735 Rnd = DAG.getConstant(X86::STATIC_ROUNDING::CUR_DIRECTION, dl, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
21828 SDValue FPclass = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Imm); in LowerINTRINSIC_WO_CHAIN()
21833 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1, in LowerINTRINSIC_WO_CHAIN()
21834 DAG.getConstant(0, dl, MVT::v8i1), in LowerINTRINSIC_WO_CHAIN()
21836 return DAG.getBitcast(MVT::i8, Ins); in LowerINTRINSIC_WO_CHAIN()
21840 MVT MaskVT = Op.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN()
21843 CC = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, CC); in LowerINTRINSIC_WO_CHAIN()
21863 SDValue CC = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op.getOperand(3)); in LowerINTRINSIC_WO_CHAIN()
21870 Cmp = DAG.getNode(IntrData->Opc1, dl, MVT::v1i1, Src1, Src2, CC, Rnd); in LowerINTRINSIC_WO_CHAIN()
21874 Cmp = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Src2, CC); in LowerINTRINSIC_WO_CHAIN()
21880 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1, in LowerINTRINSIC_WO_CHAIN()
21881 DAG.getConstant(0, dl, MVT::v8i1), in LowerINTRINSIC_WO_CHAIN()
21883 return DAG.getBitcast(MVT::i8, Ins); in LowerINTRINSIC_WO_CHAIN()
21889 SDValue Comi = DAG.getNode(IntrData->Opc0, dl, MVT::i32, LHS, RHS); in LowerINTRINSIC_WO_CHAIN()
21890 SDValue InvComi = DAG.getNode(IntrData->Opc0, dl, MVT::i32, RHS, LHS); in LowerINTRINSIC_WO_CHAIN()
21896 SetCC = DAG.getNode(ISD::AND, dl, MVT::i8, SetCC, SetNP); in LowerINTRINSIC_WO_CHAIN()
21902 SetCC = DAG.getNode(ISD::OR, dl, MVT::i8, SetCC, SetP); in LowerINTRINSIC_WO_CHAIN()
21921 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
21931 FCmp = DAG.getNode(X86ISD::FSETCCM, dl, MVT::v1i1, LHS, RHS, in LowerINTRINSIC_WO_CHAIN()
21932 DAG.getConstant(CondVal, dl, MVT::i8)); in LowerINTRINSIC_WO_CHAIN()
21934 FCmp = DAG.getNode(X86ISD::FSETCCM_RND, dl, MVT::v1i1, LHS, RHS, in LowerINTRINSIC_WO_CHAIN()
21935 DAG.getConstant(CondVal, dl, MVT::i8), Sae); in LowerINTRINSIC_WO_CHAIN()
21938 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1, in LowerINTRINSIC_WO_CHAIN()
21939 DAG.getConstant(0, dl, MVT::v16i1), in LowerINTRINSIC_WO_CHAIN()
21941 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, in LowerINTRINSIC_WO_CHAIN()
21942 DAG.getBitcast(MVT::i16, Ins)); in LowerINTRINSIC_WO_CHAIN()
21978 Rnd = DAG.getConstant(X86::STATIC_ROUNDING::CUR_DIRECTION, dl, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
21992 SDValue RoundingMode = DAG.getNode(ISD::AND, dl, MVT::i32, in LowerINTRINSIC_WO_CHAIN()
21994 DAG.getConstant(0xf, dl, MVT::i32)); in LowerINTRINSIC_WO_CHAIN()
22002 SDValue RoundingMode = DAG.getNode(ISD::AND, dl, MVT::i32, in LowerINTRINSIC_WO_CHAIN()
22004 DAG.getConstant(0xf, dl, MVT::i32)); in LowerINTRINSIC_WO_CHAIN()
22010 SDVTList CFVTs = DAG.getVTList(Op->getValueType(0), MVT::i32); in LowerINTRINSIC_WO_CHAIN()
22011 SDVTList VTs = DAG.getVTList(Op.getOperand(2).getValueType(), MVT::i32); in LowerINTRINSIC_WO_CHAIN()
22021 DAG.getConstant(-1, dl, MVT::i8)); in LowerINTRINSIC_WO_CHAIN()
22039 MVT SrcVT = Src.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN()
22040 MVT MaskVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorNumElements()); in LowerINTRINSIC_WO_CHAIN()
22054 MVT SrcVT = Src.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN()
22055 MVT MaskVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorNumElements()); in LowerINTRINSIC_WO_CHAIN()
22154 SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS); in LowerINTRINSIC_WO_CHAIN()
22156 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
22215 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
22218 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
22230 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
22243 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
22250 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); in LowerINTRINSIC_WO_CHAIN()
22307 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, MVT::i8); in getAVX2GatherNode()
22309 SDVTList VTs = DAG.getVTList(Op.getValueType(), MaskVT, MVT::Other); in getAVX2GatherNode()
22310 SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32); in getAVX2GatherNode()
22311 SDValue Segment = DAG.getRegister(0, MVT::i32); in getAVX2GatherNode()
22327 MVT VT = Op.getSimpleValueType(); in getGatherNode()
22333 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, MVT::i8); in getGatherNode()
22336 MVT MaskVT = MVT::getVectorVT(MVT::i1, MinElts); in getGatherNode()
22343 SDVTList VTs = DAG.getVTList(Op.getValueType(), MaskVT, MVT::Other); in getGatherNode()
22344 SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32); in getGatherNode()
22345 SDValue Segment = DAG.getRegister(0, MVT::i32); in getGatherNode()
22366 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, MVT::i8); in getScatterNode()
22367 SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32); in getScatterNode()
22368 SDValue Segment = DAG.getRegister(0, MVT::i32); in getScatterNode()
22371 MVT MaskVT = MVT::getVectorVT(MVT::i1, MinElts); in getScatterNode()
22378 SDVTList VTs = DAG.getVTList(MaskVT, MVT::Other); in getScatterNode()
22393 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, MVT::i8); in getPrefetchNode()
22394 SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32); in getPrefetchNode()
22395 SDValue Segment = DAG.getRegister(0, MVT::i32); in getPrefetchNode()
22396 MVT MaskVT = in getPrefetchNode()
22397 MVT::getVectorVT(MVT::i1, Index.getSimpleValueType().getVectorNumElements()); in getPrefetchNode()
22400 SDNode *Res = DAG.getMachineNode(Opc, dl, MVT::Other, Ops); in getPrefetchNode()
22411 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in getExtendedControlRegister()
22423 LO = DAG.getCopyFromReg(Chain, DL, X86::RAX, MVT::i64, SDValue(N1, 1)); in getExtendedControlRegister()
22424 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64, in getExtendedControlRegister()
22427 LO = DAG.getCopyFromReg(Chain, DL, X86::EAX, MVT::i32, SDValue(N1, 1)); in getExtendedControlRegister()
22428 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32, in getExtendedControlRegister()
22435 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI, in getExtendedControlRegister()
22436 DAG.getConstant(32, DL, MVT::i8)); in getExtendedControlRegister()
22437 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp)); in getExtendedControlRegister()
22444 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops); in getExtendedControlRegister()
22456 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in getReadPerformanceCounter()
22468 LO = DAG.getCopyFromReg(rd, DL, X86::RAX, MVT::i64, rd.getValue(1)); in getReadPerformanceCounter()
22469 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64, in getReadPerformanceCounter()
22472 LO = DAG.getCopyFromReg(rd, DL, X86::EAX, MVT::i32, rd.getValue(1)); in getReadPerformanceCounter()
22473 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32, in getReadPerformanceCounter()
22481 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI, in getReadPerformanceCounter()
22482 DAG.getConstant(32, DL, MVT::i8)); in getReadPerformanceCounter()
22483 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp)); in getReadPerformanceCounter()
22490 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops); in getReadPerformanceCounter()
22502 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in getReadTimeStampCounter()
22510 LO = DAG.getCopyFromReg(rd, DL, X86::RAX, MVT::i64, rd.getValue(1)); in getReadTimeStampCounter()
22511 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64, in getReadTimeStampCounter()
22514 LO = DAG.getCopyFromReg(rd, DL, X86::EAX, MVT::i32, rd.getValue(1)); in getReadTimeStampCounter()
22515 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32, in getReadTimeStampCounter()
22524 TSC = DAG.getNode(ISD::SHL, DL, MVT::i64, HI, in getReadTimeStampCounter()
22525 DAG.getConstant(32, DL, MVT::i8)); in getReadTimeStampCounter()
22526 TSC = DAG.getNode(ISD::OR, DL, MVT::i64, LO, TSC); in getReadTimeStampCounter()
22529 TSC = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, { LO, HI }); in getReadTimeStampCounter()
22537 SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32, in getReadTimeStampCounter()
22601 SDVTList VTs = DAG.getVTList(MVT::Other); in EmitTruncSStore()
22615 SDVTList VTs = DAG.getVTList(MVT::Other); in EmitMaskedTruncSStore()
22651 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other); in LowerINTRINSIC_W_CHAIN()
22672 SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i8, SetCC); in LowerINTRINSIC_W_CHAIN()
22686 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32, MVT::Other); in LowerINTRINSIC_W_CHAIN()
22693 DAG.getConstant(X86::COND_B, dl, MVT::i8), in LowerINTRINSIC_W_CHAIN()
22768 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Other); in LowerINTRINSIC_W_CHAIN()
22796 MVT MaskVT = MVT::getVectorVT(MVT::i1, MemVT.getVectorNumElements()); in LowerINTRINSIC_W_CHAIN()
22809 MVT MaskVT = MVT::getVectorVT(MVT::i1, MemVT.getVectorNumElements()); in LowerINTRINSIC_W_CHAIN()
22883 assert(((FrameReg == X86::RBP && VT == MVT::i64) || in LowerFRAMEADDR()
22884 (FrameReg == X86::EBP && VT == MVT::i32)) && in LowerFRAMEADDR()
22962 assert(((FrameReg == X86::RBP && PtrVT == MVT::i64) || in LowerEH_RETURN()
22963 (FrameReg == X86::EBP && PtrVT == MVT::i32)) && in LowerEH_RETURN()
22966 unsigned StoreAddrReg = (PtrVT == MVT::i64) ? X86::RCX : X86::ECX; in LowerEH_RETURN()
22975 return DAG.getNode(X86ISD::EH_RETURN, dl, MVT::Other, Chain, in LowerEH_RETURN()
22993 DAG.getVTList(MVT::i32, MVT::Other), in lowerEH_SJLJ_SETJMP()
23000 return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other, in lowerEH_SJLJ_LONGJMP()
23007 return DAG.getNode(X86ISD::EH_SJLJ_SETUP_DISPATCH, DL, MVT::Other, in lowerEH_SJLJ_SETUP_DISPATCH()
23041 OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16), in LowerINIT_TRAMPOLINE()
23044 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
23045 DAG.getConstant(2, dl, MVT::i64)); in LowerINIT_TRAMPOLINE()
23053 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
23054 DAG.getConstant(10, dl, MVT::i64)); in LowerINIT_TRAMPOLINE()
23055 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16), in LowerINIT_TRAMPOLINE()
23058 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
23059 DAG.getConstant(12, dl, MVT::i64)); in LowerINIT_TRAMPOLINE()
23066 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
23067 DAG.getConstant(20, dl, MVT::i64)); in LowerINIT_TRAMPOLINE()
23068 OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16), in LowerINIT_TRAMPOLINE()
23072 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
23073 DAG.getConstant(22, dl, MVT::i64)); in LowerINIT_TRAMPOLINE()
23074 OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, dl, MVT::i8), in LowerINIT_TRAMPOLINE()
23077 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in LowerINIT_TRAMPOLINE()
23128 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
23129 DAG.getConstant(10, dl, MVT::i32)); in LowerINIT_TRAMPOLINE()
23130 Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr); in LowerINIT_TRAMPOLINE()
23136 DAG.getStore(Root, dl, DAG.getConstant(MOV32ri | N86Reg, dl, MVT::i8), in LowerINIT_TRAMPOLINE()
23139 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
23140 DAG.getConstant(1, dl, MVT::i32)); in LowerINIT_TRAMPOLINE()
23146 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
23147 DAG.getConstant(5, dl, MVT::i32)); in LowerINIT_TRAMPOLINE()
23148 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(JMP, dl, MVT::i8), in LowerINIT_TRAMPOLINE()
23152 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
23153 DAG.getConstant(6, dl, MVT::i32)); in LowerINIT_TRAMPOLINE()
23158 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in LowerINIT_TRAMPOLINE()
23186 MVT VT = Op.getSimpleValueType(); in LowerFLT_ROUNDS_()
23200 DAG.getVTList(MVT::Other), in LowerFLT_ROUNDS_()
23201 Ops, MVT::i16, MMO); in LowerFLT_ROUNDS_()
23205 DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MachinePointerInfo()); in LowerFLT_ROUNDS_()
23209 DAG.getNode(ISD::SRL, DL, MVT::i16, in LowerFLT_ROUNDS_()
23210 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerFLT_ROUNDS_()
23211 CWD, DAG.getConstant(0x800, DL, MVT::i16)), in LowerFLT_ROUNDS_()
23212 DAG.getConstant(11, DL, MVT::i8)); in LowerFLT_ROUNDS_()
23214 DAG.getNode(ISD::SRL, DL, MVT::i16, in LowerFLT_ROUNDS_()
23215 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerFLT_ROUNDS_()
23216 CWD, DAG.getConstant(0x400, DL, MVT::i16)), in LowerFLT_ROUNDS_()
23217 DAG.getConstant(9, DL, MVT::i8)); in LowerFLT_ROUNDS_()
23220 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerFLT_ROUNDS_()
23221 DAG.getNode(ISD::ADD, DL, MVT::i16, in LowerFLT_ROUNDS_()
23222 DAG.getNode(ISD::OR, DL, MVT::i16, CWD1, CWD2), in LowerFLT_ROUNDS_()
23223 DAG.getConstant(1, DL, MVT::i16)), in LowerFLT_ROUNDS_()
23224 DAG.getConstant(3, DL, MVT::i16)); in LowerFLT_ROUNDS_()
23232 MVT VT = Op.getSimpleValueType(); in LowerVectorIntUnary()
23235 MVT EltVT = VT.getVectorElementType(); in LowerVectorIntUnary()
23245 MVT NewVT = MVT::getVectorVT(EltVT, NumElems / 2); in LowerVectorIntUnary()
23277 MVT VT = Op.getSimpleValueType(); in LowerVectorCTLZ_AVX512CDI()
23278 MVT EltVT = VT.getVectorElementType(); in LowerVectorCTLZ_AVX512CDI()
23281 assert((EltVT == MVT::i8 || EltVT == MVT::i16) && in LowerVectorCTLZ_AVX512CDI()
23289 MVT NewVT = MVT::getVectorVT(MVT::i32, NumElems); in LowerVectorCTLZ_AVX512CDI()
23306 MVT VT = Op.getSimpleValueType(); in LowerVectorCTLZInRegLUT()
23309 MVT CurrVT = MVT::getVectorVT(MVT::i8, NumBytes); in LowerVectorCTLZInRegLUT()
23319 LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8)); in LowerVectorCTLZInRegLUT()
23335 MVT MaskVT = MVT::getVectorVT(MVT::i1, CurrVT.getVectorNumElements()); in LowerVectorCTLZInRegLUT()
23355 MVT NextSVT = MVT::getIntegerVT(CurrScalarSizeInBits * 2); in LowerVectorCTLZInRegLUT()
23356 MVT NextVT = MVT::getVectorVT(NextSVT, CurrNumElts / 2); in LowerVectorCTLZInRegLUT()
23361 MVT MaskVT = MVT::getVectorVT(MVT::i1, CurrVT.getVectorNumElements()); in LowerVectorCTLZInRegLUT()
23388 MVT VT = Op.getSimpleValueType(); in LowerVectorCTLZ()
23392 (Subtarget.canExtendTo512DQ() || VT.getVectorElementType() != MVT::i8)) in LowerVectorCTLZ()
23409 MVT VT = Op.getSimpleValueType(); in LowerCTLZ()
23410 MVT OpVT = VT; in LowerCTLZ()
23419 if (VT == MVT::i8) { in LowerCTLZ()
23421 OpVT = MVT::i32; in LowerCTLZ()
23426 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); in LowerCTLZ()
23434 DAG.getConstant(X86::COND_E, dl, MVT::i8), in LowerCTLZ()
23444 if (VT == MVT::i8) in LowerCTLZ()
23445 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op); in LowerCTLZ()
23451 MVT VT = Op.getSimpleValueType(); in LowerCTTZ()
23464 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerCTTZ()
23471 DAG.getConstant(X86::COND_E, dl, MVT::i8), in LowerCTTZ()
23480 MVT VT = Op.getSimpleValueType(); in split256IntArith()
23498 MVT EltVT = VT.getVectorElementType(); in split256IntArith()
23499 MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); in split256IntArith()
23509 MVT VT = Op.getSimpleValueType(); in split512IntArith()
23527 MVT EltVT = VT.getVectorElementType(); in split512IntArith()
23528 MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); in split512IntArith()
23537 MVT VT = Op.getSimpleValueType(); in lowerAddSub()
23538 if (VT == MVT::i16 || VT == MVT::i32) in lowerAddSub()
23541 if (VT.getScalarType() == MVT::i1) in lowerAddSub()
23552 MVT VT = Op.getSimpleValueType(); in LowerADDSAT_SUBSAT()
23553 if (VT.getScalarType() == MVT::i1) { in LowerADDSAT_SUBSAT()
23575 MVT VT = Op.getSimpleValueType(); in LowerABS()
23576 if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) { in LowerABS()
23581 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32), in LowerABS()
23583 SDValue Ops[] = {N0, Neg, DAG.getConstant(X86::COND_GE, DL, MVT::i8), in LowerABS()
23589 if ((VT == MVT::v2i64 || VT == MVT::v4i64) && Subtarget.hasSSE41()) { in LowerABS()
23608 MVT VT = Op.getSimpleValueType(); in LowerMINMAX()
23611 if (VT.getScalarType() != MVT::i64 && VT.is256BitVector()) in LowerMINMAX()
23621 if (VT == MVT::v8i16) { in LowerMINMAX()
23649 MVT VT = Op.getSimpleValueType(); in LowerMUL()
23651 if (VT.getScalarType() == MVT::i1) in LowerMUL()
23663 if (VT == MVT::v16i8 || VT == MVT::v32i8 || VT == MVT::v64i8) { in LowerMUL()
23666 if ((VT == MVT::v16i8 && Subtarget.hasInt256()) || in LowerMUL()
23667 (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) { in LowerMUL()
23668 MVT ExVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements()); in LowerMUL()
23676 MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts / 2); in LowerMUL()
23693 MVT::i16)); in LowerMUL()
23695 MVT::i16)); in LowerMUL()
23715 if (VT == MVT::v4i32) { in LowerMUL()
23725 SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, in LowerMUL()
23726 DAG.getBitcast(MVT::v2i64, A), in LowerMUL()
23727 DAG.getBitcast(MVT::v2i64, B)); in LowerMUL()
23729 SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, in LowerMUL()
23730 DAG.getBitcast(MVT::v2i64, Aodds), in LowerMUL()
23731 DAG.getBitcast(MVT::v2i64, Bodds)); in LowerMUL()
23742 assert((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) && in LowerMUL()
23794 MVT VT = Op.getSimpleValueType(); in LowerMULH()
23804 if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) { in LowerMULH()
23805 assert((VT == MVT::v4i32 && Subtarget.hasSSE2()) || in LowerMULH()
23806 (VT == MVT::v8i32 && Subtarget.hasInt256()) || in LowerMULH()
23807 (VT == MVT::v16i32 && Subtarget.hasAVX512())); in LowerMULH()
23832 MVT MulVT = MVT::getVectorVT(MVT::i64, NumElts / 2); in LowerMULH()
23870 assert((VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) || in LowerMULH()
23871 (VT == MVT::v64i8 && Subtarget.hasBWI())) && in LowerMULH()
23881 if ((VT == MVT::v16i8 && Subtarget.hasInt256()) || in LowerMULH()
23882 (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) { in LowerMULH()
23883 MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts); in LowerMULH()
23893 if (VT == MVT::v64i8 && IsSigned) in LowerMULH()
23897 if (VT == MVT::v32i8 && IsSigned) { in LowerMULH()
23901 MVT ExVT = MVT::v16i16; in LowerMULH()
23930 MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts / 2); in LowerMULH()
23943 if (IsSigned && VT == MVT::v16i8 && Subtarget.hasSSE41()) { in LowerMULH()
23971 LoOp = DAG.getSExtOrTrunc(LoOp, dl, MVT::i16); in LowerMULH()
23972 HiOp = DAG.getSExtOrTrunc(HiOp, dl, MVT::i16); in LowerMULH()
23974 LoOp = DAG.getZExtOrTrunc(LoOp, dl, MVT::i16); in LowerMULH()
23975 HiOp = DAG.getZExtOrTrunc(HiOp, dl, MVT::i16); in LowerMULH()
23985 } else if (IsSigned && VT == MVT::v16i8 && Subtarget.hasSSE41()) { in LowerMULH()
24060 static_cast<EVT>(MVT::v2i64).getTypeForEVT(*DAG.getContext()), Callee, in LowerWin64_i128OP()
24072 static bool SupportedVectorShiftWithImm(MVT VT, const X86Subtarget &Subtarget, in SupportedVectorShiftWithImm()
24085 (VT != MVT::v2i64 && VT != MVT::v4i64)); in SupportedVectorShiftWithImm()
24092 bool SupportedVectorShiftWithBaseAmnt(MVT VT, const X86Subtarget &Subtarget, in SupportedVectorShiftWithBaseAmnt()
24099 static bool SupportedVectorVarShift(MVT VT, const X86Subtarget &Subtarget, in SupportedVectorVarShift()
24113 bool AShift = LShift && VT != MVT::v2i64 && VT != MVT::v4i64; in SupportedVectorVarShift()
24119 MVT VT = Op.getSimpleValueType(); in LowerScalarImmediateShift()
24126 assert((VT == MVT::v2i64 || VT == MVT::v4i64) && "Unexpected SRA type"); in LowerScalarImmediateShift()
24127 MVT ExVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() * 2); in LowerScalarImmediateShift()
24132 assert((VT != MVT::v4i64 || Subtarget.hasInt256()) && in LowerScalarImmediateShift()
24143 if (VT == MVT::v2i64) in LowerScalarImmediateShift()
24145 if (VT == MVT::v4i64) in LowerScalarImmediateShift()
24155 if (VT == MVT::v2i64) in LowerScalarImmediateShift()
24157 if (VT == MVT::v4i64) in LowerScalarImmediateShift()
24174 if (((!Subtarget.hasXOP() && VT == MVT::v2i64) || in LowerScalarImmediateShift()
24175 (Subtarget.hasInt256() && VT == MVT::v4i64)) && in LowerScalarImmediateShift()
24179 if (VT == MVT::v16i8 || (Subtarget.hasInt256() && VT == MVT::v32i8) || in LowerScalarImmediateShift()
24180 VT == MVT::v64i8) { in LowerScalarImmediateShift()
24182 MVT ShiftVT = MVT::getVectorVT(MVT::i16, NumElts / 2); in LowerScalarImmediateShift()
24192 assert(VT == MVT::v64i8 && "Unexpected element type!"); in LowerScalarImmediateShift()
24193 SDValue CMP = DAG.getSetCC(dl, MVT::v64i1, Zeros, R, ISD::SETGT); in LowerScalarImmediateShift()
24200 if (VT == MVT::v16i8 && Subtarget.hasXOP()) in LowerScalarImmediateShift()
24286 MVT VT = Op.getSimpleValueType(); in LowerScalarVariableShift()
24296 MVT EltVT = VT.getVectorElementType(); in LowerScalarVariableShift()
24297 assert(EltVT.bitsLE(MVT::i64) && "Unexpected element type!"); in LowerScalarVariableShift()
24298 if (EltVT != MVT::i64 && EltVT.bitsGT(MVT::i32)) in LowerScalarVariableShift()
24299 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, BaseShAmt); in LowerScalarVariableShift()
24300 else if (EltVT.bitsLT(MVT::i32)) in LowerScalarVariableShift()
24301 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, BaseShAmt); in LowerScalarVariableShift()
24307 if (((VT == MVT::v16i8 && !Subtarget.canExtendTo512DQ()) || in LowerScalarVariableShift()
24308 (VT == MVT::v32i8 && !Subtarget.canExtendTo512BW()) || in LowerScalarVariableShift()
24309 VT == MVT::v64i8) && in LowerScalarVariableShift()
24312 MVT ExtVT = MVT::getVectorVT(MVT::i16, NumElts / 2); in LowerScalarVariableShift()
24316 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, BaseShAmt); in LowerScalarVariableShift()
24352 if (VT == MVT::v2i64 && Amt.getOpcode() == ISD::BITCAST && in LowerScalarVariableShift()
24375 MVT VT = Amt.getSimpleValueType(); in convertShiftLeftToScale()
24376 if (!(VT == MVT::v8i16 || VT == MVT::v4i32 || in convertShiftLeftToScale()
24377 (Subtarget.hasInt256() && VT == MVT::v16i16) || in convertShiftLeftToScale()
24378 (!Subtarget.hasAVX512() && VT == MVT::v16i8))) in convertShiftLeftToScale()
24383 MVT SVT = VT.getVectorElementType(); in convertShiftLeftToScale()
24409 if (VT == MVT::v4i32) { in convertShiftLeftToScale()
24413 Amt = DAG.getBitcast(MVT::v4f32, Amt); in convertShiftLeftToScale()
24418 if (VT == MVT::v8i16 && !Subtarget.hasAVX2()) { in convertShiftLeftToScale()
24420 SDValue Lo = DAG.getBitcast(MVT::v4i32, getUnpackl(DAG, dl, VT, Amt, Z)); in convertShiftLeftToScale()
24421 SDValue Hi = DAG.getBitcast(MVT::v4i32, getUnpackh(DAG, dl, VT, Amt, Z)); in convertShiftLeftToScale()
24437 MVT VT = Op.getSimpleValueType(); in LowerShift()
24462 if (Subtarget.hasXOP() && (VT == MVT::v2i64 || VT == MVT::v4i32 || in LowerShift()
24463 VT == MVT::v8i16 || VT == MVT::v16i8)) { in LowerShift()
24476 if (VT == MVT::v2i64 && Opc != ISD::SRA) { in LowerShift()
24488 if ((VT == MVT::v2i64 || (VT == MVT::v4i64 && Subtarget.hasInt256())) && in LowerShift()
24508 if (ConstantAmt && (VT == MVT::v8i16 || VT == MVT::v4i32 || in LowerShift()
24509 (VT == MVT::v16i16 && Subtarget.hasInt256()))) { in LowerShift()
24534 (VT != MVT::v16i16 || in LowerShift()
24536 (VT == MVT::v4i32 || Subtarget.hasSSE41() || Opc != ISD::SHL || in LowerShift()
24560 (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256()))) { in LowerShift()
24576 (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256())) && in LowerShift()
24600 if (VT == MVT::v4i32) { in LowerShift()
24619 SDValue Amt01 = DAG.getBitcast(MVT::v8i16, Amt); in LowerShift()
24620 SDValue Amt23 = DAG.getVectorShuffle(MVT::v8i16, dl, Amt01, Amt01, in LowerShift()
24622 Amt0 = DAG.getVectorShuffle(MVT::v8i16, dl, Amt01, Amt01, in LowerShift()
24624 Amt1 = DAG.getVectorShuffle(MVT::v8i16, dl, Amt01, Amt01, in LowerShift()
24626 Amt2 = DAG.getVectorShuffle(MVT::v8i16, dl, Amt23, Amt23, in LowerShift()
24628 Amt3 = DAG.getVectorShuffle(MVT::v8i16, dl, Amt23, Amt23, in LowerShift()
24655 if ((Subtarget.hasInt256() && VT == MVT::v8i16) || in LowerShift()
24656 (Subtarget.canExtendTo512DQ() && VT == MVT::v16i16) || in LowerShift()
24657 (Subtarget.canExtendTo512DQ() && VT == MVT::v16i8) || in LowerShift()
24658 (Subtarget.canExtendTo512BW() && VT == MVT::v32i8) || in LowerShift()
24659 (Subtarget.hasBWI() && Subtarget.hasVLX() && VT == MVT::v16i8)) { in LowerShift()
24660 assert((!Subtarget.hasBWI() || VT == MVT::v32i8 || VT == MVT::v16i8) && in LowerShift()
24662 MVT EvtSVT = Subtarget.hasBWI() ? MVT::i16 : MVT::i32; in LowerShift()
24663 MVT ExtVT = MVT::getVectorVT(EvtSVT, VT.getVectorNumElements()); in LowerShift()
24674 (VT == MVT::v16i8 || VT == MVT::v64i8 || in LowerShift()
24675 (VT == MVT::v32i8 && Subtarget.hasInt256())) && in LowerShift()
24678 SDValue Cst8 = DAG.getConstant(8, dl, MVT::i8); in LowerShift()
24682 MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts); in LowerShift()
24689 if (VT == MVT::v16i8 && Subtarget.hasInt256()) { in LowerShift()
24705 MVT VT16 = MVT::getVectorVT(MVT::i16, NumElts / 2); in LowerShift()
24720 if (VT == MVT::v16i8 || in LowerShift()
24721 (VT == MVT::v32i8 && Subtarget.hasInt256() && !Subtarget.hasXOP()) || in LowerShift()
24722 (VT == MVT::v64i8 && Subtarget.hasBWI())) { in LowerShift()
24723 MVT ExtVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements() / 2); in LowerShift()
24725 auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) { in LowerShift()
24730 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements()); in LowerShift()
24828 if (Subtarget.hasInt256() && !Subtarget.hasXOP() && VT == MVT::v16i16) { in LowerShift()
24829 MVT ExtVT = MVT::v8i32; in LowerShift()
24846 if (VT == MVT::v8i16) { in LowerShift()
24856 MVT ExtVT = MVT::getVectorVT(MVT::i8, VT.getVectorNumElements() * 2); in LowerShift()
24918 MVT VT = Op.getSimpleValueType(); in LowerRotate()
24950 DAG.getConstant(RotateAmt, DL, MVT::i8)); in LowerRotate()
24971 DAG.getConstant(RotateAmt, DL, MVT::i8)); in LowerRotate()
24982 assert((VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 || in LowerRotate()
24983 ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8) && in LowerRotate()
25000 MVT ExtVT = MVT::getVectorVT(MVT::i16, NumElts / 2); in LowerRotate()
25002 auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) { in LowerRotate()
25087 assert(VT == MVT::v4i32 && "Only v4i32 vector rotate expected"); in LowerRotate()
25092 SDValue Res02 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64, in LowerRotate()
25093 DAG.getBitcast(MVT::v2i64, R), in LowerRotate()
25094 DAG.getBitcast(MVT::v2i64, Scale)); in LowerRotate()
25095 SDValue Res13 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64, in LowerRotate()
25096 DAG.getBitcast(MVT::v2i64, R13), in LowerRotate()
25097 DAG.getBitcast(MVT::v2i64, Scale13)); in LowerRotate()
25245 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0)); in LowerATOMIC_FENCE()
25248 SDValue Zero = DAG.getTargetConstant(0, dl, MVT::i32); in LowerATOMIC_FENCE()
25250 DAG.getRegister(X86::ESP, MVT::i32), // Base in LowerATOMIC_FENCE()
25251 DAG.getTargetConstant(1, dl, MVT::i8), // Scale in LowerATOMIC_FENCE()
25252 DAG.getRegister(0, MVT::i32), // Index in LowerATOMIC_FENCE()
25253 DAG.getTargetConstant(0, dl, MVT::i32), // Disp in LowerATOMIC_FENCE()
25254 DAG.getRegister(0, MVT::i32), // Segment. in LowerATOMIC_FENCE()
25258 SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, dl, MVT::Other, Ops); in LowerATOMIC_FENCE()
25263 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0)); in LowerATOMIC_FENCE()
25268 MVT T = Op.getSimpleValueType(); in LowerCMP_SWAP()
25274 case MVT::i8: Reg = X86::AL; size = 1; break; in LowerCMP_SWAP()
25275 case MVT::i16: Reg = X86::AX; size = 2; break; in LowerCMP_SWAP()
25276 case MVT::i32: Reg = X86::EAX; size = 4; break; in LowerCMP_SWAP()
25277 case MVT::i64: in LowerCMP_SWAP()
25287 DAG.getTargetConstant(size, DL, MVT::i8), in LowerCMP_SWAP()
25289 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerCMP_SWAP()
25297 MVT::i32, cpOut.getValue(2)); in LowerCMP_SWAP()
25309 MVT InVT = V.getSimpleValueType(); in getPMOVMSKB()
25311 if (InVT == MVT::v32i8 && !Subtarget.hasInt256()) { in getPMOVMSKB()
25314 Lo = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Lo); in getPMOVMSKB()
25315 Hi = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Hi); in getPMOVMSKB()
25316 Hi = DAG.getNode(ISD::SHL, DL, MVT::i32, Hi, in getPMOVMSKB()
25317 DAG.getConstant(16, DL, MVT::i8)); in getPMOVMSKB()
25318 return DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi); in getPMOVMSKB()
25321 return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V); in getPMOVMSKB()
25327 MVT SrcVT = Src.getSimpleValueType(); in LowerBITCAST()
25328 MVT DstVT = Op.getSimpleValueType(); in LowerBITCAST()
25332 if (SrcVT == MVT::i64 && DstVT == MVT::v64i1) { in LowerBITCAST()
25336 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Src, in LowerBITCAST()
25338 Lo = DAG.getBitcast(MVT::v32i1, Lo); in LowerBITCAST()
25339 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Src, in LowerBITCAST()
25341 Hi = DAG.getBitcast(MVT::v32i1, Hi); in LowerBITCAST()
25342 return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi); in LowerBITCAST()
25346 if ((SrcVT == MVT::v32i16 || SrcVT == MVT::v64i8) && DstVT.isVector() && in LowerBITCAST()
25351 EVT CastVT = MVT::getVectorVT(DstVT.getVectorElementType(), in LowerBITCAST()
25359 if ((SrcVT == MVT::v16i1 || SrcVT == MVT::v32i1) && DstVT.isScalarInteger()) { in LowerBITCAST()
25361 MVT SExtVT = SrcVT == MVT::v16i1 ? MVT::v16i8 : MVT::v32i8; in LowerBITCAST()
25368 if (SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8 || in LowerBITCAST()
25369 SrcVT == MVT::i64) { in LowerBITCAST()
25371 if (DstVT != MVT::f64 && DstVT != MVT::i64 && in LowerBITCAST()
25372 !(DstVT == MVT::x86mmx && SrcVT.isVector())) in LowerBITCAST()
25380 MVT NewVT = MVT::getVectorVT(SrcVT.getVectorElementType(), in LowerBITCAST()
25385 assert(SrcVT == MVT::i64 && !Subtarget.is64Bit() && in LowerBITCAST()
25387 Src = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Src); in LowerBITCAST()
25390 MVT V2X64VT = DstVT == MVT::f64 ? MVT::v2f64 : MVT::v2i64; in LowerBITCAST()
25393 if (DstVT == MVT::x86mmx) in LowerBITCAST()
25402 assert((DstVT == MVT::i64 || in LowerBITCAST()
25406 if (SrcVT==MVT::i64 && DstVT.isVector()) in LowerBITCAST()
25408 if (DstVT==MVT::i64 && SrcVT.isVector()) in LowerBITCAST()
25423 static SDValue LowerHorizontalByteSum(SDValue V, MVT VT, in LowerHorizontalByteSum()
25427 MVT ByteVecVT = V.getSimpleValueType(); in LowerHorizontalByteSum()
25428 MVT EltVT = VT.getVectorElementType(); in LowerHorizontalByteSum()
25429 assert(ByteVecVT.getVectorElementType() == MVT::i8 && in LowerHorizontalByteSum()
25431 assert(EltVT != MVT::i8 && in LowerHorizontalByteSum()
25438 if (EltVT == MVT::i64) { in LowerHorizontalByteSum()
25440 MVT SadVecVT = MVT::getVectorVT(MVT::i64, VecSize / 64); in LowerHorizontalByteSum()
25445 if (EltVT == MVT::i32) { in LowerHorizontalByteSum()
25458 MVT SadVecVT = MVT::getVectorVT(MVT::i64, VecSize / 64); in LowerHorizontalByteSum()
25465 MVT ShortVecVT = MVT::getVectorVT(MVT::i16, VecSize / 16); in LowerHorizontalByteSum()
25474 assert(EltVT == MVT::i16 && "Unknown how to handle type"); in LowerHorizontalByteSum()
25490 MVT VT = Op.getSimpleValueType(); in LowerVectorCTPOPInRegLUT()
25491 MVT EltVT = VT.getVectorElementType(); in LowerVectorCTPOPInRegLUT()
25494 assert(EltVT == MVT::i8 && "Only vXi8 vector CTPOP lowering supported."); in LowerVectorCTPOPInRegLUT()
25513 LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8)); in LowerVectorCTPOPInRegLUT()
25536 MVT VT = Op.getSimpleValueType(); in LowerVectorCTPOP()
25545 assert((VT.getVectorElementType() == MVT::i8 || in LowerVectorCTPOP()
25546 VT.getVectorElementType() == MVT::i16) && "Unexpected type"); in LowerVectorCTPOP()
25548 MVT NewVT = MVT::getVectorVT(MVT::i32, NumElems); in LowerVectorCTPOP()
25564 if (VT.getScalarType() != MVT::i8) { in LowerVectorCTPOP()
25565 MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8); in LowerVectorCTPOP()
25586 MVT VT = Op.getSimpleValueType(); in LowerBITREVERSE_XOP()
25593 MVT VecVT = MVT::getVectorVT(VT, 128 / VT.getSizeInBits()); in LowerBITREVERSE_XOP()
25619 MaskElts.push_back(DAG.getConstant(PermuteByte, DL, MVT::i8)); in LowerBITREVERSE_XOP()
25623 SDValue Mask = DAG.getBuildVector(MVT::v16i8, DL, MaskElts); in LowerBITREVERSE_XOP()
25624 SDValue Res = DAG.getBitcast(MVT::v16i8, In); in LowerBITREVERSE_XOP()
25625 Res = DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, DAG.getUNDEF(MVT::v16i8), in LowerBITREVERSE_XOP()
25632 MVT VT = Op.getSimpleValueType(); in LowerBITREVERSE()
25643 assert(VT.getScalarType() == MVT::i8 && in LowerBITREVERSE()
25670 LoMaskElts.push_back(DAG.getConstant(LoLUT[i % 16], DL, MVT::i8)); in LowerBITREVERSE()
25671 HiMaskElts.push_back(DAG.getConstant(HiLUT[i % 16], DL, MVT::i8)); in LowerBITREVERSE()
25707 NewOpc, SDLoc(N), DAG.getVTList(MVT::i32, MVT::Other), in lowerAtomicArithWithLOCK()
25719 MVT VT = N->getSimpleValueType(0); in lowerAtomicArith()
25772 MVT VT = N->getSimpleValueType(0); in LowerADDSUBCARRY()
25778 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerADDSUBCARRY()
25785 Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32), in LowerADDSUBCARRY()
25793 if (N->getValueType(1) == MVT::i1) in LowerADDSUBCARRY()
25794 SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC); in LowerADDSUBCARRY()
25820 bool isF64 = ArgVT == MVT::f64; in LowerFSINCOS()
25855 static SDValue ExtendToType(SDValue InOp, MVT NVT, SelectionDAG &DAG, in ExtendToType()
25858 MVT InVT = InOp.getSimpleValueType(); in ExtendToType()
25911 MVT VT = Src.getSimpleValueType(); in LowerMSCATTER()
25921 if (VT == MVT::v2f32) { in LowerMSCATTER()
25922 assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type"); in LowerMSCATTER()
25924 if (Index.getValueType() == MVT::v2i64 && Subtarget.hasVLX()) { in LowerMSCATTER()
25925 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src, in LowerMSCATTER()
25926 DAG.getUNDEF(MVT::v2f32)); in LowerMSCATTER()
25927 SDVTList VTs = DAG.getVTList(MVT::v2i1, MVT::Other); in LowerMSCATTER()
25937 if (VT == MVT::v2i32) { in LowerMSCATTER()
25938 assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type"); in LowerMSCATTER()
25939 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src, in LowerMSCATTER()
25940 DAG.getUNDEF(MVT::v2i32)); in LowerMSCATTER()
25942 if (Index.getValueType() == MVT::v2i64 && Subtarget.hasVLX()) { in LowerMSCATTER()
25943 SDVTList VTs = DAG.getVTList(MVT::v2i1, MVT::Other); in LowerMSCATTER()
25955 Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i1, Mask, in LowerMSCATTER()
25956 DAG.getConstant(0, dl, MVT::v2i1)); in LowerMSCATTER()
25958 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(), dl, in LowerMSCATTER()
25962 MVT IndexVT = Index.getSimpleValueType(); in LowerMSCATTER()
25963 MVT MaskVT = Mask.getSimpleValueType(); in LowerMSCATTER()
25967 if (IndexVT == MVT::v2i32) in LowerMSCATTER()
25979 VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts); in LowerMSCATTER()
25980 IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(), NumElts); in LowerMSCATTER()
25981 MaskVT = MVT::getVectorVT(MVT::i1, NumElts); in LowerMSCATTER()
25988 SDVTList VTs = DAG.getVTList(MaskVT, MVT::Other); in LowerMSCATTER()
26000 MVT VT = Op.getSimpleValueType(); in LowerMLOAD()
26001 MVT ScalarVT = VT.getScalarType(); in LowerMLOAD()
26016 (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) && in LowerMLOAD()
26022 MVT WideDataVT = MVT::getVectorVT(ScalarVT, NumEltsInWideVec); in LowerMLOAD()
26026 assert(Mask.getSimpleValueType().getScalarType() == MVT::i1 && in LowerMLOAD()
26029 MVT WideMaskVT = MVT::getVectorVT(MVT::i1, NumEltsInWideVec); in LowerMLOAD()
26049 MVT VT = DataToStore.getSimpleValueType(); in LowerMSTORE()
26050 MVT ScalarVT = VT.getScalarType(); in LowerMSTORE()
26065 (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) && in LowerMSTORE()
26071 MVT WideDataVT = MVT::getVectorVT(ScalarVT, NumEltsInWideVec); in LowerMSTORE()
26074 assert(Mask.getSimpleValueType().getScalarType() == MVT::i1 && in LowerMSTORE()
26077 MVT WideMaskVT = MVT::getVectorVT(MVT::i1, NumEltsInWideVec); in LowerMSTORE()
26093 MVT VT = Op.getSimpleValueType(); in LowerMGATHER()
26097 MVT IndexVT = Index.getSimpleValueType(); in LowerMGATHER()
26098 MVT MaskVT = Mask.getSimpleValueType(); in LowerMGATHER()
26103 if (IndexVT == MVT::v2i32) in LowerMGATHER()
26108 MVT OrigVT = VT; in LowerMGATHER()
26117 VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts); in LowerMGATHER()
26118 IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(), NumElts); in LowerMGATHER()
26119 MaskVT = MVT::getVectorVT(MVT::i1, NumElts); in LowerMGATHER()
26129 DAG.getVTList(VT, MaskVT, MVT::Other), Ops, dl, N->getMemoryVT(), in LowerMGATHER()
26151 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue); in LowerGC_TRANSITION_START()
26172 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue); in LowerGC_TRANSITION_END()
26338 SDValue N0 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::v2i64, in ReplaceNodeResults()
26340 SDValue N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::v2i64, in ReplaceNodeResults()
26342 SDValue Mul = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, N0, N1); in ReplaceNodeResults()
26345 VT.getVectorElementType() == MVT::i8) { in ReplaceNodeResults()
26348 MVT MulVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements()); in ReplaceNodeResults()
26356 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i8, ConcatOps); in ReplaceNodeResults()
26401 assert(N->getValueType(0) == MVT::v2i32 && "Unexpected type"); in ReplaceNodeResults()
26402 if (N->getOperand(0).getValueType() != MVT::v2f32 || in ReplaceNodeResults()
26403 getTypeAction(*DAG.getContext(), MVT::v2i32) == TypeWidenVector) in ReplaceNodeResults()
26405 SDValue UNDEF = DAG.getUNDEF(MVT::v2f32); in ReplaceNodeResults()
26406 SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, in ReplaceNodeResults()
26408 SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, in ReplaceNodeResults()
26410 SDValue Res = DAG.getNode(ISD::SETCC, dl, MVT::v4i32, LHS, RHS, in ReplaceNodeResults()
26412 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i32, Res, in ReplaceNodeResults()
26423 assert(VT == MVT::v2f32 && "Unexpected type (!= v2f32) on FMIN/FMAX."); in ReplaceNodeResults()
26425 SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, in ReplaceNodeResults()
26427 SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, in ReplaceNodeResults()
26429 Results.push_back(DAG.getNode(N->getOpcode(), dl, MVT::v4f32, LHS, RHS)); in ReplaceNodeResults()
26455 if (VT == MVT::v2i32) { in ReplaceNodeResults()
26475 MVT VT = N->getSimpleValueType(0); in ReplaceNodeResults()
26482 MVT WidenVT = getTypeToTransformTo(*DAG.getContext(), VT).getSimpleVT(); in ReplaceNodeResults()
26491 MVT InEltVT = InVT.getSimpleVT().getVectorElementType(); in ReplaceNodeResults()
26516 if (InVT == MVT::v4i64 && VT == MVT::v4i8 && isTypeLegal(MVT::v8i64)) { in ReplaceNodeResults()
26517 In = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i64, In, in ReplaceNodeResults()
26518 DAG.getUNDEF(MVT::v4i64)); in ReplaceNodeResults()
26532 if (!Subtarget.hasSSE41() && VT == MVT::v4i64 && in ReplaceNodeResults()
26533 (InVT == MVT::v16i16 || InVT == MVT::v32i8)) { in ReplaceNodeResults()
26541 MVT ExtendVT = MVT::getVectorVT(MVT::i32, in ReplaceNodeResults()
26545 In = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, MVT::v4i32, In); in ReplaceNodeResults()
26575 if (!Subtarget.hasSSE41() && VT == MVT::v4i64 && in ReplaceNodeResults()
26576 (InVT == MVT::v4i16 || InVT == MVT::v4i8)) { in ReplaceNodeResults()
26581 In = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, In); in ReplaceNodeResults()
26584 SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32); in ReplaceNodeResults()
26585 SDValue SignBits = DAG.getSetCC(dl, MVT::v4i32, Zero, In, ISD::SETGT); in ReplaceNodeResults()
26589 SDValue Lo = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits, in ReplaceNodeResults()
26591 Lo = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Lo); in ReplaceNodeResults()
26592 SDValue Hi = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits, in ReplaceNodeResults()
26594 Hi = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Hi); in ReplaceNodeResults()
26601 if ((VT == MVT::v16i32 || VT == MVT::v8i64) && InVT.is128BitVector()) { in ReplaceNodeResults()
26636 if ((VT == MVT::v2i8 || VT == MVT::v2i16) && in ReplaceNodeResults()
26642 SDValue Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v2i32, Src); in ReplaceNodeResults()
26645 dl, MVT::v2i32, Res, in ReplaceNodeResults()
26658 MVT PromoteVT = MVT::getVectorVT(MVT::getIntegerVT(NewEltWidth), in ReplaceNodeResults()
26664 if (PromoteVT != MVT::v2i32) in ReplaceNodeResults()
26675 MVT ConcatVT = MVT::getVectorVT(VT.getSimpleVT().getVectorElementType(), in ReplaceNodeResults()
26685 if (VT == MVT::v2i32) { in ReplaceNodeResults()
26690 getTypeAction(*DAG.getContext(), MVT::v2i32) == TypeWidenVector; in ReplaceNodeResults()
26691 if (Src.getValueType() == MVT::v2f64) { in ReplaceNodeResults()
26700 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f64, in ReplaceNodeResults()
26701 Src, DAG.getUNDEF(MVT::v2f64)); in ReplaceNodeResults()
26703 SDValue Res = DAG.getNode(Opc, dl, MVT::v4i32, Src); in ReplaceNodeResults()
26705 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i32, Res, in ReplaceNodeResults()
26710 if (SrcVT == MVT::v2f32 && in ReplaceNodeResults()
26713 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src, in ReplaceNodeResults()
26714 DAG.getUNDEF(MVT::v2f32)); in ReplaceNodeResults()
26716 : ISD::FP_TO_UINT, dl, MVT::v4i32, Res); in ReplaceNodeResults()
26717 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i32, Res, Idx); in ReplaceNodeResults()
26727 if (Subtarget.hasDQI() && VT == MVT::i64 && in ReplaceNodeResults()
26728 (SrcVT == MVT::f32 || SrcVT == MVT::f64)) { in ReplaceNodeResults()
26734 MVT VecVT = MVT::getVectorVT(MVT::i64, NumElts); in ReplaceNodeResults()
26735 MVT VecInVT = MVT::getVectorVT(SrcVT.getSimpleVT(), NumElts); in ReplaceNodeResults()
26763 if (N->getValueType(0) != MVT::v2f32 || Src.getValueType() != MVT::v2i64) in ReplaceNodeResults()
26765 Results.push_back(DAG.getNode(X86ISD::CVTSI2P, dl, MVT::v4f32, Src)); in ReplaceNodeResults()
26771 if (VT != MVT::v2f32) in ReplaceNodeResults()
26775 if (Subtarget.hasDQI() && Subtarget.hasVLX() && SrcVT == MVT::v2i64) { in ReplaceNodeResults()
26776 Results.push_back(DAG.getNode(X86ISD::CVTUI2P, dl, MVT::v4f32, Src)); in ReplaceNodeResults()
26779 if (SrcVT != MVT::v2i32) in ReplaceNodeResults()
26781 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64, Src); in ReplaceNodeResults()
26783 DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL), dl, MVT::v2f64); in ReplaceNodeResults()
26784 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn, in ReplaceNodeResults()
26785 DAG.getBitcast(MVT::v2i64, VBias)); in ReplaceNodeResults()
26786 Or = DAG.getBitcast(MVT::v2f64, Or); in ReplaceNodeResults()
26788 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias); in ReplaceNodeResults()
26789 Results.push_back(DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, Sub)); in ReplaceNodeResults()
26795 SDValue V = DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, N->getOperand(0)); in ReplaceNodeResults()
26802 assert(N->getValueType(0) == MVT::v2f32 && in ReplaceNodeResults()
26835 assert((T == MVT::i64 || T == MVT::i128) && "can only expand cmpxchg pair"); in ReplaceNodeResults()
26836 bool Regs64bit = T == MVT::i128; in ReplaceNodeResults()
26837 MVT HalfT = Regs64bit ? MVT::i64 : MVT::i32; in ReplaceNodeResults()
26865 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in ReplaceNodeResults()
26906 MVT::i32, cpOutH.getValue(2)); in ReplaceNodeResults()
26938 if (SrcVT == MVT::v64i1 && DstVT == MVT::i64 && Subtarget.hasBWI()) { in ReplaceNodeResults()
26942 Lo = DAG.getBitcast(MVT::i32, Lo); in ReplaceNodeResults()
26943 Hi = DAG.getBitcast(MVT::i32, Hi); in ReplaceNodeResults()
26944 SDValue Res = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi); in ReplaceNodeResults()
26950 if ((DstVT == MVT::v32i16 || DstVT == MVT::v64i8) && in ReplaceNodeResults()
26954 MVT CastVT = (DstVT == MVT::v32i16) ? MVT::v16i16 : MVT::v32i8; in ReplaceNodeResults()
26962 if (SrcVT != MVT::f64 || in ReplaceNodeResults()
26963 (DstVT != MVT::v2i32 && DstVT != MVT::v4i16 && DstVT != MVT::v8i8) || in ReplaceNodeResults()
26971 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, N->getOperand(0)); in ReplaceNodeResults()
26980 if (VT == MVT::v2f32 && (Subtarget.hasVLX() || !Subtarget.hasAVX512())) { in ReplaceNodeResults()
26983 if (Index.getValueType() != MVT::v2i64) in ReplaceNodeResults()
26986 assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type"); in ReplaceNodeResults()
26987 SDValue PassThru = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, in ReplaceNodeResults()
26989 DAG.getUNDEF(MVT::v2f32)); in ReplaceNodeResults()
26993 Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i1, Mask, in ReplaceNodeResults()
26994 DAG.getUNDEF(MVT::v2i1)); in ReplaceNodeResults()
26995 Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Mask); in ReplaceNodeResults()
27000 DAG.getVTList(MVT::v4f32, Mask.getValueType(), MVT::Other), Ops, dl, in ReplaceNodeResults()
27006 if (VT == MVT::v2i32) { in ReplaceNodeResults()
27010 assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type"); in ReplaceNodeResults()
27011 SDValue PassThru = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, in ReplaceNodeResults()
27013 DAG.getUNDEF(MVT::v2i32)); in ReplaceNodeResults()
27015 if (Index.getValueType() == MVT::v2i64 && in ReplaceNodeResults()
27020 Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i1, Mask, in ReplaceNodeResults()
27021 DAG.getUNDEF(MVT::v2i1)); in ReplaceNodeResults()
27022 Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Mask); in ReplaceNodeResults()
27027 DAG.getVTList(MVT::v4i32, Mask.getValueType(), MVT::Other), Ops, dl, in ReplaceNodeResults()
27031 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i32, Res, in ReplaceNodeResults()
27044 Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i1, Mask, in ReplaceNodeResults()
27045 DAG.getConstant(0, dl, MVT::v2i1)); in ReplaceNodeResults()
27048 SDValue Res = DAG.getMaskedGather(DAG.getVTList(MVT::v4i32, MVT::Other), in ReplaceNodeResults()
27052 if (getTypeAction(*DAG.getContext(), MVT::v2i32) != TypeWidenVector) in ReplaceNodeResults()
27053 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i32, Res, in ReplaceNodeResults()
27066 MVT VT = N->getSimpleValueType(0); in ReplaceNodeResults()
27073 MVT LdVT = Subtarget.is64Bit() && VT.isInteger() ? MVT::i64 : MVT::f64; in ReplaceNodeResults()
27079 MVT WideVT = MVT::getVectorVT(LdVT, 2); in ReplaceNodeResults()
27081 MVT CastVT = MVT::getVectorVT(VT.getVectorElementType(), in ReplaceNodeResults()
27542 return VT1 == MVT::i32 && VT2 == MVT::i64 && Subtarget.is64Bit(); in isZExtFree()
27559 case MVT::i8: in isZExtFree()
27560 case MVT::i16: in isZExtFree()
27561 case MVT::i32: in isZExtFree()
27573 if (SrcVT.getScalarType() == MVT::i1) in isVectorLoadExtDesirable()
27590 case MVT::f32: in isFMAFasterThanFMulAndFAdd()
27591 case MVT::f64: in isFMAFasterThanFMulAndFAdd()
27602 return !(VT1 == MVT::i32 && VT2 == MVT::i16); in isNarrowingProfitable()
27614 if (VT.getSimpleVT().getScalarType() == MVT::i1) in isShuffleMaskLegal()
27631 if (VT == MVT::v32i8 || VT == MVT::v16i16) in isVectorClearMaskLegal()
27845 const TargetRegisterClass *AddrRegClass = getRegClassFor(MVT::i64); in EmitVAARG64WithCustomInserter()
27846 const TargetRegisterClass *OffsetRegClass = getRegClassFor(MVT::i32); in EmitVAARG64WithCustomInserter()
29023 MVT PVT = getPointerTy(MF->getDataLayout()); in emitSetJmpShadowStackFix()
29026 unsigned XorRROpc = (PVT == MVT::i64) ? X86::XOR64rr : X86::XOR32rr; in emitSetJmpShadowStackFix()
29034 unsigned RdsspOpc = (PVT == MVT::i64) ? X86::RDSSPQ : X86::RDSSPD; in emitSetJmpShadowStackFix()
29038 unsigned PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr; in emitSetJmpShadowStackFix()
29075 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!"); in emitEHSjLjSetJmp()
29082 MVT PVT = getPointerTy(MF->getDataLayout()); in emitEHSjLjSetJmp()
29083 assert((PVT == MVT::i64 || PVT == MVT::i32) && in emitEHSjLjSetJmp()
29127 PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr; in emitEHSjLjSetJmp()
29147 PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi; in emitEHSjLjSetJmp()
29224 MVT PVT = getPointerTy(MF->getDataLayout()); in emitLongJmpShadowStackFix()
29276 unsigned XorRROpc = (PVT == MVT::i64) ? X86::XOR64rr : X86::XOR32rr; in emitLongJmpShadowStackFix()
29284 unsigned RdsspOpc = (PVT == MVT::i64) ? X86::RDSSPQ : X86::RDSSPD; in emitLongJmpShadowStackFix()
29289 unsigned TestRROpc = (PVT == MVT::i64) ? X86::TEST64rr : X86::TEST32rr; in emitLongJmpShadowStackFix()
29299 unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm; in emitLongJmpShadowStackFix()
29317 unsigned SubRROpc = (PVT == MVT::i64) ? X86::SUB64rr : X86::SUB32rr; in emitLongJmpShadowStackFix()
29328 unsigned ShrRIOpc = (PVT == MVT::i64) ? X86::SHR64ri : X86::SHR32ri; in emitLongJmpShadowStackFix()
29329 unsigned Offset = (PVT == MVT::i64) ? 3 : 2; in emitLongJmpShadowStackFix()
29336 unsigned IncsspOpc = (PVT == MVT::i64) ? X86::INCSSPQ : X86::INCSSPD; in emitLongJmpShadowStackFix()
29351 unsigned ShlR1Opc = (PVT == MVT::i64) ? X86::SHL64r1 : X86::SHL32r1; in emitLongJmpShadowStackFix()
29358 unsigned MovRIOpc = (PVT == MVT::i64) ? X86::MOV64ri32 : X86::MOV32ri; in emitLongJmpShadowStackFix()
29377 unsigned DecROpc = (PVT == MVT::i64) ? X86::DEC64r : X86::DEC32r; in emitLongJmpShadowStackFix()
29400 MVT PVT = getPointerTy(MF->getDataLayout()); in emitEHSjLjLongJmp()
29401 assert((PVT == MVT::i64 || PVT == MVT::i32) && in emitEHSjLjLongJmp()
29405 (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass; in emitEHSjLjLongJmp()
29409 unsigned FP = (PVT == MVT::i64) ? X86::RBP : X86::EBP; in emitEHSjLjLongJmp()
29417 unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm; in emitEHSjLjLongJmp()
29418 unsigned IJmpOpc = (PVT == MVT::i64) ? X86::JMP64r : X86::JMP32r; in emitEHSjLjLongJmp()
29480 MVT PVT = getPointerTy(MF->getDataLayout()); in SetupEntryBlockForSjLj()
29481 assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!"); in SetupEntryBlockForSjLj()
29490 Op = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi; in SetupEntryBlockForSjLj()
29493 (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass; in SetupEntryBlockForSjLj()
29495 Op = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr; in SetupEntryBlockForSjLj()
29974 MVT SPTy = getPointerTy(MF->getDataLayout()); in EmitInstrWithCustomInserter()
30319 static bool matchUnaryVectorShuffle(MVT MaskVT, ArrayRef<int> Mask, in matchUnaryVectorShuffle()
30324 unsigned &Shuffle, MVT &SrcVT, MVT &DstVT) { in matchUnaryVectorShuffle()
30332 SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT; in matchUnaryVectorShuffle()
30350 MVT ScalarTy = MaskVT.isInteger() ? MaskVT.getScalarType() : in matchUnaryVectorShuffle()
30351 MVT::getIntegerVT(MaskEltSize); in matchUnaryVectorShuffle()
30352 SrcVT = MVT::getVectorVT(ScalarTy, SrcSize / MaskEltSize); in matchUnaryVectorShuffle()
30362 DstVT = MVT::getIntegerVT(Scale * MaskEltSize); in matchUnaryVectorShuffle()
30363 DstVT = MVT::getVectorVT(DstVT, NumDstElts); in matchUnaryVectorShuffle()
30374 SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT; in matchUnaryVectorShuffle()
30384 SrcVT = DstVT = MVT::v2f64; in matchUnaryVectorShuffle()
30389 SrcVT = DstVT = MVT::v4f32; in matchUnaryVectorShuffle()
30394 SrcVT = DstVT = MVT::v4f32; in matchUnaryVectorShuffle()
30403 SrcVT = DstVT = MVT::v4f64; in matchUnaryVectorShuffle()
30408 SrcVT = DstVT = MVT::v8f32; in matchUnaryVectorShuffle()
30413 SrcVT = DstVT = MVT::v8f32; in matchUnaryVectorShuffle()
30423 SrcVT = DstVT = MVT::v8f64; in matchUnaryVectorShuffle()
30429 SrcVT = DstVT = MVT::v16f32; in matchUnaryVectorShuffle()
30435 SrcVT = DstVT = MVT::v16f32; in matchUnaryVectorShuffle()
30456 static bool matchUnaryPermuteVectorShuffle(MVT MaskVT, ArrayRef<int> Mask, in matchUnaryPermuteVectorShuffle()
30461 unsigned &Shuffle, MVT &ShuffleVT, in matchUnaryPermuteVectorShuffle()
30466 MVT MaskEltVT = MVT::getIntegerVT(MaskScalarSizeInBits); in matchUnaryPermuteVectorShuffle()
30478 ShuffleVT = (AllowFloatDomain ? MVT::v4f64 : MVT::v4i64); in matchUnaryPermuteVectorShuffle()
30484 if (is256BitLaneRepeatedShuffleMask(MVT::v8f64, Mask, RepeatedMask)) { in matchUnaryPermuteVectorShuffle()
30486 ShuffleVT = (AllowFloatDomain ? MVT::v8f64 : MVT::v8i64); in matchUnaryPermuteVectorShuffle()
30494 ShuffleVT = MVT::getVectorVT(MVT::f64, Mask.size()); in matchUnaryPermuteVectorShuffle()
30520 ShuffleVT = (AllowIntDomain ? MVT::i32 : MVT::f32); in matchUnaryPermuteVectorShuffle()
30521 ShuffleVT = MVT::getVectorVT(ShuffleVT, InputSizeInBits / 32); in matchUnaryPermuteVectorShuffle()
30538 ShuffleVT = MVT::getVectorVT(MVT::i16, InputSizeInBits / 16); in matchUnaryPermuteVectorShuffle()
30552 ShuffleVT = MVT::getVectorVT(MVT::i16, InputSizeInBits / 16); in matchUnaryPermuteVectorShuffle()
30578 static bool matchBinaryVectorShuffle(MVT MaskVT, ArrayRef<int> Mask, in matchBinaryVectorShuffle()
30583 unsigned &Shuffle, MVT &SrcVT, MVT &DstVT, in matchBinaryVectorShuffle()
30590 V1 = (SM_SentinelUndef == Mask[0] ? DAG.getUNDEF(MVT::v4f32) : V1); in matchBinaryVectorShuffle()
30592 SrcVT = DstVT = Subtarget.hasSSE2() ? MVT::v2f64 : MVT::v4f32; in matchBinaryVectorShuffle()
30598 SrcVT = DstVT = Subtarget.hasSSE2() ? MVT::v2f64 : MVT::v4f32; in matchBinaryVectorShuffle()
30605 SrcVT = DstVT = MVT::v2f64; in matchBinaryVectorShuffle()
30611 SrcVT = DstVT = MVT::v4f32; in matchBinaryVectorShuffle()
30617 if (((MaskVT == MVT::v8i16 || MaskVT == MVT::v16i8) && Subtarget.hasSSE2()) || in matchBinaryVectorShuffle()
30618 ((MaskVT == MVT::v16i16 || MaskVT == MVT::v32i8) && Subtarget.hasInt256()) || in matchBinaryVectorShuffle()
30619 ((MaskVT == MVT::v32i16 || MaskVT == MVT::v64i8) && Subtarget.hasBWI())) { in matchBinaryVectorShuffle()
30628 if ((MaskVT == MVT::v4f32 && Subtarget.hasSSE1()) || in matchBinaryVectorShuffle()
30637 SrcVT = DstVT = (32 == EltSizeInBits ? MVT::v8f32 : MVT::v4f64); in matchBinaryVectorShuffle()
30646 MVT MaskVT, ArrayRef<int> Mask, const APInt &Zeroable, in matchBinaryPermuteVectorShuffle()
30649 unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm) { in matchBinaryPermuteVectorShuffle()
30659 ShuffleVT = MVT::getVectorVT(MVT::i8, MaskVT.getSizeInBits() / 8); in matchBinaryPermuteVectorShuffle()
30668 (MaskVT == MVT::v16i16 && Subtarget.hasAVX2())) { in matchBinaryPermuteVectorShuffle()
30674 if (MaskVT == MVT::v16i16) { in matchBinaryPermuteVectorShuffle()
30695 if (ShuffleVT == MVT::v4i64) in matchBinaryPermuteVectorShuffle()
30696 ShuffleVT = MVT::v8i32; in matchBinaryPermuteVectorShuffle()
30697 else if (ShuffleVT == MVT::v2i64) in matchBinaryPermuteVectorShuffle()
30698 ShuffleVT = MVT::v4i32; in matchBinaryPermuteVectorShuffle()
30700 if (ShuffleVT == MVT::v2i64 || ShuffleVT == MVT::v4i32) in matchBinaryPermuteVectorShuffle()
30701 ShuffleVT = MVT::v8i16; in matchBinaryPermuteVectorShuffle()
30702 else if (ShuffleVT == MVT::v4i64) in matchBinaryPermuteVectorShuffle()
30703 ShuffleVT = MVT::v4f64; in matchBinaryPermuteVectorShuffle()
30704 else if (ShuffleVT == MVT::v8i32) in matchBinaryPermuteVectorShuffle()
30705 ShuffleVT = MVT::v8f32; in matchBinaryPermuteVectorShuffle()
30712 ShuffleVT = MVT::getIntegerVT(EltSizeInBits / Scale); in matchBinaryPermuteVectorShuffle()
30713 ShuffleVT = MVT::getVectorVT(ShuffleVT, NumMaskElts * Scale); in matchBinaryPermuteVectorShuffle()
30731 ShuffleVT = MVT::v4f32; in matchBinaryPermuteVectorShuffle()
30743 ShuffleVT = MVT::getVectorVT(MVT::f64, MaskVT.getSizeInBits() / 64); in matchBinaryPermuteVectorShuffle()
30788 ShuffleVT = MVT::getVectorVT(MVT::f32, MaskVT.getSizeInBits() / 32); in matchBinaryPermuteVectorShuffle()
30823 MVT VT1 = V1.getSimpleValueType(); in combineX86ShuffleChain()
30824 MVT VT2 = V2.getSimpleValueType(); in combineX86ShuffleChain()
30825 MVT RootVT = Root.getSimpleValueType(); in combineX86ShuffleChain()
30866 MVT ShuffleVT = (FloatDomain ? MVT::v4f64 : MVT::v4i64); in combineX86ShuffleChain()
30874 DAG.getConstant(PermMask, DL, MVT::i8)); in combineX86ShuffleChain()
30894 MVT MaskVT = FloatDomain ? MVT::getFloatingPointVT(MaskEltSizeInBits) in combineX86ShuffleChain()
30895 : MVT::getIntegerVT(MaskEltSizeInBits); in combineX86ShuffleChain()
30896 MaskVT = MVT::getVectorVT(MaskVT, NumMaskElts); in combineX86ShuffleChain()
30903 MVT ShuffleSrcVT, ShuffleVT; in combineX86ShuffleChain()
30952 DAG.getConstant(PermuteImm, DL, MVT::i8)); in combineX86ShuffleChain()
30982 DAG.getConstant(PermuteImm, DL, MVT::i8)); in combineX86ShuffleChain()
30987 MVT IntMaskVT = MVT::getIntegerVT(MaskEltSizeInBits); in combineX86ShuffleChain()
30988 IntMaskVT = MVT::getVectorVT(IntMaskVT, NumMaskElts); in combineX86ShuffleChain()
30999 DAG.getConstant(BitLen, DL, MVT::i8), in combineX86ShuffleChain()
31000 DAG.getConstant(BitIdx, DL, MVT::i8)); in combineX86ShuffleChain()
31010 DAG.getConstant(BitLen, DL, MVT::i8), in combineX86ShuffleChain()
31011 DAG.getConstant(BitIdx, DL, MVT::i8)); in combineX86ShuffleChain()
31032 (MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) || in combineX86ShuffleChain()
31034 (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 || in combineX86ShuffleChain()
31035 MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) || in combineX86ShuffleChain()
31036 (Subtarget.hasBWI() && MaskVT == MVT::v32i16) || in combineX86ShuffleChain()
31037 (Subtarget.hasBWI() && Subtarget.hasVLX() && MaskVT == MVT::v16i16) || in combineX86ShuffleChain()
31038 (Subtarget.hasVBMI() && MaskVT == MVT::v64i8) || in combineX86ShuffleChain()
31039 (Subtarget.hasVBMI() && Subtarget.hasVLX() && MaskVT == MVT::v32i8))) { in combineX86ShuffleChain()
31050 (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 || in combineX86ShuffleChain()
31051 MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) || in combineX86ShuffleChain()
31053 (MaskVT == MVT::v4f64 || MaskVT == MVT::v4i64 || in combineX86ShuffleChain()
31054 MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) || in combineX86ShuffleChain()
31055 (Subtarget.hasBWI() && MaskVT == MVT::v32i16) || in combineX86ShuffleChain()
31056 (Subtarget.hasBWI() && Subtarget.hasVLX() && MaskVT == MVT::v16i16) || in combineX86ShuffleChain()
31057 (Subtarget.hasVBMI() && MaskVT == MVT::v64i8) || in combineX86ShuffleChain()
31058 (Subtarget.hasVBMI() && Subtarget.hasVLX() && MaskVT == MVT::v32i8))) { in combineX86ShuffleChain()
31074 (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 || in combineX86ShuffleChain()
31075 MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) || in combineX86ShuffleChain()
31077 (MaskVT == MVT::v4f64 || MaskVT == MVT::v4i64 || in combineX86ShuffleChain()
31078 MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) || in combineX86ShuffleChain()
31079 (Subtarget.hasBWI() && MaskVT == MVT::v32i16) || in combineX86ShuffleChain()
31080 (Subtarget.hasBWI() && Subtarget.hasVLX() && MaskVT == MVT::v16i16) || in combineX86ShuffleChain()
31081 (Subtarget.hasVBMI() && MaskVT == MVT::v64i8) || in combineX86ShuffleChain()
31082 (Subtarget.hasVBMI() && Subtarget.hasVLX() && MaskVT == MVT::v32i8))) { in combineX86ShuffleChain()
31123 ((MaskVT == MVT::v8f32 && Subtarget.hasAVX()) || in combineX86ShuffleChain()
31124 (MaskVT == MVT::v16f32 && Subtarget.hasAVX512()))) { in combineX86ShuffleChain()
31128 M < 0 ? DAG.getUNDEF(MVT::i32) : DAG.getConstant(M % 4, DL, MVT::i32); in combineX86ShuffleChain()
31140 (MaskVT == MVT::v2f64 || MaskVT == MVT::v4f64 || MaskVT == MVT::v4f32 || in combineX86ShuffleChain()
31141 MaskVT == MVT::v8f32)) { in combineX86ShuffleChain()
31168 DAG.getConstant(M2ZImm, DL, MVT::i8)); in combineX86ShuffleChain()
31187 PSHUFBMask.push_back(DAG.getUNDEF(MVT::i8)); in combineX86ShuffleChain()
31191 PSHUFBMask.push_back(DAG.getConstant(255, DL, MVT::i8)); in combineX86ShuffleChain()
31196 PSHUFBMask.push_back(DAG.getConstant(M, DL, MVT::i8)); in combineX86ShuffleChain()
31198 MVT ByteVT = MVT::getVectorVT(MVT::i8, NumBytes); in combineX86ShuffleChain()
31218 VPPERMMask.push_back(DAG.getUNDEF(MVT::i8)); in combineX86ShuffleChain()
31222 VPPERMMask.push_back(DAG.getConstant(128, DL, MVT::i8)); in combineX86ShuffleChain()
31226 VPPERMMask.push_back(DAG.getConstant(M, DL, MVT::i8)); in combineX86ShuffleChain()
31228 MVT ByteVT = MVT::v16i8; in combineX86ShuffleChain()
31248 MVT VT = Root.getSimpleValueType(); in combineX86ShufflesConstants()
31312 MVT MaskSVT; in combineX86ShufflesConstants()
31314 MaskSVT = MVT::getFloatingPointVT(MaskSizeInBits); in combineX86ShufflesConstants()
31316 MaskSVT = MVT::getIntegerVT(MaskSizeInBits); in combineX86ShufflesConstants()
31318 MVT MaskVT = MVT::getVectorVT(MaskSVT, NumMaskElts); in combineX86ShufflesConstants()
31369 MVT VT = Op.getSimpleValueType(); in combineX86ShufflesRecursively()
31574 MVT VT = N.getSimpleValueType(); in getPSHUFShuffleMask()
31667 if (V.getSimpleValueType().getVectorElementType() != MVT::i8 && in combineRedundantDWordShuffle()
31668 V.getSimpleValueType().getVectorElementType() != MVT::i16) in combineRedundantDWordShuffle()
31750 MVT VT = N.getSimpleValueType(); in combineTargetShuffle()
31828 MVT SVT = VT.getVectorElementType(); in combineTargetShuffle()
31841 assert(VT == MVT::v4f32 && "INSERTPS ValueType must be MVT::v4f32"); in combineTargetShuffle()
31853 DAG.getConstant(InsertPSMask, DL, MVT::i8)); in combineTargetShuffle()
31858 DAG.getConstant(InsertPSMask, DL, MVT::i8)); in combineTargetShuffle()
31869 DAG.getConstant(InsertPSMask, DL, MVT::i8)); in combineTargetShuffle()
31876 DAG.getConstant(InsertPSMask, DL, MVT::i8)); in combineTargetShuffle()
31921 DAG.getConstant(InsertPSMask, DL, MVT::i8)); in combineTargetShuffle()
31940 assert(VT.getVectorElementType() == MVT::i16 && "Bad word shuffle type!"); in combineTargetShuffle()
31950 MVT DVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2); in combineTargetShuffle()
32112 MVT VT = N->getSimpleValueType(0); in combineShuffleToFMAddSub()
32157 MVT VT = N->getSimpleValueType(0); in combineShuffleToAddSubOrFMAddSub()
32193 if (VT.getVectorElementType() != MVT::i32 && in combineShuffleOfConcatUndef()
32194 VT.getVectorElementType() != MVT::i64 && in combineShuffleOfConcatUndef()
32195 VT.getVectorElementType() != MVT::f32 && in combineShuffleOfConcatUndef()
32196 VT.getVectorElementType() != MVT::f64) in combineShuffleOfConcatUndef()
32395 if (Subtarget.hasSSE41() && VT == MVT::v4i32 && in combineShuffle()
32409 Op0.getOperand(0).getValueType() == MVT::v4i32) { in combineShuffle()
32416 Op1 = DAG.getBitcast(MVT::v4i32, Op1); in combineShuffle()
32417 Op1 = DAG.getVectorShuffle(MVT::v4i32, dl, Op1, Op1, Mask); in combineShuffle()
32418 return DAG.getNode(ISD::MUL, dl, MVT::v4i32, Op0, Op1); in combineShuffle()
32423 Op1.getOperand(0).getValueType() == MVT::v4i32) { in combineShuffle()
32429 Op0 = DAG.getBitcast(MVT::v4i32, Op0); in combineShuffle()
32430 Op0 = DAG.getVectorShuffle(MVT::v4i32, dl, Op0, Op0, Mask); in combineShuffle()
32432 return DAG.getNode(ISD::MUL, dl, MVT::v4i32, Op0, Op1); in combineShuffle()
32455 MVT AmtVT = Amt.getSimpleValueType(); in SimplifyDemandedVectorEltsForTargetNode()
32479 MVT SrcVT = Src.getSimpleValueType(); in SimplifyDemandedVectorEltsForTargetNode()
32503 MVT SrcVT = Src.getSimpleValueType(); in SimplifyDemandedVectorEltsForTargetNode()
32651 TLO.DAG.getConstant(std::abs(Diff), SDLoc(Op), MVT::i8)); in SimplifyDemandedBitsForTargetNode()
32743 MVT SrcVT = Src.getSimpleValueType(); in SimplifyDemandedBitsForTargetNode()
32902 (N0.getOperand(0).getValueType() == MVT::v16i8 || in combineBitcastvxi1()
32903 N0.getOperand(0).getValueType() == MVT::v32i8 || in combineBitcastvxi1()
32904 N0.getOperand(0).getValueType() == MVT::v64i8); in combineBitcastvxi1()
32921 MVT SExtVT; in combineBitcastvxi1()
32925 case MVT::v2i1: in combineBitcastvxi1()
32926 SExtVT = MVT::v2i64; in combineBitcastvxi1()
32928 case MVT::v4i1: in combineBitcastvxi1()
32929 SExtVT = MVT::v4i32; in combineBitcastvxi1()
32934 SExtVT = MVT::v4i64; in combineBitcastvxi1()
32937 case MVT::v8i1: in combineBitcastvxi1()
32938 SExtVT = MVT::v8i16; in combineBitcastvxi1()
32947 SExtVT = MVT::v8i32; in combineBitcastvxi1()
32950 case MVT::v16i1: in combineBitcastvxi1()
32951 SExtVT = MVT::v16i8; in combineBitcastvxi1()
32957 case MVT::v32i1: in combineBitcastvxi1()
32958 SExtVT = MVT::v32i8; in combineBitcastvxi1()
32960 case MVT::v64i1: in combineBitcastvxi1()
32964 SExtVT = MVT::v64i8; in combineBitcastvxi1()
32973 if (SExtVT == MVT::v64i8) { in combineBitcastvxi1()
32976 Lo = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Lo); in combineBitcastvxi1()
32977 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Lo); in combineBitcastvxi1()
32978 Hi = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Hi); in combineBitcastvxi1()
32979 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Hi); in combineBitcastvxi1()
32980 Hi = DAG.getNode(ISD::SHL, DL, MVT::i64, Hi, in combineBitcastvxi1()
32981 DAG.getConstant(32, DL, MVT::i8)); in combineBitcastvxi1()
32982 V = DAG.getNode(ISD::OR, DL, MVT::i64, Lo, Hi); in combineBitcastvxi1()
32983 } else if (SExtVT == MVT::v16i8 || SExtVT == MVT::v32i8) { in combineBitcastvxi1()
32986 if (SExtVT == MVT::v8i16) in combineBitcastvxi1()
32987 V = DAG.getNode(X86ISD::PACKSS, DL, MVT::v16i8, V, in combineBitcastvxi1()
32988 DAG.getUNDEF(MVT::v8i16)); in combineBitcastvxi1()
32989 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V); in combineBitcastvxi1()
32997 assert(SrcVT.getVectorElementType() == MVT::i1 && in combinevXi1ConstantToInteger()
33038 if (!(SrcVT.isVector() && SrcVT.getVectorElementType() == MVT::i1 && in combineCastedMaskArithmetic()
33040 !(DstVT.isVector() && DstVT.getVectorElementType() == MVT::i1 && in combineCastedMaskArithmetic()
33079 return DAG.getUNDEF(MVT::x86mmx); in createMMXBuildVector()
33082 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, V); in createMMXBuildVector()
33083 V = DAG.getBitcast(MVT::v2i64, V); in createMMXBuildVector()
33084 return DAG.getNode(X86ISD::MOVDQ2Q, DL, MVT::x86mmx, V); in createMMXBuildVector()
33086 V = DAG.getBitcast(MVT::i32, V); in createMMXBuildVector()
33088 V = DAG.getAnyExtOrTrunc(V, DL, MVT::i32); in createMMXBuildVector()
33090 return DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, V); in createMMXBuildVector()
33099 return DAG.getUNDEF(MVT::x86mmx); in createMMXBuildVector()
33107 ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx, in createMMXBuildVector()
33108 DAG.getConstant(Intrinsic::x86_mmx_punpcklbw, DL, MVT::i32), Splat, in createMMXBuildVector()
33114 ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx, in createMMXBuildVector()
33115 DAG.getConstant(Intrinsic::x86_sse_pshuf_w, DL, MVT::i32), Splat, in createMMXBuildVector()
33116 DAG.getConstant(ShufMask, DL, MVT::i8)); in createMMXBuildVector()
33131 SDValue Intrin = DAG.getConstant(IntrinOp, DL, MVT::i32); in createMMXBuildVector()
33133 Ops[i / 2] = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx, Intrin, in createMMXBuildVector()
33160 if ((VT == MVT::v4i1 || VT == MVT::v2i1) && SrcVT.isScalarInteger() && in combineBitcast()
33163 N0 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i8, N0); in combineBitcast()
33164 N0 = DAG.getBitcast(MVT::v8i1, N0); in combineBitcast()
33171 if ((SrcVT == MVT::v4i1 || SrcVT == MVT::v2i1) && VT.isScalarInteger() && in combineBitcast()
33177 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops); in combineBitcast()
33178 N0 = DAG.getBitcast(MVT::i8, N0); in combineBitcast()
33186 if (VT == MVT::x86mmx) { in combineBitcast()
33195 DAG.getConstant(EltBits[0].trunc(32), DL, MVT::i32)); in combineBitcast()
33199 return DAG.getBitcast(VT, DAG.getConstantFP(F64, DL, MVT::f64)); in combineBitcast()
33204 (SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8) && in combineBitcast()
33215 N00 = LowUndef ? DAG.getAnyExtOrTrunc(N00, dl, MVT::i32) in combineBitcast()
33216 : DAG.getZExtOrTrunc(N00, dl, MVT::i32); in combineBitcast()
33225 (SrcVT == MVT::v2f32 || SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || in combineBitcast()
33226 SrcVT == MVT::v8i8)) in combineBitcast()
33236 DAG.getBitcast(MVT::v2i64, N00)); in combineBitcast()
33240 if (SrcVT == MVT::v2i32 && N0.getOpcode() == ISD::FP_TO_SINT) { in combineBitcast()
33242 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0, in combineBitcast()
33243 DAG.getUNDEF(MVT::v2i32)); in combineBitcast()
33245 DAG.getBitcast(MVT::v2i64, Res)); in combineBitcast()
33252 SrcVT.isVector() && SrcVT.getVectorElementType() == MVT::i1 && in combineBitcast()
33258 VT.isVector() && VT.getVectorElementType() == MVT::i1 && in combineBitcast()
33285 if (!((Subtarget.hasSSE1() && VT == MVT::f32) || in combineBitcast()
33286 (Subtarget.hasSSE2() && VT == MVT::f64))) in combineBitcast()
33374 Op0.getOperand(0).getValueType().getVectorElementType() != MVT::i8 || in detectZextAbsDiff()
33376 Op1.getOperand(0).getValueType().getVectorElementType() != MVT::i8) in detectZextAbsDiff()
33396 MVT ExtendedVT = MVT::getVectorVT(MVT::i8, RegSize / 8); in createPSADBW()
33404 MVT VT = MVT::getVectorVT(MVT::i64, Ops[0].getValueSizeInBits() / 64); in createPSADBW()
33407 MVT SadVT = MVT::getVectorVT(MVT::i64, RegSize / 64); in createPSADBW()
33421 if (ExtractVT != MVT::i16 && ExtractVT != MVT::i8) in combineHorizontalMinMaxResult()
33449 assert(((SrcVT == MVT::v8i16 && ExtractVT == MVT::i16) || in combineHorizontalMinMaxResult()
33450 (SrcVT == MVT::v16i8 && ExtractVT == MVT::i8)) && in combineHorizontalMinMaxResult()
33471 if (ExtractVT == MVT::i8) { in combineHorizontalMinMaxResult()
33473 SrcVT, DL, MinPos, DAG.getConstant(0, DL, MVT::v16i8), in combineHorizontalMinMaxResult()
33479 MinPos = DAG.getBitcast(MVT::v8i16, MinPos); in combineHorizontalMinMaxResult()
33480 MinPos = DAG.getNode(X86ISD::PHMINPOS, DL, MVT::v8i16, MinPos); in combineHorizontalMinMaxResult()
33500 if (ExtractVT != MVT::i64 && ExtractVT != MVT::i32 && ExtractVT != MVT::i16 && in combineHorizontalPredicateResult()
33501 ExtractVT != MVT::i8) in combineHorizontalPredicateResult()
33531 MVT MaskVT; in combineHorizontalPredicateResult()
33533 MaskVT = MVT::getVectorVT(MVT::getFloatingPointVT(BitWidth), in combineHorizontalPredicateResult()
33536 MaskVT = MVT::getVectorVT(MVT::i8, MatchSizeInBits / 8); in combineHorizontalPredicateResult()
33558 Res = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Res); in combineHorizontalPredicateResult()
33559 Res = DAG.getSelectCC(DL, Res, DAG.getConstant(CompareBits, DL, MVT::i32), in combineHorizontalPredicateResult()
33621 MVT SadVT = SAD.getSimpleValueType(); in combineBasicSADPattern()
33636 MVT Type = Extract->getSimpleValueType(0); in combineBasicSADPattern()
33639 MVT ResVT = MVT::getVectorVT(Type, SadVT.getSizeInBits() / TypeSizeInBits); in combineBasicSADPattern()
33662 if (SrcSVT == MVT::i1 || !isa<ConstantSDNode>(Idx)) in combineExtractWithShuffle()
33725 if ((SrcVT == MVT::v4i32 || SrcVT == MVT::v2i64) && in combineExtractWithShuffle()
33732 if ((SrcVT == MVT::v8i16 && Subtarget.hasSSE2()) || in combineExtractWithShuffle()
33733 (SrcVT == MVT::v16i8 && Subtarget.hasSSE41())) { in combineExtractWithShuffle()
33736 unsigned OpCode = (SrcVT == MVT::v8i16 ? X86ISD::PEXTRW : X86ISD::PEXTRB); in combineExtractWithShuffle()
33737 SDValue ExtOp = DAG.getNode(OpCode, dl, MVT::i32, SrcOp, in combineExtractWithShuffle()
33774 VT == MVT::i64 && SrcVT == MVT::v1i64 && isNullConstant(EltIdx)) { in combineExtractVectorElt()
33778 if (MMXSrc.getValueType() == MVT::x86mmx) in combineExtractVectorElt()
33784 VT == MVT::i32 && SrcVT == MVT::v2i32 && isNullConstant(EltIdx)) { in combineExtractVectorElt()
33788 if (MMXSrc.getValueType() == MVT::x86mmx) in combineExtractVectorElt()
33789 return DAG.getNode(X86ISD::MMX_MOVD2W, dl, MVT::i32, MMXSrc); in combineExtractVectorElt()
33792 if (VT == MVT::i1 && InputVector.getOpcode() == ISD::BITCAST && in combineExtractVectorElt()
33799 return DAG.getConstant(Res, dl, MVT::i1); in combineExtractVectorElt()
33843 CondVT.getVectorElementType() == MVT::i1) { in combineVSelectWithAllOnesOrZeros()
33911 MVT AndNVT = MVT::getVectorVT(MVT::i64, CondVT.getSizeInBits() / 64); in combineVSelectWithAllOnesOrZeros()
33940 if (Cond.getValueType() != MVT::i1) in combineSelectOfTwoConstants()
33956 ((VT == MVT::i32 || VT == MVT::i64) && in combineSelectOfTwoConstants()
33963 Cond = DAG.getNOT(DL, Cond, MVT::i1); in combineSelectOfTwoConstants()
34021 if (VT.getVectorElementType() == MVT::i16) in combineVSelectToBLENDV()
34027 if (VT == MVT::v32i8 && !Subtarget.hasAVX2()) in combineVSelectToBLENDV()
34098 VT != MVT::f80 && VT != MVT::f128 && in combineSelect()
34099 (TLI.isTypeLegal(VT) || VT == MVT::v2f32) && in combineSelect()
34101 (Subtarget.hasSSE1() && VT.getScalarType() == MVT::f32))) { in combineSelect()
34252 Cond.getOpcode() == ISD::SETCC && (VT == MVT::f32 || VT == MVT::f64)) { in combineSelect()
34260 AndNode = DAG.getZExtOrTrunc(AndNode, DL, MVT::i8); in combineSelect()
34273 CondVT.getVectorElementType() == MVT::i1 && in combineSelect()
34276 (VT.getVectorElementType() == MVT::i8 || in combineSelect()
34277 VT.getVectorElementType() == MVT::i16)) { in combineSelect()
34321 (VT.getVectorElementType() == MVT::i8 || in combineSelect()
34322 VT.getVectorElementType() == MVT::i16)) { in combineSelect()
34388 (VT.getVectorElementType() == MVT::i8 || in combineSelect()
34389 VT.getVectorElementType() == MVT::i16)) { in combineSelect()
34447 if (VT == MVT::x86mmx) { in combineSelect()
34448 LHS = DAG.getBitcast(MVT::i64, LHS); in combineSelect()
34449 RHS = DAG.getBitcast(MVT::i64, RHS); in combineSelect()
34450 SDValue newSelect = DAG.getNode(ISD::SELECT, DL, MVT::i64, Cond, LHS, RHS); in combineSelect()
34780 if (FalseOp.getValueType() != MVT::f80 || hasFPCMov(CC)) { in combineCMov()
34781 SDValue Ops[] = {FalseOp, TrueOp, DAG.getConstant(CC, DL, MVT::i8), in combineCMov()
34811 DAG.getConstant(ShAmt, DL, MVT::i8)); in combineCMov()
34830 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) { in combineCMov()
34832 if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff; in combineCMov()
34903 DAG.getConstant(CC, DL, MVT::i8), Cond }; in combineCMov()
34937 SDValue LOps[] = {FalseOp, TrueOp, DAG.getConstant(CC0, DL, MVT::i8), in combineCMov()
34940 SDValue Ops[] = {LCMOV, TrueOp, DAG.getConstant(CC1, DL, MVT::i8), Flags}; in combineCMov()
34973 DAG.getConstant(X86::COND_NE, DL, MVT::i8), in combineCMov()
35075 MVT OpsVT = MVT::getVectorVT(MVT::i16, RegSize / 16); in reduceVMULWidth()
35076 EVT ReducedVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts); in reduceVMULWidth()
35091 MVT ResVT = MVT::getVectorVT(MVT::i32, NumElts / 2); in reduceVMULWidth()
35145 MVT ResVT = MVT::getVectorVT(MVT::i32, RegSize / 32); in reduceVMULWidth()
35161 MVT ResVT = MVT::getVectorVT(MVT::i32, RegSize / 32); in reduceVMULWidth()
35175 DAG.getConstant(Shift, DL, MVT::i8)); in combineMulSpecial()
35244 DAG.getConstant(ShiftAmt, DL, MVT::i8)); in combineMulSpecial()
35246 DAG.getConstant(ScaleShift, DL, MVT::i8)); in combineMulSpecial()
35267 if (!VT.isVector() || VT.getVectorElementType() != MVT::i32) in combineMulToPMADDWD()
35272 MVT WVT = MVT::getVectorVT(MVT::i16, 2 * VT.getVectorNumElements()); in combineMulToPMADDWD()
35273 if (!((ExperimentalVectorWideningLegalization && VT == MVT::v2i32) || in combineMulToPMADDWD()
35297 MVT VT = MVT::getVectorVT(MVT::i32, Ops[0].getValueSizeInBits() / 32); in combineMulToPMADDWD()
35313 if (!VT.isVector() || VT.getVectorElementType() != MVT::i64 || in combineMulToPMULDQ()
35372 if (VT != MVT::i64 && VT != MVT::i32) in combineMul()
35426 DAG.getConstant(Log2_64(MulAmt1), DL, MVT::i8)); in combineMul()
35433 DAG.getConstant(Log2_64(MulAmt2), DL, MVT::i8)); in combineMul()
35447 C->getZExtValue() != (VT == MVT::i64 ? UINT64_MAX : UINT32_MAX) && in combineMul()
35456 MVT::i8))); in combineMul()
35465 DL, MVT::i8)); in combineMul()
35475 DL, MVT::i8)); in combineMul()
35482 DL, MVT::i8)); in combineMul()
35581 for (MVT SVT : { MVT::i8, MVT::i16, MVT::i32 }) { in combineShiftRightArithmetic()
35746 N0.getOpcode() == ISD::TRUNCATE && N1.isUndef() && VT == MVT::v16i8 && in combineVectorPack()
35747 N0.getOperand(0).getValueType() == MVT::v8i32) { in combineVectorPack()
35756 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i32, in combineVectorPack()
35757 N0.getOperand(0), DAG.getUNDEF(MVT::v8i32)); in combineVectorPack()
35809 assert(N1.getValueType() == MVT::i8 && "Unexpected shift amount type"); in combineVectorShiftImm()
35837 DAG.getConstant(NewShiftVal, SDLoc(N), MVT::i8)); in combineVectorShiftImm()
35879 ((N->getOpcode() == X86ISD::PINSRB && N->getValueType(0) == MVT::v16i8) || in combineVectorInsert()
35881 N->getValueType(0) == MVT::v8i16)) && in combineVectorInsert()
35920 if (VT == MVT::f32 || VT == MVT::f64) { in combineCompareEqual()
35956 DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CMP00, CMP01, in combineCompareEqual()
35957 DAG.getConstant(x86cc, DL, MVT::i8)); in combineCompareEqual()
35960 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v16i1, in combineCompareEqual()
35961 DAG.getConstant(0, DL, MVT::v16i1), in combineCompareEqual()
35963 return DAG.getZExtOrTrunc(DAG.getBitcast(MVT::i16, Ins), DL, in combineCompareEqual()
35969 MVT::i8)); in combineCompareEqual()
35971 bool is64BitFP = (CMP00.getValueType() == MVT::f64); in combineCompareEqual()
35972 MVT IntVT = is64BitFP ? MVT::i64 : MVT::i32; in combineCompareEqual()
35980 SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, in combineCompareEqual()
35982 SDValue Vector32 = DAG.getBitcast(MVT::v4f32, Vector64); in combineCompareEqual()
35983 OnesOrZeroesF = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, in combineCompareEqual()
35985 IntVT = MVT::i32; in combineCompareEqual()
35991 SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, in combineCompareEqual()
36005 MVT VT = N->getSimpleValueType(0); in combineANDXORWithAllOnesIntoANDNP()
36118 ((Subtarget.hasSSE1() && VT == MVT::i32) || in convertIntLogicToFPLogic()
36119 (Subtarget.hasSSE2() && VT == MVT::i64))) { in convertIntLogicToFPLogic()
36163 SDValue ShAmt = DAG.getConstant(EltBitWidth - ShiftVal, DL, MVT::i8); in combineAndMaskToShift()
36188 static bool hasBZHI(const X86Subtarget &Subtarget, MVT VT) { in hasBZHI()
36211 MVT VT = Node->getSimpleValueType(0); in combineAndLoadToBZHI()
36265 SDValue SizeC = DAG.getConstant(VT.getSizeInBits(), dl, MVT::i32); in combineAndLoadToBZHI()
36271 Index = DAG.getZExtOrTrunc(Index, dl, MVT::i32); in combineAndLoadToBZHI()
36273 SDValue Sub = DAG.getNode(ISD::SUB, dl, MVT::i32, SizeC, Index); in combineAndLoadToBZHI()
36274 Sub = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Sub); in combineAndLoadToBZHI()
36295 if (VT != MVT::i32 && VT != MVT::i64) in combineParity()
36318 if (VT == MVT::i64) { in combineParity()
36319 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, in combineParity()
36321 DAG.getConstant(32, DL, MVT::i8))); in combineParity()
36322 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, X); in combineParity()
36323 X = DAG.getNode(ISD::XOR, DL, MVT::i32, Lo, Hi); in combineParity()
36326 SDValue Parity = DAG.getNode(ISD::AND, DL, MVT::i32, in combineParity()
36327 DAG.getNode(ISD::CTPOP, DL, MVT::i32, X), in combineParity()
36328 DAG.getConstant(1, DL, MVT::i32)); in combineParity()
36331 assert(VT == MVT::i32 && "Unexpected VT!"); in combineParity()
36335 DAG.getConstant(16, DL, MVT::i8)); in combineParity()
36341 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, in combineParity()
36343 DAG.getConstant(8, DL, MVT::i8))); in combineParity()
36344 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X); in combineParity()
36345 SDVTList VTs = DAG.getVTList(MVT::i8, MVT::i32); in combineParity()
36360 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) { in combineAnd()
36362 MVT::v4i32, DAG.getNode(X86ISD::FAND, SDLoc(N), MVT::v4f32, in combineAnd()
36363 DAG.getBitcast(MVT::v4f32, N->getOperand(0)), in combineAnd()
36364 DAG.getBitcast(MVT::v4f32, N->getOperand(1)))); in combineAnd()
36368 if (VT == MVT::i64 && Subtarget.is64Bit() && in combineAnd()
36374 SDValue LHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N->getOperand(0)); in combineAnd()
36375 SDValue RHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N->getOperand(1)); in combineAnd()
36376 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, in combineAnd()
36377 DAG.getNode(ISD::AND, dl, MVT::i32, LHS, RHS)); in combineAnd()
36575 MVT BlendVT = VT.is256BitVector() ? MVT::v32i8 : MVT::v16i8; in combineLogicBlendIntoPBLENDV()
36599 SDValue Trunc = DAG.getZExtOrTrunc(Clz, dl, MVT::i32); in lowerX86CmpEqZeroToCtlzSrl()
36600 SDValue Scc = DAG.getNode(ISD::SRL, dl, MVT::i32, Trunc, in lowerX86CmpEqZeroToCtlzSrl()
36601 DAG.getConstant(Log2b, dl, MVT::i8)); in lowerX86CmpEqZeroToCtlzSrl()
36625 if (!N->hasOneUse() || !N->getSimpleValueType(0).bitsGE(MVT::i32) || in combineOrCmpEqZeroToCtlzSrl()
36635 N->getOperand(1).getValueType().bitsGE(MVT::i32); in combineOrCmpEqZeroToCtlzSrl()
36700 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) { in combineOr()
36701 return DAG.getBitcast(MVT::v4i32, in combineOr()
36702 DAG.getNode(X86ISD::FOR, SDLoc(N), MVT::v4f32, in combineOr()
36703 DAG.getBitcast(MVT::v4f32, N0), in combineOr()
36704 DAG.getBitcast(MVT::v4f32, N1))); in combineOr()
36728 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64) in combineOr()
36751 if (ShAmt0.getValueType() != MVT::i8) in combineOr()
36754 if (ShAmt1.getValueType() != MVT::i8) in combineOr()
36806 DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, ShAmt0)); in combineOr()
36814 MVT::i8, ShAmt0)); in combineOr()
36828 DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, ShAmt0)); in combineOr()
36834 DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, ShAmt0)); in combineOr()
36850 if (ResultType != MVT::i8 && ResultType != MVT::i1) in foldXorTruncShiftIntoCmp()
36871 if (ShiftTy != MVT::i16 && ShiftTy != MVT::i32 && ShiftTy != MVT::i64) in foldXorTruncShiftIntoCmp()
36910 case MVT::v16i8: in foldVectorXorShiftIntoCmp()
36911 case MVT::v8i16: in foldVectorXorShiftIntoCmp()
36912 case MVT::v4i32: if (!Subtarget.hasSSE2()) return SDValue(); break; in foldVectorXorShiftIntoCmp()
36913 case MVT::v2i64: if (!Subtarget.hasSSE42()) return SDValue(); break; in foldVectorXorShiftIntoCmp()
36914 case MVT::v32i8: in foldVectorXorShiftIntoCmp()
36915 case MVT::v16i16: in foldVectorXorShiftIntoCmp()
36916 case MVT::v8i32: in foldVectorXorShiftIntoCmp()
36917 case MVT::v4i64: if (!Subtarget.hasAVX2()) return SDValue(); break; in foldVectorXorShiftIntoCmp()
36956 if (DstElVT != MVT::i8 && DstElVT != MVT::i16 && DstElVT != MVT::i32) in isSATValidOnAVX512Subtarget()
37106 (SVT == MVT::i8 || SVT == MVT::i16) && in combineTruncateWithSat()
37107 (InSVT == MVT::i16 || InSVT == MVT::i32)) { in combineTruncateWithSat()
37110 if (SVT == MVT::i8 && InSVT == MVT::i32) { in combineTruncateWithSat()
37111 EVT MidVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, in combineTruncateWithSat()
37118 } else if (SVT == MVT::i8 || Subtarget.hasSSE41()) in combineTruncateWithSat()
37141 if (!((ScalarVT == MVT::i8 || ScalarVT == MVT::i16) && in detectAVGPattern()
37204 if (IsConstVectorInRange(Operands[1], 1, ScalarVT == MVT::i8 ? 256 : 65536) && in detectAVGPattern()
37284 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in combineLoad()
37307 if (!BV || BV->getValueType(0).getVectorElementType() != MVT::i1) in getOneTrueElt()
37501 assert(Mask.getValueType().getVectorElementType() == MVT::i1); in combineMaskedLoad()
37504 EVT NewMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, in combineMaskedLoad()
37639 assert(Mask.getValueType().getVectorElementType() == MVT::i1); in combineMaskedStore()
37642 EVT NewMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, in combineMaskedStore()
37668 VT.getVectorElementType() == MVT::i1) { in combineStore()
37680 if (VT == MVT::v1i1 && VT == StVT && Subtarget.hasAVX512() && in combineStore()
37682 StoredVal.getOperand(0).getValueType() == MVT::i8) { in combineStore()
37689 if ((VT == MVT::v2i1 || VT == MVT::v4i1) && VT == StVT && in combineStore()
37694 StoredVal = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops); in combineStore()
37701 if ((VT == MVT::v8i1 || VT == MVT::v16i1 || VT == MVT::v32i1 || in combineStore()
37702 VT == MVT::v64i1) && VT == StVT && TLI.isTypeLegal(VT) && in combineStore()
37705 if (VT == MVT::v64i1 && !Subtarget.is64Bit()) { in combineStore()
37706 SDValue Lo = DAG.getBuildVector(MVT::v32i1, dl, in combineStore()
37709 SDValue Hi = DAG.getBuildVector(MVT::v32i1, dl, in combineStore()
37726 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ch0, Ch1); in combineStore()
37761 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ch0, Ch1); in combineStore()
37834 MVT StoreType = MVT::i8; in combineStore()
37835 for (MVT Tp : MVT::integer_valuetypes()) { in combineStore()
37841 if (TLI.isTypeLegal(MVT::f64) && StoreType.getSizeInBits() < 64 && in combineStore()
37843 StoreType = MVT::f64; in combineStore()
37865 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); in combineStore()
37882 (VT == MVT::i64 && F64IsLegal && !Subtarget.is64Bit())) && in combineStore()
37904 MVT LdVT = (Subtarget.is64Bit() && in combineStore()
37905 (!VT.isFloatingPoint() || !F64IsLegal)) ? MVT::i64 : MVT::f64; in combineStore()
37919 SDValue LoLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), LoAddr, in combineStore()
37922 SDValue HiLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), HiAddr, in combineStore()
37940 return DAG.getNode(ISD::TokenFactor, StDL, MVT::Other, LoSt, HiSt); in combineStore()
37949 if (VT == MVT::i64 && F64IsLegal && !Subtarget.is64Bit() && in combineStore()
37954 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64, VecSize / 64); in combineStore()
37956 SDValue NewExtract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in combineStore()
37998 MVT VT = LHS.getSimpleValueType(); in isHorizontalBinOp()
38100 if (((Subtarget.hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) || in combineFaddFsub()
38101 (Subtarget.hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) && in combineFaddFsub()
38177 if (SrcVT.getScalarType() == MVT::i64 && TLI.isOperationLegal(Opcode, VT) && in combineTruncatedArithmetic()
38276 if (!((InSVT == MVT::i32 || InSVT == MVT::i64) && in combineVectorTruncation()
38277 (OutSVT == MVT::i8 || OutSVT == MVT::i16) && isPowerOf2_32(NumElems) && in combineVectorTruncation()
38283 ((OutSVT == MVT::i8 && InSVT != MVT::i64) || in combineVectorTruncation()
38284 (InSVT == MVT::i32 && OutSVT == MVT::i16))) in combineVectorTruncation()
38291 if (Subtarget.hasSSE41() || OutSVT == MVT::i8) in combineVectorTruncation()
38293 if (InSVT == MVT::i32) in combineVectorTruncation()
38316 MVT VT = N->getValueType(0).getSimpleVT(); in combineVectorSignBitsTruncation()
38317 MVT SVT = VT.getScalarType(); in combineVectorSignBitsTruncation()
38319 MVT InVT = In.getValueType().getSimpleVT(); in combineVectorSignBitsTruncation()
38320 MVT InSVT = InVT.getScalarType(); in combineVectorSignBitsTruncation()
38325 if (SVT != MVT::i8 && SVT != MVT::i16 && SVT != MVT::i32) in combineVectorSignBitsTruncation()
38327 if (InSVT != MVT::i16 && InSVT != MVT::i32 && InSVT != MVT::i64) in combineVectorSignBitsTruncation()
38368 if (!VT.isVector() || VT.getVectorElementType() != MVT::i16 || in combinePMULH()
38375 if (InVT.getVectorElementType() != MVT::i32) in combinePMULH()
38420 if (ScalarVT != MVT::i16 || NumElems < 8 || !isPowerOf2_32(NumElems)) in detectPMADDUBSW()
38461 if (N00.getValueType().getVectorElementType() != MVT::i8 || in detectPMADDUBSW()
38462 N01.getValueType().getVectorElementType() != MVT::i8 || in detectPMADDUBSW()
38463 N10.getValueType().getVectorElementType() != MVT::i8 || in detectPMADDUBSW()
38464 N11.getValueType().getVectorElementType() != MVT::i8) in detectPMADDUBSW()
38532 assert(InVT.getScalarType() == MVT::i8 && in detectPMADDUBSW()
38535 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, in detectPMADDUBSW()
38571 if (Src.getOpcode() == ISD::BITCAST && VT == MVT::i32) { in combineTruncate()
38573 if (BCSrc.getValueType() == MVT::x86mmx) in combineTruncate()
38574 return DAG.getNode(X86ISD::MMX_MOVD2W, DL, MVT::i32, BCSrc); in combineTruncate()
38675 if (Arg.getOpcode() == ISD::FMUL && (SVT == MVT::f32 || SVT == MVT::f64) && in combineFneg()
38710 MVT VT = N->getSimpleValueType(0); in lowerX86FPLogicOp()
38718 MVT IntSVT = MVT::getIntegerVT(IntBits); in lowerX86FPLogicOp()
38719 MVT IntVT = MVT::getVectorVT(IntSVT, VT.getSizeInBits() / IntBits); in lowerX86FPLogicOp()
38757 N->getValueType(0) == MVT::v4i32) { in combineXor()
38759 MVT::v4i32, DAG.getNode(X86ISD::FXOR, SDLoc(N), MVT::v4f32, in combineXor()
38760 DAG.getBitcast(MVT::v4f32, N->getOperand(0)), in combineXor()
38761 DAG.getBitcast(MVT::v4f32, N->getOperand(1)))); in combineXor()
38840 if (!((VT == MVT::f32 && Subtarget.hasSSE1()) || in combineFAndFNotToFAndn()
38841 (VT == MVT::f64 && Subtarget.hasSSE2()) || in combineFAndFNotToFAndn()
38842 (VT == MVT::v4f32 && Subtarget.hasSSE1() && !Subtarget.hasSSE2()))) in combineFAndFNotToFAndn()
38943 if (!((Subtarget.hasSSE1() && (VT == MVT::f32 || VT == MVT::v4f32)) || in combineFMinNumFMaxNum()
38944 (Subtarget.hasSSE2() && (VT == MVT::f64 || VT == MVT::v2f64)) || in combineFMinNumFMaxNum()
38945 (Subtarget.hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64)))) in combineFMinNumFMaxNum()
39011 MVT VT = N->getSimpleValueType(0); in combineAndnp()
39049 return DAG.getNode(X86ISD::BT, SDLoc(N), MVT::i32, N0, DemandedN1); in combineBT()
39062 if (ExtraVT != MVT::i16) in combineSextInRegCmov()
39112 if (VT == MVT::v4i64 && (N0.getOpcode() == ISD::ANY_EXTEND || in combineSignExtendInReg()
39122 if (N00.getValueType() == MVT::v4i32 && ExtraVT.getSizeInBits() < 128) { in combineSignExtendInReg()
39123 SDValue Tmp = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32, in combineSignExtendInReg()
39125 return DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i64, Tmp); in combineSignExtendInReg()
39144 if (VT != MVT::i64) in promoteExtBeforeAdd()
39226 if (TargetVT != MVT::i32 && TargetVT != MVT::i64) in combineToExtendCMOV()
39231 if (VT != MVT::i16 && !(ExtendOpcode == ISD::SIGN_EXTEND && VT == MVT::i32)) in combineToExtendCMOV()
39237 if (TargetVT == MVT::i64 && ExtendOpcode != ISD::SIGN_EXTEND) in combineToExtendCMOV()
39238 ExtendVT = MVT::i32; in combineToExtendCMOV()
39278 if (SVT != MVT::i64 && SVT != MVT::i32 && SVT != MVT::i16 && SVT != MVT::i8) in combineToExtendBoolVectorInReg()
39280 if (InSVT != MVT::i1 || N0.getOpcode() != ISD::BITCAST) in combineToExtendBoolVectorInReg()
39330 EVT CCVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElts); in combineToExtendBoolVectorInReg()
39385 if (SVT != MVT::i64 && SVT != MVT::i32 && SVT != MVT::i16) in combineToExtendVectorInReg()
39387 if (InSVT != MVT::i32 && InSVT != MVT::i16 && InSVT != MVT::i8) in combineToExtendVectorInReg()
39479 if (SVT != MVT::i8 && SVT != MVT::i16 && SVT != MVT::i32 && in combineExtSetcc()
39480 SVT != MVT::i64 && SVT != MVT::f32 && SVT != MVT::f64) in combineExtSetcc()
39525 if (InVT == MVT::i1 && N0.getOpcode() == ISD::XOR && in combineSext()
39593 if ((ScalarVT != MVT::f32 && ScalarVT != MVT::f64) || !Subtarget.hasAnyFMA()) in combineFMA()
39752 if (peekThroughBitcasts(X).getValueType() == MVT::f128 || in combineVectorSizedSetCCEquality()
39753 peekThroughBitcasts(Y).getValueType() == MVT::f128) in combineVectorSizedSetCCEquality()
39762 EVT VecVT = OpSize == 512 ? MVT::v16i32 : in combineVectorSizedSetCCEquality()
39763 OpSize == 256 ? MVT::v32i8 : in combineVectorSizedSetCCEquality()
39764 MVT::v16i8; in combineVectorSizedSetCCEquality()
39765 EVT CmpVT = OpSize == 512 ? MVT::v16i1 : VecVT; in combineVectorSizedSetCCEquality()
39786 return DAG.getSetCC(DL, VT, DAG.getBitcast(MVT::i16, Cmp), in combineVectorSizedSetCCEquality()
39787 DAG.getConstant(0xFFFF, DL, MVT::i16), CC); in combineVectorSizedSetCCEquality()
39793 SDValue MovMsk = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Cmp); in combineVectorSizedSetCCEquality()
39795 MVT::i32); in combineVectorSizedSetCCEquality()
39831 if (VT.isVector() && VT.getVectorElementType() == MVT::i1 && in combineSetCC()
39841 (LHS.getOperand(0).getValueType().getVectorElementType() == MVT::i1); in combineSetCC()
39866 VT.getVectorElementType() == MVT::i1 && in combineSetCC()
39869 (OpVT.getVectorElementType() == MVT::i8 || in combineSetCC()
39870 OpVT.getVectorElementType() == MVT::i16)) { in combineSetCC()
39878 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32 && in combineSetCC()
39879 LHS.getValueType() == MVT::v4f32) in combineSetCC()
39888 MVT SrcVT = Src.getSimpleValueType(); in combineMOVMSK()
39889 MVT VT = N->getSimpleValueType(0); in combineMOVMSK()
39893 assert(VT== MVT::i32 && "Unexpected result type"); in combineMOVMSK()
39931 MVT VT = Src.getSimpleValueType(); in combineMOVMSK()
39939 if (VT.getVectorElementType() == MVT::i8) { in combineMOVMSK()
39940 VT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements() / 2); in combineMOVMSK()
39982 MVT EltVT = ScalarSize > 32 ? MVT::i64 : MVT::i32; in combineGatherScatter()
40051 SDValue Cond = DAG.getConstant(CC, DL, MVT::i8); in combineBrCond()
40116 EVT DstVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, in combineUIntToFP()
40150 EVT DstVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, in combineSIntToFP()
40181 if (VT == MVT::f16 || VT == MVT::f128) in combineSIntToFP()
40186 if (Subtarget.hasDQI() && VT != MVT::f80) in combineSIntToFP()
40191 !Subtarget.is64Bit() && LdVT == MVT::i64) { in combineSIntToFP()
40202 assert(Flags.getValueType() == MVT::i32 && "Unexpected VT!"); in needCarryOrOverflowFlag()
40240 assert(Flags.getValueType() == MVT::i32 && "Unexpected VT!"); in onlyZeroFlagUsed()
40294 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in combineCMP()
40342 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in combineCMP()
40347 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in combineCMP()
40356 MVT VT = N->getSimpleValueType(0); in combineSBB()
40357 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in combineSBB()
40383 MVT::i8), in combineADC()
40390 MVT VT = N->getSimpleValueType(0); in combineADC()
40391 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in combineADC()
40445 DAG.getConstant(X86::COND_B, DL, MVT::i8), in combineAddOrSubToADCOrSBB()
40463 DAG.getConstant(X86::COND_B, DL, MVT::i8), in combineAddOrSubToADCOrSBB()
40473 DAG.getVTList(VT, MVT::i32), X, in combineAddOrSubToADCOrSBB()
40493 DAG.getVTList(VT, MVT::i32), X, in combineAddOrSubToADCOrSBB()
40520 SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32); in combineAddOrSubToADCOrSBB()
40523 DAG.getConstant(X86::COND_B, DL, MVT::i8), in combineAddOrSubToADCOrSBB()
40534 SDValue Cmp1 = DAG.getNode(X86ISD::CMP, DL, MVT::i32, Z, One); in combineAddOrSubToADCOrSBB()
40536 DAG.getConstant(X86::COND_B, DL, MVT::i8), Cmp1); in combineAddOrSubToADCOrSBB()
40542 SDValue Cmp1 = DAG.getNode(X86ISD::CMP, DL, MVT::i32, Z, One); in combineAddOrSubToADCOrSBB()
40545 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in combineAddOrSubToADCOrSBB()
40584 EVT ReducedVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, in combineLoopMAddPattern()
40586 EVT MAddVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, in combineLoopMAddPattern()
40592 MVT VT = MVT::getVectorVT(MVT::i32, Ops[0].getValueSizeInBits() / 32); in combineLoopMAddPattern()
40632 !(VT.getVectorElementType() == MVT::i32)) in combineLoopSADPattern()
40666 MVT ResVT = MVT::getVectorVT(MVT::i32, Sad.getValueSizeInBits() / 32); in combineLoopSADPattern()
40744 if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 || in matchPMADDWD()
40816 assert(InVT.getScalarType() == MVT::i32 && in matchPMADDWD()
40819 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, in matchPMADDWD()
40821 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, in matchPMADDWD()
40843 !(VT.getVectorElementType() == MVT::i8 || in combineAddToSUBUS()
40844 VT.getVectorElementType() == MVT::i16)) in combineAddToSUBUS()
40877 if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 || in matchPMADDWD_2()
40903 if (InVT.getVectorElementType() != MVT::i16 || N01.getValueType() != InVT || in matchPMADDWD_2()
40975 assert(InVT.getScalarType() == MVT::i16 && in matchPMADDWD_2()
40978 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, in matchPMADDWD_2()
41005 if ((VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v16i16 || in combineAdd()
41006 VT == MVT::v8i32) && in combineAdd()
41034 if (!(Subtarget.hasSSE2() && (VT == MVT::v16i8 || VT == MVT::v8i16)) && in combineSubToSubus()
41035 !(Subtarget.hasSSSE3() && (VT == MVT::v8i32 || VT == MVT::v8i64)) && in combineSubToSubus()
41036 !(Subtarget.hasAVX() && (VT == MVT::v32i8 || VT == MVT::v16i16)) && in combineSubToSubus()
41037 !(Subtarget.useBWIRegs() && (VT == MVT::v64i8 || VT == MVT::v32i16 || in combineSubToSubus()
41038 VT == MVT::v16i32 || VT == MVT::v8i64))) in combineSubToSubus()
41075 if (VT != MVT::v8i32 && VT != MVT::v16i32 && VT != MVT::v8i64) in combineSubToSubus()
41085 if ((VT == MVT::v8i64 && NumZeros < 48) || NumZeros < 16) in combineSubToSubus()
41090 if (VT == MVT::v8i32 || VT == MVT::v8i64) in combineSubToSubus()
41091 ShrinkedType = MVT::v8i16; in combineSubToSubus()
41093 ShrinkedType = NumZeros >= 24 ? MVT::v16i8 : MVT::v16i16; in combineSubToSubus()
41139 if ((VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v16i16 || in combineSub()
41140 VT == MVT::v8i32) && in combineSub()
41163 MVT VT = N->getSimpleValueType(0); in combineVectorCompare()
41182 MVT OpVT = N->getSimpleValueType(0); in combineInsertSubvector()
41184 bool IsI1Vector = OpVT.getVectorElementType() == MVT::i1; in combineInsertSubvector()
41191 MVT SubVecVT = SubVec.getSimpleValueType(); in combineInsertSubvector()
41230 MVT CastVT = SubVec.getOperand(0).getSimpleValueType(); in combineInsertSubvector()
41232 MVT NewVT = MVT::getVectorVT(CastVT.getVectorElementType(), NumElems); in combineInsertSubvector()
41377 MVT OpVT = N->getSimpleValueType(0); in combineExtractSubvector()
41385 if (OpVT.getScalarType() == MVT::i1) in combineExtractSubvector()
41399 if (OpVT == MVT::v2f64 && InVec.getValueType() == MVT::v4f64) { in combineExtractSubvector()
41402 InVec.getOperand(0).getValueType() == MVT::v4i32) { in combineExtractSubvector()
41407 InVec.getOperand(0).getValueType() == MVT::v4f32) { in combineExtractSubvector()
41431 DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, in combineExtractSubvector()
41450 if (VT == MVT::v1i1 && Src.getOpcode() == ISD::AND && Src.hasOneUse()) in combineScalarToVector()
41453 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), MVT::v1i1, in combineScalarToVector()
41457 if (VT == MVT::v1i1 && Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in combineScalarToVector()
41459 Src.getOperand(0).getValueType().getVectorElementType() == MVT::i1) in combineScalarToVector()
41638 if (Opc == ISD::SHL && VT.isVector() && VT.getVectorElementType() == MVT::i8) in isTypeDesirableForOp()
41645 if (Opc == ISD::MUL && VT == MVT::i8) in isTypeDesirableForOp()
41650 if (VT == MVT::i16) { in isTypeDesirableForOp()
41684 return DAG.getNode(X86ISD::NT_BRIND, dl, MVT::Other, Value, Addr); in expandIndirectJTBranch()
41692 bool Is8BitMulByConstant = VT == MVT::i8 && Op.getOpcode() == ISD::MUL && in IsDesirableToPromoteOp()
41699 if (VT != MVT::i16 && !Is8BitMulByConstant) in IsDesirableToPromoteOp()
41766 PVT = MVT::i32; in IsDesirableToPromoteOp()
42221 Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op), MVT::i64); in LowerAsmOperandForConstraint()
42247 Result = DAG.getTargetConstant(CST->getSExtValue(), SDLoc(Op), MVT::i64); in LowerAsmOperandForConstraint()
42327 MVT VT) const { in getRegForInlineAsmConstraint()
42342 case MVT::i32: in getRegForInlineAsmConstraint()
42344 case MVT::i16: in getRegForInlineAsmConstraint()
42346 case MVT::i8: in getRegForInlineAsmConstraint()
42348 case MVT::i1: in getRegForInlineAsmConstraint()
42350 case MVT::i64: in getRegForInlineAsmConstraint()
42357 if (VT == MVT::i32 || VT == MVT::f32) in getRegForInlineAsmConstraint()
42359 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
42361 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
42363 if (VT == MVT::i64 || VT == MVT::f64) in getRegForInlineAsmConstraint()
42370 if (VT == MVT::i32 || VT == MVT::f32) in getRegForInlineAsmConstraint()
42372 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
42374 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
42376 if (VT == MVT::i64) in getRegForInlineAsmConstraint()
42381 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
42383 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
42385 if (VT == MVT::i32 || VT == MVT::f32 || !Subtarget.is64Bit()) in getRegForInlineAsmConstraint()
42389 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
42391 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
42393 if (VT == MVT::i32 || !Subtarget.is64Bit()) in getRegForInlineAsmConstraint()
42399 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT)) in getRegForInlineAsmConstraint()
42401 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT)) in getRegForInlineAsmConstraint()
42418 case MVT::f32: in getRegForInlineAsmConstraint()
42419 case MVT::i32: in getRegForInlineAsmConstraint()
42423 case MVT::f64: in getRegForInlineAsmConstraint()
42424 case MVT::i64: in getRegForInlineAsmConstraint()
42430 case MVT::v16i8: in getRegForInlineAsmConstraint()
42431 case MVT::v8i16: in getRegForInlineAsmConstraint()
42432 case MVT::v4i32: in getRegForInlineAsmConstraint()
42433 case MVT::v2i64: in getRegForInlineAsmConstraint()
42434 case MVT::v4f32: in getRegForInlineAsmConstraint()
42435 case MVT::v2f64: in getRegForInlineAsmConstraint()
42440 case MVT::v32i8: in getRegForInlineAsmConstraint()
42441 case MVT::v16i16: in getRegForInlineAsmConstraint()
42442 case MVT::v8i32: in getRegForInlineAsmConstraint()
42443 case MVT::v4i64: in getRegForInlineAsmConstraint()
42444 case MVT::v8f32: in getRegForInlineAsmConstraint()
42445 case MVT::v4f64: in getRegForInlineAsmConstraint()
42449 case MVT::v8f64: in getRegForInlineAsmConstraint()
42450 case MVT::v16f32: in getRegForInlineAsmConstraint()
42451 case MVT::v16i32: in getRegForInlineAsmConstraint()
42452 case MVT::v8i64: in getRegForInlineAsmConstraint()
42477 case MVT::i32: in getRegForInlineAsmConstraint()
42479 case MVT::i16: in getRegForInlineAsmConstraint()
42481 case MVT::i8: in getRegForInlineAsmConstraint()
42483 case MVT::i1: in getRegForInlineAsmConstraint()
42485 case MVT::i64: in getRegForInlineAsmConstraint()
42560 if (TRI->isTypeLegalForClass(*Res.second, VT) || VT == MVT::Other) in getRegForInlineAsmConstraint()
42618 if (VT == MVT::f32 || VT == MVT::i32) in getRegForInlineAsmConstraint()
42620 else if (VT == MVT::f64 || VT == MVT::i64) in getRegForInlineAsmConstraint()