Lines Matching refs:ISD
246 if (ISD::isConstantSplatVectorAllOnes(Op.getNode())) in isZeroingInactiveLanes()
252 case ISD::INTRINSIC_WO_CHAIN: in isZeroingInactiveLanes()
300 setOperationAction(ISD::LOAD, MVT::i64x8, Custom); in AArch64TargetLowering()
301 setOperationAction(ISD::STORE, MVT::i64x8, Custom); in AArch64TargetLowering()
379 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); in AArch64TargetLowering()
380 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); in AArch64TargetLowering()
381 setOperationAction(ISD::SETCC, MVT::i32, Custom); in AArch64TargetLowering()
382 setOperationAction(ISD::SETCC, MVT::i64, Custom); in AArch64TargetLowering()
383 setOperationAction(ISD::SETCC, MVT::f16, Custom); in AArch64TargetLowering()
384 setOperationAction(ISD::SETCC, MVT::f32, Custom); in AArch64TargetLowering()
385 setOperationAction(ISD::SETCC, MVT::f64, Custom); in AArch64TargetLowering()
386 setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Custom); in AArch64TargetLowering()
387 setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Custom); in AArch64TargetLowering()
388 setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Custom); in AArch64TargetLowering()
389 setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Custom); in AArch64TargetLowering()
390 setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Custom); in AArch64TargetLowering()
391 setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Custom); in AArch64TargetLowering()
392 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); in AArch64TargetLowering()
393 setOperationAction(ISD::BITREVERSE, MVT::i64, Legal); in AArch64TargetLowering()
394 setOperationAction(ISD::BRCOND, MVT::Other, Custom); in AArch64TargetLowering()
395 setOperationAction(ISD::BR_CC, MVT::i32, Custom); in AArch64TargetLowering()
396 setOperationAction(ISD::BR_CC, MVT::i64, Custom); in AArch64TargetLowering()
397 setOperationAction(ISD::BR_CC, MVT::f16, Custom); in AArch64TargetLowering()
398 setOperationAction(ISD::BR_CC, MVT::f32, Custom); in AArch64TargetLowering()
399 setOperationAction(ISD::BR_CC, MVT::f64, Custom); in AArch64TargetLowering()
400 setOperationAction(ISD::SELECT, MVT::i32, Custom); in AArch64TargetLowering()
401 setOperationAction(ISD::SELECT, MVT::i64, Custom); in AArch64TargetLowering()
402 setOperationAction(ISD::SELECT, MVT::f16, Custom); in AArch64TargetLowering()
403 setOperationAction(ISD::SELECT, MVT::f32, Custom); in AArch64TargetLowering()
404 setOperationAction(ISD::SELECT, MVT::f64, Custom); in AArch64TargetLowering()
405 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); in AArch64TargetLowering()
406 setOperationAction(ISD::SELECT_CC, MVT::i64, Custom); in AArch64TargetLowering()
407 setOperationAction(ISD::SELECT_CC, MVT::f16, Custom); in AArch64TargetLowering()
408 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); in AArch64TargetLowering()
409 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom); in AArch64TargetLowering()
410 setOperationAction(ISD::BR_JT, MVT::Other, Custom); in AArch64TargetLowering()
411 setOperationAction(ISD::JumpTable, MVT::i64, Custom); in AArch64TargetLowering()
413 setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom); in AArch64TargetLowering()
414 setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom); in AArch64TargetLowering()
415 setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom); in AArch64TargetLowering()
417 setOperationAction(ISD::FREM, MVT::f32, Expand); in AArch64TargetLowering()
418 setOperationAction(ISD::FREM, MVT::f64, Expand); in AArch64TargetLowering()
419 setOperationAction(ISD::FREM, MVT::f80, Expand); in AArch64TargetLowering()
421 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand); in AArch64TargetLowering()
425 setOperationAction(ISD::XOR, MVT::i32, Custom); in AArch64TargetLowering()
426 setOperationAction(ISD::XOR, MVT::i64, Custom); in AArch64TargetLowering()
430 setOperationAction(ISD::FABS, MVT::f128, Expand); in AArch64TargetLowering()
431 setOperationAction(ISD::FADD, MVT::f128, LibCall); in AArch64TargetLowering()
432 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand); in AArch64TargetLowering()
433 setOperationAction(ISD::FCOS, MVT::f128, Expand); in AArch64TargetLowering()
434 setOperationAction(ISD::FDIV, MVT::f128, LibCall); in AArch64TargetLowering()
435 setOperationAction(ISD::FMA, MVT::f128, Expand); in AArch64TargetLowering()
436 setOperationAction(ISD::FMUL, MVT::f128, LibCall); in AArch64TargetLowering()
437 setOperationAction(ISD::FNEG, MVT::f128, Expand); in AArch64TargetLowering()
438 setOperationAction(ISD::FPOW, MVT::f128, Expand); in AArch64TargetLowering()
439 setOperationAction(ISD::FREM, MVT::f128, Expand); in AArch64TargetLowering()
440 setOperationAction(ISD::FRINT, MVT::f128, Expand); in AArch64TargetLowering()
441 setOperationAction(ISD::FSIN, MVT::f128, Expand); in AArch64TargetLowering()
442 setOperationAction(ISD::FSINCOS, MVT::f128, Expand); in AArch64TargetLowering()
443 setOperationAction(ISD::FSQRT, MVT::f128, Expand); in AArch64TargetLowering()
444 setOperationAction(ISD::FSUB, MVT::f128, LibCall); in AArch64TargetLowering()
445 setOperationAction(ISD::FTRUNC, MVT::f128, Expand); in AArch64TargetLowering()
446 setOperationAction(ISD::SETCC, MVT::f128, Custom); in AArch64TargetLowering()
447 setOperationAction(ISD::STRICT_FSETCC, MVT::f128, Custom); in AArch64TargetLowering()
448 setOperationAction(ISD::STRICT_FSETCCS, MVT::f128, Custom); in AArch64TargetLowering()
449 setOperationAction(ISD::BR_CC, MVT::f128, Custom); in AArch64TargetLowering()
450 setOperationAction(ISD::SELECT, MVT::f128, Custom); in AArch64TargetLowering()
451 setOperationAction(ISD::SELECT_CC, MVT::f128, Custom); in AArch64TargetLowering()
452 setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom); in AArch64TargetLowering()
458 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); in AArch64TargetLowering()
459 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); in AArch64TargetLowering()
460 setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom); in AArch64TargetLowering()
461 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom); in AArch64TargetLowering()
462 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom); in AArch64TargetLowering()
463 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i128, Custom); in AArch64TargetLowering()
464 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); in AArch64TargetLowering()
465 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); in AArch64TargetLowering()
466 setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom); in AArch64TargetLowering()
467 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom); in AArch64TargetLowering()
468 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom); in AArch64TargetLowering()
469 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i128, Custom); in AArch64TargetLowering()
470 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); in AArch64TargetLowering()
471 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); in AArch64TargetLowering()
472 setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom); in AArch64TargetLowering()
473 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom); in AArch64TargetLowering()
474 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom); in AArch64TargetLowering()
475 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, Custom); in AArch64TargetLowering()
476 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom); in AArch64TargetLowering()
477 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom); in AArch64TargetLowering()
478 setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom); in AArch64TargetLowering()
479 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom); in AArch64TargetLowering()
480 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom); in AArch64TargetLowering()
481 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, Custom); in AArch64TargetLowering()
482 setOperationAction(ISD::FP_ROUND, MVT::f16, Custom); in AArch64TargetLowering()
483 setOperationAction(ISD::FP_ROUND, MVT::f32, Custom); in AArch64TargetLowering()
484 setOperationAction(ISD::FP_ROUND, MVT::f64, Custom); in AArch64TargetLowering()
485 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom); in AArch64TargetLowering()
486 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom); in AArch64TargetLowering()
487 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Custom); in AArch64TargetLowering()
489 setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i32, Custom); in AArch64TargetLowering()
490 setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i64, Custom); in AArch64TargetLowering()
491 setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i32, Custom); in AArch64TargetLowering()
492 setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i64, Custom); in AArch64TargetLowering()
495 setOperationAction(ISD::VASTART, MVT::Other, Custom); in AArch64TargetLowering()
496 setOperationAction(ISD::VAARG, MVT::Other, Custom); in AArch64TargetLowering()
497 setOperationAction(ISD::VACOPY, MVT::Other, Custom); in AArch64TargetLowering()
498 setOperationAction(ISD::VAEND, MVT::Other, Expand); in AArch64TargetLowering()
501 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); in AArch64TargetLowering()
502 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); in AArch64TargetLowering()
505 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Custom); in AArch64TargetLowering()
507 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand); in AArch64TargetLowering()
510 setOperationAction(ISD::ConstantPool, MVT::i64, Custom); in AArch64TargetLowering()
513 setOperationAction(ISD::BlockAddress, MVT::i64, Custom); in AArch64TargetLowering()
516 setOperationAction(ISD::ROTL, MVT::i32, Expand); in AArch64TargetLowering()
517 setOperationAction(ISD::ROTL, MVT::i64, Expand); in AArch64TargetLowering()
519 setOperationAction(ISD::ROTL, VT, Expand); in AArch64TargetLowering()
520 setOperationAction(ISD::ROTR, VT, Expand); in AArch64TargetLowering()
524 setOperationAction(ISD::MULHU, MVT::i32, Expand); in AArch64TargetLowering()
525 setOperationAction(ISD::MULHS, MVT::i32, Expand); in AArch64TargetLowering()
528 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand); in AArch64TargetLowering()
529 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand); in AArch64TargetLowering()
531 setOperationAction(ISD::CTPOP, MVT::i32, Custom); in AArch64TargetLowering()
532 setOperationAction(ISD::CTPOP, MVT::i64, Custom); in AArch64TargetLowering()
533 setOperationAction(ISD::CTPOP, MVT::i128, Custom); in AArch64TargetLowering()
535 setOperationAction(ISD::PARITY, MVT::i64, Custom); in AArch64TargetLowering()
536 setOperationAction(ISD::PARITY, MVT::i128, Custom); in AArch64TargetLowering()
538 setOperationAction(ISD::ABS, MVT::i32, Custom); in AArch64TargetLowering()
539 setOperationAction(ISD::ABS, MVT::i64, Custom); in AArch64TargetLowering()
541 setOperationAction(ISD::SDIVREM, MVT::i32, Expand); in AArch64TargetLowering()
542 setOperationAction(ISD::SDIVREM, MVT::i64, Expand); in AArch64TargetLowering()
544 setOperationAction(ISD::SDIVREM, VT, Expand); in AArch64TargetLowering()
545 setOperationAction(ISD::UDIVREM, VT, Expand); in AArch64TargetLowering()
547 setOperationAction(ISD::SREM, MVT::i32, Expand); in AArch64TargetLowering()
548 setOperationAction(ISD::SREM, MVT::i64, Expand); in AArch64TargetLowering()
549 setOperationAction(ISD::UDIVREM, MVT::i32, Expand); in AArch64TargetLowering()
550 setOperationAction(ISD::UDIVREM, MVT::i64, Expand); in AArch64TargetLowering()
551 setOperationAction(ISD::UREM, MVT::i32, Expand); in AArch64TargetLowering()
552 setOperationAction(ISD::UREM, MVT::i64, Expand); in AArch64TargetLowering()
555 setOperationAction(ISD::SADDO, MVT::i32, Custom); in AArch64TargetLowering()
556 setOperationAction(ISD::SADDO, MVT::i64, Custom); in AArch64TargetLowering()
557 setOperationAction(ISD::UADDO, MVT::i32, Custom); in AArch64TargetLowering()
558 setOperationAction(ISD::UADDO, MVT::i64, Custom); in AArch64TargetLowering()
559 setOperationAction(ISD::SSUBO, MVT::i32, Custom); in AArch64TargetLowering()
560 setOperationAction(ISD::SSUBO, MVT::i64, Custom); in AArch64TargetLowering()
561 setOperationAction(ISD::USUBO, MVT::i32, Custom); in AArch64TargetLowering()
562 setOperationAction(ISD::USUBO, MVT::i64, Custom); in AArch64TargetLowering()
563 setOperationAction(ISD::SMULO, MVT::i32, Custom); in AArch64TargetLowering()
564 setOperationAction(ISD::SMULO, MVT::i64, Custom); in AArch64TargetLowering()
565 setOperationAction(ISD::UMULO, MVT::i32, Custom); in AArch64TargetLowering()
566 setOperationAction(ISD::UMULO, MVT::i64, Custom); in AArch64TargetLowering()
568 setOperationAction(ISD::ADDCARRY, MVT::i32, Custom); in AArch64TargetLowering()
569 setOperationAction(ISD::ADDCARRY, MVT::i64, Custom); in AArch64TargetLowering()
570 setOperationAction(ISD::SUBCARRY, MVT::i32, Custom); in AArch64TargetLowering()
571 setOperationAction(ISD::SUBCARRY, MVT::i64, Custom); in AArch64TargetLowering()
572 setOperationAction(ISD::SADDO_CARRY, MVT::i32, Custom); in AArch64TargetLowering()
573 setOperationAction(ISD::SADDO_CARRY, MVT::i64, Custom); in AArch64TargetLowering()
574 setOperationAction(ISD::SSUBO_CARRY, MVT::i32, Custom); in AArch64TargetLowering()
575 setOperationAction(ISD::SSUBO_CARRY, MVT::i64, Custom); in AArch64TargetLowering()
577 setOperationAction(ISD::FSIN, MVT::f32, Expand); in AArch64TargetLowering()
578 setOperationAction(ISD::FSIN, MVT::f64, Expand); in AArch64TargetLowering()
579 setOperationAction(ISD::FCOS, MVT::f32, Expand); in AArch64TargetLowering()
580 setOperationAction(ISD::FCOS, MVT::f64, Expand); in AArch64TargetLowering()
581 setOperationAction(ISD::FPOW, MVT::f32, Expand); in AArch64TargetLowering()
582 setOperationAction(ISD::FPOW, MVT::f64, Expand); in AArch64TargetLowering()
583 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); in AArch64TargetLowering()
584 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); in AArch64TargetLowering()
586 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Custom); in AArch64TargetLowering()
588 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote); in AArch64TargetLowering()
590 for (auto Op : {ISD::FREM, ISD::FPOW, ISD::FPOWI, in AArch64TargetLowering()
591 ISD::FCOS, ISD::FSIN, ISD::FSINCOS, in AArch64TargetLowering()
592 ISD::FEXP, ISD::FEXP2, ISD::FLOG, in AArch64TargetLowering()
593 ISD::FLOG2, ISD::FLOG10, ISD::STRICT_FREM, in AArch64TargetLowering()
594 ISD::STRICT_FPOW, ISD::STRICT_FPOWI, ISD::STRICT_FCOS, in AArch64TargetLowering()
595 ISD::STRICT_FSIN, ISD::STRICT_FEXP, ISD::STRICT_FEXP2, in AArch64TargetLowering()
596 ISD::STRICT_FLOG, ISD::STRICT_FLOG2, ISD::STRICT_FLOG10}) { in AArch64TargetLowering()
604 {ISD::SELECT, ISD::SELECT_CC, ISD::SETCC, in AArch64TargetLowering()
605 ISD::BR_CC, ISD::FADD, ISD::FSUB, in AArch64TargetLowering()
606 ISD::FMUL, ISD::FDIV, ISD::FMA, in AArch64TargetLowering()
607 ISD::FNEG, ISD::FABS, ISD::FCEIL, in AArch64TargetLowering()
608 ISD::FSQRT, ISD::FFLOOR, ISD::FNEARBYINT, in AArch64TargetLowering()
609 ISD::FRINT, ISD::FROUND, ISD::FROUNDEVEN, in AArch64TargetLowering()
610 ISD::FTRUNC, ISD::FMINNUM, ISD::FMAXNUM, in AArch64TargetLowering()
611 ISD::FMINIMUM, ISD::FMAXIMUM, ISD::STRICT_FADD, in AArch64TargetLowering()
612 ISD::STRICT_FSUB, ISD::STRICT_FMUL, ISD::STRICT_FDIV, in AArch64TargetLowering()
613 ISD::STRICT_FMA, ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, in AArch64TargetLowering()
614 ISD::STRICT_FSQRT, ISD::STRICT_FRINT, ISD::STRICT_FNEARBYINT, in AArch64TargetLowering()
615 ISD::STRICT_FROUND, ISD::STRICT_FTRUNC, ISD::STRICT_FROUNDEVEN, in AArch64TargetLowering()
616 ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM, ISD::STRICT_FMINIMUM, in AArch64TargetLowering()
617 ISD::STRICT_FMAXIMUM}) in AArch64TargetLowering()
622 for (auto Op : {ISD::STRICT_LROUND, ISD::STRICT_LLROUND, ISD::STRICT_LRINT, in AArch64TargetLowering()
623 ISD::STRICT_LLRINT}) in AArch64TargetLowering()
627 setOperationAction(ISD::FADD, MVT::v4f16, Promote); in AArch64TargetLowering()
628 setOperationAction(ISD::FSUB, MVT::v4f16, Promote); in AArch64TargetLowering()
629 setOperationAction(ISD::FMUL, MVT::v4f16, Promote); in AArch64TargetLowering()
630 setOperationAction(ISD::FDIV, MVT::v4f16, Promote); in AArch64TargetLowering()
631 AddPromotedToType(ISD::FADD, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
632 AddPromotedToType(ISD::FSUB, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
633 AddPromotedToType(ISD::FMUL, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
634 AddPromotedToType(ISD::FDIV, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
636 setOperationAction(ISD::FABS, MVT::v4f16, Expand); in AArch64TargetLowering()
637 setOperationAction(ISD::FNEG, MVT::v4f16, Expand); in AArch64TargetLowering()
638 setOperationAction(ISD::FROUND, MVT::v4f16, Expand); in AArch64TargetLowering()
639 setOperationAction(ISD::FROUNDEVEN, MVT::v4f16, Expand); in AArch64TargetLowering()
640 setOperationAction(ISD::FMA, MVT::v4f16, Expand); in AArch64TargetLowering()
641 setOperationAction(ISD::SETCC, MVT::v4f16, Expand); in AArch64TargetLowering()
642 setOperationAction(ISD::BR_CC, MVT::v4f16, Expand); in AArch64TargetLowering()
643 setOperationAction(ISD::SELECT, MVT::v4f16, Expand); in AArch64TargetLowering()
644 setOperationAction(ISD::SELECT_CC, MVT::v4f16, Expand); in AArch64TargetLowering()
645 setOperationAction(ISD::FTRUNC, MVT::v4f16, Expand); in AArch64TargetLowering()
646 setOperationAction(ISD::FCOPYSIGN, MVT::v4f16, Expand); in AArch64TargetLowering()
647 setOperationAction(ISD::FFLOOR, MVT::v4f16, Expand); in AArch64TargetLowering()
648 setOperationAction(ISD::FCEIL, MVT::v4f16, Expand); in AArch64TargetLowering()
649 setOperationAction(ISD::FRINT, MVT::v4f16, Expand); in AArch64TargetLowering()
650 setOperationAction(ISD::FNEARBYINT, MVT::v4f16, Expand); in AArch64TargetLowering()
651 setOperationAction(ISD::FSQRT, MVT::v4f16, Expand); in AArch64TargetLowering()
653 setOperationAction(ISD::FABS, MVT::v8f16, Expand); in AArch64TargetLowering()
654 setOperationAction(ISD::FADD, MVT::v8f16, Expand); in AArch64TargetLowering()
655 setOperationAction(ISD::FCEIL, MVT::v8f16, Expand); in AArch64TargetLowering()
656 setOperationAction(ISD::FCOPYSIGN, MVT::v8f16, Expand); in AArch64TargetLowering()
657 setOperationAction(ISD::FDIV, MVT::v8f16, Expand); in AArch64TargetLowering()
658 setOperationAction(ISD::FFLOOR, MVT::v8f16, Expand); in AArch64TargetLowering()
659 setOperationAction(ISD::FMA, MVT::v8f16, Expand); in AArch64TargetLowering()
660 setOperationAction(ISD::FMUL, MVT::v8f16, Expand); in AArch64TargetLowering()
661 setOperationAction(ISD::FNEARBYINT, MVT::v8f16, Expand); in AArch64TargetLowering()
662 setOperationAction(ISD::FNEG, MVT::v8f16, Expand); in AArch64TargetLowering()
663 setOperationAction(ISD::FROUND, MVT::v8f16, Expand); in AArch64TargetLowering()
664 setOperationAction(ISD::FROUNDEVEN, MVT::v8f16, Expand); in AArch64TargetLowering()
665 setOperationAction(ISD::FRINT, MVT::v8f16, Expand); in AArch64TargetLowering()
666 setOperationAction(ISD::FSQRT, MVT::v8f16, Expand); in AArch64TargetLowering()
667 setOperationAction(ISD::FSUB, MVT::v8f16, Expand); in AArch64TargetLowering()
668 setOperationAction(ISD::FTRUNC, MVT::v8f16, Expand); in AArch64TargetLowering()
669 setOperationAction(ISD::SETCC, MVT::v8f16, Expand); in AArch64TargetLowering()
670 setOperationAction(ISD::BR_CC, MVT::v8f16, Expand); in AArch64TargetLowering()
671 setOperationAction(ISD::SELECT, MVT::v8f16, Expand); in AArch64TargetLowering()
672 setOperationAction(ISD::SELECT_CC, MVT::v8f16, Expand); in AArch64TargetLowering()
673 setOperationAction(ISD::FP_EXTEND, MVT::v8f16, Expand); in AArch64TargetLowering()
678 {ISD::FFLOOR, ISD::FNEARBYINT, ISD::FCEIL, in AArch64TargetLowering()
679 ISD::FRINT, ISD::FTRUNC, ISD::FROUND, in AArch64TargetLowering()
680 ISD::FROUNDEVEN, ISD::FMINNUM, ISD::FMAXNUM, in AArch64TargetLowering()
681 ISD::FMINIMUM, ISD::FMAXIMUM, ISD::LROUND, in AArch64TargetLowering()
682 ISD::LLROUND, ISD::LRINT, ISD::LLRINT, in AArch64TargetLowering()
683 ISD::STRICT_FFLOOR, ISD::STRICT_FCEIL, ISD::STRICT_FNEARBYINT, in AArch64TargetLowering()
684 ISD::STRICT_FRINT, ISD::STRICT_FTRUNC, ISD::STRICT_FROUNDEVEN, in AArch64TargetLowering()
685 ISD::STRICT_FROUND, ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM, in AArch64TargetLowering()
686 ISD::STRICT_FMINIMUM, ISD::STRICT_FMAXIMUM, ISD::STRICT_LROUND, in AArch64TargetLowering()
687 ISD::STRICT_LLROUND, ISD::STRICT_LRINT, ISD::STRICT_LLRINT}) { in AArch64TargetLowering()
695 for (auto Op : {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL, in AArch64TargetLowering()
696 ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FSQRT}) { in AArch64TargetLowering()
705 setOperationAction(ISD::STRICT_FP_EXTEND, VT, Legal); in AArch64TargetLowering()
707 setOperationAction(ISD::PREFETCH, MVT::Other, Custom); in AArch64TargetLowering()
709 setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom); in AArch64TargetLowering()
710 setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom); in AArch64TargetLowering()
712 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, Custom); in AArch64TargetLowering()
713 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom); in AArch64TargetLowering()
714 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom); in AArch64TargetLowering()
715 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom); in AArch64TargetLowering()
716 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom); in AArch64TargetLowering()
721 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i8, LibCall); in AArch64TargetLowering()
722 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i16, LibCall); in AArch64TargetLowering()
723 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, LibCall); in AArch64TargetLowering()
724 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, LibCall); in AArch64TargetLowering()
725 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, LibCall); in AArch64TargetLowering()
726 setOperationAction(ISD::ATOMIC_SWAP, MVT::i8, LibCall); in AArch64TargetLowering()
727 setOperationAction(ISD::ATOMIC_SWAP, MVT::i16, LibCall); in AArch64TargetLowering()
728 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, LibCall); in AArch64TargetLowering()
729 setOperationAction(ISD::ATOMIC_SWAP, MVT::i64, LibCall); in AArch64TargetLowering()
730 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i8, LibCall); in AArch64TargetLowering()
731 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i16, LibCall); in AArch64TargetLowering()
732 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, LibCall); in AArch64TargetLowering()
733 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i64, LibCall); in AArch64TargetLowering()
734 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i8, LibCall); in AArch64TargetLowering()
735 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i16, LibCall); in AArch64TargetLowering()
736 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, LibCall); in AArch64TargetLowering()
737 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i64, LibCall); in AArch64TargetLowering()
738 setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i8, LibCall); in AArch64TargetLowering()
739 setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i16, LibCall); in AArch64TargetLowering()
740 setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i32, LibCall); in AArch64TargetLowering()
741 setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i64, LibCall); in AArch64TargetLowering()
742 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i8, LibCall); in AArch64TargetLowering()
743 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i16, LibCall); in AArch64TargetLowering()
744 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, LibCall); in AArch64TargetLowering()
745 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i64, LibCall); in AArch64TargetLowering()
770 setOperationAction(ISD::LOAD, MVT::i128, Custom); in AArch64TargetLowering()
771 setOperationAction(ISD::STORE, MVT::i128, Custom); in AArch64TargetLowering()
776 setOperationAction(ISD::ATOMIC_LOAD, MVT::i128, Custom); in AArch64TargetLowering()
777 setOperationAction(ISD::ATOMIC_STORE, MVT::i128, Custom); in AArch64TargetLowering()
783 setOperationAction(ISD::STORE, MVT::v32i8, Custom); in AArch64TargetLowering()
784 setOperationAction(ISD::STORE, MVT::v16i16, Custom); in AArch64TargetLowering()
785 setOperationAction(ISD::STORE, MVT::v16f16, Custom); in AArch64TargetLowering()
786 setOperationAction(ISD::STORE, MVT::v8i32, Custom); in AArch64TargetLowering()
787 setOperationAction(ISD::STORE, MVT::v8f32, Custom); in AArch64TargetLowering()
788 setOperationAction(ISD::STORE, MVT::v4f64, Custom); in AArch64TargetLowering()
789 setOperationAction(ISD::STORE, MVT::v4i64, Custom); in AArch64TargetLowering()
794 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal); in AArch64TargetLowering()
799 setOperationAction(ISD::FSINCOS, MVT::f64, Custom); in AArch64TargetLowering()
800 setOperationAction(ISD::FSINCOS, MVT::f32, Custom); in AArch64TargetLowering()
802 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); in AArch64TargetLowering()
803 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); in AArch64TargetLowering()
815 setOperationAction(ISD::ConstantFP, MVT::f32, Legal); in AArch64TargetLowering()
816 setOperationAction(ISD::ConstantFP, MVT::f64, Legal); in AArch64TargetLowering()
822 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand); in AArch64TargetLowering()
823 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand); in AArch64TargetLowering()
824 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand); in AArch64TargetLowering()
825 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand); in AArch64TargetLowering()
828 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand); in AArch64TargetLowering()
838 setOperationAction(ISD::BITCAST, MVT::i16, Custom); in AArch64TargetLowering()
839 setOperationAction(ISD::BITCAST, MVT::f16, Custom); in AArch64TargetLowering()
840 setOperationAction(ISD::BITCAST, MVT::bf16, Custom); in AArch64TargetLowering()
843 for (unsigned im = (unsigned)ISD::PRE_INC; in AArch64TargetLowering()
844 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) { in AArch64TargetLowering()
864 setOperationAction(ISD::TRAP, MVT::Other, Legal); in AArch64TargetLowering()
865 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal); in AArch64TargetLowering()
866 setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal); in AArch64TargetLowering()
869 setTargetDAGCombine(ISD::OR); in AArch64TargetLowering()
871 setTargetDAGCombine(ISD::AND); in AArch64TargetLowering()
875 setTargetDAGCombine({ISD::ADD, ISD::ABS, ISD::SUB, ISD::XOR, ISD::SINT_TO_FP, in AArch64TargetLowering()
876 ISD::UINT_TO_FP}); in AArch64TargetLowering()
878 setTargetDAGCombine({ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT, in AArch64TargetLowering()
879 ISD::FP_TO_UINT_SAT, ISD::FDIV}); in AArch64TargetLowering()
882 setTargetDAGCombine(ISD::SETCC); in AArch64TargetLowering()
884 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN); in AArch64TargetLowering()
886 setTargetDAGCombine({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND, in AArch64TargetLowering()
887 ISD::VECTOR_SPLICE, ISD::SIGN_EXTEND_INREG, in AArch64TargetLowering()
888 ISD::CONCAT_VECTORS, ISD::EXTRACT_SUBVECTOR, in AArch64TargetLowering()
889 ISD::INSERT_SUBVECTOR, ISD::STORE, ISD::BUILD_VECTOR}); in AArch64TargetLowering()
891 setTargetDAGCombine(ISD::LOAD); in AArch64TargetLowering()
893 setTargetDAGCombine(ISD::MSTORE); in AArch64TargetLowering()
895 setTargetDAGCombine(ISD::MUL); in AArch64TargetLowering()
897 setTargetDAGCombine({ISD::SELECT, ISD::VSELECT}); in AArch64TargetLowering()
899 setTargetDAGCombine({ISD::INTRINSIC_VOID, ISD::INTRINSIC_W_CHAIN, in AArch64TargetLowering()
900 ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT, in AArch64TargetLowering()
901 ISD::VECREDUCE_ADD, ISD::STEP_VECTOR}); in AArch64TargetLowering()
903 setTargetDAGCombine({ISD::MGATHER, ISD::MSCATTER}); in AArch64TargetLowering()
905 setTargetDAGCombine(ISD::FP_EXTEND); in AArch64TargetLowering()
907 setTargetDAGCombine(ISD::GlobalAddress); in AArch64TargetLowering()
947 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); in AArch64TargetLowering()
953 {ISD::SELECT, ISD::SELECT_CC, ISD::SETCC, in AArch64TargetLowering()
954 ISD::BR_CC, ISD::FADD, ISD::FSUB, in AArch64TargetLowering()
955 ISD::FMUL, ISD::FDIV, ISD::FMA, in AArch64TargetLowering()
956 ISD::FNEG, ISD::FABS, ISD::FCEIL, in AArch64TargetLowering()
957 ISD::FSQRT, ISD::FFLOOR, ISD::FNEARBYINT, in AArch64TargetLowering()
958 ISD::FRINT, ISD::FROUND, ISD::FROUNDEVEN, in AArch64TargetLowering()
959 ISD::FTRUNC, ISD::FMINNUM, ISD::FMAXNUM, in AArch64TargetLowering()
960 ISD::FMINIMUM, ISD::FMAXIMUM, ISD::STRICT_FADD, in AArch64TargetLowering()
961 ISD::STRICT_FSUB, ISD::STRICT_FMUL, ISD::STRICT_FDIV, in AArch64TargetLowering()
962 ISD::STRICT_FMA, ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, in AArch64TargetLowering()
963 ISD::STRICT_FSQRT, ISD::STRICT_FRINT, ISD::STRICT_FNEARBYINT, in AArch64TargetLowering()
964 ISD::STRICT_FROUND, ISD::STRICT_FTRUNC, ISD::STRICT_FROUNDEVEN, in AArch64TargetLowering()
965 ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM, ISD::STRICT_FMINIMUM, in AArch64TargetLowering()
966 ISD::STRICT_FMAXIMUM}) in AArch64TargetLowering()
970 {ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::SINT_TO_FP, ISD::UINT_TO_FP, in AArch64TargetLowering()
971 ISD::FP_ROUND, ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT, ISD::MUL, in AArch64TargetLowering()
972 ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT, in AArch64TargetLowering()
973 ISD::STRICT_SINT_TO_FP, ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_ROUND}) in AArch64TargetLowering()
978 setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i8, MVT::v4i32); in AArch64TargetLowering()
979 setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i8, MVT::v4i32); in AArch64TargetLowering()
984 for (auto Op : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP, in AArch64TargetLowering()
985 ISD::STRICT_UINT_TO_FP}) in AArch64TargetLowering()
990 setOperationAction(ISD::ConstantFP, MVT::f16, Legal); in AArch64TargetLowering()
992 setOperationAction(ISD::SINT_TO_FP, MVT::v8i8, Custom); in AArch64TargetLowering()
993 setOperationAction(ISD::UINT_TO_FP, MVT::v8i8, Custom); in AArch64TargetLowering()
994 setOperationAction(ISD::SINT_TO_FP, MVT::v16i8, Custom); in AArch64TargetLowering()
995 setOperationAction(ISD::UINT_TO_FP, MVT::v16i8, Custom); in AArch64TargetLowering()
996 setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom); in AArch64TargetLowering()
997 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom); in AArch64TargetLowering()
998 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Custom); in AArch64TargetLowering()
999 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom); in AArch64TargetLowering()
1003 setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i8, MVT::v8i32); in AArch64TargetLowering()
1004 setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i8, MVT::v8i32); in AArch64TargetLowering()
1005 setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v16i8, MVT::v16i32); in AArch64TargetLowering()
1006 setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v16i8, MVT::v16i32); in AArch64TargetLowering()
1007 setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i16, MVT::v4i32); in AArch64TargetLowering()
1008 setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i16, MVT::v4i32); in AArch64TargetLowering()
1009 setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i16, MVT::v8i32); in AArch64TargetLowering()
1010 setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i16, MVT::v8i32); in AArch64TargetLowering()
1013 setOperationAction(ISD::CTLZ, MVT::v1i64, Expand); in AArch64TargetLowering()
1014 setOperationAction(ISD::CTLZ, MVT::v2i64, Expand); in AArch64TargetLowering()
1015 setOperationAction(ISD::BITREVERSE, MVT::v8i8, Legal); in AArch64TargetLowering()
1016 setOperationAction(ISD::BITREVERSE, MVT::v16i8, Legal); in AArch64TargetLowering()
1017 setOperationAction(ISD::BITREVERSE, MVT::v2i32, Custom); in AArch64TargetLowering()
1018 setOperationAction(ISD::BITREVERSE, MVT::v4i32, Custom); in AArch64TargetLowering()
1019 setOperationAction(ISD::BITREVERSE, MVT::v1i64, Custom); in AArch64TargetLowering()
1020 setOperationAction(ISD::BITREVERSE, MVT::v2i64, Custom); in AArch64TargetLowering()
1022 setOperationAction(ISD::UMAX, VT, Custom); in AArch64TargetLowering()
1023 setOperationAction(ISD::SMAX, VT, Custom); in AArch64TargetLowering()
1024 setOperationAction(ISD::UMIN, VT, Custom); in AArch64TargetLowering()
1025 setOperationAction(ISD::SMIN, VT, Custom); in AArch64TargetLowering()
1029 setOperationAction(ISD::MUL, MVT::v2i64, Expand); in AArch64TargetLowering()
1031 setOperationAction(ISD::MUL, MVT::v8i16, Custom); in AArch64TargetLowering()
1032 setOperationAction(ISD::MUL, MVT::v4i32, Custom); in AArch64TargetLowering()
1033 setOperationAction(ISD::MUL, MVT::v2i64, Custom); in AArch64TargetLowering()
1038 setOperationAction(ISD::SADDSAT, VT, Legal); in AArch64TargetLowering()
1039 setOperationAction(ISD::UADDSAT, VT, Legal); in AArch64TargetLowering()
1040 setOperationAction(ISD::SSUBSAT, VT, Legal); in AArch64TargetLowering()
1041 setOperationAction(ISD::USUBSAT, VT, Legal); in AArch64TargetLowering()
1046 setOperationAction(ISD::AVGFLOORS, VT, Legal); in AArch64TargetLowering()
1047 setOperationAction(ISD::AVGFLOORU, VT, Legal); in AArch64TargetLowering()
1048 setOperationAction(ISD::AVGCEILS, VT, Legal); in AArch64TargetLowering()
1049 setOperationAction(ISD::AVGCEILU, VT, Legal); in AArch64TargetLowering()
1050 setOperationAction(ISD::ABDS, VT, Legal); in AArch64TargetLowering()
1051 setOperationAction(ISD::ABDU, VT, Legal); in AArch64TargetLowering()
1058 setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); in AArch64TargetLowering()
1059 setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); in AArch64TargetLowering()
1061 setOperationAction(ISD::VECREDUCE_FADD, VT, Legal); in AArch64TargetLowering()
1066 setOperationAction(ISD::VECREDUCE_ADD, VT, Custom); in AArch64TargetLowering()
1067 setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom); in AArch64TargetLowering()
1068 setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom); in AArch64TargetLowering()
1069 setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom); in AArch64TargetLowering()
1070 setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom); in AArch64TargetLowering()
1072 setOperationAction(ISD::VECREDUCE_ADD, MVT::v2i64, Custom); in AArch64TargetLowering()
1074 setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Legal); in AArch64TargetLowering()
1079 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); in AArch64TargetLowering()
1082 setOperationAction(ISD::MULHS, VT, Legal); in AArch64TargetLowering()
1083 setOperationAction(ISD::MULHU, VT, Legal); in AArch64TargetLowering()
1085 setOperationAction(ISD::MULHS, VT, Expand); in AArch64TargetLowering()
1086 setOperationAction(ISD::MULHU, VT, Expand); in AArch64TargetLowering()
1088 setOperationAction(ISD::SMUL_LOHI, VT, Expand); in AArch64TargetLowering()
1089 setOperationAction(ISD::UMUL_LOHI, VT, Expand); in AArch64TargetLowering()
1091 setOperationAction(ISD::BSWAP, VT, Expand); in AArch64TargetLowering()
1092 setOperationAction(ISD::CTTZ, VT, Expand); in AArch64TargetLowering()
1096 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
1097 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
1098 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
1104 {ISD::FFLOOR, ISD::FNEARBYINT, ISD::FCEIL, ISD::FRINT, ISD::FTRUNC, in AArch64TargetLowering()
1105 ISD::FROUND, ISD::FROUNDEVEN, ISD::STRICT_FFLOOR, in AArch64TargetLowering()
1106 ISD::STRICT_FNEARBYINT, ISD::STRICT_FCEIL, ISD::STRICT_FRINT, in AArch64TargetLowering()
1107 ISD::STRICT_FTRUNC, ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN}) { in AArch64TargetLowering()
1117 setLoadExtAction(ISD::EXTLOAD, MVT::v4i16, MVT::v4i8, Custom); in AArch64TargetLowering()
1118 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i16, MVT::v4i8, Custom); in AArch64TargetLowering()
1119 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, MVT::v4i8, Custom); in AArch64TargetLowering()
1120 setLoadExtAction(ISD::EXTLOAD, MVT::v4i32, MVT::v4i8, Custom); in AArch64TargetLowering()
1121 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i8, Custom); in AArch64TargetLowering()
1122 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i8, Custom); in AArch64TargetLowering()
1126 setOperationAction(ISD::ADD, VT, Custom); in AArch64TargetLowering()
1129 setOperationAction(ISD::FADD, VT, Custom); in AArch64TargetLowering()
1133 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); in AArch64TargetLowering()
1141 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); in AArch64TargetLowering()
1142 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in AArch64TargetLowering()
1148 setOperationAction(ISD::BITREVERSE, VT, Custom); in AArch64TargetLowering()
1149 setOperationAction(ISD::BSWAP, VT, Custom); in AArch64TargetLowering()
1150 setOperationAction(ISD::CTLZ, VT, Custom); in AArch64TargetLowering()
1151 setOperationAction(ISD::CTPOP, VT, Custom); in AArch64TargetLowering()
1152 setOperationAction(ISD::CTTZ, VT, Custom); in AArch64TargetLowering()
1153 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); in AArch64TargetLowering()
1154 setOperationAction(ISD::UINT_TO_FP, VT, Custom); in AArch64TargetLowering()
1155 setOperationAction(ISD::SINT_TO_FP, VT, Custom); in AArch64TargetLowering()
1156 setOperationAction(ISD::FP_TO_UINT, VT, Custom); in AArch64TargetLowering()
1157 setOperationAction(ISD::FP_TO_SINT, VT, Custom); in AArch64TargetLowering()
1158 setOperationAction(ISD::MGATHER, VT, Custom); in AArch64TargetLowering()
1159 setOperationAction(ISD::MSCATTER, VT, Custom); in AArch64TargetLowering()
1160 setOperationAction(ISD::MLOAD, VT, Custom); in AArch64TargetLowering()
1161 setOperationAction(ISD::MUL, VT, Custom); in AArch64TargetLowering()
1162 setOperationAction(ISD::MULHS, VT, Custom); in AArch64TargetLowering()
1163 setOperationAction(ISD::MULHU, VT, Custom); in AArch64TargetLowering()
1164 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); in AArch64TargetLowering()
1165 setOperationAction(ISD::VECTOR_SPLICE, VT, Custom); in AArch64TargetLowering()
1166 setOperationAction(ISD::SELECT, VT, Custom); in AArch64TargetLowering()
1167 setOperationAction(ISD::SETCC, VT, Custom); in AArch64TargetLowering()
1168 setOperationAction(ISD::SDIV, VT, Custom); in AArch64TargetLowering()
1169 setOperationAction(ISD::UDIV, VT, Custom); in AArch64TargetLowering()
1170 setOperationAction(ISD::SMIN, VT, Custom); in AArch64TargetLowering()
1171 setOperationAction(ISD::UMIN, VT, Custom); in AArch64TargetLowering()
1172 setOperationAction(ISD::SMAX, VT, Custom); in AArch64TargetLowering()
1173 setOperationAction(ISD::UMAX, VT, Custom); in AArch64TargetLowering()
1174 setOperationAction(ISD::SHL, VT, Custom); in AArch64TargetLowering()
1175 setOperationAction(ISD::SRL, VT, Custom); in AArch64TargetLowering()
1176 setOperationAction(ISD::SRA, VT, Custom); in AArch64TargetLowering()
1177 setOperationAction(ISD::ABS, VT, Custom); in AArch64TargetLowering()
1178 setOperationAction(ISD::ABDS, VT, Custom); in AArch64TargetLowering()
1179 setOperationAction(ISD::ABDU, VT, Custom); in AArch64TargetLowering()
1180 setOperationAction(ISD::VECREDUCE_ADD, VT, Custom); in AArch64TargetLowering()
1181 setOperationAction(ISD::VECREDUCE_AND, VT, Custom); in AArch64TargetLowering()
1182 setOperationAction(ISD::VECREDUCE_OR, VT, Custom); in AArch64TargetLowering()
1183 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom); in AArch64TargetLowering()
1184 setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom); in AArch64TargetLowering()
1185 setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom); in AArch64TargetLowering()
1186 setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom); in AArch64TargetLowering()
1187 setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom); in AArch64TargetLowering()
1189 setOperationAction(ISD::UMUL_LOHI, VT, Expand); in AArch64TargetLowering()
1190 setOperationAction(ISD::SMUL_LOHI, VT, Expand); in AArch64TargetLowering()
1191 setOperationAction(ISD::SELECT_CC, VT, Expand); in AArch64TargetLowering()
1192 setOperationAction(ISD::ROTL, VT, Expand); in AArch64TargetLowering()
1193 setOperationAction(ISD::ROTR, VT, Expand); in AArch64TargetLowering()
1195 setOperationAction(ISD::SADDSAT, VT, Legal); in AArch64TargetLowering()
1196 setOperationAction(ISD::UADDSAT, VT, Legal); in AArch64TargetLowering()
1197 setOperationAction(ISD::SSUBSAT, VT, Legal); in AArch64TargetLowering()
1198 setOperationAction(ISD::USUBSAT, VT, Legal); in AArch64TargetLowering()
1199 setOperationAction(ISD::UREM, VT, Expand); in AArch64TargetLowering()
1200 setOperationAction(ISD::SREM, VT, Expand); in AArch64TargetLowering()
1201 setOperationAction(ISD::SDIVREM, VT, Expand); in AArch64TargetLowering()
1202 setOperationAction(ISD::UDIVREM, VT, Expand); in AArch64TargetLowering()
1207 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); in AArch64TargetLowering()
1208 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); in AArch64TargetLowering()
1214 setOperationAction(ISD::BITCAST, VT, Custom); in AArch64TargetLowering()
1219 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Legal); in AArch64TargetLowering()
1223 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in AArch64TargetLowering()
1224 setOperationAction(ISD::SELECT, VT, Custom); in AArch64TargetLowering()
1225 setOperationAction(ISD::SETCC, VT, Custom); in AArch64TargetLowering()
1226 setOperationAction(ISD::TRUNCATE, VT, Custom); in AArch64TargetLowering()
1227 setOperationAction(ISD::VECREDUCE_AND, VT, Custom); in AArch64TargetLowering()
1228 setOperationAction(ISD::VECREDUCE_OR, VT, Custom); in AArch64TargetLowering()
1229 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom); in AArch64TargetLowering()
1231 setOperationAction(ISD::SELECT_CC, VT, Expand); in AArch64TargetLowering()
1232 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in AArch64TargetLowering()
1233 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); in AArch64TargetLowering()
1237 setOperationAction(ISD::SINT_TO_FP, VT, Custom); in AArch64TargetLowering()
1238 setOperationAction(ISD::UINT_TO_FP, VT, Custom); in AArch64TargetLowering()
1246 setOperationAction(ISD::MLOAD, VT, Custom); in AArch64TargetLowering()
1247 setOperationAction(ISD::MSTORE, VT, Custom); in AArch64TargetLowering()
1248 setOperationAction(ISD::MGATHER, VT, Custom); in AArch64TargetLowering()
1249 setOperationAction(ISD::MSCATTER, VT, Custom); in AArch64TargetLowering()
1257 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
1258 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
1259 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
1270 for (auto Op : {ISD::ZEXTLOAD, ISD::SEXTLOAD, ISD::EXTLOAD}) { in AArch64TargetLowering()
1288 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in AArch64TargetLowering()
1289 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); in AArch64TargetLowering()
1290 setOperationAction(ISD::MGATHER, VT, Custom); in AArch64TargetLowering()
1291 setOperationAction(ISD::MSCATTER, VT, Custom); in AArch64TargetLowering()
1292 setOperationAction(ISD::MLOAD, VT, Custom); in AArch64TargetLowering()
1293 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); in AArch64TargetLowering()
1294 setOperationAction(ISD::SELECT, VT, Custom); in AArch64TargetLowering()
1295 setOperationAction(ISD::FADD, VT, Custom); in AArch64TargetLowering()
1296 setOperationAction(ISD::FCOPYSIGN, VT, Custom); in AArch64TargetLowering()
1297 setOperationAction(ISD::FDIV, VT, Custom); in AArch64TargetLowering()
1298 setOperationAction(ISD::FMA, VT, Custom); in AArch64TargetLowering()
1299 setOperationAction(ISD::FMAXIMUM, VT, Custom); in AArch64TargetLowering()
1300 setOperationAction(ISD::FMAXNUM, VT, Custom); in AArch64TargetLowering()
1301 setOperationAction(ISD::FMINIMUM, VT, Custom); in AArch64TargetLowering()
1302 setOperationAction(ISD::FMINNUM, VT, Custom); in AArch64TargetLowering()
1303 setOperationAction(ISD::FMUL, VT, Custom); in AArch64TargetLowering()
1304 setOperationAction(ISD::FNEG, VT, Custom); in AArch64TargetLowering()
1305 setOperationAction(ISD::FSUB, VT, Custom); in AArch64TargetLowering()
1306 setOperationAction(ISD::FCEIL, VT, Custom); in AArch64TargetLowering()
1307 setOperationAction(ISD::FFLOOR, VT, Custom); in AArch64TargetLowering()
1308 setOperationAction(ISD::FNEARBYINT, VT, Custom); in AArch64TargetLowering()
1309 setOperationAction(ISD::FRINT, VT, Custom); in AArch64TargetLowering()
1310 setOperationAction(ISD::FROUND, VT, Custom); in AArch64TargetLowering()
1311 setOperationAction(ISD::FROUNDEVEN, VT, Custom); in AArch64TargetLowering()
1312 setOperationAction(ISD::FTRUNC, VT, Custom); in AArch64TargetLowering()
1313 setOperationAction(ISD::FSQRT, VT, Custom); in AArch64TargetLowering()
1314 setOperationAction(ISD::FABS, VT, Custom); in AArch64TargetLowering()
1315 setOperationAction(ISD::FP_EXTEND, VT, Custom); in AArch64TargetLowering()
1316 setOperationAction(ISD::FP_ROUND, VT, Custom); in AArch64TargetLowering()
1317 setOperationAction(ISD::VECREDUCE_FADD, VT, Custom); in AArch64TargetLowering()
1318 setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); in AArch64TargetLowering()
1319 setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); in AArch64TargetLowering()
1320 setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom); in AArch64TargetLowering()
1321 setOperationAction(ISD::VECTOR_SPLICE, VT, Custom); in AArch64TargetLowering()
1323 setOperationAction(ISD::SELECT_CC, VT, Expand); in AArch64TargetLowering()
1324 setOperationAction(ISD::FREM, VT, Expand); in AArch64TargetLowering()
1325 setOperationAction(ISD::FPOW, VT, Expand); in AArch64TargetLowering()
1326 setOperationAction(ISD::FPOWI, VT, Expand); in AArch64TargetLowering()
1327 setOperationAction(ISD::FCOS, VT, Expand); in AArch64TargetLowering()
1328 setOperationAction(ISD::FSIN, VT, Expand); in AArch64TargetLowering()
1329 setOperationAction(ISD::FSINCOS, VT, Expand); in AArch64TargetLowering()
1330 setOperationAction(ISD::FEXP, VT, Expand); in AArch64TargetLowering()
1331 setOperationAction(ISD::FEXP2, VT, Expand); in AArch64TargetLowering()
1332 setOperationAction(ISD::FLOG, VT, Expand); in AArch64TargetLowering()
1333 setOperationAction(ISD::FLOG2, VT, Expand); in AArch64TargetLowering()
1334 setOperationAction(ISD::FLOG10, VT, Expand); in AArch64TargetLowering()
1336 setCondCodeAction(ISD::SETO, VT, Expand); in AArch64TargetLowering()
1337 setCondCodeAction(ISD::SETOLT, VT, Expand); in AArch64TargetLowering()
1338 setCondCodeAction(ISD::SETLT, VT, Expand); in AArch64TargetLowering()
1339 setCondCodeAction(ISD::SETOLE, VT, Expand); in AArch64TargetLowering()
1340 setCondCodeAction(ISD::SETLE, VT, Expand); in AArch64TargetLowering()
1341 setCondCodeAction(ISD::SETULT, VT, Expand); in AArch64TargetLowering()
1342 setCondCodeAction(ISD::SETULE, VT, Expand); in AArch64TargetLowering()
1343 setCondCodeAction(ISD::SETUGE, VT, Expand); in AArch64TargetLowering()
1344 setCondCodeAction(ISD::SETUGT, VT, Expand); in AArch64TargetLowering()
1345 setCondCodeAction(ISD::SETUEQ, VT, Expand); in AArch64TargetLowering()
1346 setCondCodeAction(ISD::SETONE, VT, Expand); in AArch64TargetLowering()
1350 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in AArch64TargetLowering()
1351 setOperationAction(ISD::MGATHER, VT, Custom); in AArch64TargetLowering()
1352 setOperationAction(ISD::MSCATTER, VT, Custom); in AArch64TargetLowering()
1353 setOperationAction(ISD::MLOAD, VT, Custom); in AArch64TargetLowering()
1354 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); in AArch64TargetLowering()
1355 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); in AArch64TargetLowering()
1358 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom); in AArch64TargetLowering()
1359 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom); in AArch64TargetLowering()
1364 setOperationAction(ISD::SDIV, VT, Custom); in AArch64TargetLowering()
1365 setOperationAction(ISD::UDIV, VT, Custom); in AArch64TargetLowering()
1369 setOperationAction(ISD::MUL, MVT::v1i64, Custom); in AArch64TargetLowering()
1370 setOperationAction(ISD::MUL, MVT::v2i64, Custom); in AArch64TargetLowering()
1384 setOperationAction(ISD::TRUNCATE, VT, Custom); in AArch64TargetLowering()
1385 setOperationAction(ISD::FP_ROUND, MVT::v4f16, Custom); in AArch64TargetLowering()
1389 setOperationAction(ISD::TRUNCATE, VT, Custom); in AArch64TargetLowering()
1391 setOperationAction(ISD::FP_ROUND, VT, Custom); in AArch64TargetLowering()
1394 setOperationAction(ISD::BITREVERSE, MVT::v1i64, Custom); in AArch64TargetLowering()
1395 setOperationAction(ISD::CTLZ, MVT::v1i64, Custom); in AArch64TargetLowering()
1396 setOperationAction(ISD::CTLZ, MVT::v2i64, Custom); in AArch64TargetLowering()
1397 setOperationAction(ISD::CTTZ, MVT::v1i64, Custom); in AArch64TargetLowering()
1398 setOperationAction(ISD::MULHS, MVT::v1i64, Custom); in AArch64TargetLowering()
1399 setOperationAction(ISD::MULHS, MVT::v2i64, Custom); in AArch64TargetLowering()
1400 setOperationAction(ISD::MULHU, MVT::v1i64, Custom); in AArch64TargetLowering()
1401 setOperationAction(ISD::MULHU, MVT::v2i64, Custom); in AArch64TargetLowering()
1402 setOperationAction(ISD::SMAX, MVT::v1i64, Custom); in AArch64TargetLowering()
1403 setOperationAction(ISD::SMAX, MVT::v2i64, Custom); in AArch64TargetLowering()
1404 setOperationAction(ISD::SMIN, MVT::v1i64, Custom); in AArch64TargetLowering()
1405 setOperationAction(ISD::SMIN, MVT::v2i64, Custom); in AArch64TargetLowering()
1406 setOperationAction(ISD::UMAX, MVT::v1i64, Custom); in AArch64TargetLowering()
1407 setOperationAction(ISD::UMAX, MVT::v2i64, Custom); in AArch64TargetLowering()
1408 setOperationAction(ISD::UMIN, MVT::v1i64, Custom); in AArch64TargetLowering()
1409 setOperationAction(ISD::UMIN, MVT::v2i64, Custom); in AArch64TargetLowering()
1410 setOperationAction(ISD::VECREDUCE_SMAX, MVT::v2i64, Custom); in AArch64TargetLowering()
1411 setOperationAction(ISD::VECREDUCE_SMIN, MVT::v2i64, Custom); in AArch64TargetLowering()
1412 setOperationAction(ISD::VECREDUCE_UMAX, MVT::v2i64, Custom); in AArch64TargetLowering()
1413 setOperationAction(ISD::VECREDUCE_UMIN, MVT::v2i64, Custom); in AArch64TargetLowering()
1418 setOperationAction(ISD::BITREVERSE, VT, Custom); in AArch64TargetLowering()
1419 setOperationAction(ISD::CTTZ, VT, Custom); in AArch64TargetLowering()
1420 setOperationAction(ISD::VECREDUCE_AND, VT, Custom); in AArch64TargetLowering()
1421 setOperationAction(ISD::VECREDUCE_OR, VT, Custom); in AArch64TargetLowering()
1422 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom); in AArch64TargetLowering()
1428 setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom); in AArch64TargetLowering()
1432 setOperationAction(ISD::VECREDUCE_FADD, VT, Custom); in AArch64TargetLowering()
1435 setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv2i1, MVT::nxv2i64); in AArch64TargetLowering()
1436 setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv4i1, MVT::nxv4i32); in AArch64TargetLowering()
1437 setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv8i1, MVT::nxv8i16); in AArch64TargetLowering()
1438 setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv16i1, MVT::nxv16i8); in AArch64TargetLowering()
1440 setOperationAction(ISD::VSCALE, MVT::i32, Custom); in AArch64TargetLowering()
1445 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom); in AArch64TargetLowering()
1458 setOperationPromotedToType(ISD::LOAD, VT, PromoteTo); in addTypeForNEON()
1459 setOperationPromotedToType(ISD::STORE, VT, PromoteTo); in addTypeForNEON()
1464 setOperationAction(ISD::FSIN, VT, Expand); in addTypeForNEON()
1465 setOperationAction(ISD::FCOS, VT, Expand); in addTypeForNEON()
1466 setOperationAction(ISD::FPOW, VT, Expand); in addTypeForNEON()
1467 setOperationAction(ISD::FLOG, VT, Expand); in addTypeForNEON()
1468 setOperationAction(ISD::FLOG2, VT, Expand); in addTypeForNEON()
1469 setOperationAction(ISD::FLOG10, VT, Expand); in addTypeForNEON()
1470 setOperationAction(ISD::FEXP, VT, Expand); in addTypeForNEON()
1471 setOperationAction(ISD::FEXP2, VT, Expand); in addTypeForNEON()
1477 setOperationAction(ISD::FCOPYSIGN, VT, Custom); in addTypeForNEON()
1479 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in addTypeForNEON()
1480 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in addTypeForNEON()
1481 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in addTypeForNEON()
1482 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in addTypeForNEON()
1483 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); in addTypeForNEON()
1484 setOperationAction(ISD::SRA, VT, Custom); in addTypeForNEON()
1485 setOperationAction(ISD::SRL, VT, Custom); in addTypeForNEON()
1486 setOperationAction(ISD::SHL, VT, Custom); in addTypeForNEON()
1487 setOperationAction(ISD::OR, VT, Custom); in addTypeForNEON()
1488 setOperationAction(ISD::SETCC, VT, Custom); in addTypeForNEON()
1489 setOperationAction(ISD::CONCAT_VECTORS, VT, Legal); in addTypeForNEON()
1491 setOperationAction(ISD::SELECT, VT, Expand); in addTypeForNEON()
1492 setOperationAction(ISD::SELECT_CC, VT, Expand); in addTypeForNEON()
1493 setOperationAction(ISD::VSELECT, VT, Expand); in addTypeForNEON()
1495 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand); in addTypeForNEON()
1499 setOperationAction(ISD::CTPOP, VT, Custom); in addTypeForNEON()
1501 setOperationAction(ISD::UDIV, VT, Expand); in addTypeForNEON()
1502 setOperationAction(ISD::SDIV, VT, Expand); in addTypeForNEON()
1503 setOperationAction(ISD::UREM, VT, Expand); in addTypeForNEON()
1504 setOperationAction(ISD::SREM, VT, Expand); in addTypeForNEON()
1505 setOperationAction(ISD::FREM, VT, Expand); in addTypeForNEON()
1508 {ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT, in addTypeForNEON()
1509 ISD::FP_TO_UINT_SAT, ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT}) in addTypeForNEON()
1513 setOperationAction(ISD::ABS, VT, Legal); in addTypeForNEON()
1517 for (unsigned Opcode : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) in addTypeForNEON()
1526 {ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FMINNUM, ISD::FMAXNUM, in addTypeForNEON()
1527 ISD::STRICT_FMINIMUM, ISD::STRICT_FMAXIMUM, ISD::STRICT_FMINNUM, in addTypeForNEON()
1528 ISD::STRICT_FMAXNUM, ISD::STRICT_FADD, ISD::STRICT_FSUB, in addTypeForNEON()
1529 ISD::STRICT_FMUL, ISD::STRICT_FDIV, ISD::STRICT_FMA, in addTypeForNEON()
1530 ISD::STRICT_FSQRT}) in addTypeForNEON()
1535 setOperationAction(ISD::STRICT_FP_EXTEND, VT, Legal); in addTypeForNEON()
1537 setOperationAction(ISD::STRICT_FP_ROUND, VT, Legal); in addTypeForNEON()
1550 setOperationAction(ISD::STRICT_FSETCC, VT, Expand); in addTypeForNEON()
1551 setOperationAction(ISD::STRICT_FSETCCS, VT, Expand); in addTypeForNEON()
1554 for (unsigned im = (unsigned)ISD::PRE_INC; in addTypeForNEON()
1555 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) { in addTypeForNEON()
1586 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) in addTypeForFixedLengthSVE()
1590 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); in addTypeForFixedLengthSVE()
1593 setCondCodeAction(ISD::SETO, VT, Expand); in addTypeForFixedLengthSVE()
1594 setCondCodeAction(ISD::SETOLT, VT, Expand); in addTypeForFixedLengthSVE()
1595 setCondCodeAction(ISD::SETLT, VT, Expand); in addTypeForFixedLengthSVE()
1596 setCondCodeAction(ISD::SETOLE, VT, Expand); in addTypeForFixedLengthSVE()
1597 setCondCodeAction(ISD::SETLE, VT, Expand); in addTypeForFixedLengthSVE()
1598 setCondCodeAction(ISD::SETULT, VT, Expand); in addTypeForFixedLengthSVE()
1599 setCondCodeAction(ISD::SETULE, VT, Expand); in addTypeForFixedLengthSVE()
1600 setCondCodeAction(ISD::SETUGE, VT, Expand); in addTypeForFixedLengthSVE()
1601 setCondCodeAction(ISD::SETUGT, VT, Expand); in addTypeForFixedLengthSVE()
1602 setCondCodeAction(ISD::SETUEQ, VT, Expand); in addTypeForFixedLengthSVE()
1603 setCondCodeAction(ISD::SETONE, VT, Expand); in addTypeForFixedLengthSVE()
1611 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Custom); in addTypeForFixedLengthSVE()
1612 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Custom); in addTypeForFixedLengthSVE()
1624 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Custom); in addTypeForFixedLengthSVE()
1631 setOperationAction(ISD::ABS, VT, Custom); in addTypeForFixedLengthSVE()
1632 setOperationAction(ISD::ADD, VT, Custom); in addTypeForFixedLengthSVE()
1633 setOperationAction(ISD::AND, VT, Custom); in addTypeForFixedLengthSVE()
1634 setOperationAction(ISD::ANY_EXTEND, VT, Custom); in addTypeForFixedLengthSVE()
1635 setOperationAction(ISD::BITCAST, VT, Custom); in addTypeForFixedLengthSVE()
1636 setOperationAction(ISD::BITREVERSE, VT, Custom); in addTypeForFixedLengthSVE()
1637 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in addTypeForFixedLengthSVE()
1638 setOperationAction(ISD::BSWAP, VT, Custom); in addTypeForFixedLengthSVE()
1639 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in addTypeForFixedLengthSVE()
1640 setOperationAction(ISD::CTLZ, VT, Custom); in addTypeForFixedLengthSVE()
1641 setOperationAction(ISD::CTPOP, VT, Custom); in addTypeForFixedLengthSVE()
1642 setOperationAction(ISD::CTTZ, VT, Custom); in addTypeForFixedLengthSVE()
1643 setOperationAction(ISD::FABS, VT, Custom); in addTypeForFixedLengthSVE()
1644 setOperationAction(ISD::FADD, VT, Custom); in addTypeForFixedLengthSVE()
1645 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in addTypeForFixedLengthSVE()
1646 setOperationAction(ISD::FCEIL, VT, Custom); in addTypeForFixedLengthSVE()
1647 setOperationAction(ISD::FDIV, VT, Custom); in addTypeForFixedLengthSVE()
1648 setOperationAction(ISD::FFLOOR, VT, Custom); in addTypeForFixedLengthSVE()
1649 setOperationAction(ISD::FMA, VT, Custom); in addTypeForFixedLengthSVE()
1650 setOperationAction(ISD::FMAXIMUM, VT, Custom); in addTypeForFixedLengthSVE()
1651 setOperationAction(ISD::FMAXNUM, VT, Custom); in addTypeForFixedLengthSVE()
1652 setOperationAction(ISD::FMINIMUM, VT, Custom); in addTypeForFixedLengthSVE()
1653 setOperationAction(ISD::FMINNUM, VT, Custom); in addTypeForFixedLengthSVE()
1654 setOperationAction(ISD::FMUL, VT, Custom); in addTypeForFixedLengthSVE()
1655 setOperationAction(ISD::FNEARBYINT, VT, Custom); in addTypeForFixedLengthSVE()
1656 setOperationAction(ISD::FNEG, VT, Custom); in addTypeForFixedLengthSVE()
1657 setOperationAction(ISD::FP_EXTEND, VT, Custom); in addTypeForFixedLengthSVE()
1658 setOperationAction(ISD::FP_ROUND, VT, Custom); in addTypeForFixedLengthSVE()
1659 setOperationAction(ISD::FP_TO_SINT, VT, Custom); in addTypeForFixedLengthSVE()
1660 setOperationAction(ISD::FP_TO_UINT, VT, Custom); in addTypeForFixedLengthSVE()
1661 setOperationAction(ISD::FRINT, VT, Custom); in addTypeForFixedLengthSVE()
1662 setOperationAction(ISD::FROUND, VT, Custom); in addTypeForFixedLengthSVE()
1663 setOperationAction(ISD::FROUNDEVEN, VT, Custom); in addTypeForFixedLengthSVE()
1664 setOperationAction(ISD::FSQRT, VT, Custom); in addTypeForFixedLengthSVE()
1665 setOperationAction(ISD::FSUB, VT, Custom); in addTypeForFixedLengthSVE()
1666 setOperationAction(ISD::FTRUNC, VT, Custom); in addTypeForFixedLengthSVE()
1667 setOperationAction(ISD::LOAD, VT, Custom); in addTypeForFixedLengthSVE()
1668 setOperationAction(ISD::MGATHER, VT, Custom); in addTypeForFixedLengthSVE()
1669 setOperationAction(ISD::MLOAD, VT, Custom); in addTypeForFixedLengthSVE()
1670 setOperationAction(ISD::MSCATTER, VT, Custom); in addTypeForFixedLengthSVE()
1671 setOperationAction(ISD::MSTORE, VT, Custom); in addTypeForFixedLengthSVE()
1672 setOperationAction(ISD::MUL, VT, Custom); in addTypeForFixedLengthSVE()
1673 setOperationAction(ISD::MULHS, VT, Custom); in addTypeForFixedLengthSVE()
1674 setOperationAction(ISD::MULHU, VT, Custom); in addTypeForFixedLengthSVE()
1675 setOperationAction(ISD::OR, VT, Custom); in addTypeForFixedLengthSVE()
1676 setOperationAction(ISD::SDIV, VT, Custom); in addTypeForFixedLengthSVE()
1677 setOperationAction(ISD::SELECT, VT, Custom); in addTypeForFixedLengthSVE()
1678 setOperationAction(ISD::SETCC, VT, Custom); in addTypeForFixedLengthSVE()
1679 setOperationAction(ISD::SHL, VT, Custom); in addTypeForFixedLengthSVE()
1680 setOperationAction(ISD::SIGN_EXTEND, VT, Custom); in addTypeForFixedLengthSVE()
1681 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Custom); in addTypeForFixedLengthSVE()
1682 setOperationAction(ISD::SINT_TO_FP, VT, Custom); in addTypeForFixedLengthSVE()
1683 setOperationAction(ISD::SMAX, VT, Custom); in addTypeForFixedLengthSVE()
1684 setOperationAction(ISD::SMIN, VT, Custom); in addTypeForFixedLengthSVE()
1685 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); in addTypeForFixedLengthSVE()
1686 setOperationAction(ISD::VECTOR_SPLICE, VT, Custom); in addTypeForFixedLengthSVE()
1687 setOperationAction(ISD::SRA, VT, Custom); in addTypeForFixedLengthSVE()
1688 setOperationAction(ISD::SRL, VT, Custom); in addTypeForFixedLengthSVE()
1689 setOperationAction(ISD::STORE, VT, Custom); in addTypeForFixedLengthSVE()
1690 setOperationAction(ISD::SUB, VT, Custom); in addTypeForFixedLengthSVE()
1691 setOperationAction(ISD::TRUNCATE, VT, Custom); in addTypeForFixedLengthSVE()
1692 setOperationAction(ISD::UDIV, VT, Custom); in addTypeForFixedLengthSVE()
1693 setOperationAction(ISD::UINT_TO_FP, VT, Custom); in addTypeForFixedLengthSVE()
1694 setOperationAction(ISD::UMAX, VT, Custom); in addTypeForFixedLengthSVE()
1695 setOperationAction(ISD::UMIN, VT, Custom); in addTypeForFixedLengthSVE()
1696 setOperationAction(ISD::VECREDUCE_ADD, VT, Custom); in addTypeForFixedLengthSVE()
1697 setOperationAction(ISD::VECREDUCE_AND, VT, Custom); in addTypeForFixedLengthSVE()
1698 setOperationAction(ISD::VECREDUCE_FADD, VT, Custom); in addTypeForFixedLengthSVE()
1699 setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom); in addTypeForFixedLengthSVE()
1700 setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); in addTypeForFixedLengthSVE()
1701 setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); in addTypeForFixedLengthSVE()
1702 setOperationAction(ISD::VECREDUCE_OR, VT, Custom); in addTypeForFixedLengthSVE()
1703 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in addTypeForFixedLengthSVE()
1704 setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom); in addTypeForFixedLengthSVE()
1705 setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom); in addTypeForFixedLengthSVE()
1706 setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom); in addTypeForFixedLengthSVE()
1707 setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom); in addTypeForFixedLengthSVE()
1708 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom); in addTypeForFixedLengthSVE()
1709 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in addTypeForFixedLengthSVE()
1710 setOperationAction(ISD::VSELECT, VT, Custom); in addTypeForFixedLengthSVE()
1711 setOperationAction(ISD::XOR, VT, Custom); in addTypeForFixedLengthSVE()
1712 setOperationAction(ISD::ZERO_EXTEND, VT, Custom); in addTypeForFixedLengthSVE()
1855 case ISD::AND: in targetShrinkDemandedConstant()
1858 case ISD::OR: in targetShrinkDemandedConstant()
1861 case ISD::XOR: in targetShrinkDemandedConstant()
1932 case ISD::INTRINSIC_W_CHAIN: { in computeKnownBitsForTargetNode()
1948 case ISD::INTRINSIC_WO_CHAIN: in computeKnownBitsForTargetNode()
1949 case ISD::INTRINSIC_VOID: { in computeKnownBitsForTargetNode()
2672 while (N->getOpcode() == ISD::BITCAST) in isZerosVector()
2675 if (ISD::isConstantSplatVectorAllZeros(N)) in isZerosVector()
2689 static AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC) { in changeIntCCToAArch64CC()
2693 case ISD::SETNE: in changeIntCCToAArch64CC()
2695 case ISD::SETEQ: in changeIntCCToAArch64CC()
2697 case ISD::SETGT: in changeIntCCToAArch64CC()
2699 case ISD::SETGE: in changeIntCCToAArch64CC()
2701 case ISD::SETLT: in changeIntCCToAArch64CC()
2703 case ISD::SETLE: in changeIntCCToAArch64CC()
2705 case ISD::SETUGT: in changeIntCCToAArch64CC()
2707 case ISD::SETUGE: in changeIntCCToAArch64CC()
2709 case ISD::SETULT: in changeIntCCToAArch64CC()
2711 case ISD::SETULE: in changeIntCCToAArch64CC()
2717 static void changeFPCCToAArch64CC(ISD::CondCode CC, in changeFPCCToAArch64CC()
2724 case ISD::SETEQ: in changeFPCCToAArch64CC()
2725 case ISD::SETOEQ: in changeFPCCToAArch64CC()
2728 case ISD::SETGT: in changeFPCCToAArch64CC()
2729 case ISD::SETOGT: in changeFPCCToAArch64CC()
2732 case ISD::SETGE: in changeFPCCToAArch64CC()
2733 case ISD::SETOGE: in changeFPCCToAArch64CC()
2736 case ISD::SETOLT: in changeFPCCToAArch64CC()
2739 case ISD::SETOLE: in changeFPCCToAArch64CC()
2742 case ISD::SETONE: in changeFPCCToAArch64CC()
2746 case ISD::SETO: in changeFPCCToAArch64CC()
2749 case ISD::SETUO: in changeFPCCToAArch64CC()
2752 case ISD::SETUEQ: in changeFPCCToAArch64CC()
2756 case ISD::SETUGT: in changeFPCCToAArch64CC()
2759 case ISD::SETUGE: in changeFPCCToAArch64CC()
2762 case ISD::SETLT: in changeFPCCToAArch64CC()
2763 case ISD::SETULT: in changeFPCCToAArch64CC()
2766 case ISD::SETLE: in changeFPCCToAArch64CC()
2767 case ISD::SETULE: in changeFPCCToAArch64CC()
2770 case ISD::SETNE: in changeFPCCToAArch64CC()
2771 case ISD::SETUNE: in changeFPCCToAArch64CC()
2780 static void changeFPCCToANDAArch64CC(ISD::CondCode CC, in changeFPCCToANDAArch64CC()
2789 case ISD::SETONE: in changeFPCCToANDAArch64CC()
2796 case ISD::SETUEQ: in changeFPCCToANDAArch64CC()
2810 static void changeVectorFPCCToAArch64CC(ISD::CondCode CC, in changeVectorFPCCToAArch64CC()
2820 case ISD::SETUO: in changeVectorFPCCToAArch64CC()
2823 case ISD::SETO: in changeVectorFPCCToAArch64CC()
2827 case ISD::SETUEQ: in changeVectorFPCCToAArch64CC()
2828 case ISD::SETULT: in changeVectorFPCCToAArch64CC()
2829 case ISD::SETULE: in changeVectorFPCCToAArch64CC()
2830 case ISD::SETUGT: in changeVectorFPCCToAArch64CC()
2831 case ISD::SETUGE: in changeVectorFPCCToAArch64CC()
2859 static bool isCMN(SDValue Op, ISD::CondCode CC) { in isCMN()
2860 return Op.getOpcode() == ISD::SUB && isNullConstant(Op.getOperand(0)) && in isCMN()
2861 (CC == ISD::SETEQ || CC == ISD::SETNE); in isCMN()
2873 LHS = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {MVT::f32, MVT::Other}, in emitStrictFPComparison()
2875 RHS = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {MVT::f32, MVT::Other}, in emitStrictFPComparison()
2885 static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC, in emitComparison()
2893 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS); in emitComparison()
2894 RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS); in emitComparison()
2916 if (LHS.getOpcode() == ISD::AND) { in emitComparison()
2991 ISD::CondCode CC, SDValue CCOp, in emitConditionalComparison()
3001 LHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, LHS); in emitConditionalComparison()
3002 RHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, RHS); in emitConditionalComparison()
3005 } else if (RHS.getOpcode() == ISD::SUB) { in emitConditionalComparison()
3007 if (isNullConstant(SubOp0) && (CC == ISD::SETEQ || CC == ISD::SETNE)) { in emitConditionalComparison()
3043 if (Opcode == ISD::SETCC) { in canEmitConjunction()
3053 if (Opcode == ISD::AND || Opcode == ISD::OR) { in canEmitConjunction()
3054 bool IsOR = Opcode == ISD::OR; in canEmitConjunction()
3081 assert(Opcode == ISD::AND && "Must be OR or AND"); in canEmitConjunction()
3104 if (Opcode == ISD::SETCC) { in emitConjunctionRec()
3107 ISD::CondCode CC = cast<CondCodeSDNode>(Val->getOperand(2))->get(); in emitConjunctionRec()
3142 bool IsOR = Opcode == ISD::OR; in emitConjunctionRec()
3170 if (Opcode == ISD::OR) { in emitConjunctionRec()
3187 assert(Opcode == ISD::AND && "Valid conjunction/disjunction tree"); in emitConjunctionRec()
3227 if (V.getOpcode() == ISD::SIGN_EXTEND_INREG) in getCmpOperandFoldingProfit()
3230 if (V.getOpcode() == ISD::AND) in getCmpOperandFoldingProfit()
3246 if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA) in getCmpOperandFoldingProfit()
3259 static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, in getAArch64Cmp()
3270 case ISD::SETLT: in getAArch64Cmp()
3271 case ISD::SETGE: in getAArch64Cmp()
3276 CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT; in getAArch64Cmp()
3281 case ISD::SETULT: in getAArch64Cmp()
3282 case ISD::SETUGE: in getAArch64Cmp()
3286 CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT; in getAArch64Cmp()
3291 case ISD::SETLE: in getAArch64Cmp()
3292 case ISD::SETGT: in getAArch64Cmp()
3297 CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE; in getAArch64Cmp()
3302 case ISD::SETULE: in getAArch64Cmp()
3303 case ISD::SETUGT: in getAArch64Cmp()
3308 CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE; in getAArch64Cmp()
3333 CC = ISD::getSetCCSwappedOperands(CC); in getAArch64Cmp()
3339 if ((CC == ISD::SETEQ || CC == ISD::SETNE) && isa<ConstantSDNode>(RHS)) { in getAArch64Cmp()
3359 cast<LoadSDNode>(LHS)->getExtensionType() == ISD::ZEXTLOAD && in getAArch64Cmp()
3365 DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, LHS.getValueType(), LHS, in getAArch64Cmp()
3376 if ((CC == ISD::SETNE) ^ RHSC->isZero()) in getAArch64Cmp()
3402 case ISD::SADDO: in getAArch64XALUOOp()
3406 case ISD::UADDO: in getAArch64XALUOOp()
3410 case ISD::SSUBO: in getAArch64XALUOOp()
3414 case ISD::USUBO: in getAArch64XALUOOp()
3419 case ISD::SMULO: in getAArch64XALUOOp()
3420 case ISD::UMULO: { in getAArch64XALUOOp()
3422 bool IsSigned = Op.getOpcode() == ISD::SMULO; in getAArch64XALUOOp()
3425 unsigned ExtendOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in getAArch64XALUOOp()
3428 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
3429 Value = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul); in getAArch64XALUOOp()
3435 SDValue SExtMul = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Value); in getAArch64XALUOOp()
3448 Value = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
3450 SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
3451 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value, in getAArch64XALUOOp()
3459 SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
3495 if (isOneConstant(Other) && ISD::isOverflowIntrOpRes(Sel)) { in LowerXOR()
3510 if (Sel.getOpcode() != ISD::SELECT_CC) in LowerXOR()
3512 if (Sel.getOpcode() != ISD::SELECT_CC) in LowerXOR()
3522 ISD::CondCode CC = cast<CondCodeSDNode>(Sel.getOperand(4))->get(); in LowerXOR()
3544 CC = ISD::getSetCCInverse(CC, LHS.getValueType()); in LowerXOR()
3553 TVal = DAG.getNode(ISD::XOR, dl, Other.getValueType(), Other, in LowerXOR()
3624 return DAG.getNode(ISD::MERGE_VALUES, DL, VTs, Sum, OutFlag); in lowerADDSUBCARRY()
3650 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow); in LowerXALUO()
3731 unsigned Opcode = Op.getOpcode() == ISD::FP_TO_UINT in LowerVectorFP_TO_INT()
3748 SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NewVT, MVT::Other}, in LowerVectorFP_TO_INT()
3755 DAG.getNode(ISD::FP_EXTEND, dl, NewVT, Op.getOperand(0))); in LowerVectorFP_TO_INT()
3766 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, VT, Cv); in LowerVectorFP_TO_INT()
3772 return DAG.getNode(ISD::TRUNCATE, dl, VT, Cv); in LowerVectorFP_TO_INT()
3781 SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {ExtVT, MVT::Other}, in LowerVectorFP_TO_INT()
3786 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, dl, ExtVT, Op.getOperand(0)); in LowerVectorFP_TO_INT()
3795 ISD::EXTRACT_VECTOR_ELT, dl, InVT.getScalarType(), in LowerVectorFP_TO_INT()
3821 DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {MVT::f32, MVT::Other}, in LowerFP_TO_INT()
3828 DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, SrcVal)); in LowerFP_TO_INT()
3867 SrcVal = DAG.getNode(ISD::FP_EXTEND, SDLoc(Op), F32VT, SrcVal); in LowerVectorFP_TO_INT_SAT()
3892 if (Op.getOpcode() == ISD::FP_TO_SINT_SAT) { in LowerVectorFP_TO_INT_SAT()
3895 SDValue Min = DAG.getNode(ISD::SMIN, DL, IntVT, NativeCvt, MinC); in LowerVectorFP_TO_INT_SAT()
3898 Sat = DAG.getNode(ISD::SMAX, DL, IntVT, Min, MaxC); in LowerVectorFP_TO_INT_SAT()
3902 Sat = DAG.getNode(ISD::UMIN, DL, IntVT, NativeCvt, MinC); in LowerVectorFP_TO_INT_SAT()
3905 return DAG.getNode(ISD::TRUNCATE, DL, DstVT, Sat); in LowerVectorFP_TO_INT_SAT()
3926 SrcVal = DAG.getNode(ISD::FP_EXTEND, SDLoc(Op), MVT::f32, SrcVal); in LowerFP_TO_INT_SAT()
3948 if (Op.getOpcode() == ISD::FP_TO_SINT_SAT) { in LowerFP_TO_INT_SAT()
3951 SDValue Min = DAG.getNode(ISD::SMIN, DL, DstVT, NativeCvt, MinC); in LowerFP_TO_INT_SAT()
3954 Sat = DAG.getNode(ISD::SMAX, DL, DstVT, Min, MaxC); in LowerFP_TO_INT_SAT()
3958 Sat = DAG.getNode(ISD::UMIN, DL, DstVT, NativeCvt, MinC); in LowerFP_TO_INT_SAT()
3961 return DAG.getNode(ISD::TRUNCATE, DL, DstVT, Sat); in LowerFP_TO_INT_SAT()
3975 bool IsSigned = Opc == ISD::SINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP; in LowerVectorINT_TO_FP()
3980 unsigned CastOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in LowerVectorINT_TO_FP()
4004 ISD::STRICT_FP_ROUND, dl, {VT, MVT::Other}, in LowerVectorINT_TO_FP()
4008 return DAG.getNode(ISD::FP_ROUND, dl, VT, In, DAG.getIntPtrConstant(0, dl)); in LowerVectorINT_TO_FP()
4012 unsigned CastOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in LowerVectorINT_TO_FP()
4024 ISD::EXTRACT_VECTOR_ELT, dl, InVT.getScalarType(), in LowerVectorINT_TO_FP()
4051 ISD::STRICT_FP_ROUND, dl, {MVT::f16, MVT::Other}, in LowerINT_TO_FP()
4055 ISD::FP_ROUND, dl, MVT::f16, in LowerINT_TO_FP()
4128 DAG.getNode(ISD::ANY_EXTEND, SDLoc(Op), getSVEContainerType(ArgVT), in LowerBITCAST()
4145 Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op.getOperand(0)); in LowerBITCAST()
4146 Op = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Op); in LowerBITCAST()
4191 if (N->getOpcode() != ISD::BUILD_VECTOR) in isExtendedBUILD_VECTOR()
4214 if (N->getOpcode() == ISD::SIGN_EXTEND || in skipExtensionForVectorMULL()
4215 N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND) in skipExtensionForVectorMULL()
4221 assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR"); in skipExtensionForVectorMULL()
4239 return N->getOpcode() == ISD::SIGN_EXTEND || in isSignExtended()
4240 N->getOpcode() == ISD::ANY_EXTEND || in isSignExtended()
4245 return N->getOpcode() == ISD::ZERO_EXTEND || in isZeroExtended()
4246 N->getOpcode() == ISD::ANY_EXTEND || in isZeroExtended()
4252 if (Opcode == ISD::ADD || Opcode == ISD::SUB) { in isAddSubSExt()
4263 if (Opcode == ISD::ADD || Opcode == ISD::SUB) { in isAddSubZExt()
4282 ISD::INTRINSIC_W_CHAIN, dl, {MVT::i64, MVT::Other}, in LowerFLT_ROUNDS_()
4285 SDValue FPCR_32 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, FPCR_64); in LowerFLT_ROUNDS_()
4286 SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPCR_32, in LowerFLT_ROUNDS_()
4288 SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds, in LowerFLT_ROUNDS_()
4290 SDValue AND = DAG.getNode(ISD::AND, dl, MVT::i32, RMODE, in LowerFLT_ROUNDS_()
4311 RMValue = DAG.getNode(ISD::SUB, DL, MVT::i32, RMValue, in LowerSET_ROUNDING()
4313 RMValue = DAG.getNode(ISD::AND, DL, MVT::i32, RMValue, in LowerSET_ROUNDING()
4316 DAG.getNode(ISD::SHL, DL, MVT::i32, RMValue, in LowerSET_ROUNDING()
4318 RMValue = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, RMValue); in LowerSET_ROUNDING()
4324 DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i64, MVT::Other}, Ops); in LowerSET_ROUNDING()
4330 FPCR = DAG.getNode(ISD::AND, DL, MVT::i64, FPCR, in LowerSET_ROUNDING()
4332 FPCR = DAG.getNode(ISD::OR, DL, MVT::i64, FPCR, RMValue); in LowerSET_ROUNDING()
4336 return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2); in LowerSET_ROUNDING()
4410 DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1), in LowerMUL()
4412 DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1)); in LowerMUL()
4459 return DAG.getNode(ISD::AND, DL, VT, Reinterpret, Mask); in getSVEPredicateBitCast()
4498 SDValue And = DAG.getNode(ISD::AND, DL, MVT::i64, MRS, Mask); in LowerINTRINSIC_W_CHAIN()
4517 SDValue Result = DAG.getNode(ISD::BITCAST, dl, MVT::v1i64, in LowerINTRINSIC_WO_CHAIN()
4519 Result = DAG.getNode(ISD::ABS, dl, MVT::v1i64, Result); in LowerINTRINSIC_WO_CHAIN()
4520 return DAG.getNode(ISD::BITCAST, dl, MVT::i64, Result); in LowerINTRINSIC_WO_CHAIN()
4522 return DAG.getNode(ISD::ABS, dl, Ty, Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
4528 return DAG.getNode(ISD::SMAX, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4531 return DAG.getNode(ISD::UMAX, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4534 return DAG.getNode(ISD::SMIN, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4537 return DAG.getNode(ISD::UMIN, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4565 return DAG.getNode(ISD::VECTOR_REVERSE, dl, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
4603 return DAG.getNode(ISD::SRL, dl, Op.getValueType(), Bytes, One); in LowerINTRINSIC_WO_CHAIN()
4608 return DAG.getNode(ISD::SRL, dl, Op.getValueType(), Bytes, in LowerINTRINSIC_WO_CHAIN()
4614 return DAG.getNode(ISD::SRL, dl, Op.getValueType(), Bytes, in LowerINTRINSIC_WO_CHAIN()
4621 Data = DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Data); in LowerINTRINSIC_WO_CHAIN()
4702 Scalar = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Scalar); in LowerINTRINSIC_WO_CHAIN()
4805 ? (IsRoundingAdd ? ISD::AVGCEILS : ISD::AVGFLOORS) in LowerINTRINSIC_WO_CHAIN()
4806 : (IsRoundingAdd ? ISD::AVGCEILU : ISD::AVGFLOORU); in LowerINTRINSIC_WO_CHAIN()
4812 unsigned Opcode = IntNo == Intrinsic::aarch64_neon_uabd ? ISD::ABDU in LowerINTRINSIC_WO_CHAIN()
4813 : ISD::ABDS; in LowerINTRINSIC_WO_CHAIN()
4838 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, Op.getValueType(), ID, in LowerINTRINSIC_WO_CHAIN()
4933 ISD::LoadExtType ExtType = MGT->getExtensionType(); in LowerMGATHER()
4934 ISD::MemIndexType IndexType = MGT->getIndexType(); in LowerMGATHER()
4956 Index = DAG.getNode(ISD::SHL, DL, IndexVT, Index, in LowerMGATHER()
4983 unsigned ExtOpcode = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in LowerMGATHER()
4985 Mask = DAG.getNode(ISD::SIGN_EXTEND, DL, PromotedVT, Mask); in LowerMGATHER()
4988 if (PromotedVT != DataVT && ExtType == ISD::NON_EXTLOAD) in LowerMGATHER()
4989 ExtType = ISD::EXTLOAD; in LowerMGATHER()
5008 Result = DAG.getNode(ISD::TRUNCATE, DL, DataVT, Result); in LowerMGATHER()
5010 Result = DAG.getNode(ISD::BITCAST, DL, VT, Result); in LowerMGATHER()
5032 ISD::MemIndexType IndexType = MSC->getIndexType(); in LowerMSCATTER()
5044 Index = DAG.getNode(ISD::SHL, DL, IndexVT, Index, in LowerMSCATTER()
5062 StoreVal = DAG.getNode(ISD::BITCAST, DL, VT, StoreVal); in LowerMSCATTER()
5073 unsigned ExtOpcode = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in LowerMSCATTER()
5075 Mask = DAG.getNode(ISD::SIGN_EXTEND, DL, PromotedVT, Mask); in LowerMSCATTER()
5076 StoreVal = DAG.getNode(ISD::ANY_EXTEND, DL, PromotedVT, StoreVal); in LowerMSCATTER()
5148 SDValue TruncExt = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i16, in LowerTruncateVectorStore()
5150 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, TruncExt); in LowerTruncateVectorStore()
5152 Trunc = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Trunc); in LowerTruncateVectorStore()
5153 SDValue ExtractTrunc = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, in LowerTruncateVectorStore()
5204 DAG.getNode(ISD::EXTRACT_SUBVECTOR, Dl, in LowerSTORE()
5208 DAG.getNode(ISD::EXTRACT_SUBVECTOR, Dl, in LowerSTORE()
5229 SDValue Ptr = DAG.getNode(ISD::ADD, Dl, PtrVT, Base, in LowerSTORE()
5250 SDValue Value = StoreNode->getOpcode() == ISD::STORE in LowerStore128()
5254 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, Value, in LowerStore128()
5256 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, Value, in LowerStore128()
5277 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, Base, in LowerLOAD()
5297 if (LoadNode->getExtensionType() == ISD::SEXTLOAD) in LowerLOAD()
5298 ExtType = ISD::SIGN_EXTEND; in LowerLOAD()
5299 else if (LoadNode->getExtensionType() == ISD::ZEXTLOAD || in LowerLOAD()
5300 LoadNode->getExtensionType() == ISD::EXTLOAD) in LowerLOAD()
5301 ExtType = ISD::ZERO_EXTEND; in LowerLOAD()
5308 SDValue Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f32, Load); in LowerLOAD()
5309 SDValue BC = DAG.getNode(ISD::BITCAST, DL, MVT::v8i8, Vec); in LowerLOAD()
5311 Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i16, Ext, in LowerLOAD()
5326 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in LowerABS()
5362 case ISD::BITCAST: in LowerOperation()
5364 case ISD::GlobalAddress: in LowerOperation()
5366 case ISD::GlobalTLSAddress: in LowerOperation()
5368 case ISD::SETCC: in LowerOperation()
5369 case ISD::STRICT_FSETCC: in LowerOperation()
5370 case ISD::STRICT_FSETCCS: in LowerOperation()
5372 case ISD::BRCOND: in LowerOperation()
5374 case ISD::BR_CC: in LowerOperation()
5376 case ISD::SELECT: in LowerOperation()
5378 case ISD::SELECT_CC: in LowerOperation()
5380 case ISD::JumpTable: in LowerOperation()
5382 case ISD::BR_JT: in LowerOperation()
5384 case ISD::ConstantPool: in LowerOperation()
5386 case ISD::BlockAddress: in LowerOperation()
5388 case ISD::VASTART: in LowerOperation()
5390 case ISD::VACOPY: in LowerOperation()
5392 case ISD::VAARG: in LowerOperation()
5394 case ISD::ADDCARRY: in LowerOperation()
5396 case ISD::SUBCARRY: in LowerOperation()
5398 case ISD::SADDO_CARRY: in LowerOperation()
5400 case ISD::SSUBO_CARRY: in LowerOperation()
5402 case ISD::SADDO: in LowerOperation()
5403 case ISD::UADDO: in LowerOperation()
5404 case ISD::SSUBO: in LowerOperation()
5405 case ISD::USUBO: in LowerOperation()
5406 case ISD::SMULO: in LowerOperation()
5407 case ISD::UMULO: in LowerOperation()
5409 case ISD::FADD: in LowerOperation()
5411 case ISD::FSUB: in LowerOperation()
5413 case ISD::FMUL: in LowerOperation()
5415 case ISD::FMA: in LowerOperation()
5417 case ISD::FDIV: in LowerOperation()
5419 case ISD::FNEG: in LowerOperation()
5421 case ISD::FCEIL: in LowerOperation()
5423 case ISD::FFLOOR: in LowerOperation()
5425 case ISD::FNEARBYINT: in LowerOperation()
5427 case ISD::FRINT: in LowerOperation()
5429 case ISD::FROUND: in LowerOperation()
5431 case ISD::FROUNDEVEN: in LowerOperation()
5433 case ISD::FTRUNC: in LowerOperation()
5435 case ISD::FSQRT: in LowerOperation()
5437 case ISD::FABS: in LowerOperation()
5439 case ISD::FP_ROUND: in LowerOperation()
5440 case ISD::STRICT_FP_ROUND: in LowerOperation()
5442 case ISD::FP_EXTEND: in LowerOperation()
5444 case ISD::FRAMEADDR: in LowerOperation()
5446 case ISD::SPONENTRY: in LowerOperation()
5448 case ISD::RETURNADDR: in LowerOperation()
5450 case ISD::ADDROFRETURNADDR: in LowerOperation()
5452 case ISD::CONCAT_VECTORS: in LowerOperation()
5454 case ISD::INSERT_VECTOR_ELT: in LowerOperation()
5456 case ISD::EXTRACT_VECTOR_ELT: in LowerOperation()
5458 case ISD::BUILD_VECTOR: in LowerOperation()
5460 case ISD::VECTOR_SHUFFLE: in LowerOperation()
5462 case ISD::SPLAT_VECTOR: in LowerOperation()
5464 case ISD::EXTRACT_SUBVECTOR: in LowerOperation()
5466 case ISD::INSERT_SUBVECTOR: in LowerOperation()
5468 case ISD::SDIV: in LowerOperation()
5469 case ISD::UDIV: in LowerOperation()
5471 case ISD::SMIN: in LowerOperation()
5472 case ISD::UMIN: in LowerOperation()
5473 case ISD::SMAX: in LowerOperation()
5474 case ISD::UMAX: in LowerOperation()
5476 case ISD::SRA: in LowerOperation()
5477 case ISD::SRL: in LowerOperation()
5478 case ISD::SHL: in LowerOperation()
5480 case ISD::SHL_PARTS: in LowerOperation()
5481 case ISD::SRL_PARTS: in LowerOperation()
5482 case ISD::SRA_PARTS: in LowerOperation()
5484 case ISD::CTPOP: in LowerOperation()
5485 case ISD::PARITY: in LowerOperation()
5487 case ISD::FCOPYSIGN: in LowerOperation()
5489 case ISD::OR: in LowerOperation()
5491 case ISD::XOR: in LowerOperation()
5493 case ISD::PREFETCH: in LowerOperation()
5495 case ISD::SINT_TO_FP: in LowerOperation()
5496 case ISD::UINT_TO_FP: in LowerOperation()
5497 case ISD::STRICT_SINT_TO_FP: in LowerOperation()
5498 case ISD::STRICT_UINT_TO_FP: in LowerOperation()
5500 case ISD::FP_TO_SINT: in LowerOperation()
5501 case ISD::FP_TO_UINT: in LowerOperation()
5502 case ISD::STRICT_FP_TO_SINT: in LowerOperation()
5503 case ISD::STRICT_FP_TO_UINT: in LowerOperation()
5505 case ISD::FP_TO_SINT_SAT: in LowerOperation()
5506 case ISD::FP_TO_UINT_SAT: in LowerOperation()
5508 case ISD::FSINCOS: in LowerOperation()
5510 case ISD::FLT_ROUNDS_: in LowerOperation()
5512 case ISD::SET_ROUNDING: in LowerOperation()
5514 case ISD::MUL: in LowerOperation()
5516 case ISD::MULHS: in LowerOperation()
5518 case ISD::MULHU: in LowerOperation()
5520 case ISD::INTRINSIC_W_CHAIN: in LowerOperation()
5522 case ISD::INTRINSIC_WO_CHAIN: in LowerOperation()
5524 case ISD::ATOMIC_STORE: in LowerOperation()
5530 case ISD::STORE: in LowerOperation()
5532 case ISD::MSTORE: in LowerOperation()
5534 case ISD::MGATHER: in LowerOperation()
5536 case ISD::MSCATTER: in LowerOperation()
5538 case ISD::VECREDUCE_SEQ_FADD: in LowerOperation()
5540 case ISD::VECREDUCE_ADD: in LowerOperation()
5541 case ISD::VECREDUCE_AND: in LowerOperation()
5542 case ISD::VECREDUCE_OR: in LowerOperation()
5543 case ISD::VECREDUCE_XOR: in LowerOperation()
5544 case ISD::VECREDUCE_SMAX: in LowerOperation()
5545 case ISD::VECREDUCE_SMIN: in LowerOperation()
5546 case ISD::VECREDUCE_UMAX: in LowerOperation()
5547 case ISD::VECREDUCE_UMIN: in LowerOperation()
5548 case ISD::VECREDUCE_FADD: in LowerOperation()
5549 case ISD::VECREDUCE_FMAX: in LowerOperation()
5550 case ISD::VECREDUCE_FMIN: in LowerOperation()
5552 case ISD::ATOMIC_LOAD_SUB: in LowerOperation()
5554 case ISD::ATOMIC_LOAD_AND: in LowerOperation()
5556 case ISD::DYNAMIC_STACKALLOC: in LowerOperation()
5558 case ISD::VSCALE: in LowerOperation()
5560 case ISD::ANY_EXTEND: in LowerOperation()
5561 case ISD::SIGN_EXTEND: in LowerOperation()
5562 case ISD::ZERO_EXTEND: in LowerOperation()
5564 case ISD::SIGN_EXTEND_INREG: { in LowerOperation()
5575 case ISD::TRUNCATE: in LowerOperation()
5577 case ISD::MLOAD: in LowerOperation()
5579 case ISD::LOAD: in LowerOperation()
5583 case ISD::ADD: in LowerOperation()
5584 case ISD::AND: in LowerOperation()
5585 case ISD::SUB: in LowerOperation()
5587 case ISD::FMAXIMUM: in LowerOperation()
5589 case ISD::FMAXNUM: in LowerOperation()
5591 case ISD::FMINIMUM: in LowerOperation()
5593 case ISD::FMINNUM: in LowerOperation()
5595 case ISD::VSELECT: in LowerOperation()
5597 case ISD::ABS: in LowerOperation()
5599 case ISD::ABDS: in LowerOperation()
5601 case ISD::ABDU: in LowerOperation()
5603 case ISD::BITREVERSE: in LowerOperation()
5605 case ISD::BSWAP: in LowerOperation()
5607 case ISD::CTLZ: in LowerOperation()
5609 case ISD::CTTZ: in LowerOperation()
5611 case ISD::VECTOR_SPLICE: in LowerOperation()
5613 case ISD::STRICT_LROUND: in LowerOperation()
5614 case ISD::STRICT_LLROUND: in LowerOperation()
5615 case ISD::STRICT_LRINT: in LowerOperation()
5616 case ISD::STRICT_LLRINT: { in LowerOperation()
5620 SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other}, in LowerOperation()
5687 case ISD::INTRINSIC_WO_CHAIN: { in getIntrinsicID()
5707 return N0.getOpcode() != ISD::ADD; in isReassocProfitable()
5755 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, in LowerFormalArguments()
5763 SmallVector<ISD::OutputArg, 4> Outs; in LowerFormalArguments()
5766 if (any_of(Outs, [](ISD::OutputArg &Out){ return Out.VT.isScalableVector(); })) in LowerFormalArguments()
5878 ArgValue = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), ArgValue); in LowerFormalArguments()
5885 ArgValue = DAG.getNode(ISD::SRL, DL, RegVT, ArgValue, in LowerFormalArguments()
5908 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD; in LowerFormalArguments()
5924 ExtType = ISD::SEXTLOAD; in LowerFormalArguments()
5927 ExtType = ISD::ZEXTLOAD; in LowerFormalArguments()
5930 ExtType = ISD::EXTLOAD; in LowerFormalArguments()
5966 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, in LowerFormalArguments()
5974 ArgValue = DAG.getNode(ISD::AssertZext, DL, ArgValue.getValueType(), in LowerFormalArguments()
6043 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Copy, Chain); in LowerFormalArguments()
6116 DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getConstant(8, DL, PtrVT)); in saveVarArgRegisters()
6143 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, in saveVarArgRegisters()
6152 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in saveVarArgRegisters()
6194 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val); in LowerCallResult()
6197 Val = DAG.getNode(ISD::SRL, DL, VA.getLocVT(), Val, in LowerCallResult()
6242 const SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs; in analyzeCallOperands()
6248 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; in analyzeCallOperands()
6287 const SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs; in isEligibleForTailCallOptimization()
6289 const SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins; in isEligibleForTailCallOptimization()
6453 return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); in addTokenForArgument()
6481 SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs; in LowerCall()
6483 SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins; in LowerCall()
6625 ISD::ArgFlagsTy Flags = Outs[i].Flags; in LowerCall()
6634 Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg); in LowerCall()
6637 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg); in LowerCall()
6655 Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg); in LowerCall()
6656 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i8, Arg); in LowerCall()
6659 Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg); in LowerCall()
6663 Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg); in LowerCall()
6664 Arg = DAG.getNode(ISD::SHL, DL, VA.getLocVT(), Arg, in LowerCall()
6674 Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg); in LowerCall()
6714 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, in LowerCall()
6745 Bits = DAG.getNode(ISD::OR, DL, Bits.getValueType(), Bits, Arg); in LowerCall()
6784 PtrOff = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff); in LowerCall()
6800 DstAddr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff); in LowerCall()
6820 Arg = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Arg); in LowerCall()
6829 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); in LowerCall()
6970 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const { in CanLowerReturn()
6980 const SmallVectorImpl<ISD::OutputArg> &Outs, in LowerReturn()
7009 Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg); in LowerReturn()
7010 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg); in LowerReturn()
7014 Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg); in LowerReturn()
7023 Arg = DAG.getNode(ISD::SHL, DL, VA.getLocVT(), Arg, in LowerReturn()
7033 Bits = DAG.getNode(ISD::OR, DL, Bits.getValueType(), Bits, Arg); in LowerReturn()
7340 return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff); in LowerELFTLSLocalExec()
7366 return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff); in LowerELFTLSLocalExec()
7488 return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff); in LowerELFGlobalTLSAddress()
7505 DAG.getNode(ISD::ADD, DL, PtrVT, TEB, DAG.getIntPtrConstant(0x58, DL)); in LowerWindowsGlobalTLSAddress()
7525 TLSIndex = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TLSIndex); in LowerWindowsGlobalTLSAddress()
7526 SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex, in LowerWindowsGlobalTLSAddress()
7529 DAG.getNode(ISD::ADD, DL, PtrVT, TLSArray, Slot), in LowerWindowsGlobalTLSAddress()
7570 if (Val.getOpcode() == ISD::SIGN_EXTEND_INREG) in lookThroughSignExtension()
7575 if (Val.getOpcode() == ISD::SIGN_EXTEND) in lookThroughSignExtension()
7584 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); in LowerBR_CC()
7607 CC = ISD::SETNE; in LowerBR_CC()
7613 if (ISD::isOverflowIntrOpRes(LHS) && isOneConstant(RHS) && in LowerBR_CC()
7614 (CC == ISD::SETEQ || CC == ISD::SETNE)) { in LowerBR_CC()
7624 if (CC == ISD::SETNE) in LowerBR_CC()
7640 if (CC == ISD::SETEQ) { in LowerBR_CC()
7645 if (LHS.getOpcode() == ISD::AND && in LowerBR_CC()
7656 } else if (CC == ISD::SETNE) { in LowerBR_CC()
7661 if (LHS.getOpcode() == ISD::AND && in LowerBR_CC()
7672 } else if (CC == ISD::SETLT && LHS.getOpcode() != ISD::AND) { in LowerBR_CC()
7682 if (RHSC && RHSC->getSExtValue() == -1 && CC == ISD::SETGT && in LowerBR_CC()
7683 LHS.getOpcode() != ISD::AND && ProduceNonFlagSettingCondBr) { in LowerBR_CC()
7733 In2 = DAG.getNode(ISD::FP_EXTEND, DL, VT, In2); in LowerFCOPYSIGN()
7735 In2 = DAG.getNode(ISD::FP_ROUND, DL, VT, In2, DAG.getIntPtrConstant(0, DL)); in LowerFCOPYSIGN()
7788 SignMaskV = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, SignMaskV); in LowerFCOPYSIGN()
7789 SignMaskV = DAG.getNode(ISD::FNEG, DL, MVT::v2f64, SignMaskV); in LowerFCOPYSIGN()
7790 SignMaskV = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, SignMaskV); in LowerFCOPYSIGN()
7814 bool IsParity = Op.getOpcode() == ISD::PARITY; in LowerCTPOP_PARITY()
7830 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Val); in LowerCTPOP_PARITY()
7831 Val = DAG.getNode(ISD::BITCAST, DL, MVT::v8i8, Val); in LowerCTPOP_PARITY()
7833 SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, Val); in LowerCTPOP_PARITY()
7835 ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32, in LowerCTPOP_PARITY()
7839 UaddLV = DAG.getNode(ISD::AND, DL, MVT::i32, UaddLV, in LowerCTPOP_PARITY()
7843 UaddLV = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, UaddLV); in LowerCTPOP_PARITY()
7846 Val = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Val); in LowerCTPOP_PARITY()
7848 SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v16i8, Val); in LowerCTPOP_PARITY()
7850 ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32, in LowerCTPOP_PARITY()
7854 UaddLV = DAG.getNode(ISD::AND, DL, MVT::i32, UaddLV, in LowerCTPOP_PARITY()
7857 return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i128, UaddLV); in LowerCTPOP_PARITY()
7871 Val = DAG.getNode(ISD::CTPOP, DL, VT8Bit, Val); in LowerCTPOP_PARITY()
7881 ISD::INTRINSIC_WO_CHAIN, DL, WidenVT, in LowerCTPOP_PARITY()
7895 SDValue RBIT = DAG.getNode(ISD::BITREVERSE, DL, VT, Op.getOperand(0)); in LowerCTTZ()
7896 return DAG.getNode(ISD::CTLZ, DL, VT, RBIT); in LowerCTTZ()
7905 ISD::CondCode CC; in LowerMinMax()
7909 case ISD::SMAX: in LowerMinMax()
7910 CC = ISD::SETGT; in LowerMinMax()
7912 case ISD::SMIN: in LowerMinMax()
7913 CC = ISD::SETLT; in LowerMinMax()
7915 case ISD::UMAX: in LowerMinMax()
7916 CC = ISD::SETUGT; in LowerMinMax()
7918 case ISD::UMIN: in LowerMinMax()
7919 CC = ISD::SETULT; in LowerMinMax()
7929 case ISD::SMAX: in LowerMinMax()
7931 case ISD::SMIN: in LowerMinMax()
7933 case ISD::UMAX: in LowerMinMax()
7935 case ISD::UMIN: in LowerMinMax()
7993 DAG.getNode(ISD::BITREVERSE, DL, VST, REVB)); in LowerBitreverse()
8002 bool IsSignaling = Op.getOpcode() == ISD::STRICT_FSETCCS; in LowerSETCC()
8009 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(OpNo + 2))->get(); in LowerSETCC()
8034 LHS, RHS, ISD::getSetCCInverse(CC, LHS.getValueType()), CCVal, DAG, dl); in LowerSETCC()
8059 changeFPCCToAArch64CC(ISD::getSetCCInverse(CC, LHS.getValueType()), CC1, in LowerSETCC()
8084 SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS, in LowerSELECT_CC()
8097 CC = ISD::SETNE; in LowerSELECT_CC()
8103 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS); in LowerSELECT_CC()
8104 RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS); in LowerSELECT_CC()
8118 if (CC == ISD::SETGT && RHSC && RHSC->isAllOnes() && CTVal && CFVal && in LowerSELECT_CC()
8123 DAG.getNode(ISD::SRA, dl, VT, LHS, in LowerSELECT_CC()
8125 return DAG.getNode(ISD::OR, dl, VT, Shift, DAG.getConstant(1, dl, VT)); in LowerSELECT_CC()
8135 CC = ISD::getSetCCInverse(CC, LHS.getValueType()); in LowerSELECT_CC()
8139 CC = ISD::getSetCCInverse(CC, LHS.getValueType()); in LowerSELECT_CC()
8140 } else if (TVal.getOpcode() == ISD::XOR) { in LowerSELECT_CC()
8146 CC = ISD::getSetCCInverse(CC, LHS.getValueType()); in LowerSELECT_CC()
8148 } else if (TVal.getOpcode() == ISD::SUB) { in LowerSELECT_CC()
8154 CC = ISD::getSetCCInverse(CC, LHS.getValueType()); in LowerSELECT_CC()
8198 CC = ISD::getSetCCInverse(CC, LHS.getValueType()); in LowerSELECT_CC()
8261 if ((CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETUEQ) && in LowerSELECT_CC()
8264 else if ((CC == ISD::SETNE || CC == ISD::SETONE || CC == ISD::SETUNE) && in LowerSELECT_CC()
8310 Pred = DAG.getNode(ISD::VECTOR_REVERSE, DL, PredVT, Pred); in LowerVECTOR_SPLICE()
8328 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); in LowerSELECT_CC()
8346 SDValue TruncCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, CCVal); in LowerSELECT()
8348 SDValue SplatPred = DAG.getNode(ISD::SPLAT_VECTOR, DL, PredVT, TruncCC); in LowerSELECT()
8349 return DAG.getNode(ISD::VSELECT, DL, Ty, SplatPred, TVal, FVal); in LowerSELECT()
8359 SDValue SplatPred = DAG.getNode(ISD::SPLAT_VECTOR, DL, PredVT, SplatVal); in LowerSELECT()
8360 return DAG.getNode(ISD::VSELECT, DL, Ty, SplatPred, TVal, FVal); in LowerSELECT()
8365 if (ISD::isOverflowIntrOpRes(CCVal)) { in LowerSELECT()
8380 ISD::CondCode CC; in LowerSELECT()
8382 if (CCVal.getOpcode() == ISD::SETCC) { in LowerSELECT()
8389 CC = ISD::SETNE; in LowerSELECT()
8424 return DAG.getNode(ISD::BRIND, DL, MVT::Other, Op.getOperand(0), in LowerBR_JT()
8515 GRTopAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList, in LowerAAPCS_VASTART()
8519 GRTop = DAG.getNode(ISD::ADD, DL, PtrVT, GRTop, in LowerAAPCS_VASTART()
8533 VRTopAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList, in LowerAAPCS_VASTART()
8537 VRTop = DAG.getNode(ISD::ADD, DL, PtrVT, VRTop, in LowerAAPCS_VASTART()
8548 SDValue GROffsAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList, in LowerAAPCS_VASTART()
8556 SDValue VROffsAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList, in LowerAAPCS_VASTART()
8562 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in LowerAAPCS_VASTART()
8619 VAList = DAG.getNode(ISD::ADD, DL, PtrVT, VAList, in LowerVAARG()
8621 VAList = DAG.getNode(ISD::AND, DL, PtrVT, VAList, in LowerVAARG()
8641 SDValue VANext = DAG.getNode(ISD::ADD, DL, PtrVT, VAList, in LowerVAARG()
8655 SDValue NarrowFP = DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0), in LowerVAARG()
8680 FrameAddr = DAG.getNode(ISD::AssertZext, DL, MVT::i64, FrameAddr, in LowerFRAMEADDR()
8727 return DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset); in LowerADDROFRETURNADDR()
8745 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset), MachinePointerInfo()); in LowerRETURNADDR()
8857 return DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ); in getSqrtInputTest()
8884 SDValue Step = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Estimate, in getSqrtEstimate()
8887 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags); in getSqrtEstimate()
8890 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Operand, Estimate, Flags); in getSqrtEstimate()
8916 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags); in getRecipEstimate()
9329 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideTy, DAG.getUNDEF(WideTy), in WidenVector()
9356 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!"); in ReconstructShuffle()
9393 else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in ReconstructShuffle()
9459 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i8, Src, in ReconstructShuffle()
9473 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, in ReconstructShuffle()
9521 DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
9541 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
9547 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
9552 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
9555 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
9581 Src.ShuffleVec = DAG.getNode(ISD::BITCAST, dl, ShuffleVT, Src.ShuffleVec); in ReconstructShuffle()
9633 SDValue V = DAG.getNode(ISD::BITCAST, dl, VT, Shuffle); in ReconstructShuffle()
9684 if (BaseExt.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in ReconstructTruncateFromBuildVector()
9694 if (Ext.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in ReconstructTruncateFromBuildVector()
9712 Trunc[I] = DAG.getNode(ISD::TRUNCATE, DL, MVT::v4i16, Trunc[I]); in ReconstructTruncateFromBuildVector()
9714 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i16, Trunc[0], Trunc[1]); in ReconstructTruncateFromBuildVector()
9716 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i16, Trunc[2], Trunc[3]); in ReconstructTruncateFromBuildVector()
9717 SDValue Trunc0 = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, Concat0); in ReconstructTruncateFromBuildVector()
9718 SDValue Trunc1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, Concat1); in ReconstructTruncateFromBuildVector()
9719 return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, Trunc0, Trunc1); in ReconstructTruncateFromBuildVector()
10040 V0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V0, in tryFormConcatFromShuffle()
10044 V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V1, in tryFormConcatFromShuffle()
10047 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, V0, V1); in tryFormConcatFromShuffle()
10140 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, in GeneratePerfectShuffle()
10144 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Input.getValueType(), OpLHS, in GeneratePerfectShuffle()
10261 SDValue V1Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V1); in GenerateTBL()
10262 SDValue V2Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V2); in GenerateTBL()
10267 V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V1Cst); in GenerateTBL()
10269 ISD::INTRINSIC_WO_CHAIN, DL, IndexVT, in GenerateTBL()
10275 V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V2Cst); in GenerateTBL()
10277 ISD::INTRINSIC_WO_CHAIN, DL, IndexVT, in GenerateTBL()
10289 ISD::INTRINSIC_WO_CHAIN, DL, IndexVT, in GenerateTBL()
10295 return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Shuffle); in GenerateTBL()
10316 if (BitCast.getOpcode() != ISD::BITCAST || in constructDup()
10317 BitCast.getOperand(0).getOpcode() != ISD::EXTRACT_SUBVECTOR) in constructDup()
10351 } else if (V.getOpcode() == ISD::EXTRACT_SUBVECTOR && in constructDup()
10357 } else if (V.getOpcode() == ISD::CONCAT_VECTORS) { in constructDup()
10479 if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR) in LowerVECTOR_SHUFFLE()
10484 if (V1.getOpcode() == ISD::BUILD_VECTOR && in LowerVECTOR_SHUFFLE()
10592 ISD::INSERT_VECTOR_ELT, dl, VT, DstVec, in LowerVECTOR_SHUFFLE()
10593 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, SrcVec, SrcLaneV), in LowerVECTOR_SHUFFLE()
10641 SplatVal = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, SplatVal, in LowerSPLAT_VECTOR()
10647 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::nxv1i1, in LowerSPLAT_VECTOR()
10648 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::nxv2i1, ID, in LowerSPLAT_VECTOR()
10651 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, ID, Zero, SplatVal); in LowerSPLAT_VECTOR()
10676 SDValue V = DAG.getNode(ISD::BITCAST, DL, MVT::nxv2i64, Op.getOperand(1)); in LowerDUPQLane()
10683 SDValue SplatOne = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, One); in LowerDUPQLane()
10687 SV = DAG.getNode(ISD::AND, DL, MVT::nxv2i64, SV, SplatOne); in LowerDUPQLane()
10690 SDValue Idx64 = DAG.getNode(ISD::ADD, DL, MVT::i64, Idx128, Idx128); in LowerDUPQLane()
10691 SDValue SplatIdx64 = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, Idx64); in LowerDUPQLane()
10692 SDValue ShuffleMask = DAG.getNode(ISD::ADD, DL, MVT::nxv2i64, SV, SplatIdx64); in LowerDUPQLane()
10696 return DAG.getNode(ISD::BITCAST, DL, VT, TBL); in LowerDUPQLane()
10960 if ((FirstOpc == ISD::AND || FirstOpc == AArch64ISD::BICi) && in tryLowerToSLI()
10965 } else if ((SecondOpc == ISD::AND || SecondOpc == AArch64ISD::BICi) && in tryLowerToSLI()
10972 bool IsAnd = And.getOpcode() == ISD::AND; in tryLowerToSLI()
11070 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!"); in NormalizeBuildVector()
11150 SDValue Seq = DAG.getNode(ISD::ADD, DL, ContainerVT, Start, Steps); in LowerBUILD_VECTOR()
11205 if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT) in LowerBUILD_VECTOR()
11244 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value); in LowerBUILD_VECTOR()
11293 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0), in LowerBUILD_VECTOR()
11296 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0), in LowerBUILD_VECTOR()
11312 if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in LowerBUILD_VECTOR()
11344 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, NewType, Op.getOperand(i))); in LowerBUILD_VECTOR()
11351 return DAG.getNode(ISD::BITCAST, dl, VT, Val); in LowerBUILD_VECTOR()
11385 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Val, V, LaneIdx); in LowerBUILD_VECTOR()
11418 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, NewVector, in LowerBUILD_VECTOR()
11451 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op0); in LowerBUILD_VECTOR()
11461 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx); in LowerBUILD_VECTOR()
11498 DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), PairVT, V1, V2); in LowerCONCAT_VECTORS()
11510 assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT && "Unknown opcode!"); in LowerINSERT_VECTOR_ELT()
11529 DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VectorVT, ExtendedVector, in LowerINSERT_VECTOR_ELT()
11555 SDValue Node = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideTy, WideVec, in LowerINSERT_VECTOR_ELT()
11564 assert(Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Unknown opcode!"); in LowerEXTRACT_VECTOR_ELT()
11573 DAG.getNode(ISD::ANY_EXTEND, DL, VectorVT, Op.getOperand(0)); in LowerEXTRACT_VECTOR_ELT()
11575 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractTy, in LowerEXTRACT_VECTOR_ELT()
11610 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtrTy, WideVec, in LowerEXTRACT_VECTOR_ELT()
11652 SDValue Splice = DAG.getNode(ISD::VECTOR_SPLICE, DL, ContainerVT, NewInVec, in LowerEXTRACT_SUBVECTOR()
11683 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Vec0, in LowerINSERT_SUBVECTOR()
11685 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Vec0, in LowerINSERT_SUBVECTOR()
11688 SDValue NewLo = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, HalfVT, Lo, Vec1, in LowerINSERT_SUBVECTOR()
11693 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, HalfVT, Hi, Vec1, in LowerINSERT_SUBVECTOR()
11715 Vec1 = DAG.getNode(ISD::ANY_EXTEND, DL, WideVT, Vec1); in LowerINSERT_SUBVECTOR()
11745 return DAG.getNode(ISD::VSELECT, DL, VT, PTrue, ScalableVec1, Vec0); in LowerINSERT_SUBVECTOR()
11753 Op.getOpcode() != ISD::SPLAT_VECTOR && in isPow2Splat()
11754 Op.getOpcode() != ISD::BUILD_VECTOR) in isPow2Splat()
11757 if (Op.getOpcode() == ISD::BUILD_VECTOR && in isPow2Splat()
11761 if (Op.getOpcode() != ISD::BUILD_VECTOR && in isPow2Splat()
11791 bool Signed = Op.getOpcode() == ISD::SDIV; in LowerDIV()
11802 Res = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), Res); in LowerDIV()
11871 while (Op.getOpcode() == ISD::BITCAST) in getVShiftImm()
11918 SDValue And = DAG.getNode(ISD::AND, dl, OpVT, Op.getOperand(0), One); in LowerTRUNCATE()
11919 return DAG.getSetCC(dl, VT, And, Zero, ISD::SETNE); in LowerTRUNCATE()
11942 case ISD::SHL: in LowerVectorSRA_SRL_SHL()
11949 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in LowerVectorSRA_SRL_SHL()
11953 case ISD::SRA: in LowerVectorSRA_SRL_SHL()
11954 case ISD::SRL: in LowerVectorSRA_SRL_SHL()
11956 unsigned Opc = Op.getOpcode() == ISD::SRA ? AArch64ISD::SRA_PRED in LowerVectorSRA_SRL_SHL()
11964 (Op.getOpcode() == ISD::SRA) ? AArch64ISD::VASHR : AArch64ISD::VLSHR; in LowerVectorSRA_SRL_SHL()
11972 unsigned Opc = (Op.getOpcode() == ISD::SRA) ? Intrinsic::aarch64_neon_sshl in LowerVectorSRA_SRL_SHL()
11975 SDValue NegShift = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in LowerVectorSRA_SRL_SHL()
11978 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in LowerVectorSRA_SRL_SHL()
12095 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); in LowerVSETCC()
12115 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, LHS); in LowerVSETCC()
12116 RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, RHS); in LowerVSETCC()
12145 Cmp = DAG.getNode(ISD::OR, dl, CmpVT, Cmp, Cmp2); in LowerVSETCC()
12160 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarOp.getValueType(), Rdx, in getReductionSDNode()
12170 bool OverrideNEON = Op.getOpcode() == ISD::VECREDUCE_AND || in LowerVECREDUCE()
12171 Op.getOpcode() == ISD::VECREDUCE_OR || in LowerVECREDUCE()
12172 Op.getOpcode() == ISD::VECREDUCE_XOR || in LowerVECREDUCE()
12173 Op.getOpcode() == ISD::VECREDUCE_FADD || in LowerVECREDUCE()
12174 (Op.getOpcode() != ISD::VECREDUCE_ADD && in LowerVECREDUCE()
12184 case ISD::VECREDUCE_ADD: in LowerVECREDUCE()
12186 case ISD::VECREDUCE_AND: in LowerVECREDUCE()
12188 case ISD::VECREDUCE_OR: in LowerVECREDUCE()
12190 case ISD::VECREDUCE_SMAX: in LowerVECREDUCE()
12192 case ISD::VECREDUCE_SMIN: in LowerVECREDUCE()
12194 case ISD::VECREDUCE_UMAX: in LowerVECREDUCE()
12196 case ISD::VECREDUCE_UMIN: in LowerVECREDUCE()
12198 case ISD::VECREDUCE_XOR: in LowerVECREDUCE()
12200 case ISD::VECREDUCE_FADD: in LowerVECREDUCE()
12202 case ISD::VECREDUCE_FMAX: in LowerVECREDUCE()
12204 case ISD::VECREDUCE_FMIN: in LowerVECREDUCE()
12214 case ISD::VECREDUCE_ADD: in LowerVECREDUCE()
12216 case ISD::VECREDUCE_SMAX: in LowerVECREDUCE()
12218 case ISD::VECREDUCE_SMIN: in LowerVECREDUCE()
12220 case ISD::VECREDUCE_UMAX: in LowerVECREDUCE()
12222 case ISD::VECREDUCE_UMIN: in LowerVECREDUCE()
12224 case ISD::VECREDUCE_FMAX: { in LowerVECREDUCE()
12226 ISD::INTRINSIC_WO_CHAIN, dl, Op.getValueType(), in LowerVECREDUCE()
12230 case ISD::VECREDUCE_FMIN: { in LowerVECREDUCE()
12232 ISD::INTRINSIC_WO_CHAIN, dl, Op.getValueType(), in LowerVECREDUCE()
12252 RHS = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), RHS); in LowerATOMIC_LOAD_SUB()
12253 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, dl, AN->getMemoryVT(), in LowerATOMIC_LOAD_SUB()
12269 RHS = DAG.getNode(ISD::XOR, dl, VT, DAG.getConstant(-1ULL, dl, VT), RHS); in LowerATOMIC_LOAD_AND()
12270 return DAG.getAtomic(ISD::ATOMIC_LOAD_CLR, dl, AN->getMemoryVT(), in LowerATOMIC_LOAD_AND()
12286 Size = DAG.getNode(ISD::SRL, dl, MVT::i64, Size, in LowerWindowsDYNAMIC_STACKALLOC()
12298 Size = DAG.getNode(ISD::SHL, dl, MVT::i64, Size, in LowerWindowsDYNAMIC_STACKALLOC()
12321 SP = DAG.getNode(ISD::SUB, dl, MVT::i64, SP, Size); in LowerDYNAMIC_STACKALLOC()
12323 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0), in LowerDYNAMIC_STACKALLOC()
12336 SP = DAG.getNode(ISD::SUB, dl, MVT::i64, SP, Size); in LowerDYNAMIC_STACKALLOC()
12338 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0), in LowerDYNAMIC_STACKALLOC()
12365 Info.opc = ISD::INTRINSIC_VOID; in setInfoSVEStN()
12412 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
12432 Info.opc = ISD::INTRINSIC_VOID; in getTgtMemIntrinsic()
12452 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
12463 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
12473 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
12482 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
12491 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
12502 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
12513 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
12531 ISD::LoadExtType ExtTy, in shouldReduceLoadWidth()
12539 if (ExtTy != ISD::NON_EXTLOAD) in shouldReduceLoadWidth()
12546 if (Base.getOpcode() == ISD::ADD && in shouldReduceLoadWidth()
12547 Base.getOperand(1).getOpcode() == ISD::SHL && in shouldReduceLoadWidth()
12549 Base.getOperand(1).getOperand(1).getOpcode() == ISD::Constant) { in shouldReduceLoadWidth()
12602 isOperationLegalOrCustom(ISD::FMA, getValueType(DL, Ty)) && in isProfitableToHoist()
12630 if (Val.getOpcode() != ISD::LOAD) in isZExtFree()
13410 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, PseudoLoadOps); in LowerSVEStructLoad()
13653 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA || in isDesirableToCommuteWithShift()
13654 N->getOpcode() == ISD::SRL) && in isDesirableToCommuteWithShift()
13662 if (ShiftLHS.getOpcode() == ISD::AND && (VT == MVT::i32 || VT == MVT::i64) && in isDesirableToCommuteWithShift()
13666 ShiftLHS.getOperand(0).getOpcode() == ISD::SRL && in isDesirableToCommuteWithShift()
13675 assert(N->getOpcode() == ISD::XOR && in isDesirableToCommuteXorWithShift()
13676 (N->getOperand(0).getOpcode() == ISD::SHL || in isDesirableToCommuteXorWithShift()
13677 N->getOperand(0).getOpcode() == ISD::SRL) && in isDesirableToCommuteXorWithShift()
13688 if (N->getOperand(0).getOpcode() == ISD::SHL) in isDesirableToCommuteXorWithShift()
13699 assert(((N->getOpcode() == ISD::SHL && in shouldFoldConstantShiftPairToMask()
13700 N->getOperand(0).getOpcode() == ISD::SRL) || in shouldFoldConstantShiftPairToMask()
13701 (N->getOpcode() == ISD::SRL && in shouldFoldConstantShiftPairToMask()
13702 N->getOperand(0).getOpcode() == ISD::SHL)) && in shouldFoldConstantShiftPairToMask()
13710 if (N->getOpcode() == ISD::SRL && (VT == MVT::i32 || VT == MVT::i64)) { in shouldFoldConstantShiftPairToMask()
13744 if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT)) in isExtractSubvectorCheap()
13765 !ISD::isBuildVectorAllOnes(Ones.getNode())) in foldVectorXorShiftIntoCmp()
13801 if (Opcode != ISD::ABS || VecReduceOp0->getValueType(0) != MVT::v16i32) in performVecReduceAddCombineWithUADDLP()
13806 if (ABS->getOperand(0)->getOpcode() != ISD::SUB || in performVecReduceAddCombineWithUADDLP()
13820 if (Opcode0 == ISD::ZERO_EXTEND && Opcode1 == ISD::ZERO_EXTEND) { in performVecReduceAddCombineWithUADDLP()
13822 } else if (Opcode0 == ISD::SIGN_EXTEND && Opcode1 == ISD::SIGN_EXTEND) { in performVecReduceAddCombineWithUADDLP()
13839 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i8, EXT0->getOperand(0), in performVecReduceAddCombineWithUADDLP()
13842 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i8, EXT1->getOperand(0), in performVecReduceAddCombineWithUADDLP()
13844 SDValue UABDHigh8 = DAG.getNode(IsZExt ? ISD::ABDU : ISD::ABDS, DL, MVT::v8i8, in performVecReduceAddCombineWithUADDLP()
13846 SDValue UABDL = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, UABDHigh8); in performVecReduceAddCombineWithUADDLP()
13850 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i8, EXT0->getOperand(0), in performVecReduceAddCombineWithUADDLP()
13853 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i8, EXT1->getOperand(0), in performVecReduceAddCombineWithUADDLP()
13855 SDValue UABDLo8 = DAG.getNode(IsZExt ? ISD::ABDU : ISD::ABDS, DL, MVT::v8i8, in performVecReduceAddCombineWithUADDLP()
13857 SDValue ZExtUABD = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, UABDLo8); in performVecReduceAddCombineWithUADDLP()
13858 SDValue UABAL = DAG.getNode(ISD::ADD, DL, MVT::v8i16, UABDL, ZExtUABD); in performVecReduceAddCombineWithUADDLP()
13864 return DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i32, UADDLP); in performVecReduceAddCombineWithUADDLP()
13883 if (ExtOpcode == ISD::MUL) { in performVecReduceAddCombine()
13891 if (ExtOpcode != ISD::ZERO_EXTEND && ExtOpcode != ISD::SIGN_EXTEND) in performVecReduceAddCombine()
13909 (ExtOpcode == ISD::ZERO_EXTEND) ? AArch64ISD::UDOT : AArch64ISD::SDOT; in performVecReduceAddCombine()
13912 return DAG.getNode(ISD::VECREDUCE_ADD, DL, N->getValueType(0), Dot); in performVecReduceAddCombine()
13923 if (A.getOpcode() != ISD::ADD) in performUADDVCombine()
13929 (Op0.getOpcode() != ISD::ZERO_EXTEND && in performUADDVCombine()
13930 Op0.getOpcode() != ISD::SIGN_EXTEND)) in performUADDVCombine()
13934 if (Ext0.getOpcode() != ISD::EXTRACT_SUBVECTOR || in performUADDVCombine()
13935 Ext1.getOpcode() != ISD::EXTRACT_SUBVECTOR || in performUADDVCombine()
13948 unsigned Opcode = Op0.getOpcode() == ISD::ZERO_EXTEND ? AArch64ISD::UADDLP in performUADDVCombine()
13956 if (A.getOpcode() == ISD::ADD) { in performUADDVCombine()
13959 DAG.getNode(ISD::ADD, SDLoc(A), A.getValueType(), R, in performUADDVCombine()
13963 DAG.getNode(ISD::ADD, SDLoc(A), A.getValueType(), R, in performUADDVCombine()
14007 SDValue Cmp = getAArch64Cmp(N0, Zero, ISD::SETLT, CCVal, DAG, DL); in BuildSDIVPow2()
14008 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne); in BuildSDIVPow2()
14017 DAG.getNode(ISD::SRA, DL, VT, CSel, DAG.getConstant(Lg2, DL, MVT::i64)); in BuildSDIVPow2()
14025 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA); in BuildSDIVPow2()
14058 SDValue Cmp = getAArch64Cmp(N0, Zero, ISD::SETGE, CCVal, DAG, DL); in BuildSREMPow2()
14059 SDValue And = DAG.getNode(ISD::AND, DL, VT, N0, Pow2MinusOne); in BuildSREMPow2()
14069 SDValue AndPos = DAG.getNode(ISD::AND, DL, VT, N0, Pow2MinusOne); in BuildSREMPow2()
14070 SDValue AndNeg = DAG.getNode(ISD::AND, DL, VT, Negs, Pow2MinusOne); in BuildSREMPow2()
14108 case ISD::SIGN_EXTEND: in calculatePreExtendType()
14109 case ISD::ZERO_EXTEND: in calculatePreExtendType()
14111 case ISD::AssertSext: in calculatePreExtendType()
14112 case ISD::AssertZext: in calculatePreExtendType()
14113 case ISD::SIGN_EXTEND_INREG: { in calculatePreExtendType()
14119 case ISD::AND: { in calculatePreExtendType()
14146 if (BV.getOpcode() != ISD::BUILD_VECTOR && in performBuildShuffleExtendCombine()
14147 BV.getOpcode() != ISD::VECTOR_SHUFFLE) in performBuildShuffleExtendCombine()
14154 bool IsSExt = ExtendOpcode == ISD::SIGN_EXTEND || in performBuildShuffleExtendCombine()
14155 ExtendOpcode == ISD::SIGN_EXTEND_INREG || in performBuildShuffleExtendCombine()
14156 ExtendOpcode == ISD::AssertSext; in performBuildShuffleExtendCombine()
14157 if (!IsSExt && ExtendOpcode != ISD::ZERO_EXTEND && in performBuildShuffleExtendCombine()
14158 ExtendOpcode != ISD::AssertZext && ExtendOpcode != ISD::AND) in performBuildShuffleExtendCombine()
14162 if (BV.getOpcode() == ISD::VECTOR_SHUFFLE && in performBuildShuffleExtendCombine()
14163 ExtendOpcode != ISD::SIGN_EXTEND && ExtendOpcode != ISD::ZERO_EXTEND) in performBuildShuffleExtendCombine()
14177 bool OpcIsSExt = Opc == ISD::SIGN_EXTEND || Opc == ISD::SIGN_EXTEND_INREG || in performBuildShuffleExtendCombine()
14178 Opc == ISD::AssertSext; in performBuildShuffleExtendCombine()
14185 if (BV.getOpcode() == ISD::BUILD_VECTOR) { in performBuildShuffleExtendCombine()
14194 NBV = DAG.getNode(ISD::BUILD_VECTOR, DL, PreExtendVT, NewOps); in performBuildShuffleExtendCombine()
14203 return DAG.getNode(IsSExt ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL, VT, NBV); in performBuildShuffleExtendCombine()
14248 if ((AddSubOpc == ISD::ADD || AddSubOpc == ISD::SUB) && V->hasOneUse()) { in performMulCombine()
14251 if (AddSubOpc == ISD::SUB) in performMulCombine()
14260 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N1, MulOper); in performMulCombine()
14265 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N0, MulOper); in performMulCombine()
14279 (N0->getOpcode() == ISD::TRUNCATE && in performMulCombine()
14306 if (N->hasOneUse() && (N->use_begin()->getOpcode() == ISD::ADD || in performMulCombine()
14307 N->use_begin()->getOpcode() == ISD::SUB)) in performMulCombine()
14328 AddSubOpc = ISD::ADD; in performMulCombine()
14331 AddSubOpc = ISD::SUB; in performMulCombine()
14341 AddSubOpc = ISD::SUB; in performMulCombine()
14345 AddSubOpc = ISD::ADD; in performMulCombine()
14351 SDValue ShiftedVal = DAG.getNode(ISD::SHL, DL, VT, N0, in performMulCombine()
14361 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Res); in performMulCombine()
14364 return DAG.getNode(ISD::SHL, DL, VT, Res, in performMulCombine()
14383 if (!VT.isVector() || N->getOperand(0)->getOpcode() != ISD::AND || in performVectorCompareAndMaskUnaryOpCombine()
14384 N->getOperand(0)->getOperand(0)->getOpcode() != ISD::SETCC || in performVectorCompareAndMaskUnaryOpCombine()
14405 SDValue MaskConst = DAG.getNode(ISD::BITCAST, DL, IntVT, SourceConst); in performVectorCompareAndMaskUnaryOpCombine()
14406 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT, in performVectorCompareAndMaskUnaryOpCombine()
14408 SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd); in performVectorCompareAndMaskUnaryOpCombine()
14434 if (Subtarget->hasNEON() && ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && in performIntToFpCombine()
14447 (N->getOpcode() == ISD::SINT_TO_FP) ? AArch64ISD::SITOF : AArch64ISD::UITOF; in performIntToFpCombine()
14466 if (!Op.getValueType().isSimple() || Op.getOpcode() != ISD::FMUL) in performFpToIntCombine()
14502 if (N->getOpcode() == ISD::FP_TO_SINT_SAT || in performFpToIntCombine()
14503 N->getOpcode() == ISD::FP_TO_UINT_SAT) { in performFpToIntCombine()
14510 bool IsSigned = (N->getOpcode() == ISD::FP_TO_SINT || in performFpToIntCombine()
14511 N->getOpcode() == ISD::FP_TO_SINT_SAT); in performFpToIntCombine()
14515 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ResTy, in performFpToIntCombine()
14520 FixConv = DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), FixConv); in performFpToIntCombine()
14537 (Opc != ISD::SINT_TO_FP && Opc != ISD::UINT_TO_FP)) in performFDivCombine()
14582 bool IsSigned = Opc == ISD::SINT_TO_FP; in performFDivCombine()
14584 ConvInput = DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL, in performFDivCombine()
14589 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(), in performFDivCombine()
14598 if (N.getOpcode() == ISD::SHL) in findEXTRHalf()
14600 else if (N.getOpcode() == ISD::SRL) in findEXTRHalf()
14624 assert(N->getOpcode() == ISD::OR && "Unexpected root"); in tryCombineToEXTR()
14673 if (N0.getOpcode() != ISD::AND) in tryCombineToBSL()
14677 if (N1.getOpcode() != ISD::AND) in tryCombineToBSL()
14690 if (O0.getOpcode() == ISD::SUB && O1.getOpcode() == ISD::ADD) { in tryCombineToBSL()
14695 } else if (O0.getOpcode() == ISD::ADD && O1.getOpcode() == ISD::SUB) { in tryCombineToBSL()
14703 if (!ISD::isBuildVectorAllZeros(Sub.getOperand(0).getNode())) in tryCombineToBSL()
14707 if (!ISD::isBuildVectorAllOnes(Add.getOperand(1).getNode())) in tryCombineToBSL()
14794 if (N->getOpcode() == ISD::AND) { in performANDORCSELCombine()
14856 if (N->getOpcode() == AArch64ISD::DUP || N->getOpcode() == ISD::SPLAT_VECTOR) in isConstantSplatVectorMaskForType()
14877 if (Dup.getOpcode() != ISD::SPLAT_VECTOR) in performSVEAndCombine()
14900 Dup = DAG.getNode(ISD::SPLAT_VECTOR, DL, UnpkOp->getValueType(0), in performSVEAndCombine()
14903 SDValue And = DAG.getNode(ISD::AND, DL, in performSVEAndCombine()
15008 case ISD::STRICT_FADD: in hasPairwiseAdd()
15009 case ISD::FADD: in hasPairwiseAdd()
15011 case ISD::ADD: in hasPairwiseAdd()
15022 if ((N.getOpcode() == ISD::SETCC) || in isPredicateCCSettingOp()
15023 (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN && in isPredicateCCSettingOp()
15045 assert(N->getOpcode() == ISD::EXTRACT_VECTOR_ELT); in performFirstTrueTestVectorCombine()
15075 assert(N->getOpcode() == ISD::EXTRACT_VECTOR_ELT); in performLastTrueTestVectorCombine()
15088 if (Idx.getOpcode() != ISD::ADD || !isAllOnesConstant(Idx.getOperand(1))) in performLastTrueTestVectorCombine()
15092 if (VS.getOpcode() != ISD::VSCALE) in performLastTrueTestVectorCombine()
15108 assert(N->getOpcode() == ISD::EXTRACT_VECTOR_ELT); in performExtractVectorEltCombine()
15153 SDValue Extract1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Other, in performExtractVectorEltCombine()
15155 SDValue Extract2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Other, in performExtractVectorEltCombine()
15198 if (N->getNumOperands() == 2 && N0Opc == ISD::TRUNCATE && in performConcatVectorsCombine()
15199 N1Opc == ISD::TRUNCATE) { in performConcatVectorsCombine()
15211 return DAG.getNode(ISD::TRUNCATE, dl, VT, in performConcatVectorsCombine()
15214 DAG.getNode(ISD::BITCAST, dl, MidVT, N00), in performConcatVectorsCombine()
15215 DAG.getNode(ISD::BITCAST, dl, MidVT, N10), Mask)); in performConcatVectorsCombine()
15230 LD->getExtensionType() == ISD::NON_EXTLOAD; in performConcatVectorsCombine()
15272 (N0Opc == ISD::AVGCEILU || N0Opc == ISD::AVGCEILS || in performConcatVectorsCombine()
15273 N0Opc == ISD::AVGFLOORU || N0Opc == ISD::AVGFLOORS)) { in performConcatVectorsCombine()
15282 if (N00->getOpcode() == ISD::EXTRACT_SUBVECTOR && in performConcatVectorsCombine()
15283 N01->getOpcode() == ISD::EXTRACT_SUBVECTOR && in performConcatVectorsCombine()
15284 N10->getOpcode() == ISD::EXTRACT_SUBVECTOR && in performConcatVectorsCombine()
15285 N11->getOpcode() == ISD::EXTRACT_SUBVECTOR && N00VT == N10VT) { in performConcatVectorsCombine()
15325 if (N->getNumOperands() != 2 || N1Opc != ISD::BITCAST) in performConcatVectorsCombine()
15338 return DAG.getNode(ISD::BITCAST, dl, VT, in performConcatVectorsCombine()
15339 DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatTy, in performConcatVectorsCombine()
15340 DAG.getNode(ISD::BITCAST, dl, RHSTy, N0), in performConcatVectorsCombine()
15360 if (V.getOpcode() == ISD::SPLAT_VECTOR) in performExtractSubvectorCombine()
15362 return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), VT, V.getOperand(0)); in performExtractSubvectorCombine()
15399 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, Vec, in performInsertSubvectorCombine()
15402 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, Vec, in performInsertSubvectorCombine()
15406 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Lo, Hi); in performInsertSubvectorCombine()
15427 if (Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT) in tryCombineFixedPointConvert()
15453 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VecResTy, IID, Vec, Shift); in tryCombineFixedPointConvert()
15454 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResTy, Convert, Lane); in tryCombineFixedPointConvert()
15474 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR && in tryExtendDUPToExtractHigh()
15509 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, N, in tryExtendDUPToExtractHigh()
15514 if (N.getOpcode() == ISD::BITCAST) in isEssentiallyExtractHighSubvector()
15516 if (N.getOpcode() != ISD::EXTRACT_SUBVECTOR) in isEssentiallyExtractHighSubvector()
15528 ISD::CondCode CC;
15559 if (Op.getOpcode() == ISD::SETCC) { in isSetCC()
15603 return ((Op.getOpcode() == ISD::ZERO_EXTEND) && in isSetCCOrZExtSetCC()
15614 assert(Op && Op->getOpcode() == ISD::ADD && "Unexpected operation!"); in performSetccAddFolding()
15651 ISD::getSetCCInverse(InfoAndKind.Info.Generic.CC, CmpVT), CCVal, DAG, in performSetccAddFolding()
15655 LHS = DAG.getNode(ISD::ADD, dl, VT, RHS, DAG.getConstant(1, dl, VT)); in performSetccAddFolding()
15663 if (N->getOpcode() != ISD::ADD || !VT.isScalarInteger()) in performAddUADDVCombine()
15668 if (LHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in performAddUADDVCombine()
15669 RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT || LHS.getValueType() != VT) in performAddUADDVCombine()
15690 SDValue AddVal = DAG.getNode(ISD::ADD, DL, ValVT, Val1, Val2); in performAddUADDVCombine()
15691 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in performAddUADDVCombine()
15701 if (!VT.isScalarInteger() || N->getOpcode() != ISD::ADD) in performAddCSelIntoCSinc()
15764 SDValue NewNode = DAG.getNode(ISD::ADD, DL, VT, RHS, SDValue(CTVal, 0)); in performAddCSelIntoCSinc()
15774 if (N->getOpcode() != ISD::ADD) in performAddDotCombine()
15795 return Op.getOpcode() == ISD::SUB && isNullConstant(Op.getOperand(0)); in isNegatedInteger()
15802 return DAG.getNode(ISD::SUB, DL, VT, Zero, Op); in getNegatedInteger()
15856 if (N->getOpcode() == ISD::ADD) in performAddSubLongCombine()
15864 if ((LHS.getOpcode() != ISD::ZERO_EXTEND && in performAddSubLongCombine()
15865 LHS.getOpcode() != ISD::SIGN_EXTEND) || in performAddSubLongCombine()
15962 (N->getOperand(0).getOpcode() != ISD::ZERO_EXTEND && in performVectorAddSubExtCombine()
15963 N->getOperand(0).getOpcode() != ISD::SIGN_EXTEND) || in performVectorAddSubExtCombine()
15964 (N->getOperand(1).getOpcode() != ISD::ZERO_EXTEND && in performVectorAddSubExtCombine()
15965 N->getOperand(1).getOpcode() != ISD::SIGN_EXTEND) || in performVectorAddSubExtCombine()
15985 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NewOp); in performVectorAddSubExtCombine()
16009 if (Elt0->getOpcode() == ISD::EXTRACT_VECTOR_ELT && in performBuildVectorCombine()
16010 Elt1->getOpcode() == ISD::EXTRACT_VECTOR_ELT && in performBuildVectorCombine()
16025 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, DL, ExtVT, VecToExtend); in performBuildVectorCombine()
16026 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i32, Ext, in performBuildVectorCombine()
16086 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), N->getValueType(0), in tryCombineLongOpWithDup()
16163 if (AndN.getOpcode() != ISD::AND) in tryCombineCRC32()
16170 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), MVT::i32, in tryCombineCRC32()
16177 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, N->getValueType(0), in combineAcrossLanesIntrinsic()
16194 SDValue Step = DAG.getNode(ISD::SPLAT_VECTOR, DL, N->getValueType(0), Op2); in LowerSVEIntrinsicIndex()
16195 SDValue Mul = DAG.getNode(ISD::MUL, DL, N->getValueType(0), StepVector, Step); in LowerSVEIntrinsicIndex()
16196 SDValue Base = DAG.getNode(ISD::SPLAT_VECTOR, DL, N->getValueType(0), Op1); in LowerSVEIntrinsicIndex()
16197 return DAG.getNode(ISD::ADD, DL, N->getValueType(0), Mul, Base); in LowerSVEIntrinsicIndex()
16206 Scalar = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Scalar); in LowerSVEIntrinsicDUP()
16231 SDValue Op0 = DAG.getNode(ISD::BITCAST, dl, ByteVT, N->getOperand(1)); in LowerSVEIntrinsicEXT()
16232 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, ByteVT, N->getOperand(2)); in LowerSVEIntrinsicEXT()
16233 SDValue Op2 = DAG.getNode(ISD::MUL, dl, MVT::i32, N->getOperand(3), in LowerSVEIntrinsicEXT()
16237 return DAG.getNode(ISD::BITCAST, dl, VT, EXT); in LowerSVEIntrinsicEXT()
16240 static SDValue tryConvertSVEWideCompare(SDNode *N, ISD::CondCode CC, in tryConvertSVEWideCompare()
16248 Comparator.getOpcode() == ISD::SPLAT_VECTOR) { in tryConvertSVEWideCompare()
16296 SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, DL, CmpVT, Imm); in tryConvertSVEWideCompare()
16355 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, N->getValueType(0), Reduce, in combineSVEReductionInt()
16372 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, N->getValueType(0), Reduce, in combineSVEReductionFP()
16388 InitVal = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ReduceVT, in combineSVEReductionOrderedFP()
16395 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, N->getValueType(0), Reduce, in combineSVEReductionOrderedFP()
16404 return ISD::isConstantSplatVectorAllZeros(N.getNode()); in isAllInactivePredicate()
16419 if (ISD::isConstantSplatVectorAllOnes(N.getNode())) in isAllActivePredicate()
16452 assert(N->getOpcode() == ISD::INTRINSIC_WO_CHAIN && "Expected intrinsic!"); in convertMergedOpToPredOp()
16502 Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, WhileVT, ID, in performIntrinsicCombine()
16504 Res = DAG.getNode(ISD::SIGN_EXTEND, DL, PromVT, Res); in performIntrinsicCombine()
16505 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtVT, Res, in performIntrinsicCombine()
16507 Res = DAG.getNode(ISD::TRUNCATE, DL, VT, Res); in performIntrinsicCombine()
16527 return DAG.getNode(ISD::FMAXIMUM, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16530 return DAG.getNode(ISD::FMINIMUM, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16533 return DAG.getNode(ISD::FMAXNUM, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16536 return DAG.getNode(ISD::FMINNUM, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16588 return DAG.getNode(ISD::SPLAT_VECTOR, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16619 return convertMergedOpToPredOp(N, ISD::ADD, DAG, true); in performIntrinsicCombine()
16621 return convertMergedOpToPredOp(N, ISD::SUB, DAG, true); in performIntrinsicCombine()
16623 return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true); in performIntrinsicCombine()
16625 return convertMergedOpToPredOp(N, ISD::AND, DAG, true); in performIntrinsicCombine()
16629 return convertMergedOpToPredOp(N, ISD::XOR, DAG, true); in performIntrinsicCombine()
16631 return convertMergedOpToPredOp(N, ISD::OR, DAG, true); in performIntrinsicCombine()
16633 return convertMergedOpToPredOp(N, ISD::ABDS, DAG, true); in performIntrinsicCombine()
16635 return convertMergedOpToPredOp(N, ISD::ABDU, DAG, true); in performIntrinsicCombine()
16637 return convertMergedOpToPredOp(N, ISD::SADDSAT, DAG, true); in performIntrinsicCombine()
16639 return convertMergedOpToPredOp(N, ISD::SSUBSAT, DAG, true); in performIntrinsicCombine()
16641 return convertMergedOpToPredOp(N, ISD::UADDSAT, DAG, true); in performIntrinsicCombine()
16643 return convertMergedOpToPredOp(N, ISD::USUBSAT, DAG, true); in performIntrinsicCombine()
16645 return DAG.getNode(ISD::SADDSAT, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16648 return DAG.getNode(ISD::SSUBSAT, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16651 return DAG.getNode(ISD::UADDSAT, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16654 return DAG.getNode(ISD::USUBSAT, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16663 N->getOperand(3), DAG.getCondCode(ISD::SETUGE)); in performIntrinsicCombine()
16669 N->getOperand(3), DAG.getCondCode(ISD::SETUGT)); in performIntrinsicCombine()
16675 N->getOperand(3), DAG.getCondCode(ISD::SETGE)); in performIntrinsicCombine()
16681 N->getOperand(3), DAG.getCondCode(ISD::SETGT)); in performIntrinsicCombine()
16687 N->getOperand(3), DAG.getCondCode(ISD::SETEQ)); in performIntrinsicCombine()
16693 N->getOperand(3), DAG.getCondCode(ISD::SETNE)); in performIntrinsicCombine()
16698 N->getOperand(3), DAG.getCondCode(ISD::SETUO)); in performIntrinsicCombine()
16713 return DAG.getNode(ISD::VSELECT, SDLoc(N), N->getValueType(0), in performIntrinsicCombine()
16716 return tryConvertSVEWideCompare(N, ISD::SETEQ, DCI, DAG); in performIntrinsicCombine()
16718 return tryConvertSVEWideCompare(N, ISD::SETNE, DCI, DAG); in performIntrinsicCombine()
16720 return tryConvertSVEWideCompare(N, ISD::SETGE, DCI, DAG); in performIntrinsicCombine()
16722 return tryConvertSVEWideCompare(N, ISD::SETGT, DCI, DAG); in performIntrinsicCombine()
16724 return tryConvertSVEWideCompare(N, ISD::SETLT, DCI, DAG); in performIntrinsicCombine()
16726 return tryConvertSVEWideCompare(N, ISD::SETLE, DCI, DAG); in performIntrinsicCombine()
16728 return tryConvertSVEWideCompare(N, ISD::SETUGE, DCI, DAG); in performIntrinsicCombine()
16730 return tryConvertSVEWideCompare(N, ISD::SETUGT, DCI, DAG); in performIntrinsicCombine()
16732 return tryConvertSVEWideCompare(N, ISD::SETULT, DCI, DAG); in performIntrinsicCombine()
16734 return tryConvertSVEWideCompare(N, ISD::SETULE, DCI, DAG); in performIntrinsicCombine()
16750 return OC == ISD::LOAD || OC == ISD::MLOAD || in isCheapToExtend()
16751 ISD::isConstantSplatVectorAllZeros(N.getNode()); in isCheapToExtend()
16763 assert(N->getOpcode() == ISD::SIGN_EXTEND && in performSignExtendSetCCCombine()
16764 N->getOperand(0)->getOpcode() == ISD::SETCC); in performSignExtendSetCCCombine()
16773 ISD::CondCode Code = in performSignExtendSetCCCombine()
16776 ISD::NodeType ExtType = in performSignExtendSetCCCombine()
16777 isSignedIntSetCC(Code) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in performSignExtendSetCCCombine()
16801 if (!DCI.isBeforeLegalizeOps() && N->getOpcode() == ISD::ZERO_EXTEND && in performExtendCombine()
16802 (N->getOperand(0).getOpcode() == ISD::ABDU || in performExtendCombine()
16803 N->getOperand(0).getOpcode() == ISD::ABDS)) { in performExtendCombine()
16810 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), NewABD); in performExtendCombine()
16814 N->getOpcode() == ISD::SIGN_EXTEND && in performExtendCombine()
16815 N->getOperand(0)->getOpcode() == ISD::SETCC) in performExtendCombine()
16841 if (BasePtr->getOpcode() == ISD::ADD && in splitStoreSplat()
16851 DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, in splitStoreSplat()
16912 Load = DAG.getNode(ISD::TRUNCATE, DL, VT, Load.getValue(0)); in performLD1Combine()
16932 ISD::UNINDEXED, ISD::NON_EXTLOAD, false); in performLDNT1Combine()
16935 SDValue Ops[] = { DAG.getNode(ISD::BITCAST, DL, VT, L), L.getValue(1) }; in performLDNT1Combine()
16959 Load = DAG.getNode(ISD::BITCAST, DL, VT, Load.getValue(0)); in performLD1ReplicateCombine()
16976 SrcNew = DAG.getNode(ISD::BITCAST, DL, HwSrcVt, Data); in performST1Combine()
16978 SrcNew = DAG.getNode(ISD::ANY_EXTEND, DL, HwSrcVt, Data); in performST1Combine()
16998 Data = DAG.getNode(ISD::BITCAST, DL, DataVT.changeTypeToInteger(), Data); in performSTNT1Combine()
17004 ISD::UNINDEXED, false, false); in performSTNT1Combine()
17039 if (StVal.getOpcode() != ISD::BUILD_VECTOR) in replaceZeroVectorStore()
17115 if (StVal.getOpcode() != ISD::INSERT_VECTOR_ELT) in replaceSplatVectorStore()
17198 SDValue SubVector0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal, in splitStores()
17200 SDValue SubVector1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal, in splitStores()
17206 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, in splitStores()
17236 if (N->getOperand(0).getOpcode() == ISD::MLOAD && in performUnpackCombine()
17242 if (MLD->isUnindexed() && MLD->getExtensionType() != ISD::SEXTLOAD && in performUnpackCombine()
17260 MLD->getAddressingMode(), ISD::ZEXTLOAD); in performUnpackCombine()
17279 if (Op1.getOpcode() == ISD::UNDEF) { in performUzpCombine()
17299 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, HalfVT, BC); in performUzpCombine()
17300 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Trunc, in performUzpCombine()
17408 if (N->getOperand(0).getOpcode() == ISD::SIGN_EXTEND && in performSunpkloCombine()
17413 SDValue Unpk = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT, CC, in performSunpkloCombine()
17415 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), N->getValueType(0), Unpk); in performSunpkloCombine()
17438 if (LD->getOpcode() != ISD::LOAD) in performPostLD1Combine()
17472 if (User->getOpcode() != ISD::ADD in performPostLD1Combine()
17546 assert((N->getOpcode() == ISD::STORE || N->getOpcode() == ISD::MSTORE) && in foldTruncStoreOfExt()
17554 if (ExtOpCode != ISD::ZERO_EXTEND && ExtOpCode != ISD::SIGN_EXTEND && in foldTruncStoreOfExt()
17555 ExtOpCode != ISD::ANY_EXTEND) in foldTruncStoreOfExt()
17580 if (DCI.isBeforeLegalizeOps() && Value.getOpcode() == ISD::FP_ROUND && in performSTORECombine()
17618 if (Value.getOpcode() == ISD::BITCAST) { in performMSTORECombine()
17660 if (Index.getOpcode() == ISD::ADD) { in foldIndexIntoBase()
17662 Offset = DAG.getNode(ISD::MUL, DL, MVT::i64, Offset, Scale); in foldIndexIntoBase()
17663 BasePtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, Offset); in foldIndexIntoBase()
17675 if (Index.getOpcode() == ISD::SHL && in foldIndexIntoBase()
17676 Index.getOperand(0).getOpcode() == ISD::ADD) { in foldIndexIntoBase()
17682 Offset = DAG.getNode(ISD::SHL, DL, MVT::i64, Offset, Shift); in foldIndexIntoBase()
17683 Offset = DAG.getNode(ISD::MUL, DL, MVT::i64, Offset, Scale); in foldIndexIntoBase()
17684 BasePtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, Offset); in foldIndexIntoBase()
17685 Index = DAG.getNode(ISD::SHL, DL, Index.getValueType(), in foldIndexIntoBase()
17715 if (Index.getOpcode() == ISD::STEP_VECTOR) in findMoreOptimalIndexType()
17720 else if (Index.getOpcode() == ISD::SHL && in findMoreOptimalIndexType()
17721 Index.getOperand(0).getOpcode() == ISD::STEP_VECTOR) { in findMoreOptimalIndexType()
17751 Index = DAG.getNode(ISD::STEP_VECTOR, SDLoc(N), NewIndexVT, in findMoreOptimalIndexType()
17770 ISD::MemIndexType IndexType = MGS->getIndexType(); in performMaskedGatherScatterCombine()
17807 if (User->getOpcode() != ISD::ADD || in performNEONPostLDSTCombine()
17933 bool checkValueWidth(SDValue V, unsigned width, ISD::LoadExtType &ExtType) { in checkValueWidth()
17934 ExtType = ISD::NON_EXTLOAD; in checkValueWidth()
17938 case ISD::LOAD: { in checkValueWidth()
17947 case ISD::AssertSext: { in checkValueWidth()
17951 ExtType = ISD::SEXTLOAD; in checkValueWidth()
17956 case ISD::AssertZext: { in checkValueWidth()
17960 ExtType = ISD::ZEXTLOAD; in checkValueWidth()
17965 case ISD::Constant: in checkValueWidth()
17966 case ISD::TargetConstant: { in checkValueWidth()
18037 ISD::LoadExtType ExtType, int AddConstant, in isEquivalentMaskless()
18049 if (ExtType == ISD::SEXTLOAD) in isEquivalentMaskless()
18129 if (AndNode->getOpcode() != ISD::AND) in performCONDCombine()
18145 if (AddValue.getOpcode() != ISD::ADD) in performCONDCombine()
18161 ISD::LoadExtType ExtType; in performCONDCombine()
18231 if (LHS.getOpcode() == ISD::SHL || LHS.getOpcode() == ISD::SRA || in performBRCONDCombine()
18232 LHS.getOpcode() == ISD::SRL) in performBRCONDCombine()
18262 if ((CTTZ.getOpcode() != ISD::CTTZ && CTTZ.getOpcode() != ISD::TRUNCATE) || in foldCSELofCTTZ()
18263 (CTTZ.getOpcode() == ISD::TRUNCATE && in foldCSELofCTTZ()
18264 CTTZ.getOperand(0).getOpcode() != ISD::CTTZ)) in foldCSELofCTTZ()
18273 SDValue X = CTTZ.getOpcode() == ISD::TRUNCATE in foldCSELofCTTZ()
18280 unsigned BitWidth = CTTZ.getOpcode() == ISD::TRUNCATE in foldCSELofCTTZ()
18285 return DAG.getNode(ISD::AND, SDLoc(N), CTTZ.getValueType(), CTTZ, in foldCSELofCTTZ()
18316 if (FirstUse->getOpcode() != ISD::VSELECT) in tryToWidenSetCCOperands()
18322 return N->getOpcode() != ISD::VSELECT || N->getValueType(0) != UseMVT; in tryToWidenSetCCOperands()
18327 if (!ISD::isConstantSplatVector(Op->getOperand(1).getNode(), V)) in tryToWidenSetCCOperands()
18333 ISD::CondCode CC = cast<CondCodeSDNode>(Op->getOperand(2))->get(); in tryToWidenSetCCOperands()
18336 SDNode *Op0SExt = DAG.getNodeIfExists(ISD::SIGN_EXTEND, DAG.getVTList(UseMVT), in tryToWidenSetCCOperands()
18338 SDNode *Op0ZExt = DAG.getNodeIfExists(ISD::ZERO_EXTEND, DAG.getVTList(UseMVT), in tryToWidenSetCCOperands()
18342 Op1ExtV = DAG.getNode(ISD::SIGN_EXTEND, DL, UseMVT, Op->getOperand(1)); in tryToWidenSetCCOperands()
18345 Op1ExtV = DAG.getNode(ISD::ZERO_EXTEND, DL, UseMVT, Op->getOperand(1)); in tryToWidenSetCCOperands()
18349 return DAG.getNode(ISD::SETCC, DL, UseMVT.changeVectorElementType(MVT::i1), in tryToWidenSetCCOperands()
18356 assert(N->getOpcode() == ISD::SETCC && "Unexpected opcode!"); in performSETCCCombine()
18359 ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(2))->get(); in performSETCCCombine()
18367 if (Cond == ISD::SETNE && isOneConstant(RHS) && in performSETCCCombine()
18385 if (Cond == ISD::SETNE && isNullConstant(RHS) && in performSETCCCombine()
18386 LHS->getOpcode() == ISD::SRL && isa<ConstantSDNode>(LHS->getOperand(1)) && in performSETCCCombine()
18392 SDValue TST = DAG.getNode(ISD::AND, DL, TstVT, LHS->getOperand(0), in performSETCCCombine()
18394 return DAG.getNode(ISD::SETCC, DL, VT, TST, RHS, N->getOperand(2)); in performSETCCCombine()
18401 (Cond == ISD::SETEQ || Cond == ISD::SETNE) && isNullConstant(RHS) && in performSETCCCombine()
18402 LHS->getOpcode() == ISD::BITCAST) { in performSETCCCombine()
18407 LHS = DAG.getNode(ISD::VECREDUCE_OR, DL, MVT::i1, LHS->getOperand(0)); in performSETCCCombine()
18408 LHS = DAG.getNode(ISD::ZERO_EXTEND, DL, ToVT, LHS); in performSETCCCombine()
18448 ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(3))->get(); in performSetCCPunpkCombine()
18450 if (Cond != ISD::SETNE || !isZerosVector(RHS.getNode()) || in performSetCCPunpkCombine()
18451 LHS->getOpcode() != ISD::SIGN_EXTEND) in performSetCCPunpkCombine()
18455 if (Extract->getOpcode() != ISD::EXTRACT_SUBVECTOR || in performSetCCPunpkCombine()
18488 ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(3))->get(); in performSetccMergeZeroCombine()
18493 if (Cond == ISD::SETNE && isZerosVector(RHS.getNode()) && in performSetccMergeZeroCombine()
18494 LHS->getOpcode() == ISD::SIGN_EXTEND && in performSetccMergeZeroCombine()
18515 return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), in performSetccMergeZeroCombine()
18538 if (Op->getOpcode() == ISD::TRUNCATE && in getTestBitOperand()
18544 if (Op->getOpcode() == ISD::ANY_EXTEND && in getTestBitOperand()
18561 case ISD::AND: in getTestBitOperand()
18567 case ISD::SHL: in getTestBitOperand()
18576 case ISD::SRA: in getTestBitOperand()
18583 case ISD::SRL: in getTestBitOperand()
18591 case ISD::XOR: in getTestBitOperand()
18638 if (SetCC.getOpcode() != ISD::SETCC || !SetCC.hasOneUse()) in trySwapVSelectOperands()
18644 case ISD::FMUL: in trySwapVSelectOperands()
18645 case ISD::FSUB: in trySwapVSelectOperands()
18646 case ISD::FADD: in trySwapVSelectOperands()
18652 ISD::CondCode CC = cast<CondCodeSDNode>(SetCC.getOperand(2))->get(); in trySwapVSelectOperands()
18653 ISD::CondCode InverseCC = in trySwapVSelectOperands()
18654 ISD::getSetCCInverse(CC, SetCC.getOperand(0).getValueType()); in trySwapVSelectOperands()
18659 return DAG.getNode(ISD::VSELECT, SDLoc(N), NTy, in trySwapVSelectOperands()
18685 if (SetCC.getOpcode() == ISD::SETCC && in performVSelectCombine()
18686 SetCC.getOperand(2) == DAG.getCondCode(ISD::SETGT)) { in performVSelectCombine()
18699 ISD::isConstantSplatVector(SplatLHS, SplatLHSVal) && in performVSelectCombine()
18700 SplatLHSVal.isOne() && ISD::isConstantSplatVectorAllOnes(CmpRHS) && in performVSelectCombine()
18701 ISD::isConstantSplatVectorAllOnes(SplatRHS)) { in performVSelectCombine()
18708 auto Shift = DAG.getNode(ISD::SRA, SDLoc(N), VT, CmpLHS, Val); in performVSelectCombine()
18709 auto Or = DAG.getNode(ISD::OR, SDLoc(N), VT, Shift, N->getOperand(1)); in performVSelectCombine()
18714 if (N0.getOpcode() != ISD::SETCC || in performVSelectCombine()
18731 return DAG.getNode(ISD::VSELECT, SDLoc(N), ResVT, SetCC, in performVSelectCombine()
18745 if (N0.getOpcode() != ISD::SETCC) in performSelectCombine()
18788 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, SrcVT, N0.getOperand(0)); in performSelectCombine()
18790 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, SrcVT, N0.getOperand(1)); in performSelectCombine()
18791 SDValue SetCC = DAG.getNode(ISD::SETCC, DL, CCVT, LHS, RHS, N0.getOperand(2)); in performSelectCombine()
18796 Mask = DAG.getNode(ISD::BITCAST, DL, in performSelectCombine()
18812 return DCI.DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SDValue(LN, 0), in performDUPCombine()
18841 if (N->getOpcode() != ISD::ADD) in performGlobalAddressCombine()
18878 return DAG.getNode(ISD::SUB, DL, MVT::i64, Result, in performGlobalAddressCombine()
18890 SDValue SplatShift = DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::nxv2i64, Shift); in getScaledOffsetForBitWidth()
18892 return DAG.getNode(ISD::SHL, DL, MVT::nxv2i64, Offset, SplatShift); in getScaledOffsetForBitWidth()
19000 Offset = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::nxv2i64, Offset).getValue(0); in performScatterStoreCombine()
19019 SrcNew = DAG.getNode(ISD::BITCAST, DL, HwSrcVt, Src); in performScatterStoreCombine()
19021 SrcNew = DAG.getNode(ISD::ANY_EXTEND, DL, HwSrcVt, Src); in performScatterStoreCombine()
19103 Offset = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::nxv2i64, Offset).getValue(0); in performGatherLoadCombine()
19124 Load = DAG.getNode(ISD::TRUNCATE, DL, RetVT, Load.getValue(0)); in performGatherLoadCombine()
19129 Load = DAG.getNode(ISD::BITCAST, DL, RetVT, Load.getValue(0)); in performGatherLoadCombine()
19165 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ExtOp.getValueType(), in performSignExtendInRegCombine()
19277 Offset = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::nxv2i64, Offset); in legalizeSVEGatherPrefetchOffsVec()
19335 assert(N->getOpcode() == ISD::INSERT_VECTOR_ELT && "Unexpected node!"); in removeRedundantInsertVectorElt()
19344 if (!ISD::isConstantSplatVectorAllZeros(InsertVec.getNode())) in removeRedundantInsertVectorElt()
19347 if (InsertElt.getOpcode() != ISD::EXTRACT_VECTOR_ELT) in removeRedundantInsertVectorElt()
19395 SDValue Splice = DAG.getNode(ISD::VECTOR_SPLICE, DL, ExtIntTy, LHS, RHS, Idx); in performSVESpliceCombine()
19407 if (N->hasOneUse() && N->use_begin()->getOpcode() == ISD::FP_ROUND) in performFPExtendCombine()
19413 if (DCI.isBeforeLegalizeOps() && ISD::isNormalLoad(N0.getNode()) && in performFPExtendCombine()
19418 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, in performFPExtendCombine()
19423 DAG.getNode(ISD::FP_ROUND, SDLoc(N0), N0.getValueType(), in performFPExtendCombine()
19452 SDValue Sel = DAG.getNode(ISD::AND, DL, VT, Mask, In1); in performBSPExpandForSVE()
19453 SDValue SelInv = DAG.getNode(ISD::AND, DL, VT, InvMask, In2); in performBSPExpandForSVE()
19454 return DAG.getNode(ISD::OR, DL, VT, Sel, SelInv); in performBSPExpandForSVE()
19461 if (Insert.getOpcode() != ISD::INSERT_SUBVECTOR) in performDupLane128Combine()
19473 if (Bitcast.getOpcode() != ISD::BITCAST) in performDupLane128Combine()
19485 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, NewSubvecVT, in performDupLane128Combine()
19489 return DAG.getNode(ISD::BITCAST, DL, VT, NewDuplane128); in performDupLane128Combine()
19499 case ISD::ADD: in PerformDAGCombine()
19500 case ISD::SUB: in PerformDAGCombine()
19502 case ISD::BUILD_VECTOR: in PerformDAGCombine()
19505 return performFlagSettingCombine(N, DCI, ISD::AND); in PerformDAGCombine()
19520 case ISD::XOR: in PerformDAGCombine()
19522 case ISD::MUL: in PerformDAGCombine()
19524 case ISD::SINT_TO_FP: in PerformDAGCombine()
19525 case ISD::UINT_TO_FP: in PerformDAGCombine()
19527 case ISD::FP_TO_SINT: in PerformDAGCombine()
19528 case ISD::FP_TO_UINT: in PerformDAGCombine()
19529 case ISD::FP_TO_SINT_SAT: in PerformDAGCombine()
19530 case ISD::FP_TO_UINT_SAT: in PerformDAGCombine()
19532 case ISD::FDIV: in PerformDAGCombine()
19534 case ISD::OR: in PerformDAGCombine()
19536 case ISD::AND: in PerformDAGCombine()
19538 case ISD::INTRINSIC_WO_CHAIN: in PerformDAGCombine()
19540 case ISD::ANY_EXTEND: in PerformDAGCombine()
19541 case ISD::ZERO_EXTEND: in PerformDAGCombine()
19542 case ISD::SIGN_EXTEND: in PerformDAGCombine()
19544 case ISD::SIGN_EXTEND_INREG: in PerformDAGCombine()
19546 case ISD::CONCAT_VECTORS: in PerformDAGCombine()
19548 case ISD::EXTRACT_SUBVECTOR: in PerformDAGCombine()
19550 case ISD::INSERT_SUBVECTOR: in PerformDAGCombine()
19552 case ISD::SELECT: in PerformDAGCombine()
19554 case ISD::VSELECT: in PerformDAGCombine()
19556 case ISD::SETCC: in PerformDAGCombine()
19558 case ISD::LOAD: in PerformDAGCombine()
19562 case ISD::STORE: in PerformDAGCombine()
19564 case ISD::MSTORE: in PerformDAGCombine()
19566 case ISD::MGATHER: in PerformDAGCombine()
19567 case ISD::MSCATTER: in PerformDAGCombine()
19569 case ISD::VECTOR_SPLICE: in PerformDAGCombine()
19571 case ISD::FP_EXTEND: in PerformDAGCombine()
19618 case ISD::INSERT_VECTOR_ELT: in PerformDAGCombine()
19620 case ISD::EXTRACT_VECTOR_ELT: in PerformDAGCombine()
19622 case ISD::VECREDUCE_ADD: in PerformDAGCombine()
19629 case ISD::INTRINSIC_VOID: in PerformDAGCombine()
19630 case ISD::INTRINSIC_W_CHAIN: in PerformDAGCombine()
19780 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResVT, Src1, ExtIdx); in PerformDAGCombine()
19808 Opnds.push_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, in PerformDAGCombine()
19813 DAG.getNode(ISD::CONCAT_VECTORS, DL, Tuple.getValueType(), Opnds); in PerformDAGCombine()
19831 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, DestVT, Opnds); in PerformDAGCombine()
19870 case ISD::GlobalAddress: in PerformDAGCombine()
19889 if (Copy->getOpcode() == ISD::CopyToReg) { in isUsedByReturnOnly()
19896 } else if (Copy->getOpcode() != ISD::FP_EXTEND) in isUsedByReturnOnly()
19923 ISD::MemIndexedMode &AM, in getIndexedAddressParts()
19926 if (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB) in getIndexedAddressParts()
19934 if (Op->getOpcode() == ISD::SUB) in getIndexedAddressParts()
19938 IsInc = (Op->getOpcode() == ISD::ADD); in getIndexedAddressParts()
19947 ISD::MemIndexedMode &AM, in getPreIndexedAddressParts()
19963 AM = IsInc ? ISD::PRE_INC : ISD::PRE_DEC; in getPreIndexedAddressParts()
19969 ISD::MemIndexedMode &AM, SelectionDAG &DAG) const { in getPostIndexedAddressParts()
19988 AM = IsInc ? ISD::POST_INC : ISD::POST_DEC; in getPostIndexedAddressParts()
20012 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, CastResult)); in ReplaceBITCASTResults()
20024 Op = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Op); in ReplaceBITCASTResults()
20025 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Op)); in ReplaceBITCASTResults()
20070 DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Addp, in ReplaceAddWithADDP()
20091 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, N); in splitInt128()
20092 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, in splitInt128()
20093 DAG.getNode(ISD::SRL, DL, MVT::i128, N, in splitInt128()
20129 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Half)); in ReplaceExtractSubVectorResults()
20137 DAG.getNode(ISD::SRL, dl, MVT::i128, V, DAG.getConstant(64, dl, MVT::i64)), in createGPRPairNode()
20199 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i128, Lo, Hi)); in ReplaceCMP_SWAP_128Results()
20232 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i128, in ReplaceCMP_SWAP_128Results()
20242 case ISD::BITCAST: in ReplaceNodeResults()
20245 case ISD::VECREDUCE_ADD: in ReplaceNodeResults()
20246 case ISD::VECREDUCE_SMAX: in ReplaceNodeResults()
20247 case ISD::VECREDUCE_SMIN: in ReplaceNodeResults()
20248 case ISD::VECREDUCE_UMAX: in ReplaceNodeResults()
20249 case ISD::VECREDUCE_UMIN: in ReplaceNodeResults()
20252 case ISD::ADD: in ReplaceNodeResults()
20253 case ISD::FADD: in ReplaceNodeResults()
20257 case ISD::CTPOP: in ReplaceNodeResults()
20258 case ISD::PARITY: in ReplaceNodeResults()
20263 ReplaceReductionResults(N, Results, DAG, ISD::ADD, AArch64ISD::SADDV); in ReplaceNodeResults()
20266 ReplaceReductionResults(N, Results, DAG, ISD::ADD, AArch64ISD::UADDV); in ReplaceNodeResults()
20269 ReplaceReductionResults(N, Results, DAG, ISD::SMIN, AArch64ISD::SMINV); in ReplaceNodeResults()
20272 ReplaceReductionResults(N, Results, DAG, ISD::UMIN, AArch64ISD::UMINV); in ReplaceNodeResults()
20275 ReplaceReductionResults(N, Results, DAG, ISD::SMAX, AArch64ISD::SMAXV); in ReplaceNodeResults()
20278 ReplaceReductionResults(N, Results, DAG, ISD::UMAX, AArch64ISD::UMAXV); in ReplaceNodeResults()
20280 case ISD::FP_TO_UINT: in ReplaceNodeResults()
20281 case ISD::FP_TO_SINT: in ReplaceNodeResults()
20282 case ISD::STRICT_FP_TO_SINT: in ReplaceNodeResults()
20283 case ISD::STRICT_FP_TO_UINT: in ReplaceNodeResults()
20287 case ISD::ATOMIC_CMP_SWAP: in ReplaceNodeResults()
20290 case ISD::ATOMIC_LOAD: in ReplaceNodeResults()
20291 case ISD::LOAD: { in ReplaceNodeResults()
20308 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i128, in ReplaceNodeResults()
20313 case ISD::EXTRACT_SUBVECTOR: in ReplaceNodeResults()
20316 case ISD::INSERT_SUBVECTOR: in ReplaceNodeResults()
20317 case ISD::CONCAT_VECTORS: in ReplaceNodeResults()
20322 case ISD::INTRINSIC_WO_CHAIN: { in ReplaceNodeResults()
20334 auto Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, N->getOperand(2)); in ReplaceNodeResults()
20337 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V)); in ReplaceNodeResults()
20342 auto Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, N->getOperand(2)); in ReplaceNodeResults()
20345 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V)); in ReplaceNodeResults()
20352 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V)); in ReplaceNodeResults()
20359 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, V)); in ReplaceNodeResults()
20710 return X.getValueType().isScalarInteger() || NewShiftOpcode == ISD::SHL; in shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd()
20966 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero); in convertToScalableVector()
20977 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero); in convertFromScalableVector()
20993 if (VT.isFloatingPoint() && Load->getExtensionType() == ISD::EXTLOAD) { in LowerFixedLengthVectorLoadToSVE()
21004 if (VT.isFloatingPoint() && Load->getExtensionType() == ISD::EXTLOAD) { in LowerFixedLengthVectorLoadToSVE()
21026 if (ISD::isBuildVectorAllOnes(Mask.getNode())) in convertFixedMaskToScalableVector()
21033 {Pg, Op1, Op2, DAG.getCondCode(ISD::SETNE)}); in convertFixedMaskToScalableVector()
21132 bool Signed = Op.getOpcode() == ISD::SDIV; in LowerFixedLengthVectorIntDivideToSVE()
21145 Res = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), Res); in LowerFixedLengthVectorIntDivideToSVE()
21163 unsigned ExtendOpcode = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in LowerFixedLengthVectorIntDivideToSVE()
21167 return DAG.getNode(ISD::TRUNCATE, dl, VT, Div); in LowerFixedLengthVectorIntDivideToSVE()
21211 bool Signed = Op.getOpcode() == ISD::SIGN_EXTEND; in LowerFixedLengthVectorIntExtendToSVE()
21252 Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv4i32, Val); in LowerFixedLengthVectorTruncateToSVE()
21258 Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv8i16, Val); in LowerFixedLengthVectorTruncateToSVE()
21264 Val = DAG.getNode(ISD::BITCAST, DL, MVT::nxv16i8, Val); in LowerFixedLengthVectorTruncateToSVE()
21283 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Op.getOperand(1)); in LowerFixedLengthExtractVectorElt()
21296 auto ScalableRes = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ContainerVT, Op0, in LowerFixedLengthInsertVectorElt()
21408 AccOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ContainerVT, in LowerVECREDUCE_SEQ_FADD()
21415 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Rdx, Zero); in LowerVECREDUCE_SEQ_FADD()
21433 case ISD::VECREDUCE_OR: in LowerPredReductionToSVE()
21440 case ISD::VECREDUCE_AND: { in LowerPredReductionToSVE()
21441 Op = DAG.getNode(ISD::XOR, DL, OpVT, Op, Pg); in LowerPredReductionToSVE()
21444 case ISD::VECREDUCE_XOR: { in LowerPredReductionToSVE()
21453 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64, ID, Pg, Op); in LowerPredReductionToSVE()
21484 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, in LowerReductionToSVE()
21510 Mask = DAG.getNode(ISD::TRUNCATE, DL, in LowerFixedLengthVectorSelectToSVE()
21513 auto ScalableRes = DAG.getNode(ISD::VSELECT, DL, ContainerVT, in LowerFixedLengthVectorSelectToSVE()
21554 Op = DAG.getNode(ISD::BITCAST, DL, ContainerDstVT, SrcOp); in LowerFixedLengthBitcastToSVE()
21575 Ops.push_back(DAG.getNode(ISD::CONCAT_VECTORS, DL, PairVT, in LowerFixedLengthConcatVectorsToSVE()
21578 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Ops); in LowerFixedLengthConcatVectorsToSVE()
21606 Val = DAG.getNode(ISD::BITCAST, DL, SrcVT.changeTypeToInteger(), Val); in LowerFixedLengthFPExtendToSVE()
21607 Val = DAG.getNode(ISD::ANY_EXTEND, DL, VT.changeTypeToInteger(), Val); in LowerFixedLengthFPExtendToSVE()
21637 Val = DAG.getNode(ISD::TRUNCATE, DL, VT.changeTypeToInteger(), Val); in LowerFixedLengthFPRoundToSVE()
21638 return DAG.getNode(ISD::BITCAST, DL, VT, Val); in LowerFixedLengthFPRoundToSVE()
21647 bool IsSigned = Op.getOpcode() == ISD::SINT_TO_FP; in LowerFixedLengthIntToFPToSVE()
21661 Val = DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL, in LowerFixedLengthIntToFPToSVE()
21681 Val = DAG.getNode(ISD::TRUNCATE, DL, VT.changeTypeToInteger(), Val); in LowerFixedLengthIntToFPToSVE()
21682 return DAG.getNode(ISD::BITCAST, DL, VT, Val); in LowerFixedLengthIntToFPToSVE()
21692 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT; in LowerFixedLengthFPToIntToSVE()
21708 Val = DAG.getNode(ISD::BITCAST, DL, SrcVT.changeTypeToInteger(), Val); in LowerFixedLengthFPToIntToSVE()
21709 Val = DAG.getNode(ISD::ANY_EXTEND, DL, VT, Val); in LowerFixedLengthFPToIntToSVE()
21726 return DAG.getNode(ISD::TRUNCATE, DL, VT, Val); in LowerFixedLengthFPToIntToSVE()
21757 ISD::EXTRACT_VECTOR_ELT, DL, ScalarTy, Op1, in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21776 Op = DAG.getNode(ISD::BITCAST, DL, NewVT, Op1); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21778 Op = DAG.getNode(ISD::BITCAST, DL, ContainerVT, Op); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21826 Op = DAG.getNode(ISD::VECTOR_REVERSE, DL, ContainerVT, Op1); in LowerFixedLengthVECTOR_SHUFFLEToSVE()
21887 Op = DAG.getNode(ISD::BITCAST, DL, PackedVT, Op); in getSVESafeBitCast()
21951 (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR && in isTargetCanonicalConstantNode()