Lines Matching refs:ISD
234 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, XLenVT, in RISCVTargetLowering()
237 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::i32, in RISCVTargetLowering()
241 setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Expand); in RISCVTargetLowering()
243 setOperationAction(ISD::BR_JT, MVT::Other, Expand); in RISCVTargetLowering()
244 setOperationAction(ISD::BR_CC, XLenVT, Expand); in RISCVTargetLowering()
246 setOperationAction(ISD::BR_CC, MVT::i32, Expand); in RISCVTargetLowering()
247 setOperationAction(ISD::BRCOND, MVT::Other, Custom); in RISCVTargetLowering()
248 setOperationAction(ISD::SELECT_CC, XLenVT, Expand); in RISCVTargetLowering()
250 setOperationAction(ISD::SELECT_CC, MVT::i32, Expand); in RISCVTargetLowering()
252 setCondCodeAction(ISD::SETLE, XLenVT, Expand); in RISCVTargetLowering()
253 setCondCodeAction(ISD::SETGT, XLenVT, Custom); in RISCVTargetLowering()
254 setCondCodeAction(ISD::SETGE, XLenVT, Expand); in RISCVTargetLowering()
255 setCondCodeAction(ISD::SETULE, XLenVT, Expand); in RISCVTargetLowering()
256 setCondCodeAction(ISD::SETUGT, XLenVT, Custom); in RISCVTargetLowering()
257 setCondCodeAction(ISD::SETUGE, XLenVT, Expand); in RISCVTargetLowering()
260 setOperationAction(ISD::SETCC, MVT::i32, Promote); in RISCVTargetLowering()
262 setOperationAction({ISD::STACKSAVE, ISD::STACKRESTORE}, MVT::Other, Expand); in RISCVTargetLowering()
264 setOperationAction(ISD::VASTART, MVT::Other, Custom); in RISCVTargetLowering()
265 setOperationAction({ISD::VAARG, ISD::VACOPY, ISD::VAEND}, MVT::Other, Expand); in RISCVTargetLowering()
267 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); in RISCVTargetLowering()
269 setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom); in RISCVTargetLowering()
272 setOperationAction(ISD::SIGN_EXTEND_INREG, {MVT::i8, MVT::i16}, Expand); in RISCVTargetLowering()
275 setOperationAction(ISD::EH_DWARF_CFA, MVT::i64, Custom); in RISCVTargetLowering()
278 setOperationAction(ISD::LOAD, MVT::i32, Custom); in RISCVTargetLowering()
279 setOperationAction({ISD::ADD, ISD::SUB, ISD::SHL, ISD::SRA, ISD::SRL}, in RISCVTargetLowering()
281 setOperationAction(ISD::SADDO, MVT::i32, Custom); in RISCVTargetLowering()
282 setOperationAction({ISD::UADDO, ISD::USUBO, ISD::UADDSAT, ISD::USUBSAT}, in RISCVTargetLowering()
293 setOperationAction({ISD::MUL, ISD::MULHS, ISD::MULHU}, XLenVT, Expand); in RISCVTargetLowering()
295 setOperationAction(ISD::MUL, MVT::i32, Promote); in RISCVTargetLowering()
297 setOperationAction(ISD::MUL, MVT::i128, Custom); in RISCVTargetLowering()
299 setOperationAction(ISD::MUL, MVT::i32, Custom); in RISCVTargetLowering()
301 setOperationAction(ISD::MUL, MVT::i64, Custom); in RISCVTargetLowering()
305 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, in RISCVTargetLowering()
308 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, MVT::i32, in RISCVTargetLowering()
312 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::UREM}, in RISCVTargetLowering()
317 setOperationAction({ISD::MULHS, ISD::MULHU}, MVT::i32, Expand); in RISCVTargetLowering()
319 {ISD::SDIVREM, ISD::UDIVREM, ISD::SMUL_LOHI, ISD::UMUL_LOHI}, MVT::i32, in RISCVTargetLowering()
324 {ISD::SDIVREM, ISD::UDIVREM, ISD::SMUL_LOHI, ISD::UMUL_LOHI}, XLenVT, in RISCVTargetLowering()
327 setOperationAction({ISD::SHL_PARTS, ISD::SRL_PARTS, ISD::SRA_PARTS}, XLenVT, in RISCVTargetLowering()
332 setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom); in RISCVTargetLowering()
335 setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom); in RISCVTargetLowering()
336 setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Custom); in RISCVTargetLowering()
338 setOperationAction(ISD::ROTL, XLenVT, Expand); in RISCVTargetLowering()
340 setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Expand); in RISCVTargetLowering()
342 setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Expand); in RISCVTargetLowering()
347 setOperationAction(ISD::BSWAP, XLenVT, in RISCVTargetLowering()
353 setOperationAction(ISD::BSWAP, MVT::i32, in RISCVTargetLowering()
361 setOperationAction(ISD::BITREVERSE, XLenVT, Legal); in RISCVTargetLowering()
364 setOperationAction(ISD::BITREVERSE, XLenVT, in RISCVTargetLowering()
369 setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, XLenVT, in RISCVTargetLowering()
372 setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, MVT::i32, in RISCVTargetLowering()
377 setOperationAction(ISD::CTTZ, MVT::i32, Legal); in RISCVTargetLowering()
379 setOperationAction({ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF}, MVT::i32, Custom); in RISCVTargetLowering()
382 setOperationAction({ISD::CTTZ, ISD::CTPOP}, XLenVT, Expand); in RISCVTargetLowering()
384 setOperationAction({ISD::CTTZ, ISD::CTPOP}, MVT::i32, Expand); in RISCVTargetLowering()
393 setOperationAction(ISD::CTLZ, MVT::i32, in RISCVTargetLowering()
396 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Promote); in RISCVTargetLowering()
398 setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, MVT::i32, Custom); in RISCVTargetLowering()
401 setOperationAction(ISD::CTLZ, XLenVT, Expand); in RISCVTargetLowering()
403 setOperationAction(ISD::CTLZ, MVT::i32, Expand); in RISCVTargetLowering()
408 setOperationAction(ISD::ABS, MVT::i32, Custom); in RISCVTargetLowering()
412 setOperationAction(ISD::ABS, XLenVT, Legal); in RISCVTargetLowering()
415 setOperationAction(ISD::SELECT, XLenVT, Custom); in RISCVTargetLowering()
418 setOperationAction(ISD::SELECT, MVT::i32, Promote); in RISCVTargetLowering()
421 ISD::FMINNUM, ISD::FMAXNUM, ISD::LRINT, in RISCVTargetLowering()
422 ISD::LLRINT, ISD::LROUND, ISD::LLROUND, in RISCVTargetLowering()
423 ISD::STRICT_LRINT, ISD::STRICT_LLRINT, ISD::STRICT_LROUND, in RISCVTargetLowering()
424 ISD::STRICT_LLROUND, ISD::STRICT_FMA, ISD::STRICT_FADD, in RISCVTargetLowering()
425 ISD::STRICT_FSUB, ISD::STRICT_FMUL, ISD::STRICT_FDIV, in RISCVTargetLowering()
426 ISD::STRICT_FSQRT, ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS}; in RISCVTargetLowering()
428 static const ISD::CondCode FPCCToExpand[] = { in RISCVTargetLowering()
429 ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT, in RISCVTargetLowering()
430 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT, in RISCVTargetLowering()
431 ISD::SETGE, ISD::SETNE, ISD::SETO, ISD::SETUO}; in RISCVTargetLowering()
434 ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, in RISCVTargetLowering()
435 ISD::FREM}; in RISCVTargetLowering()
438 ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FRINT, ISD::FROUND, in RISCVTargetLowering()
439 ISD::FROUNDEVEN}; in RISCVTargetLowering()
442 setOperationAction(ISD::BITCAST, MVT::i16, Custom); in RISCVTargetLowering()
445 ISD::FMINNUM, ISD::FMAXNUM, ISD::FADD, in RISCVTargetLowering()
446 ISD::FSUB, ISD::FMUL, ISD::FMA, in RISCVTargetLowering()
447 ISD::FDIV, ISD::FSQRT, ISD::FABS, in RISCVTargetLowering()
448 ISD::FNEG, ISD::STRICT_FMA, ISD::STRICT_FADD, in RISCVTargetLowering()
449 ISD::STRICT_FSUB, ISD::STRICT_FMUL, ISD::STRICT_FDIV, in RISCVTargetLowering()
450 ISD::STRICT_FSQRT, ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, in RISCVTargetLowering()
451 ISD::SETCC, ISD::FCEIL, ISD::FFLOOR, in RISCVTargetLowering()
452 ISD::FTRUNC, ISD::FRINT, ISD::FROUND, in RISCVTargetLowering()
453 ISD::FROUNDEVEN, ISD::SELECT}; in RISCVTargetLowering()
456 setOperationAction(ISD::BITCAST, MVT::i16, Custom); in RISCVTargetLowering()
457 setOperationAction(ISD::BITCAST, MVT::bf16, Custom); in RISCVTargetLowering()
458 setOperationAction(ISD::FP_ROUND, MVT::bf16, Custom); in RISCVTargetLowering()
459 setOperationAction(ISD::FP_EXTEND, MVT::f32, Custom); in RISCVTargetLowering()
460 setOperationAction(ISD::FP_EXTEND, MVT::f64, Custom); in RISCVTargetLowering()
461 setOperationAction(ISD::ConstantFP, MVT::bf16, Expand); in RISCVTargetLowering()
462 setOperationAction(ISD::SELECT_CC, MVT::bf16, Expand); in RISCVTargetLowering()
463 setOperationAction(ISD::BR_CC, MVT::bf16, Expand); in RISCVTargetLowering()
465 setOperationAction(ISD::FREM, MVT::bf16, Promote); in RISCVTargetLowering()
468 setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Expand); in RISCVTargetLowering()
476 setOperationAction(ISD::SELECT, MVT::f16, Custom); in RISCVTargetLowering()
477 setOperationAction(ISD::IS_FPCLASS, MVT::f16, Custom); in RISCVTargetLowering()
480 setOperationAction({ISD::STRICT_LRINT, ISD::STRICT_LLRINT, in RISCVTargetLowering()
481 ISD::STRICT_LROUND, ISD::STRICT_LLROUND}, in RISCVTargetLowering()
485 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Expand); in RISCVTargetLowering()
488 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Legal); in RISCVTargetLowering()
489 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal); in RISCVTargetLowering()
491 setOperationAction(ISD::SELECT_CC, MVT::f16, Expand); in RISCVTargetLowering()
492 setOperationAction(ISD::BR_CC, MVT::f16, Expand); in RISCVTargetLowering()
494 setOperationAction(ISD::FNEARBYINT, MVT::f16, in RISCVTargetLowering()
496 setOperationAction({ISD::FREM, ISD::FPOW, ISD::FPOWI, in RISCVTargetLowering()
497 ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP, in RISCVTargetLowering()
498 ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2, in RISCVTargetLowering()
499 ISD::FLOG10}, in RISCVTargetLowering()
504 setOperationAction({ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, in RISCVTargetLowering()
505 ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT, in RISCVTargetLowering()
506 ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN, in RISCVTargetLowering()
507 ISD::STRICT_FTRUNC}, in RISCVTargetLowering()
512 setOperationAction(ISD::FPOWI, MVT::i32, Custom); in RISCVTargetLowering()
515 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f16, Custom); in RISCVTargetLowering()
523 setOperationAction(ISD::SELECT_CC, MVT::f32, Expand); in RISCVTargetLowering()
524 setOperationAction(ISD::SELECT, MVT::f32, Custom); in RISCVTargetLowering()
525 setOperationAction(ISD::BR_CC, MVT::f32, Expand); in RISCVTargetLowering()
527 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand); in RISCVTargetLowering()
529 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::bf16, Expand); in RISCVTargetLowering()
531 setOperationAction(ISD::IS_FPCLASS, MVT::f32, Custom); in RISCVTargetLowering()
532 setOperationAction(ISD::BF16_TO_FP, MVT::f32, Custom); in RISCVTargetLowering()
533 setOperationAction(ISD::FP_TO_BF16, MVT::f32, in RISCVTargetLowering()
535 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Custom); in RISCVTargetLowering()
536 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Custom); in RISCVTargetLowering()
539 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal); in RISCVTargetLowering()
541 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f32, Custom); in RISCVTargetLowering()
545 setOperationAction(ISD::BITCAST, MVT::i32, Custom); in RISCVTargetLowering()
552 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal); in RISCVTargetLowering()
553 setOperationAction(ISD::BITCAST, MVT::i64, Custom); in RISCVTargetLowering()
554 setOperationAction(ISD::BITCAST, MVT::f64, Custom); in RISCVTargetLowering()
559 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f64, Custom); in RISCVTargetLowering()
562 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal); in RISCVTargetLowering()
563 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal); in RISCVTargetLowering()
565 setOperationAction(ISD::SELECT_CC, MVT::f64, Expand); in RISCVTargetLowering()
566 setOperationAction(ISD::SELECT, MVT::f64, Custom); in RISCVTargetLowering()
567 setOperationAction(ISD::BR_CC, MVT::f64, Expand); in RISCVTargetLowering()
568 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand); in RISCVTargetLowering()
571 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand); in RISCVTargetLowering()
573 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::bf16, Expand); in RISCVTargetLowering()
575 setOperationAction(ISD::IS_FPCLASS, MVT::f64, Custom); in RISCVTargetLowering()
576 setOperationAction(ISD::BF16_TO_FP, MVT::f64, Custom); in RISCVTargetLowering()
577 setOperationAction(ISD::FP_TO_BF16, MVT::f64, in RISCVTargetLowering()
579 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Custom); in RISCVTargetLowering()
580 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); in RISCVTargetLowering()
584 setOperationAction({ISD::FP_TO_UINT, ISD::FP_TO_SINT, in RISCVTargetLowering()
585 ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT}, in RISCVTargetLowering()
587 setOperationAction(ISD::LROUND, MVT::i32, Custom); in RISCVTargetLowering()
591 setOperationAction({ISD::FP_TO_UINT_SAT, ISD::FP_TO_SINT_SAT}, XLenVT, in RISCVTargetLowering()
594 setOperationAction({ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT, in RISCVTargetLowering()
595 ISD::STRICT_UINT_TO_FP, ISD::STRICT_SINT_TO_FP}, in RISCVTargetLowering()
599 setOperationAction({ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT, in RISCVTargetLowering()
600 ISD::STRICT_UINT_TO_FP, ISD::STRICT_SINT_TO_FP}, in RISCVTargetLowering()
603 setOperationAction(ISD::GET_ROUNDING, XLenVT, Custom); in RISCVTargetLowering()
604 setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom); in RISCVTargetLowering()
607 setOperationAction({ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool, in RISCVTargetLowering()
608 ISD::JumpTable}, in RISCVTargetLowering()
611 setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom); in RISCVTargetLowering()
614 setOperationAction(ISD::Constant, MVT::i64, Custom); in RISCVTargetLowering()
618 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, in RISCVTargetLowering()
621 setOperationAction({ISD::TRAP, ISD::DEBUGTRAP}, MVT::Other, Legal); in RISCVTargetLowering()
622 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); in RISCVTargetLowering()
624 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom); in RISCVTargetLowering()
627 setOperationAction(ISD::PREFETCH, MVT::Other, Legal); in RISCVTargetLowering()
639 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom); in RISCVTargetLowering()
646 setOperationAction(ISD::VSCALE, XLenVT, Custom); in RISCVTargetLowering()
648 setOperationAction(ISD::VSCALE, MVT::i32, Custom); in RISCVTargetLowering()
652 setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN, in RISCVTargetLowering()
653 ISD::INTRINSIC_VOID}, in RISCVTargetLowering()
656 setOperationAction({ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID}, in RISCVTargetLowering()
659 setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN}, in RISCVTargetLowering()
662 setOperationAction({ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID}, in RISCVTargetLowering()
666 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL, in RISCVTargetLowering()
667 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM, in RISCVTargetLowering()
668 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR, in RISCVTargetLowering()
669 ISD::VP_XOR, ISD::VP_ASHR, ISD::VP_LSHR, in RISCVTargetLowering()
670 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND, in RISCVTargetLowering()
671 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX, in RISCVTargetLowering()
672 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN, in RISCVTargetLowering()
673 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT, in RISCVTargetLowering()
674 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND, in RISCVTargetLowering()
675 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN, in RISCVTargetLowering()
676 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX, in RISCVTargetLowering()
677 ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE}; in RISCVTargetLowering()
680 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL, in RISCVTargetLowering()
681 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS, in RISCVTargetLowering()
682 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD, in RISCVTargetLowering()
683 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE, in RISCVTargetLowering()
684 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP, in RISCVTargetLowering()
685 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND, in RISCVTargetLowering()
686 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM, in RISCVTargetLowering()
687 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND, in RISCVTargetLowering()
688 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, in RISCVTargetLowering()
689 ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS, in RISCVTargetLowering()
690 ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::EXPERIMENTAL_VP_REVERSE, in RISCVTargetLowering()
691 ISD::EXPERIMENTAL_VP_SPLICE}; in RISCVTargetLowering()
694 ISD::VECREDUCE_ADD, ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, in RISCVTargetLowering()
695 ISD::VECREDUCE_XOR, ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN, in RISCVTargetLowering()
696 ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN}; in RISCVTargetLowering()
699 ISD::VECREDUCE_FADD, ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_FMIN, in RISCVTargetLowering()
700 ISD::VECREDUCE_FMAX}; in RISCVTargetLowering()
705 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, in RISCVTargetLowering()
710 setOperationAction({ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND, in RISCVTargetLowering()
711 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, in RISCVTargetLowering()
712 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN, in RISCVTargetLowering()
713 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN}, in RISCVTargetLowering()
721 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); in RISCVTargetLowering()
724 setOperationAction({ISD::TRUNCATE, ISD::CONCAT_VECTORS, in RISCVTargetLowering()
725 ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR, in RISCVTargetLowering()
726 ISD::SCALAR_TO_VECTOR}, in RISCVTargetLowering()
729 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT, in RISCVTargetLowering()
732 setOperationAction(ISD::SELECT, VT, Custom); in RISCVTargetLowering()
734 {ISD::SELECT_CC, ISD::VSELECT, ISD::VP_MERGE, ISD::VP_SELECT}, VT, in RISCVTargetLowering()
737 setOperationAction({ISD::VP_AND, ISD::VP_OR, ISD::VP_XOR}, VT, Custom); in RISCVTargetLowering()
740 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT, in RISCVTargetLowering()
744 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT, in RISCVTargetLowering()
751 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, in RISCVTargetLowering()
752 ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP, in RISCVTargetLowering()
753 ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT, in RISCVTargetLowering()
754 ISD::STRICT_FP_TO_UINT}, in RISCVTargetLowering()
756 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT, in RISCVTargetLowering()
763 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT, in RISCVTargetLowering()
767 setOperationAction({ISD::VP_FP_TO_SINT, ISD::VP_FP_TO_UINT, in RISCVTargetLowering()
768 ISD::VP_TRUNCATE, ISD::VP_SETCC}, in RISCVTargetLowering()
771 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom); in RISCVTargetLowering()
772 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom); in RISCVTargetLowering()
774 setOperationAction(ISD::VECTOR_REVERSE, VT, Custom); in RISCVTargetLowering()
776 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom); in RISCVTargetLowering()
777 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom); in RISCVTargetLowering()
780 ISD::VECTOR_SPLICE, VT, in RISCVTargetLowering()
788 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); in RISCVTargetLowering()
789 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom); in RISCVTargetLowering()
792 setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Expand); in RISCVTargetLowering()
796 setOperationAction({ISD::MULHU, ISD::MULHS}, VT, Expand); in RISCVTargetLowering()
798 setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT, in RISCVTargetLowering()
802 setOperationAction({ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND}, in RISCVTargetLowering()
809 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, in RISCVTargetLowering()
810 ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP, in RISCVTargetLowering()
811 ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT, in RISCVTargetLowering()
812 ISD::STRICT_FP_TO_UINT}, in RISCVTargetLowering()
814 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT, in RISCVTargetLowering()
816 setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom); in RISCVTargetLowering()
817 setOperationAction({ISD::AVGFLOORU, ISD::AVGCEILU, ISD::SADDSAT, in RISCVTargetLowering()
818 ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT}, in RISCVTargetLowering()
823 setOperationAction(ISD::TRUNCATE, VT, Custom); in RISCVTargetLowering()
826 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT, in RISCVTargetLowering()
835 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom); in RISCVTargetLowering()
837 setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, in RISCVTargetLowering()
841 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD, in RISCVTargetLowering()
842 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER}, in RISCVTargetLowering()
845 setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR, in RISCVTargetLowering()
846 ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR}, in RISCVTargetLowering()
849 setOperationAction(ISD::SELECT, VT, Custom); in RISCVTargetLowering()
850 setOperationAction(ISD::SELECT_CC, VT, Expand); in RISCVTargetLowering()
852 setOperationAction({ISD::STEP_VECTOR, ISD::VECTOR_REVERSE}, VT, Custom); in RISCVTargetLowering()
856 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT, in RISCVTargetLowering()
860 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom); in RISCVTargetLowering()
861 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom); in RISCVTargetLowering()
864 setOperationAction(ISD::VECTOR_SPLICE, VT, Custom); in RISCVTargetLowering()
867 setOperationAction(ISD::BSWAP, VT, Legal); in RISCVTargetLowering()
868 setOperationAction(ISD::VP_BSWAP, VT, Custom); in RISCVTargetLowering()
870 setOperationAction({ISD::BSWAP, ISD::VP_BSWAP}, VT, Expand); in RISCVTargetLowering()
871 setOperationAction({ISD::ROTL, ISD::ROTR}, VT, Expand); in RISCVTargetLowering()
875 setOperationAction(ISD::BITREVERSE, VT, Legal); in RISCVTargetLowering()
876 setOperationAction(ISD::VP_BITREVERSE, VT, Custom); in RISCVTargetLowering()
877 setOperationAction({ISD::VP_CTLZ, ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ, in RISCVTargetLowering()
878 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP}, in RISCVTargetLowering()
881 setOperationAction({ISD::BITREVERSE, ISD::VP_BITREVERSE}, VT, Expand); in RISCVTargetLowering()
882 setOperationAction({ISD::CTLZ, ISD::CTTZ, ISD::CTPOP}, VT, Expand); in RISCVTargetLowering()
883 setOperationAction({ISD::VP_CTLZ, ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ, in RISCVTargetLowering()
884 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP}, in RISCVTargetLowering()
891 setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF, in RISCVTargetLowering()
892 ISD::CTTZ_ZERO_UNDEF, ISD::VP_CTLZ, in RISCVTargetLowering()
893 ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF}, in RISCVTargetLowering()
906 static const ISD::CondCode VFPCCToExpand[] = { in RISCVTargetLowering()
907 ISD::SETO, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT, in RISCVTargetLowering()
908 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO, in RISCVTargetLowering()
909 ISD::SETGT, ISD::SETOGT, ISD::SETGE, ISD::SETOGE, in RISCVTargetLowering()
914 ISD::FMINNUM, ISD::FMAXNUM, ISD::FADD, ISD::FSUB, in RISCVTargetLowering()
915 ISD::FMUL, ISD::FMA, ISD::FDIV, ISD::FSQRT, in RISCVTargetLowering()
916 ISD::FABS, ISD::FNEG, ISD::FCOPYSIGN, ISD::FCEIL, in RISCVTargetLowering()
917 ISD::FFLOOR, ISD::FROUND, ISD::FROUNDEVEN, ISD::FRINT, in RISCVTargetLowering()
918 ISD::FNEARBYINT, ISD::IS_FPCLASS, ISD::SETCC, ISD::FMAXIMUM, in RISCVTargetLowering()
919 ISD::FMINIMUM, ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL, in RISCVTargetLowering()
920 ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA}; in RISCVTargetLowering()
924 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL, in RISCVTargetLowering()
925 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS, in RISCVTargetLowering()
926 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD, in RISCVTargetLowering()
927 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_SQRT, in RISCVTargetLowering()
928 ISD::VP_FMINNUM, ISD::VP_FMAXNUM, ISD::VP_FCEIL, in RISCVTargetLowering()
929 ISD::VP_FFLOOR, ISD::VP_FROUND, ISD::VP_FROUNDEVEN, in RISCVTargetLowering()
930 ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, ISD::VP_FRINT, in RISCVTargetLowering()
931 ISD::VP_FNEARBYINT, ISD::VP_SETCC, ISD::VP_FMINIMUM, in RISCVTargetLowering()
932 ISD::VP_FMAXIMUM}; in RISCVTargetLowering()
936 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); in RISCVTargetLowering()
941 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom); in RISCVTargetLowering()
943 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT, in RISCVTargetLowering()
948 setOperationAction({ISD::FMINNUM, ISD::FMAXNUM}, VT, Legal); in RISCVTargetLowering()
949 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Custom); in RISCVTargetLowering()
951 setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND, in RISCVTargetLowering()
952 ISD::FROUNDEVEN, ISD::FRINT, ISD::FNEARBYINT, in RISCVTargetLowering()
953 ISD::IS_FPCLASS}, in RISCVTargetLowering()
959 setOperationAction(ISD::FREM, VT, Expand); in RISCVTargetLowering()
960 setOperationAction(ISD::FPOW, VT, Expand); in RISCVTargetLowering()
961 setOperationAction(ISD::FCOS, VT, Expand); in RISCVTargetLowering()
962 setOperationAction(ISD::FSIN, VT, Expand); in RISCVTargetLowering()
963 setOperationAction(ISD::FSINCOS, VT, Expand); in RISCVTargetLowering()
964 setOperationAction(ISD::FEXP, VT, Expand); in RISCVTargetLowering()
965 setOperationAction(ISD::FEXP2, VT, Expand); in RISCVTargetLowering()
966 setOperationAction(ISD::FEXP10, VT, Expand); in RISCVTargetLowering()
967 setOperationAction(ISD::FLOG, VT, Expand); in RISCVTargetLowering()
968 setOperationAction(ISD::FLOG2, VT, Expand); in RISCVTargetLowering()
969 setOperationAction(ISD::FLOG10, VT, Expand); in RISCVTargetLowering()
971 setOperationAction(ISD::FCOPYSIGN, VT, Legal); in RISCVTargetLowering()
973 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom); in RISCVTargetLowering()
975 setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, in RISCVTargetLowering()
979 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD, in RISCVTargetLowering()
980 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER}, in RISCVTargetLowering()
983 setOperationAction(ISD::SELECT, VT, Custom); in RISCVTargetLowering()
984 setOperationAction(ISD::SELECT_CC, VT, Expand); in RISCVTargetLowering()
986 setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR, in RISCVTargetLowering()
987 ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR}, in RISCVTargetLowering()
990 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom); in RISCVTargetLowering()
991 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom); in RISCVTargetLowering()
993 setOperationAction({ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE}, VT, Custom); in RISCVTargetLowering()
997 setOperationAction({ISD::STRICT_FP_EXTEND, ISD::STRICT_FP_ROUND}, VT, in RISCVTargetLowering()
999 setOperationAction({ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL, in RISCVTargetLowering()
1000 ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA}, in RISCVTargetLowering()
1002 setOperationAction({ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, in RISCVTargetLowering()
1003 ISD::STRICT_FTRUNC, ISD::STRICT_FCEIL, in RISCVTargetLowering()
1004 ISD::STRICT_FFLOOR, ISD::STRICT_FROUND, in RISCVTargetLowering()
1005 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT}, in RISCVTargetLowering()
1015 setLoadExtAction(ISD::EXTLOAD, VT, SmallVT, Expand); in RISCVTargetLowering()
1029 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom); in RISCVTargetLowering()
1030 setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT, in RISCVTargetLowering()
1032 setOperationAction({ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Custom); in RISCVTargetLowering()
1033 setOperationAction({ISD::VP_MERGE, ISD::VP_SELECT, ISD::SELECT}, VT, in RISCVTargetLowering()
1035 setOperationAction(ISD::SELECT_CC, VT, Expand); in RISCVTargetLowering()
1036 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, in RISCVTargetLowering()
1037 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP}, in RISCVTargetLowering()
1039 setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR, in RISCVTargetLowering()
1040 ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR}, in RISCVTargetLowering()
1042 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); in RISCVTargetLowering()
1044 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom); in RISCVTargetLowering()
1084 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) in RISCVTargetLowering()
1088 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT, in RISCVTargetLowering()
1094 setOperationAction(ISD::UNDEF, VT, Custom); in RISCVTargetLowering()
1097 setOperationAction({ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT, in RISCVTargetLowering()
1100 setOperationAction({ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS}, VT, in RISCVTargetLowering()
1103 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, in RISCVTargetLowering()
1106 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom); in RISCVTargetLowering()
1108 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom); in RISCVTargetLowering()
1110 setOperationAction(ISD::SETCC, VT, Custom); in RISCVTargetLowering()
1112 setOperationAction(ISD::SELECT, VT, Custom); in RISCVTargetLowering()
1114 setOperationAction(ISD::TRUNCATE, VT, Custom); in RISCVTargetLowering()
1116 setOperationAction(ISD::BITCAST, VT, Custom); in RISCVTargetLowering()
1119 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT, in RISCVTargetLowering()
1123 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT, in RISCVTargetLowering()
1128 ISD::SINT_TO_FP, in RISCVTargetLowering()
1129 ISD::UINT_TO_FP, in RISCVTargetLowering()
1130 ISD::FP_TO_SINT, in RISCVTargetLowering()
1131 ISD::FP_TO_UINT, in RISCVTargetLowering()
1132 ISD::STRICT_SINT_TO_FP, in RISCVTargetLowering()
1133 ISD::STRICT_UINT_TO_FP, in RISCVTargetLowering()
1134 ISD::STRICT_FP_TO_SINT, in RISCVTargetLowering()
1135 ISD::STRICT_FP_TO_UINT, in RISCVTargetLowering()
1138 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT, in RISCVTargetLowering()
1141 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in RISCVTargetLowering()
1145 setOperationAction({ISD::VP_AND, ISD::VP_OR, ISD::VP_XOR, ISD::AND, in RISCVTargetLowering()
1146 ISD::OR, ISD::XOR}, in RISCVTargetLowering()
1149 setOperationAction({ISD::VP_FP_TO_SINT, ISD::VP_FP_TO_UINT, in RISCVTargetLowering()
1150 ISD::VP_SETCC, ISD::VP_TRUNCATE}, in RISCVTargetLowering()
1153 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom); in RISCVTargetLowering()
1154 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom); in RISCVTargetLowering()
1164 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); in RISCVTargetLowering()
1165 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom); in RISCVTargetLowering()
1169 {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, VT, Custom); in RISCVTargetLowering()
1171 setOperationAction({ISD::VP_LOAD, ISD::VP_STORE, in RISCVTargetLowering()
1172 ISD::EXPERIMENTAL_VP_STRIDED_LOAD, in RISCVTargetLowering()
1173 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, in RISCVTargetLowering()
1174 ISD::VP_SCATTER}, in RISCVTargetLowering()
1177 setOperationAction({ISD::ADD, ISD::MUL, ISD::SUB, ISD::AND, ISD::OR, in RISCVTargetLowering()
1178 ISD::XOR, ISD::SDIV, ISD::SREM, ISD::UDIV, in RISCVTargetLowering()
1179 ISD::UREM, ISD::SHL, ISD::SRA, ISD::SRL}, in RISCVTargetLowering()
1183 {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX, ISD::ABS}, VT, Custom); in RISCVTargetLowering()
1187 setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Custom); in RISCVTargetLowering()
1189 setOperationAction({ISD::AVGFLOORU, ISD::AVGCEILU, ISD::SADDSAT, in RISCVTargetLowering()
1190 ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT}, in RISCVTargetLowering()
1193 setOperationAction(ISD::VSELECT, VT, Custom); in RISCVTargetLowering()
1194 setOperationAction(ISD::SELECT_CC, VT, Expand); in RISCVTargetLowering()
1197 {ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND}, VT, Custom); in RISCVTargetLowering()
1201 setOperationAction({ISD::VECREDUCE_ADD, ISD::VECREDUCE_SMAX, in RISCVTargetLowering()
1202 ISD::VECREDUCE_SMIN, ISD::VECREDUCE_UMAX, in RISCVTargetLowering()
1203 ISD::VECREDUCE_UMIN}, in RISCVTargetLowering()
1209 setOperationAction({ISD::BSWAP, ISD::ROTL, ISD::ROTR}, VT, Custom); in RISCVTargetLowering()
1212 setOperationAction({ISD::BITREVERSE, ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF, in RISCVTargetLowering()
1213 ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF, ISD::CTPOP}, in RISCVTargetLowering()
1221 {ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF, ISD::CTTZ_ZERO_UNDEF}, VT, in RISCVTargetLowering()
1229 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); in RISCVTargetLowering()
1237 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) in RISCVTargetLowering()
1242 setOperationAction(ISD::UNDEF, VT, Custom); in RISCVTargetLowering()
1246 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom); in RISCVTargetLowering()
1247 setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT, in RISCVTargetLowering()
1249 setOperationAction({ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Custom); in RISCVTargetLowering()
1251 {ISD::VP_MERGE, ISD::VP_SELECT, ISD::VSELECT, ISD::SELECT}, VT, in RISCVTargetLowering()
1253 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, in RISCVTargetLowering()
1254 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP}, in RISCVTargetLowering()
1256 setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR, in RISCVTargetLowering()
1257 ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR}, in RISCVTargetLowering()
1259 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom); in RISCVTargetLowering()
1260 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); in RISCVTargetLowering()
1273 setOperationAction({ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT, in RISCVTargetLowering()
1276 setOperationAction({ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS, in RISCVTargetLowering()
1277 ISD::VECTOR_SHUFFLE, ISD::INSERT_VECTOR_ELT, in RISCVTargetLowering()
1278 ISD::EXTRACT_VECTOR_ELT}, in RISCVTargetLowering()
1281 setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE, in RISCVTargetLowering()
1282 ISD::MGATHER, ISD::MSCATTER}, in RISCVTargetLowering()
1285 setOperationAction({ISD::VP_LOAD, ISD::VP_STORE, in RISCVTargetLowering()
1286 ISD::EXPERIMENTAL_VP_STRIDED_LOAD, in RISCVTargetLowering()
1287 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, in RISCVTargetLowering()
1288 ISD::VP_SCATTER}, in RISCVTargetLowering()
1291 setOperationAction({ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV, in RISCVTargetLowering()
1292 ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN, ISD::FSQRT, in RISCVTargetLowering()
1293 ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM, in RISCVTargetLowering()
1294 ISD::IS_FPCLASS, ISD::FMAXIMUM, ISD::FMINIMUM}, in RISCVTargetLowering()
1297 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom); in RISCVTargetLowering()
1299 setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND, in RISCVTargetLowering()
1300 ISD::FROUNDEVEN, ISD::FRINT, ISD::FNEARBYINT}, in RISCVTargetLowering()
1305 setOperationAction(ISD::SETCC, VT, Custom); in RISCVTargetLowering()
1306 setOperationAction({ISD::VSELECT, ISD::SELECT}, VT, Custom); in RISCVTargetLowering()
1307 setOperationAction(ISD::SELECT_CC, VT, Expand); in RISCVTargetLowering()
1309 setOperationAction(ISD::BITCAST, VT, Custom); in RISCVTargetLowering()
1315 setOperationAction({ISD::STRICT_FP_EXTEND, ISD::STRICT_FP_ROUND}, VT, in RISCVTargetLowering()
1318 {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL, in RISCVTargetLowering()
1319 ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA, in RISCVTargetLowering()
1320 ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::STRICT_FTRUNC, in RISCVTargetLowering()
1321 ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, ISD::STRICT_FROUND, in RISCVTargetLowering()
1322 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT}, in RISCVTargetLowering()
1327 setOperationAction(ISD::BITCAST, {MVT::i8, MVT::i16, MVT::i32, MVT::i64}, in RISCVTargetLowering()
1330 setOperationAction(ISD::BITCAST, MVT::f16, Custom); in RISCVTargetLowering()
1332 setOperationAction(ISD::BITCAST, MVT::f32, Custom); in RISCVTargetLowering()
1334 setOperationAction(ISD::BITCAST, MVT::f64, Custom); in RISCVTargetLowering()
1339 setOperationAction(ISD::ATOMIC_LOAD_SUB, XLenVT, Expand); in RISCVTargetLowering()
1341 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Expand); in RISCVTargetLowering()
1347 {ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD_ADD, in RISCVTargetLowering()
1348 ISD::ATOMIC_LOAD_SUB, ISD::ATOMIC_LOAD_AND, ISD::ATOMIC_LOAD_OR, in RISCVTargetLowering()
1349 ISD::ATOMIC_LOAD_XOR, ISD::ATOMIC_LOAD_NAND, ISD::ATOMIC_LOAD_MIN, in RISCVTargetLowering()
1350 ISD::ATOMIC_LOAD_MAX, ISD::ATOMIC_LOAD_UMIN, ISD::ATOMIC_LOAD_UMAX}, in RISCVTargetLowering()
1355 for (unsigned im : {ISD::PRE_INC, ISD::POST_INC}) { in RISCVTargetLowering()
1377 setTargetDAGCombine({ISD::INTRINSIC_VOID, ISD::INTRINSIC_W_CHAIN, in RISCVTargetLowering()
1378 ISD::INTRINSIC_WO_CHAIN, ISD::ADD, ISD::SUB, ISD::MUL, in RISCVTargetLowering()
1379 ISD::AND, ISD::OR, ISD::XOR, ISD::SETCC, ISD::SELECT}); in RISCVTargetLowering()
1381 setTargetDAGCombine(ISD::SRA); in RISCVTargetLowering()
1384 setTargetDAGCombine({ISD::FADD, ISD::FMAXNUM, ISD::FMINNUM}); in RISCVTargetLowering()
1387 setTargetDAGCombine({ISD::UMAX, ISD::UMIN, ISD::SMAX, ISD::SMIN}); in RISCVTargetLowering()
1390 setTargetDAGCombine(ISD::TRUNCATE); in RISCVTargetLowering()
1393 setTargetDAGCombine(ISD::BITREVERSE); in RISCVTargetLowering()
1395 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG); in RISCVTargetLowering()
1397 setTargetDAGCombine({ISD::ZERO_EXTEND, ISD::FP_TO_SINT, ISD::FP_TO_UINT, in RISCVTargetLowering()
1398 ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}); in RISCVTargetLowering()
1400 setTargetDAGCombine({ISD::FCOPYSIGN, ISD::MGATHER, ISD::MSCATTER, in RISCVTargetLowering()
1401 ISD::VP_GATHER, ISD::VP_SCATTER, ISD::SRA, ISD::SRL, in RISCVTargetLowering()
1402 ISD::SHL, ISD::STORE, ISD::SPLAT_VECTOR, in RISCVTargetLowering()
1403 ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS, in RISCVTargetLowering()
1404 ISD::EXPERIMENTAL_VP_REVERSE, ISD::MUL, in RISCVTargetLowering()
1405 ISD::INSERT_VECTOR_ELT}); in RISCVTargetLowering()
1407 setTargetDAGCombine({ISD::LOAD, ISD::STORE}); in RISCVTargetLowering()
1409 setTargetDAGCombine(ISD::BITCAST); in RISCVTargetLowering()
1468 Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
1507 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
1793 (LD->getExtensionType() == ISD::NON_EXTLOAD || in isZExtFree()
1794 LD->getExtensionType() == ISD::ZEXTLOAD)) in isZExtFree()
1910 if (XC && OldShiftOpcode == ISD::SRL && XC->isOne()) in shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd()
1914 if (NewShiftOpcode == ISD::SRL && CC->isOne()) in shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd()
2043 if (Opc >= ISD::BUILTIN_OP_END) in shouldScalarizeBinop()
2142 if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT)) in isExtractSubvectorCheap()
2221 ISD::CondCode &CC, SelectionDAG &DAG) { in translateSetCCForBranch()
2225 LHS.getOpcode() == ISD::AND && LHS.hasOneUse() && in translateSetCCForBranch()
2231 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT; in translateSetCCForBranch()
2239 LHS = DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS, in translateSetCCForBranch()
2249 case ISD::SETGT: in translateSetCCForBranch()
2253 CC = ISD::SETGE; in translateSetCCForBranch()
2257 case ISD::SETLT: in translateSetCCForBranch()
2262 CC = ISD::SETGE; in translateSetCCForBranch()
2272 case ISD::SETGT: in translateSetCCForBranch()
2273 case ISD::SETLE: in translateSetCCForBranch()
2274 case ISD::SETUGT: in translateSetCCForBranch()
2275 case ISD::SETULE: in translateSetCCForBranch()
2276 CC = ISD::getSetCCSwappedOperands(CC); in translateSetCCForBranch()
2427 assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in getVLOperand()
2428 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) && in getVLOperand()
2430 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN; in getVLOperand()
2564 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero); in convertToScalableVector()
2576 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero); in convertFromScalableVector()
2743 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT; in lowerFP_TO_INT_SAT()
2750 Src = DAG.getNode(ISD::FP_EXTEND, SDLoc(Op), MVT::f32, Src); in lowerFP_TO_INT_SAT()
2772 ISD::CondCode::SETUO); in lowerFP_TO_INT_SAT()
2807 {Src, Src, DAG.getCondCode(ISD::SETNE), in lowerFP_TO_INT_SAT()
2836 case ISD::FROUNDEVEN: in matchRoundingOp()
2837 case ISD::STRICT_FROUNDEVEN: in matchRoundingOp()
2838 case ISD::VP_FROUNDEVEN: in matchRoundingOp()
2840 case ISD::FTRUNC: in matchRoundingOp()
2841 case ISD::STRICT_FTRUNC: in matchRoundingOp()
2842 case ISD::VP_FROUNDTOZERO: in matchRoundingOp()
2844 case ISD::FFLOOR: in matchRoundingOp()
2845 case ISD::STRICT_FFLOOR: in matchRoundingOp()
2846 case ISD::VP_FFLOOR: in matchRoundingOp()
2848 case ISD::FCEIL: in matchRoundingOp()
2849 case ISD::STRICT_FCEIL: in matchRoundingOp()
2850 case ISD::VP_FCEIL: in matchRoundingOp()
2852 case ISD::FROUND: in matchRoundingOp()
2853 case ISD::STRICT_FROUND: in matchRoundingOp()
2854 case ISD::VP_FROUND: in matchRoundingOp()
2856 case ISD::FRINT: in matchRoundingOp()
2917 {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT), in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2928 case ISD::FCEIL: in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2929 case ISD::VP_FCEIL: in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2930 case ISD::FFLOOR: in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2931 case ISD::VP_FFLOOR: in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2932 case ISD::FROUND: in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2933 case ISD::FROUNDEVEN: in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2934 case ISD::VP_FROUND: in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2935 case ISD::VP_FROUNDEVEN: in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2936 case ISD::VP_FROUNDTOZERO: { in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2943 case ISD::FTRUNC: in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2947 case ISD::FRINT: in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2948 case ISD::VP_FRINT: in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2951 case ISD::FNEARBYINT: in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2952 case ISD::VP_FNEARBYINT: in lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()
2999 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE), in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3026 {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT), Mask, Mask, VL}); in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3036 case ISD::STRICT_FCEIL: in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3037 case ISD::STRICT_FFLOOR: in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3038 case ISD::STRICT_FROUND: in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3039 case ISD::STRICT_FROUNDEVEN: { in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3047 case ISD::STRICT_FTRUNC: in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3052 case ISD::STRICT_FNEARBYINT: in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3061 if (Op.getOpcode() != ISD::STRICT_FNEARBYINT) { in lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()
3199 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR"); in isSimpleVIDSequence()
3306 if (SplatVal.getOpcode() != ISD::EXTRACT_VECTOR_ELT) in matchSplatAsGather()
3397 (NumElts != 2 || ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) && in lowerBuildVectorViaDominantValues()
3417 LastOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, LastOp); in lowerBuildVectorViaDominantValues()
3430 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Vec, V, in lowerBuildVectorViaDominantValues()
3440 Vec = DAG.getNode(ISD::VSELECT, DL, VT, in lowerBuildVectorViaDominantValues()
3466 if (ISD::isBuildVectorAllZeros(Op.getNode())) { in lowerBuildVectorOfConstants()
3471 if (ISD::isBuildVectorAllOnes(Op.getNode())) { in lowerBuildVectorOfConstants()
3530 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Vec, in lowerBuildVectorOfConstants()
3546 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat); in lowerBuildVectorOfConstants()
3563 unsigned StepOpcode = ISD::MUL; in lowerBuildVectorOfConstants()
3569 StepOpcode = ISD::SHL; in lowerBuildVectorOfConstants()
3577 if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) || in lowerBuildVectorOfConstants()
3578 (StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) && in lowerBuildVectorOfConstants()
3591 if ((StepOpcode == ISD::MUL && SplatStepVal != 1) || in lowerBuildVectorOfConstants()
3592 (StepOpcode == ISD::SHL && SplatStepVal != 0)) { in lowerBuildVectorOfConstants()
3599 VID = DAG.getNode(ISD::SRL, DL, VIDVT, VID, SplatStep); in lowerBuildVectorOfConstants()
3603 VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VIDVT, SplatAddend, in lowerBuildVectorOfConstants()
3608 VID = DAG.getNode(ISD::SINT_TO_FP, DL, VT, VID); in lowerBuildVectorOfConstants()
3618 ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) { in lowerBuildVectorOfConstants()
3644 SDValue Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ViaVecVT, in lowerBuildVectorOfConstants()
3649 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, in lowerBuildVectorOfConstants()
3666 ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) && in lowerBuildVectorOfConstants()
3718 Splat = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, in lowerBuildVectorOfConstants()
3758 if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) || in lowerBUILD_VECTOR()
3759 ISD::isBuildVectorOfConstantFPSDNodes(Op.getNode())) in lowerBUILD_VECTOR()
3780 Splat = DAG.getNode(ISD::AND, DL, Splat.getValueType(), Splat, in lowerBUILD_VECTOR()
3787 WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne); in lowerBUILD_VECTOR()
3790 return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE); in lowerBUILD_VECTOR()
3799 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat); in lowerBUILD_VECTOR()
3830 DAG.getNode(ISD::BUILD_VECTOR, DL, OneRegVT, OneVRegOfOps); in lowerBUILD_VECTOR()
3833 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, Vec, SubBV, in lowerBUILD_VECTOR()
3917 V = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), V); in lowerBUILD_VECTOR()
3954 NewVL = DAG.getNode(ISD::ADD, DL, VL.getValueType(), VL, VL); in splatPartsI64WithVL()
3962 return DAG.getNode(ISD::BITCAST, DL, VT, InterVec); in splatPartsI64WithVL()
3968 if (Hi.getOpcode() == ISD::SRA && Hi.getOperand(0) == Lo && in splatPartsI64WithVL()
4016 isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND; in lowerScalarSplat()
4044 if (Scalar.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in lowerScalarInsert()
4059 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Passthru, in lowerScalarInsert()
4061 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, ExtractedVal, in lowerScalarInsert()
4083 isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND; in lowerScalarInsert()
4104 if (V1.getOpcode() != ISD::EXTRACT_SUBVECTOR || in isDeinterleaveShuffle()
4105 V2.getOpcode() != ISD::EXTRACT_SUBVECTOR) in isDeinterleaveShuffle()
4318 while (Parent.getOpcode() == ISD::EXTRACT_SUBVECTOR && in lowerVECTOR_SHUFFLEAsVSlidedown()
4370 ISD::EXTRACT_SUBVECTOR, DL, VT, in lowerVECTOR_SHUFFLEAsVSlidedown()
4474 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Splat); in lowerVECTOR_SHUFFLEAsVSlide1()
4592 if (!Subtarget.getTargetLowering()->isOperationLegalOrCustom(ISD::BITREVERSE, in lowerBitreverseShuffle()
4601 V = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ViaBitVT, DAG.getUNDEF(ViaBitVT), in lowerBitreverseShuffle()
4605 DAG.getNode(ISD::BITREVERSE, DL, ViaVT, DAG.getBitcast(ViaVT, V)); in lowerBitreverseShuffle()
4610 Res = DAG.getNode(ISD::SRL, DL, ViaVT, Res, in lowerBitreverseShuffle()
4616 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in lowerBitreverseShuffle()
4649 Rotate = DAG.getNode(ISD::BSWAP, DL, RotateVT, Op); in lowerVECTOR_SHUFFLEAsRotate()
4651 Rotate = DAG.getNode(ISD::ROTL, DL, RotateVT, Op, in lowerVECTOR_SHUFFLEAsRotate()
4724 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, SrcVec, in lowerShuffleViaVRegSplitting()
4730 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, Vec, SubVec, in lowerShuffleViaVRegSplitting()
4756 V1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V1); in lowerVECTOR_SHUFFLE()
4758 : DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V2); in lowerVECTOR_SHUFFLE()
4761 ISD::SETNE); in lowerVECTOR_SHUFFLE()
4779 if (V.getOpcode() == ISD::CONCAT_VECTORS) { in lowerVECTOR_SHUFFLE()
4787 if (ISD::isNormalLoad(V.getNode()) && cast<LoadSDNode>(V)->isSimple()) { in lowerVECTOR_SHUFFLE()
4805 ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, SVT, in lowerVECTOR_SHUFFLE()
4821 V = DAG.getExtLoad(ISD::SEXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr, in lowerVECTOR_SHUFFLE()
4920 EvenV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, EvenV, in lowerVECTOR_SHUFFLE()
4925 OddV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, OddV, in lowerVECTOR_SHUFFLE()
4961 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2); in lowerVECTOR_SHUFFLE()
5149 if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF) { in lowerCTLZ_CTTZ_ZERO_UNDEF()
5151 Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5152 } else if (Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF) { in lowerCTLZ_CTTZ_ZERO_UNDEF()
5153 SDValue Neg = DAG.getNode(ISD::VP_SUB, DL, VT, DAG.getConstant(0, DL, VT), in lowerCTLZ_CTTZ_ZERO_UNDEF()
5155 Src = DAG.getNode(ISD::VP_AND, DL, VT, Src, Neg, Mask, VL); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5162 FloatVal = DAG.getNode(ISD::VP_UINT_TO_FP, DL, FloatVT, Src, Mask, VL); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5164 FloatVal = DAG.getNode(ISD::UINT_TO_FP, DL, FloatVT, Src); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5190 Exp = DAG.getNode(ISD::VP_LSHR, DL, IntVT, Bitcast, in lowerCTLZ_CTTZ_ZERO_UNDEF()
5194 Exp = DAG.getNode(ISD::SRL, DL, IntVT, Bitcast, in lowerCTLZ_CTTZ_ZERO_UNDEF()
5197 Exp = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Exp); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5199 Exp = DAG.getNode(ISD::TRUNCATE, DL, VT, Exp); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5205 if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF) in lowerCTLZ_CTTZ_ZERO_UNDEF()
5206 return DAG.getNode(ISD::SUB, DL, VT, Exp, in lowerCTLZ_CTTZ_ZERO_UNDEF()
5208 if (Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF) in lowerCTLZ_CTTZ_ZERO_UNDEF()
5209 return DAG.getNode(ISD::VP_SUB, DL, VT, Exp, in lowerCTLZ_CTTZ_ZERO_UNDEF()
5217 Res = DAG.getNode(ISD::VP_SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp, in lowerCTLZ_CTTZ_ZERO_UNDEF()
5220 Res = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5224 if (Op.getOpcode() == ISD::CTLZ) in lowerCTLZ_CTTZ_ZERO_UNDEF()
5225 Res = DAG.getNode(ISD::UMIN, DL, VT, Res, DAG.getConstant(EltSize, DL, VT)); in lowerCTLZ_CTTZ_ZERO_UNDEF()
5226 else if (Op.getOpcode() == ISD::VP_CTLZ) in lowerCTLZ_CTTZ_ZERO_UNDEF()
5227 Res = DAG.getNode(ISD::VP_UMIN, DL, VT, Res, in lowerCTLZ_CTTZ_ZERO_UNDEF()
5348 return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0)); in LowerATOMIC_FENCE()
5356 return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0)); in LowerATOMIC_FENCE()
5400 if (Op.getOpcode() == ISD::VP_IS_FPCLASS) { in LowerIS_FPCLASS()
5409 ISD::CondCode::SETEQ); in LowerIS_FPCLASS()
5410 SDValue AND = DAG.getNode(ISD::AND, DL, DstVT, FPCLASS, in LowerIS_FPCLASS()
5413 ISD::SETNE); in LowerIS_FPCLASS()
5420 if (Op.getOpcode() == ISD::VP_IS_FPCLASS) { in LowerIS_FPCLASS()
5437 {FPCLASS, TDCMaskV, DAG.getCondCode(ISD::SETEQ), in LowerIS_FPCLASS()
5449 {AND, SplatZero, DAG.getCondCode(ISD::SETNE), in LowerIS_FPCLASS()
5455 SDValue AND = DAG.getNode(ISD::AND, DL, XLenVT, FCLASS, TDCMaskV); in LowerIS_FPCLASS()
5457 ISD::CondCode::SETNE); in LowerIS_FPCLASS()
5458 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res); in LowerIS_FPCLASS()
5481 SDValue XIsNonNan = DAG.getSetCC(DL, XLenVT, X, X, ISD::SETOEQ); in lowerFMAXIMUM_FMINIMUM()
5487 SDValue YIsNonNan = DAG.getSetCC(DL, XLenVT, Y, Y, ISD::SETOEQ); in lowerFMAXIMUM_FMINIMUM()
5492 Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::FMAX : RISCVISD::FMIN; in lowerFMAXIMUM_FMINIMUM()
5521 {X, X, DAG.getCondCode(ISD::SETOEQ), in lowerFMAXIMUM_FMINIMUM()
5530 {Y, Y, DAG.getCondCode(ISD::SETOEQ), in lowerFMAXIMUM_FMINIMUM()
5537 Op.getOpcode() == ISD::FMAXIMUM || Op->getOpcode() == ISD::VP_FMAXIMUM in lowerFMAXIMUM_FMINIMUM()
5550 case ISD::NODE: \ in getRISCVVLOp()
5553 case ISD::VP_##NODE: \ in getRISCVVLOp()
5627 case ISD::CTLZ_ZERO_UNDEF: in getRISCVVLOp()
5628 case ISD::VP_CTLZ_ZERO_UNDEF: in getRISCVVLOp()
5630 case ISD::CTTZ_ZERO_UNDEF: in getRISCVVLOp()
5631 case ISD::VP_CTTZ_ZERO_UNDEF: in getRISCVVLOp()
5633 case ISD::FMA: in getRISCVVLOp()
5634 case ISD::VP_FMA: in getRISCVVLOp()
5636 case ISD::STRICT_FMA: in getRISCVVLOp()
5638 case ISD::AND: in getRISCVVLOp()
5639 case ISD::VP_AND: in getRISCVVLOp()
5643 case ISD::OR: in getRISCVVLOp()
5644 case ISD::VP_OR: in getRISCVVLOp()
5648 case ISD::XOR: in getRISCVVLOp()
5649 case ISD::VP_XOR: in getRISCVVLOp()
5653 case ISD::VP_SELECT: in getRISCVVLOp()
5654 case ISD::VP_MERGE: in getRISCVVLOp()
5656 case ISD::VP_ASHR: in getRISCVVLOp()
5658 case ISD::VP_LSHR: in getRISCVVLOp()
5660 case ISD::VP_SQRT: in getRISCVVLOp()
5662 case ISD::VP_SIGN_EXTEND: in getRISCVVLOp()
5664 case ISD::VP_ZERO_EXTEND: in getRISCVVLOp()
5666 case ISD::VP_FP_TO_SINT: in getRISCVVLOp()
5668 case ISD::VP_FP_TO_UINT: in getRISCVVLOp()
5670 case ISD::FMINNUM: in getRISCVVLOp()
5671 case ISD::VP_FMINNUM: in getRISCVVLOp()
5673 case ISD::FMAXNUM: in getRISCVVLOp()
5674 case ISD::VP_FMAXNUM: in getRISCVVLOp()
5690 ISD::FIRST_TARGET_STRICTFP_OPCODE == in hasMergeOp()
5716 ISD::FIRST_TARGET_STRICTFP_OPCODE == in hasMaskOp()
5751 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Op.getValueType(), LoRes, HiRes); in SplitVectorOp()
5755 assert(ISD::isVPOpcode(Op.getOpcode()) && "Not a VP op"); in SplitVPOp()
5763 if (ISD::getVPExplicitVectorLengthIdx(Op.getOpcode()) == j) { in SplitVPOp()
5782 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Op.getValueType(), LoRes, HiRes); in SplitVPOp()
5830 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, Op->getValueType(0), in SplitStrictFPVectorOp()
5840 case ISD::ATOMIC_FENCE: in LowerOperation()
5842 case ISD::GlobalAddress: in LowerOperation()
5844 case ISD::BlockAddress: in LowerOperation()
5846 case ISD::ConstantPool: in LowerOperation()
5848 case ISD::JumpTable: in LowerOperation()
5850 case ISD::GlobalTLSAddress: in LowerOperation()
5852 case ISD::Constant: in LowerOperation()
5854 case ISD::SELECT: in LowerOperation()
5856 case ISD::BRCOND: in LowerOperation()
5858 case ISD::VASTART: in LowerOperation()
5860 case ISD::FRAMEADDR: in LowerOperation()
5862 case ISD::RETURNADDR: in LowerOperation()
5864 case ISD::SHL_PARTS: in LowerOperation()
5866 case ISD::SRA_PARTS: in LowerOperation()
5868 case ISD::SRL_PARTS: in LowerOperation()
5870 case ISD::ROTL: in LowerOperation()
5871 case ISD::ROTR: in LowerOperation()
5883 case ISD::BITCAST: { in LowerOperation()
5891 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0); in LowerOperation()
5897 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0); in LowerOperation()
5903 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0); in LowerOperation()
5940 return DAG.getBitcast(VT, DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, BVT, in LowerOperation()
5954 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec, in LowerOperation()
5959 case ISD::INTRINSIC_WO_CHAIN: in LowerOperation()
5961 case ISD::INTRINSIC_W_CHAIN: in LowerOperation()
5963 case ISD::INTRINSIC_VOID: in LowerOperation()
5965 case ISD::IS_FPCLASS: in LowerOperation()
5967 case ISD::BITREVERSE: { in LowerOperation()
5975 assert(Op.getOpcode() == ISD::BITREVERSE && "Unexpected opcode"); in LowerOperation()
5977 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Op.getOperand(0)); in LowerOperation()
5980 case ISD::TRUNCATE: in LowerOperation()
5985 case ISD::ANY_EXTEND: in LowerOperation()
5986 case ISD::ZERO_EXTEND: in LowerOperation()
5991 case ISD::SIGN_EXTEND: in LowerOperation()
5996 case ISD::SPLAT_VECTOR_PARTS: in LowerOperation()
5998 case ISD::INSERT_VECTOR_ELT: in LowerOperation()
6000 case ISD::EXTRACT_VECTOR_ELT: in LowerOperation()
6002 case ISD::SCALAR_TO_VECTOR: { in LowerOperation()
6008 SDValue V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, WideVT, Scalar); in LowerOperation()
6009 return DAG.getNode(ISD::TRUNCATE, DL, VT, V); in LowerOperation()
6015 Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Scalar); in LowerOperation()
6022 case ISD::VSCALE: { in LowerOperation()
6039 Res = DAG.getNode(ISD::SRL, DL, XLenVT, Res, in LowerOperation()
6042 Res = DAG.getNode(ISD::SHL, DL, XLenVT, Res, in LowerOperation()
6047 Res = DAG.getNode(ISD::MUL, DL, XLenVT, Res, in LowerOperation()
6050 SDValue VScale = DAG.getNode(ISD::SRL, DL, XLenVT, Res, in LowerOperation()
6052 Res = DAG.getNode(ISD::MUL, DL, XLenVT, VScale, in LowerOperation()
6055 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res); in LowerOperation()
6057 case ISD::FPOWI: { in LowerOperation()
6063 SDValue Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0)); in LowerOperation()
6065 DAG.getNode(ISD::FPOWI, DL, MVT::f32, Op0, Op.getOperand(1)); in LowerOperation()
6066 return DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, Powi, in LowerOperation()
6071 case ISD::FMAXIMUM: in LowerOperation()
6072 case ISD::FMINIMUM: in LowerOperation()
6078 case ISD::FP_EXTEND: { in LowerOperation()
6088 return DAG.getNode(ISD::FP_EXTEND, DL, MVT::f64, FloatVal); in LowerOperation()
6095 case ISD::FP_ROUND: { in LowerOperation()
6105 DAG.getNode(ISD::FP_ROUND, DL, MVT::f32, Op0, in LowerOperation()
6114 case ISD::STRICT_FP_ROUND: in LowerOperation()
6115 case ISD::STRICT_FP_EXTEND: in LowerOperation()
6117 case ISD::SINT_TO_FP: in LowerOperation()
6118 case ISD::UINT_TO_FP: in LowerOperation()
6131 return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), NC, in LowerOperation()
6135 case ISD::FP_TO_SINT: in LowerOperation()
6136 case ISD::FP_TO_UINT: in LowerOperation()
6148 SDValue WidenVec = DAG.getNode(ISD::FP_EXTEND, DL, NVT, Op1); in LowerOperation()
6153 case ISD::STRICT_FP_TO_SINT: in LowerOperation()
6154 case ISD::STRICT_FP_TO_UINT: in LowerOperation()
6155 case ISD::STRICT_SINT_TO_FP: in LowerOperation()
6156 case ISD::STRICT_UINT_TO_FP: { in LowerOperation()
6181 unsigned ExtOpcode = (Op.getOpcode() == ISD::UINT_TO_FP || in LowerOperation()
6182 Op.getOpcode() == ISD::STRICT_UINT_TO_FP) in LowerOperation()
6183 ? ISD::ZERO_EXTEND in LowerOperation()
6184 : ISD::SIGN_EXTEND; in LowerOperation()
6230 SDValue Res = DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int); in LowerOperation()
6234 return DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int); in LowerOperation()
6247 case ISD::FP_TO_SINT: in LowerOperation()
6250 case ISD::FP_TO_UINT: in LowerOperation()
6253 case ISD::SINT_TO_FP: in LowerOperation()
6256 case ISD::UINT_TO_FP: in LowerOperation()
6259 case ISD::STRICT_FP_TO_SINT: in LowerOperation()
6262 case ISD::STRICT_FP_TO_UINT: in LowerOperation()
6265 case ISD::STRICT_SINT_TO_FP: in LowerOperation()
6268 case ISD::STRICT_UINT_TO_FP: in LowerOperation()
6290 case ISD::FP_TO_SINT_SAT: in LowerOperation()
6291 case ISD::FP_TO_UINT_SAT: in LowerOperation()
6293 case ISD::FP_TO_BF16: { in LowerOperation()
6307 case ISD::BF16_TO_FP: { in LowerOperation()
6312 ISD::SHL, DL, Op.getOperand(0).getValueType(), Op.getOperand(0), in LowerOperation()
6319 return DAG.getNode(ISD::FP_EXTEND, DL, VT, Res); in LowerOperation()
6322 case ISD::FP_TO_FP16: { in LowerOperation()
6336 case ISD::FP16_TO_FP: { in LowerOperation()
6351 case ISD::FTRUNC: in LowerOperation()
6352 case ISD::FCEIL: in LowerOperation()
6353 case ISD::FFLOOR: in LowerOperation()
6354 case ISD::FNEARBYINT: in LowerOperation()
6355 case ISD::FRINT: in LowerOperation()
6356 case ISD::FROUND: in LowerOperation()
6357 case ISD::FROUNDEVEN: in LowerOperation()
6359 case ISD::LRINT: in LowerOperation()
6360 case ISD::LLRINT: in LowerOperation()
6362 case ISD::VECREDUCE_ADD: in LowerOperation()
6363 case ISD::VECREDUCE_UMAX: in LowerOperation()
6364 case ISD::VECREDUCE_SMAX: in LowerOperation()
6365 case ISD::VECREDUCE_UMIN: in LowerOperation()
6366 case ISD::VECREDUCE_SMIN: in LowerOperation()
6368 case ISD::VECREDUCE_AND: in LowerOperation()
6369 case ISD::VECREDUCE_OR: in LowerOperation()
6370 case ISD::VECREDUCE_XOR: in LowerOperation()
6374 case ISD::VECREDUCE_FADD: in LowerOperation()
6375 case ISD::VECREDUCE_SEQ_FADD: in LowerOperation()
6376 case ISD::VECREDUCE_FMIN: in LowerOperation()
6377 case ISD::VECREDUCE_FMAX: in LowerOperation()
6379 case ISD::VP_REDUCE_ADD: in LowerOperation()
6380 case ISD::VP_REDUCE_UMAX: in LowerOperation()
6381 case ISD::VP_REDUCE_SMAX: in LowerOperation()
6382 case ISD::VP_REDUCE_UMIN: in LowerOperation()
6383 case ISD::VP_REDUCE_SMIN: in LowerOperation()
6384 case ISD::VP_REDUCE_FADD: in LowerOperation()
6385 case ISD::VP_REDUCE_SEQ_FADD: in LowerOperation()
6386 case ISD::VP_REDUCE_FMIN: in LowerOperation()
6387 case ISD::VP_REDUCE_FMAX: in LowerOperation()
6393 case ISD::VP_REDUCE_AND: in LowerOperation()
6394 case ISD::VP_REDUCE_OR: in LowerOperation()
6395 case ISD::VP_REDUCE_XOR: in LowerOperation()
6399 case ISD::UNDEF: { in LowerOperation()
6404 case ISD::INSERT_SUBVECTOR: in LowerOperation()
6406 case ISD::EXTRACT_SUBVECTOR: in LowerOperation()
6408 case ISD::VECTOR_DEINTERLEAVE: in LowerOperation()
6410 case ISD::VECTOR_INTERLEAVE: in LowerOperation()
6412 case ISD::STEP_VECTOR: in LowerOperation()
6414 case ISD::VECTOR_REVERSE: in LowerOperation()
6416 case ISD::VECTOR_SPLICE: in LowerOperation()
6418 case ISD::BUILD_VECTOR: in LowerOperation()
6420 case ISD::SPLAT_VECTOR: in LowerOperation()
6428 DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0)); in LowerOperation()
6430 ISD::SPLAT_VECTOR, DL, in LowerOperation()
6433 return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), NewSplat, in LowerOperation()
6439 case ISD::VECTOR_SHUFFLE: in LowerOperation()
6441 case ISD::CONCAT_VECTORS: { in LowerOperation()
6454 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, SubVec, in LowerOperation()
6459 case ISD::LOAD: in LowerOperation()
6465 case ISD::STORE: in LowerOperation()
6471 case ISD::MLOAD: in LowerOperation()
6472 case ISD::VP_LOAD: in LowerOperation()
6474 case ISD::MSTORE: in LowerOperation()
6475 case ISD::VP_STORE: in LowerOperation()
6477 case ISD::SELECT_CC: { in LowerOperation()
6492 DAG.getNode(ISD::SETCC, DL, CCVT, Tmp1, Tmp2, CC, Op->getFlags()); in LowerOperation()
6495 case ISD::SETCC: { in LowerOperation()
6501 ISD::CondCode CCVal = cast<CondCodeSDNode>(Op.getOperand(2))->get(); in LowerOperation()
6502 assert((CCVal == ISD::SETGT || CCVal == ISD::SETUGT) && in LowerOperation()
6516 if (CCVal == ISD::SETUGT && Imm == -1) in LowerOperation()
6519 CCVal = ISD::getSetCCSwappedOperands(CCVal); in LowerOperation()
6528 CCVal = ISD::getSetCCSwappedOperands(CCVal); in LowerOperation()
6539 case ISD::ADD: in LowerOperation()
6540 case ISD::SUB: in LowerOperation()
6541 case ISD::MUL: in LowerOperation()
6542 case ISD::MULHS: in LowerOperation()
6543 case ISD::MULHU: in LowerOperation()
6544 case ISD::AND: in LowerOperation()
6545 case ISD::OR: in LowerOperation()
6546 case ISD::XOR: in LowerOperation()
6547 case ISD::SDIV: in LowerOperation()
6548 case ISD::SREM: in LowerOperation()
6549 case ISD::UDIV: in LowerOperation()
6550 case ISD::UREM: in LowerOperation()
6551 case ISD::BSWAP: in LowerOperation()
6552 case ISD::CTPOP: in LowerOperation()
6554 case ISD::SHL: in LowerOperation()
6555 case ISD::SRA: in LowerOperation()
6556 case ISD::SRL: in LowerOperation()
6563 case ISD::FADD: in LowerOperation()
6564 case ISD::FSUB: in LowerOperation()
6565 case ISD::FMUL: in LowerOperation()
6566 case ISD::FDIV: in LowerOperation()
6567 case ISD::FNEG: in LowerOperation()
6568 case ISD::FABS: in LowerOperation()
6569 case ISD::FSQRT: in LowerOperation()
6570 case ISD::FMA: in LowerOperation()
6571 case ISD::FMINNUM: in LowerOperation()
6572 case ISD::FMAXNUM: in LowerOperation()
6578 case ISD::AVGFLOORU: in LowerOperation()
6579 case ISD::AVGCEILU: in LowerOperation()
6580 case ISD::SADDSAT: in LowerOperation()
6581 case ISD::UADDSAT: in LowerOperation()
6582 case ISD::SSUBSAT: in LowerOperation()
6583 case ISD::USUBSAT: in LowerOperation()
6584 case ISD::SMIN: in LowerOperation()
6585 case ISD::SMAX: in LowerOperation()
6586 case ISD::UMIN: in LowerOperation()
6587 case ISD::UMAX: in LowerOperation()
6589 case ISD::ABS: in LowerOperation()
6590 case ISD::VP_ABS: in LowerOperation()
6592 case ISD::CTLZ: in LowerOperation()
6593 case ISD::CTLZ_ZERO_UNDEF: in LowerOperation()
6594 case ISD::CTTZ: in LowerOperation()
6595 case ISD::CTTZ_ZERO_UNDEF: in LowerOperation()
6598 assert(Op.getOpcode() != ISD::CTTZ); in LowerOperation()
6600 case ISD::VSELECT: in LowerOperation()
6602 case ISD::FCOPYSIGN: in LowerOperation()
6608 case ISD::STRICT_FADD: in LowerOperation()
6609 case ISD::STRICT_FSUB: in LowerOperation()
6610 case ISD::STRICT_FMUL: in LowerOperation()
6611 case ISD::STRICT_FDIV: in LowerOperation()
6612 case ISD::STRICT_FSQRT: in LowerOperation()
6613 case ISD::STRICT_FMA: in LowerOperation()
6619 case ISD::STRICT_FSETCC: in LowerOperation()
6620 case ISD::STRICT_FSETCCS: in LowerOperation()
6622 case ISD::STRICT_FCEIL: in LowerOperation()
6623 case ISD::STRICT_FRINT: in LowerOperation()
6624 case ISD::STRICT_FFLOOR: in LowerOperation()
6625 case ISD::STRICT_FTRUNC: in LowerOperation()
6626 case ISD::STRICT_FNEARBYINT: in LowerOperation()
6627 case ISD::STRICT_FROUND: in LowerOperation()
6628 case ISD::STRICT_FROUNDEVEN: in LowerOperation()
6630 case ISD::MGATHER: in LowerOperation()
6631 case ISD::VP_GATHER: in LowerOperation()
6633 case ISD::MSCATTER: in LowerOperation()
6634 case ISD::VP_SCATTER: in LowerOperation()
6636 case ISD::GET_ROUNDING: in LowerOperation()
6638 case ISD::SET_ROUNDING: in LowerOperation()
6640 case ISD::EH_DWARF_CFA: in LowerOperation()
6642 case ISD::VP_SELECT: in LowerOperation()
6643 case ISD::VP_MERGE: in LowerOperation()
6644 case ISD::VP_ADD: in LowerOperation()
6645 case ISD::VP_SUB: in LowerOperation()
6646 case ISD::VP_MUL: in LowerOperation()
6647 case ISD::VP_SDIV: in LowerOperation()
6648 case ISD::VP_UDIV: in LowerOperation()
6649 case ISD::VP_SREM: in LowerOperation()
6650 case ISD::VP_UREM: in LowerOperation()
6652 case ISD::VP_AND: in LowerOperation()
6653 case ISD::VP_OR: in LowerOperation()
6654 case ISD::VP_XOR: in LowerOperation()
6656 case ISD::VP_FADD: in LowerOperation()
6657 case ISD::VP_FSUB: in LowerOperation()
6658 case ISD::VP_FMUL: in LowerOperation()
6659 case ISD::VP_FDIV: in LowerOperation()
6660 case ISD::VP_FNEG: in LowerOperation()
6661 case ISD::VP_FABS: in LowerOperation()
6662 case ISD::VP_SQRT: in LowerOperation()
6663 case ISD::VP_FMA: in LowerOperation()
6664 case ISD::VP_FMINNUM: in LowerOperation()
6665 case ISD::VP_FMAXNUM: in LowerOperation()
6666 case ISD::VP_FCOPYSIGN: in LowerOperation()
6672 case ISD::VP_ASHR: in LowerOperation()
6673 case ISD::VP_LSHR: in LowerOperation()
6674 case ISD::VP_SHL: in LowerOperation()
6676 case ISD::VP_IS_FPCLASS: in LowerOperation()
6678 case ISD::VP_SIGN_EXTEND: in LowerOperation()
6679 case ISD::VP_ZERO_EXTEND: in LowerOperation()
6683 case ISD::VP_TRUNCATE: in LowerOperation()
6685 case ISD::VP_FP_EXTEND: in LowerOperation()
6686 case ISD::VP_FP_ROUND: in LowerOperation()
6688 case ISD::VP_SINT_TO_FP: in LowerOperation()
6689 case ISD::VP_UINT_TO_FP: in LowerOperation()
6702 return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), NC, in LowerOperation()
6706 case ISD::VP_FP_TO_SINT: in LowerOperation()
6707 case ISD::VP_FP_TO_UINT: in LowerOperation()
6719 SDValue WidenVec = DAG.getNode(ISD::FP_EXTEND, DL, NVT, Op1); in LowerOperation()
6725 case ISD::VP_SETCC: in LowerOperation()
6733 case ISD::VP_SMIN: in LowerOperation()
6734 case ISD::VP_SMAX: in LowerOperation()
6735 case ISD::VP_UMIN: in LowerOperation()
6736 case ISD::VP_UMAX: in LowerOperation()
6737 case ISD::VP_BITREVERSE: in LowerOperation()
6738 case ISD::VP_BSWAP: in LowerOperation()
6740 case ISD::VP_CTLZ: in LowerOperation()
6741 case ISD::VP_CTLZ_ZERO_UNDEF: in LowerOperation()
6745 case ISD::VP_CTTZ: in LowerOperation()
6746 case ISD::VP_CTTZ_ZERO_UNDEF: in LowerOperation()
6750 case ISD::VP_CTPOP: in LowerOperation()
6752 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: in LowerOperation()
6754 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: in LowerOperation()
6756 case ISD::VP_FCEIL: in LowerOperation()
6757 case ISD::VP_FFLOOR: in LowerOperation()
6758 case ISD::VP_FRINT: in LowerOperation()
6759 case ISD::VP_FNEARBYINT: in LowerOperation()
6760 case ISD::VP_FROUND: in LowerOperation()
6761 case ISD::VP_FROUNDEVEN: in LowerOperation()
6762 case ISD::VP_FROUNDTOZERO: in LowerOperation()
6768 case ISD::VP_FMAXIMUM: in LowerOperation()
6769 case ISD::VP_FMINIMUM: in LowerOperation()
6775 case ISD::EXPERIMENTAL_VP_SPLICE: in LowerOperation()
6777 case ISD::EXPERIMENTAL_VP_REVERSE: in LowerOperation()
6930 return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg); in getStaticTLSAddr()
7036 ISD::CondCode CC, SDValue Val) { in matchSetCC()
7037 assert(Val->getOpcode() == ISD::SETCC); in matchSetCC()
7040 ISD::CondCode CC2 = cast<CondCodeSDNode>(Val.getOperand(2))->get(); in matchSetCC()
7045 if (CC == ISD::getSetCCInverse(CC2, LHS2.getValueType())) in matchSetCC()
7048 CC2 = ISD::getSetCCSwappedOperands(CC2); in matchSetCC()
7051 if (CC == ISD::getSetCCInverse(CC2, LHS2.getValueType())) in matchSetCC()
7070 return DAG.getNode(ISD::OR, DL, VT, Neg, FalseV); in combineSelectToBinOp()
7074 SDValue Neg = DAG.getNode(ISD::ADD, DL, VT, CondV, in combineSelectToBinOp()
7076 return DAG.getNode(ISD::OR, DL, VT, Neg, TrueV); in combineSelectToBinOp()
7081 SDValue Neg = DAG.getNode(ISD::ADD, DL, VT, CondV, in combineSelectToBinOp()
7083 return DAG.getNode(ISD::AND, DL, VT, Neg, FalseV); in combineSelectToBinOp()
7088 return DAG.getNode(ISD::AND, DL, VT, Neg, TrueV); in combineSelectToBinOp()
7094 if (CondV.getOpcode() == ISD::SETCC && TrueV.getOpcode() == ISD::SETCC && in combineSelectToBinOp()
7095 FalseV.getOpcode() == ISD::SETCC) { in combineSelectToBinOp()
7098 ISD::CondCode CC = cast<CondCodeSDNode>(CondV.getOperand(2))->get(); in combineSelectToBinOp()
7103 return DAG.getNode(*MatchResult ? ISD::OR : ISD::AND, DL, VT, TrueV, in combineSelectToBinOp()
7109 return DAG.getNode(*MatchResult ? ISD::AND : ISD::OR, DL, VT, TrueV, in combineSelectToBinOp()
7131 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) { in foldBinOpIntoSelectIfProfitable()
7136 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) in foldBinOpIntoSelectIfProfitable()
7194 return DAG.getNode(ISD::VSELECT, DL, VT, CondSplat, TrueV, FalseV); in lowerSELECT()
7211 if (TrueV.getOpcode() == ISD::AND && in lowerSELECT()
7214 ISD::OR, DL, VT, TrueV, in lowerSELECT()
7217 if (FalseV.getOpcode() == ISD::AND && in lowerSELECT()
7220 ISD::OR, DL, VT, FalseV, in lowerSELECT()
7231 ISD::OR, DL, VT, in lowerSELECT()
7257 return DAG.getNode(ISD::SINT_TO_FP, DL, VT, CondV); in lowerSELECT()
7259 SDValue XOR = DAG.getNode(ISD::XOR, DL, XLenVT, CondV, in lowerSELECT()
7261 return DAG.getNode(ISD::SINT_TO_FP, DL, VT, XOR); in lowerSELECT()
7269 if (CondV.getOpcode() != ISD::SETCC || in lowerSELECT()
7272 SDValue SetNE = DAG.getCondCode(ISD::SETNE); in lowerSELECT()
7286 ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get(); in lowerSELECT()
7296 CCVal == ISD::SETLT) { in lowerSELECT()
7300 return DAG.getNode(ISD::ADD, DL, VT, CondV, FalseV); in lowerSELECT()
7302 return DAG.getNode(ISD::SUB, DL, VT, FalseV, CondV); in lowerSELECT()
7307 if (isOneConstant(LHS) && (CCVal == ISD::SETLT || CCVal == ISD::SETULT) && in lowerSELECT()
7311 if (CCVal == ISD::SETULT) { in lowerSELECT()
7313 CCVal = ISD::SETNE; in lowerSELECT()
7318 if (isAllOnesConstant(RHS) && CCVal == ISD::SETLT && LHS == TrueV && in lowerSELECT()
7329 TargetCC = DAG.getCondCode(ISD::getSetCCInverse(CCVal, LHS.getValueType())); in lowerSELECT()
7341 if (CondV.getOpcode() == ISD::SETCC && in lowerBRCOND()
7345 ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get(); in lowerBRCOND()
7356 DAG.getCondCode(ISD::SETNE), Op.getOperand(2)); in lowerBRCOND()
7389 SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr, in lowerFRAMEADDR()
7417 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset), in lowerRETURNADDR()
7446 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen); in lowerShiftLeftParts()
7447 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt); in lowerShiftLeftParts()
7449 SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt); in lowerShiftLeftParts()
7450 SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One); in lowerShiftLeftParts()
7452 DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt); in lowerShiftLeftParts()
7453 SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt); in lowerShiftLeftParts()
7454 SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo); in lowerShiftLeftParts()
7455 SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen); in lowerShiftLeftParts()
7457 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT); in lowerShiftLeftParts()
7459 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero); in lowerShiftLeftParts()
7460 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse); in lowerShiftLeftParts()
7490 unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL; in lowerShiftRightParts()
7496 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen); in lowerShiftRightParts()
7497 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt); in lowerShiftRightParts()
7499 SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt); in lowerShiftRightParts()
7500 SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One); in lowerShiftRightParts()
7502 DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt); in lowerShiftRightParts()
7503 SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi); in lowerShiftRightParts()
7507 IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero; in lowerShiftRightParts()
7509 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT); in lowerShiftRightParts()
7511 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse); in lowerShiftRightParts()
7512 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse); in lowerShiftRightParts()
7526 if (ISD::isConstantSplatVectorAllOnes(Op.getNode())) { in lowerVectorMaskSplat()
7530 if (ISD::isConstantSplatVectorAllZeros(Op.getNode())) { in lowerVectorMaskSplat()
7535 SplatVal = DAG.getNode(ISD::AND, DL, SplatVal.getValueType(), SplatVal, in lowerVectorMaskSplat()
7539 return DAG.getSetCC(DL, VT, LHS, Zero, ISD::SETNE); in lowerVectorMaskSplat()
7588 return DAG.getNode(ISD::VSELECT, DL, VecVT, Src, SplatTrueVal, SplatZero); in lowerVectorMaskExt()
7646 bool IsVPTrunc = Op.getOpcode() == ISD::VP_TRUNCATE; in lowerVectorMaskTruncLike()
7689 {Trunc, SplatZero, DAG.getCondCode(ISD::SETNE), in lowerVectorMaskTruncLike()
7698 bool IsVPTrunc = Op.getOpcode() == ISD::VP_TRUNCATE; in lowerVectorTruncLike()
7782 unsigned InterConvOpc = Op.getOpcode() == ISD::STRICT_FP_EXTEND in lowerStrictFPExtendOrRoundLike()
7791 unsigned ConvOpc = Op.getOpcode() == ISD::STRICT_FP_EXTEND in lowerStrictFPExtendOrRoundLike()
7809 Op.getOpcode() == ISD::VP_FP_ROUND || Op.getOpcode() == ISD::VP_FP_EXTEND; in lowerVectorFPExtendOrRoundLike()
7811 Op.getOpcode() == ISD::VP_FP_EXTEND || Op.getOpcode() == ISD::FP_EXTEND; in lowerVectorFPExtendOrRoundLike()
7918 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec); in lowerINSERT_VECTOR_ELT()
7919 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideVT, Vec, Val, Idx); in lowerINSERT_VECTOR_ELT()
7920 return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Vec); in lowerINSERT_VECTOR_ELT()
7963 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec, in lowerINSERT_VECTOR_ELT()
7991 Val = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Val); in lowerINSERT_VECTOR_ELT()
7995 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, OrigContainerVT, OrigVec, in lowerINSERT_VECTOR_ELT()
8031 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, OrigContainerVT, OrigVec, in lowerINSERT_VECTOR_ELT()
8053 DAG.getNode(ISD::ADD, DL, XLenVT, Idx, DAG.getConstant(1, DL, XLenVT)); in lowerINSERT_VECTOR_ELT()
8064 Slideup = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, OrigContainerVT, OrigVec, in lowerINSERT_VECTOR_ELT()
8096 DAG.getConstant(0, DL, XLenVT), ISD::SETEQ); in lowerEXTRACT_VECTOR_ELT()
8097 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Res); in lowerEXTRACT_VECTOR_ELT()
8121 ISD::SRL, DL, XLenVT, Idx, in lowerEXTRACT_VECTOR_ELT()
8125 ISD::AND, DL, XLenVT, Idx, in lowerEXTRACT_VECTOR_ELT()
8129 Vec = DAG.getNode(ISD::BITCAST, DL, WideVT, Vec); in lowerEXTRACT_VECTOR_ELT()
8130 SDValue ExtractElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, XLenVT, in lowerEXTRACT_VECTOR_ELT()
8134 DAG.getNode(ISD::SRL, DL, XLenVT, ExtractElt, ExtractBitIdx); in lowerEXTRACT_VECTOR_ELT()
8135 SDValue Res = DAG.getNode(ISD::AND, DL, XLenVT, ShiftRight, in lowerEXTRACT_VECTOR_ELT()
8137 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Res); in lowerEXTRACT_VECTOR_ELT()
8142 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec); in lowerEXTRACT_VECTOR_ELT()
8143 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec, Idx); in lowerEXTRACT_VECTOR_ELT()
8170 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, Vec, in lowerEXTRACT_VECTOR_ELT()
8187 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec, in lowerEXTRACT_VECTOR_ELT()
8218 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec, in lowerEXTRACT_VECTOR_ELT()
8223 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Elt0); in lowerEXTRACT_VECTOR_ELT()
8230 assert((Op.getOpcode() == ISD::INTRINSIC_VOID || in lowerVectorIntrinsicScalars()
8231 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in lowerVectorIntrinsicScalars()
8232 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) && in lowerVectorIntrinsicScalars()
8238 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_VOID || in lowerVectorIntrinsicScalars()
8239 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN; in lowerVectorIntrinsicScalars()
8268 isa<ConstantSDNode>(ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND; in lowerVectorIntrinsicScalars()
8289 ScalarOp = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, ScalarOp); in lowerVectorIntrinsicScalars()
8329 I32VL = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, SETVLMAX, SEW, in lowerVectorIntrinsicScalars()
8346 SDValue VL = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, SETVL, AVL, in lowerVectorIntrinsicScalars()
8349 DAG.getNode(ISD::SHL, DL, XLenVT, VL, DAG.getConstant(1, DL, XLenVT)); in lowerVectorIntrinsicScalars()
8443 SDValue AVL = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, N->getOperand(1)); in lowerGetVectorLength()
8447 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, ID, AVL, Sew, LMul); in lowerGetVectorLength()
8448 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), Res); in lowerGetVectorLength()
8516 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
8518 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res); in LowerINTRINSIC_WO_CHAIN()
8530 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
8532 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2)); in LowerINTRINSIC_WO_CHAIN()
8535 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res); in LowerINTRINSIC_WO_CHAIN()
8550 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
8552 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2)); in LowerINTRINSIC_WO_CHAIN()
8554 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res); in LowerINTRINSIC_WO_CHAIN()
8564 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
8566 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2)); in LowerINTRINSIC_WO_CHAIN()
8567 NewOp0 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0, in LowerINTRINSIC_WO_CHAIN()
8569 NewOp1 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp1, in LowerINTRINSIC_WO_CHAIN()
8572 Res = DAG.getNode(ISD::SRL, DL, MVT::i64, Res, in LowerINTRINSIC_WO_CHAIN()
8574 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res); in LowerINTRINSIC_WO_CHAIN()
8583 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Res); in LowerINTRINSIC_WO_CHAIN()
8586 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getValueType(), in LowerINTRINSIC_WO_CHAIN()
8599 Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Scalar); in LowerINTRINSIC_WO_CHAIN()
8637 {VID, SplattedIdx, DAG.getCondCode(ISD::SETEQ), in LowerINTRINSIC_WO_CHAIN()
8713 SDValue NewNode = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, RetVT, Ops); in LowerINTRINSIC_WO_CHAIN()
8743 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode()); in LowerINTRINSIC_W_CHAIN()
8770 DAG.getExtLoad(ISD::ZEXTLOAD, DL, XLenVT, Load->getChain(), Ptr, in LowerINTRINSIC_W_CHAIN()
8798 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, in LowerINTRINSIC_W_CHAIN()
8837 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, in LowerINTRINSIC_W_CHAIN()
8873 SDValue NewNode = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops); in LowerINTRINSIC_W_CHAIN()
8907 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode()); in LowerINTRINSIC_VOID()
8937 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, Store->getVTList(), in LowerINTRINSIC_VOID()
8974 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops, in LowerINTRINSIC_VOID()
9037 DAG.getNode(ISD::INTRINSIC_VOID, SDLoc(Op), Op->getVTList(), Ops); in LowerINTRINSIC_VOID()
9053 case ISD::VP_REDUCE_ADD: in getRVVReductionOp()
9054 case ISD::VECREDUCE_ADD: in getRVVReductionOp()
9056 case ISD::VP_REDUCE_UMAX: in getRVVReductionOp()
9057 case ISD::VECREDUCE_UMAX: in getRVVReductionOp()
9059 case ISD::VP_REDUCE_SMAX: in getRVVReductionOp()
9060 case ISD::VECREDUCE_SMAX: in getRVVReductionOp()
9062 case ISD::VP_REDUCE_UMIN: in getRVVReductionOp()
9063 case ISD::VECREDUCE_UMIN: in getRVVReductionOp()
9065 case ISD::VP_REDUCE_SMIN: in getRVVReductionOp()
9066 case ISD::VECREDUCE_SMIN: in getRVVReductionOp()
9068 case ISD::VP_REDUCE_AND: in getRVVReductionOp()
9069 case ISD::VECREDUCE_AND: in getRVVReductionOp()
9071 case ISD::VP_REDUCE_OR: in getRVVReductionOp()
9072 case ISD::VECREDUCE_OR: in getRVVReductionOp()
9074 case ISD::VP_REDUCE_XOR: in getRVVReductionOp()
9075 case ISD::VECREDUCE_XOR: in getRVVReductionOp()
9077 case ISD::VP_REDUCE_FADD: in getRVVReductionOp()
9079 case ISD::VP_REDUCE_SEQ_FADD: in getRVVReductionOp()
9081 case ISD::VP_REDUCE_FMAX: in getRVVReductionOp()
9083 case ISD::VP_REDUCE_FMIN: in getRVVReductionOp()
9095 assert((Op.getOpcode() == ISD::VECREDUCE_AND || in lowerVectorMaskVecReduction()
9096 Op.getOpcode() == ISD::VECREDUCE_OR || in lowerVectorMaskVecReduction()
9097 Op.getOpcode() == ISD::VECREDUCE_XOR || in lowerVectorMaskVecReduction()
9098 Op.getOpcode() == ISD::VP_REDUCE_AND || in lowerVectorMaskVecReduction()
9099 Op.getOpcode() == ISD::VP_REDUCE_OR || in lowerVectorMaskVecReduction()
9100 Op.getOpcode() == ISD::VP_REDUCE_XOR) && in lowerVectorMaskVecReduction()
9121 ISD::CondCode CC; in lowerVectorMaskVecReduction()
9127 case ISD::VECREDUCE_AND: in lowerVectorMaskVecReduction()
9128 case ISD::VP_REDUCE_AND: { in lowerVectorMaskVecReduction()
9133 CC = ISD::SETEQ; in lowerVectorMaskVecReduction()
9134 BaseOpc = ISD::AND; in lowerVectorMaskVecReduction()
9137 case ISD::VECREDUCE_OR: in lowerVectorMaskVecReduction()
9138 case ISD::VP_REDUCE_OR: in lowerVectorMaskVecReduction()
9141 CC = ISD::SETNE; in lowerVectorMaskVecReduction()
9142 BaseOpc = ISD::OR; in lowerVectorMaskVecReduction()
9144 case ISD::VECREDUCE_XOR: in lowerVectorMaskVecReduction()
9145 case ISD::VP_REDUCE_XOR: { in lowerVectorMaskVecReduction()
9149 Vec = DAG.getNode(ISD::AND, DL, XLenVT, Vec, One); in lowerVectorMaskVecReduction()
9150 CC = ISD::SETNE; in lowerVectorMaskVecReduction()
9151 BaseOpc = ISD::XOR; in lowerVectorMaskVecReduction()
9157 SetCC = DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), SetCC); in lowerVectorMaskVecReduction()
9199 InitialValue = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, M1VT, in lowerReductionSeq()
9206 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Reduction, in lowerReductionSeq()
9216 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op.getOpcode()); in lowerVECREDUCE()
9246 case ISD::AND: in lowerVECREDUCE()
9247 case ISD::OR: in lowerVECREDUCE()
9248 case ISD::UMAX: in lowerVECREDUCE()
9249 case ISD::UMIN: in lowerVECREDUCE()
9250 case ISD::SMAX: in lowerVECREDUCE()
9251 case ISD::SMIN: in lowerVECREDUCE()
9253 StartV = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Vec, in lowerVECREDUCE()
9272 case ISD::VECREDUCE_FADD: { in getRVVFPReductionOpAndOperands()
9278 case ISD::VECREDUCE_SEQ_FADD: in getRVVFPReductionOpAndOperands()
9281 case ISD::VECREDUCE_FMIN: in getRVVFPReductionOpAndOperands()
9282 case ISD::VECREDUCE_FMAX: { in getRVVFPReductionOpAndOperands()
9285 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op.getOperand(0), in getRVVFPReductionOpAndOperands()
9287 unsigned RVVOpc = (Opcode == ISD::VECREDUCE_FMIN) in getRVVFPReductionOpAndOperands()
9382 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec); in lowerINSERT_SUBVECTOR()
9383 SubVec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtSubVecVT, SubVec); in lowerINSERT_SUBVECTOR()
9384 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ExtVecVT, Vec, SubVec, in lowerINSERT_SUBVECTOR()
9387 return DAG.getSetCC(DL, VecVT, Vec, SplatZero, ISD::SETNE); in lowerINSERT_SUBVECTOR()
9406 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, in lowerINSERT_SUBVECTOR()
9413 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, in lowerINSERT_SUBVECTOR()
9481 AlignedExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec, in lowerINSERT_SUBVECTOR()
9485 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InterSubVT, in lowerINSERT_SUBVECTOR()
9503 VL = DAG.getNode(ISD::ADD, DL, XLenVT, SlideupAmt, VL); in lowerINSERT_SUBVECTOR()
9512 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, Vec, SubVec, in lowerINSERT_SUBVECTOR()
9560 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec); in lowerEXTRACT_SUBVECTOR()
9561 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtSubVecVT, Vec, in lowerEXTRACT_SUBVECTOR()
9564 return DAG.getSetCC(DL, SubVecVT, Vec, SplatZero, ISD::SETNE); in lowerEXTRACT_SUBVECTOR()
9590 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec, in lowerEXTRACT_SUBVECTOR()
9605 Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown, in lowerEXTRACT_SUBVECTOR()
9647 Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown, in lowerEXTRACT_SUBVECTOR()
9665 WideOps.push_back(DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Op)); in widenVectorOpsToi8()
9677 DAG.getConstant(0, DL, WideVT), ISD::SETNE)); in widenVectorOpsToi8()
9705 SDValue ResLo = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, in lowerVECTOR_DEINTERLEAVE()
9707 SDValue ResHi = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, in lowerVECTOR_DEINTERLEAVE()
9710 SDValue Even = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, in lowerVECTOR_DEINTERLEAVE()
9712 SDValue Odd = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, ResLo.getValue(1), in lowerVECTOR_DEINTERLEAVE()
9721 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, in lowerVECTOR_DEINTERLEAVE()
9745 DAG.getNode(ISD::ADD, DL, IdxVT, EvenIdx, DAG.getConstant(1, DL, IdxVT)); in lowerVECTOR_DEINTERLEAVE()
9754 SDValue Even = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, EvenWide, in lowerVECTOR_DEINTERLEAVE()
9756 SDValue Odd = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, OddWide, in lowerVECTOR_DEINTERLEAVE()
9783 SDValue ResLo = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, in lowerVECTOR_INTERLEAVE()
9785 SDValue ResHi = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, in lowerVECTOR_INTERLEAVE()
9788 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, in lowerVECTOR_INTERLEAVE()
9790 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, in lowerVECTOR_INTERLEAVE()
9807 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, in lowerVECTOR_INTERLEAVE()
9819 SDValue OddMask = DAG.getNode(ISD::AND, DL, IdxVT, StepVec, Ones); in lowerVECTOR_INTERLEAVE()
9823 ISD::CondCode::SETNE); in lowerVECTOR_INTERLEAVE()
9829 SDValue Idx = DAG.getNode(ISD::SRL, DL, IdxVT, StepVec, Ones); in lowerVECTOR_INTERLEAVE()
9842 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, Interleaved, in lowerVECTOR_INTERLEAVE()
9845 ISD::EXTRACT_SUBVECTOR, DL, VecVT, Interleaved, in lowerVECTOR_INTERLEAVE()
9867 StepVec = DAG.getNode(ISD::SHL, DL, VT, StepVec, StepVal); in lowerSTEP_VECTOR()
9872 StepVec = DAG.getNode(ISD::MUL, DL, VT, StepVec, StepVal); in lowerSTEP_VECTOR()
9889 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, Op.getOperand(0)); in lowerVECTOR_REVERSE()
9890 SDValue Op2 = DAG.getNode(ISD::VECTOR_REVERSE, DL, WidenVT, Op1); in lowerVECTOR_REVERSE()
9891 return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Op2); in lowerVECTOR_REVERSE()
9914 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo); in lowerVECTOR_REVERSE()
9915 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi); in lowerVECTOR_REVERSE()
9919 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, DAG.getUNDEF(VecVT), Hi, in lowerVECTOR_REVERSE()
9922 ISD::INSERT_SUBVECTOR, DL, VecVT, Res, Lo, in lowerVECTOR_REVERSE()
9935 SDValue VLMinus1 = DAG.getNode(ISD::SUB, DL, XLenVT, in lowerVECTOR_REVERSE()
9973 UpOffset = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, DownOffset); in lowerVECTOR_SPLICE()
9978 DownOffset = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, UpOffset); in lowerVECTOR_SPLICE()
10031 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, in lowerFixedLengthVectorLoadToRVV()
10056 StoreVal = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in lowerFixedLengthVectorStoreToRVV()
10083 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), in lowerFixedLengthVectorStoreToRVV()
10110 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode()); in lowerMaskedLoad()
10144 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO); in lowerMaskedLoad()
10174 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode()); in lowerMaskedStore()
10202 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, in lowerMaskedStore()
10239 ISD::CondCode CCVal = cast<CondCodeSDNode>(CC)->get(); in lowerVectorStrictFSetcc()
10245 if (Opc == ISD::STRICT_FSETCCS) { in lowerVectorStrictFSetcc()
10249 if (CCVal == ISD::SETEQ || CCVal == ISD::SETOEQ) { in lowerVectorStrictFSetcc()
10250 SDValue OLECCVal = DAG.getCondCode(ISD::SETOLE); in lowerVectorStrictFSetcc()
10251 SDValue Tmp1 = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op1, in lowerVectorStrictFSetcc()
10253 SDValue Tmp2 = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op2, in lowerVectorStrictFSetcc()
10255 SDValue OutChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in lowerVectorStrictFSetcc()
10259 Tmp1 = DAG.getNode(ISD::AND, DL, VT, Tmp1, Tmp2); in lowerVectorStrictFSetcc()
10264 if (CCVal == ISD::SETNE || CCVal == ISD::SETUNE) { in lowerVectorStrictFSetcc()
10265 SDValue OEQCCVal = DAG.getCondCode(ISD::SETOEQ); in lowerVectorStrictFSetcc()
10266 SDValue OEQ = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op1, in lowerVectorStrictFSetcc()
10284 if (Opc == ISD::STRICT_FSETCC && in lowerVectorStrictFSetcc()
10285 (CCVal == ISD::SETLT || CCVal == ISD::SETOLT || CCVal == ISD::SETLE || in lowerVectorStrictFSetcc()
10286 CCVal == ISD::SETOLE)) { in lowerVectorStrictFSetcc()
10292 {Chain, Op1, Op1, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT), in lowerVectorStrictFSetcc()
10296 {Chain, Op2, Op2, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT), in lowerVectorStrictFSetcc()
10306 unsigned RVVOpc = Opc == ISD::STRICT_FSETCC ? RISCVISD::STRICT_FSETCC_VL in lowerVectorStrictFSetcc()
10325 assert((Op.getOpcode() == ISD::VP_ABS || VT.isFixedLengthVector()) && in lowerABS()
10335 if (Op->getOpcode() == ISD::VP_ABS) { in lowerABS()
10474 auto MaskIdx = ISD::getVPMaskIdx(Op.getOpcode()); in lowerVPOp()
10478 } else if (ISD::getVPExplicitVectorLengthIdx(Op.getOpcode()) == in lowerVPOp()
10480 if (Op.getOpcode() == ISD::VP_MERGE) { in lowerVPOp()
10484 assert(Op.getOpcode() == ISD::VP_SELECT); in lowerVPOp()
10533 Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1, DL, XLenVT); in lowerVPExtMaskOp()
10551 ISD::CondCode Condition = cast<CondCodeSDNode>(Op.getOperand(2))->get(); in lowerVPSetCCMaskOp()
10569 case ISD::SETNE: in lowerVPSetCCMaskOp()
10573 case ISD::SETEQ: { in lowerVPSetCCMaskOp()
10582 case ISD::SETGT: in lowerVPSetCCMaskOp()
10583 case ISD::SETULT: { in lowerVPSetCCMaskOp()
10591 case ISD::SETLT: in lowerVPSetCCMaskOp()
10592 case ISD::SETUGT: { in lowerVPSetCCMaskOp()
10600 case ISD::SETGE: in lowerVPSetCCMaskOp()
10601 case ISD::SETULE: { in lowerVPSetCCMaskOp()
10609 case ISD::SETLE: in lowerVPSetCCMaskOp()
10610 case ISD::SETUGE: { in lowerVPSetCCMaskOp()
10731 {Result, SplatZero, DAG.getCondCode(ISD::SETNE), in lowerVPFPIntConvOp()
10810 UpOffset = DAG.getNode(ISD::SUB, DL, XLenVT, EVL1, DownOffset); in lowerVPSpliceExperimental()
10815 DownOffset = DAG.getNode(ISD::SUB, DL, XLenVT, EVL1, UpOffset); in lowerVPSpliceExperimental()
10829 DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)), in lowerVPSpliceExperimental()
10897 SDValue LoRev = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo); in lowerVPReverseExperimental()
10898 SDValue HiRev = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi); in lowerVPReverseExperimental()
10905 DAG.getNode(ISD::CONCAT_VECTORS, DL, GatherVT, HiRev, LoRev); in lowerVPReverseExperimental()
10910 SDValue VLMax = DAG.getNode(ISD::VSCALE, DL, XLenVT, in lowerVPReverseExperimental()
10912 SDValue Diff = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, EVL); in lowerVPReverseExperimental()
10922 DAG.getCondCode(ISD::SETNE), in lowerVPReverseExperimental()
10938 DAG.getNode(ISD::SUB, DL, XLenVT, EVL, DAG.getConstant(1, DL, XLenVT)); in lowerVPReverseExperimental()
10950 {Result, DAG.getConstant(0, DL, GatherVT), DAG.getCondCode(ISD::SETNE), in lowerVPReverseExperimental()
10999 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode()); in lowerVPStridedLoad()
11021 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, in lowerVPStridedLoad()
11047 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode()); in lowerVPStridedStore()
11063 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, VPNode->getVTList(), in lowerVPStridedStore()
11085 ISD::LoadExtType LoadExtType; in lowerMaskedGather()
11094 LoadExtType = ISD::NON_EXTLOAD; in lowerMaskedGather()
11111 assert(LoadExtType == ISD::NON_EXTLOAD && in lowerMaskedGather()
11117 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode()); in lowerMaskedGather()
11139 Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index); in lowerMaskedGather()
11159 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO); in lowerMaskedGather()
11216 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode()); in lowerMaskedScatter()
11238 Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index); in lowerMaskedScatter()
11251 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, in lowerMaskedScatter()
11277 DAG.getNode(ISD::SHL, DL, XLenVT, RM, DAG.getConstant(2, DL, XLenVT)); in lowerGET_ROUNDING()
11278 SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT, in lowerGET_ROUNDING()
11280 SDValue Masked = DAG.getNode(ISD::AND, DL, XLenVT, Shifted, in lowerGET_ROUNDING()
11306 RMValue = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, RMValue); in lowerSET_ROUNDING()
11308 SDValue Shift = DAG.getNode(ISD::SHL, DL, XLenVT, RMValue, in lowerSET_ROUNDING()
11310 SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT, in lowerSET_ROUNDING()
11312 RMValue = DAG.getNode(ISD::AND, DL, XLenVT, Shifted, in lowerSET_ROUNDING()
11335 case ISD::SHL: in getRISCVWOpcode()
11337 case ISD::SRA: in getRISCVWOpcode()
11339 case ISD::SRL: in getRISCVWOpcode()
11341 case ISD::SDIV: in getRISCVWOpcode()
11343 case ISD::UDIV: in getRISCVWOpcode()
11345 case ISD::UREM: in getRISCVWOpcode()
11347 case ISD::ROTL: in getRISCVWOpcode()
11349 case ISD::ROTR: in getRISCVWOpcode()
11360 unsigned ExtOpc = ISD::ANY_EXTEND) { in customLegalizeToWOp()
11367 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes); in customLegalizeToWOp()
11374 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)); in customLegalizeToWOpWithSExt()
11375 SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); in customLegalizeToWOpWithSExt()
11377 SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp, in customLegalizeToWOpWithSExt()
11379 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes); in customLegalizeToWOpWithSExt()
11389 case ISD::STRICT_FP_TO_SINT: in ReplaceNodeResults()
11390 case ISD::STRICT_FP_TO_UINT: in ReplaceNodeResults()
11391 case ISD::FP_TO_SINT: in ReplaceNodeResults()
11392 case ISD::FP_TO_UINT: { in ReplaceNodeResults()
11396 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT || in ReplaceNodeResults()
11397 N->getOpcode() == ISD::STRICT_FP_TO_SINT; in ReplaceNodeResults()
11408 Op0 = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other}, in ReplaceNodeResults()
11418 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11427 Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op0); in ReplaceNodeResults()
11433 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11457 case ISD::LROUND: { in ReplaceNodeResults()
11467 Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op0); in ReplaceNodeResults()
11472 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11484 Result = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Result); in ReplaceNodeResults()
11488 case ISD::READCYCLECOUNTER: { in ReplaceNodeResults()
11497 DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, RCW, RCW.getValue(1))); in ReplaceNodeResults()
11501 case ISD::LOAD: { in ReplaceNodeResults()
11502 if (!ISD::isNON_EXTLoad(N)) in ReplaceNodeResults()
11510 SDValue Res = DAG.getExtLoad(ISD::SEXTLOAD, dl, MVT::i64, Ld->getChain(), in ReplaceNodeResults()
11513 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Res)); in ReplaceNodeResults()
11517 case ISD::MUL: { in ReplaceNodeResults()
11535 S = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, S); in ReplaceNodeResults()
11536 U = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, U); in ReplaceNodeResults()
11537 SDValue Lo = DAG.getNode(ISD::MUL, DL, XLenVT, S, U); in ReplaceNodeResults()
11539 return DAG.getNode(ISD::BUILD_PAIR, DL, N->getValueType(0), Lo, Hi); in ReplaceNodeResults()
11556 case ISD::ADD: in ReplaceNodeResults()
11557 case ISD::SUB: in ReplaceNodeResults()
11562 case ISD::SHL: in ReplaceNodeResults()
11563 case ISD::SRA: in ReplaceNodeResults()
11564 case ISD::SRL: in ReplaceNodeResults()
11567 if (N->getOperand(1).getOpcode() != ISD::Constant) { in ReplaceNodeResults()
11569 if (N->getOpcode() == ISD::SHL && Subtarget.hasStdExtZbs() && in ReplaceNodeResults()
11579 if (N->getOpcode() == ISD::SHL) { in ReplaceNodeResults()
11582 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
11584 DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
11585 SDValue NewWOp = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0, NewOp1); in ReplaceNodeResults()
11586 SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp, in ReplaceNodeResults()
11588 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes)); in ReplaceNodeResults()
11592 case ISD::ROTL: in ReplaceNodeResults()
11593 case ISD::ROTR: in ReplaceNodeResults()
11604 case ISD::CTTZ: in ReplaceNodeResults()
11605 case ISD::CTTZ_ZERO_UNDEF: in ReplaceNodeResults()
11606 case ISD::CTLZ: in ReplaceNodeResults()
11607 case ISD::CTLZ_ZERO_UNDEF: { in ReplaceNodeResults()
11612 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
11614 N->getOpcode() == ISD::CTTZ || N->getOpcode() == ISD::CTTZ_ZERO_UNDEF; in ReplaceNodeResults()
11617 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11620 case ISD::SDIV: in ReplaceNodeResults()
11621 case ISD::UDIV: in ReplaceNodeResults()
11622 case ISD::UREM: { in ReplaceNodeResults()
11630 if (N->getOperand(1).getOpcode() == ISD::Constant && in ReplaceNodeResults()
11637 unsigned ExtOpc = ISD::ANY_EXTEND; in ReplaceNodeResults()
11639 ExtOpc = N->getOpcode() == ISD::SDIV ? ISD::SIGN_EXTEND in ReplaceNodeResults()
11640 : ISD::ZERO_EXTEND; in ReplaceNodeResults()
11645 case ISD::SADDO: { in ReplaceNodeResults()
11654 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
11655 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
11656 SDValue Res = DAG.getNode(ISD::ADD, DL, MVT::i64, LHS, RHS); in ReplaceNodeResults()
11657 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res, in ReplaceNodeResults()
11669 SDValue ResultLowerThanLHS = DAG.getSetCC(DL, OType, Res, LHS, ISD::SETLT); in ReplaceNodeResults()
11670 SDValue ConditionRHS = DAG.getSetCC(DL, OType, RHS, Zero, ISD::SETLT); in ReplaceNodeResults()
11673 DAG.getNode(ISD::XOR, DL, OType, ConditionRHS, ResultLowerThanLHS); in ReplaceNodeResults()
11674 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11678 case ISD::UADDO: in ReplaceNodeResults()
11679 case ISD::USUBO: { in ReplaceNodeResults()
11682 bool IsAdd = N->getOpcode() == ISD::UADDO; in ReplaceNodeResults()
11684 SDValue LHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
11685 SDValue RHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
11687 DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS); in ReplaceNodeResults()
11688 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res, in ReplaceNodeResults()
11699 DAG.getConstant(0, DL, MVT::i64), ISD::SETEQ); in ReplaceNodeResults()
11703 DAG.getConstant(0, DL, MVT::i32), ISD::SETNE); in ReplaceNodeResults()
11708 LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
11710 IsAdd ? ISD::SETULT : ISD::SETUGT); in ReplaceNodeResults()
11713 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11717 case ISD::UADDSAT: in ReplaceNodeResults()
11718 case ISD::USUBSAT: { in ReplaceNodeResults()
11726 DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
11728 DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
11730 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11739 case ISD::ABS: { in ReplaceNodeResults()
11747 SDValue Src = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, in ReplaceNodeResults()
11750 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Abs)); in ReplaceNodeResults()
11755 SDValue Src = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
11761 SDValue SignFill = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Src, in ReplaceNodeResults()
11763 SignFill = DAG.getNode(ISD::SRA, DL, MVT::i64, SignFill, in ReplaceNodeResults()
11766 SDValue NewRes = DAG.getNode(ISD::XOR, DL, MVT::i64, Src, SignFill); in ReplaceNodeResults()
11767 NewRes = DAG.getNode(ISD::SUB, DL, MVT::i64, NewRes, SignFill); in ReplaceNodeResults()
11771 NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewRes, in ReplaceNodeResults()
11773 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes)); in ReplaceNodeResults()
11776 case ISD::BITCAST: { in ReplaceNodeResults()
11785 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv)); in ReplaceNodeResults()
11789 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv)); in ReplaceNodeResults()
11794 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv)); in ReplaceNodeResults()
11799 SDValue RetReg = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, in ReplaceNodeResults()
11811 Results.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec, in ReplaceNodeResults()
11823 SDValue NewOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0)); in ReplaceNodeResults()
11827 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NewRes)); in ReplaceNodeResults()
11830 case ISD::EXTRACT_VECTOR_ELT: { in ReplaceNodeResults()
11885 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi)); in ReplaceNodeResults()
11888 case ISD::INTRINSIC_WO_CHAIN: { in ReplaceNodeResults()
11896 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11922 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
11924 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11932 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
11934 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2)); in ReplaceNodeResults()
11937 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11945 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
11947 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2)); in ReplaceNodeResults()
11949 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11967 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
11969 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2)); in ReplaceNodeResults()
11970 NewOp0 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0, in ReplaceNodeResults()
11972 NewOp1 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp1, in ReplaceNodeResults()
11977 Res = DAG.getNode(ISD::SRL, DL, MVT::i64, Res, in ReplaceNodeResults()
11979 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
11989 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Extract)); in ReplaceNodeResults()
12015 DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi)); in ReplaceNodeResults()
12021 case ISD::VECREDUCE_ADD: in ReplaceNodeResults()
12022 case ISD::VECREDUCE_AND: in ReplaceNodeResults()
12023 case ISD::VECREDUCE_OR: in ReplaceNodeResults()
12024 case ISD::VECREDUCE_XOR: in ReplaceNodeResults()
12025 case ISD::VECREDUCE_SMAX: in ReplaceNodeResults()
12026 case ISD::VECREDUCE_UMAX: in ReplaceNodeResults()
12027 case ISD::VECREDUCE_SMIN: in ReplaceNodeResults()
12028 case ISD::VECREDUCE_UMIN: in ReplaceNodeResults()
12032 case ISD::VP_REDUCE_ADD: in ReplaceNodeResults()
12033 case ISD::VP_REDUCE_AND: in ReplaceNodeResults()
12034 case ISD::VP_REDUCE_OR: in ReplaceNodeResults()
12035 case ISD::VP_REDUCE_XOR: in ReplaceNodeResults()
12036 case ISD::VP_REDUCE_SMAX: in ReplaceNodeResults()
12037 case ISD::VP_REDUCE_UMAX: in ReplaceNodeResults()
12038 case ISD::VP_REDUCE_SMIN: in ReplaceNodeResults()
12039 case ISD::VP_REDUCE_UMIN: in ReplaceNodeResults()
12043 case ISD::GET_ROUNDING: { in ReplaceNodeResults()
12045 SDValue Res = DAG.getNode(ISD::GET_ROUNDING, DL, VTs, N->getOperand(0)); in ReplaceNodeResults()
12059 case ISD::ADD: in getVecReduceOpcode()
12060 return ISD::VECREDUCE_ADD; in getVecReduceOpcode()
12061 case ISD::UMAX: in getVecReduceOpcode()
12062 return ISD::VECREDUCE_UMAX; in getVecReduceOpcode()
12063 case ISD::SMAX: in getVecReduceOpcode()
12064 return ISD::VECREDUCE_SMAX; in getVecReduceOpcode()
12065 case ISD::UMIN: in getVecReduceOpcode()
12066 return ISD::VECREDUCE_UMIN; in getVecReduceOpcode()
12067 case ISD::SMIN: in getVecReduceOpcode()
12068 return ISD::VECREDUCE_SMIN; in getVecReduceOpcode()
12069 case ISD::AND: in getVecReduceOpcode()
12070 return ISD::VECREDUCE_AND; in getVecReduceOpcode()
12071 case ISD::OR: in getVecReduceOpcode()
12072 return ISD::VECREDUCE_OR; in getVecReduceOpcode()
12073 case ISD::XOR: in getVecReduceOpcode()
12074 return ISD::VECREDUCE_XOR; in getVecReduceOpcode()
12075 case ISD::FADD: in getVecReduceOpcode()
12077 return ISD::VECREDUCE_FADD; in getVecReduceOpcode()
12111 (Opc != ISD::FADD || !N->getFlags().hasAllowReassociation())) in combineBinOpOfExtractToReduceTree()
12115 assert(Opc == ISD::getVecReduceBaseOpcode(ReduceOpc) && in combineBinOpOfExtractToReduceTree()
12123 if (RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT) in combineBinOpOfExtractToReduceTree()
12126 if (RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in combineBinOpOfExtractToReduceTree()
12144 if (LHS.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in combineBinOpOfExtractToReduceTree()
12150 SDValue Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ReduceVT, SrcVec, in combineBinOpOfExtractToReduceTree()
12163 if (ReduceVec.getOpcode() == ISD::EXTRACT_SUBVECTOR && in combineBinOpOfExtractToReduceTree()
12172 SDValue Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ReduceVT, SrcVec, in combineBinOpOfExtractToReduceTree()
12190 case ISD::ADD: in combineBinOpToReduce()
12192 case ISD::UMAX: in combineBinOpToReduce()
12194 case ISD::SMAX: in combineBinOpToReduce()
12196 case ISD::UMIN: in combineBinOpToReduce()
12198 case ISD::SMIN: in combineBinOpToReduce()
12200 case ISD::AND: in combineBinOpToReduce()
12202 case ISD::OR: in combineBinOpToReduce()
12204 case ISD::XOR: in combineBinOpToReduce()
12206 case ISD::FADD: in combineBinOpToReduce()
12208 case ISD::FMAXNUM: in combineBinOpToReduce()
12210 case ISD::FMINNUM: in combineBinOpToReduce()
12216 return V.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in combineBinOpToReduce()
12231 if (Opc == ISD::FADD && !N->getFlags().hasAllowReassociation()) in combineBinOpToReduce()
12241 if (ScalarV.getOpcode() == ISD::INSERT_SUBVECTOR && in combineBinOpToReduce()
12276 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ScalarVT, DAG.getUNDEF(ScalarVT), in combineBinOpToReduce()
12304 if (N0->getOpcode() != ISD::SHL || N1->getOpcode() != ISD::SHL || in transformAddShlImm()
12329 DAG.getNode(ISD::SHL, DL, VT, NL, DAG.getConstant(Diff, DL, VT)); in transformAddShlImm()
12330 SDValue NA1 = DAG.getNode(ISD::ADD, DL, VT, NA0, NS); in transformAddShlImm()
12331 return DAG.getNode(ISD::SHL, DL, VT, NA1, DAG.getConstant(Bits, DL, VT)); in transformAddShlImm()
12359 N->getOpcode() != ISD::AND) in combineSelectAndUse()
12363 if (Slct.getOpcode() == ISD::SELECT && !Slct.getOperand(0).hasOneUse()) in combineSelectAndUse()
12371 if ((Slct.getOpcode() != ISD::SELECT && in combineSelectAndUse()
12406 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, in combineSelectAndUse()
12449 if (!N0->hasOneUse() || N0->getOpcode() != ISD::MUL) in transformAddImmMulImm()
12483 SDValue New0 = DAG.getNode(ISD::ADD, DL, VT, N0->getOperand(0), in transformAddImmMulImm()
12486 DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getConstant(C0, DL, VT)); in transformAddImmMulImm()
12487 return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT)); in transformAddImmMulImm()
12502 if (N0.getOpcode() != ISD::XOR || !isOneConstant(N0.getOperand(1))) in combineAddOfBooleanXor()
12511 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), in combineAddOfBooleanXor()
12552 if (N1.getOpcode() == ISD::SETCC && N1.hasOneUse()) { in combineSubOfBoolean()
12555 ISD::CondCode CCVal = cast<CondCodeSDNode>(N1.getOperand(2))->get(); in combineSubOfBoolean()
12559 CCVal = ISD::getSetCCInverse(CCVal, SetCCOpVT); in combineSubOfBoolean()
12562 } else if (N1.getOpcode() == ISD::XOR && isOneConstant(N1.getOperand(1)) && in combineSubOfBoolean()
12563 N1.getOperand(0).getOpcode() == ISD::SETCC) { in combineSubOfBoolean()
12571 return DAG.getNode(ISD::ADD, DL, VT, NewLHS, NewRHS); in combineSubOfBoolean()
12582 if (isNullConstant(N0) && N1.getOpcode() == ISD::SETCC && N1.hasOneUse() && in performSUBCombine()
12584 ISD::CondCode CCVal = cast<CondCodeSDNode>(N1.getOperand(2))->get(); in performSUBCombine()
12585 if (CCVal == ISD::SETLT) { in performSUBCombine()
12589 return DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), in performSUBCombine()
12605 bool IsAnd = N->getOpcode() == ISD::AND; in combineDeMorganOfBoolean()
12607 if (N0.getOpcode() != ISD::XOR || N1.getOpcode() != ISD::XOR) in combineDeMorganOfBoolean()
12641 unsigned Opc = IsAnd ? ISD::OR : ISD::AND; in combineDeMorganOfBoolean()
12643 return DAG.getNode(ISD::XOR, DL, VT, Logic, DAG.getConstant(1, DL, VT)); in combineDeMorganOfBoolean()
12657 N0.getValueType() == MVT::i32 && N0.getOpcode() == ISD::SRL && in performTRUNCATECombine()
12660 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0)); in performTRUNCATECombine()
12661 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1)); in performTRUNCATECombine()
12662 SDValue Srl = DAG.getNode(ISD::SRL, DL, MVT::i64, Op0, Op1); in performTRUNCATECombine()
12663 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Srl); in performTRUNCATECombine()
12685 N0.getOpcode() == ISD::SRL && !isa<ConstantSDNode>(N0.getOperand(1)) && in performANDCombine()
12688 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0)); in performANDCombine()
12689 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1)); in performANDCombine()
12690 SDValue Srl = DAG.getNode(ISD::SRL, DL, MVT::i64, Op0, Op1); in performANDCombine()
12691 SDValue And = DAG.getNode(ISD::AND, DL, MVT::i64, Srl, in performANDCombine()
12693 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, And); in performANDCombine()
12714 assert(N->getOpcode() == ISD::OR && "Unexpected opcode"); in combineOrOfCZERO()
12729 if (TrueV.getOpcode() != ISD::XOR || FalseV.getOpcode() != ISD::XOR || in combineOrOfCZERO()
12742 SDValue NewOr = DAG.getNode(ISD::OR, DL, VT, NewN0, NewN1); in combineOrOfCZERO()
12743 return DAG.getNode(ISD::XOR, DL, VT, NewOr, TrueV.getOperand(1)); in combineOrOfCZERO()
12783 N0.getOpcode() == ISD::SHL && isAllOnesConstant(N0.getOperand(0)) && in performXORCombine()
12786 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0)); in performXORCombine()
12787 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1)); in performXORCombine()
12788 SDValue Shl = DAG.getNode(ISD::SHL, DL, MVT::i64, Op0, Op1); in performXORCombine()
12790 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, And); in performXORCombine()
12798 TLI.isOperationLegal(ISD::ROTL, MVT::i64)) { in performXORCombine()
12805 if (N0.getOpcode() == ISD::SETCC && isOneConstant(N1) && N0.hasOneUse()) { in performXORCombine()
12807 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in performXORCombine()
12808 if (ConstN00 && CC == ISD::SETLT) { in performXORCombine()
12845 if ((AddSubOpc == ISD::ADD || AddSubOpc == ISD::SUB) && V->hasOneUse()) { in performMULCombine()
12848 if (AddSubOpc == ISD::SUB) in performMULCombine()
12857 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N1, MulOper); in performMULCombine()
12862 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N0, MulOper); in performMULCombine()
12871 static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG) { in narrowIndex()
12888 if (ISD::isBuildVectorOfConstantSDNodes(N.getNode())) { in narrowIndex()
12894 N = DAG.getNode(ISD::TRUNCATE, DL, in narrowIndex()
12901 if (N.getOpcode() != ISD::SHL) in narrowIndex()
12905 if (N0.getOpcode() != ISD::ZERO_EXTEND && in narrowIndex()
12913 if (!ISD::isConstantSplatVector(N1.getNode(), ShAmt)) in narrowIndex()
12932 N = DAG.getNode(ISD::SHL, DL, NewVT, NewExt, NewShAmtVec); in narrowIndex()
12956 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() || in performSETCCCombine()
12962 ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(2))->get(); in performSETCCCombine()
12978 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT); in performSETCCCombine()
12980 SDValue SExtOp = DAG.getNode(ISD::SIGN_EXTEND_INREG, N, OpVT, in performSETCCCombine()
13055 case ISD::ZERO_EXTEND: in getSource()
13056 case ISD::SIGN_EXTEND: in getSource()
13091 case ISD::ZERO_EXTEND: in getOrCreateExtendedOp()
13092 case ISD::SIGN_EXTEND: in getOrCreateExtendedOp()
13132 case ISD::ADD: in getSameExtensionOpcode()
13137 case ISD::MUL: in getSameExtensionOpcode()
13140 case ISD::SUB: in getSameExtensionOpcode()
13153 assert((Opcode == RISCVISD::MUL_VL || Opcode == ISD::MUL) && in getSUOpcode()
13162 case ISD::ADD: in getWOpcode()
13165 case ISD::SUB: in getWOpcode()
13191 case ISD::ZERO_EXTEND: in fillUpExtensionSupport()
13192 case ISD::SIGN_EXTEND: { in fillUpExtensionSupport()
13211 SupportsZExt = Opc == ISD::ZERO_EXTEND; in fillUpExtensionSupport()
13212 SupportsSExt = Opc == ISD::SIGN_EXTEND; in fillUpExtensionSupport()
13275 case ISD::ADD: in isSupportedRoot()
13276 case ISD::SUB: in isSupportedRoot()
13277 case ISD::MUL: { in isSupportedRoot()
13346 case ISD::ADD: in getMaskAndVL()
13347 case ISD::SUB: in getMaskAndVL()
13348 case ISD::MUL: { in getMaskAndVL()
13369 case ISD::ADD: in isCommutative()
13370 case ISD::MUL: in isCommutative()
13376 case ISD::SUB: in isCommutative()
13430 case ISD::ADD: in materialize()
13431 case ISD::SUB: in materialize()
13432 case ISD::MUL: in materialize()
13560 case ISD::ADD: in getSupportedFoldings()
13561 case ISD::SUB: in getSupportedFoldings()
13569 case ISD::MUL: in getSupportedFoldings()
13717 if (LSNode1->getOpcode() == ISD::LOAD) { in tryMemPairCombine()
13721 Opcode = (Ext == ISD::ZEXTLOAD) ? RISCVISD::TH_LWUD : RISCVISD::TH_LWD; in tryMemPairCombine()
13766 unsigned OpNum = LSNode1->getOpcode() == ISD::LOAD ? 1 : 2; in performMemPairCombine()
13774 if (Ptr->getOpcode() == ISD::ADD) in performMemPairCombine()
13795 if (LSNode1->getOpcode() == ISD::LOAD) in performMemPairCombine()
13868 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT; in performFP_TO_INTCombine()
13934 return DAG.getNode(ISD::TRUNCATE, DL, VT, FpToInt); in performFP_TO_INTCombine()
13977 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT_SAT; in performFP_TO_INT_SATCombine()
14002 return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO); in performFP_TO_INT_SATCombine()
14012 if (Src.getOpcode() != ISD::BSWAP) in performBITREVERSECombine()
14267 assert(N->getOpcode() == ISD::SRA && "Unexpected opcode"); in performSRACombine()
14284 N0.getOpcode() == ISD::SIGN_EXTEND_INREG && N0.hasOneUse() && in performSRACombine()
14286 N0.getOperand(0).getOpcode() == ISD::SHL && N0.getOperand(0).hasOneUse() && in performSRACombine()
14291 SDValue Shl = DAG.getNode(ISD::SHL, ShlDL, MVT::i64, in performSRACombine()
14295 return DAG.getNode(ISD::SRA, DL, MVT::i64, Shl, in performSRACombine()
14310 bool IsAdd = N0.getOpcode() == ISD::ADD; in performSRACombine()
14311 if ((IsAdd || N0.getOpcode() == ISD::SUB)) { in performSRACombine()
14325 if (U->getOpcode() != ISD::SRA || in performSRACombine()
14338 if (Shl.getOpcode() != ISD::SHL || !isa<ConstantSDNode>(Shl.getOperand(1)) || in performSRACombine()
14358 In = DAG.getNode(ISD::ADD, DL, MVT::i64, In, ShiftedAddC); in performSRACombine()
14360 In = DAG.getNode(ISD::SUB, DL, MVT::i64, ShiftedAddC, In); in performSRACombine()
14363 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, In, in performSRACombine()
14369 ISD::SHL, DL, MVT::i64, SExt, in performSRACombine()
14378 bool IsAnd = Cond.getOpcode() == ISD::AND; in tryDemorganOfBooleanCondition()
14379 if (!IsAnd && Cond.getOpcode() != ISD::OR) in tryDemorganOfBooleanCondition()
14388 if (Setcc.getOpcode() != ISD::SETCC) in tryDemorganOfBooleanCondition()
14391 if (Setcc.getOpcode() != ISD::SETCC || !Setcc.hasOneUse() || in tryDemorganOfBooleanCondition()
14392 Xor.getOpcode() != ISD::XOR || !Xor.hasOneUse()) in tryDemorganOfBooleanCondition()
14414 ISD::CondCode CCVal = cast<CondCodeSDNode>(Setcc.getOperand(2))->get(); in tryDemorganOfBooleanCondition()
14415 if (ISD::isIntEqualitySetCC(CCVal)) { in tryDemorganOfBooleanCondition()
14416 CCVal = ISD::getSetCCInverse(CCVal, SetCCOpVT); in tryDemorganOfBooleanCondition()
14419 } else if (CCVal == ISD::SETLT && isNullConstant(Setcc.getOperand(0))) { in tryDemorganOfBooleanCondition()
14423 } else if (CCVal == ISD::SETLT && isOneConstant(Setcc.getOperand(1))) { in tryDemorganOfBooleanCondition()
14431 unsigned Opc = IsAnd ? ISD::OR : ISD::AND; in tryDemorganOfBooleanCondition()
14438 ISD::CondCode CCVal = cast<CondCodeSDNode>(CC)->get(); in combine_CC()
14444 if (isNullConstant(RHS) && (CCVal == ISD::SETGE || CCVal == ISD::SETLT) && in combine_CC()
14445 LHS.getOpcode() == ISD::SRA) { in combine_CC()
14450 if (!ISD::isIntEqualitySetCC(CCVal)) in combine_CC()
14455 if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) && in combine_CC()
14459 bool Invert = CCVal == ISD::SETEQ; in combine_CC()
14462 CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType()); in combine_CC()
14473 if (LHS.getOpcode() == ISD::XOR && isNullConstant(RHS)) { in combine_CC()
14480 if (isNullConstant(RHS) && LHS.getOpcode() == ISD::SRL && LHS.hasOneUse() && in combine_CC()
14481 LHS.getOperand(1).getOpcode() == ISD::Constant) { in combine_CC()
14483 if (LHS0.getOpcode() == ISD::AND && in combine_CC()
14484 LHS0.getOperand(1).getOpcode() == ISD::Constant) { in combine_CC()
14488 CCVal = CCVal == ISD::SETEQ ? ISD::SETGE : ISD::SETLT; in combine_CC()
14495 DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS0.getOperand(0), in combine_CC()
14506 CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType()); in combine_CC()
14514 CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType()); in combine_CC()
14537 case ISD::SHL: in tryFoldSelectIntoOp()
14538 case ISD::SRA: in tryFoldSelectIntoOp()
14539 case ISD::SRL: in tryFoldSelectIntoOp()
14540 case ISD::SUB: in tryFoldSelectIntoOp()
14543 case ISD::ADD: in tryFoldSelectIntoOp()
14544 case ISD::OR: in tryFoldSelectIntoOp()
14545 case ISD::XOR: in tryFoldSelectIntoOp()
14587 if (Cond.getOpcode() != ISD::SETCC) in foldSelectOfCTTZOrCTLZ()
14593 ISD::CondCode CCVal = cast<CondCodeSDNode>(Cond->getOperand(2))->get(); in foldSelectOfCTTZOrCTLZ()
14594 if (CCVal == ISD::CondCode::SETEQ) { in foldSelectOfCTTZOrCTLZ()
14597 } else if (CCVal == ISD::CondCode::SETNE) { in foldSelectOfCTTZOrCTLZ()
14604 if (CountZeroes.getOpcode() == ISD::TRUNCATE || in foldSelectOfCTTZOrCTLZ()
14605 CountZeroes.getOpcode() == ISD::ZERO_EXTEND) in foldSelectOfCTTZOrCTLZ()
14608 if (CountZeroes.getOpcode() != ISD::CTTZ && in foldSelectOfCTTZOrCTLZ()
14609 CountZeroes.getOpcode() != ISD::CTTZ_ZERO_UNDEF && in foldSelectOfCTTZOrCTLZ()
14610 CountZeroes.getOpcode() != ISD::CTLZ && in foldSelectOfCTTZOrCTLZ()
14611 CountZeroes.getOpcode() != ISD::CTLZ_ZERO_UNDEF) in foldSelectOfCTTZOrCTLZ()
14621 if (CountZeroes.getOpcode() == ISD::CTTZ_ZERO_UNDEF) { in foldSelectOfCTTZOrCTLZ()
14622 CountZeroes = DAG.getNode(ISD::CTTZ, SDLoc(CountZeroes), in foldSelectOfCTTZOrCTLZ()
14624 } else if (CountZeroes.getOpcode() == ISD::CTLZ_ZERO_UNDEF) { in foldSelectOfCTTZOrCTLZ()
14625 CountZeroes = DAG.getNode(ISD::CTLZ, SDLoc(CountZeroes), in foldSelectOfCTTZOrCTLZ()
14633 auto AndNode = DAG.getNode(ISD::AND, SDLoc(N), CountZeroes.getValueType(), in foldSelectOfCTTZOrCTLZ()
14647 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse()) in useInversedSetcc()
14656 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in useInversedSetcc()
14657 if (CC == ISD::SETEQ && LHS.getOpcode() == ISD::AND && in useInversedSetcc()
14662 DAG.getSetCC(DL, CondVT, LHS, RHS, ISD::SETNE), in useInversedSetcc()
14770 if (!ISD::isBuildVectorOfConstantSDNodes(InVecRHS.getNode())) in performINSERT_VECTOR_ELTCombine()
14778 SDValue LHS = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, in performINSERT_VECTOR_ELTCombine()
14780 SDValue RHS = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, in performINSERT_VECTOR_ELTCombine()
14787 if (InVec.getOpcode() != ISD::CONCAT_VECTORS) in performINSERT_VECTOR_ELTCombine()
14804 ConcatOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ConcatVT, in performINSERT_VECTOR_ELTCombine()
14810 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); in performINSERT_VECTOR_ELTCombine()
14832 if (!BaseLd || !BaseLd->isSimple() || !ISD::isNormalLoad(BaseLd) || in performCONCAT_VECTORSCombine()
14845 Ld->getChain() != BaseLd->getChain() || !ISD::isNormalLoad(Ld) || in performCONCAT_VECTORSCombine()
14868 if (P2.getOpcode() == ISD::ADD && P2.getOperand(0) == P1) in performCONCAT_VECTORSCombine()
14870 if (P1.getOpcode() == ISD::ADD && P1.getOperand(0) == P2) in performCONCAT_VECTORSCombine()
14939 SDValue StridedLoad = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, in performCONCAT_VECTORSCombine()
14950 assert(N->getOpcode() == RISCVISD::ADD_VL || N->getOpcode() == ISD::ADD); in combineToVWMACC()
14988 if (N->getOpcode() == ISD::ADD) { in combineToVWMACC()
15020 ISD::MemIndexType &IndexType, in legalizeScatterGatherIndexType()
15041 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, in legalizeScatterGatherIndexType()
15044 IndexType = ISD::UNSIGNED_SCALED; in legalizeScatterGatherIndexType()
15054 if (!ISD::isConstantSplatVectorAllOnes(Mask.getNode())) in matchIndexAsShuffle()
15056 if (!ISD::isBuildVectorOfConstantSDNodes(Index.getNode())) in matchIndexAsShuffle()
15089 if (!ISD::isConstantSplatVectorAllOnes(Mask.getNode())) in matchIndexAsWiderOp()
15091 if (!ISD::isBuildVectorOfConstantSDNodes(Index.getNode())) in matchIndexAsWiderOp()
15144 if (N->getOpcode() != ISD::DELETED_NODE) in PerformDAGCombine()
15179 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) || in PerformDAGCombine()
15188 if (Op0.getOpcode() == ISD::FNEG) { in PerformDAGCombine()
15189 SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi, in PerformDAGCombine()
15193 assert(Op0.getOpcode() == ISD::FABS); in PerformDAGCombine()
15194 SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi, in PerformDAGCombine()
15247 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) || in PerformDAGCombine()
15253 if (Op0.getOpcode() == ISD::FNEG) in PerformDAGCombine()
15254 return DAG.getNode(ISD::XOR, DL, VT, NewFMV, in PerformDAGCombine()
15257 assert(Op0.getOpcode() == ISD::FABS); in PerformDAGCombine()
15258 return DAG.getNode(ISD::AND, DL, VT, NewFMV, in PerformDAGCombine()
15261 case ISD::ADD: { in PerformDAGCombine()
15268 case ISD::SUB: { in PerformDAGCombine()
15273 case ISD::AND: in PerformDAGCombine()
15275 case ISD::OR: in PerformDAGCombine()
15277 case ISD::XOR: in PerformDAGCombine()
15279 case ISD::MUL: in PerformDAGCombine()
15283 case ISD::FADD: in PerformDAGCombine()
15284 case ISD::UMAX: in PerformDAGCombine()
15285 case ISD::UMIN: in PerformDAGCombine()
15286 case ISD::SMAX: in PerformDAGCombine()
15287 case ISD::SMIN: in PerformDAGCombine()
15288 case ISD::FMAXNUM: in PerformDAGCombine()
15289 case ISD::FMINNUM: { in PerformDAGCombine()
15296 case ISD::SETCC: in PerformDAGCombine()
15298 case ISD::SIGN_EXTEND_INREG: in PerformDAGCombine()
15300 case ISD::ZERO_EXTEND: in PerformDAGCombine()
15306 if (Src.getOpcode() == ISD::FP_TO_UINT && in PerformDAGCombine()
15308 return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), MVT::i64, in PerformDAGCombine()
15310 if (Src.getOpcode() == ISD::STRICT_FP_TO_UINT && Src.hasOneUse() && in PerformDAGCombine()
15313 SDValue Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, SDLoc(N), VTs, in PerformDAGCombine()
15352 if (Op.getOpcode() == ISD::SRA && Op.hasOneUse()) { in PerformDAGCombine()
15355 if (N0.getOpcode() == ISD::SIGN_EXTEND && N0.hasOneUse() && in PerformDAGCombine()
15356 N1.getOpcode() == ISD::ZERO_EXTEND && N1.hasOneUse()) { in PerformDAGCombine()
15364 ISD::SMIN, SDLoc(N1), N->getValueType(0), N10, in PerformDAGCombine()
15366 return DAG.getNode(ISD::SRA, SDLoc(N), N->getValueType(0), N00, SMin); in PerformDAGCombine()
15372 case ISD::TRUNCATE: in PerformDAGCombine()
15374 case ISD::SELECT: in PerformDAGCombine()
15380 if (N->getOperand(1).getOpcode() == ISD::XOR && in PerformDAGCombine()
15399 ISD::CondCode CCVal = cast<CondCodeSDNode>(CC)->get(); in PerformDAGCombine()
15413 (CCVal == ISD::CondCode::SETLT || CCVal == ISD::CondCode::SETGE)) { in PerformDAGCombine()
15414 if (CCVal == ISD::CondCode::SETGE) in PerformDAGCombine()
15424 DAG.getNode(ISD::SRA, DL, VT, LHS, in PerformDAGCombine()
15427 DAG.getNode(ISD::AND, DL, VT, SRA, in PerformDAGCombine()
15429 return DAG.getNode(ISD::ADD, DL, VT, AND, FalseV); in PerformDAGCombine()
15432 if (CCVal == ISD::CondCode::SETGE) in PerformDAGCombine()
15445 return DAG.getNode(ISD::OR, DL, VT, Neg, FalseV); in PerformDAGCombine()
15450 DAG.getSetCC(DL, VT, LHS, RHS, ISD::getSetCCInverse(CCVal, VT)); in PerformDAGCombine()
15452 return DAG.getNode(ISD::OR, DL, VT, Neg, TrueV); in PerformDAGCombine()
15458 DAG.getSetCC(DL, VT, LHS, RHS, ISD::getSetCCInverse(CCVal, VT)); in PerformDAGCombine()
15460 return DAG.getNode(ISD::AND, DL, VT, Neg, FalseV); in PerformDAGCombine()
15466 return DAG.getNode(ISD::AND, DL, VT, Neg, TrueV); in PerformDAGCombine()
15471 CCVal == ISD::CondCode::SETNE) || in PerformDAGCombine()
15473 CCVal == ISD::CondCode::SETEQ)) && in PerformDAGCombine()
15477 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, ISD::CondCode::SETEQ); in PerformDAGCombine()
15478 return DAG.getNode(ISD::ADD, DL, VT, LHS, C); in PerformDAGCombine()
15486 if (TrueV.getOpcode() == ISD::XOR && FalseV.getOpcode() == ISD::XOR && in PerformDAGCombine()
15492 return DAG.getNode(ISD::XOR, DL, VT, NewSel, TrueV.getOperand(1)); in PerformDAGCombine()
15509 case ISD::BITREVERSE: in PerformDAGCombine()
15511 case ISD::FP_TO_SINT: in PerformDAGCombine()
15512 case ISD::FP_TO_UINT: in PerformDAGCombine()
15514 case ISD::FP_TO_SINT_SAT: in PerformDAGCombine()
15515 case ISD::FP_TO_UINT_SAT: in PerformDAGCombine()
15517 case ISD::FCOPYSIGN: { in PerformDAGCombine()
15530 if (In2.getOpcode() != ISD::FP_EXTEND && in PerformDAGCombine()
15531 (In2.getOpcode() != ISD::FP_ROUND || In2.getConstantOperandVal(1) != 0)) in PerformDAGCombine()
15534 if (In2.getOpcode() != ISD::FNEG) in PerformDAGCombine()
15538 return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N->getOperand(0), in PerformDAGCombine()
15539 DAG.getNode(ISD::FNEG, DL, VT, NewFPExtRound)); in PerformDAGCombine()
15541 case ISD::MGATHER: { in PerformDAGCombine()
15546 ISD::MemIndexType IndexType = MGN->getIndexType(); in PerformDAGCombine()
15565 if (Index.getOpcode() == ISD::BUILD_VECTOR && in PerformDAGCombine()
15566 MGN->getExtensionType() == ISD::NON_EXTLOAD && isTypeLegal(VT)) { in PerformDAGCombine()
15581 assert(IndexType == ISD::UNSIGNED_SCALED); in PerformDAGCombine()
15582 SDValue BasePtr = DAG.getNode(ISD::ADD, DL, PtrVT, MGN->getBasePtr(), in PerformDAGCombine()
15592 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, in PerformDAGCombine()
15598 if (MGN->getExtensionType() == ISD::NON_EXTLOAD && in PerformDAGCombine()
15604 ISD::UNINDEXED, ISD::NON_EXTLOAD); in PerformDAGCombine()
15610 if (MGN->getExtensionType() == ISD::NON_EXTLOAD && in PerformDAGCombine()
15635 MGN->getMemOperand(), IndexType, ISD::NON_EXTLOAD); in PerformDAGCombine()
15641 case ISD::MSCATTER:{ in PerformDAGCombine()
15645 ISD::MemIndexType IndexType = MSN->getIndexType(); in PerformDAGCombine()
15673 ISD::UNINDEXED, false); in PerformDAGCombine()
15677 case ISD::VP_GATHER: { in PerformDAGCombine()
15681 ISD::MemIndexType IndexType = VPGN->getIndexType(); in PerformDAGCombine()
15702 case ISD::VP_SCATTER: { in PerformDAGCombine()
15706 ISD::MemIndexType IndexType = VPSN->getIndexType(); in PerformDAGCombine()
15742 case ISD::SRA: in PerformDAGCombine()
15746 case ISD::SRL: in PerformDAGCombine()
15747 case ISD::SHL: { in PerformDAGCombine()
15785 case ISD::LOAD: in PerformDAGCombine()
15786 case ISD::STORE: { in PerformDAGCombine()
15791 if (N->getOpcode() != ISD::STORE) in PerformDAGCombine()
15801 MemVT.isFixedLengthVector() && ISD::isNormalStore(Store) && in PerformDAGCombine()
15817 ISD::isBuildVectorOfConstantSDNodes(Val.getNode())) { in PerformDAGCombine()
15847 Store->getChain() == SDValue(L, 1) && ISD::isNormalLoad(L) && in PerformDAGCombine()
15868 Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in PerformDAGCombine()
15888 case ISD::SPLAT_VECTOR: { in PerformDAGCombine()
15898 case ISD::BUILD_VECTOR: in PerformDAGCombine()
15902 case ISD::CONCAT_VECTORS: in PerformDAGCombine()
15906 case ISD::INSERT_VECTOR_ELT: in PerformDAGCombine()
15951 Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in PerformDAGCombine()
15974 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, Passthru, in PerformDAGCombine()
15978 Result = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Passthru, Result, in PerformDAGCombine()
15998 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, Vec, in PerformDAGCombine()
16004 case ISD::INTRINSIC_VOID: in PerformDAGCombine()
16005 case ISD::INTRINSIC_W_CHAIN: in PerformDAGCombine()
16006 case ISD::INTRINSIC_WO_CHAIN: { in PerformDAGCombine()
16007 unsigned IntOpNo = N->getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 0 : 1; in PerformDAGCombine()
16029 ISD::UNINDEXED, ISD::NON_EXTLOAD); in PerformDAGCombine()
16047 ISD::UNINDEXED, false); in PerformDAGCombine()
16070 case ISD::BITCAST: { in PerformDAGCombine()
16083 N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i1, Ops); in PerformDAGCombine()
16085 return DAG.getNode(ISD::TRUNCATE, DL, VT, N0); in PerformDAGCombine()
16116 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA || in isDesirableToCommuteWithShift()
16117 N->getOpcode() == ISD::SRL) && in isDesirableToCommuteWithShift()
16128 (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) { in isDesirableToCommuteWithShift()
16178 if (Opcode != ISD::AND && Opcode != ISD::OR && Opcode != ISD::XOR) in targetShrinkDemandedConstant()
16213 if (Opcode == ISD::AND) { in targetShrinkDemandedConstant()
16278 assert((Opc >= ISD::BUILTIN_OP_END || in computeKnownBitsForTargetNode()
16279 Opc == ISD::INTRINSIC_WO_CHAIN || in computeKnownBitsForTargetNode()
16280 Opc == ISD::INTRINSIC_W_CHAIN || in computeKnownBitsForTargetNode()
16281 Opc == ISD::INTRINSIC_VOID) && in computeKnownBitsForTargetNode()
16378 case ISD::INTRINSIC_W_CHAIN: in computeKnownBitsForTargetNode()
16379 case ISD::INTRINSIC_WO_CHAIN: { in computeKnownBitsForTargetNode()
16381 Op.getConstantOperandVal(Opc == ISD::INTRINSIC_WO_CHAIN ? 0 : 1); in computeKnownBitsForTargetNode()
16467 case ISD::INTRINSIC_W_CHAIN: { in ComputeNumSignBitsForTargetNode()
16501 if (!ISD::isNormalLoad(Ld)) in getTargetConstantFromLoad()
17360 ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2, in CC_RISCVAssign2XLen()
17362 ISD::ArgFlagsTy ArgFlags2, bool EABI) { in CC_RISCVAssign2XLen()
17427 ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, in CC_RISCV()
17516 SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags = in CC_RISCV()
17578 ISD::ArgFlagsTy AF = PendingArgFlags[0]; in CC_RISCV()
17679 const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet, in analyzeInputArgs()
17690 ISD::ArgFlagsTy ArgFlags = Ins[i].Flags; in analyzeInputArgs()
17711 const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet, in analyzeOutputArgs()
17721 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; in analyzeOutputArgs()
17753 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Val); in convertLocVTToValVT()
17754 Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val); in convertLocVTToValVT()
17759 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val); in convertLocVTToValVT()
17770 const ISD::InputArg &In, in unpackFromRegLoc()
17819 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val); in convertValVTToLocVT()
17820 Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Val); in convertValVTToLocVT()
17825 Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val); in convertValVTToLocVT()
17852 ISD::LoadExtType ExtType; in unpackFromMemLoc()
17859 ExtType = ISD::NON_EXTLOAD; in unpackFromMemLoc()
17905 ISD::ArgFlagsTy ArgFlags, CCState &State, in CC_RISCV_FastCC()
18020 ISD::ArgFlagsTy ArgFlags, CCState &State) { in CC_RISCV_GHC()
18082 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, in LowerFormalArguments()
18163 Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset); in LowerFormalArguments()
18164 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, Offset); in LowerFormalArguments()
18237 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains); in LowerFormalArguments()
18316 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; in LowerCall()
18318 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; in LowerCall()
18358 ISD::ArgFlagsTy Flags = Outs[i].Flags; in LowerCall()
18389 ISD::ArgFlagsTy Flags = Outs[OutIdx].Flags; in LowerCall()
18411 DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, in LowerCall()
18450 Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset); in LowerCall()
18466 DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, PartOffset); in LowerCall()
18492 DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, in LowerCall()
18503 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); in LowerCall()
18615 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const { in CanLowerReturn()
18625 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; in CanLowerReturn()
18638 const SmallVectorImpl<ISD::OutputArg> &Outs, in LowerReturn()
18760 if (Copy->getOpcode() == ISD::BITCAST) { in isUsedByReturnOnly()
18766 if (Copy->getOpcode() != ISD::CopyToReg) { in isUsedByReturnOnly()
19517 return Extend.getOpcode() == ISD::ZERO_EXTEND && in shouldRemoveExtendFromGSIndex()
19571 ISD::MemIndexedMode &AM, in getIndexedAddressParts()
19577 if (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB) in getIndexedAddressParts()
19583 if (Op->getOpcode() == ISD::SUB) in getIndexedAddressParts()
19607 ISD::MemIndexedMode &AM, in getPreIndexedAddressParts()
19623 AM = ISD::PRE_INC; in getPreIndexedAddressParts()
19630 ISD::MemIndexedMode &AM, in getPostIndexedAddressParts()
19650 AM = ISD::POST_INC; in getPostIndexedAddressParts()
19676 ISD::NodeType RISCVTargetLowering::getExtendForAtomicCmpSwapArg() const { in getExtendForAtomicCmpSwapArg()
19678 return Subtarget.hasStdExtZacas() ? ISD::ANY_EXTEND : ISD::SIGN_EXTEND; in getExtendForAtomicCmpSwapArg()
19851 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Val); in splitValueIntoRegisterParts()
19852 Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Val); in splitValueIntoRegisterParts()
19853 Val = DAG.getNode(ISD::OR, DL, MVT::i32, Val, in splitValueIntoRegisterParts()
19855 Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val); in splitValueIntoRegisterParts()
19880 Val = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SameEltTypeVT, in splitValueIntoRegisterParts()
19884 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); in splitValueIntoRegisterParts()
19887 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT), in splitValueIntoRegisterParts()
19906 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val); in joinRegisterPartsIntoValue()
19907 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Val); in joinRegisterPartsIntoValue()
19908 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); in joinRegisterPartsIntoValue()
19933 Val = DAG.getNode(ISD::BITCAST, DL, SameEltTypeVT, Val); in joinRegisterPartsIntoValue()
19935 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val, in joinRegisterPartsIntoValue()
19955 if (Opc == ISD::ZERO_EXTEND || Opc == ISD::SIGN_EXTEND) in preferScalarizeSplat()
20333 ISD::CondCode Cond) const { in getCustomCtpopCost()