Lines Matching refs:ISD
180 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, XLenVT, in RISCVTargetLowering()
184 setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Expand); in RISCVTargetLowering()
186 setOperationAction(ISD::BR_JT, MVT::Other, Expand); in RISCVTargetLowering()
187 setOperationAction(ISD::BR_CC, XLenVT, Expand); in RISCVTargetLowering()
188 setOperationAction(ISD::BRCOND, MVT::Other, Custom); in RISCVTargetLowering()
189 setOperationAction(ISD::SELECT_CC, XLenVT, Expand); in RISCVTargetLowering()
191 setOperationAction({ISD::STACKSAVE, ISD::STACKRESTORE}, MVT::Other, Expand); in RISCVTargetLowering()
193 setOperationAction(ISD::VASTART, MVT::Other, Custom); in RISCVTargetLowering()
194 setOperationAction({ISD::VAARG, ISD::VACOPY, ISD::VAEND}, MVT::Other, Expand); in RISCVTargetLowering()
196 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); in RISCVTargetLowering()
198 setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom); in RISCVTargetLowering()
201 setOperationAction(ISD::SIGN_EXTEND_INREG, {MVT::i8, MVT::i16}, Expand); in RISCVTargetLowering()
204 setOperationAction(ISD::EH_DWARF_CFA, MVT::i64, Custom); in RISCVTargetLowering()
206 setOperationAction({ISD::ADD, ISD::SUB, ISD::SHL, ISD::SRA, ISD::SRL}, in RISCVTargetLowering()
209 setOperationAction({ISD::UADDO, ISD::USUBO, ISD::UADDSAT, ISD::USUBSAT}, in RISCVTargetLowering()
219 setOperationAction({ISD::MUL, ISD::MULHS, ISD::MULHU}, XLenVT, Expand); in RISCVTargetLowering()
222 setOperationAction(ISD::MUL, {MVT::i32, MVT::i128}, Custom); in RISCVTargetLowering()
224 setOperationAction(ISD::MUL, MVT::i64, Custom); in RISCVTargetLowering()
229 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, in RISCVTargetLowering()
233 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::UREM}, in RISCVTargetLowering()
239 {ISD::SDIVREM, ISD::UDIVREM, ISD::SMUL_LOHI, ISD::UMUL_LOHI}, XLenVT, in RISCVTargetLowering()
242 setOperationAction({ISD::SHL_PARTS, ISD::SRL_PARTS, ISD::SRA_PARTS}, XLenVT, in RISCVTargetLowering()
248 setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom); in RISCVTargetLowering()
250 setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Expand); in RISCVTargetLowering()
256 setOperationAction({ISD::BITREVERSE, ISD::BSWAP}, XLenVT, Custom); in RISCVTargetLowering()
259 setOperationAction(ISD::BITREVERSE, MVT::i8, Custom); in RISCVTargetLowering()
261 setOperationAction({ISD::BITREVERSE, ISD::BSWAP}, MVT::i16, Custom); in RISCVTargetLowering()
264 setOperationAction({ISD::BITREVERSE, ISD::BSWAP}, MVT::i32, Custom); in RISCVTargetLowering()
268 setOperationAction(ISD::BSWAP, XLenVT, in RISCVTargetLowering()
273 setOperationAction(ISD::BITREVERSE, XLenVT, in RISCVTargetLowering()
278 setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, XLenVT, in RISCVTargetLowering()
283 {ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF, ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, in RISCVTargetLowering()
286 setOperationAction({ISD::CTTZ, ISD::CTLZ, ISD::CTPOP}, XLenVT, Expand); in RISCVTargetLowering()
289 setOperationAction(ISD::ABS, MVT::i32, Custom); in RISCVTargetLowering()
293 setOperationAction({ISD::FSHL, ISD::FSHR}, XLenVT, Custom); in RISCVTargetLowering()
294 setOperationAction(ISD::SELECT, XLenVT, Legal); in RISCVTargetLowering()
297 setOperationAction({ISD::FSHL, ISD::FSHR}, MVT::i32, Custom); in RISCVTargetLowering()
299 setOperationAction(ISD::SELECT, XLenVT, Custom); in RISCVTargetLowering()
303 ISD::FMINNUM, ISD::FMAXNUM, ISD::LRINT, in RISCVTargetLowering()
304 ISD::LLRINT, ISD::LROUND, ISD::LLROUND, in RISCVTargetLowering()
305 ISD::STRICT_LRINT, ISD::STRICT_LLRINT, ISD::STRICT_LROUND, in RISCVTargetLowering()
306 ISD::STRICT_LLROUND, ISD::STRICT_FMA, ISD::STRICT_FADD, in RISCVTargetLowering()
307 ISD::STRICT_FSUB, ISD::STRICT_FMUL, ISD::STRICT_FDIV, in RISCVTargetLowering()
308 ISD::STRICT_FSQRT, ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS}; in RISCVTargetLowering()
310 static const ISD::CondCode FPCCToExpand[] = { in RISCVTargetLowering()
311 ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT, in RISCVTargetLowering()
312 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT, in RISCVTargetLowering()
313 ISD::SETGE, ISD::SETNE, ISD::SETO, ISD::SETUO}; in RISCVTargetLowering()
316 ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, in RISCVTargetLowering()
317 ISD::FREM, ISD::FP16_TO_FP, ISD::FP_TO_FP16}; in RISCVTargetLowering()
320 setOperationAction(ISD::BITCAST, MVT::i16, Custom); in RISCVTargetLowering()
324 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Legal); in RISCVTargetLowering()
325 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal); in RISCVTargetLowering()
327 setOperationAction(ISD::SELECT_CC, MVT::f16, Expand); in RISCVTargetLowering()
328 setOperationAction(ISD::SELECT, MVT::f16, Custom); in RISCVTargetLowering()
329 setOperationAction(ISD::BR_CC, MVT::f16, Expand); in RISCVTargetLowering()
331 setOperationAction({ISD::FREM, ISD::FCEIL, ISD::FFLOOR, ISD::FNEARBYINT, in RISCVTargetLowering()
332 ISD::FRINT, ISD::FROUND, ISD::FROUNDEVEN, ISD::FTRUNC, in RISCVTargetLowering()
333 ISD::FPOW, ISD::FPOWI, ISD::FCOS, ISD::FSIN, in RISCVTargetLowering()
334 ISD::FSINCOS, ISD::FEXP, ISD::FEXP2, ISD::FLOG, in RISCVTargetLowering()
335 ISD::FLOG2, ISD::FLOG10}, in RISCVTargetLowering()
343 setOperationAction(ISD::FPOWI, MVT::i32, Custom); in RISCVTargetLowering()
349 setOperationAction(ISD::SELECT_CC, MVT::f32, Expand); in RISCVTargetLowering()
350 setOperationAction(ISD::SELECT, MVT::f32, Custom); in RISCVTargetLowering()
351 setOperationAction(ISD::BR_CC, MVT::f32, Expand); in RISCVTargetLowering()
353 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand); in RISCVTargetLowering()
358 setOperationAction(ISD::BITCAST, MVT::i32, Custom); in RISCVTargetLowering()
362 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal); in RISCVTargetLowering()
363 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal); in RISCVTargetLowering()
365 setOperationAction(ISD::SELECT_CC, MVT::f64, Expand); in RISCVTargetLowering()
366 setOperationAction(ISD::SELECT, MVT::f64, Custom); in RISCVTargetLowering()
367 setOperationAction(ISD::BR_CC, MVT::f64, Expand); in RISCVTargetLowering()
368 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand); in RISCVTargetLowering()
371 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand); in RISCVTargetLowering()
376 setOperationAction({ISD::FP_TO_UINT, ISD::FP_TO_SINT, in RISCVTargetLowering()
377 ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT}, in RISCVTargetLowering()
381 setOperationAction({ISD::FP_TO_UINT_SAT, ISD::FP_TO_SINT_SAT}, XLenVT, in RISCVTargetLowering()
384 setOperationAction({ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT, in RISCVTargetLowering()
385 ISD::STRICT_UINT_TO_FP, ISD::STRICT_SINT_TO_FP}, in RISCVTargetLowering()
388 setOperationAction(ISD::FLT_ROUNDS_, XLenVT, Custom); in RISCVTargetLowering()
389 setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom); in RISCVTargetLowering()
392 setOperationAction({ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool, in RISCVTargetLowering()
393 ISD::JumpTable}, in RISCVTargetLowering()
396 setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom); in RISCVTargetLowering()
399 setOperationAction(ISD::Constant, MVT::i64, Custom); in RISCVTargetLowering()
403 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, in RISCVTargetLowering()
406 setOperationAction({ISD::TRAP, ISD::DEBUGTRAP}, MVT::Other, Legal); in RISCVTargetLowering()
407 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); in RISCVTargetLowering()
409 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom); in RISCVTargetLowering()
423 setOperationAction(ISD::VSCALE, XLenVT, Custom); in RISCVTargetLowering()
427 setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN}, in RISCVTargetLowering()
430 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i32, Custom); in RISCVTargetLowering()
432 setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN}, in RISCVTargetLowering()
435 setOperationAction({ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID}, in RISCVTargetLowering()
439 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL, in RISCVTargetLowering()
440 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM, in RISCVTargetLowering()
441 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR, in RISCVTargetLowering()
442 ISD::VP_XOR, ISD::VP_ASHR, ISD::VP_LSHR, in RISCVTargetLowering()
443 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND, in RISCVTargetLowering()
444 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX, in RISCVTargetLowering()
445 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN, in RISCVTargetLowering()
446 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FPTOSI, in RISCVTargetLowering()
447 ISD::VP_FPTOUI, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND, in RISCVTargetLowering()
448 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE}; in RISCVTargetLowering()
451 ISD::VP_FADD, ISD::VP_FSUB, in RISCVTargetLowering()
452 ISD::VP_FMUL, ISD::VP_FDIV, in RISCVTargetLowering()
453 ISD::VP_FNEG, ISD::VP_FMA, in RISCVTargetLowering()
454 ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD, in RISCVTargetLowering()
455 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, in RISCVTargetLowering()
456 ISD::VP_MERGE, ISD::VP_SELECT, in RISCVTargetLowering()
457 ISD::VP_SITOFP, ISD::VP_UITOFP, in RISCVTargetLowering()
458 ISD::VP_SETCC, ISD::VP_FP_ROUND, in RISCVTargetLowering()
459 ISD::VP_FP_EXTEND}; in RISCVTargetLowering()
462 ISD::VECREDUCE_ADD, ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, in RISCVTargetLowering()
463 ISD::VECREDUCE_XOR, ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN, in RISCVTargetLowering()
464 ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN}; in RISCVTargetLowering()
467 ISD::VECREDUCE_FADD, ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_FMIN, in RISCVTargetLowering()
468 ISD::VECREDUCE_FMAX}; in RISCVTargetLowering()
473 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, in RISCVTargetLowering()
478 setOperationAction({ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND, in RISCVTargetLowering()
479 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, in RISCVTargetLowering()
480 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN, in RISCVTargetLowering()
481 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN}, in RISCVTargetLowering()
489 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); in RISCVTargetLowering()
492 setOperationAction({ISD::TRUNCATE, ISD::CONCAT_VECTORS, in RISCVTargetLowering()
493 ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, in RISCVTargetLowering()
496 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT, in RISCVTargetLowering()
499 setOperationAction(ISD::SELECT, VT, Custom); in RISCVTargetLowering()
501 {ISD::SELECT_CC, ISD::VSELECT, ISD::VP_MERGE, ISD::VP_SELECT}, VT, in RISCVTargetLowering()
504 setOperationAction({ISD::VP_AND, ISD::VP_OR, ISD::VP_XOR}, VT, Custom); in RISCVTargetLowering()
507 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT, in RISCVTargetLowering()
511 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT, in RISCVTargetLowering()
519 {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT}, in RISCVTargetLowering()
526 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, OtherVT, in RISCVTargetLowering()
531 {ISD::VP_FPTOSI, ISD::VP_FPTOUI, ISD::VP_TRUNCATE, ISD::VP_SETCC}, VT, in RISCVTargetLowering()
533 setOperationAction(ISD::VECTOR_REVERSE, VT, Custom); in RISCVTargetLowering()
536 ISD::VECTOR_SPLICE, VT, in RISCVTargetLowering()
544 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); in RISCVTargetLowering()
545 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom); in RISCVTargetLowering()
548 setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Expand); in RISCVTargetLowering()
552 setOperationAction({ISD::MULHU, ISD::MULHS}, VT, Expand); in RISCVTargetLowering()
554 setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT, in RISCVTargetLowering()
557 setOperationAction({ISD::ROTL, ISD::ROTR}, VT, Expand); in RISCVTargetLowering()
559 setOperationAction({ISD::CTTZ, ISD::CTLZ, ISD::CTPOP, ISD::BSWAP}, VT, in RISCVTargetLowering()
562 setOperationAction(ISD::BSWAP, VT, Expand); in RISCVTargetLowering()
565 setOperationAction({ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND}, in RISCVTargetLowering()
573 {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT}, in RISCVTargetLowering()
577 {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT}, VT, Legal); in RISCVTargetLowering()
581 setOperationAction(ISD::TRUNCATE, VT, Custom); in RISCVTargetLowering()
584 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT, in RISCVTargetLowering()
593 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom); in RISCVTargetLowering()
595 setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, in RISCVTargetLowering()
599 {ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER, ISD::VP_SCATTER}, VT, in RISCVTargetLowering()
603 {ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, in RISCVTargetLowering()
606 setOperationAction(ISD::SELECT, VT, Custom); in RISCVTargetLowering()
607 setOperationAction(ISD::SELECT_CC, VT, Expand); in RISCVTargetLowering()
609 setOperationAction({ISD::STEP_VECTOR, ISD::VECTOR_REVERSE}, VT, Custom); in RISCVTargetLowering()
613 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, OtherVT, in RISCVTargetLowering()
618 setOperationAction(ISD::VECTOR_SPLICE, VT, Custom); in RISCVTargetLowering()
627 setOperationAction({ISD::CTLZ_ZERO_UNDEF, ISD::CTTZ_ZERO_UNDEF}, VT, in RISCVTargetLowering()
640 static const ISD::CondCode VFPCCToExpand[] = { in RISCVTargetLowering()
641 ISD::SETO, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT, in RISCVTargetLowering()
642 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO, in RISCVTargetLowering()
643 ISD::SETGT, ISD::SETOGT, ISD::SETGE, ISD::SETOGE, in RISCVTargetLowering()
648 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); in RISCVTargetLowering()
653 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom); in RISCVTargetLowering()
655 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT, in RISCVTargetLowering()
660 setOperationAction({ISD::FMINNUM, ISD::FMAXNUM}, VT, Legal); in RISCVTargetLowering()
662 setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND}, in RISCVTargetLowering()
668 setOperationAction(ISD::FREM, VT, Expand); in RISCVTargetLowering()
669 setOperationAction(ISD::FPOW, VT, Expand); in RISCVTargetLowering()
670 setOperationAction(ISD::FCOS, VT, Expand); in RISCVTargetLowering()
671 setOperationAction(ISD::FSIN, VT, Expand); in RISCVTargetLowering()
672 setOperationAction(ISD::FSINCOS, VT, Expand); in RISCVTargetLowering()
673 setOperationAction(ISD::FEXP, VT, Expand); in RISCVTargetLowering()
674 setOperationAction(ISD::FEXP2, VT, Expand); in RISCVTargetLowering()
675 setOperationAction(ISD::FLOG, VT, Expand); in RISCVTargetLowering()
676 setOperationAction(ISD::FLOG2, VT, Expand); in RISCVTargetLowering()
677 setOperationAction(ISD::FLOG10, VT, Expand); in RISCVTargetLowering()
678 setOperationAction(ISD::FRINT, VT, Expand); in RISCVTargetLowering()
679 setOperationAction(ISD::FNEARBYINT, VT, Expand); in RISCVTargetLowering()
681 setOperationAction(ISD::VECREDUCE_FADD, VT, Custom); in RISCVTargetLowering()
682 setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom); in RISCVTargetLowering()
683 setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); in RISCVTargetLowering()
684 setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); in RISCVTargetLowering()
686 setOperationAction(ISD::FCOPYSIGN, VT, Legal); in RISCVTargetLowering()
688 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom); in RISCVTargetLowering()
690 setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, in RISCVTargetLowering()
694 {ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER, ISD::VP_SCATTER}, VT, in RISCVTargetLowering()
697 setOperationAction(ISD::SELECT, VT, Custom); in RISCVTargetLowering()
698 setOperationAction(ISD::SELECT_CC, VT, Expand); in RISCVTargetLowering()
701 {ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, in RISCVTargetLowering()
704 setOperationAction({ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE}, VT, Custom); in RISCVTargetLowering()
715 setLoadExtAction(ISD::EXTLOAD, VT, SmallVT, Expand); in RISCVTargetLowering()
752 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) in RISCVTargetLowering()
756 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, in RISCVTargetLowering()
761 setOperationAction({ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT, in RISCVTargetLowering()
764 setOperationAction({ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS}, VT, in RISCVTargetLowering()
767 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, in RISCVTargetLowering()
770 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom); in RISCVTargetLowering()
772 setOperationAction(ISD::SETCC, VT, Custom); in RISCVTargetLowering()
774 setOperationAction(ISD::SELECT, VT, Custom); in RISCVTargetLowering()
776 setOperationAction(ISD::TRUNCATE, VT, Custom); in RISCVTargetLowering()
778 setOperationAction(ISD::BITCAST, VT, Custom); in RISCVTargetLowering()
781 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT, in RISCVTargetLowering()
785 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT, in RISCVTargetLowering()
788 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, in RISCVTargetLowering()
789 ISD::FP_TO_UINT}, in RISCVTargetLowering()
794 setOperationAction({ISD::VP_AND, ISD::VP_OR, ISD::VP_XOR, ISD::AND, in RISCVTargetLowering()
795 ISD::OR, ISD::XOR}, in RISCVTargetLowering()
799 {ISD::VP_FPTOSI, ISD::VP_FPTOUI, ISD::VP_SETCC, ISD::VP_TRUNCATE}, in RISCVTargetLowering()
810 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal); in RISCVTargetLowering()
811 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom); in RISCVTargetLowering()
814 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in RISCVTargetLowering()
815 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in RISCVTargetLowering()
818 {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, VT, Custom); in RISCVTargetLowering()
821 {ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER, ISD::VP_SCATTER}, VT, in RISCVTargetLowering()
824 setOperationAction({ISD::ADD, ISD::MUL, ISD::SUB, ISD::AND, ISD::OR, in RISCVTargetLowering()
825 ISD::XOR, ISD::SDIV, ISD::SREM, ISD::UDIV, in RISCVTargetLowering()
826 ISD::UREM, ISD::SHL, ISD::SRA, ISD::SRL}, in RISCVTargetLowering()
830 {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX, ISD::ABS}, VT, Custom); in RISCVTargetLowering()
834 setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Custom); in RISCVTargetLowering()
837 {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT}, VT, in RISCVTargetLowering()
840 setOperationAction(ISD::VSELECT, VT, Custom); in RISCVTargetLowering()
841 setOperationAction(ISD::SELECT_CC, VT, Expand); in RISCVTargetLowering()
844 {ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND}, VT, Custom); in RISCVTargetLowering()
848 setOperationAction({ISD::VECREDUCE_ADD, ISD::VECREDUCE_SMAX, in RISCVTargetLowering()
849 ISD::VECREDUCE_SMIN, ISD::VECREDUCE_UMAX, in RISCVTargetLowering()
850 ISD::VECREDUCE_UMIN}, in RISCVTargetLowering()
863 setOperationAction({ISD::CTLZ_ZERO_UNDEF, ISD::CTTZ_ZERO_UNDEF}, VT, in RISCVTargetLowering()
873 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) in RISCVTargetLowering()
876 setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand); in RISCVTargetLowering()
881 setOperationAction({ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT, in RISCVTargetLowering()
884 setOperationAction({ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS, in RISCVTargetLowering()
885 ISD::VECTOR_SHUFFLE, ISD::INSERT_VECTOR_ELT, in RISCVTargetLowering()
886 ISD::EXTRACT_VECTOR_ELT}, in RISCVTargetLowering()
889 setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE, in RISCVTargetLowering()
890 ISD::MGATHER, ISD::MSCATTER}, in RISCVTargetLowering()
894 {ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER, ISD::VP_SCATTER}, VT, in RISCVTargetLowering()
897 setOperationAction({ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV, in RISCVTargetLowering()
898 ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN, ISD::FSQRT, in RISCVTargetLowering()
899 ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM}, in RISCVTargetLowering()
902 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom); in RISCVTargetLowering()
904 setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND}, in RISCVTargetLowering()
909 setOperationAction({ISD::VSELECT, ISD::SELECT}, VT, Custom); in RISCVTargetLowering()
910 setOperationAction(ISD::SELECT_CC, VT, Expand); in RISCVTargetLowering()
912 setOperationAction(ISD::BITCAST, VT, Custom); in RISCVTargetLowering()
920 setOperationAction(ISD::BITCAST, {MVT::i8, MVT::i16, MVT::i32, MVT::i64}, in RISCVTargetLowering()
923 setOperationAction(ISD::BITCAST, MVT::f16, Custom); in RISCVTargetLowering()
925 setOperationAction(ISD::BITCAST, MVT::f32, Custom); in RISCVTargetLowering()
927 setOperationAction(ISD::BITCAST, MVT::f64, Custom); in RISCVTargetLowering()
941 setTargetDAGCombine({ISD::INTRINSIC_WO_CHAIN, ISD::ADD, ISD::SUB, ISD::AND, in RISCVTargetLowering()
942 ISD::OR, ISD::XOR, ISD::SETCC}); in RISCVTargetLowering()
944 setTargetDAGCombine(ISD::SRA); in RISCVTargetLowering()
947 setTargetDAGCombine({ISD::FADD, ISD::FMAXNUM, ISD::FMINNUM}); in RISCVTargetLowering()
950 setTargetDAGCombine({ISD::ROTL, ISD::ROTR}); in RISCVTargetLowering()
953 setTargetDAGCombine({ISD::UMAX, ISD::UMIN, ISD::SMAX, ISD::SMIN}); in RISCVTargetLowering()
956 setTargetDAGCombine(ISD::BITREVERSE); in RISCVTargetLowering()
958 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG); in RISCVTargetLowering()
960 setTargetDAGCombine({ISD::ZERO_EXTEND, ISD::FP_TO_SINT, ISD::FP_TO_UINT, in RISCVTargetLowering()
961 ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}); in RISCVTargetLowering()
963 setTargetDAGCombine({ISD::FCOPYSIGN, ISD::MGATHER, ISD::MSCATTER, in RISCVTargetLowering()
964 ISD::VP_GATHER, ISD::VP_SCATTER, ISD::SRA, ISD::SRL, in RISCVTargetLowering()
965 ISD::SHL, ISD::STORE, ISD::SPLAT_VECTOR}); in RISCVTargetLowering()
967 setTargetDAGCombine(ISD::BITCAST); in RISCVTargetLowering()
1005 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
1014 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
1022 Info.opc = ISD::INTRINSIC_VOID; in getTgtMemIntrinsic()
1039 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
1118 (LD->getExtensionType() == ISD::NON_EXTLOAD || in isZExtFree()
1119 LD->getExtensionType() == ISD::ZEXTLOAD)) in isZExtFree()
1201 if (XC && OldShiftOpcode == ISD::SRL && XC->isOne()) in shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd()
1205 if (NewShiftOpcode == ISD::SRL && CC->isOne()) in shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd()
1321 if (Opc >= ISD::BUILTIN_OP_END) in shouldScalarizeBinop()
1391 ISD::CondCode &CC, SelectionDAG &DAG) { in translateSetCCForBranch()
1395 LHS.getOpcode() == ISD::AND && LHS.hasOneUse() && in translateSetCCForBranch()
1399 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT; in translateSetCCForBranch()
1403 LHS = DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS, in translateSetCCForBranch()
1413 case ISD::SETGT: in translateSetCCForBranch()
1417 CC = ISD::SETGE; in translateSetCCForBranch()
1421 case ISD::SETLT: in translateSetCCForBranch()
1426 CC = ISD::SETGE; in translateSetCCForBranch()
1436 case ISD::SETGT: in translateSetCCForBranch()
1437 case ISD::SETLE: in translateSetCCForBranch()
1438 case ISD::SETUGT: in translateSetCCForBranch()
1439 case ISD::SETULE: in translateSetCCForBranch()
1440 CC = ISD::getSetCCSwappedOperands(CC); in translateSetCCForBranch()
1584 assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in getVLOperand()
1585 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) && in getVLOperand()
1587 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN; in getVLOperand()
1721 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero); in convertToScalableVector()
1733 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero); in convertFromScalableVector()
1799 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT; in lowerFP_TO_INT_SAT()
1815 return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO); in lowerFP_TO_INT_SAT()
1834 SDValue Truncated = DAG.getNode(ISD::FP_TO_SINT, DL, IntVT, Src); in lowerFTRUNC_FCEIL_FFLOOR()
1835 Truncated = DAG.getNode(ISD::SINT_TO_FP, DL, VT, Truncated); in lowerFTRUNC_FCEIL_FFLOOR()
1839 if (Op.getOpcode() == ISD::FCEIL) { in lowerFTRUNC_FCEIL_FFLOOR()
1844 SDValue Adjust = DAG.getNode(ISD::FADD, DL, VT, Truncated, in lowerFTRUNC_FCEIL_FFLOOR()
1846 SDValue NeedAdjust = DAG.getSetCC(DL, SetccVT, Truncated, Src, ISD::SETOLT); in lowerFTRUNC_FCEIL_FFLOOR()
1848 } else if (Op.getOpcode() == ISD::FFLOOR) { in lowerFTRUNC_FCEIL_FFLOOR()
1853 SDValue Adjust = DAG.getNode(ISD::FSUB, DL, VT, Truncated, in lowerFTRUNC_FCEIL_FFLOOR()
1855 SDValue NeedAdjust = DAG.getSetCC(DL, SetccVT, Truncated, Src, ISD::SETOGT); in lowerFTRUNC_FCEIL_FFLOOR()
1860 Truncated = DAG.getNode(ISD::FCOPYSIGN, DL, VT, Truncated, Src); in lowerFTRUNC_FCEIL_FFLOOR()
1873 SDValue Abs = DAG.getNode(ISD::FABS, DL, VT, Src); in lowerFTRUNC_FCEIL_FFLOOR()
1874 SDValue Setcc = DAG.getSetCC(DL, SetccVT, Abs, MaxValNode, ISD::SETOLT); in lowerFTRUNC_FCEIL_FFLOOR()
1895 SDValue Abs = DAG.getNode(ISD::FABS, DL, VT, Src); in lowerFROUND()
1904 SDValue Adjust = DAG.getNode(ISD::FADD, DL, VT, Abs, in lowerFROUND()
1909 SDValue Truncated = DAG.getNode(ISD::FP_TO_SINT, DL, IntVT, Adjust); in lowerFROUND()
1910 Truncated = DAG.getNode(ISD::SINT_TO_FP, DL, VT, Truncated); in lowerFROUND()
1913 Truncated = DAG.getNode(ISD::FCOPYSIGN, DL, VT, Truncated, Src); in lowerFROUND()
1926 SDValue Setcc = DAG.getSetCC(DL, SetccVT, Abs, MaxValNode, ISD::SETOLT); in lowerFROUND()
1948 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR"); in isSimpleVIDSequence()
2036 if (SplatVal.getOpcode() != ISD::EXTRACT_VECTOR_ELT) in matchSplatAsGather()
2082 if (ISD::isBuildVectorAllZeros(Op.getNode())) { in lowerBUILD_VECTOR()
2087 if (ISD::isBuildVectorAllOnes(Op.getNode())) { in lowerBUILD_VECTOR()
2103 if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) { in lowerBUILD_VECTOR()
2126 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec, in lowerBUILD_VECTOR()
2142 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec, Elt, in lowerBUILD_VECTOR()
2151 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Vec, in lowerBUILD_VECTOR()
2175 Splat = DAG.getNode(ISD::AND, DL, XLenVT, Splat, in lowerBUILD_VECTOR()
2182 WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne); in lowerBUILD_VECTOR()
2185 return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE); in lowerBUILD_VECTOR()
2209 unsigned StepOpcode = ISD::MUL; in lowerBUILD_VECTOR()
2213 StepOpcode = ISD::SHL; in lowerBUILD_VECTOR()
2222 if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) || in lowerBUILD_VECTOR()
2223 (StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) && in lowerBUILD_VECTOR()
2232 if ((StepOpcode == ISD::MUL && SplatStepVal != 1) || in lowerBUILD_VECTOR()
2233 (StepOpcode == ISD::SHL && SplatStepVal != 0)) { in lowerBUILD_VECTOR()
2241 VID = DAG.getNode(ISD::SRL, DL, VT, VID, SplatStep); in lowerBUILD_VECTOR()
2246 VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VT, SplatAddend, VID); in lowerBUILD_VECTOR()
2263 ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) && in lowerBUILD_VECTOR()
2369 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Vec, V, in lowerBUILD_VECTOR()
2379 Vec = DAG.getNode(ISD::VSELECT, DL, VT, in lowerBUILD_VECTOR()
2414 return DAG.getNode(ISD::BITCAST, DL, VT, InterVec); in splatPartsI64WithVL()
2430 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar, in splatSplitI64WithVL()
2432 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar, in splatSplitI64WithVL()
2462 isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND; in lowerScalarSplat()
2625 if (V.getOpcode() == ISD::CONCAT_VECTORS) { in lowerVECTOR_SHUFFLE()
2633 if (ISD::isNormalLoad(V.getNode()) && cast<LoadSDNode>(V)->isSimple()) { in lowerVECTOR_SHUFFLE()
2651 ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, SVT, in lowerVECTOR_SHUFFLE()
2667 V = DAG.getExtLoad(ISD::SEXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr, in lowerVECTOR_SHUFFLE()
2740 V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1, in lowerVECTOR_SHUFFLE()
2742 V2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V2, in lowerVECTOR_SHUFFLE()
2857 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2); in lowerVECTOR_SHUFFLE()
2974 if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF) { in lowerCTLZ_CTTZ_ZERO_UNDEF()
2976 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Src); in lowerCTLZ_CTTZ_ZERO_UNDEF()
2977 Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg); in lowerCTLZ_CTTZ_ZERO_UNDEF()
2981 SDValue FloatVal = DAG.getNode(ISD::UINT_TO_FP, DL, FloatVT, Src); in lowerCTLZ_CTTZ_ZERO_UNDEF()
2986 SDValue Shift = DAG.getNode(ISD::SRL, DL, IntVT, Bitcast, in lowerCTLZ_CTTZ_ZERO_UNDEF()
2989 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, Shift); in lowerCTLZ_CTTZ_ZERO_UNDEF()
2994 if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF) in lowerCTLZ_CTTZ_ZERO_UNDEF()
2995 return DAG.getNode(ISD::SUB, DL, VT, Trunc, in lowerCTLZ_CTTZ_ZERO_UNDEF()
3001 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Trunc); in lowerCTLZ_CTTZ_ZERO_UNDEF()
3095 case ISD::GlobalAddress: in LowerOperation()
3097 case ISD::BlockAddress: in LowerOperation()
3099 case ISD::ConstantPool: in LowerOperation()
3101 case ISD::JumpTable: in LowerOperation()
3103 case ISD::GlobalTLSAddress: in LowerOperation()
3105 case ISD::Constant: in LowerOperation()
3107 case ISD::SELECT: in LowerOperation()
3109 case ISD::BRCOND: in LowerOperation()
3111 case ISD::VASTART: in LowerOperation()
3113 case ISD::FRAMEADDR: in LowerOperation()
3115 case ISD::RETURNADDR: in LowerOperation()
3117 case ISD::SHL_PARTS: in LowerOperation()
3119 case ISD::SRA_PARTS: in LowerOperation()
3121 case ISD::SRL_PARTS: in LowerOperation()
3123 case ISD::BITCAST: { in LowerOperation()
3130 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0); in LowerOperation()
3136 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0); in LowerOperation()
3165 return DAG.getBitcast(VT, DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, BVT, in LowerOperation()
3179 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec, in LowerOperation()
3184 case ISD::INTRINSIC_WO_CHAIN: in LowerOperation()
3186 case ISD::INTRINSIC_W_CHAIN: in LowerOperation()
3188 case ISD::INTRINSIC_VOID: in LowerOperation()
3190 case ISD::BSWAP: in LowerOperation()
3191 case ISD::BITREVERSE: { in LowerOperation()
3199 if (Op.getOpcode() == ISD::BSWAP) in LowerOperation()
3205 assert(Op.getOpcode() == ISD::BITREVERSE && "Unexpected opcode"); in LowerOperation()
3207 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Op.getOperand(0)); in LowerOperation()
3213 case ISD::FSHL: in LowerOperation()
3214 case ISD::FSHR: { in LowerOperation()
3222 SDValue ShAmt = DAG.getNode(ISD::AND, DL, VT, Op.getOperand(2), in LowerOperation()
3232 if (Op.getOpcode() == ISD::FSHR) { in LowerOperation()
3238 case ISD::TRUNCATE: in LowerOperation()
3243 case ISD::ANY_EXTEND: in LowerOperation()
3244 case ISD::ZERO_EXTEND: in LowerOperation()
3249 case ISD::SIGN_EXTEND: in LowerOperation()
3254 case ISD::SPLAT_VECTOR_PARTS: in LowerOperation()
3256 case ISD::INSERT_VECTOR_ELT: in LowerOperation()
3258 case ISD::EXTRACT_VECTOR_ELT: in LowerOperation()
3260 case ISD::VSCALE: { in LowerOperation()
3276 return DAG.getNode(ISD::SRL, DL, VT, VLENB, in LowerOperation()
3279 return DAG.getNode(ISD::SHL, DL, VT, VLENB, in LowerOperation()
3286 return DAG.getNode(ISD::MUL, DL, VT, VLENB, in LowerOperation()
3289 SDValue VScale = DAG.getNode(ISD::SRL, DL, VT, VLENB, in LowerOperation()
3291 return DAG.getNode(ISD::MUL, DL, VT, VScale, Op.getOperand(0)); in LowerOperation()
3293 case ISD::FPOWI: { in LowerOperation()
3299 SDValue Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0)); in LowerOperation()
3301 DAG.getNode(ISD::FPOWI, DL, MVT::f32, Op0, Op.getOperand(1)); in LowerOperation()
3302 return DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, Powi, in LowerOperation()
3307 case ISD::FP_EXTEND: in LowerOperation()
3308 case ISD::FP_ROUND: in LowerOperation()
3312 case ISD::FP_TO_SINT: in LowerOperation()
3313 case ISD::FP_TO_UINT: in LowerOperation()
3314 case ISD::SINT_TO_FP: in LowerOperation()
3315 case ISD::UINT_TO_FP: { in LowerOperation()
3339 unsigned ExtOpcode = Op.getOpcode() == ISD::UINT_TO_FP in LowerOperation()
3340 ? ISD::ZERO_EXTEND in LowerOperation()
3341 : ISD::SIGN_EXTEND; in LowerOperation()
3369 return DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int); in LowerOperation()
3382 case ISD::FP_TO_SINT: in LowerOperation()
3385 case ISD::FP_TO_UINT: in LowerOperation()
3388 case ISD::SINT_TO_FP: in LowerOperation()
3391 case ISD::UINT_TO_FP: in LowerOperation()
3414 case ISD::FP_TO_SINT_SAT: in LowerOperation()
3415 case ISD::FP_TO_UINT_SAT: in LowerOperation()
3417 case ISD::FTRUNC: in LowerOperation()
3418 case ISD::FCEIL: in LowerOperation()
3419 case ISD::FFLOOR: in LowerOperation()
3421 case ISD::FROUND: in LowerOperation()
3423 case ISD::VECREDUCE_ADD: in LowerOperation()
3424 case ISD::VECREDUCE_UMAX: in LowerOperation()
3425 case ISD::VECREDUCE_SMAX: in LowerOperation()
3426 case ISD::VECREDUCE_UMIN: in LowerOperation()
3427 case ISD::VECREDUCE_SMIN: in LowerOperation()
3429 case ISD::VECREDUCE_AND: in LowerOperation()
3430 case ISD::VECREDUCE_OR: in LowerOperation()
3431 case ISD::VECREDUCE_XOR: in LowerOperation()
3435 case ISD::VECREDUCE_FADD: in LowerOperation()
3436 case ISD::VECREDUCE_SEQ_FADD: in LowerOperation()
3437 case ISD::VECREDUCE_FMIN: in LowerOperation()
3438 case ISD::VECREDUCE_FMAX: in LowerOperation()
3440 case ISD::VP_REDUCE_ADD: in LowerOperation()
3441 case ISD::VP_REDUCE_UMAX: in LowerOperation()
3442 case ISD::VP_REDUCE_SMAX: in LowerOperation()
3443 case ISD::VP_REDUCE_UMIN: in LowerOperation()
3444 case ISD::VP_REDUCE_SMIN: in LowerOperation()
3445 case ISD::VP_REDUCE_FADD: in LowerOperation()
3446 case ISD::VP_REDUCE_SEQ_FADD: in LowerOperation()
3447 case ISD::VP_REDUCE_FMIN: in LowerOperation()
3448 case ISD::VP_REDUCE_FMAX: in LowerOperation()
3450 case ISD::VP_REDUCE_AND: in LowerOperation()
3451 case ISD::VP_REDUCE_OR: in LowerOperation()
3452 case ISD::VP_REDUCE_XOR: in LowerOperation()
3456 case ISD::INSERT_SUBVECTOR: in LowerOperation()
3458 case ISD::EXTRACT_SUBVECTOR: in LowerOperation()
3460 case ISD::STEP_VECTOR: in LowerOperation()
3462 case ISD::VECTOR_REVERSE: in LowerOperation()
3464 case ISD::VECTOR_SPLICE: in LowerOperation()
3466 case ISD::BUILD_VECTOR: in LowerOperation()
3468 case ISD::SPLAT_VECTOR: in LowerOperation()
3472 case ISD::VECTOR_SHUFFLE: in LowerOperation()
3474 case ISD::CONCAT_VECTORS: { in LowerOperation()
3487 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, SubVec, in LowerOperation()
3492 case ISD::LOAD: in LowerOperation()
3498 case ISD::STORE: in LowerOperation()
3504 case ISD::MLOAD: in LowerOperation()
3505 case ISD::VP_LOAD: in LowerOperation()
3507 case ISD::MSTORE: in LowerOperation()
3508 case ISD::VP_STORE: in LowerOperation()
3510 case ISD::SETCC: in LowerOperation()
3512 case ISD::ADD: in LowerOperation()
3514 case ISD::SUB: in LowerOperation()
3516 case ISD::MUL: in LowerOperation()
3518 case ISD::MULHS: in LowerOperation()
3520 case ISD::MULHU: in LowerOperation()
3522 case ISD::AND: in LowerOperation()
3525 case ISD::OR: in LowerOperation()
3528 case ISD::XOR: in LowerOperation()
3531 case ISD::SDIV: in LowerOperation()
3533 case ISD::SREM: in LowerOperation()
3535 case ISD::UDIV: in LowerOperation()
3537 case ISD::UREM: in LowerOperation()
3539 case ISD::SHL: in LowerOperation()
3540 case ISD::SRA: in LowerOperation()
3541 case ISD::SRL: in LowerOperation()
3548 case ISD::SADDSAT: in LowerOperation()
3550 case ISD::UADDSAT: in LowerOperation()
3552 case ISD::SSUBSAT: in LowerOperation()
3554 case ISD::USUBSAT: in LowerOperation()
3556 case ISD::FADD: in LowerOperation()
3558 case ISD::FSUB: in LowerOperation()
3560 case ISD::FMUL: in LowerOperation()
3562 case ISD::FDIV: in LowerOperation()
3564 case ISD::FNEG: in LowerOperation()
3566 case ISD::FABS: in LowerOperation()
3568 case ISD::FSQRT: in LowerOperation()
3570 case ISD::FMA: in LowerOperation()
3572 case ISD::SMIN: in LowerOperation()
3574 case ISD::SMAX: in LowerOperation()
3576 case ISD::UMIN: in LowerOperation()
3578 case ISD::UMAX: in LowerOperation()
3580 case ISD::FMINNUM: in LowerOperation()
3582 case ISD::FMAXNUM: in LowerOperation()
3584 case ISD::ABS: in LowerOperation()
3586 case ISD::CTLZ_ZERO_UNDEF: in LowerOperation()
3587 case ISD::CTTZ_ZERO_UNDEF: in LowerOperation()
3589 case ISD::VSELECT: in LowerOperation()
3591 case ISD::FCOPYSIGN: in LowerOperation()
3593 case ISD::MGATHER: in LowerOperation()
3594 case ISD::VP_GATHER: in LowerOperation()
3596 case ISD::MSCATTER: in LowerOperation()
3597 case ISD::VP_SCATTER: in LowerOperation()
3599 case ISD::FLT_ROUNDS_: in LowerOperation()
3601 case ISD::SET_ROUNDING: in LowerOperation()
3603 case ISD::EH_DWARF_CFA: in LowerOperation()
3605 case ISD::VP_SELECT: in LowerOperation()
3607 case ISD::VP_MERGE: in LowerOperation()
3609 case ISD::VP_ADD: in LowerOperation()
3611 case ISD::VP_SUB: in LowerOperation()
3613 case ISD::VP_MUL: in LowerOperation()
3615 case ISD::VP_SDIV: in LowerOperation()
3617 case ISD::VP_UDIV: in LowerOperation()
3619 case ISD::VP_SREM: in LowerOperation()
3621 case ISD::VP_UREM: in LowerOperation()
3623 case ISD::VP_AND: in LowerOperation()
3625 case ISD::VP_OR: in LowerOperation()
3627 case ISD::VP_XOR: in LowerOperation()
3629 case ISD::VP_ASHR: in LowerOperation()
3631 case ISD::VP_LSHR: in LowerOperation()
3633 case ISD::VP_SHL: in LowerOperation()
3635 case ISD::VP_FADD: in LowerOperation()
3637 case ISD::VP_FSUB: in LowerOperation()
3639 case ISD::VP_FMUL: in LowerOperation()
3641 case ISD::VP_FDIV: in LowerOperation()
3643 case ISD::VP_FNEG: in LowerOperation()
3645 case ISD::VP_FMA: in LowerOperation()
3647 case ISD::VP_SIGN_EXTEND: in LowerOperation()
3648 case ISD::VP_ZERO_EXTEND: in LowerOperation()
3652 Op.getOpcode() == ISD::VP_SIGN_EXTEND in LowerOperation()
3655 case ISD::VP_TRUNCATE: in LowerOperation()
3657 case ISD::VP_FP_EXTEND: in LowerOperation()
3658 case ISD::VP_FP_ROUND: in LowerOperation()
3660 case ISD::VP_FPTOSI: in LowerOperation()
3662 case ISD::VP_FPTOUI: in LowerOperation()
3664 case ISD::VP_SITOFP: in LowerOperation()
3666 case ISD::VP_UITOFP: in LowerOperation()
3668 case ISD::VP_SETCC: in LowerOperation()
3802 return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg); in getStaticTLSAddr()
3897 return DAG.getNode(ISD::VSELECT, DL, VT, CondSplat, TrueV, FalseV); in lowerSELECT()
3906 if (VT == XLenVT && CondV.getOpcode() == ISD::SETCC && in lowerSELECT()
3911 ISD::CondCode CCVal = CC->get(); in lowerSELECT()
3921 CCVal == ISD::SETLT) { in lowerSELECT()
3925 return DAG.getNode(ISD::ADD, DL, Op.getValueType(), CondV, FalseV); in lowerSELECT()
3927 return DAG.getNode(ISD::SUB, DL, Op.getValueType(), FalseV, CondV); in lowerSELECT()
3941 SDValue SetNE = DAG.getCondCode(ISD::SETNE); in lowerSELECT()
3953 if (CondV.getOpcode() == ISD::SETCC && in lowerBRCOND()
3957 ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get(); in lowerBRCOND()
3968 DAG.getCondCode(ISD::SETNE), Op.getOperand(2)); in lowerBRCOND()
4001 SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr, in lowerFRAMEADDR()
4029 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset), in lowerRETURNADDR()
4058 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen); in lowerShiftLeftParts()
4059 SDValue XLenMinus1Shamt = DAG.getNode(ISD::XOR, DL, VT, Shamt, XLenMinus1); in lowerShiftLeftParts()
4061 SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt); in lowerShiftLeftParts()
4062 SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One); in lowerShiftLeftParts()
4064 DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt); in lowerShiftLeftParts()
4065 SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt); in lowerShiftLeftParts()
4066 SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo); in lowerShiftLeftParts()
4067 SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen); in lowerShiftLeftParts()
4069 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT); in lowerShiftLeftParts()
4071 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero); in lowerShiftLeftParts()
4072 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse); in lowerShiftLeftParts()
4102 unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL; in lowerShiftRightParts()
4108 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen); in lowerShiftRightParts()
4109 SDValue XLenMinus1Shamt = DAG.getNode(ISD::XOR, DL, VT, Shamt, XLenMinus1); in lowerShiftRightParts()
4111 SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt); in lowerShiftRightParts()
4112 SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One); in lowerShiftRightParts()
4114 DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt); in lowerShiftRightParts()
4115 SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi); in lowerShiftRightParts()
4119 IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero; in lowerShiftRightParts()
4121 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT); in lowerShiftRightParts()
4123 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse); in lowerShiftRightParts()
4124 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse); in lowerShiftRightParts()
4138 if (ISD::isConstantSplatVectorAllOnes(Op.getNode())) { in lowerVectorMaskSplat()
4142 if (ISD::isConstantSplatVectorAllZeros(Op.getNode())) { in lowerVectorMaskSplat()
4150 SplatVal = DAG.getNode(ISD::AND, DL, XLenVT, SplatVal, in lowerVectorMaskSplat()
4154 return DAG.getSetCC(DL, VT, LHS, Zero, ISD::SETNE); in lowerVectorMaskSplat()
4195 if (Hi.getOpcode() == ISD::SRA && Hi.getOperand(0) == Lo && in lowerSPLAT_VECTOR_PARTS()
4223 return DAG.getNode(ISD::VSELECT, DL, VecVT, Src, SplatTrueVal, SplatZero); in lowerVectorMaskExt()
4282 bool IsVPTrunc = Op.getOpcode() == ISD::VP_TRUNCATE; in lowerVectorMaskTruncLike()
4325 DAG.getCondCode(ISD::SETNE), Mask, VL); in lowerVectorMaskTruncLike()
4333 bool IsVPTrunc = Op.getOpcode() == ISD::VP_TRUNCATE; in lowerVectorTruncLike()
4397 Op.getOpcode() == ISD::VP_FP_ROUND || Op.getOpcode() == ISD::VP_FP_EXTEND; in lowerVectorFPExtendOrRoundLike()
4399 Op.getOpcode() == ISD::VP_FP_EXTEND || Op.getOpcode() == ISD::FP_EXTEND; in lowerVectorFPExtendOrRoundLike()
4480 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec); in lowerINSERT_VECTOR_ELT()
4481 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideVT, Vec, Val, Idx); in lowerINSERT_VECTOR_ELT()
4482 return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Vec); in lowerINSERT_VECTOR_ELT()
4530 SDValue ValLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Val, Zero); in lowerINSERT_VECTOR_ELT()
4531 SDValue ValHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Val, One); in lowerINSERT_VECTOR_ELT()
4555 DAG.getNode(ISD::ADD, DL, XLenVT, Idx, DAG.getConstant(1, DL, XLenVT)); in lowerINSERT_VECTOR_ELT()
4599 ISD::SRL, DL, XLenVT, Idx, in lowerEXTRACT_VECTOR_ELT()
4603 ISD::AND, DL, XLenVT, Idx, in lowerEXTRACT_VECTOR_ELT()
4607 Vec = DAG.getNode(ISD::BITCAST, DL, WideVT, Vec); in lowerEXTRACT_VECTOR_ELT()
4608 SDValue ExtractElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, XLenVT, in lowerEXTRACT_VECTOR_ELT()
4612 DAG.getNode(ISD::SRL, DL, XLenVT, ExtractElt, ExtractBitIdx); in lowerEXTRACT_VECTOR_ELT()
4613 return DAG.getNode(ISD::AND, DL, XLenVT, ShiftRight, in lowerEXTRACT_VECTOR_ELT()
4619 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec); in lowerEXTRACT_VECTOR_ELT()
4620 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec, Idx); in lowerEXTRACT_VECTOR_ELT()
4641 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec, in lowerEXTRACT_VECTOR_ELT()
4646 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Elt0); in lowerEXTRACT_VECTOR_ELT()
4653 assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || in lowerVectorIntrinsicScalars()
4654 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) && in lowerVectorIntrinsicScalars()
4660 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN; in lowerVectorIntrinsicScalars()
4688 isa<ConstantSDNode>(ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND; in lowerVectorIntrinsicScalars()
4709 ScalarOp = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, ScalarOp); in lowerVectorIntrinsicScalars()
4726 SDValue ScalarLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, ScalarOp, in lowerVectorIntrinsicScalars()
4728 SDValue ScalarHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, ScalarOp, in lowerVectorIntrinsicScalars()
4759 I32VL = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, SETVLMAX, SEW, in lowerVectorIntrinsicScalars()
4776 SDValue VL = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, SETVL, AVL, in lowerVectorIntrinsicScalars()
4779 DAG.getNode(ISD::SHL, DL, XLenVT, VL, DAG.getConstant(1, DL, XLenVT)); in lowerVectorIntrinsicScalars()
4911 Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Scalar); in LowerINTRINSIC_WO_CHAIN()
4949 DAG.getCondCode(ISD::SETEQ), Mask, VL); in LowerINTRINSIC_WO_CHAIN()
4971 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode()); in LowerINTRINSIC_W_CHAIN()
5007 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, in LowerINTRINSIC_W_CHAIN()
5043 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, in LowerINTRINSIC_W_CHAIN()
5070 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode()); in LowerINTRINSIC_VOID()
5097 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, Store->getVTList(), in LowerINTRINSIC_VOID()
5118 case ISD::VECREDUCE_ADD: in getRVVReductionOp()
5120 case ISD::VECREDUCE_UMAX: in getRVVReductionOp()
5122 case ISD::VECREDUCE_SMAX: in getRVVReductionOp()
5124 case ISD::VECREDUCE_UMIN: in getRVVReductionOp()
5126 case ISD::VECREDUCE_SMIN: in getRVVReductionOp()
5128 case ISD::VECREDUCE_AND: in getRVVReductionOp()
5130 case ISD::VECREDUCE_OR: in getRVVReductionOp()
5132 case ISD::VECREDUCE_XOR: in getRVVReductionOp()
5143 assert((Op.getOpcode() == ISD::VECREDUCE_AND || in lowerVectorMaskVecReduction()
5144 Op.getOpcode() == ISD::VECREDUCE_OR || in lowerVectorMaskVecReduction()
5145 Op.getOpcode() == ISD::VECREDUCE_XOR || in lowerVectorMaskVecReduction()
5146 Op.getOpcode() == ISD::VP_REDUCE_AND || in lowerVectorMaskVecReduction()
5147 Op.getOpcode() == ISD::VP_REDUCE_OR || in lowerVectorMaskVecReduction()
5148 Op.getOpcode() == ISD::VP_REDUCE_XOR) && in lowerVectorMaskVecReduction()
5171 ISD::CondCode CC; in lowerVectorMaskVecReduction()
5177 case ISD::VECREDUCE_AND: in lowerVectorMaskVecReduction()
5178 case ISD::VP_REDUCE_AND: { in lowerVectorMaskVecReduction()
5183 CC = ISD::SETEQ; in lowerVectorMaskVecReduction()
5184 BaseOpc = ISD::AND; in lowerVectorMaskVecReduction()
5187 case ISD::VECREDUCE_OR: in lowerVectorMaskVecReduction()
5188 case ISD::VP_REDUCE_OR: in lowerVectorMaskVecReduction()
5191 CC = ISD::SETNE; in lowerVectorMaskVecReduction()
5192 BaseOpc = ISD::OR; in lowerVectorMaskVecReduction()
5194 case ISD::VECREDUCE_XOR: in lowerVectorMaskVecReduction()
5195 case ISD::VP_REDUCE_XOR: { in lowerVectorMaskVecReduction()
5199 Vec = DAG.getNode(ISD::AND, DL, XLenVT, Vec, One); in lowerVectorMaskVecReduction()
5200 CC = ISD::SETNE; in lowerVectorMaskVecReduction()
5201 BaseOpc = ISD::XOR; in lowerVectorMaskVecReduction()
5226 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op.getOpcode()); in lowerVECREDUCE()
5266 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction, in lowerVECREDUCE()
5279 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Opcode); in getRVVFPReductionOpAndOperands()
5283 case ISD::VECREDUCE_FADD: { in getRVVFPReductionOpAndOperands()
5289 case ISD::VECREDUCE_SEQ_FADD: in getRVVFPReductionOpAndOperands()
5292 case ISD::VECREDUCE_FMIN: in getRVVFPReductionOpAndOperands()
5295 case ISD::VECREDUCE_FMAX: in getRVVFPReductionOpAndOperands()
5329 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction, in lowerFPVECREDUCE()
5337 case ISD::VP_REDUCE_ADD: in getRVVVPReductionOp()
5339 case ISD::VP_REDUCE_UMAX: in getRVVVPReductionOp()
5341 case ISD::VP_REDUCE_SMAX: in getRVVVPReductionOp()
5343 case ISD::VP_REDUCE_UMIN: in getRVVVPReductionOp()
5345 case ISD::VP_REDUCE_SMIN: in getRVVVPReductionOp()
5347 case ISD::VP_REDUCE_AND: in getRVVVPReductionOp()
5349 case ISD::VP_REDUCE_OR: in getRVVVPReductionOp()
5351 case ISD::VP_REDUCE_XOR: in getRVVVPReductionOp()
5353 case ISD::VP_REDUCE_FADD: in getRVVVPReductionOp()
5355 case ISD::VP_REDUCE_SEQ_FADD: in getRVVVPReductionOp()
5357 case ISD::VP_REDUCE_FMAX: in getRVVVPReductionOp()
5359 case ISD::VP_REDUCE_FMIN: in getRVVVPReductionOp()
5397 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Reduction, in lowerVPREDUCE()
5444 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec); in lowerINSERT_SUBVECTOR()
5445 SubVec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtSubVecVT, SubVec); in lowerINSERT_SUBVECTOR()
5446 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ExtVecVT, Vec, SubVec, in lowerINSERT_SUBVECTOR()
5449 return DAG.getSetCC(DL, VecVT, Vec, SplatZero, ISD::SETNE); in lowerINSERT_SUBVECTOR()
5466 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, in lowerINSERT_SUBVECTOR()
5524 AlignedExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec, in lowerINSERT_SUBVECTOR()
5530 SlideupAmt = DAG.getNode(ISD::VSCALE, DL, XLenVT, SlideupAmt); in lowerINSERT_SUBVECTOR()
5537 VL = DAG.getNode(ISD::VSCALE, DL, XLenVT, VL); in lowerINSERT_SUBVECTOR()
5538 VL = DAG.getNode(ISD::ADD, DL, XLenVT, SlideupAmt, VL); in lowerINSERT_SUBVECTOR()
5540 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InterSubVT, in lowerINSERT_SUBVECTOR()
5550 Slideup = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, Vec, Slideup, in lowerINSERT_SUBVECTOR()
5598 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec); in lowerEXTRACT_SUBVECTOR()
5599 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtSubVecVT, Vec, in lowerEXTRACT_SUBVECTOR()
5602 return DAG.getSetCC(DL, SubVecVT, Vec, SplatZero, ISD::SETNE); in lowerEXTRACT_SUBVECTOR()
5631 Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown, in lowerEXTRACT_SUBVECTOR()
5656 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec, in lowerEXTRACT_SUBVECTOR()
5664 SlidedownAmt = DAG.getNode(ISD::VSCALE, DL, XLenVT, SlidedownAmt); in lowerEXTRACT_SUBVECTOR()
5674 Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown, in lowerEXTRACT_SUBVECTOR()
5698 StepVec = DAG.getNode(ISD::SHL, DL, VT, StepVec, StepVal); in lowerSTEP_VECTOR()
5703 StepVec = DAG.getNode(ISD::MUL, DL, VT, StepVec, StepVal); in lowerSTEP_VECTOR()
5720 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, Op.getOperand(0)); in lowerVECTOR_REVERSE()
5721 SDValue Op2 = DAG.getNode(ISD::VECTOR_REVERSE, DL, WidenVT, Op1); in lowerVECTOR_REVERSE()
5722 return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Op2); in lowerVECTOR_REVERSE()
5747 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo); in lowerVECTOR_REVERSE()
5748 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi); in lowerVECTOR_REVERSE()
5752 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, DAG.getUNDEF(VecVT), Hi, in lowerVECTOR_REVERSE()
5755 ISD::INSERT_SUBVECTOR, DL, VecVT, Res, Lo, in lowerVECTOR_REVERSE()
5770 SDValue VLMax = DAG.getNode(ISD::VSCALE, DL, XLenVT, in lowerVECTOR_REVERSE()
5773 DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, DAG.getConstant(1, DL, XLenVT)); in lowerVECTOR_REVERSE()
5802 SDValue VLMax = DAG.getNode(ISD::VSCALE, DL, XLenVT, in lowerVECTOR_SPLICE()
5811 UpOffset = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, DownOffset); in lowerVECTOR_SPLICE()
5816 DownOffset = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, UpOffset); in lowerVECTOR_SPLICE()
5855 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, in lowerFixedLengthVectorLoadToRVV()
5880 StoreVal = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in lowerFixedLengthVectorStoreToRVV()
5896 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), in lowerFixedLengthVectorStoreToRVV()
5923 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode()); in lowerMaskedLoad()
5957 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO); in lowerMaskedLoad()
5987 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode()); in lowerMaskedStore()
6015 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, in lowerMaskedStore()
6061 case ISD::SHL: Opc = RISCVISD::SHL_VL; break; in lowerFixedLengthVectorShiftToRVV()
6062 case ISD::SRA: Opc = RISCVISD::SRA_VL; break; in lowerFixedLengthVectorShiftToRVV()
6063 case ISD::SRL: Opc = RISCVISD::SRL_VL; break; in lowerFixedLengthVectorShiftToRVV()
6234 Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1, DL, XLenVT); in lowerVPExtMaskOp()
6252 ISD::CondCode Condition = cast<CondCodeSDNode>(Op.getOperand(2))->get(); in lowerVPSetCCMaskOp()
6270 case ISD::SETNE: in lowerVPSetCCMaskOp()
6274 case ISD::SETEQ: { in lowerVPSetCCMaskOp()
6283 case ISD::SETGT: in lowerVPSetCCMaskOp()
6284 case ISD::SETULT: { in lowerVPSetCCMaskOp()
6292 case ISD::SETLT: in lowerVPSetCCMaskOp()
6293 case ISD::SETUGT: { in lowerVPSetCCMaskOp()
6301 case ISD::SETGE: in lowerVPSetCCMaskOp()
6302 case ISD::SETULE: { in lowerVPSetCCMaskOp()
6310 case ISD::SETLE: in lowerVPSetCCMaskOp()
6311 case ISD::SETUGE: { in lowerVPSetCCMaskOp()
6432 DAG.getCondCode(ISD::SETNE), Mask, VL); in lowerVPFPIntConvOp()
6501 ISD::LoadExtType LoadExtType; in lowerMaskedGather()
6510 LoadExtType = ISD::NON_EXTLOAD; in lowerMaskedGather()
6527 assert(LoadExtType == ISD::NON_EXTLOAD && in lowerMaskedGather()
6533 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode()); in lowerMaskedGather()
6578 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO); in lowerMaskedGather()
6635 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode()); in lowerMaskedScatter()
6673 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, in lowerMaskedScatter()
6699 DAG.getNode(ISD::SHL, DL, XLenVT, RM, DAG.getConstant(2, DL, XLenVT)); in lowerGET_ROUNDING()
6700 SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT, in lowerGET_ROUNDING()
6702 SDValue Masked = DAG.getNode(ISD::AND, DL, XLenVT, Shifted, in lowerGET_ROUNDING()
6728 SDValue Shift = DAG.getNode(ISD::SHL, DL, XLenVT, RMValue, in lowerSET_ROUNDING()
6730 SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT, in lowerSET_ROUNDING()
6732 RMValue = DAG.getNode(ISD::AND, DL, XLenVT, Shifted, in lowerSET_ROUNDING()
6775 NewOps.push_back(DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op)); in customLegalizeToWOpByIntr()
6778 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes); in customLegalizeToWOpByIntr()
6787 case ISD::SHL: in getRISCVWOpcode()
6789 case ISD::SRA: in getRISCVWOpcode()
6791 case ISD::SRL: in getRISCVWOpcode()
6793 case ISD::SDIV: in getRISCVWOpcode()
6795 case ISD::UDIV: in getRISCVWOpcode()
6797 case ISD::UREM: in getRISCVWOpcode()
6799 case ISD::ROTL: in getRISCVWOpcode()
6801 case ISD::ROTR: in getRISCVWOpcode()
6812 unsigned ExtOpc = ISD::ANY_EXTEND) { in customLegalizeToWOp()
6819 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes); in customLegalizeToWOp()
6826 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)); in customLegalizeToWOpWithSExt()
6827 SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); in customLegalizeToWOpWithSExt()
6829 SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp, in customLegalizeToWOpWithSExt()
6831 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes); in customLegalizeToWOpWithSExt()
6841 case ISD::STRICT_FP_TO_SINT: in ReplaceNodeResults()
6842 case ISD::STRICT_FP_TO_UINT: in ReplaceNodeResults()
6843 case ISD::FP_TO_SINT: in ReplaceNodeResults()
6844 case ISD::FP_TO_UINT: { in ReplaceNodeResults()
6848 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT || in ReplaceNodeResults()
6849 N->getOpcode() == ISD::STRICT_FP_TO_SINT; in ReplaceNodeResults()
6862 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
6870 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
6894 case ISD::READCYCLECOUNTER: { in ReplaceNodeResults()
6903 DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, RCW, RCW.getValue(1))); in ReplaceNodeResults()
6907 case ISD::MUL: { in ReplaceNodeResults()
6925 S = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, S); in ReplaceNodeResults()
6926 U = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, U); in ReplaceNodeResults()
6927 SDValue Lo = DAG.getNode(ISD::MUL, DL, XLenVT, S, U); in ReplaceNodeResults()
6929 return DAG.getNode(ISD::BUILD_PAIR, DL, N->getValueType(0), Lo, Hi); in ReplaceNodeResults()
6946 case ISD::ADD: in ReplaceNodeResults()
6947 case ISD::SUB: in ReplaceNodeResults()
6952 case ISD::SHL: in ReplaceNodeResults()
6953 case ISD::SRA: in ReplaceNodeResults()
6954 case ISD::SRL: in ReplaceNodeResults()
6957 if (N->getOperand(1).getOpcode() != ISD::Constant) { in ReplaceNodeResults()
6959 if (N->getOpcode() == ISD::SHL && Subtarget.hasStdExtZbs() && in ReplaceNodeResults()
6969 if (N->getOpcode() == ISD::SHL) { in ReplaceNodeResults()
6972 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
6974 DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
6975 SDValue NewWOp = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0, NewOp1); in ReplaceNodeResults()
6976 SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp, in ReplaceNodeResults()
6978 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes)); in ReplaceNodeResults()
6982 case ISD::ROTL: in ReplaceNodeResults()
6983 case ISD::ROTR: in ReplaceNodeResults()
6988 case ISD::CTTZ: in ReplaceNodeResults()
6989 case ISD::CTTZ_ZERO_UNDEF: in ReplaceNodeResults()
6990 case ISD::CTLZ: in ReplaceNodeResults()
6991 case ISD::CTLZ_ZERO_UNDEF: { in ReplaceNodeResults()
6996 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
6998 N->getOpcode() == ISD::CTTZ || N->getOpcode() == ISD::CTTZ_ZERO_UNDEF; in ReplaceNodeResults()
7001 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
7004 case ISD::SDIV: in ReplaceNodeResults()
7005 case ISD::UDIV: in ReplaceNodeResults()
7006 case ISD::UREM: { in ReplaceNodeResults()
7014 if (N->getOperand(1).getOpcode() == ISD::Constant) in ReplaceNodeResults()
7020 unsigned ExtOpc = ISD::ANY_EXTEND; in ReplaceNodeResults()
7022 ExtOpc = N->getOpcode() == ISD::SDIV ? ISD::SIGN_EXTEND in ReplaceNodeResults()
7023 : ISD::ZERO_EXTEND; in ReplaceNodeResults()
7028 case ISD::UADDO: in ReplaceNodeResults()
7029 case ISD::USUBO: { in ReplaceNodeResults()
7032 bool IsAdd = N->getOpcode() == ISD::UADDO; in ReplaceNodeResults()
7034 SDValue LHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
7035 SDValue RHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
7037 DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS); in ReplaceNodeResults()
7038 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res, in ReplaceNodeResults()
7049 DAG.getConstant(0, DL, MVT::i64), ISD::SETEQ); in ReplaceNodeResults()
7054 LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
7056 IsAdd ? ISD::SETULT : ISD::SETUGT); in ReplaceNodeResults()
7059 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
7063 case ISD::UADDSAT: in ReplaceNodeResults()
7064 case ISD::USUBSAT: { in ReplaceNodeResults()
7072 DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
7074 DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
7076 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
7085 case ISD::ABS: { in ReplaceNodeResults()
7091 SDValue Src = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
7097 SDValue SignFill = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Src, in ReplaceNodeResults()
7099 SignFill = DAG.getNode(ISD::SRA, DL, MVT::i64, SignFill, in ReplaceNodeResults()
7102 SDValue NewRes = DAG.getNode(ISD::XOR, DL, MVT::i64, Src, SignFill); in ReplaceNodeResults()
7103 NewRes = DAG.getNode(ISD::SUB, DL, MVT::i64, NewRes, SignFill); in ReplaceNodeResults()
7107 NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewRes, in ReplaceNodeResults()
7109 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes)); in ReplaceNodeResults()
7112 case ISD::BITCAST: { in ReplaceNodeResults()
7120 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv)); in ReplaceNodeResults()
7125 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv)); in ReplaceNodeResults()
7135 Results.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec, in ReplaceNodeResults()
7153 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0)); in ReplaceNodeResults()
7155 DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, N->getOperand(1)); in ReplaceNodeResults()
7159 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NewRes)); in ReplaceNodeResults()
7162 case ISD::BSWAP: in ReplaceNodeResults()
7163 case ISD::BITREVERSE: { in ReplaceNodeResults()
7169 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0)); in ReplaceNodeResults()
7172 if (N->getOpcode() == ISD::BSWAP) in ReplaceNodeResults()
7178 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, GREVI)); in ReplaceNodeResults()
7181 case ISD::FSHL: in ReplaceNodeResults()
7182 case ISD::FSHR: { in ReplaceNodeResults()
7186 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)); in ReplaceNodeResults()
7188 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
7190 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2)); in ReplaceNodeResults()
7193 NewShAmt = DAG.getNode(ISD::AND, DL, MVT::i64, NewShAmt, in ReplaceNodeResults()
7201 if (N->getOpcode() == ISD::FSHR) { in ReplaceNodeResults()
7206 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewOp)); in ReplaceNodeResults()
7209 case ISD::EXTRACT_VECTOR_ELT: { in ReplaceNodeResults()
7264 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi)); in ReplaceNodeResults()
7267 case ISD::INTRINSIC_WO_CHAIN: { in ReplaceNodeResults()
7278 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
7280 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2)); in ReplaceNodeResults()
7287 NewOp2 = DAG.getNode(ISD::AND, DL, MVT::i64, NewOp2, in ReplaceNodeResults()
7292 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
7308 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
7311 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
7319 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)); in ReplaceNodeResults()
7321 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2)); in ReplaceNodeResults()
7329 NewOp2 = DAG.getNode(ISD::AND, DL, MVT::i64, NewOp2, in ReplaceNodeResults()
7335 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); in ReplaceNodeResults()
7345 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Extract)); in ReplaceNodeResults()
7372 DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi)); in ReplaceNodeResults()
7378 case ISD::VECREDUCE_ADD: in ReplaceNodeResults()
7379 case ISD::VECREDUCE_AND: in ReplaceNodeResults()
7380 case ISD::VECREDUCE_OR: in ReplaceNodeResults()
7381 case ISD::VECREDUCE_XOR: in ReplaceNodeResults()
7382 case ISD::VECREDUCE_SMAX: in ReplaceNodeResults()
7383 case ISD::VECREDUCE_UMAX: in ReplaceNodeResults()
7384 case ISD::VECREDUCE_SMIN: in ReplaceNodeResults()
7385 case ISD::VECREDUCE_UMIN: in ReplaceNodeResults()
7389 case ISD::VP_REDUCE_ADD: in ReplaceNodeResults()
7390 case ISD::VP_REDUCE_AND: in ReplaceNodeResults()
7391 case ISD::VP_REDUCE_OR: in ReplaceNodeResults()
7392 case ISD::VP_REDUCE_XOR: in ReplaceNodeResults()
7393 case ISD::VP_REDUCE_SMAX: in ReplaceNodeResults()
7394 case ISD::VP_REDUCE_UMAX: in ReplaceNodeResults()
7395 case ISD::VP_REDUCE_SMIN: in ReplaceNodeResults()
7396 case ISD::VP_REDUCE_UMIN: in ReplaceNodeResults()
7400 case ISD::FLT_ROUNDS_: { in ReplaceNodeResults()
7402 SDValue Res = DAG.getNode(ISD::FLT_ROUNDS_, DL, VTs, N->getOperand(0)); in ReplaceNodeResults()
7441 if (Op.getOpcode() == ISD::AND && isa<ConstantSDNode>(Op.getOperand(1))) { in matchRISCVBitmanipPat()
7445 if (Op.getOpcode() != ISD::SHL && Op.getOpcode() != ISD::SRL) in matchRISCVBitmanipPat()
7447 bool IsSHL = Op.getOpcode() == ISD::SHL; in matchRISCVBitmanipPat()
7472 if (Src.getOpcode() == ISD::AND && isa<ConstantSDNode>(Src.getOperand(1))) { in matchRISCVBitmanipPat()
7528 case ISD::ADD: in combineBinOpToReduce()
7530 case ISD::UMAX: in combineBinOpToReduce()
7532 case ISD::SMAX: in combineBinOpToReduce()
7534 case ISD::UMIN: in combineBinOpToReduce()
7536 case ISD::SMIN: in combineBinOpToReduce()
7538 case ISD::AND: in combineBinOpToReduce()
7540 case ISD::OR: in combineBinOpToReduce()
7542 case ISD::XOR: in combineBinOpToReduce()
7544 case ISD::FADD: in combineBinOpToReduce()
7546 case ISD::FMAXNUM: in combineBinOpToReduce()
7548 case ISD::FMINNUM: in combineBinOpToReduce()
7554 return V.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in combineBinOpToReduce()
7569 if (Opc == ISD::FADD && !N->getFlags().hasAllowReassociation()) in combineBinOpToReduce()
7590 if (Opc == ISD::FADD && N->getFlags().hasNoSignedZeros() && in combineBinOpToReduce()
7670 if ((Reverse.getOpcode() == ISD::ROTL || in combineORToGORC()
7671 Reverse.getOpcode() == ISD::ROTR) && in combineORToGORC()
7689 if (Op0.getOpcode() != ISD::OR && Op1.getOpcode() == ISD::OR) in combineORToGORC()
7691 if (Op0.getOpcode() != ISD::OR) in combineORToGORC()
7745 if (Op0.getOpcode() != ISD::OR) in combineORToSHFL()
7747 if (Op0.getOpcode() != ISD::OR) in combineORToSHFL()
7783 if (C.getOpcode() != ISD::AND || !isa<ConstantSDNode>(C.getOperand(1)) || in combineORToSHFL()
7822 if (N0->getOpcode() != ISD::SHL || N1->getOpcode() != ISD::SHL || in transformAddShlImm()
7847 DAG.getNode(ISD::SHL, DL, VT, NL, DAG.getConstant(Diff, DL, VT)); in transformAddShlImm()
7848 SDValue NA1 = DAG.getNode(ISD::ADD, DL, VT, NA0, NS); in transformAddShlImm()
7849 return DAG.getNode(ISD::SHL, DL, VT, NA1, DAG.getConstant(Bits, DL, VT)); in transformAddShlImm()
7866 assert((N->getOpcode() == ISD::ROTR || N->getOpcode() == ISD::ROTL || in combineROTR_ROTL_RORW_ROLW()
7905 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Res, in combineROTR_ROTL_RORW_ROLW()
7964 if ((Slct.getOpcode() != ISD::SELECT && in combineSelectAndUse()
7999 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, in combineSelectAndUse()
8041 if (!N0->hasOneUse() || N0->getOpcode() != ISD::MUL) in transformAddImmMulImm()
8075 SDValue New0 = DAG.getNode(ISD::ADD, DL, VT, N0->getOperand(0), in transformAddImmMulImm()
8078 DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getConstant(C0, DL, VT)); in transformAddImmMulImm()
8079 return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT)); in transformAddImmMulImm()
8113 N0.getOpcode() == ISD::SRL && !isa<ConstantSDNode>(N0.getOperand(1)) && in performANDCombine()
8116 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0)); in performANDCombine()
8117 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1)); in performANDCombine()
8118 SDValue Srl = DAG.getNode(ISD::SRL, DL, MVT::i64, Op0, Op1); in performANDCombine()
8119 SDValue And = DAG.getNode(ISD::AND, DL, MVT::i64, Srl, in performANDCombine()
8121 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, And); in performANDCombine()
8159 TLI.isOperationLegal(ISD::ROTL, MVT::i64)) { in performXORCombine()
8192 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() || in performSETCCCombine()
8198 ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(2))->get(); in performSETCCCombine()
8214 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT); in performSETCCCombine()
8216 SDValue SExtOp = DAG.getNode(ISD::SIGN_EXTEND_INREG, N, OpVT, in performSETCCCombine()
8247 Src.getOpcode() == ISD::ABS && Src.hasOneUse() && VT == MVT::i64 && in performSIGN_EXTEND_INREGCombine()
8253 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, MVT::i64), Freeze); in performSIGN_EXTEND_INREGCombine()
8254 Neg = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Neg, in performSIGN_EXTEND_INREGCombine()
8256 return DAG.getNode(ISD::SMAX, DL, MVT::i64, Freeze, Neg); in performSIGN_EXTEND_INREGCombine()
8486 case ISD::FROUNDEVEN: return RISCVFPRndMode::RNE; in matchRoundingOp()
8487 case ISD::FTRUNC: return RISCVFPRndMode::RTZ; in matchRoundingOp()
8488 case ISD::FFLOOR: return RISCVFPRndMode::RDN; in matchRoundingOp()
8489 case ISD::FCEIL: return RISCVFPRndMode::RUP; in matchRoundingOp()
8490 case ISD::FROUND: return RISCVFPRndMode::RMM; in matchRoundingOp()
8529 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT; in performFP_TO_INTCombine()
8540 return DAG.getNode(ISD::TRUNCATE, DL, VT, FpToInt); in performFP_TO_INTCombine()
8578 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT_SAT; in performFP_TO_INT_SATCombine()
8598 return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO); in performFP_TO_INT_SATCombine()
8608 if (Src.getOpcode() != ISD::BSWAP) in performBITREVERSECombine()
8658 assert(N->getOpcode() == ISD::SRA && "Unexpected opcode"); in performSRACombine()
8675 N0.getOpcode() == ISD::SIGN_EXTEND_INREG && N0.hasOneUse() && in performSRACombine()
8677 N0.getOperand(0).getOpcode() == ISD::SHL && N0.getOperand(0).hasOneUse() && in performSRACombine()
8682 SDValue Shl = DAG.getNode(ISD::SHL, ShlDL, MVT::i64, in performSRACombine()
8686 return DAG.getNode(ISD::SRA, DL, MVT::i64, Shl, in performSRACombine()
8701 bool IsAdd = N0.getOpcode() == ISD::ADD; in performSRACombine()
8702 if ((IsAdd || N0.getOpcode() == ISD::SUB)) { in performSRACombine()
8721 if (Shl.getOpcode() != ISD::SHL || !Shl.hasOneUse() || in performSRACombine()
8735 In = DAG.getNode(ISD::ADD, DL, MVT::i64, In, ShiftedAddC); in performSRACombine()
8737 In = DAG.getNode(ISD::SUB, DL, MVT::i64, ShiftedAddC, In); in performSRACombine()
8740 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, In, in performSRACombine()
8746 ISD::SHL, DL, MVT::i64, SExt, in performSRACombine()
8753 ISD::CondCode CCVal = cast<CondCodeSDNode>(CC)->get(); in combine_CC()
8754 if (!ISD::isIntEqualitySetCC(CCVal)) in combine_CC()
8759 if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) && in combine_CC()
8763 bool Invert = CCVal == ISD::SETEQ; in combine_CC()
8766 CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType()); in combine_CC()
8777 if (LHS.getOpcode() == ISD::XOR && isNullConstant(RHS)) { in combine_CC()
8784 if (isNullConstant(RHS) && LHS.getOpcode() == ISD::SRL && LHS.hasOneUse() && in combine_CC()
8785 LHS.getOperand(1).getOpcode() == ISD::Constant) { in combine_CC()
8787 if (LHS0.getOpcode() == ISD::AND && in combine_CC()
8788 LHS0.getOperand(1).getOpcode() == ISD::Constant) { in combine_CC()
8792 CCVal = CCVal == ISD::SETEQ ? ISD::SETGE : ISD::SETLT; in combine_CC()
8799 DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS0.getOperand(0), in combine_CC()
8810 CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType()); in combine_CC()
8832 if (N->getOpcode() != ISD::DELETED_NODE) in PerformDAGCombine()
8869 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) || in PerformDAGCombine()
8878 if (Op0.getOpcode() == ISD::FNEG) { in PerformDAGCombine()
8879 SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi, in PerformDAGCombine()
8883 assert(Op0.getOpcode() == ISD::FABS); in PerformDAGCombine()
8884 SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi, in PerformDAGCombine()
8898 case ISD::ROTR: in PerformDAGCombine()
8899 case ISD::ROTL: in PerformDAGCombine()
9001 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) || in PerformDAGCombine()
9007 if (Op0.getOpcode() == ISD::FNEG) in PerformDAGCombine()
9008 return DAG.getNode(ISD::XOR, DL, VT, NewFMV, in PerformDAGCombine()
9011 assert(Op0.getOpcode() == ISD::FABS); in PerformDAGCombine()
9012 return DAG.getNode(ISD::AND, DL, VT, NewFMV, in PerformDAGCombine()
9015 case ISD::ADD: in PerformDAGCombine()
9017 case ISD::SUB: in PerformDAGCombine()
9019 case ISD::AND: in PerformDAGCombine()
9021 case ISD::OR: in PerformDAGCombine()
9023 case ISD::XOR: in PerformDAGCombine()
9025 case ISD::FADD: in PerformDAGCombine()
9026 case ISD::UMAX: in PerformDAGCombine()
9027 case ISD::UMIN: in PerformDAGCombine()
9028 case ISD::SMAX: in PerformDAGCombine()
9029 case ISD::SMIN: in PerformDAGCombine()
9030 case ISD::FMAXNUM: in PerformDAGCombine()
9031 case ISD::FMINNUM: in PerformDAGCombine()
9033 case ISD::SETCC: in PerformDAGCombine()
9035 case ISD::SIGN_EXTEND_INREG: in PerformDAGCombine()
9037 case ISD::ZERO_EXTEND: in PerformDAGCombine()
9043 if (Src.getOpcode() == ISD::FP_TO_UINT && in PerformDAGCombine()
9045 return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), MVT::i64, in PerformDAGCombine()
9047 if (Src.getOpcode() == ISD::STRICT_FP_TO_UINT && Src.hasOneUse() && in PerformDAGCombine()
9050 SDValue Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, SDLoc(N), VTs, in PerformDAGCombine()
9090 case ISD::BITREVERSE: in PerformDAGCombine()
9092 case ISD::FP_TO_SINT: in PerformDAGCombine()
9093 case ISD::FP_TO_UINT: in PerformDAGCombine()
9095 case ISD::FP_TO_SINT_SAT: in PerformDAGCombine()
9096 case ISD::FP_TO_UINT_SAT: in PerformDAGCombine()
9098 case ISD::FCOPYSIGN: { in PerformDAGCombine()
9111 if (In2.getOpcode() != ISD::FP_EXTEND && in PerformDAGCombine()
9112 (In2.getOpcode() != ISD::FP_ROUND || In2.getConstantOperandVal(1) != 0)) in PerformDAGCombine()
9115 if (In2.getOpcode() != ISD::FNEG) in PerformDAGCombine()
9119 return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N->getOperand(0), in PerformDAGCombine()
9120 DAG.getNode(ISD::FNEG, DL, VT, NewFPExtRound)); in PerformDAGCombine()
9122 case ISD::MGATHER: in PerformDAGCombine()
9123 case ISD::MSCATTER: in PerformDAGCombine()
9124 case ISD::VP_GATHER: in PerformDAGCombine()
9125 case ISD::VP_SCATTER: { in PerformDAGCombine()
9161 Index = DAG.getNode(IsIndexSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, in PerformDAGCombine()
9172 Index = DAG.getNode(ISD::SHL, DL, IndexVT, Index, SplatScale); in PerformDAGCombine()
9176 ISD::MemIndexType NewIndexTy = ISD::UNSIGNED_SCALED; in PerformDAGCombine()
9218 case ISD::SRA: in PerformDAGCombine()
9222 case ISD::SRL: in PerformDAGCombine()
9223 case ISD::SHL: { in PerformDAGCombine()
9286 case ISD::STORE: { in PerformDAGCombine()
9310 case ISD::SPLAT_VECTOR: { in PerformDAGCombine()
9331 case ISD::INTRINSIC_WO_CHAIN: { in PerformDAGCombine()
9357 case ISD::BITCAST: { in PerformDAGCombine()
9370 N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i1, Ops); in PerformDAGCombine()
9372 return DAG.getNode(ISD::TRUNCATE, DL, VT, N0); in PerformDAGCombine()
9384 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA || in isDesirableToCommuteWithShift()
9385 N->getOpcode() == ISD::SRL) && in isDesirableToCommuteWithShift()
9396 (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) { in isDesirableToCommuteWithShift()
9447 if (Opcode != ISD::AND && Opcode != ISD::OR && Opcode != ISD::XOR) in targetShrinkDemandedConstant()
9482 if (Opcode == ISD::AND) { in targetShrinkDemandedConstant()
9548 assert((Opc >= ISD::BUILTIN_OP_END || in computeKnownBitsForTargetNode()
9549 Opc == ISD::INTRINSIC_WO_CHAIN || in computeKnownBitsForTargetNode()
9550 Opc == ISD::INTRINSIC_W_CHAIN || in computeKnownBitsForTargetNode()
9551 Opc == ISD::INTRINSIC_VOID) && in computeKnownBitsForTargetNode()
9628 case ISD::INTRINSIC_W_CHAIN: in computeKnownBitsForTargetNode()
9629 case ISD::INTRINSIC_WO_CHAIN: { in computeKnownBitsForTargetNode()
9631 Op.getConstantOperandVal(Opc == ISD::INTRINSIC_WO_CHAIN ? 0 : 1); in computeKnownBitsForTargetNode()
9725 if (!ISD::isNormalLoad(Ld)) in getTargetConstantFromLoad()
10287 ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2, in CC_RISCVAssign2XLen()
10289 ISD::ArgFlagsTy ArgFlags2) { in CC_RISCVAssign2XLen()
10347 ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, in CC_RISCV()
10418 SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags = in CC_RISCV()
10474 ISD::ArgFlagsTy AF = PendingArgFlags[0]; in CC_RISCV()
10573 const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet, in analyzeInputArgs()
10584 ISD::ArgFlagsTy ArgFlags = Ins[i].Flags; in analyzeInputArgs()
10605 const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet, in analyzeOutputArgs()
10615 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; in analyzeOutputArgs()
10647 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val); in convertLocVTToValVT()
10691 Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val); in convertValVTToLocVT()
10717 ISD::LoadExtType ExtType; in unpackFromMemLoc()
10724 ExtType = ISD::NON_EXTLOAD; in unpackFromMemLoc()
10776 ISD::ArgFlagsTy ArgFlags, CCState &State, in CC_RISCV_FastCC()
10877 ISD::ArgFlagsTy ArgFlags, CCState &State) { in CC_RISCV_GHC()
10922 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, in LowerFormalArguments()
10999 Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset); in LowerFormalArguments()
11000 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, Offset); in LowerFormalArguments()
11069 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains); in LowerFormalArguments()
11159 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; in LowerCall()
11161 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; in LowerCall()
11199 ISD::ArgFlagsTy Flags = Outs[i].Flags; in LowerCall()
11229 ISD::ArgFlagsTy Flags = Outs[i].Flags; in LowerCall()
11289 Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset); in LowerCall()
11304 DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, PartOffset); in LowerCall()
11330 DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, in LowerCall()
11341 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); in LowerCall()
11457 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const { in CanLowerReturn()
11467 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; in CanLowerReturn()
11480 const SmallVectorImpl<ISD::OutputArg> &Outs, in LowerReturn()
12397 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Val); in splitValueIntoRegisterParts()
12398 Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Val); in splitValueIntoRegisterParts()
12399 Val = DAG.getNode(ISD::OR, DL, MVT::i32, Val, in splitValueIntoRegisterParts()
12401 Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val); in splitValueIntoRegisterParts()
12426 Val = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SameEltTypeVT, in splitValueIntoRegisterParts()
12430 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); in splitValueIntoRegisterParts()
12433 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT), in splitValueIntoRegisterParts()
12451 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val); in joinRegisterPartsIntoValue()
12452 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Val); in joinRegisterPartsIntoValue()
12453 Val = DAG.getNode(ISD::BITCAST, DL, MVT::f16, Val); in joinRegisterPartsIntoValue()
12478 Val = DAG.getNode(ISD::BITCAST, DL, SameEltTypeVT, Val); in joinRegisterPartsIntoValue()
12480 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val, in joinRegisterPartsIntoValue()
12521 SDValue Cmp = DAG.getSetCC(DL, VT, N0, Zero, ISD::SETLT); in BuildSDIVPow2()
12522 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne); in BuildSDIVPow2()
12523 SDValue Sel = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0); in BuildSDIVPow2()
12531 DAG.getNode(ISD::SRA, DL, VT, Sel, DAG.getConstant(Lg2, DL, VT)); in BuildSDIVPow2()
12539 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA); in BuildSDIVPow2()