Lines Matching refs:ISD
284 case ISD::LOAD: { in LegalizeOp()
286 ISD::LoadExtType ExtType = LD->getExtensionType(); in LegalizeOp()
288 if (LoadedVT.isVector() && ExtType != ISD::NON_EXTLOAD) in LegalizeOp()
292 case ISD::STORE: { in LegalizeOp()
300 case ISD::MERGE_VALUES: in LegalizeOp()
308 case ISD::STRICT_##DAGN: in LegalizeOp()
311 if (Op.getOpcode() == ISD::STRICT_SINT_TO_FP || in LegalizeOp()
312 Op.getOpcode() == ISD::STRICT_UINT_TO_FP) in LegalizeOp()
314 if (Op.getOpcode() == ISD::STRICT_FSETCC || in LegalizeOp()
315 Op.getOpcode() == ISD::STRICT_FSETCCS) { in LegalizeOp()
317 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(3))->get(); in LegalizeOp()
340 case ISD::ADD: in LegalizeOp()
341 case ISD::SUB: in LegalizeOp()
342 case ISD::MUL: in LegalizeOp()
343 case ISD::MULHS: in LegalizeOp()
344 case ISD::MULHU: in LegalizeOp()
345 case ISD::SDIV: in LegalizeOp()
346 case ISD::UDIV: in LegalizeOp()
347 case ISD::SREM: in LegalizeOp()
348 case ISD::UREM: in LegalizeOp()
349 case ISD::SDIVREM: in LegalizeOp()
350 case ISD::UDIVREM: in LegalizeOp()
351 case ISD::FADD: in LegalizeOp()
352 case ISD::FSUB: in LegalizeOp()
353 case ISD::FMUL: in LegalizeOp()
354 case ISD::FDIV: in LegalizeOp()
355 case ISD::FREM: in LegalizeOp()
356 case ISD::AND: in LegalizeOp()
357 case ISD::OR: in LegalizeOp()
358 case ISD::XOR: in LegalizeOp()
359 case ISD::SHL: in LegalizeOp()
360 case ISD::SRA: in LegalizeOp()
361 case ISD::SRL: in LegalizeOp()
362 case ISD::FSHL: in LegalizeOp()
363 case ISD::FSHR: in LegalizeOp()
364 case ISD::ROTL: in LegalizeOp()
365 case ISD::ROTR: in LegalizeOp()
366 case ISD::ABS: in LegalizeOp()
367 case ISD::BSWAP: in LegalizeOp()
368 case ISD::BITREVERSE: in LegalizeOp()
369 case ISD::CTLZ: in LegalizeOp()
370 case ISD::CTTZ: in LegalizeOp()
371 case ISD::CTLZ_ZERO_UNDEF: in LegalizeOp()
372 case ISD::CTTZ_ZERO_UNDEF: in LegalizeOp()
373 case ISD::CTPOP: in LegalizeOp()
374 case ISD::SELECT: in LegalizeOp()
375 case ISD::VSELECT: in LegalizeOp()
376 case ISD::SELECT_CC: in LegalizeOp()
377 case ISD::ZERO_EXTEND: in LegalizeOp()
378 case ISD::ANY_EXTEND: in LegalizeOp()
379 case ISD::TRUNCATE: in LegalizeOp()
380 case ISD::SIGN_EXTEND: in LegalizeOp()
381 case ISD::FP_TO_SINT: in LegalizeOp()
382 case ISD::FP_TO_UINT: in LegalizeOp()
383 case ISD::FNEG: in LegalizeOp()
384 case ISD::FABS: in LegalizeOp()
385 case ISD::FMINNUM: in LegalizeOp()
386 case ISD::FMAXNUM: in LegalizeOp()
387 case ISD::FMINNUM_IEEE: in LegalizeOp()
388 case ISD::FMAXNUM_IEEE: in LegalizeOp()
389 case ISD::FMINIMUM: in LegalizeOp()
390 case ISD::FMAXIMUM: in LegalizeOp()
391 case ISD::FCOPYSIGN: in LegalizeOp()
392 case ISD::FSQRT: in LegalizeOp()
393 case ISD::FSIN: in LegalizeOp()
394 case ISD::FCOS: in LegalizeOp()
395 case ISD::FLDEXP: in LegalizeOp()
396 case ISD::FPOWI: in LegalizeOp()
397 case ISD::FPOW: in LegalizeOp()
398 case ISD::FLOG: in LegalizeOp()
399 case ISD::FLOG2: in LegalizeOp()
400 case ISD::FLOG10: in LegalizeOp()
401 case ISD::FEXP: in LegalizeOp()
402 case ISD::FEXP2: in LegalizeOp()
403 case ISD::FEXP10: in LegalizeOp()
404 case ISD::FCEIL: in LegalizeOp()
405 case ISD::FTRUNC: in LegalizeOp()
406 case ISD::FRINT: in LegalizeOp()
407 case ISD::LRINT: in LegalizeOp()
408 case ISD::LLRINT: in LegalizeOp()
409 case ISD::FNEARBYINT: in LegalizeOp()
410 case ISD::FROUND: in LegalizeOp()
411 case ISD::FROUNDEVEN: in LegalizeOp()
412 case ISD::FFLOOR: in LegalizeOp()
413 case ISD::FP_ROUND: in LegalizeOp()
414 case ISD::FP_EXTEND: in LegalizeOp()
415 case ISD::FMA: in LegalizeOp()
416 case ISD::SIGN_EXTEND_INREG: in LegalizeOp()
417 case ISD::ANY_EXTEND_VECTOR_INREG: in LegalizeOp()
418 case ISD::SIGN_EXTEND_VECTOR_INREG: in LegalizeOp()
419 case ISD::ZERO_EXTEND_VECTOR_INREG: in LegalizeOp()
420 case ISD::SMIN: in LegalizeOp()
421 case ISD::SMAX: in LegalizeOp()
422 case ISD::UMIN: in LegalizeOp()
423 case ISD::UMAX: in LegalizeOp()
424 case ISD::SMUL_LOHI: in LegalizeOp()
425 case ISD::UMUL_LOHI: in LegalizeOp()
426 case ISD::SADDO: in LegalizeOp()
427 case ISD::UADDO: in LegalizeOp()
428 case ISD::SSUBO: in LegalizeOp()
429 case ISD::USUBO: in LegalizeOp()
430 case ISD::SMULO: in LegalizeOp()
431 case ISD::UMULO: in LegalizeOp()
432 case ISD::FCANONICALIZE: in LegalizeOp()
433 case ISD::FFREXP: in LegalizeOp()
434 case ISD::SADDSAT: in LegalizeOp()
435 case ISD::UADDSAT: in LegalizeOp()
436 case ISD::SSUBSAT: in LegalizeOp()
437 case ISD::USUBSAT: in LegalizeOp()
438 case ISD::SSHLSAT: in LegalizeOp()
439 case ISD::USHLSAT: in LegalizeOp()
440 case ISD::FP_TO_SINT_SAT: in LegalizeOp()
441 case ISD::FP_TO_UINT_SAT: in LegalizeOp()
442 case ISD::MGATHER: in LegalizeOp()
445 case ISD::SMULFIX: in LegalizeOp()
446 case ISD::SMULFIXSAT: in LegalizeOp()
447 case ISD::UMULFIX: in LegalizeOp()
448 case ISD::UMULFIXSAT: in LegalizeOp()
449 case ISD::SDIVFIX: in LegalizeOp()
450 case ISD::SDIVFIXSAT: in LegalizeOp()
451 case ISD::UDIVFIX: in LegalizeOp()
452 case ISD::UDIVFIXSAT: { in LegalizeOp()
458 case ISD::SINT_TO_FP: in LegalizeOp()
459 case ISD::UINT_TO_FP: in LegalizeOp()
460 case ISD::VECREDUCE_ADD: in LegalizeOp()
461 case ISD::VECREDUCE_MUL: in LegalizeOp()
462 case ISD::VECREDUCE_AND: in LegalizeOp()
463 case ISD::VECREDUCE_OR: in LegalizeOp()
464 case ISD::VECREDUCE_XOR: in LegalizeOp()
465 case ISD::VECREDUCE_SMAX: in LegalizeOp()
466 case ISD::VECREDUCE_SMIN: in LegalizeOp()
467 case ISD::VECREDUCE_UMAX: in LegalizeOp()
468 case ISD::VECREDUCE_UMIN: in LegalizeOp()
469 case ISD::VECREDUCE_FADD: in LegalizeOp()
470 case ISD::VECREDUCE_FMUL: in LegalizeOp()
471 case ISD::VECREDUCE_FMAX: in LegalizeOp()
472 case ISD::VECREDUCE_FMIN: in LegalizeOp()
473 case ISD::VECREDUCE_FMAXIMUM: in LegalizeOp()
474 case ISD::VECREDUCE_FMINIMUM: in LegalizeOp()
478 case ISD::VECREDUCE_SEQ_FADD: in LegalizeOp()
479 case ISD::VECREDUCE_SEQ_FMUL: in LegalizeOp()
483 case ISD::SETCC: { in LegalizeOp()
485 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); in LegalizeOp()
493 case ISD::VPID: { \ in LegalizeOp()
496 if (ISD::VPID == ISD::VP_SETCC) { \ in LegalizeOp()
497 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \ in LegalizeOp()
513 assert((Op.getOpcode() != ISD::LOAD && Op.getOpcode() != ISD::STORE) && in LegalizeOp()
585 DAG.getNode(ISD::FP_EXTEND, DL, NewScalarVT, Node->getOperand(0)); in PromoteReduction()
588 DAG.getNode(ISD::ANY_EXTEND, DL, NewScalarVT, Node->getOperand(0)); in PromoteReduction()
592 !(ISD::isVPOpcode(Node->getOpcode()) && in PromoteReduction()
593 ISD::getVPMaskIdx(Node->getOpcode()) == j)) // Skip mask operand. in PromoteReduction()
597 DAG.getNode(ISD::FP_EXTEND, DL, NewVecVT, Node->getOperand(j)); in PromoteReduction()
600 DAG.getNode(ISD::ANY_EXTEND, DL, NewVecVT, Node->getOperand(j)); in PromoteReduction()
608 Res = DAG.getNode(ISD::FP_ROUND, DL, ScalarVT, Res, in PromoteReduction()
611 Res = DAG.getNode(ISD::TRUNCATE, DL, ScalarVT, Res); in PromoteReduction()
621 unsigned ExtOp = VecVT.isFloatingPoint() ? ISD::FP_EXTEND : ISD::ANY_EXTEND; in PromoteSETCC()
630 if (Node->getOpcode() == ISD::VP_SETCC) { in PromoteSETCC()
654 !(ISD::isVPOpcode(Node->getOpcode()) && in PromoteSTRICT()
655 ISD::getVPMaskIdx(Node->getOpcode()) == j)) // Skip mask operand. in PromoteSTRICT()
659 DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {NewVecVT, MVT::Other}, in PromoteSTRICT()
668 Operands[0] = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); in PromoteSTRICT()
674 DAG.getNode(ISD::STRICT_FP_ROUND, DL, {VecVT, MVT::Other}, in PromoteSTRICT()
686 case ISD::SINT_TO_FP: in Promote()
687 case ISD::UINT_TO_FP: in Promote()
688 case ISD::STRICT_SINT_TO_FP: in Promote()
689 case ISD::STRICT_UINT_TO_FP: in Promote()
693 case ISD::FP_TO_UINT: in Promote()
694 case ISD::FP_TO_SINT: in Promote()
695 case ISD::STRICT_FP_TO_UINT: in Promote()
696 case ISD::STRICT_FP_TO_SINT: in Promote()
700 case ISD::VP_REDUCE_ADD: in Promote()
701 case ISD::VP_REDUCE_MUL: in Promote()
702 case ISD::VP_REDUCE_AND: in Promote()
703 case ISD::VP_REDUCE_OR: in Promote()
704 case ISD::VP_REDUCE_XOR: in Promote()
705 case ISD::VP_REDUCE_SMAX: in Promote()
706 case ISD::VP_REDUCE_SMIN: in Promote()
707 case ISD::VP_REDUCE_UMAX: in Promote()
708 case ISD::VP_REDUCE_UMIN: in Promote()
709 case ISD::VP_REDUCE_FADD: in Promote()
710 case ISD::VP_REDUCE_FMUL: in Promote()
711 case ISD::VP_REDUCE_FMAX: in Promote()
712 case ISD::VP_REDUCE_FMIN: in Promote()
713 case ISD::VP_REDUCE_SEQ_FADD: in Promote()
717 case ISD::VP_SETCC: in Promote()
718 case ISD::SETCC: in Promote()
722 case ISD::STRICT_FADD: in Promote()
723 case ISD::STRICT_FSUB: in Promote()
724 case ISD::STRICT_FMUL: in Promote()
725 case ISD::STRICT_FDIV: in Promote()
726 case ISD::STRICT_FSQRT: in Promote()
727 case ISD::STRICT_FMA: in Promote()
730 case ISD::FP_ROUND: in Promote()
731 case ISD::FP_EXTEND: in Promote()
751 bool SkipPromote = ISD::isVPOpcode(Node->getOpcode()) && in Promote()
752 ISD::getVPMaskIdx(Node->getOpcode()) == j; in Promote()
759 Operands[j] = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(j)); in Promote()
761 Operands[j] = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(j)); in Promote()
772 Res = DAG.getNode(ISD::FP_ROUND, dl, VT, Res, in Promote()
775 Res = DAG.getNode(ISD::BITCAST, dl, VT, Res); in Promote()
793 unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP || in PromoteINT_TO_FP()
794 Node->getOpcode() == ISD::STRICT_UINT_TO_FP) in PromoteINT_TO_FP()
795 ? ISD::ZERO_EXTEND in PromoteINT_TO_FP()
796 : ISD::SIGN_EXTEND; in PromoteINT_TO_FP()
832 if (NewOpc == ISD::FP_TO_UINT && in PromoteFP_TO_INT()
833 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) in PromoteFP_TO_INT()
834 NewOpc = ISD::FP_TO_SINT; in PromoteFP_TO_INT()
836 if (NewOpc == ISD::STRICT_FP_TO_UINT && in PromoteFP_TO_INT()
837 TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT)) in PromoteFP_TO_INT()
838 NewOpc = ISD::STRICT_FP_TO_SINT; in PromoteFP_TO_INT()
852 if (Node->getOpcode() == ISD::FP_TO_UINT || in PromoteFP_TO_INT()
853 Node->getOpcode() == ISD::STRICT_FP_TO_UINT) in PromoteFP_TO_INT()
854 NewOpc = ISD::AssertZext; in PromoteFP_TO_INT()
856 NewOpc = ISD::AssertSext; in PromoteFP_TO_INT()
860 Promoted = DAG.getNode(ISD::TRUNCATE, dl, VT, Promoted); in PromoteFP_TO_INT()
879 case ISD::LOAD: { in Expand()
885 case ISD::STORE: in Expand()
888 case ISD::MERGE_VALUES: in Expand()
892 case ISD::SIGN_EXTEND_INREG: in Expand()
895 case ISD::ANY_EXTEND_VECTOR_INREG: in Expand()
898 case ISD::SIGN_EXTEND_VECTOR_INREG: in Expand()
901 case ISD::ZERO_EXTEND_VECTOR_INREG: in Expand()
904 case ISD::BSWAP: in Expand()
907 case ISD::VP_BSWAP: in Expand()
910 case ISD::VSELECT: in Expand()
913 case ISD::VP_SELECT: in Expand()
916 case ISD::VP_SREM: in Expand()
917 case ISD::VP_UREM: in Expand()
923 case ISD::SELECT: in Expand()
926 case ISD::SELECT_CC: { in Expand()
931 DAG.getNode(ISD::SETCC, SDLoc(Node), CondVT, Node->getOperand(0), in Expand()
940 case ISD::FP_TO_UINT: in Expand()
943 case ISD::UINT_TO_FP: in Expand()
946 case ISD::FNEG: in Expand()
949 case ISD::FSUB: in Expand()
952 case ISD::SETCC: in Expand()
953 case ISD::VP_SETCC: in Expand()
956 case ISD::ABS: in Expand()
962 case ISD::ABDS: in Expand()
963 case ISD::ABDU: in Expand()
969 case ISD::BITREVERSE: in Expand()
972 case ISD::VP_BITREVERSE: in Expand()
978 case ISD::CTPOP: in Expand()
984 case ISD::VP_CTPOP: in Expand()
990 case ISD::CTLZ: in Expand()
991 case ISD::CTLZ_ZERO_UNDEF: in Expand()
997 case ISD::VP_CTLZ: in Expand()
998 case ISD::VP_CTLZ_ZERO_UNDEF: in Expand()
1004 case ISD::CTTZ: in Expand()
1005 case ISD::CTTZ_ZERO_UNDEF: in Expand()
1011 case ISD::VP_CTTZ: in Expand()
1012 case ISD::VP_CTTZ_ZERO_UNDEF: in Expand()
1018 case ISD::FSHL: in Expand()
1019 case ISD::VP_FSHL: in Expand()
1020 case ISD::FSHR: in Expand()
1021 case ISD::VP_FSHR: in Expand()
1027 case ISD::ROTL: in Expand()
1028 case ISD::ROTR: in Expand()
1034 case ISD::FMINNUM: in Expand()
1035 case ISD::FMAXNUM: in Expand()
1041 case ISD::SMIN: in Expand()
1042 case ISD::SMAX: in Expand()
1043 case ISD::UMIN: in Expand()
1044 case ISD::UMAX: in Expand()
1050 case ISD::UADDO: in Expand()
1051 case ISD::USUBO: in Expand()
1054 case ISD::SADDO: in Expand()
1055 case ISD::SSUBO: in Expand()
1058 case ISD::UMULO: in Expand()
1059 case ISD::SMULO: in Expand()
1062 case ISD::USUBSAT: in Expand()
1063 case ISD::SSUBSAT: in Expand()
1064 case ISD::UADDSAT: in Expand()
1065 case ISD::SADDSAT: in Expand()
1071 case ISD::USHLSAT: in Expand()
1072 case ISD::SSHLSAT: in Expand()
1078 case ISD::FP_TO_SINT_SAT: in Expand()
1079 case ISD::FP_TO_UINT_SAT: in Expand()
1088 case ISD::SMULFIX: in Expand()
1089 case ISD::UMULFIX: in Expand()
1095 case ISD::SMULFIXSAT: in Expand()
1096 case ISD::UMULFIXSAT: in Expand()
1102 case ISD::SDIVFIX: in Expand()
1103 case ISD::UDIVFIX: in Expand()
1106 case ISD::SDIVFIXSAT: in Expand()
1107 case ISD::UDIVFIXSAT: in Expand()
1110 case ISD::STRICT_##DAGN: in Expand()
1114 case ISD::VECREDUCE_ADD: in Expand()
1115 case ISD::VECREDUCE_MUL: in Expand()
1116 case ISD::VECREDUCE_AND: in Expand()
1117 case ISD::VECREDUCE_OR: in Expand()
1118 case ISD::VECREDUCE_XOR: in Expand()
1119 case ISD::VECREDUCE_SMAX: in Expand()
1120 case ISD::VECREDUCE_SMIN: in Expand()
1121 case ISD::VECREDUCE_UMAX: in Expand()
1122 case ISD::VECREDUCE_UMIN: in Expand()
1123 case ISD::VECREDUCE_FADD: in Expand()
1124 case ISD::VECREDUCE_FMUL: in Expand()
1125 case ISD::VECREDUCE_FMAX: in Expand()
1126 case ISD::VECREDUCE_FMIN: in Expand()
1127 case ISD::VECREDUCE_FMAXIMUM: in Expand()
1128 case ISD::VECREDUCE_FMINIMUM: in Expand()
1131 case ISD::VECREDUCE_SEQ_FADD: in Expand()
1132 case ISD::VECREDUCE_SEQ_FMUL: in Expand()
1135 case ISD::SREM: in Expand()
1136 case ISD::UREM: in Expand()
1139 case ISD::VP_MERGE: in Expand()
1171 if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand || in ExpandSELECT()
1172 TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand || in ExpandSELECT()
1173 TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand || in ExpandSELECT()
1174 TLI.getOperationAction(VT.isFixedLengthVector() ? ISD::BUILD_VECTOR in ExpandSELECT()
1175 : ISD::SPLAT_VECTOR, in ExpandSELECT()
1194 Op1 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op1); in ExpandSELECT()
1195 Op2 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op2); in ExpandSELECT()
1199 Op1 = DAG.getNode(ISD::AND, DL, MaskTy, Op1, Mask); in ExpandSELECT()
1200 Op2 = DAG.getNode(ISD::AND, DL, MaskTy, Op2, NotMask); in ExpandSELECT()
1201 SDValue Val = DAG.getNode(ISD::OR, DL, MaskTy, Op1, Op2); in ExpandSELECT()
1202 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val); in ExpandSELECT()
1209 if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Expand || in ExpandSEXTINREG()
1210 TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Expand) in ExpandSEXTINREG()
1220 SDValue Op = DAG.getNode(ISD::SHL, DL, VT, Node->getOperand(0), ShiftSz); in ExpandSEXTINREG()
1221 return DAG.getNode(ISD::SRA, DL, VT, Op, ShiftSz); in ExpandSEXTINREG()
1242 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT), in ExpandANY_EXTEND_VECTOR_INREG()
1257 ISD::BITCAST, DL, VT, in ExpandANY_EXTEND_VECTOR_INREG()
1269 SDValue Op = DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Src); in ExpandSIGN_EXTEND_VECTOR_INREG()
1277 return DAG.getNode(ISD::SRA, DL, VT, in ExpandSIGN_EXTEND_VECTOR_INREG()
1278 DAG.getNode(ISD::SHL, DL, VT, Op, ShiftAmount), in ExpandSIGN_EXTEND_VECTOR_INREG()
1301 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT), in ExpandZERO_EXTEND_VECTOR_INREG()
1317 return DAG.getNode(ISD::BITCAST, DL, VT, in ExpandZERO_EXTEND_VECTOR_INREG()
1343 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0)); in ExpandBSWAP()
1345 return DAG.getNode(ISD::BITCAST, DL, VT, Op); in ExpandBSWAP()
1350 if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) && in ExpandBSWAP()
1351 TLI.isOperationLegalOrCustom(ISD::SRL, VT) && in ExpandBSWAP()
1352 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) && in ExpandBSWAP()
1353 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT)) in ExpandBSWAP()
1371 if (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, VT.getScalarType())) { in ExpandBITREVERSE()
1387 (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, ByteVT) || in ExpandBITREVERSE()
1388 (TLI.isOperationLegalOrCustom(ISD::SHL, ByteVT) && in ExpandBITREVERSE()
1389 TLI.isOperationLegalOrCustom(ISD::SRL, ByteVT) && in ExpandBITREVERSE()
1390 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, ByteVT) && in ExpandBITREVERSE()
1391 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, ByteVT)))) { in ExpandBITREVERSE()
1393 SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0)); in ExpandBITREVERSE()
1396 Op = DAG.getNode(ISD::BITREVERSE, DL, ByteVT, Op); in ExpandBITREVERSE()
1397 Op = DAG.getNode(ISD::BITCAST, DL, VT, Op); in ExpandBITREVERSE()
1405 if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) && in ExpandBITREVERSE()
1406 TLI.isOperationLegalOrCustom(ISD::SRL, VT) && in ExpandBITREVERSE()
1407 TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) && in ExpandBITREVERSE()
1408 TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT)) { in ExpandBITREVERSE()
1433 if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand || in ExpandVSELECT()
1434 TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand || in ExpandVSELECT()
1435 TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand) in ExpandVSELECT()
1457 Op1 = DAG.getNode(ISD::BITCAST, DL, VT, Op1); in ExpandVSELECT()
1458 Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2); in ExpandVSELECT()
1462 Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask); in ExpandVSELECT()
1463 Op2 = DAG.getNode(ISD::AND, DL, VT, Op2, NotMask); in ExpandVSELECT()
1464 SDValue Val = DAG.getNode(ISD::OR, DL, VT, Op1, Op2); in ExpandVSELECT()
1465 return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val); in ExpandVSELECT()
1482 if (TLI.getOperationAction(ISD::VP_AND, VT) == TargetLowering::Expand || in ExpandVP_SELECT()
1483 TLI.getOperationAction(ISD::VP_XOR, VT) == TargetLowering::Expand || in ExpandVP_SELECT()
1484 TLI.getOperationAction(ISD::VP_OR, VT) == TargetLowering::Expand) in ExpandVP_SELECT()
1492 SDValue NotMask = DAG.getNode(ISD::VP_XOR, DL, VT, Mask, Ones, Ones, EVL); in ExpandVP_SELECT()
1494 Op1 = DAG.getNode(ISD::VP_AND, DL, VT, Op1, Mask, Ones, EVL); in ExpandVP_SELECT()
1495 Op2 = DAG.getNode(ISD::VP_AND, DL, VT, Op2, NotMask, Ones, EVL); in ExpandVP_SELECT()
1496 return DAG.getNode(ISD::VP_OR, DL, VT, Op1, Op2, Ones, EVL); in ExpandVP_SELECT()
1519 !TLI.isOperationLegalOrCustom(ISD::BUILD_VECTOR, EVLVecVT)) || in ExpandVP_MERGE()
1521 (!TLI.isOperationLegalOrCustom(ISD::STEP_VECTOR, EVLVecVT) || in ExpandVP_MERGE()
1522 !TLI.isOperationLegalOrCustom(ISD::SPLAT_VECTOR, EVLVecVT)))) in ExpandVP_MERGE()
1534 DAG.getSetCC(DL, MaskVT, StepVec, SplatEVL, ISD::CondCode::SETULT); in ExpandVP_MERGE()
1536 SDValue FullMask = DAG.getNode(ISD::AND, DL, MaskVT, Mask, EVLMask); in ExpandVP_MERGE()
1544 unsigned DivOpc = Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV; in ExpandVP_REM()
1547 !TLI.isOperationLegalOrCustom(ISD::VP_MUL, VT) || in ExpandVP_REM()
1548 !TLI.isOperationLegalOrCustom(ISD::VP_SUB, VT)) in ExpandVP_REM()
1560 SDValue Mul = DAG.getNode(ISD::VP_MUL, DL, VT, Divisor, Div, Mask, EVL); in ExpandVP_REM()
1561 return DAG.getNode(ISD::VP_SUB, DL, VT, Dividend, Mul, Mask, EVL); in ExpandVP_REM()
1603 if (((!IsStrict && TLI.getOperationAction(ISD::SINT_TO_FP, VT) == in ExpandUINT_TO_FLOAT()
1605 (IsStrict && TLI.getOperationAction(ISD::STRICT_SINT_TO_FP, VT) == in ExpandUINT_TO_FLOAT()
1607 TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Expand) { in ExpandUINT_TO_FLOAT()
1634 SDValue HI = DAG.getNode(ISD::SRL, DL, VT, Src, HalfWord); in ExpandUINT_TO_FLOAT()
1635 SDValue LO = DAG.getNode(ISD::AND, DL, VT, Src, HalfWordMask); in ExpandUINT_TO_FLOAT()
1641 SDValue fHI = DAG.getNode(ISD::STRICT_SINT_TO_FP, DL, in ExpandUINT_TO_FLOAT()
1644 fHI = DAG.getNode(ISD::STRICT_FMUL, DL, {Node->getValueType(0), MVT::Other}, in ExpandUINT_TO_FLOAT()
1646 SDValue fLO = DAG.getNode(ISD::STRICT_SINT_TO_FP, DL, in ExpandUINT_TO_FLOAT()
1650 SDValue TF = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, fHI.getValue(1), in ExpandUINT_TO_FLOAT()
1655 DAG.getNode(ISD::STRICT_FADD, DL, {Node->getValueType(0), MVT::Other}, in ExpandUINT_TO_FLOAT()
1666 SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), HI); in ExpandUINT_TO_FLOAT()
1667 fHI = DAG.getNode(ISD::FMUL, DL, Node->getValueType(0), fHI, TWOHW); in ExpandUINT_TO_FLOAT()
1668 SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), LO); in ExpandUINT_TO_FLOAT()
1672 DAG.getNode(ISD::FADD, DL, Node->getValueType(0), fHI, fLO)); in ExpandUINT_TO_FLOAT()
1676 if (TLI.isOperationLegalOrCustom(ISD::FSUB, Node->getValueType(0))) { in ExpandFNEG()
1680 return DAG.getNode(ISD::FSUB, DL, Node->getValueType(0), Zero, in ExpandFNEG()
1692 if (TLI.isOperationLegalOrCustom(ISD::FNEG, VT) && in ExpandFSUB()
1693 TLI.isOperationLegalOrCustom(ISD::FADD, VT)) in ExpandFSUB()
1703 bool IsVP = Node->getOpcode() == ISD::VP_SETCC; in ExpandSETCC()
1704 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC || in ExpandSETCC()
1705 Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandSETCC()
1706 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandSETCC()
1715 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); in ExpandSETCC()
1746 LHS = DAG.getNode(ISD::VP_SETCC, dl, Node->getValueType(0), in ExpandSETCC()
1749 LHS = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), LHS, RHS, CC, in ExpandSETCC()
1769 DAG.getNode(ISD::SELECT_CC, dl, VT, LHS, RHS, in ExpandSETCC()
1816 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP) { in ExpandStrictFPOp()
1820 if (Node->getOpcode() == ISD::STRICT_FP_TO_UINT) { in ExpandStrictFPOp()
1825 if (Node->getOpcode() == ISD::STRICT_FSETCC || in ExpandStrictFPOp()
1826 Node->getOpcode() == ISD::STRICT_FSETCCS) { in ExpandStrictFPOp()
1836 assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) && in ExpandREM()
1854 if (Node->getOpcode() == ISD::STRICT_FSETCC || in UnrollStrictFPOp()
1855 Node->getOpcode() == ISD::STRICT_FSETCCS) in UnrollStrictFPOp()
1878 Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, in UnrollStrictFPOp()
1888 if (Node->getOpcode() == ISD::STRICT_FSETCC || in UnrollStrictFPOp()
1889 Node->getOpcode() == ISD::STRICT_FSETCCS) in UnrollStrictFPOp()
1899 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains); in UnrollStrictFPOp()
1916 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS, in UnrollVSETCC()
1918 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS, in UnrollVSETCC()
1920 Ops[i] = DAG.getNode(ISD::SETCC, dl, in UnrollVSETCC()