Lines Matching refs:ISD
325 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) && in ExpandConstantFP()
341 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx, in ExpandConstantFP()
415 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, in ExpandINSERT_VECTOR_ELT()
433 if (!ISD::isNormalStore(ST)) in OptimizeFloatStore()
451 if (Value.getOpcode() == ISD::TargetConstantFP) in OptimizeFloatStore()
491 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); in OptimizeFloatStore()
516 switch (TLI.getOperationAction(ISD::STORE, VT)) { in LegalizeStoreOps()
540 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT); in LegalizeStoreOps()
543 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value); in LegalizeStoreOps()
598 ISD::SRL, dl, Value.getValueType(), Value, in LegalizeStoreOps()
609 ISD::SRL, dl, Value.getValueType(), Value, in LegalizeStoreOps()
617 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, in LegalizeStoreOps()
626 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); in LegalizeStoreOps()
656 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value); in LegalizeStoreOps()
662 Value = DAG.getNode(ISD::TRUNCATE, dl, in LegalizeStoreOps()
683 ISD::LoadExtType ExtType = LD->getExtensionType(); in LegalizeLoadOps()
684 if (ExtType == ISD::NON_EXTLOAD) { in LegalizeLoadOps()
716 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res); in LegalizeLoadOps()
760 ISD::LoadExtType NewExtType = in LegalizeLoadOps()
761 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD; in LegalizeLoadOps()
769 if (ExtType == ISD::SEXTLOAD) in LegalizeLoadOps()
771 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, in LegalizeLoadOps()
774 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()) in LegalizeLoadOps()
776 Result = DAG.getNode(ISD::AssertZext, dl, in LegalizeLoadOps()
803 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
817 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), in LegalizeLoadOps()
822 ISD::SHL, dl, Hi.getValueType(), Hi, in LegalizeLoadOps()
827 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
840 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
846 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), in LegalizeLoadOps()
851 ISD::SHL, dl, Hi.getValueType(), Hi, in LegalizeLoadOps()
856 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
891 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) { in LegalizeLoadOps()
900 ISD::LoadExtType MidExtType = in LegalizeLoadOps()
901 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType; in LegalizeLoadOps()
906 ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType); in LegalizeLoadOps()
922 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain, in LegalizeLoadOps()
925 DAG.getNode(SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP, in LegalizeLoadOps()
939 assert(ExtType != ISD::EXTLOAD && in LegalizeLoadOps()
943 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, in LegalizeLoadOps()
948 if (ExtType == ISD::SEXTLOAD) in LegalizeLoadOps()
949 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, in LegalizeLoadOps()
980 if (Node->getOpcode() == ISD::TargetConstant || in LegalizeOp()
981 Node->getOpcode() == ISD::Register) in LegalizeOp()
993 Op.getOpcode() == ISD::TargetConstant || in LegalizeOp()
994 Op.getOpcode() == ISD::Register) && in LegalizeOp()
1002 case ISD::INTRINSIC_W_CHAIN: in LegalizeOp()
1003 case ISD::INTRINSIC_WO_CHAIN: in LegalizeOp()
1004 case ISD::INTRINSIC_VOID: in LegalizeOp()
1005 case ISD::STACKSAVE: in LegalizeOp()
1008 case ISD::GET_DYNAMIC_AREA_OFFSET: in LegalizeOp()
1012 case ISD::VAARG: in LegalizeOp()
1018 case ISD::SET_FPENV: in LegalizeOp()
1019 case ISD::SET_FPMODE: in LegalizeOp()
1023 case ISD::FP_TO_FP16: in LegalizeOp()
1024 case ISD::FP_TO_BF16: in LegalizeOp()
1025 case ISD::SINT_TO_FP: in LegalizeOp()
1026 case ISD::UINT_TO_FP: in LegalizeOp()
1027 case ISD::EXTRACT_VECTOR_ELT: in LegalizeOp()
1028 case ISD::LROUND: in LegalizeOp()
1029 case ISD::LLROUND: in LegalizeOp()
1030 case ISD::LRINT: in LegalizeOp()
1031 case ISD::LLRINT: in LegalizeOp()
1035 case ISD::STRICT_FP_TO_FP16: in LegalizeOp()
1036 case ISD::STRICT_SINT_TO_FP: in LegalizeOp()
1037 case ISD::STRICT_UINT_TO_FP: in LegalizeOp()
1038 case ISD::STRICT_LRINT: in LegalizeOp()
1039 case ISD::STRICT_LLRINT: in LegalizeOp()
1040 case ISD::STRICT_LROUND: in LegalizeOp()
1041 case ISD::STRICT_LLROUND: in LegalizeOp()
1048 case ISD::SIGN_EXTEND_INREG: { in LegalizeOp()
1053 case ISD::ATOMIC_STORE: in LegalizeOp()
1057 case ISD::SELECT_CC: in LegalizeOp()
1058 case ISD::STRICT_FSETCC: in LegalizeOp()
1059 case ISD::STRICT_FSETCCS: in LegalizeOp()
1060 case ISD::SETCC: in LegalizeOp()
1061 case ISD::SETCCCARRY: in LegalizeOp()
1062 case ISD::VP_SETCC: in LegalizeOp()
1063 case ISD::BR_CC: { in LegalizeOp()
1065 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4 in LegalizeOp()
1066 : Opc == ISD::STRICT_FSETCC ? 3 in LegalizeOp()
1067 : Opc == ISD::STRICT_FSETCCS ? 3 in LegalizeOp()
1068 : Opc == ISD::SETCCCARRY ? 3 in LegalizeOp()
1069 : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2 in LegalizeOp()
1071 unsigned CompareOperand = Opc == ISD::BR_CC ? 2 in LegalizeOp()
1072 : Opc == ISD::STRICT_FSETCC ? 1 in LegalizeOp()
1073 : Opc == ISD::STRICT_FSETCCS ? 1 in LegalizeOp()
1076 ISD::CondCode CCCode = in LegalizeOp()
1080 if (Node->getOpcode() == ISD::SELECT_CC) in LegalizeOp()
1088 case ISD::LOAD: in LegalizeOp()
1089 case ISD::STORE: in LegalizeOp()
1094 case ISD::CALLSEQ_START: in LegalizeOp()
1095 case ISD::CALLSEQ_END: in LegalizeOp()
1101 case ISD::EXTRACT_ELEMENT: in LegalizeOp()
1102 case ISD::GET_ROUNDING: in LegalizeOp()
1103 case ISD::MERGE_VALUES: in LegalizeOp()
1104 case ISD::EH_RETURN: in LegalizeOp()
1105 case ISD::FRAME_TO_ARGS_OFFSET: in LegalizeOp()
1106 case ISD::EH_DWARF_CFA: in LegalizeOp()
1107 case ISD::EH_SJLJ_SETJMP: in LegalizeOp()
1108 case ISD::EH_SJLJ_LONGJMP: in LegalizeOp()
1109 case ISD::EH_SJLJ_SETUP_DISPATCH: in LegalizeOp()
1116 case ISD::INIT_TRAMPOLINE: in LegalizeOp()
1117 case ISD::ADJUST_TRAMPOLINE: in LegalizeOp()
1118 case ISD::FRAMEADDR: in LegalizeOp()
1119 case ISD::RETURNADDR: in LegalizeOp()
1120 case ISD::ADDROFRETURNADDR: in LegalizeOp()
1121 case ISD::SPONENTRY: in LegalizeOp()
1128 case ISD::READCYCLECOUNTER: in LegalizeOp()
1133 case ISD::READ_REGISTER: in LegalizeOp()
1134 case ISD::WRITE_REGISTER: in LegalizeOp()
1140 case ISD::UBSANTRAP: in LegalizeOp()
1145 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), in LegalizeOp()
1152 case ISD::DEBUGTRAP: in LegalizeOp()
1157 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), in LegalizeOp()
1164 case ISD::SADDSAT: in LegalizeOp()
1165 case ISD::UADDSAT: in LegalizeOp()
1166 case ISD::SSUBSAT: in LegalizeOp()
1167 case ISD::USUBSAT: in LegalizeOp()
1168 case ISD::SSHLSAT: in LegalizeOp()
1169 case ISD::USHLSAT: in LegalizeOp()
1170 case ISD::FP_TO_SINT_SAT: in LegalizeOp()
1171 case ISD::FP_TO_UINT_SAT: in LegalizeOp()
1174 case ISD::SMULFIX: in LegalizeOp()
1175 case ISD::SMULFIXSAT: in LegalizeOp()
1176 case ISD::UMULFIX: in LegalizeOp()
1177 case ISD::UMULFIXSAT: in LegalizeOp()
1178 case ISD::SDIVFIX: in LegalizeOp()
1179 case ISD::SDIVFIXSAT: in LegalizeOp()
1180 case ISD::UDIVFIX: in LegalizeOp()
1181 case ISD::UDIVFIXSAT: { in LegalizeOp()
1187 case ISD::MSCATTER: in LegalizeOp()
1191 case ISD::MSTORE: in LegalizeOp()
1195 case ISD::VP_SCATTER: in LegalizeOp()
1200 case ISD::VP_STORE: in LegalizeOp()
1205 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: in LegalizeOp()
1210 case ISD::VECREDUCE_FADD: in LegalizeOp()
1211 case ISD::VECREDUCE_FMUL: in LegalizeOp()
1212 case ISD::VECREDUCE_ADD: in LegalizeOp()
1213 case ISD::VECREDUCE_MUL: in LegalizeOp()
1214 case ISD::VECREDUCE_AND: in LegalizeOp()
1215 case ISD::VECREDUCE_OR: in LegalizeOp()
1216 case ISD::VECREDUCE_XOR: in LegalizeOp()
1217 case ISD::VECREDUCE_SMAX: in LegalizeOp()
1218 case ISD::VECREDUCE_SMIN: in LegalizeOp()
1219 case ISD::VECREDUCE_UMAX: in LegalizeOp()
1220 case ISD::VECREDUCE_UMIN: in LegalizeOp()
1221 case ISD::VECREDUCE_FMAX: in LegalizeOp()
1222 case ISD::VECREDUCE_FMIN: in LegalizeOp()
1223 case ISD::VECREDUCE_FMAXIMUM: in LegalizeOp()
1224 case ISD::VECREDUCE_FMINIMUM: in LegalizeOp()
1225 case ISD::IS_FPCLASS: in LegalizeOp()
1229 case ISD::VECREDUCE_SEQ_FADD: in LegalizeOp()
1230 case ISD::VECREDUCE_SEQ_FMUL: in LegalizeOp()
1231 case ISD::VP_REDUCE_FADD: in LegalizeOp()
1232 case ISD::VP_REDUCE_FMUL: in LegalizeOp()
1233 case ISD::VP_REDUCE_ADD: in LegalizeOp()
1234 case ISD::VP_REDUCE_MUL: in LegalizeOp()
1235 case ISD::VP_REDUCE_AND: in LegalizeOp()
1236 case ISD::VP_REDUCE_OR: in LegalizeOp()
1237 case ISD::VP_REDUCE_XOR: in LegalizeOp()
1238 case ISD::VP_REDUCE_SMAX: in LegalizeOp()
1239 case ISD::VP_REDUCE_SMIN: in LegalizeOp()
1240 case ISD::VP_REDUCE_UMAX: in LegalizeOp()
1241 case ISD::VP_REDUCE_UMIN: in LegalizeOp()
1242 case ISD::VP_REDUCE_FMAX: in LegalizeOp()
1243 case ISD::VP_REDUCE_FMIN: in LegalizeOp()
1244 case ISD::VP_REDUCE_SEQ_FADD: in LegalizeOp()
1245 case ISD::VP_REDUCE_SEQ_FMUL: in LegalizeOp()
1250 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { in LegalizeOp()
1262 case ISD::SHL: in LegalizeOp()
1263 case ISD::SRL: in LegalizeOp()
1264 case ISD::SRA: in LegalizeOp()
1265 case ISD::ROTL: in LegalizeOp()
1266 case ISD::ROTR: { in LegalizeOp()
1283 case ISD::FSHL: in LegalizeOp()
1284 case ISD::FSHR: in LegalizeOp()
1285 case ISD::SRL_PARTS: in LegalizeOp()
1286 case ISD::SRA_PARTS: in LegalizeOp()
1287 case ISD::SHL_PARTS: { in LegalizeOp()
1370 case ISD::CALLSEQ_START: in LegalizeOp()
1371 case ISD::CALLSEQ_END: in LegalizeOp()
1373 case ISD::LOAD: in LegalizeOp()
1375 case ISD::STORE: in LegalizeOp()
1446 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, in ExpandExtractFromVectorThroughStack()
1497 assert((Node->getOpcode() == ISD::BUILD_VECTOR || in ExpandVectorBuildThroughStack()
1498 Node->getOpcode() == ISD::CONCAT_VECTORS) && in ExpandVectorBuildThroughStack()
1545 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); in ExpandVectorBuildThroughStack()
1565 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value); in getSignAsIntValue()
1600 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr, in getSignAsIntValue()
1612 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue); in modifySignAsInt()
1632 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue, in ExpandFCOPYSIGN()
1637 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) && in ExpandFCOPYSIGN()
1638 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) { in ExpandFCOPYSIGN()
1639 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag); in ExpandFCOPYSIGN()
1640 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue); in ExpandFCOPYSIGN()
1642 DAG.getConstant(0, DL, IntVT), ISD::SETNE); in ExpandFCOPYSIGN()
1651 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue, in ExpandFCOPYSIGN()
1659 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit); in ExpandFCOPYSIGN()
1664 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst); in ExpandFCOPYSIGN()
1667 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst); in ExpandFCOPYSIGN()
1671 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit); in ExpandFCOPYSIGN()
1675 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit); in ExpandFCOPYSIGN()
1689 DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask); in ExpandFNEG()
1701 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) { in ExpandFABS()
1703 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero); in ExpandFABS()
1711 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue, in ExpandFABS()
1739 ISD::ADD : ISD::SUB; in ExpandDYNAMIC_STACKALLOC()
1744 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1, in ExpandDYNAMIC_STACKALLOC()
1774 !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT))) in EmitStackConvert()
1804 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT, in EmitStackConvert()
1850 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V); in ExpandBVWithShuffles()
1958 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); in ExpandBUILD_VECTOR()
2013 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1); in ExpandBUILD_VECTOR()
2016 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2); in ExpandBUILD_VECTOR()
2145 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in ExpandFrexpLibCall()
2229 bool isSigned = Opcode == ISD::SDIVREM; in ExpandDivRemLibCall()
2306 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN in useSinCos()
2307 ? ISD::FCOS : ISD::FSIN; in useSinCos()
2314 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS) in useSinCos()
2395 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO in expandLdexp()
2428 SDValue NGtMaxExp = DAG.getSetCC(dl, SetCCVT, N, MaxExp, ISD::SETGT); in expandLdexp()
2430 SDValue DecN0 = DAG.getNode(ISD::SUB, dl, ExpVT, N, MaxExp, NSW); in expandLdexp()
2432 SDValue ClampN_Big = DAG.getNode(ISD::SMIN, dl, ExpVT, N, ClampMaxVal); in expandLdexp()
2434 DAG.getNode(ISD::SUB, dl, ExpVT, ClampN_Big, DoubleMaxExp, NSW); in expandLdexp()
2437 DAG.getSetCC(dl, SetCCVT, N, DoubleMaxExp, ISD::SETUGT); in expandLdexp()
2440 SDValue ScaleUp0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleUpVal); in expandLdexp()
2441 SDValue ScaleUp1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleUp0, ScaleUpVal); in expandLdexp()
2444 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleUpTwice, DecN1, DecN0); in expandLdexp()
2446 DAG.getNode(ISD::SELECT, dl, VT, ScaleUpTwice, ScaleUp1, ScaleUp0); in expandLdexp()
2449 SDValue NLtMinExp = DAG.getSetCC(dl, SetCCVT, N, MinExp, ISD::SETLT); in expandLdexp()
2454 SDValue IncN0 = DAG.getNode(ISD::ADD, dl, ExpVT, N, Increment0, NUW_NSW); in expandLdexp()
2458 SDValue ClampN_Small = DAG.getNode(ISD::SMAX, dl, ExpVT, N, ClampMinVal); in expandLdexp()
2460 DAG.getNode(ISD::ADD, dl, ExpVT, ClampN_Small, Increment1, NSW); in expandLdexp()
2463 SDValue ScaleDown0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleDownVal); in expandLdexp()
2464 SDValue ScaleDown1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleDown0, ScaleDownVal); in expandLdexp()
2468 ISD::SETULT); in expandLdexp()
2471 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleDownTwice, IncN1, IncN0); in expandLdexp()
2473 DAG.getNode(ISD::SELECT, dl, VT, ScaleDownTwice, ScaleDown1, ScaleDown0); in expandLdexp()
2478 ISD::SELECT, dl, VT, NGtMaxExp, SelectX_Big, in expandLdexp()
2479 DAG.getNode(ISD::SELECT, dl, VT, NLtMinExp, SelectX_Small, X)); in expandLdexp()
2482 ISD::SELECT, dl, ExpVT, NGtMaxExp, SelectN_Big, in expandLdexp()
2483 DAG.getNode(ISD::SELECT, dl, ExpVT, NLtMinExp, SelectN_Small, N)); in expandLdexp()
2485 SDValue BiasedN = DAG.getNode(ISD::ADD, dl, ExpVT, NewN, MaxExp, NSW); in expandLdexp()
2491 SDValue AsInt = DAG.getNode(ISD::SHL, dl, AsIntVT, CastExpToValTy, in expandLdexp()
2493 SDValue AsFP = DAG.getNode(ISD::BITCAST, dl, VT, AsInt); in expandLdexp()
2494 return DAG.getNode(ISD::FMUL, dl, VT, NewX, AsFP); in expandLdexp()
2555 SDValue ScaleUp = DAG.getNode(ISD::FMUL, dl, VT, Val, ScaleUpK); in expandFrexp()
2560 SDValue AsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, Val); in expandFrexp()
2562 SDValue Abs = DAG.getNode(ISD::AND, dl, AsIntVT, AsInt, SignMask); in expandFrexp()
2565 DAG.getNode(ISD::ADD, dl, AsIntVT, Abs, NegSmallestNormalizedInt); in expandFrexp()
2567 NegSmallestNormalizedInt, ISD::SETULE); in expandFrexp()
2570 DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt, ISD::SETULT); in expandFrexp()
2575 SDValue ScaledAsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, ScaleUp); in expandFrexp()
2577 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ScaledAsInt, AsInt); in expandFrexp()
2580 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledAsInt, ExpMask); in expandFrexp()
2583 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ExpMaskScaled, Abs); in expandFrexp()
2589 DAG.getNode(ISD::SRL, dl, AsIntVT, ScaledValue, ExponentShiftAmt); in expandFrexp()
2592 SDValue NormalBiasedExp = DAG.getNode(ISD::ADD, dl, ExpVT, Exp, MinExp); in expandFrexp()
2595 DAG.getNode(ISD::SELECT, dl, ExpVT, IsDenormal, DenormalOffset, Zero); in expandFrexp()
2598 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledSelect, FractSignMask); in expandFrexp()
2601 SDValue Or = DAG.getNode(ISD::OR, dl, AsIntVT, MaskedFractAsInt, FPHalf); in expandFrexp()
2602 SDValue MaskedFract = DAG.getNode(ISD::BITCAST, dl, VT, Or); in expandFrexp()
2605 DAG.getNode(ISD::ADD, dl, ExpVT, NormalBiasedExp, DenormalExpBias); in expandFrexp()
2608 DAG.getNode(ISD::SELECT, dl, VT, DenormOrZero, Val, MaskedFract); in expandFrexp()
2611 DAG.getNode(ISD::SELECT, dl, ExpVT, DenormOrZero, Zero, ComputedExp); in expandFrexp()
2622 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP || in ExpandLegalINT_TO_FP()
2623 Node->getOpcode() == ISD::SINT_TO_FP); in ExpandLegalINT_TO_FP()
2634 TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND in ExpandLegalINT_TO_FP()
2635 : ISD::FP_EXTEND, in ExpandLegalINT_TO_FP()
2647 Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo, in ExpandLegalINT_TO_FP()
2667 MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); in ExpandLegalINT_TO_FP()
2681 Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other}, in ExpandLegalINT_TO_FP()
2694 Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); in ExpandLegalINT_TO_FP()
2724 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT); in ExpandLegalINT_TO_FP()
2728 SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst); in ExpandLegalINT_TO_FP()
2730 SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst); in ExpandLegalINT_TO_FP()
2731 SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr); in ExpandLegalINT_TO_FP()
2738 Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other }, in ExpandLegalINT_TO_FP()
2740 Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other }, in ExpandLegalINT_TO_FP()
2752 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or); in ExpandLegalINT_TO_FP()
2753 Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt); in ExpandLegalINT_TO_FP()
2754 Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); in ExpandLegalINT_TO_FP()
2762 Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT)) in ExpandLegalINT_TO_FP()
2774 Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other }, in ExpandLegalINT_TO_FP()
2777 Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); in ExpandLegalINT_TO_FP()
2780 DAG.getConstant(0, dl, SrcVT), ISD::SETLT); in ExpandLegalINT_TO_FP()
2806 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset); in ExpandLegalINT_TO_FP()
2816 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx, in ExpandLegalINT_TO_FP()
2825 SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other }, in ExpandLegalINT_TO_FP()
2831 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); in ExpandLegalINT_TO_FP()
2842 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP || in PromoteLegalINT_TO_FP()
2843 N->getOpcode() == ISD::STRICT_SINT_TO_FP; in PromoteLegalINT_TO_FP()
2846 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP; in PromoteLegalINT_TO_FP()
2847 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP; in PromoteLegalINT_TO_FP()
2882 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, in PromoteLegalINT_TO_FP()
2891 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, in PromoteLegalINT_TO_FP()
2903 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT || in PromoteLegalFP_TO_INT()
2904 N->getOpcode() == ISD::STRICT_FP_TO_SINT; in PromoteLegalFP_TO_INT()
2919 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT; in PromoteLegalFP_TO_INT()
2924 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT; in PromoteLegalFP_TO_INT()
2941 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); in PromoteLegalFP_TO_INT()
2968 return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result); in PromoteLegalFP_TO_INT_SAT()
2979 if (TLI.isOperationLegalOrPromote(ISD::CTPOP, VT)) { in ExpandPARITY()
2980 Result = DAG.getNode(ISD::CTPOP, dl, VT, Op); in ExpandPARITY()
2984 SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result, in ExpandPARITY()
2986 Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift); in ExpandPARITY()
2990 return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT)); in ExpandPARITY()
3000 case ISD::ABS: in ExpandNode()
3004 case ISD::ABDS: in ExpandNode()
3005 case ISD::ABDU: in ExpandNode()
3009 case ISD::CTPOP: in ExpandNode()
3013 case ISD::CTLZ: in ExpandNode()
3014 case ISD::CTLZ_ZERO_UNDEF: in ExpandNode()
3018 case ISD::CTTZ: in ExpandNode()
3019 case ISD::CTTZ_ZERO_UNDEF: in ExpandNode()
3023 case ISD::BITREVERSE: in ExpandNode()
3027 case ISD::BSWAP: in ExpandNode()
3031 case ISD::PARITY: in ExpandNode()
3034 case ISD::FRAMEADDR: in ExpandNode()
3035 case ISD::RETURNADDR: in ExpandNode()
3036 case ISD::FRAME_TO_ARGS_OFFSET: in ExpandNode()
3039 case ISD::EH_DWARF_CFA: { in ExpandNode()
3042 SDValue Offset = DAG.getNode(ISD::ADD, dl, in ExpandNode()
3044 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl, in ExpandNode()
3048 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()), in ExpandNode()
3050 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(), in ExpandNode()
3054 case ISD::GET_ROUNDING: in ExpandNode()
3058 case ISD::EH_RETURN: in ExpandNode()
3059 case ISD::EH_LABEL: in ExpandNode()
3060 case ISD::PREFETCH: in ExpandNode()
3061 case ISD::VAEND: in ExpandNode()
3062 case ISD::EH_SJLJ_LONGJMP: in ExpandNode()
3067 case ISD::READCYCLECOUNTER: in ExpandNode()
3074 case ISD::EH_SJLJ_SETJMP: in ExpandNode()
3080 case ISD::ATOMIC_LOAD: { in ExpandNode()
3085 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, in ExpandNode()
3092 case ISD::ATOMIC_STORE: { in ExpandNode()
3095 ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), in ExpandNode()
3101 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { in ExpandNode()
3107 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, in ExpandNode()
3118 case ISD::SIGN_EXTEND: in ExpandNode()
3119 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res, in ExpandNode()
3121 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType, in ExpandNode()
3125 case ISD::ZERO_EXTEND: in ExpandNode()
3126 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res, in ExpandNode()
3131 case ISD::ANY_EXTEND: in ExpandNode()
3140 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ); in ExpandNode()
3147 case ISD::ATOMIC_LOAD_SUB: { in ExpandNode()
3152 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG && in ExpandNode()
3156 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS); in ExpandNode()
3157 SDValue Res = DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, AN->getMemoryVT(), in ExpandNode()
3164 case ISD::DYNAMIC_STACKALLOC: in ExpandNode()
3167 case ISD::MERGE_VALUES: in ExpandNode()
3171 case ISD::UNDEF: { in ExpandNode()
3181 case ISD::STRICT_FP_ROUND: in ExpandNode()
3202 case ISD::FP_ROUND: { in ExpandNode()
3206 DAG.getNode(ISD::FP_TO_BF16, SDLoc(Node), VT, Node->getOperand(0))); in ExpandNode()
3212 case ISD::BITCAST: in ExpandNode()
3217 case ISD::STRICT_FP_EXTEND: in ExpandNode()
3238 case ISD::FP_EXTEND: { in ExpandNode()
3243 Results.push_back(DAG.getNode(ISD::BF16_TO_FP, SDLoc(Node), DstVT, Op)); in ExpandNode()
3251 case ISD::BF16_TO_FP: { in ExpandNode()
3258 Op = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, in ExpandNode()
3259 DAG.getNode(ISD::BITCAST, dl, MVT::i16, Op)); in ExpandNode()
3264 ISD::SHL, dl, MVT::i32, Op, in ExpandNode()
3267 Op = DAG.getNode(ISD::BITCAST, dl, MVT::f32, Op); in ExpandNode()
3270 Op = DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Op); in ExpandNode()
3274 case ISD::FP_TO_BF16: { in ExpandNode()
3277 Op = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op, in ExpandNode()
3280 ISD::SRL, dl, MVT::i32, DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op), in ExpandNode()
3286 Op = DAG.getNode(ISD::BITCAST, dl, MVT::bf16, in ExpandNode()
3287 DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Op)); in ExpandNode()
3294 case ISD::SIGN_EXTEND_INREG: { in ExpandNode()
3307 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One); in ExpandNode()
3309 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And); in ExpandNode()
3320 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), in ExpandNode()
3322 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); in ExpandNode()
3326 case ISD::UINT_TO_FP: in ExpandNode()
3327 case ISD::STRICT_UINT_TO_FP: in ExpandNode()
3335 case ISD::SINT_TO_FP: in ExpandNode()
3336 case ISD::STRICT_SINT_TO_FP: in ExpandNode()
3343 case ISD::FP_TO_SINT: in ExpandNode()
3347 case ISD::STRICT_FP_TO_SINT: in ExpandNode()
3354 case ISD::FP_TO_UINT: in ExpandNode()
3358 case ISD::STRICT_FP_TO_UINT: in ExpandNode()
3368 case ISD::FP_TO_SINT_SAT: in ExpandNode()
3369 case ISD::FP_TO_UINT_SAT: in ExpandNode()
3372 case ISD::VAARG: in ExpandNode()
3376 case ISD::VACOPY: in ExpandNode()
3379 case ISD::EXTRACT_VECTOR_ELT: in ExpandNode()
3382 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), in ExpandNode()
3388 case ISD::EXTRACT_SUBVECTOR: in ExpandNode()
3391 case ISD::INSERT_SUBVECTOR: in ExpandNode()
3394 case ISD::CONCAT_VECTORS: in ExpandNode()
3397 case ISD::SCALAR_TO_VECTOR: in ExpandNode()
3400 case ISD::INSERT_VECTOR_ELT: in ExpandNode()
3405 case ISD::VECTOR_SHUFFLE: { in ExpandNode()
3431 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0); in ExpandNode()
3432 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1); in ExpandNode()
3465 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0, in ExpandNode()
3469 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1, in ExpandNode()
3475 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3479 case ISD::VECTOR_SPLICE: { in ExpandNode()
3483 case ISD::EXTRACT_ELEMENT: { in ExpandNode()
3487 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), in ExpandNode()
3492 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3495 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), in ExpandNode()
3501 case ISD::STACKSAVE: in ExpandNode()
3513 case ISD::STACKRESTORE: in ExpandNode()
3523 case ISD::GET_DYNAMIC_AREA_OFFSET: in ExpandNode()
3527 case ISD::FCOPYSIGN: in ExpandNode()
3530 case ISD::FNEG: in ExpandNode()
3533 case ISD::FABS: in ExpandNode()
3536 case ISD::IS_FPCLASS: { in ExpandNode()
3544 case ISD::SMIN: in ExpandNode()
3545 case ISD::SMAX: in ExpandNode()
3546 case ISD::UMIN: in ExpandNode()
3547 case ISD::UMAX: { in ExpandNode()
3549 ISD::CondCode Pred; in ExpandNode()
3552 case ISD::SMAX: Pred = ISD::SETGT; break; in ExpandNode()
3553 case ISD::SMIN: Pred = ISD::SETLT; break; in ExpandNode()
3554 case ISD::UMAX: Pred = ISD::SETUGT; break; in ExpandNode()
3555 case ISD::UMIN: Pred = ISD::SETULT; break; in ExpandNode()
3563 case ISD::FMINNUM: in ExpandNode()
3564 case ISD::FMAXNUM: { in ExpandNode()
3569 case ISD::FSIN: in ExpandNode()
3570 case ISD::FCOS: { in ExpandNode()
3574 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) || in ExpandNode()
3578 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); in ExpandNode()
3579 if (Node->getOpcode() == ISD::FCOS) in ExpandNode()
3585 case ISD::FLDEXP: in ExpandNode()
3586 case ISD::STRICT_FLDEXP: { in ExpandNode()
3596 if (Node->getOpcode() == ISD::STRICT_FLDEXP) in ExpandNode()
3602 case ISD::FFREXP: { in ExpandNode()
3615 case ISD::FMAD: in ExpandNode()
3618 case ISD::FP16_TO_FP: in ExpandNode()
3624 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); in ExpandNode()
3626 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res)); in ExpandNode()
3629 case ISD::STRICT_FP16_TO_FP: in ExpandNode()
3635 DAG.getNode(ISD::STRICT_FP16_TO_FP, dl, {MVT::f32, MVT::Other}, in ExpandNode()
3637 Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, in ExpandNode()
3644 case ISD::FP_TO_FP16: in ExpandNode()
3650 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) { in ExpandNode()
3654 DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op, in ExpandNode()
3657 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal)); in ExpandNode()
3661 case ISD::ConstantFP: { in ExpandNode()
3670 case ISD::Constant: { in ExpandNode()
3675 case ISD::FSUB: { in ExpandNode()
3677 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) && in ExpandNode()
3678 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) { in ExpandNode()
3680 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); in ExpandNode()
3681 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags); in ExpandNode()
3686 case ISD::SUB: { in ExpandNode()
3688 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) && in ExpandNode()
3689 TLI.isOperationLegalOrCustom(ISD::XOR, VT) && in ExpandNode()
3692 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT)); in ExpandNode()
3693 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); in ExpandNode()
3696 case ISD::UREM: in ExpandNode()
3697 case ISD::SREM: in ExpandNode()
3701 case ISD::UDIV: in ExpandNode()
3702 case ISD::SDIV: { in ExpandNode()
3703 bool isSigned = Node->getOpcode() == ISD::SDIV; in ExpandNode()
3704 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; in ExpandNode()
3714 case ISD::MULHU: in ExpandNode()
3715 case ISD::MULHS: { in ExpandNode()
3717 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI; in ExpandNode()
3726 case ISD::UMUL_LOHI: in ExpandNode()
3727 case ISD::SMUL_LOHI: { in ExpandNode()
3732 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS; in ExpandNode()
3735 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS)); in ExpandNode()
3747 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]); in ExpandNode()
3748 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]); in ExpandNode()
3752 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); in ExpandNode()
3753 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); in ExpandNode()
3759 case ISD::MUL: { in ExpandNode()
3767 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT); in ExpandNode()
3768 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT); in ExpandNode()
3769 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT); in ExpandNode()
3770 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT); in ExpandNode()
3773 OpToUse = ISD::SMUL_LOHI; in ExpandNode()
3775 OpToUse = ISD::UMUL_LOHI; in ExpandNode()
3777 OpToUse = ISD::SMUL_LOHI; in ExpandNode()
3779 OpToUse = ISD::UMUL_LOHI; in ExpandNode()
3789 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) && in ExpandNode()
3790 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) && in ExpandNode()
3791 TLI.isOperationLegalOrCustom(ISD::SHL, VT) && in ExpandNode()
3792 TLI.isOperationLegalOrCustom(ISD::OR, VT) && in ExpandNode()
3795 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); in ExpandNode()
3796 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi); in ExpandNode()
3800 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); in ExpandNode()
3801 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); in ExpandNode()
3805 case ISD::FSHL: in ExpandNode()
3806 case ISD::FSHR: in ExpandNode()
3810 case ISD::ROTL: in ExpandNode()
3811 case ISD::ROTR: in ExpandNode()
3815 case ISD::SADDSAT: in ExpandNode()
3816 case ISD::UADDSAT: in ExpandNode()
3817 case ISD::SSUBSAT: in ExpandNode()
3818 case ISD::USUBSAT: in ExpandNode()
3821 case ISD::SSHLSAT: in ExpandNode()
3822 case ISD::USHLSAT: in ExpandNode()
3825 case ISD::SMULFIX: in ExpandNode()
3826 case ISD::SMULFIXSAT: in ExpandNode()
3827 case ISD::UMULFIX: in ExpandNode()
3828 case ISD::UMULFIXSAT: in ExpandNode()
3831 case ISD::SDIVFIX: in ExpandNode()
3832 case ISD::SDIVFIXSAT: in ExpandNode()
3833 case ISD::UDIVFIX: in ExpandNode()
3834 case ISD::UDIVFIXSAT: in ExpandNode()
3851 case ISD::UADDO_CARRY: in ExpandNode()
3852 case ISD::USUBO_CARRY: { in ExpandNode()
3857 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY; in ExpandNode()
3860 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB; in ExpandNode()
3867 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT; in ExpandNode()
3873 DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One); in ExpandNode()
3882 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ) in ExpandNode()
3883 : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ); in ExpandNode()
3884 Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2, in ExpandNode()
3888 DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2); in ExpandNode()
3894 case ISD::SADDO: in ExpandNode()
3895 case ISD::SSUBO: { in ExpandNode()
3902 case ISD::UADDO: in ExpandNode()
3903 case ISD::USUBO: { in ExpandNode()
3910 case ISD::UMULO: in ExpandNode()
3911 case ISD::SMULO: { in ExpandNode()
3919 case ISD::BUILD_PAIR: { in ExpandNode()
3921 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); in ExpandNode()
3922 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); in ExpandNode()
3924 ISD::SHL, dl, PairTy, Tmp2, in ExpandNode()
3927 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2)); in ExpandNode()
3930 case ISD::SELECT: in ExpandNode()
3934 if (Tmp1.getOpcode() == ISD::SETCC) { in ExpandNode()
3941 Tmp2, Tmp3, ISD::SETNE); in ExpandNode()
3946 case ISD::BR_JT: { in ExpandNode()
3964 ISD::SHL, dl, Index.getValueType(), Index, in ExpandNode()
3967 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index, in ExpandNode()
3969 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(), in ExpandNode()
3974 ISD::SEXTLOAD, dl, PTy, Chain, Addr, in ExpandNode()
3981 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, in ExpandNode()
3989 case ISD::BRCOND: in ExpandNode()
3994 if (Tmp2.getOpcode() == ISD::SETCC && in ExpandNode()
3995 TLI.isOperationLegalOrCustom(ISD::BR_CC, in ExpandNode()
3997 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, Tmp2.getOperand(2), in ExpandNode()
4003 (Tmp2.getOpcode() == ISD::AND && isOneConstant(Tmp2.getOperand(1)))) in ExpandNode()
4006 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2, in ExpandNode()
4008 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, in ExpandNode()
4009 DAG.getCondCode(ISD::SETNE), Tmp3, in ExpandNode()
4015 case ISD::SETCC: in ExpandNode()
4016 case ISD::VP_SETCC: in ExpandNode()
4017 case ISD::STRICT_FSETCC: in ExpandNode()
4018 case ISD::STRICT_FSETCCS: { in ExpandNode()
4019 bool IsVP = Node->getOpcode() == ISD::VP_SETCC; in ExpandNode()
4020 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC || in ExpandNode()
4021 Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandNode()
4022 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandNode()
4080 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, in ExpandNode()
4087 case ISD::SELECT_CC: { in ExpandNode()
4096 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get(); in ExpandNode()
4102 assert(!TLI.isOperationExpand(ISD::SELECT, VT) && in ExpandNode()
4106 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags()); in ExpandNode()
4116 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, Tmp1.getValueType()); in ExpandNode()
4125 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC); in ExpandNode()
4150 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), in ExpandNode()
4154 CC = DAG.getCondCode(ISD::SETNE); in ExpandNode()
4155 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
4163 case ISD::BR_CC: { in ExpandNode()
4182 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
4186 Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE); in ExpandNode()
4187 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, in ExpandNode()
4193 case ISD::BUILD_VECTOR: in ExpandNode()
4196 case ISD::SPLAT_VECTOR: in ExpandNode()
4199 case ISD::SRA: in ExpandNode()
4200 case ISD::SRL: in ExpandNode()
4201 case ISD::SHL: { in ExpandNode()
4211 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), in ExpandNode()
4214 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), in ExpandNode()
4224 case ISD::VECREDUCE_FADD: in ExpandNode()
4225 case ISD::VECREDUCE_FMUL: in ExpandNode()
4226 case ISD::VECREDUCE_ADD: in ExpandNode()
4227 case ISD::VECREDUCE_MUL: in ExpandNode()
4228 case ISD::VECREDUCE_AND: in ExpandNode()
4229 case ISD::VECREDUCE_OR: in ExpandNode()
4230 case ISD::VECREDUCE_XOR: in ExpandNode()
4231 case ISD::VECREDUCE_SMAX: in ExpandNode()
4232 case ISD::VECREDUCE_SMIN: in ExpandNode()
4233 case ISD::VECREDUCE_UMAX: in ExpandNode()
4234 case ISD::VECREDUCE_UMIN: in ExpandNode()
4235 case ISD::VECREDUCE_FMAX: in ExpandNode()
4236 case ISD::VECREDUCE_FMIN: in ExpandNode()
4237 case ISD::VECREDUCE_FMAXIMUM: in ExpandNode()
4238 case ISD::VECREDUCE_FMINIMUM: in ExpandNode()
4241 case ISD::GLOBAL_OFFSET_TABLE: in ExpandNode()
4242 case ISD::GlobalAddress: in ExpandNode()
4243 case ISD::GlobalTLSAddress: in ExpandNode()
4244 case ISD::ExternalSymbol: in ExpandNode()
4245 case ISD::ConstantPool: in ExpandNode()
4246 case ISD::JumpTable: in ExpandNode()
4247 case ISD::INTRINSIC_W_CHAIN: in ExpandNode()
4248 case ISD::INTRINSIC_WO_CHAIN: in ExpandNode()
4249 case ISD::INTRINSIC_VOID: in ExpandNode()
4269 case ISD::STRICT_FSUB: { in ExpandNode()
4271 ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal) in ExpandNode()
4274 ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal) in ExpandNode()
4279 SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags); in ExpandNode()
4280 SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(), in ExpandNode()
4288 case ISD::STRICT_SINT_TO_FP: in ExpandNode()
4289 case ISD::STRICT_UINT_TO_FP: in ExpandNode()
4290 case ISD::STRICT_LRINT: in ExpandNode()
4291 case ISD::STRICT_LLRINT: in ExpandNode()
4292 case ISD::STRICT_LROUND: in ExpandNode()
4293 case ISD::STRICT_LLROUND: in ExpandNode()
4322 case ISD::ATOMIC_FENCE: { in ConvertNodeToLibcall()
4344 case ISD::ATOMIC_SWAP: in ConvertNodeToLibcall()
4345 case ISD::ATOMIC_LOAD_ADD: in ConvertNodeToLibcall()
4346 case ISD::ATOMIC_LOAD_SUB: in ConvertNodeToLibcall()
4347 case ISD::ATOMIC_LOAD_AND: in ConvertNodeToLibcall()
4348 case ISD::ATOMIC_LOAD_CLR: in ConvertNodeToLibcall()
4349 case ISD::ATOMIC_LOAD_OR: in ConvertNodeToLibcall()
4350 case ISD::ATOMIC_LOAD_XOR: in ConvertNodeToLibcall()
4351 case ISD::ATOMIC_LOAD_NAND: in ConvertNodeToLibcall()
4352 case ISD::ATOMIC_LOAD_MIN: in ConvertNodeToLibcall()
4353 case ISD::ATOMIC_LOAD_MAX: in ConvertNodeToLibcall()
4354 case ISD::ATOMIC_LOAD_UMIN: in ConvertNodeToLibcall()
4355 case ISD::ATOMIC_LOAD_UMAX: in ConvertNodeToLibcall()
4356 case ISD::ATOMIC_CMP_SWAP: { in ConvertNodeToLibcall()
4383 case ISD::TRAP: { in ConvertNodeToLibcall()
4398 case ISD::FMINNUM: in ConvertNodeToLibcall()
4399 case ISD::STRICT_FMINNUM: in ConvertNodeToLibcall()
4407 case ISD::FMAXNUM: in ConvertNodeToLibcall()
4408 case ISD::STRICT_FMAXNUM: in ConvertNodeToLibcall()
4413 case ISD::FSQRT: in ConvertNodeToLibcall()
4414 case ISD::STRICT_FSQRT: in ConvertNodeToLibcall()
4419 case ISD::FCBRT: in ConvertNodeToLibcall()
4424 case ISD::FSIN: in ConvertNodeToLibcall()
4425 case ISD::STRICT_FSIN: in ConvertNodeToLibcall()
4430 case ISD::FCOS: in ConvertNodeToLibcall()
4431 case ISD::STRICT_FCOS: in ConvertNodeToLibcall()
4436 case ISD::FSINCOS: in ConvertNodeToLibcall()
4440 case ISD::FLOG: in ConvertNodeToLibcall()
4441 case ISD::STRICT_FLOG: in ConvertNodeToLibcall()
4445 case ISD::FLOG2: in ConvertNodeToLibcall()
4446 case ISD::STRICT_FLOG2: in ConvertNodeToLibcall()
4450 case ISD::FLOG10: in ConvertNodeToLibcall()
4451 case ISD::STRICT_FLOG10: in ConvertNodeToLibcall()
4455 case ISD::FEXP: in ConvertNodeToLibcall()
4456 case ISD::STRICT_FEXP: in ConvertNodeToLibcall()
4460 case ISD::FEXP2: in ConvertNodeToLibcall()
4461 case ISD::STRICT_FEXP2: in ConvertNodeToLibcall()
4465 case ISD::FEXP10: in ConvertNodeToLibcall()
4469 case ISD::FTRUNC: in ConvertNodeToLibcall()
4470 case ISD::STRICT_FTRUNC: in ConvertNodeToLibcall()
4475 case ISD::FFLOOR: in ConvertNodeToLibcall()
4476 case ISD::STRICT_FFLOOR: in ConvertNodeToLibcall()
4481 case ISD::FCEIL: in ConvertNodeToLibcall()
4482 case ISD::STRICT_FCEIL: in ConvertNodeToLibcall()
4487 case ISD::FRINT: in ConvertNodeToLibcall()
4488 case ISD::STRICT_FRINT: in ConvertNodeToLibcall()
4493 case ISD::FNEARBYINT: in ConvertNodeToLibcall()
4494 case ISD::STRICT_FNEARBYINT: in ConvertNodeToLibcall()
4501 case ISD::FROUND: in ConvertNodeToLibcall()
4502 case ISD::STRICT_FROUND: in ConvertNodeToLibcall()
4509 case ISD::FROUNDEVEN: in ConvertNodeToLibcall()
4510 case ISD::STRICT_FROUNDEVEN: in ConvertNodeToLibcall()
4517 case ISD::FLDEXP: in ConvertNodeToLibcall()
4518 case ISD::STRICT_FLDEXP: in ConvertNodeToLibcall()
4522 case ISD::FFREXP: { in ConvertNodeToLibcall()
4526 case ISD::FPOWI: in ConvertNodeToLibcall()
4527 case ISD::STRICT_FPOWI: { in ConvertNodeToLibcall()
4534 DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(Node), in ConvertNodeToLibcall()
4538 DAG.getNode(ISD::STRICT_FPOW, SDLoc(Node), in ConvertNodeToLibcall()
4545 DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), Node->getValueType(0), in ConvertNodeToLibcall()
4547 Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node), in ConvertNodeToLibcall()
4567 case ISD::FPOW: in ConvertNodeToLibcall()
4568 case ISD::STRICT_FPOW: in ConvertNodeToLibcall()
4572 case ISD::LROUND: in ConvertNodeToLibcall()
4573 case ISD::STRICT_LROUND: in ConvertNodeToLibcall()
4579 case ISD::LLROUND: in ConvertNodeToLibcall()
4580 case ISD::STRICT_LLROUND: in ConvertNodeToLibcall()
4586 case ISD::LRINT: in ConvertNodeToLibcall()
4587 case ISD::STRICT_LRINT: in ConvertNodeToLibcall()
4593 case ISD::LLRINT: in ConvertNodeToLibcall()
4594 case ISD::STRICT_LLRINT: in ConvertNodeToLibcall()
4600 case ISD::FDIV: in ConvertNodeToLibcall()
4601 case ISD::STRICT_FDIV: in ConvertNodeToLibcall()
4606 case ISD::FREM: in ConvertNodeToLibcall()
4607 case ISD::STRICT_FREM: in ConvertNodeToLibcall()
4612 case ISD::FMA: in ConvertNodeToLibcall()
4613 case ISD::STRICT_FMA: in ConvertNodeToLibcall()
4618 case ISD::FADD: in ConvertNodeToLibcall()
4619 case ISD::STRICT_FADD: in ConvertNodeToLibcall()
4624 case ISD::FMUL: in ConvertNodeToLibcall()
4625 case ISD::STRICT_FMUL: in ConvertNodeToLibcall()
4630 case ISD::FP16_TO_FP: in ConvertNodeToLibcall()
4635 case ISD::STRICT_FP16_TO_FP: { in ConvertNodeToLibcall()
4646 case ISD::FP_TO_FP16: { in ConvertNodeToLibcall()
4653 case ISD::FP_TO_BF16: { in ConvertNodeToLibcall()
4660 case ISD::STRICT_SINT_TO_FP: in ConvertNodeToLibcall()
4661 case ISD::STRICT_UINT_TO_FP: in ConvertNodeToLibcall()
4662 case ISD::SINT_TO_FP: in ConvertNodeToLibcall()
4663 case ISD::UINT_TO_FP: { in ConvertNodeToLibcall()
4666 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP || in ConvertNodeToLibcall()
4667 Node->getOpcode() == ISD::STRICT_SINT_TO_FP; in ConvertNodeToLibcall()
4690 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, in ConvertNodeToLibcall()
4701 case ISD::FP_TO_SINT: in ConvertNodeToLibcall()
4702 case ISD::FP_TO_UINT: in ConvertNodeToLibcall()
4703 case ISD::STRICT_FP_TO_SINT: in ConvertNodeToLibcall()
4704 case ISD::STRICT_FP_TO_UINT: { in ConvertNodeToLibcall()
4707 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT || in ConvertNodeToLibcall()
4708 Node->getOpcode() == ISD::STRICT_FP_TO_SINT; in ConvertNodeToLibcall()
4737 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first)); in ConvertNodeToLibcall()
4743 case ISD::FP_ROUND: in ConvertNodeToLibcall()
4744 case ISD::STRICT_FP_ROUND: { in ConvertNodeToLibcall()
4768 case ISD::FP_EXTEND: { in ConvertNodeToLibcall()
4775 case ISD::STRICT_FP_EXTEND: in ConvertNodeToLibcall()
4776 case ISD::STRICT_FP_TO_FP16: { in ConvertNodeToLibcall()
4778 Node->getOpcode() == ISD::STRICT_FP_TO_FP16 in ConvertNodeToLibcall()
4792 case ISD::FSUB: in ConvertNodeToLibcall()
4793 case ISD::STRICT_FSUB: in ConvertNodeToLibcall()
4798 case ISD::SREM: in ConvertNodeToLibcall()
4804 case ISD::UREM: in ConvertNodeToLibcall()
4810 case ISD::SDIV: in ConvertNodeToLibcall()
4816 case ISD::UDIV: in ConvertNodeToLibcall()
4822 case ISD::SDIVREM: in ConvertNodeToLibcall()
4823 case ISD::UDIVREM: in ConvertNodeToLibcall()
4827 case ISD::MUL: in ConvertNodeToLibcall()
4833 case ISD::CTLZ_ZERO_UNDEF: in ConvertNodeToLibcall()
4848 case ISD::RESET_FPENV: { in ConvertNodeToLibcall()
4857 case ISD::GET_FPENV_MEM: { in ConvertNodeToLibcall()
4864 case ISD::SET_FPENV_MEM: { in ConvertNodeToLibcall()
4871 case ISD::GET_FPMODE: { in ConvertNodeToLibcall()
4886 case ISD::SET_FPMODE: { in ConvertNodeToLibcall()
4900 case ISD::RESET_FPMODE: { in ConvertNodeToLibcall()
4937 if (Node->getOpcode() == ISD::UINT_TO_FP || in PromoteNode()
4938 Node->getOpcode() == ISD::SINT_TO_FP || in PromoteNode()
4939 Node->getOpcode() == ISD::SETCC || in PromoteNode()
4940 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT || in PromoteNode()
4941 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) { in PromoteNode()
4944 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP || in PromoteNode()
4945 Node->getOpcode() == ISD::STRICT_SINT_TO_FP || in PromoteNode()
4946 Node->getOpcode() == ISD::STRICT_FSETCC || in PromoteNode()
4947 Node->getOpcode() == ISD::STRICT_FSETCCS) in PromoteNode()
4949 if (Node->getOpcode() == ISD::BR_CC || in PromoteNode()
4950 Node->getOpcode() == ISD::SELECT_CC) in PromoteNode()
4956 case ISD::CTTZ: in PromoteNode()
4957 case ISD::CTTZ_ZERO_UNDEF: in PromoteNode()
4958 case ISD::CTLZ: in PromoteNode()
4959 case ISD::CTLZ_ZERO_UNDEF: in PromoteNode()
4960 case ISD::CTPOP: in PromoteNode()
4962 if (Node->getOpcode() == ISD::CTTZ || in PromoteNode()
4963 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF) in PromoteNode()
4964 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4966 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4968 if (Node->getOpcode() == ISD::CTTZ) { in PromoteNode()
4974 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1, in PromoteNode()
4980 if (Node->getOpcode() == ISD::CTLZ || in PromoteNode()
4981 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) { in PromoteNode()
4983 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, in PromoteNode()
4987 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); in PromoteNode()
4989 case ISD::BITREVERSE: in PromoteNode()
4990 case ISD::BSWAP: { in PromoteNode()
4992 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4995 ISD::SRL, dl, NVT, Tmp1, in PromoteNode()
4999 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); in PromoteNode()
5002 case ISD::FP_TO_UINT: in PromoteNode()
5003 case ISD::STRICT_FP_TO_UINT: in PromoteNode()
5004 case ISD::FP_TO_SINT: in PromoteNode()
5005 case ISD::STRICT_FP_TO_SINT: in PromoteNode()
5008 case ISD::FP_TO_UINT_SAT: in PromoteNode()
5009 case ISD::FP_TO_SINT_SAT: in PromoteNode()
5012 case ISD::UINT_TO_FP: in PromoteNode()
5013 case ISD::STRICT_UINT_TO_FP: in PromoteNode()
5014 case ISD::SINT_TO_FP: in PromoteNode()
5015 case ISD::STRICT_SINT_TO_FP: in PromoteNode()
5018 case ISD::VAARG: { in PromoteNode()
5024 TruncOp = ISD::BITCAST; in PromoteNode()
5028 TruncOp = ISD::TRUNCATE; in PromoteNode()
5049 case ISD::MUL: in PromoteNode()
5050 case ISD::SDIV: in PromoteNode()
5051 case ISD::SREM: in PromoteNode()
5052 case ISD::UDIV: in PromoteNode()
5053 case ISD::UREM: in PromoteNode()
5054 case ISD::SMIN: in PromoteNode()
5055 case ISD::SMAX: in PromoteNode()
5056 case ISD::UMIN: in PromoteNode()
5057 case ISD::UMAX: in PromoteNode()
5058 case ISD::AND: in PromoteNode()
5059 case ISD::OR: in PromoteNode()
5060 case ISD::XOR: { in PromoteNode()
5063 ExtOp = ISD::BITCAST; in PromoteNode()
5064 TruncOp = ISD::BITCAST; in PromoteNode()
5070 ExtOp = ISD::ANY_EXTEND; in PromoteNode()
5072 case ISD::SDIV: in PromoteNode()
5073 case ISD::SREM: in PromoteNode()
5074 case ISD::SMIN: in PromoteNode()
5075 case ISD::SMAX: in PromoteNode()
5076 ExtOp = ISD::SIGN_EXTEND; in PromoteNode()
5078 case ISD::UDIV: in PromoteNode()
5079 case ISD::UREM: in PromoteNode()
5080 ExtOp = ISD::ZERO_EXTEND; in PromoteNode()
5082 case ISD::UMIN: in PromoteNode()
5083 case ISD::UMAX: in PromoteNode()
5085 ExtOp = ISD::SIGN_EXTEND; in PromoteNode()
5087 ExtOp = ISD::ZERO_EXTEND; in PromoteNode()
5090 TruncOp = ISD::TRUNCATE; in PromoteNode()
5100 case ISD::UMUL_LOHI: in PromoteNode()
5101 case ISD::SMUL_LOHI: { in PromoteNode()
5103 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND in PromoteNode()
5104 : ISD::SIGN_EXTEND; in PromoteNode()
5107 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2); in PromoteNode()
5112 ISD::SRL, dl, NVT, Tmp1, in PromoteNode()
5114 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); in PromoteNode()
5115 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2)); in PromoteNode()
5118 case ISD::SELECT: { in PromoteNode()
5122 ExtOp = ISD::BITCAST; in PromoteNode()
5123 TruncOp = ISD::BITCAST; in PromoteNode()
5125 ExtOp = ISD::ANY_EXTEND; in PromoteNode()
5126 TruncOp = ISD::TRUNCATE; in PromoteNode()
5128 ExtOp = ISD::FP_EXTEND; in PromoteNode()
5129 TruncOp = ISD::FP_ROUND; in PromoteNode()
5138 if (TruncOp != ISD::FP_ROUND) in PromoteNode()
5146 case ISD::VECTOR_SHUFFLE: { in PromoteNode()
5150 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); in PromoteNode()
5151 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); in PromoteNode()
5155 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1); in PromoteNode()
5159 case ISD::VECTOR_SPLICE: { in PromoteNode()
5160 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5161 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
5162 Tmp3 = DAG.getNode(ISD::VECTOR_SPLICE, dl, NVT, Tmp1, Tmp2, in PromoteNode()
5164 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp3)); in PromoteNode()
5167 case ISD::SELECT_CC: { in PromoteNode()
5169 ISD::CondCode CCCode = cast<CondCodeSDNode>(Cond)->get(); in PromoteNode()
5174 unsigned ExtOp = ISD::FP_EXTEND; in PromoteNode()
5176 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in PromoteNode()
5191 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, NVT, {Tmp1, Tmp2, Tmp3, Tmp4, Cond}, in PromoteNode()
5195 if (ExtOp != ISD::FP_EXTEND) in PromoteNode()
5196 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1); in PromoteNode()
5198 Tmp1 = DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp1, in PromoteNode()
5204 case ISD::SETCC: in PromoteNode()
5205 case ISD::STRICT_FSETCC: in PromoteNode()
5206 case ISD::STRICT_FSETCCS: { in PromoteNode()
5207 unsigned ExtOp = ISD::FP_EXTEND; in PromoteNode()
5209 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); in PromoteNode()
5212 ExtOp = ISD::SIGN_EXTEND; in PromoteNode()
5214 ExtOp = ISD::ZERO_EXTEND; in PromoteNode()
5233 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1, in PromoteNode()
5237 case ISD::BR_CC: { in PromoteNode()
5238 unsigned ExtOp = ISD::FP_EXTEND; in PromoteNode()
5240 ISD::CondCode CCCode = in PromoteNode()
5242 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in PromoteNode()
5246 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), in PromoteNode()
5251 case ISD::FADD: in PromoteNode()
5252 case ISD::FSUB: in PromoteNode()
5253 case ISD::FMUL: in PromoteNode()
5254 case ISD::FDIV: in PromoteNode()
5255 case ISD::FREM: in PromoteNode()
5256 case ISD::FMINNUM: in PromoteNode()
5257 case ISD::FMAXNUM: in PromoteNode()
5258 case ISD::FMINIMUM: in PromoteNode()
5259 case ISD::FMAXIMUM: in PromoteNode()
5260 case ISD::FPOW: in PromoteNode()
5261 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5262 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
5266 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3, in PromoteNode()
5269 case ISD::STRICT_FADD: in PromoteNode()
5270 case ISD::STRICT_FSUB: in PromoteNode()
5271 case ISD::STRICT_FMUL: in PromoteNode()
5272 case ISD::STRICT_FDIV: in PromoteNode()
5273 case ISD::STRICT_FMINNUM: in PromoteNode()
5274 case ISD::STRICT_FMAXNUM: in PromoteNode()
5275 case ISD::STRICT_FREM: in PromoteNode()
5276 case ISD::STRICT_FPOW: in PromoteNode()
5277 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5279 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5281 Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1), in PromoteNode()
5285 Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, in PromoteNode()
5290 case ISD::FMA: in PromoteNode()
5291 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5292 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
5293 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2)); in PromoteNode()
5295 DAG.getNode(ISD::FP_ROUND, dl, OVT, in PromoteNode()
5299 case ISD::STRICT_FMA: in PromoteNode()
5300 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5302 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5304 Tmp3 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5306 Tmp4 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1), in PromoteNode()
5310 Tmp4 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, in PromoteNode()
5315 case ISD::FCOPYSIGN: in PromoteNode()
5316 case ISD::FLDEXP: in PromoteNode()
5317 case ISD::FPOWI: { in PromoteNode()
5318 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5327 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN); in PromoteNode()
5329 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3, in PromoteNode()
5333 case ISD::STRICT_FPOWI: in PromoteNode()
5334 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5338 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, in PromoteNode()
5343 case ISD::FFREXP: { in PromoteNode()
5344 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5345 Tmp2 = DAG.getNode(ISD::FFREXP, dl, {NVT, Node->getValueType(1)}, Tmp1); in PromoteNode()
5348 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2, in PromoteNode()
5354 case ISD::FFLOOR: in PromoteNode()
5355 case ISD::FCEIL: in PromoteNode()
5356 case ISD::FRINT: in PromoteNode()
5357 case ISD::FNEARBYINT: in PromoteNode()
5358 case ISD::FROUND: in PromoteNode()
5359 case ISD::FROUNDEVEN: in PromoteNode()
5360 case ISD::FTRUNC: in PromoteNode()
5361 case ISD::FNEG: in PromoteNode()
5362 case ISD::FSQRT: in PromoteNode()
5363 case ISD::FSIN: in PromoteNode()
5364 case ISD::FCOS: in PromoteNode()
5365 case ISD::FLOG: in PromoteNode()
5366 case ISD::FLOG2: in PromoteNode()
5367 case ISD::FLOG10: in PromoteNode()
5368 case ISD::FABS: in PromoteNode()
5369 case ISD::FEXP: in PromoteNode()
5370 case ISD::FEXP2: in PromoteNode()
5371 case ISD::FEXP10: in PromoteNode()
5372 case ISD::FCANONICALIZE: in PromoteNode()
5373 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5376 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2, in PromoteNode()
5379 case ISD::STRICT_FFLOOR: in PromoteNode()
5380 case ISD::STRICT_FCEIL: in PromoteNode()
5381 case ISD::STRICT_FRINT: in PromoteNode()
5382 case ISD::STRICT_FNEARBYINT: in PromoteNode()
5383 case ISD::STRICT_FROUND: in PromoteNode()
5384 case ISD::STRICT_FROUNDEVEN: in PromoteNode()
5385 case ISD::STRICT_FTRUNC: in PromoteNode()
5386 case ISD::STRICT_FSQRT: in PromoteNode()
5387 case ISD::STRICT_FSIN: in PromoteNode()
5388 case ISD::STRICT_FCOS: in PromoteNode()
5389 case ISD::STRICT_FLOG: in PromoteNode()
5390 case ISD::STRICT_FLOG2: in PromoteNode()
5391 case ISD::STRICT_FLOG10: in PromoteNode()
5392 case ISD::STRICT_FEXP: in PromoteNode()
5393 case ISD::STRICT_FEXP2: in PromoteNode()
5394 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
5398 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, in PromoteNode()
5403 case ISD::BUILD_VECTOR: { in PromoteNode()
5422 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op)); in PromoteNode()
5427 DAG.getNode(MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS, in PromoteNode()
5429 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); in PromoteNode()
5433 case ISD::EXTRACT_VECTOR_ELT: { in PromoteNode()
5459 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); in PromoteNode()
5461 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); in PromoteNode()
5466 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); in PromoteNode()
5468 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, in PromoteNode()
5474 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec)); in PromoteNode()
5477 case ISD::INSERT_VECTOR_ELT: { in PromoteNode()
5507 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); in PromoteNode()
5509 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); in PromoteNode()
5510 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); in PromoteNode()
5515 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); in PromoteNode()
5517 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, in PromoteNode()
5520 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT, in PromoteNode()
5524 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec)); in PromoteNode()
5527 case ISD::SCALAR_TO_VECTOR: { in PromoteNode()
5542 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); in PromoteNode()
5550 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts); in PromoteNode()
5551 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); in PromoteNode()
5555 case ISD::ATOMIC_SWAP: { in PromoteNode()
5558 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal()); in PromoteNode()
5565 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT, in PromoteNode()
5569 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic)); in PromoteNode()
5573 case ISD::SPLAT_VECTOR: { in PromoteNode()
5578 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NewScalarType, Scalar); in PromoteNode()
5580 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2)); in PromoteNode()
5583 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NewScalarType, Scalar); in PromoteNode()
5586 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2, in PromoteNode()