Lines Matching refs:ISD
316 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) && in ExpandConstantFP()
331 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx, in ExpandConstantFP()
405 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, in ExpandINSERT_VECTOR_ELT()
423 if (!ISD::isNormalStore(ST)) in OptimizeFloatStore()
441 if (Value.getOpcode() == ISD::TargetConstantFP) in OptimizeFloatStore()
480 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); in OptimizeFloatStore()
505 switch (TLI.getOperationAction(ISD::STORE, VT)) { in LegalizeStoreOps()
529 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT); in LegalizeStoreOps()
532 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value); in LegalizeStoreOps()
586 ISD::SRL, dl, Value.getValueType(), Value, in LegalizeStoreOps()
597 ISD::SRL, dl, Value.getValueType(), Value, in LegalizeStoreOps()
605 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, in LegalizeStoreOps()
614 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); in LegalizeStoreOps()
644 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value); in LegalizeStoreOps()
650 Value = DAG.getNode(ISD::TRUNCATE, dl, in LegalizeStoreOps()
671 ISD::LoadExtType ExtType = LD->getExtensionType(); in LegalizeLoadOps()
672 if (ExtType == ISD::NON_EXTLOAD) { in LegalizeLoadOps()
704 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res); in LegalizeLoadOps()
748 ISD::LoadExtType NewExtType = in LegalizeLoadOps()
749 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD; in LegalizeLoadOps()
757 if (ExtType == ISD::SEXTLOAD) in LegalizeLoadOps()
759 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, in LegalizeLoadOps()
762 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()) in LegalizeLoadOps()
764 Result = DAG.getNode(ISD::AssertZext, dl, in LegalizeLoadOps()
791 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
804 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), in LegalizeLoadOps()
809 ISD::SHL, dl, Hi.getValueType(), Hi, in LegalizeLoadOps()
814 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
826 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
832 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), in LegalizeLoadOps()
837 ISD::SHL, dl, Hi.getValueType(), Hi, in LegalizeLoadOps()
842 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
877 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) { in LegalizeLoadOps()
885 ISD::LoadExtType MidExtType = in LegalizeLoadOps()
886 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType; in LegalizeLoadOps()
891 ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType); in LegalizeLoadOps()
906 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain, in LegalizeLoadOps()
908 Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result); in LegalizeLoadOps()
921 assert(ExtType != ISD::EXTLOAD && in LegalizeLoadOps()
925 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, in LegalizeLoadOps()
930 if (ExtType == ISD::SEXTLOAD) in LegalizeLoadOps()
931 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, in LegalizeLoadOps()
962 if (Node->getOpcode() == ISD::TargetConstant || in LegalizeOp()
963 Node->getOpcode() == ISD::Register) in LegalizeOp()
975 Op.getOpcode() == ISD::TargetConstant || in LegalizeOp()
976 Op.getOpcode() == ISD::Register) && in LegalizeOp()
984 case ISD::INTRINSIC_W_CHAIN: in LegalizeOp()
985 case ISD::INTRINSIC_WO_CHAIN: in LegalizeOp()
986 case ISD::INTRINSIC_VOID: in LegalizeOp()
987 case ISD::STACKSAVE: in LegalizeOp()
990 case ISD::GET_DYNAMIC_AREA_OFFSET: in LegalizeOp()
994 case ISD::VAARG: in LegalizeOp()
1000 case ISD::FP_TO_FP16: in LegalizeOp()
1001 case ISD::FP_TO_BF16: in LegalizeOp()
1002 case ISD::SINT_TO_FP: in LegalizeOp()
1003 case ISD::UINT_TO_FP: in LegalizeOp()
1004 case ISD::EXTRACT_VECTOR_ELT: in LegalizeOp()
1005 case ISD::LROUND: in LegalizeOp()
1006 case ISD::LLROUND: in LegalizeOp()
1007 case ISD::LRINT: in LegalizeOp()
1008 case ISD::LLRINT: in LegalizeOp()
1012 case ISD::STRICT_FP_TO_FP16: in LegalizeOp()
1013 case ISD::STRICT_SINT_TO_FP: in LegalizeOp()
1014 case ISD::STRICT_UINT_TO_FP: in LegalizeOp()
1015 case ISD::STRICT_LRINT: in LegalizeOp()
1016 case ISD::STRICT_LLRINT: in LegalizeOp()
1017 case ISD::STRICT_LROUND: in LegalizeOp()
1018 case ISD::STRICT_LLROUND: in LegalizeOp()
1025 case ISD::SIGN_EXTEND_INREG: { in LegalizeOp()
1030 case ISD::ATOMIC_STORE: in LegalizeOp()
1034 case ISD::SELECT_CC: in LegalizeOp()
1035 case ISD::STRICT_FSETCC: in LegalizeOp()
1036 case ISD::STRICT_FSETCCS: in LegalizeOp()
1037 case ISD::SETCC: in LegalizeOp()
1038 case ISD::VP_SETCC: in LegalizeOp()
1039 case ISD::BR_CC: { in LegalizeOp()
1041 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4 in LegalizeOp()
1042 : Opc == ISD::STRICT_FSETCC ? 3 in LegalizeOp()
1043 : Opc == ISD::STRICT_FSETCCS ? 3 in LegalizeOp()
1044 : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2 in LegalizeOp()
1046 unsigned CompareOperand = Opc == ISD::BR_CC ? 2 in LegalizeOp()
1047 : Opc == ISD::STRICT_FSETCC ? 1 in LegalizeOp()
1048 : Opc == ISD::STRICT_FSETCCS ? 1 in LegalizeOp()
1051 ISD::CondCode CCCode = in LegalizeOp()
1055 if (Node->getOpcode() == ISD::SELECT_CC) in LegalizeOp()
1063 case ISD::LOAD: in LegalizeOp()
1064 case ISD::STORE: in LegalizeOp()
1069 case ISD::CALLSEQ_START: in LegalizeOp()
1070 case ISD::CALLSEQ_END: in LegalizeOp()
1076 case ISD::EXTRACT_ELEMENT: in LegalizeOp()
1077 case ISD::FLT_ROUNDS_: in LegalizeOp()
1078 case ISD::MERGE_VALUES: in LegalizeOp()
1079 case ISD::EH_RETURN: in LegalizeOp()
1080 case ISD::FRAME_TO_ARGS_OFFSET: in LegalizeOp()
1081 case ISD::EH_DWARF_CFA: in LegalizeOp()
1082 case ISD::EH_SJLJ_SETJMP: in LegalizeOp()
1083 case ISD::EH_SJLJ_LONGJMP: in LegalizeOp()
1084 case ISD::EH_SJLJ_SETUP_DISPATCH: in LegalizeOp()
1091 case ISD::INIT_TRAMPOLINE: in LegalizeOp()
1092 case ISD::ADJUST_TRAMPOLINE: in LegalizeOp()
1093 case ISD::FRAMEADDR: in LegalizeOp()
1094 case ISD::RETURNADDR: in LegalizeOp()
1095 case ISD::ADDROFRETURNADDR: in LegalizeOp()
1096 case ISD::SPONENTRY: in LegalizeOp()
1103 case ISD::READCYCLECOUNTER: in LegalizeOp()
1108 case ISD::READ_REGISTER: in LegalizeOp()
1109 case ISD::WRITE_REGISTER: in LegalizeOp()
1115 case ISD::UBSANTRAP: in LegalizeOp()
1120 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), in LegalizeOp()
1127 case ISD::DEBUGTRAP: in LegalizeOp()
1132 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), in LegalizeOp()
1139 case ISD::SADDSAT: in LegalizeOp()
1140 case ISD::UADDSAT: in LegalizeOp()
1141 case ISD::SSUBSAT: in LegalizeOp()
1142 case ISD::USUBSAT: in LegalizeOp()
1143 case ISD::SSHLSAT: in LegalizeOp()
1144 case ISD::USHLSAT: in LegalizeOp()
1145 case ISD::FP_TO_SINT_SAT: in LegalizeOp()
1146 case ISD::FP_TO_UINT_SAT: in LegalizeOp()
1149 case ISD::SMULFIX: in LegalizeOp()
1150 case ISD::SMULFIXSAT: in LegalizeOp()
1151 case ISD::UMULFIX: in LegalizeOp()
1152 case ISD::UMULFIXSAT: in LegalizeOp()
1153 case ISD::SDIVFIX: in LegalizeOp()
1154 case ISD::SDIVFIXSAT: in LegalizeOp()
1155 case ISD::UDIVFIX: in LegalizeOp()
1156 case ISD::UDIVFIXSAT: { in LegalizeOp()
1162 case ISD::MSCATTER: in LegalizeOp()
1166 case ISD::MSTORE: in LegalizeOp()
1170 case ISD::VP_SCATTER: in LegalizeOp()
1175 case ISD::VP_STORE: in LegalizeOp()
1180 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: in LegalizeOp()
1185 case ISD::VECREDUCE_FADD: in LegalizeOp()
1186 case ISD::VECREDUCE_FMUL: in LegalizeOp()
1187 case ISD::VECREDUCE_ADD: in LegalizeOp()
1188 case ISD::VECREDUCE_MUL: in LegalizeOp()
1189 case ISD::VECREDUCE_AND: in LegalizeOp()
1190 case ISD::VECREDUCE_OR: in LegalizeOp()
1191 case ISD::VECREDUCE_XOR: in LegalizeOp()
1192 case ISD::VECREDUCE_SMAX: in LegalizeOp()
1193 case ISD::VECREDUCE_SMIN: in LegalizeOp()
1194 case ISD::VECREDUCE_UMAX: in LegalizeOp()
1195 case ISD::VECREDUCE_UMIN: in LegalizeOp()
1196 case ISD::VECREDUCE_FMAX: in LegalizeOp()
1197 case ISD::VECREDUCE_FMIN: in LegalizeOp()
1198 case ISD::IS_FPCLASS: in LegalizeOp()
1202 case ISD::VECREDUCE_SEQ_FADD: in LegalizeOp()
1203 case ISD::VECREDUCE_SEQ_FMUL: in LegalizeOp()
1204 case ISD::VP_REDUCE_FADD: in LegalizeOp()
1205 case ISD::VP_REDUCE_FMUL: in LegalizeOp()
1206 case ISD::VP_REDUCE_ADD: in LegalizeOp()
1207 case ISD::VP_REDUCE_MUL: in LegalizeOp()
1208 case ISD::VP_REDUCE_AND: in LegalizeOp()
1209 case ISD::VP_REDUCE_OR: in LegalizeOp()
1210 case ISD::VP_REDUCE_XOR: in LegalizeOp()
1211 case ISD::VP_REDUCE_SMAX: in LegalizeOp()
1212 case ISD::VP_REDUCE_SMIN: in LegalizeOp()
1213 case ISD::VP_REDUCE_UMAX: in LegalizeOp()
1214 case ISD::VP_REDUCE_UMIN: in LegalizeOp()
1215 case ISD::VP_REDUCE_FMAX: in LegalizeOp()
1216 case ISD::VP_REDUCE_FMIN: in LegalizeOp()
1217 case ISD::VP_REDUCE_SEQ_FADD: in LegalizeOp()
1218 case ISD::VP_REDUCE_SEQ_FMUL: in LegalizeOp()
1223 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { in LegalizeOp()
1235 case ISD::SHL: in LegalizeOp()
1236 case ISD::SRL: in LegalizeOp()
1237 case ISD::SRA: in LegalizeOp()
1238 case ISD::ROTL: in LegalizeOp()
1239 case ISD::ROTR: { in LegalizeOp()
1256 case ISD::FSHL: in LegalizeOp()
1257 case ISD::FSHR: in LegalizeOp()
1258 case ISD::SRL_PARTS: in LegalizeOp()
1259 case ISD::SRA_PARTS: in LegalizeOp()
1260 case ISD::SHL_PARTS: { in LegalizeOp()
1343 case ISD::CALLSEQ_START: in LegalizeOp()
1344 case ISD::CALLSEQ_END: in LegalizeOp()
1346 case ISD::LOAD: in LegalizeOp()
1348 case ISD::STORE: in LegalizeOp()
1419 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, in ExpandExtractFromVectorThroughStack()
1470 assert((Node->getOpcode() == ISD::BUILD_VECTOR || in ExpandVectorBuildThroughStack()
1471 Node->getOpcode() == ISD::CONCAT_VECTORS) && in ExpandVectorBuildThroughStack()
1517 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); in ExpandVectorBuildThroughStack()
1537 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value); in getSignAsIntValue()
1572 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr, in getSignAsIntValue()
1584 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue); in modifySignAsInt()
1604 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue, in ExpandFCOPYSIGN()
1609 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) && in ExpandFCOPYSIGN()
1610 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) { in ExpandFCOPYSIGN()
1611 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag); in ExpandFCOPYSIGN()
1612 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue); in ExpandFCOPYSIGN()
1614 DAG.getConstant(0, DL, IntVT), ISD::SETNE); in ExpandFCOPYSIGN()
1623 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue, in ExpandFCOPYSIGN()
1631 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit); in ExpandFCOPYSIGN()
1636 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst); in ExpandFCOPYSIGN()
1639 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst); in ExpandFCOPYSIGN()
1643 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit); in ExpandFCOPYSIGN()
1647 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit); in ExpandFCOPYSIGN()
1661 DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask); in ExpandFNEG()
1673 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) { in ExpandFABS()
1675 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero); in ExpandFABS()
1683 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue, in ExpandFABS()
1711 ISD::ADD : ISD::SUB; in ExpandDYNAMIC_STACKALLOC()
1716 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1, in ExpandDYNAMIC_STACKALLOC()
1747 !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT))) in EmitStackConvert()
1777 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT, in EmitStackConvert()
1823 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V); in ExpandBVWithShuffles()
1931 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); in ExpandBUILD_VECTOR()
1986 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1); in ExpandBUILD_VECTOR()
1989 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2); in ExpandBUILD_VECTOR()
2155 bool isSigned = Opcode == ISD::SDIVREM; in ExpandDivRemLibCall()
2236 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN in useSinCos()
2237 ? ISD::FCOS : ISD::FSIN; in useSinCos()
2244 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS) in useSinCos()
2321 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP || in ExpandLegalINT_TO_FP()
2322 Node->getOpcode() == ISD::SINT_TO_FP); in ExpandLegalINT_TO_FP()
2333 TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND in ExpandLegalINT_TO_FP()
2334 : ISD::FP_EXTEND, in ExpandLegalINT_TO_FP()
2346 Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo, in ExpandLegalINT_TO_FP()
2365 MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); in ExpandLegalINT_TO_FP()
2379 Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other}, in ExpandLegalINT_TO_FP()
2392 Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); in ExpandLegalINT_TO_FP()
2422 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT); in ExpandLegalINT_TO_FP()
2426 SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst); in ExpandLegalINT_TO_FP()
2428 SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst); in ExpandLegalINT_TO_FP()
2429 SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr); in ExpandLegalINT_TO_FP()
2436 Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other }, in ExpandLegalINT_TO_FP()
2438 Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other }, in ExpandLegalINT_TO_FP()
2450 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or); in ExpandLegalINT_TO_FP()
2451 Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt); in ExpandLegalINT_TO_FP()
2452 Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); in ExpandLegalINT_TO_FP()
2460 Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT)) in ExpandLegalINT_TO_FP()
2472 Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other }, in ExpandLegalINT_TO_FP()
2475 Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); in ExpandLegalINT_TO_FP()
2478 DAG.getConstant(0, dl, SrcVT), ISD::SETLT); in ExpandLegalINT_TO_FP()
2504 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset); in ExpandLegalINT_TO_FP()
2514 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx, in ExpandLegalINT_TO_FP()
2523 SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other }, in ExpandLegalINT_TO_FP()
2529 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); in ExpandLegalINT_TO_FP()
2540 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP || in PromoteLegalINT_TO_FP()
2541 N->getOpcode() == ISD::STRICT_SINT_TO_FP; in PromoteLegalINT_TO_FP()
2544 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP; in PromoteLegalINT_TO_FP()
2545 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP; in PromoteLegalINT_TO_FP()
2580 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, in PromoteLegalINT_TO_FP()
2589 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, in PromoteLegalINT_TO_FP()
2601 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT || in PromoteLegalFP_TO_INT()
2602 N->getOpcode() == ISD::STRICT_FP_TO_SINT; in PromoteLegalFP_TO_INT()
2617 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT; in PromoteLegalFP_TO_INT()
2622 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT; in PromoteLegalFP_TO_INT()
2639 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); in PromoteLegalFP_TO_INT()
2666 return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result); in PromoteLegalFP_TO_INT_SAT()
2677 if (TLI.isOperationLegalOrPromote(ISD::CTPOP, VT)) { in ExpandPARITY()
2678 Result = DAG.getNode(ISD::CTPOP, dl, VT, Op); in ExpandPARITY()
2682 SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result, in ExpandPARITY()
2684 Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift); in ExpandPARITY()
2688 return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT)); in ExpandPARITY()
2698 case ISD::ABS: in ExpandNode()
2702 case ISD::CTPOP: in ExpandNode()
2706 case ISD::CTLZ: in ExpandNode()
2707 case ISD::CTLZ_ZERO_UNDEF: in ExpandNode()
2711 case ISD::CTTZ: in ExpandNode()
2712 case ISD::CTTZ_ZERO_UNDEF: in ExpandNode()
2716 case ISD::BITREVERSE: in ExpandNode()
2720 case ISD::BSWAP: in ExpandNode()
2724 case ISD::PARITY: in ExpandNode()
2727 case ISD::FRAMEADDR: in ExpandNode()
2728 case ISD::RETURNADDR: in ExpandNode()
2729 case ISD::FRAME_TO_ARGS_OFFSET: in ExpandNode()
2732 case ISD::EH_DWARF_CFA: { in ExpandNode()
2735 SDValue Offset = DAG.getNode(ISD::ADD, dl, in ExpandNode()
2737 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl, in ExpandNode()
2741 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()), in ExpandNode()
2743 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(), in ExpandNode()
2747 case ISD::FLT_ROUNDS_: in ExpandNode()
2751 case ISD::EH_RETURN: in ExpandNode()
2752 case ISD::EH_LABEL: in ExpandNode()
2753 case ISD::PREFETCH: in ExpandNode()
2754 case ISD::VAEND: in ExpandNode()
2755 case ISD::EH_SJLJ_LONGJMP: in ExpandNode()
2760 case ISD::READCYCLECOUNTER: in ExpandNode()
2767 case ISD::EH_SJLJ_SETJMP: in ExpandNode()
2773 case ISD::ATOMIC_LOAD: { in ExpandNode()
2778 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, in ExpandNode()
2785 case ISD::ATOMIC_STORE: { in ExpandNode()
2787 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, in ExpandNode()
2795 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { in ExpandNode()
2801 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, in ExpandNode()
2812 case ISD::SIGN_EXTEND: in ExpandNode()
2813 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res, in ExpandNode()
2815 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType, in ExpandNode()
2819 case ISD::ZERO_EXTEND: in ExpandNode()
2820 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res, in ExpandNode()
2825 case ISD::ANY_EXTEND: in ExpandNode()
2834 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ); in ExpandNode()
2841 case ISD::DYNAMIC_STACKALLOC: in ExpandNode()
2844 case ISD::MERGE_VALUES: in ExpandNode()
2848 case ISD::UNDEF: { in ExpandNode()
2858 case ISD::STRICT_FP_ROUND: in ExpandNode()
2879 case ISD::FP_ROUND: in ExpandNode()
2880 case ISD::BITCAST: in ExpandNode()
2885 case ISD::STRICT_FP_EXTEND: in ExpandNode()
2906 case ISD::FP_EXTEND: in ExpandNode()
2912 case ISD::BF16_TO_FP: { in ExpandNode()
2914 SDValue Op = DAG.getNode(ISD::BITCAST, dl, MVT::i16, Node->getOperand(0)); in ExpandNode()
2915 Op = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op); in ExpandNode()
2917 ISD::SHL, dl, MVT::i32, Op, in ExpandNode()
2920 Op = DAG.getNode(ISD::BITCAST, dl, MVT::f32, Op); in ExpandNode()
2924 case ISD::SIGN_EXTEND_INREG: { in ExpandNode()
2937 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One); in ExpandNode()
2939 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And); in ExpandNode()
2950 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), in ExpandNode()
2952 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); in ExpandNode()
2956 case ISD::UINT_TO_FP: in ExpandNode()
2957 case ISD::STRICT_UINT_TO_FP: in ExpandNode()
2965 case ISD::SINT_TO_FP: in ExpandNode()
2966 case ISD::STRICT_SINT_TO_FP: in ExpandNode()
2973 case ISD::FP_TO_SINT: in ExpandNode()
2977 case ISD::STRICT_FP_TO_SINT: in ExpandNode()
2984 case ISD::FP_TO_UINT: in ExpandNode()
2988 case ISD::STRICT_FP_TO_UINT: in ExpandNode()
2998 case ISD::FP_TO_SINT_SAT: in ExpandNode()
2999 case ISD::FP_TO_UINT_SAT: in ExpandNode()
3002 case ISD::VAARG: in ExpandNode()
3006 case ISD::VACOPY: in ExpandNode()
3009 case ISD::EXTRACT_VECTOR_ELT: in ExpandNode()
3012 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), in ExpandNode()
3018 case ISD::EXTRACT_SUBVECTOR: in ExpandNode()
3021 case ISD::INSERT_SUBVECTOR: in ExpandNode()
3024 case ISD::CONCAT_VECTORS: in ExpandNode()
3027 case ISD::SCALAR_TO_VECTOR: in ExpandNode()
3030 case ISD::INSERT_VECTOR_ELT: in ExpandNode()
3035 case ISD::VECTOR_SHUFFLE: { in ExpandNode()
3061 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0); in ExpandNode()
3062 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1); in ExpandNode()
3095 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0, in ExpandNode()
3099 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1, in ExpandNode()
3105 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3109 case ISD::VECTOR_SPLICE: { in ExpandNode()
3113 case ISD::EXTRACT_ELEMENT: { in ExpandNode()
3117 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), in ExpandNode()
3122 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3125 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), in ExpandNode()
3131 case ISD::STACKSAVE: in ExpandNode()
3143 case ISD::STACKRESTORE: in ExpandNode()
3153 case ISD::GET_DYNAMIC_AREA_OFFSET: in ExpandNode()
3157 case ISD::FCOPYSIGN: in ExpandNode()
3160 case ISD::FNEG: in ExpandNode()
3163 case ISD::FABS: in ExpandNode()
3166 case ISD::IS_FPCLASS: { in ExpandNode()
3175 case ISD::SMIN: in ExpandNode()
3176 case ISD::SMAX: in ExpandNode()
3177 case ISD::UMIN: in ExpandNode()
3178 case ISD::UMAX: { in ExpandNode()
3180 ISD::CondCode Pred; in ExpandNode()
3183 case ISD::SMAX: Pred = ISD::SETGT; break; in ExpandNode()
3184 case ISD::SMIN: Pred = ISD::SETLT; break; in ExpandNode()
3185 case ISD::UMAX: Pred = ISD::SETUGT; break; in ExpandNode()
3186 case ISD::UMIN: Pred = ISD::SETULT; break; in ExpandNode()
3194 case ISD::FMINNUM: in ExpandNode()
3195 case ISD::FMAXNUM: { in ExpandNode()
3200 case ISD::FSIN: in ExpandNode()
3201 case ISD::FCOS: { in ExpandNode()
3205 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) || in ExpandNode()
3209 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); in ExpandNode()
3210 if (Node->getOpcode() == ISD::FCOS) in ExpandNode()
3216 case ISD::FMAD: in ExpandNode()
3219 case ISD::FP16_TO_FP: in ExpandNode()
3225 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); in ExpandNode()
3227 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res)); in ExpandNode()
3230 case ISD::STRICT_FP16_TO_FP: in ExpandNode()
3236 DAG.getNode(ISD::STRICT_FP16_TO_FP, dl, {MVT::f32, MVT::Other}, in ExpandNode()
3238 Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, in ExpandNode()
3245 case ISD::FP_TO_FP16: in ExpandNode()
3251 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) { in ExpandNode()
3254 SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op, in ExpandNode()
3257 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal)); in ExpandNode()
3261 case ISD::ConstantFP: { in ExpandNode()
3270 case ISD::Constant: { in ExpandNode()
3275 case ISD::FSUB: { in ExpandNode()
3277 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) && in ExpandNode()
3278 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) { in ExpandNode()
3280 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); in ExpandNode()
3281 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags); in ExpandNode()
3286 case ISD::SUB: { in ExpandNode()
3288 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) && in ExpandNode()
3289 TLI.isOperationLegalOrCustom(ISD::XOR, VT) && in ExpandNode()
3292 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT)); in ExpandNode()
3293 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); in ExpandNode()
3296 case ISD::UREM: in ExpandNode()
3297 case ISD::SREM: in ExpandNode()
3301 case ISD::UDIV: in ExpandNode()
3302 case ISD::SDIV: { in ExpandNode()
3303 bool isSigned = Node->getOpcode() == ISD::SDIV; in ExpandNode()
3304 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; in ExpandNode()
3314 case ISD::MULHU: in ExpandNode()
3315 case ISD::MULHS: { in ExpandNode()
3317 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI; in ExpandNode()
3326 case ISD::UMUL_LOHI: in ExpandNode()
3327 case ISD::SMUL_LOHI: { in ExpandNode()
3332 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS; in ExpandNode()
3335 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS)); in ExpandNode()
3347 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]); in ExpandNode()
3348 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]); in ExpandNode()
3352 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); in ExpandNode()
3353 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); in ExpandNode()
3359 case ISD::MUL: { in ExpandNode()
3367 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT); in ExpandNode()
3368 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT); in ExpandNode()
3369 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT); in ExpandNode()
3370 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT); in ExpandNode()
3373 OpToUse = ISD::SMUL_LOHI; in ExpandNode()
3375 OpToUse = ISD::UMUL_LOHI; in ExpandNode()
3377 OpToUse = ISD::SMUL_LOHI; in ExpandNode()
3379 OpToUse = ISD::UMUL_LOHI; in ExpandNode()
3389 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) && in ExpandNode()
3390 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) && in ExpandNode()
3391 TLI.isOperationLegalOrCustom(ISD::SHL, VT) && in ExpandNode()
3392 TLI.isOperationLegalOrCustom(ISD::OR, VT) && in ExpandNode()
3395 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); in ExpandNode()
3396 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi); in ExpandNode()
3400 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); in ExpandNode()
3401 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); in ExpandNode()
3405 case ISD::FSHL: in ExpandNode()
3406 case ISD::FSHR: in ExpandNode()
3410 case ISD::ROTL: in ExpandNode()
3411 case ISD::ROTR: in ExpandNode()
3415 case ISD::SADDSAT: in ExpandNode()
3416 case ISD::UADDSAT: in ExpandNode()
3417 case ISD::SSUBSAT: in ExpandNode()
3418 case ISD::USUBSAT: in ExpandNode()
3421 case ISD::SSHLSAT: in ExpandNode()
3422 case ISD::USHLSAT: in ExpandNode()
3425 case ISD::SMULFIX: in ExpandNode()
3426 case ISD::SMULFIXSAT: in ExpandNode()
3427 case ISD::UMULFIX: in ExpandNode()
3428 case ISD::UMULFIXSAT: in ExpandNode()
3431 case ISD::SDIVFIX: in ExpandNode()
3432 case ISD::SDIVFIXSAT: in ExpandNode()
3433 case ISD::UDIVFIX: in ExpandNode()
3434 case ISD::UDIVFIXSAT: in ExpandNode()
3451 case ISD::ADDCARRY: in ExpandNode()
3452 case ISD::SUBCARRY: { in ExpandNode()
3457 bool IsAdd = Node->getOpcode() == ISD::ADDCARRY; in ExpandNode()
3460 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB; in ExpandNode()
3467 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT; in ExpandNode()
3473 DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One); in ExpandNode()
3482 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ) in ExpandNode()
3483 : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ); in ExpandNode()
3484 Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2, in ExpandNode()
3488 DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2); in ExpandNode()
3494 case ISD::SADDO: in ExpandNode()
3495 case ISD::SSUBO: { in ExpandNode()
3502 case ISD::UADDO: in ExpandNode()
3503 case ISD::USUBO: { in ExpandNode()
3510 case ISD::UMULO: in ExpandNode()
3511 case ISD::SMULO: { in ExpandNode()
3519 case ISD::BUILD_PAIR: { in ExpandNode()
3521 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); in ExpandNode()
3522 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); in ExpandNode()
3524 ISD::SHL, dl, PairTy, Tmp2, in ExpandNode()
3527 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2)); in ExpandNode()
3530 case ISD::SELECT: in ExpandNode()
3534 if (Tmp1.getOpcode() == ISD::SETCC) { in ExpandNode()
3541 Tmp2, Tmp3, ISD::SETNE); in ExpandNode()
3546 case ISD::BR_JT: { in ExpandNode()
3563 ISD::SHL, dl, Index.getValueType(), Index, in ExpandNode()
3566 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index, in ExpandNode()
3568 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(), in ExpandNode()
3573 ISD::SEXTLOAD, dl, PTy, Chain, Addr, in ExpandNode()
3580 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, in ExpandNode()
3588 case ISD::BRCOND: in ExpandNode()
3593 if (Tmp2.getOpcode() == ISD::SETCC && in ExpandNode()
3594 TLI.isOperationLegalOrCustom(ISD::BR_CC, in ExpandNode()
3596 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, Tmp2.getOperand(2), in ExpandNode()
3602 (Tmp2.getOpcode() == ISD::AND && in ExpandNode()
3607 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2, in ExpandNode()
3609 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, in ExpandNode()
3610 DAG.getCondCode(ISD::SETNE), Tmp3, in ExpandNode()
3616 case ISD::SETCC: in ExpandNode()
3617 case ISD::VP_SETCC: in ExpandNode()
3618 case ISD::STRICT_FSETCC: in ExpandNode()
3619 case ISD::STRICT_FSETCCS: { in ExpandNode()
3620 bool IsVP = Node->getOpcode() == ISD::VP_SETCC; in ExpandNode()
3621 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC || in ExpandNode()
3622 Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandNode()
3623 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandNode()
3681 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, in ExpandNode()
3688 case ISD::SELECT_CC: { in ExpandNode()
3697 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get(); in ExpandNode()
3703 assert(!TLI.isOperationExpand(ISD::SELECT, VT) && in ExpandNode()
3707 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags()); in ExpandNode()
3717 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, Tmp1.getValueType()); in ExpandNode()
3726 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC); in ExpandNode()
3751 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), in ExpandNode()
3755 CC = DAG.getCondCode(ISD::SETNE); in ExpandNode()
3756 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
3764 case ISD::BR_CC: { in ExpandNode()
3783 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
3787 Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE); in ExpandNode()
3788 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, in ExpandNode()
3794 case ISD::BUILD_VECTOR: in ExpandNode()
3797 case ISD::SPLAT_VECTOR: in ExpandNode()
3800 case ISD::SRA: in ExpandNode()
3801 case ISD::SRL: in ExpandNode()
3802 case ISD::SHL: { in ExpandNode()
3812 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), in ExpandNode()
3815 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), in ExpandNode()
3825 case ISD::VECREDUCE_FADD: in ExpandNode()
3826 case ISD::VECREDUCE_FMUL: in ExpandNode()
3827 case ISD::VECREDUCE_ADD: in ExpandNode()
3828 case ISD::VECREDUCE_MUL: in ExpandNode()
3829 case ISD::VECREDUCE_AND: in ExpandNode()
3830 case ISD::VECREDUCE_OR: in ExpandNode()
3831 case ISD::VECREDUCE_XOR: in ExpandNode()
3832 case ISD::VECREDUCE_SMAX: in ExpandNode()
3833 case ISD::VECREDUCE_SMIN: in ExpandNode()
3834 case ISD::VECREDUCE_UMAX: in ExpandNode()
3835 case ISD::VECREDUCE_UMIN: in ExpandNode()
3836 case ISD::VECREDUCE_FMAX: in ExpandNode()
3837 case ISD::VECREDUCE_FMIN: in ExpandNode()
3840 case ISD::GLOBAL_OFFSET_TABLE: in ExpandNode()
3841 case ISD::GlobalAddress: in ExpandNode()
3842 case ISD::GlobalTLSAddress: in ExpandNode()
3843 case ISD::ExternalSymbol: in ExpandNode()
3844 case ISD::ConstantPool: in ExpandNode()
3845 case ISD::JumpTable: in ExpandNode()
3846 case ISD::INTRINSIC_W_CHAIN: in ExpandNode()
3847 case ISD::INTRINSIC_WO_CHAIN: in ExpandNode()
3848 case ISD::INTRINSIC_VOID: in ExpandNode()
3868 case ISD::STRICT_FSUB: { in ExpandNode()
3870 ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal) in ExpandNode()
3873 ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal) in ExpandNode()
3878 SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags); in ExpandNode()
3879 SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(), in ExpandNode()
3887 case ISD::STRICT_SINT_TO_FP: in ExpandNode()
3888 case ISD::STRICT_UINT_TO_FP: in ExpandNode()
3889 case ISD::STRICT_LRINT: in ExpandNode()
3890 case ISD::STRICT_LLRINT: in ExpandNode()
3891 case ISD::STRICT_LROUND: in ExpandNode()
3892 case ISD::STRICT_LLROUND: in ExpandNode()
3921 case ISD::ATOMIC_FENCE: { in ConvertNodeToLibcall()
3943 case ISD::ATOMIC_SWAP: in ConvertNodeToLibcall()
3944 case ISD::ATOMIC_LOAD_ADD: in ConvertNodeToLibcall()
3945 case ISD::ATOMIC_LOAD_SUB: in ConvertNodeToLibcall()
3946 case ISD::ATOMIC_LOAD_AND: in ConvertNodeToLibcall()
3947 case ISD::ATOMIC_LOAD_CLR: in ConvertNodeToLibcall()
3948 case ISD::ATOMIC_LOAD_OR: in ConvertNodeToLibcall()
3949 case ISD::ATOMIC_LOAD_XOR: in ConvertNodeToLibcall()
3950 case ISD::ATOMIC_LOAD_NAND: in ConvertNodeToLibcall()
3951 case ISD::ATOMIC_LOAD_MIN: in ConvertNodeToLibcall()
3952 case ISD::ATOMIC_LOAD_MAX: in ConvertNodeToLibcall()
3953 case ISD::ATOMIC_LOAD_UMIN: in ConvertNodeToLibcall()
3954 case ISD::ATOMIC_LOAD_UMAX: in ConvertNodeToLibcall()
3955 case ISD::ATOMIC_CMP_SWAP: { in ConvertNodeToLibcall()
3982 case ISD::TRAP: { in ConvertNodeToLibcall()
3997 case ISD::FMINNUM: in ConvertNodeToLibcall()
3998 case ISD::STRICT_FMINNUM: in ConvertNodeToLibcall()
4003 case ISD::FMAXNUM: in ConvertNodeToLibcall()
4004 case ISD::STRICT_FMAXNUM: in ConvertNodeToLibcall()
4009 case ISD::FSQRT: in ConvertNodeToLibcall()
4010 case ISD::STRICT_FSQRT: in ConvertNodeToLibcall()
4015 case ISD::FCBRT: in ConvertNodeToLibcall()
4020 case ISD::FSIN: in ConvertNodeToLibcall()
4021 case ISD::STRICT_FSIN: in ConvertNodeToLibcall()
4026 case ISD::FCOS: in ConvertNodeToLibcall()
4027 case ISD::STRICT_FCOS: in ConvertNodeToLibcall()
4032 case ISD::FSINCOS: in ConvertNodeToLibcall()
4036 case ISD::FLOG: in ConvertNodeToLibcall()
4037 case ISD::STRICT_FLOG: in ConvertNodeToLibcall()
4041 case ISD::FLOG2: in ConvertNodeToLibcall()
4042 case ISD::STRICT_FLOG2: in ConvertNodeToLibcall()
4046 case ISD::FLOG10: in ConvertNodeToLibcall()
4047 case ISD::STRICT_FLOG10: in ConvertNodeToLibcall()
4051 case ISD::FEXP: in ConvertNodeToLibcall()
4052 case ISD::STRICT_FEXP: in ConvertNodeToLibcall()
4056 case ISD::FEXP2: in ConvertNodeToLibcall()
4057 case ISD::STRICT_FEXP2: in ConvertNodeToLibcall()
4061 case ISD::FTRUNC: in ConvertNodeToLibcall()
4062 case ISD::STRICT_FTRUNC: in ConvertNodeToLibcall()
4067 case ISD::FFLOOR: in ConvertNodeToLibcall()
4068 case ISD::STRICT_FFLOOR: in ConvertNodeToLibcall()
4073 case ISD::FCEIL: in ConvertNodeToLibcall()
4074 case ISD::STRICT_FCEIL: in ConvertNodeToLibcall()
4079 case ISD::FRINT: in ConvertNodeToLibcall()
4080 case ISD::STRICT_FRINT: in ConvertNodeToLibcall()
4085 case ISD::FNEARBYINT: in ConvertNodeToLibcall()
4086 case ISD::STRICT_FNEARBYINT: in ConvertNodeToLibcall()
4093 case ISD::FROUND: in ConvertNodeToLibcall()
4094 case ISD::STRICT_FROUND: in ConvertNodeToLibcall()
4101 case ISD::FROUNDEVEN: in ConvertNodeToLibcall()
4102 case ISD::STRICT_FROUNDEVEN: in ConvertNodeToLibcall()
4109 case ISD::FPOWI: in ConvertNodeToLibcall()
4110 case ISD::STRICT_FPOWI: { in ConvertNodeToLibcall()
4117 DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(Node), in ConvertNodeToLibcall()
4121 DAG.getNode(ISD::STRICT_FPOW, SDLoc(Node), in ConvertNodeToLibcall()
4128 DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), Node->getValueType(0), in ConvertNodeToLibcall()
4130 Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node), in ConvertNodeToLibcall()
4150 case ISD::FPOW: in ConvertNodeToLibcall()
4151 case ISD::STRICT_FPOW: in ConvertNodeToLibcall()
4155 case ISD::LROUND: in ConvertNodeToLibcall()
4156 case ISD::STRICT_LROUND: in ConvertNodeToLibcall()
4162 case ISD::LLROUND: in ConvertNodeToLibcall()
4163 case ISD::STRICT_LLROUND: in ConvertNodeToLibcall()
4169 case ISD::LRINT: in ConvertNodeToLibcall()
4170 case ISD::STRICT_LRINT: in ConvertNodeToLibcall()
4176 case ISD::LLRINT: in ConvertNodeToLibcall()
4177 case ISD::STRICT_LLRINT: in ConvertNodeToLibcall()
4183 case ISD::FDIV: in ConvertNodeToLibcall()
4184 case ISD::STRICT_FDIV: in ConvertNodeToLibcall()
4189 case ISD::FREM: in ConvertNodeToLibcall()
4190 case ISD::STRICT_FREM: in ConvertNodeToLibcall()
4195 case ISD::FMA: in ConvertNodeToLibcall()
4196 case ISD::STRICT_FMA: in ConvertNodeToLibcall()
4201 case ISD::FADD: in ConvertNodeToLibcall()
4202 case ISD::STRICT_FADD: in ConvertNodeToLibcall()
4207 case ISD::FMUL: in ConvertNodeToLibcall()
4208 case ISD::STRICT_FMUL: in ConvertNodeToLibcall()
4213 case ISD::FP16_TO_FP: in ConvertNodeToLibcall()
4218 case ISD::STRICT_FP16_TO_FP: { in ConvertNodeToLibcall()
4229 case ISD::FP_TO_FP16: { in ConvertNodeToLibcall()
4236 case ISD::FP_TO_BF16: { in ConvertNodeToLibcall()
4243 case ISD::STRICT_SINT_TO_FP: in ConvertNodeToLibcall()
4244 case ISD::STRICT_UINT_TO_FP: in ConvertNodeToLibcall()
4245 case ISD::SINT_TO_FP: in ConvertNodeToLibcall()
4246 case ISD::UINT_TO_FP: { in ConvertNodeToLibcall()
4249 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP || in ConvertNodeToLibcall()
4250 Node->getOpcode() == ISD::STRICT_SINT_TO_FP; in ConvertNodeToLibcall()
4273 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, in ConvertNodeToLibcall()
4284 case ISD::FP_TO_SINT: in ConvertNodeToLibcall()
4285 case ISD::FP_TO_UINT: in ConvertNodeToLibcall()
4286 case ISD::STRICT_FP_TO_SINT: in ConvertNodeToLibcall()
4287 case ISD::STRICT_FP_TO_UINT: { in ConvertNodeToLibcall()
4290 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT || in ConvertNodeToLibcall()
4291 Node->getOpcode() == ISD::STRICT_FP_TO_SINT; in ConvertNodeToLibcall()
4320 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first)); in ConvertNodeToLibcall()
4326 case ISD::FP_ROUND: in ConvertNodeToLibcall()
4327 case ISD::STRICT_FP_ROUND: { in ConvertNodeToLibcall()
4351 case ISD::FP_EXTEND: { in ConvertNodeToLibcall()
4358 case ISD::STRICT_FP_EXTEND: in ConvertNodeToLibcall()
4359 case ISD::STRICT_FP_TO_FP16: { in ConvertNodeToLibcall()
4361 Node->getOpcode() == ISD::STRICT_FP_TO_FP16 in ConvertNodeToLibcall()
4375 case ISD::FSUB: in ConvertNodeToLibcall()
4376 case ISD::STRICT_FSUB: in ConvertNodeToLibcall()
4381 case ISD::SREM: in ConvertNodeToLibcall()
4386 case ISD::UREM: in ConvertNodeToLibcall()
4391 case ISD::SDIV: in ConvertNodeToLibcall()
4396 case ISD::UDIV: in ConvertNodeToLibcall()
4401 case ISD::SDIVREM: in ConvertNodeToLibcall()
4402 case ISD::UDIVREM: in ConvertNodeToLibcall()
4406 case ISD::MUL: in ConvertNodeToLibcall()
4411 case ISD::CTLZ_ZERO_UNDEF: in ConvertNodeToLibcall()
4450 if (Node->getOpcode() == ISD::UINT_TO_FP || in PromoteNode()
4451 Node->getOpcode() == ISD::SINT_TO_FP || in PromoteNode()
4452 Node->getOpcode() == ISD::SETCC || in PromoteNode()
4453 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT || in PromoteNode()
4454 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) { in PromoteNode()
4457 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP || in PromoteNode()
4458 Node->getOpcode() == ISD::STRICT_SINT_TO_FP || in PromoteNode()
4459 Node->getOpcode() == ISD::STRICT_FSETCC || in PromoteNode()
4460 Node->getOpcode() == ISD::STRICT_FSETCCS) in PromoteNode()
4462 if (Node->getOpcode() == ISD::BR_CC || in PromoteNode()
4463 Node->getOpcode() == ISD::SELECT_CC) in PromoteNode()
4469 case ISD::CTTZ: in PromoteNode()
4470 case ISD::CTTZ_ZERO_UNDEF: in PromoteNode()
4471 case ISD::CTLZ: in PromoteNode()
4472 case ISD::CTLZ_ZERO_UNDEF: in PromoteNode()
4473 case ISD::CTPOP: in PromoteNode()
4475 if (Node->getOpcode() == ISD::CTTZ || in PromoteNode()
4476 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF) in PromoteNode()
4477 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4479 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4481 if (Node->getOpcode() == ISD::CTTZ) { in PromoteNode()
4487 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1, in PromoteNode()
4493 if (Node->getOpcode() == ISD::CTLZ || in PromoteNode()
4494 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) { in PromoteNode()
4496 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, in PromoteNode()
4500 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); in PromoteNode()
4502 case ISD::BITREVERSE: in PromoteNode()
4503 case ISD::BSWAP: { in PromoteNode()
4505 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4508 ISD::SRL, dl, NVT, Tmp1, in PromoteNode()
4512 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); in PromoteNode()
4515 case ISD::FP_TO_UINT: in PromoteNode()
4516 case ISD::STRICT_FP_TO_UINT: in PromoteNode()
4517 case ISD::FP_TO_SINT: in PromoteNode()
4518 case ISD::STRICT_FP_TO_SINT: in PromoteNode()
4521 case ISD::FP_TO_UINT_SAT: in PromoteNode()
4522 case ISD::FP_TO_SINT_SAT: in PromoteNode()
4525 case ISD::UINT_TO_FP: in PromoteNode()
4526 case ISD::STRICT_UINT_TO_FP: in PromoteNode()
4527 case ISD::SINT_TO_FP: in PromoteNode()
4528 case ISD::STRICT_SINT_TO_FP: in PromoteNode()
4531 case ISD::VAARG: { in PromoteNode()
4537 TruncOp = ISD::BITCAST; in PromoteNode()
4541 TruncOp = ISD::TRUNCATE; in PromoteNode()
4562 case ISD::MUL: in PromoteNode()
4563 case ISD::SDIV: in PromoteNode()
4564 case ISD::SREM: in PromoteNode()
4565 case ISD::UDIV: in PromoteNode()
4566 case ISD::UREM: in PromoteNode()
4567 case ISD::AND: in PromoteNode()
4568 case ISD::OR: in PromoteNode()
4569 case ISD::XOR: { in PromoteNode()
4572 ExtOp = ISD::BITCAST; in PromoteNode()
4573 TruncOp = ISD::BITCAST; in PromoteNode()
4579 ExtOp = ISD::ANY_EXTEND; in PromoteNode()
4581 case ISD::SDIV: in PromoteNode()
4582 case ISD::SREM: in PromoteNode()
4583 ExtOp = ISD::SIGN_EXTEND; in PromoteNode()
4585 case ISD::UDIV: in PromoteNode()
4586 case ISD::UREM: in PromoteNode()
4587 ExtOp = ISD::ZERO_EXTEND; in PromoteNode()
4590 TruncOp = ISD::TRUNCATE; in PromoteNode()
4600 case ISD::UMUL_LOHI: in PromoteNode()
4601 case ISD::SMUL_LOHI: { in PromoteNode()
4603 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND in PromoteNode()
4604 : ISD::SIGN_EXTEND; in PromoteNode()
4607 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2); in PromoteNode()
4612 ISD::SRL, dl, NVT, Tmp1, in PromoteNode()
4614 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); in PromoteNode()
4615 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2)); in PromoteNode()
4618 case ISD::SELECT: { in PromoteNode()
4622 ExtOp = ISD::BITCAST; in PromoteNode()
4623 TruncOp = ISD::BITCAST; in PromoteNode()
4625 ExtOp = ISD::ANY_EXTEND; in PromoteNode()
4626 TruncOp = ISD::TRUNCATE; in PromoteNode()
4628 ExtOp = ISD::FP_EXTEND; in PromoteNode()
4629 TruncOp = ISD::FP_ROUND; in PromoteNode()
4638 if (TruncOp != ISD::FP_ROUND) in PromoteNode()
4646 case ISD::VECTOR_SHUFFLE: { in PromoteNode()
4650 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); in PromoteNode()
4651 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); in PromoteNode()
4655 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1); in PromoteNode()
4659 case ISD::VECTOR_SPLICE: { in PromoteNode()
4660 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4661 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
4662 Tmp3 = DAG.getNode(ISD::VECTOR_SPLICE, dl, NVT, Tmp1, Tmp2, in PromoteNode()
4664 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp3)); in PromoteNode()
4667 case ISD::SELECT_CC: { in PromoteNode()
4669 ISD::CondCode CCCode = cast<CondCodeSDNode>(Cond)->get(); in PromoteNode()
4674 unsigned ExtOp = ISD::FP_EXTEND; in PromoteNode()
4676 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in PromoteNode()
4691 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, NVT, {Tmp1, Tmp2, Tmp3, Tmp4, Cond}, in PromoteNode()
4695 if (ExtOp != ISD::FP_EXTEND) in PromoteNode()
4696 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1); in PromoteNode()
4698 Tmp1 = DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp1, in PromoteNode()
4704 case ISD::SETCC: in PromoteNode()
4705 case ISD::STRICT_FSETCC: in PromoteNode()
4706 case ISD::STRICT_FSETCCS: { in PromoteNode()
4707 unsigned ExtOp = ISD::FP_EXTEND; in PromoteNode()
4709 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); in PromoteNode()
4710 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in PromoteNode()
4729 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1, in PromoteNode()
4733 case ISD::BR_CC: { in PromoteNode()
4734 unsigned ExtOp = ISD::FP_EXTEND; in PromoteNode()
4736 ISD::CondCode CCCode = in PromoteNode()
4738 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in PromoteNode()
4742 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), in PromoteNode()
4747 case ISD::FADD: in PromoteNode()
4748 case ISD::FSUB: in PromoteNode()
4749 case ISD::FMUL: in PromoteNode()
4750 case ISD::FDIV: in PromoteNode()
4751 case ISD::FREM: in PromoteNode()
4752 case ISD::FMINNUM: in PromoteNode()
4753 case ISD::FMAXNUM: in PromoteNode()
4754 case ISD::FPOW: in PromoteNode()
4755 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4756 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
4759 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, in PromoteNode()
4762 case ISD::STRICT_FADD: in PromoteNode()
4763 case ISD::STRICT_FSUB: in PromoteNode()
4764 case ISD::STRICT_FMUL: in PromoteNode()
4765 case ISD::STRICT_FDIV: in PromoteNode()
4766 case ISD::STRICT_FMINNUM: in PromoteNode()
4767 case ISD::STRICT_FMAXNUM: in PromoteNode()
4768 case ISD::STRICT_FREM: in PromoteNode()
4769 case ISD::STRICT_FPOW: in PromoteNode()
4770 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
4772 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
4774 Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1), in PromoteNode()
4778 Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, in PromoteNode()
4783 case ISD::FMA: in PromoteNode()
4784 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4785 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
4786 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2)); in PromoteNode()
4788 DAG.getNode(ISD::FP_ROUND, dl, OVT, in PromoteNode()
4792 case ISD::STRICT_FMA: in PromoteNode()
4793 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
4795 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
4797 Tmp3 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
4799 Tmp4 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1), in PromoteNode()
4803 Tmp4 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, in PromoteNode()
4808 case ISD::FCOPYSIGN: in PromoteNode()
4809 case ISD::FPOWI: { in PromoteNode()
4810 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4819 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN); in PromoteNode()
4820 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, in PromoteNode()
4824 case ISD::STRICT_FPOWI: in PromoteNode()
4825 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
4829 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, in PromoteNode()
4834 case ISD::FFLOOR: in PromoteNode()
4835 case ISD::FCEIL: in PromoteNode()
4836 case ISD::FRINT: in PromoteNode()
4837 case ISD::FNEARBYINT: in PromoteNode()
4838 case ISD::FROUND: in PromoteNode()
4839 case ISD::FROUNDEVEN: in PromoteNode()
4840 case ISD::FTRUNC: in PromoteNode()
4841 case ISD::FNEG: in PromoteNode()
4842 case ISD::FSQRT: in PromoteNode()
4843 case ISD::FSIN: in PromoteNode()
4844 case ISD::FCOS: in PromoteNode()
4845 case ISD::FLOG: in PromoteNode()
4846 case ISD::FLOG2: in PromoteNode()
4847 case ISD::FLOG10: in PromoteNode()
4848 case ISD::FABS: in PromoteNode()
4849 case ISD::FEXP: in PromoteNode()
4850 case ISD::FEXP2: in PromoteNode()
4851 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4853 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, in PromoteNode()
4856 case ISD::STRICT_FFLOOR: in PromoteNode()
4857 case ISD::STRICT_FCEIL: in PromoteNode()
4858 case ISD::STRICT_FRINT: in PromoteNode()
4859 case ISD::STRICT_FNEARBYINT: in PromoteNode()
4860 case ISD::STRICT_FROUND: in PromoteNode()
4861 case ISD::STRICT_FROUNDEVEN: in PromoteNode()
4862 case ISD::STRICT_FTRUNC: in PromoteNode()
4863 case ISD::STRICT_FSQRT: in PromoteNode()
4864 case ISD::STRICT_FSIN: in PromoteNode()
4865 case ISD::STRICT_FCOS: in PromoteNode()
4866 case ISD::STRICT_FLOG: in PromoteNode()
4867 case ISD::STRICT_FLOG2: in PromoteNode()
4868 case ISD::STRICT_FLOG10: in PromoteNode()
4869 case ISD::STRICT_FEXP: in PromoteNode()
4870 case ISD::STRICT_FEXP2: in PromoteNode()
4871 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other}, in PromoteNode()
4875 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other}, in PromoteNode()
4880 case ISD::BUILD_VECTOR: { in PromoteNode()
4899 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op)); in PromoteNode()
4903 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps); in PromoteNode()
4904 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); in PromoteNode()
4908 case ISD::EXTRACT_VECTOR_ELT: { in PromoteNode()
4934 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); in PromoteNode()
4936 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); in PromoteNode()
4941 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); in PromoteNode()
4943 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, in PromoteNode()
4949 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec)); in PromoteNode()
4952 case ISD::INSERT_VECTOR_ELT: { in PromoteNode()
4982 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); in PromoteNode()
4984 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); in PromoteNode()
4985 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); in PromoteNode()
4990 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); in PromoteNode()
4992 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, in PromoteNode()
4995 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT, in PromoteNode()
4999 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec)); in PromoteNode()
5002 case ISD::SCALAR_TO_VECTOR: { in PromoteNode()
5017 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); in PromoteNode()
5025 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts); in PromoteNode()
5026 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); in PromoteNode()
5030 case ISD::ATOMIC_SWAP: { in PromoteNode()
5033 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal()); in PromoteNode()
5040 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT, in PromoteNode()
5044 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic)); in PromoteNode()