Lines Matching refs:Node
111 void LegalizeOp(SDNode *Node);
116 void LegalizeLoadOps(SDNode *Node);
117 void LegalizeStoreOps(SDNode *Node);
136 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
138 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
140 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
145 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, RTLIB::Libcall Call_I8,
149 void ExpandArgFPLibCall(SDNode *Node,
154 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
155 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
161 SDValue ExpandBUILD_VECTOR(SDNode *Node);
162 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
163 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
164 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
170 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
171 SDValue ExpandFABS(SDNode *Node) const;
172 SDValue ExpandFNEG(SDNode *Node) const;
173 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
178 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
184 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
190 bool ExpandNode(SDNode *Node);
191 void ConvertNodeToLibcall(SDNode *Node);
192 void PromoteNode(SDNode *Node);
487 void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { in LegalizeStoreOps() argument
488 StoreSDNode *ST = cast<StoreSDNode>(Node); in LegalizeStoreOps()
491 SDLoc dl(Node); in LegalizeStoreOps()
523 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); in LegalizeStoreOps()
524 if (Res && Res != SDValue(Node, 0)) in LegalizeStoreOps()
525 ReplaceNode(SDValue(Node, 0), Res); in LegalizeStoreOps()
535 ReplaceNode(SDValue(Node, 0), Result); in LegalizeStoreOps()
558 ReplaceNode(SDValue(Node, 0), Result); in LegalizeStoreOps()
615 ReplaceNode(SDValue(Node, 0), Result); in LegalizeStoreOps()
631 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); in LegalizeStoreOps()
632 if (Res && Res != SDValue(Node, 0)) in LegalizeStoreOps()
633 ReplaceNode(SDValue(Node, 0), Res); in LegalizeStoreOps()
658 ReplaceNode(SDValue(Node, 0), Result); in LegalizeStoreOps()
664 void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { in LegalizeLoadOps() argument
665 LoadSDNode *LD = cast<LoadSDNode>(Node); in LegalizeLoadOps()
669 SDLoc dl(Node); in LegalizeLoadOps()
674 MVT VT = Node->getSimpleValueType(0); in LegalizeLoadOps()
675 SDValue RVal = SDValue(Node, 0); in LegalizeLoadOps()
676 SDValue RChain = SDValue(Node, 1); in LegalizeLoadOps()
678 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { in LegalizeLoadOps()
699 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); in LegalizeLoadOps()
709 if (RChain.getNode() != Node) { in LegalizeLoadOps()
710 assert(RVal.getNode() != Node && "Load must be completely replaced"); in LegalizeLoadOps()
711 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal); in LegalizeLoadOps()
712 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain); in LegalizeLoadOps()
717 ReplacedNode(Node); in LegalizeLoadOps()
737 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) == in LegalizeLoadOps()
751 SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), in LegalizeLoadOps()
791 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
798 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
814 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
819 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
826 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
842 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
848 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0), in LegalizeLoadOps()
855 Value = SDValue(Node, 0); in LegalizeLoadOps()
856 Chain = SDValue(Node, 1); in LegalizeLoadOps()
859 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { in LegalizeLoadOps()
876 EVT DestVT = Node->getValueType(0); in LegalizeLoadOps()
892 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load); in LegalizeLoadOps()
926 Node->getValueType(0), in LegalizeLoadOps()
945 if (Chain.getNode() != Node) { in LegalizeLoadOps()
946 assert(Value.getNode() != Node && "Load must be completely replaced"); in LegalizeLoadOps()
947 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value); in LegalizeLoadOps()
948 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); in LegalizeLoadOps()
953 ReplacedNode(Node); in LegalizeLoadOps()
958 void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { in LegalizeOp() argument
959 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG)); in LegalizeOp()
962 if (Node->getOpcode() == ISD::TargetConstant || in LegalizeOp()
963 Node->getOpcode() == ISD::Register) in LegalizeOp()
967 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) in LegalizeOp()
968 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == in LegalizeOp()
972 for (const SDValue &Op : Node->op_values()) in LegalizeOp()
983 switch (Node->getOpcode()) { in LegalizeOp()
988 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); in LegalizeOp()
991 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
992 Node->getValueType(0)); in LegalizeOp()
995 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
996 Node->getValueType(0)); in LegalizeOp()
998 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); in LegalizeOp()
1009 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1010 Node->getOperand(0).getValueType()); in LegalizeOp()
1022 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1023 Node->getOperand(1).getValueType()); in LegalizeOp()
1026 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT(); in LegalizeOp()
1027 Action = TLI.getOperationAction(Node->getOpcode(), InnerType); in LegalizeOp()
1031 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1032 Node->getOperand(2).getValueType()); in LegalizeOp()
1040 unsigned Opc = Node->getOpcode(); in LegalizeOp()
1050 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType(); in LegalizeOp()
1052 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get(); in LegalizeOp()
1055 if (Node->getOpcode() == ISD::SELECT_CC) in LegalizeOp()
1056 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1057 Node->getValueType(0)); in LegalizeOp()
1059 Action = TLI.getOperationAction(Node->getOpcode(), OpVT); in LegalizeOp()
1087 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1099 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1106 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64); in LegalizeOp()
1116 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1120 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), in LegalizeOp()
1121 Node->getOperand(0)); in LegalizeOp()
1122 ReplaceNode(Node, NewVal.getNode()); in LegalizeOp()
1128 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1132 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), in LegalizeOp()
1133 Node->getOperand(0)); in LegalizeOp()
1134 ReplaceNode(Node, NewVal.getNode()); in LegalizeOp()
1147 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1157 unsigned Scale = Node->getConstantOperandVal(2); in LegalizeOp()
1158 Action = TLI.getFixedPointOperationAction(Node->getOpcode(), in LegalizeOp()
1159 Node->getValueType(0), Scale); in LegalizeOp()
1163 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1164 cast<MaskedScatterSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1167 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1168 cast<MaskedStoreSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1172 Node->getOpcode(), in LegalizeOp()
1173 cast<VPScatterSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1177 Node->getOpcode(), in LegalizeOp()
1178 cast<VPStoreSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1182 Node->getOpcode(), in LegalizeOp()
1183 cast<VPStridedStoreSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1200 Node->getOpcode(), Node->getOperand(0).getValueType()); in LegalizeOp()
1220 Node->getOpcode(), Node->getOperand(1).getValueType()); in LegalizeOp()
1223 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { in LegalizeOp()
1224 Action = TLI.getCustomOperationAction(*Node); in LegalizeOp()
1226 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1232 SDNode *NewNode = Node; in LegalizeOp()
1233 switch (Node->getOpcode()) { in LegalizeOp()
1242 SDValue Op0 = Node->getOperand(0); in LegalizeOp()
1243 SDValue Op1 = Node->getOperand(1); in LegalizeOp()
1252 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO); in LegalizeOp()
1263 SDValue Op0 = Node->getOperand(0); in LegalizeOp()
1264 SDValue Op1 = Node->getOperand(1); in LegalizeOp()
1265 SDValue Op2 = Node->getOperand(2); in LegalizeOp()
1272 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO); in LegalizeOp()
1278 if (NewNode != Node) { in LegalizeOp()
1279 ReplaceNode(Node, NewNode); in LegalizeOp()
1280 Node = NewNode; in LegalizeOp()
1290 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { in LegalizeOp()
1291 if (!(Res.getNode() != Node || Res.getResNo() != 0)) in LegalizeOp()
1294 if (Node->getNumValues() == 1) { in LegalizeOp()
1297 assert((Res.getValueType() == Node->getValueType(0) || in LegalizeOp()
1298 Node->getValueType(0) == MVT::Glue) && in LegalizeOp()
1302 ReplaceNode(SDValue(Node, 0), Res); in LegalizeOp()
1307 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) { in LegalizeOp()
1310 assert((Res->getValueType(i) == Node->getValueType(i) || in LegalizeOp()
1311 Node->getValueType(i) == MVT::Glue) && in LegalizeOp()
1316 ReplaceNode(Node, ResultVals.data()); in LegalizeOp()
1322 if (ExpandNode(Node)) in LegalizeOp()
1326 ConvertNodeToLibcall(Node); in LegalizeOp()
1329 PromoteNode(Node); in LegalizeOp()
1334 switch (Node->getOpcode()) { in LegalizeOp()
1338 Node->dump( &DAG); in LegalizeOp()
1347 return LegalizeLoadOps(Node); in LegalizeOp()
1349 return LegalizeStoreOps(Node); in LegalizeOp()
1469 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) { in ExpandVectorBuildThroughStack() argument
1470 assert((Node->getOpcode() == ISD::BUILD_VECTOR || in ExpandVectorBuildThroughStack()
1471 Node->getOpcode() == ISD::CONCAT_VECTORS) && in ExpandVectorBuildThroughStack()
1478 EVT VT = Node->getValueType(0); in ExpandVectorBuildThroughStack()
1479 EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType() in ExpandVectorBuildThroughStack()
1480 : Node->getOperand(0).getValueType(); in ExpandVectorBuildThroughStack()
1481 SDLoc dl(Node); in ExpandVectorBuildThroughStack()
1494 bool Truncate = isa<BuildVectorSDNode>(Node) && in ExpandVectorBuildThroughStack()
1495 MemVT.bitsLT(Node->getOperand(0).getValueType()); in ExpandVectorBuildThroughStack()
1498 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { in ExpandVectorBuildThroughStack()
1500 if (Node->getOperand(i).isUndef()) continue; in ExpandVectorBuildThroughStack()
1508 Node->getOperand(i), Idx, in ExpandVectorBuildThroughStack()
1511 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i), in ExpandVectorBuildThroughStack()
1593 SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const { in ExpandFCOPYSIGN()
1594 SDLoc DL(Node); in ExpandFCOPYSIGN()
1595 SDValue Mag = Node->getOperand(0); in ExpandFCOPYSIGN()
1596 SDValue Sign = Node->getOperand(1); in ExpandFCOPYSIGN()
1651 SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const { in ExpandFNEG()
1653 SDLoc DL(Node); in ExpandFNEG()
1655 getSignAsIntValue(SignAsInt, DL, Node->getOperand(0)); in ExpandFNEG()
1667 SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const { in ExpandFABS()
1668 SDLoc DL(Node); in ExpandFABS()
1669 SDValue Value = Node->getOperand(0); in ExpandFABS()
1688 void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node, in ExpandDYNAMIC_STACKALLOC() argument
1693 SDLoc dl(Node); in ExpandDYNAMIC_STACKALLOC()
1694 EVT VT = Node->getValueType(0); in ExpandDYNAMIC_STACKALLOC()
1695 SDValue Tmp1 = SDValue(Node, 0); in ExpandDYNAMIC_STACKALLOC()
1696 SDValue Tmp2 = SDValue(Node, 1); in ExpandDYNAMIC_STACKALLOC()
1697 SDValue Tmp3 = Node->getOperand(2); in ExpandDYNAMIC_STACKALLOC()
1781 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { in ExpandSCALAR_TO_VECTOR() argument
1782 SDLoc dl(Node); in ExpandSCALAR_TO_VECTOR()
1785 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); in ExpandSCALAR_TO_VECTOR()
1791 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr, in ExpandSCALAR_TO_VECTOR()
1793 Node->getValueType(0).getVectorElementType()); in ExpandSCALAR_TO_VECTOR()
1795 Node->getValueType(0), dl, Ch, StackPtr, in ExpandSCALAR_TO_VECTOR()
1800 ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, in ExpandBVWithShuffles() argument
1802 unsigned NumElems = Node->getNumOperands(); in ExpandBVWithShuffles()
1803 SDLoc dl(Node); in ExpandBVWithShuffles()
1804 EVT VT = Node->getValueType(0); in ExpandBVWithShuffles()
1817 SDValue V = Node->getOperand(i); in ExpandBVWithShuffles()
1895 SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { in ExpandBUILD_VECTOR() argument
1896 unsigned NumElems = Node->getNumOperands(); in ExpandBUILD_VECTOR()
1898 SDLoc dl(Node); in ExpandBUILD_VECTOR()
1899 EVT VT = Node->getValueType(0); in ExpandBUILD_VECTOR()
1900 EVT OpVT = Node->getOperand(0).getValueType(); in ExpandBUILD_VECTOR()
1909 SDValue V = Node->getOperand(i); in ExpandBUILD_VECTOR()
1931 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); in ExpandBUILD_VECTOR()
1938 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { in ExpandBUILD_VECTOR()
1941 dyn_cast<ConstantSDNode>(Node->getOperand(i))) { in ExpandBUILD_VECTOR()
1953 assert(Node->getOperand(i).isUndef()); in ExpandBUILD_VECTOR()
1970 if (Node->getOperand(i).isUndef()) in ExpandBUILD_VECTOR()
1972 DefinedValues.insert(Node->getOperand(i)); in ExpandBUILD_VECTOR()
1979 SDValue V = Node->getOperand(i); in ExpandBUILD_VECTOR()
1984 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) { in ExpandBUILD_VECTOR()
1998 if (ExpandBVWithShuffles(Node, DAG, TLI, Res)) in ExpandBUILD_VECTOR()
2004 return ExpandVectorBuildThroughStack(Node); in ExpandBUILD_VECTOR()
2007 SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) { in ExpandSPLAT_VECTOR() argument
2008 SDLoc DL(Node); in ExpandSPLAT_VECTOR()
2009 EVT VT = Node->getValueType(0); in ExpandSPLAT_VECTOR()
2010 SDValue SplatVal = Node->getOperand(0); in ExpandSPLAT_VECTOR()
2019 SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, in ExpandLibCall() argument
2023 for (const SDValue &Op : Node->op_values()) { in ExpandLibCall()
2026 Entry.Node = Op; in ExpandLibCall()
2035 EVT RetVT = Node->getValueType(0); in ExpandLibCall()
2049 TLI.isInTailCallPosition(DAG, Node, TCChain) && in ExpandLibCall()
2056 CLI.setDebugLoc(SDLoc(Node)) in ExpandLibCall()
2077 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, in ExpandFPLibCall() argument
2083 if (Node->isStrictFPOpcode()) { in ExpandFPLibCall()
2084 EVT RetVT = Node->getValueType(0); in ExpandFPLibCall()
2085 SmallVector<SDValue, 4> Ops(drop_begin(Node->ops())); in ExpandFPLibCall()
2090 SDLoc(Node), in ExpandFPLibCall()
2091 Node->getOperand(0)); in ExpandFPLibCall()
2095 SDValue Tmp = ExpandLibCall(LC, Node, false); in ExpandFPLibCall()
2101 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, in ExpandFPLibCall() argument
2108 RTLIB::Libcall LC = RTLIB::getFPLibCall(Node->getSimpleValueType(0), in ExpandFPLibCall()
2111 ExpandFPLibCall(Node, LC, Results); in ExpandFPLibCall()
2115 SDNode *Node, bool isSigned, RTLIB::Libcall Call_I8, in ExpandIntLibCall() argument
2119 switch (Node->getSimpleValueType(0).SimpleTy) { in ExpandIntLibCall()
2131 return ExpandLibCall(LC, Node, isSigned); in ExpandIntLibCall()
2136 void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node, in ExpandArgFPLibCall() argument
2143 EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType(); in ExpandArgFPLibCall()
2147 ExpandFPLibCall(Node, LC, Results); in ExpandArgFPLibCall()
2152 SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, in ExpandDivRemLibCall() argument
2154 unsigned Opcode = Node->getOpcode(); in ExpandDivRemLibCall()
2158 switch (Node->getSimpleValueType(0).SimpleTy) { in ExpandDivRemLibCall()
2176 EVT RetVT = Node->getValueType(0); in ExpandDivRemLibCall()
2181 for (const SDValue &Op : Node->op_values()) { in ExpandDivRemLibCall()
2184 Entry.Node = Op; in ExpandDivRemLibCall()
2193 Entry.Node = FIPtr; in ExpandDivRemLibCall()
2202 SDLoc dl(Node); in ExpandDivRemLibCall()
2221 static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) { in isSinCosLibcallAvailable() argument
2223 switch (Node->getSimpleValueType(0).SimpleTy) { in isSinCosLibcallAvailable()
2235 static bool useSinCos(SDNode *Node) { in useSinCos() argument
2236 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN in useSinCos()
2239 SDValue Op0 = Node->getOperand(0); in useSinCos()
2241 if (User == Node) in useSinCos()
2252 SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node, in ExpandSinCosLibCall() argument
2255 switch (Node->getSimpleValueType(0).SimpleTy) { in ExpandSinCosLibCall()
2269 EVT RetVT = Node->getValueType(0); in ExpandSinCosLibCall()
2276 Entry.Node = Node->getOperand(0); in ExpandSinCosLibCall()
2284 Entry.Node = SinPtr; in ExpandSinCosLibCall()
2292 Entry.Node = CosPtr; in ExpandSinCosLibCall()
2301 SDLoc dl(Node); in ExpandSinCosLibCall()
2319 SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node, in ExpandLegalINT_TO_FP() argument
2321 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP || in ExpandLegalINT_TO_FP()
2322 Node->getOpcode() == ISD::SINT_TO_FP); in ExpandLegalINT_TO_FP()
2323 EVT DestVT = Node->getValueType(0); in ExpandLegalINT_TO_FP()
2324 SDLoc dl(Node); in ExpandLegalINT_TO_FP()
2325 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0; in ExpandLegalINT_TO_FP()
2326 SDValue Op0 = Node->getOperand(OpNo); in ExpandLegalINT_TO_FP()
2333 TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND in ExpandLegalINT_TO_FP()
2378 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2380 {Node->getOperand(0), Load, Bias}); in ExpandLegalINT_TO_FP()
2432 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2437 { Node->getOperand(0), InCvt }); in ExpandLegalINT_TO_FP()
2445 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept()); in ExpandLegalINT_TO_FP()
2460 Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT)) in ExpandLegalINT_TO_FP()
2471 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2473 { Node->getOperand(0), Op0 }); in ExpandLegalINT_TO_FP()
2522 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2648 SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node, in PromoteLegalFP_TO_INT_SAT() argument
2650 unsigned Opcode = Node->getOpcode(); in PromoteLegalFP_TO_INT_SAT()
2653 EVT NewOutTy = Node->getValueType(0); in PromoteLegalFP_TO_INT_SAT()
2664 SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0), in PromoteLegalFP_TO_INT_SAT()
2665 Node->getOperand(1)); in PromoteLegalFP_TO_INT_SAT()
2666 return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result); in PromoteLegalFP_TO_INT_SAT()
2691 bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { in ExpandNode() argument
2694 SDLoc dl(Node); in ExpandNode()
2697 switch (Node->getOpcode()) { in ExpandNode()
2699 if ((Tmp1 = TLI.expandABS(Node, DAG))) in ExpandNode()
2703 if ((Tmp1 = TLI.expandCTPOP(Node, DAG))) in ExpandNode()
2708 if ((Tmp1 = TLI.expandCTLZ(Node, DAG))) in ExpandNode()
2713 if ((Tmp1 = TLI.expandCTTZ(Node, DAG))) in ExpandNode()
2717 if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG))) in ExpandNode()
2721 if ((Tmp1 = TLI.expandBSWAP(Node, DAG))) in ExpandNode()
2725 Results.push_back(ExpandPARITY(Node->getOperand(0), dl)); in ExpandNode()
2730 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); in ExpandNode()
2733 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl, in ExpandNode()
2748 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0))); in ExpandNode()
2749 Results.push_back(Node->getOperand(0)); in ExpandNode()
2758 Results.push_back(Node->getOperand(0)); in ExpandNode()
2763 Results.append(Node->getNumValues() - 1, in ExpandNode()
2764 DAG.getConstant(0, dl, Node->getValueType(0))); in ExpandNode()
2765 Results.push_back(Node->getOperand(0)); in ExpandNode()
2771 Results.push_back(Node->getOperand(0)); in ExpandNode()
2775 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0)); in ExpandNode()
2776 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); in ExpandNode()
2778 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, in ExpandNode()
2779 Node->getOperand(0), Node->getOperand(1), Zero, Zero, in ExpandNode()
2780 cast<AtomicSDNode>(Node)->getMemOperand()); in ExpandNode()
2788 cast<AtomicSDNode>(Node)->getMemoryVT(), in ExpandNode()
2789 Node->getOperand(0), in ExpandNode()
2790 Node->getOperand(1), Node->getOperand(2), in ExpandNode()
2791 cast<AtomicSDNode>(Node)->getMemOperand()); in ExpandNode()
2799 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); in ExpandNode()
2801 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, in ExpandNode()
2802 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), in ExpandNode()
2803 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand()); in ExpandNode()
2807 SDValue RHS = Node->getOperand(1); in ExpandNode()
2809 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT(); in ExpandNode()
2810 EVT OuterType = Node->getValueType(0); in ExpandNode()
2816 Node->getOperand(2), DAG.getValueType(AtomicType)); in ExpandNode()
2822 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); in ExpandNode()
2827 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); in ExpandNode()
2834 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ); in ExpandNode()
2842 ExpandDYNAMIC_STACKALLOC(Node, Results); in ExpandNode()
2845 for (unsigned i = 0; i < Node->getNumValues(); i++) in ExpandNode()
2846 Results.push_back(Node->getOperand(i)); in ExpandNode()
2849 EVT VT = Node->getValueType(0); in ExpandNode()
2865 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
2866 Node->getValueType(0)) in ExpandNode()
2871 if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0), in ExpandNode()
2872 Node->getValueType(0), dl, in ExpandNode()
2873 Node->getOperand(0)))) { in ExpandNode()
2874 ReplaceNode(Node, Tmp1.getNode()); in ExpandNode()
2881 if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), in ExpandNode()
2882 Node->getValueType(0), dl))) in ExpandNode()
2892 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
2893 Node->getValueType(0)) in ExpandNode()
2899 Node->getOperand(1), Node->getOperand(1).getValueType(), in ExpandNode()
2900 Node->getValueType(0), dl, Node->getOperand(0)))) { in ExpandNode()
2901 ReplaceNode(Node, Tmp1.getNode()); in ExpandNode()
2907 if ((Tmp1 = EmitStackConvert(Node->getOperand(0), in ExpandNode()
2908 Node->getOperand(0).getValueType(), in ExpandNode()
2909 Node->getValueType(0), dl))) in ExpandNode()
2914 SDValue Op = DAG.getNode(ISD::BITCAST, dl, MVT::i16, Node->getOperand(0)); in ExpandNode()
2925 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); in ExpandNode()
2926 EVT VT = Node->getValueType(0); in ExpandNode()
2937 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One); in ExpandNode()
2950 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), in ExpandNode()
2951 Node->getOperand(0), ShiftCst); in ExpandNode()
2952 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); in ExpandNode()
2958 if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) { in ExpandNode()
2960 if (Node->isStrictFPOpcode()) in ExpandNode()
2967 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) { in ExpandNode()
2969 if (Node->isStrictFPOpcode()) in ExpandNode()
2974 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) in ExpandNode()
2978 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) { in ExpandNode()
2979 ReplaceNode(Node, Tmp1.getNode()); in ExpandNode()
2985 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) in ExpandNode()
2989 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) { in ExpandNode()
2991 DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2); in ExpandNode()
2993 ReplaceNodeWithValue(SDValue(Node, 0), Tmp1); in ExpandNode()
3000 Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG)); in ExpandNode()
3003 Results.push_back(DAG.expandVAArg(Node)); in ExpandNode()
3007 Results.push_back(DAG.expandVACopy(Node)); in ExpandNode()
3010 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1) in ExpandNode()
3012 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), in ExpandNode()
3013 Node->getOperand(0)); in ExpandNode()
3015 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0)); in ExpandNode()
3019 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0))); in ExpandNode()
3022 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0))); in ExpandNode()
3025 Results.push_back(ExpandVectorBuildThroughStack(Node)); in ExpandNode()
3028 Results.push_back(ExpandSCALAR_TO_VECTOR(Node)); in ExpandNode()
3031 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0), in ExpandNode()
3032 Node->getOperand(1), in ExpandNode()
3033 Node->getOperand(2), dl)); in ExpandNode()
3037 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); in ExpandNode()
3039 EVT VT = Node->getValueType(0); in ExpandNode()
3041 SDValue Op0 = Node->getOperand(0); in ExpandNode()
3042 SDValue Op1 = Node->getOperand(1); in ExpandNode()
3105 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3110 Results.push_back(TLI.expandVectorSplice(Node, DAG)); in ExpandNode()
3114 EVT OpTy = Node->getOperand(0).getValueType(); in ExpandNode()
3115 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) { in ExpandNode()
3117 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), in ExpandNode()
3120 Node->getOperand(0).getValueType(), in ExpandNode()
3122 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3125 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), in ExpandNode()
3126 Node->getOperand(0)); in ExpandNode()
3135 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP, in ExpandNode()
3136 Node->getValueType(0))); in ExpandNode()
3139 Results.push_back(DAG.getUNDEF(Node->getValueType(0))); in ExpandNode()
3140 Results.push_back(Node->getOperand(0)); in ExpandNode()
3147 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP, in ExpandNode()
3148 Node->getOperand(1))); in ExpandNode()
3150 Results.push_back(Node->getOperand(0)); in ExpandNode()
3154 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); in ExpandNode()
3158 Results.push_back(ExpandFCOPYSIGN(Node)); in ExpandNode()
3161 Results.push_back(ExpandFNEG(Node)); in ExpandNode()
3164 Results.push_back(ExpandFABS(Node)); in ExpandNode()
3167 auto CNode = cast<ConstantSDNode>(Node->getOperand(1)); in ExpandNode()
3170 TLI.expandIS_FPCLASS(Node->getValueType(0), Node->getOperand(0), in ExpandNode()
3171 Test, Node->getFlags(), SDLoc(Node), DAG)) in ExpandNode()
3181 switch (Node->getOpcode()) { in ExpandNode()
3188 Tmp1 = Node->getOperand(0); in ExpandNode()
3189 Tmp2 = Node->getOperand(1); in ExpandNode()
3196 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) in ExpandNode()
3202 EVT VT = Node->getValueType(0); in ExpandNode()
3206 isSinCosLibcallAvailable(Node, TLI)) in ExpandNode()
3207 && useSinCos(Node)) { in ExpandNode()
3209 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); in ExpandNode()
3210 if (Node->getOpcode() == ISD::FCOS) in ExpandNode()
3220 if (Node->getValueType(0) != MVT::f32) { in ExpandNode()
3225 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); in ExpandNode()
3227 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res)); in ExpandNode()
3231 if (Node->getValueType(0) != MVT::f32) { in ExpandNode()
3237 {Node->getOperand(0), Node->getOperand(1)}); in ExpandNode()
3239 {Node->getValueType(0), MVT::Other}, in ExpandNode()
3248 SDValue Op = Node->getOperand(0); in ExpandNode()
3257 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal)); in ExpandNode()
3262 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); in ExpandNode()
3265 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0), in ExpandNode()
3271 ConstantSDNode *CP = cast<ConstantSDNode>(Node); in ExpandNode()
3276 EVT VT = Node->getValueType(0); in ExpandNode()
3279 const SDNodeFlags Flags = Node->getFlags(); in ExpandNode()
3280 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); in ExpandNode()
3281 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags); in ExpandNode()
3287 EVT VT = Node->getValueType(0); in ExpandNode()
3291 Tmp1 = DAG.getNOT(dl, Node->getOperand(1), VT); in ExpandNode()
3293 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); in ExpandNode()
3298 if (TLI.expandREM(Node, Tmp1, DAG)) in ExpandNode()
3303 bool isSigned = Node->getOpcode() == ISD::SDIV; in ExpandNode()
3305 EVT VT = Node->getValueType(0); in ExpandNode()
3308 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), in ExpandNode()
3309 Node->getOperand(1)); in ExpandNode()
3317 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI; in ExpandNode()
3318 EVT VT = Node->getValueType(0); in ExpandNode()
3321 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), in ExpandNode()
3322 Node->getOperand(1)); in ExpandNode()
3328 SDValue LHS = Node->getOperand(0); in ExpandNode()
3329 SDValue RHS = Node->getOperand(1); in ExpandNode()
3332 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS; in ExpandNode()
3343 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves, in ExpandNode()
3360 EVT VT = Node->getValueType(0); in ExpandNode()
3382 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0), in ExpandNode()
3383 Node->getOperand(1))); in ExpandNode()
3393 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG, in ExpandNode()
3407 if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG)) in ExpandNode()
3412 if (SDValue Expanded = TLI.expandROT(Node, true /*AllowVectorOps*/, DAG)) in ExpandNode()
3419 Results.push_back(TLI.expandAddSubSat(Node, DAG)); in ExpandNode()
3423 Results.push_back(TLI.expandShlSat(Node, DAG)); in ExpandNode()
3429 Results.push_back(TLI.expandFixedPointMul(Node, DAG)); in ExpandNode()
3435 if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node), in ExpandNode()
3436 Node->getOperand(0), in ExpandNode()
3437 Node->getOperand(1), in ExpandNode()
3438 Node->getConstantOperandVal(2), in ExpandNode()
3453 SDValue LHS = Node->getOperand(0); in ExpandNode()
3454 SDValue RHS = Node->getOperand(1); in ExpandNode()
3455 SDValue Carry = Node->getOperand(2); in ExpandNode()
3457 bool IsAdd = Node->getOpcode() == ISD::ADDCARRY; in ExpandNode()
3465 EVT CarryType = Node->getValueType(1); in ExpandNode()
3466 EVT SetCCType = getSetCCResultType(Node->getValueType(0)); in ExpandNode()
3497 TLI.expandSADDSUBO(Node, Result, Overflow, DAG); in ExpandNode()
3505 TLI.expandUADDSUBO(Node, Result, Overflow, DAG); in ExpandNode()
3513 if (TLI.expandMULO(Node, Result, Overflow, DAG)) { in ExpandNode()
3520 EVT PairTy = Node->getValueType(0); in ExpandNode()
3521 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); in ExpandNode()
3522 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); in ExpandNode()
3531 Tmp1 = Node->getOperand(0); in ExpandNode()
3532 Tmp2 = Node->getOperand(1); in ExpandNode()
3533 Tmp3 = Node->getOperand(2); in ExpandNode()
3543 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
3547 SDValue Chain = Node->getOperand(0); in ExpandNode()
3548 SDValue Table = Node->getOperand(1); in ExpandNode()
3549 SDValue Index = Node->getOperand(2); in ExpandNode()
3591 Tmp1 = Node->getOperand(0); in ExpandNode()
3592 Tmp2 = Node->getOperand(1); in ExpandNode()
3598 Node->getOperand(2)); in ExpandNode()
3612 Node->getOperand(2)); in ExpandNode()
3620 bool IsVP = Node->getOpcode() == ISD::VP_SETCC; in ExpandNode()
3621 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC || in ExpandNode()
3622 Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandNode()
3623 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandNode()
3624 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ExpandNode()
3626 Tmp1 = Node->getOperand(0 + Offset); in ExpandNode()
3627 Tmp2 = Node->getOperand(1 + Offset); in ExpandNode()
3628 Tmp3 = Node->getOperand(2 + Offset); in ExpandNode()
3631 Mask = Node->getOperand(3 + Offset); in ExpandNode()
3632 EVL = Node->getOperand(4 + Offset); in ExpandNode()
3635 DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl, in ExpandNode()
3643 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(), in ExpandNode()
3644 {Chain, Tmp1, Tmp2, Tmp3}, Node->getFlags()); in ExpandNode()
3647 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), in ExpandNode()
3648 {Tmp1, Tmp2, Tmp3, Mask, EVL}, Node->getFlags()); in ExpandNode()
3650 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Tmp1, in ExpandNode()
3651 Tmp2, Tmp3, Node->getFlags()); in ExpandNode()
3679 EVT VT = Node->getValueType(0); in ExpandNode()
3684 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
3690 Tmp1 = Node->getOperand(0); // LHS in ExpandNode()
3691 Tmp2 = Node->getOperand(1); // RHS in ExpandNode()
3692 Tmp3 = Node->getOperand(2); // True in ExpandNode()
3693 Tmp4 = Node->getOperand(3); // False in ExpandNode()
3694 EVT VT = Node->getValueType(0); in ExpandNode()
3696 SDValue CC = Node->getOperand(4); in ExpandNode()
3707 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags()); in ExpandNode()
3722 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
3732 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
3751 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), in ExpandNode()
3756 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
3759 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
3767 Tmp1 = Node->getOperand(0); // Chain in ExpandNode()
3768 Tmp2 = Node->getOperand(2); // LHS in ExpandNode()
3769 Tmp3 = Node->getOperand(3); // RHS in ExpandNode()
3770 Tmp4 = Node->getOperand(1); // CC in ExpandNode()
3783 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
3784 Tmp4, Tmp2, Tmp3, Node->getOperand(4)); in ExpandNode()
3788 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, in ExpandNode()
3789 Tmp2, Tmp3, Node->getOperand(4)); in ExpandNode()
3795 Results.push_back(ExpandBUILD_VECTOR(Node)); in ExpandNode()
3798 Results.push_back(ExpandSPLAT_VECTOR(Node)); in ExpandNode()
3804 EVT VT = Node->getValueType(0); in ExpandNode()
3813 Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl)); in ExpandNode()
3816 Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl)); in ExpandNode()
3817 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, in ExpandNode()
3821 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars); in ExpandNode()
3838 Results.push_back(TLI.expandVecReduce(Node, DAG)); in ExpandNode()
3855 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) { in ExpandNode()
3861 switch (Node->getOpcode()) { in ExpandNode()
3863 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
3864 Node->getValueType(0)) in ExpandNode()
3870 ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal) in ExpandNode()
3873 ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal) in ExpandNode()
3876 EVT VT = Node->getValueType(0); in ExpandNode()
3877 const SDNodeFlags Flags = Node->getFlags(); in ExpandNode()
3878 SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags); in ExpandNode()
3879 SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(), in ExpandNode()
3880 {Node->getOperand(0), Node->getOperand(1), Neg}, in ExpandNode()
3895 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
3896 Node->getOperand(1).getValueType()) in ExpandNode()
3910 ReplaceNode(Node, Results.data()); in ExpandNode()
3914 void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) { in ConvertNodeToLibcall() argument
3917 SDLoc dl(Node); in ConvertNodeToLibcall()
3919 unsigned Opc = Node->getOpcode(); in ConvertNodeToLibcall()
3928 .setChain(Node->getOperand(0)) in ConvertNodeToLibcall()
3956 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); in ConvertNodeToLibcall()
3957 AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering(); in ConvertNodeToLibcall()
3959 EVT RetVT = Node->getValueType(0); in ConvertNodeToLibcall()
3964 Ops.append(Node->op_begin() + 2, Node->op_end()); in ConvertNodeToLibcall()
3965 Ops.push_back(Node->getOperand(1)); in ConvertNodeToLibcall()
3972 Ops.append(Node->op_begin() + 1, Node->op_end()); in ConvertNodeToLibcall()
3976 SDLoc(Node), in ConvertNodeToLibcall()
3977 Node->getOperand(0)); in ConvertNodeToLibcall()
3987 .setChain(Node->getOperand(0)) in ConvertNodeToLibcall()
3999 ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64, in ConvertNodeToLibcall()
4005 ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64, in ConvertNodeToLibcall()
4011 ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64, in ConvertNodeToLibcall()
4016 ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64, in ConvertNodeToLibcall()
4022 ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64, in ConvertNodeToLibcall()
4028 ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64, in ConvertNodeToLibcall()
4034 ExpandSinCosLibCall(Node, Results); in ConvertNodeToLibcall()
4038 ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80, in ConvertNodeToLibcall()
4043 ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80, in ConvertNodeToLibcall()
4048 ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80, in ConvertNodeToLibcall()
4053 ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80, in ConvertNodeToLibcall()
4058 ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80, in ConvertNodeToLibcall()
4063 ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, in ConvertNodeToLibcall()
4069 ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, in ConvertNodeToLibcall()
4075 ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64, in ConvertNodeToLibcall()
4081 ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64, in ConvertNodeToLibcall()
4087 ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32, in ConvertNodeToLibcall()
4095 ExpandFPLibCall(Node, RTLIB::ROUND_F32, in ConvertNodeToLibcall()
4103 ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32, in ConvertNodeToLibcall()
4111 RTLIB::Libcall LC = RTLIB::getPOWI(Node->getSimpleValueType(0)); in ConvertNodeToLibcall()
4115 if (Node->isStrictFPOpcode()) { in ConvertNodeToLibcall()
4117 DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(Node), in ConvertNodeToLibcall()
4118 {Node->getValueType(0), Node->getValueType(1)}, in ConvertNodeToLibcall()
4119 {Node->getOperand(0), Node->getOperand(2)}); in ConvertNodeToLibcall()
4121 DAG.getNode(ISD::STRICT_FPOW, SDLoc(Node), in ConvertNodeToLibcall()
4122 {Node->getValueType(0), Node->getValueType(1)}, in ConvertNodeToLibcall()
4123 {Exponent.getValue(1), Node->getOperand(1), Exponent}); in ConvertNodeToLibcall()
4128 DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), Node->getValueType(0), in ConvertNodeToLibcall()
4129 Node->getOperand(1)); in ConvertNodeToLibcall()
4130 Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node), in ConvertNodeToLibcall()
4131 Node->getValueType(0), in ConvertNodeToLibcall()
4132 Node->getOperand(0), Exponent)); in ConvertNodeToLibcall()
4136 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0; in ConvertNodeToLibcall()
4139 Node->getOperand(1 + Offset).getValueType().getSizeInBits(); in ConvertNodeToLibcall()
4144 Results.push_back(DAG.getUNDEF(Node->getValueType(0))); in ConvertNodeToLibcall()
4147 ExpandFPLibCall(Node, LC, Results); in ConvertNodeToLibcall()
4152 ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80, in ConvertNodeToLibcall()
4157 ExpandArgFPLibCall(Node, RTLIB::LROUND_F32, in ConvertNodeToLibcall()
4164 ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32, in ConvertNodeToLibcall()
4171 ExpandArgFPLibCall(Node, RTLIB::LRINT_F32, in ConvertNodeToLibcall()
4178 ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32, in ConvertNodeToLibcall()
4185 ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64, in ConvertNodeToLibcall()
4191 ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, in ConvertNodeToLibcall()
4197 ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64, in ConvertNodeToLibcall()
4203 ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64, in ConvertNodeToLibcall()
4209 ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64, in ConvertNodeToLibcall()
4214 if (Node->getValueType(0) == MVT::f32) { in ConvertNodeToLibcall()
4215 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false)); in ConvertNodeToLibcall()
4219 if (Node->getValueType(0) == MVT::f32) { in ConvertNodeToLibcall()
4222 DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions, in ConvertNodeToLibcall()
4223 SDLoc(Node), Node->getOperand(0)); in ConvertNodeToLibcall()
4231 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16); in ConvertNodeToLibcall()
4233 Results.push_back(ExpandLibCall(LC, Node, false)); in ConvertNodeToLibcall()
4238 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::bf16); in ConvertNodeToLibcall()
4240 Results.push_back(ExpandLibCall(LC, Node, false)); in ConvertNodeToLibcall()
4248 bool IsStrict = Node->isStrictFPOpcode(); in ConvertNodeToLibcall()
4249 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP || in ConvertNodeToLibcall()
4250 Node->getOpcode() == ISD::STRICT_SINT_TO_FP; in ConvertNodeToLibcall()
4251 EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType(); in ConvertNodeToLibcall()
4252 EVT RVT = Node->getValueType(0); in ConvertNodeToLibcall()
4254 SDLoc dl(Node); in ConvertNodeToLibcall()
4271 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ConvertNodeToLibcall()
4274 NVT, Node->getOperand(IsStrict ? 1 : 0)); in ConvertNodeToLibcall()
4289 bool IsStrict = Node->isStrictFPOpcode(); in ConvertNodeToLibcall()
4290 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT || in ConvertNodeToLibcall()
4291 Node->getOpcode() == ISD::STRICT_FP_TO_SINT; in ConvertNodeToLibcall()
4293 SDValue Op = Node->getOperand(IsStrict ? 1 : 0); in ConvertNodeToLibcall()
4295 EVT RVT = Node->getValueType(0); in ConvertNodeToLibcall()
4297 SDLoc dl(Node); in ConvertNodeToLibcall()
4314 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ConvertNodeToLibcall()
4333 bool IsStrict = Node->isStrictFPOpcode(); in ConvertNodeToLibcall()
4334 SDValue Op = Node->getOperand(IsStrict ? 1 : 0); in ConvertNodeToLibcall()
4335 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ConvertNodeToLibcall()
4336 EVT VT = Node->getValueType(0); in ConvertNodeToLibcall()
4337 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() && in ConvertNodeToLibcall()
4345 TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, SDLoc(Node), Chain); in ConvertNodeToLibcall()
4353 ExpandLibCall(RTLIB::getFPEXT(Node->getOperand(0).getValueType(), in ConvertNodeToLibcall()
4354 Node->getValueType(0)), in ConvertNodeToLibcall()
4355 Node, false)); in ConvertNodeToLibcall()
4361 Node->getOpcode() == ISD::STRICT_FP_TO_FP16 in ConvertNodeToLibcall()
4362 ? RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16) in ConvertNodeToLibcall()
4363 : RTLIB::getFPEXT(Node->getOperand(1).getValueType(), in ConvertNodeToLibcall()
4364 Node->getValueType(0)); in ConvertNodeToLibcall()
4369 TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1), in ConvertNodeToLibcall()
4370 CallOptions, SDLoc(Node), Node->getOperand(0)); in ConvertNodeToLibcall()
4377 ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64, in ConvertNodeToLibcall()
4383 Node, true, RTLIB::SREM_I8, RTLIB::SREM_I16, RTLIB::SREM_I32, in ConvertNodeToLibcall()
4388 Node, false, RTLIB::UREM_I8, RTLIB::UREM_I16, RTLIB::UREM_I32, in ConvertNodeToLibcall()
4393 Node, true, RTLIB::SDIV_I8, RTLIB::SDIV_I16, RTLIB::SDIV_I32, in ConvertNodeToLibcall()
4398 Node, false, RTLIB::UDIV_I8, RTLIB::UDIV_I16, RTLIB::UDIV_I32, in ConvertNodeToLibcall()
4404 ExpandDivRemLibCall(Node, Results); in ConvertNodeToLibcall()
4408 Node, false, RTLIB::MUL_I8, RTLIB::MUL_I16, RTLIB::MUL_I32, in ConvertNodeToLibcall()
4412 switch (Node->getSimpleValueType(0).SimpleTy) { in ConvertNodeToLibcall()
4416 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false)); in ConvertNodeToLibcall()
4419 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false)); in ConvertNodeToLibcall()
4422 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false)); in ConvertNodeToLibcall()
4431 ReplaceNode(Node, Results.data()); in ConvertNodeToLibcall()
4446 void SelectionDAGLegalize::PromoteNode(SDNode *Node) { in PromoteNode() argument
4449 MVT OVT = Node->getSimpleValueType(0); in PromoteNode()
4450 if (Node->getOpcode() == ISD::UINT_TO_FP || in PromoteNode()
4451 Node->getOpcode() == ISD::SINT_TO_FP || in PromoteNode()
4452 Node->getOpcode() == ISD::SETCC || in PromoteNode()
4453 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT || in PromoteNode()
4454 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) { in PromoteNode()
4455 OVT = Node->getOperand(0).getSimpleValueType(); in PromoteNode()
4457 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP || in PromoteNode()
4458 Node->getOpcode() == ISD::STRICT_SINT_TO_FP || in PromoteNode()
4459 Node->getOpcode() == ISD::STRICT_FSETCC || in PromoteNode()
4460 Node->getOpcode() == ISD::STRICT_FSETCCS) in PromoteNode()
4461 OVT = Node->getOperand(1).getSimpleValueType(); in PromoteNode()
4462 if (Node->getOpcode() == ISD::BR_CC || in PromoteNode()
4463 Node->getOpcode() == ISD::SELECT_CC) in PromoteNode()
4464 OVT = Node->getOperand(2).getSimpleValueType(); in PromoteNode()
4465 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); in PromoteNode()
4466 SDLoc dl(Node); in PromoteNode()
4468 switch (Node->getOpcode()) { in PromoteNode()
4475 if (Node->getOpcode() == ISD::CTTZ || in PromoteNode()
4476 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF) in PromoteNode()
4477 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4479 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4481 if (Node->getOpcode() == ISD::CTTZ) { in PromoteNode()
4492 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
4493 if (Node->getOpcode() == ISD::CTLZ || in PromoteNode()
4494 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) { in PromoteNode()
4505 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4506 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
4519 PromoteLegalFP_TO_INT(Node, dl, Results); in PromoteNode()
4523 Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl)); in PromoteNode()
4529 PromoteLegalINT_TO_FP(Node, dl, Results); in PromoteNode()
4532 SDValue Chain = Node->getOperand(0); // Get the chain. in PromoteNode()
4533 SDValue Ptr = Node->getOperand(1); // Get the pointer. in PromoteNode()
4545 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2), in PromoteNode()
4546 Node->getConstantOperandVal(3)); in PromoteNode()
4553 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2); in PromoteNode()
4554 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); in PromoteNode()
4559 ReplacedNode(Node); in PromoteNode()
4577 switch (Node->getOpcode()) { in PromoteNode()
4593 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
4594 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
4596 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); in PromoteNode()
4603 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND in PromoteNode()
4605 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
4606 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
4620 if (Node->getValueType(0).isVector() || in PromoteNode()
4621 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) { in PromoteNode()
4624 } else if (Node->getValueType(0).isInteger()) { in PromoteNode()
4631 Tmp1 = Node->getOperand(0); in PromoteNode()
4633 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
4634 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); in PromoteNode()
4637 Tmp1->setFlags(Node->getFlags()); in PromoteNode()
4639 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); in PromoteNode()
4641 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1, in PromoteNode()
4647 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); in PromoteNode()
4650 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); in PromoteNode()
4651 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); in PromoteNode()
4660 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4661 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
4663 Node->getOperand(2)); in PromoteNode()
4668 SDValue Cond = Node->getOperand(4); in PromoteNode()
4671 MVT CVT = Node->getSimpleValueType(0); in PromoteNode()
4681 Tmp1 = Node->getOperand(0); in PromoteNode()
4682 Tmp2 = Node->getOperand(1); in PromoteNode()
4684 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
4685 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
4688 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); in PromoteNode()
4689 Tmp4 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); in PromoteNode()
4692 Node->getFlags()); in PromoteNode()
4709 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); in PromoteNode()
4712 if (Node->isStrictFPOpcode()) { in PromoteNode()
4713 SDValue InChain = Node->getOperand(0); in PromoteNode()
4715 DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT); in PromoteNode()
4717 DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT); in PromoteNode()
4720 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); in PromoteNode()
4721 Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs, in PromoteNode()
4722 {OutChain, Tmp1, Tmp2, Node->getOperand(3)}, in PromoteNode()
4723 Node->getFlags())); in PromoteNode()
4727 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
4728 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
4729 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1, in PromoteNode()
4730 Tmp2, Node->getOperand(2), Node->getFlags())); in PromoteNode()
4737 cast<CondCodeSDNode>(Node->getOperand(1))->get(); in PromoteNode()
4740 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); in PromoteNode()
4741 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); in PromoteNode()
4742 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), in PromoteNode()
4743 Node->getOperand(0), Node->getOperand(1), in PromoteNode()
4744 Tmp1, Tmp2, Node->getOperand(4))); in PromoteNode()
4755 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4756 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
4757 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, in PromoteNode()
4758 Node->getFlags()); in PromoteNode()
4771 {Node->getOperand(0), Node->getOperand(1)}); in PromoteNode()
4773 {Node->getOperand(0), Node->getOperand(2)}); in PromoteNode()
4776 Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
4784 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4785 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
4786 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2)); in PromoteNode()
4789 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3), in PromoteNode()
4794 {Node->getOperand(0), Node->getOperand(1)}); in PromoteNode()
4796 {Node->getOperand(0), Node->getOperand(2)}); in PromoteNode()
4798 {Node->getOperand(0), Node->getOperand(3)}); in PromoteNode()
4801 Tmp4 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
4810 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4811 Tmp2 = Node->getOperand(1); in PromoteNode()
4812 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); in PromoteNode()
4819 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN); in PromoteNode()
4826 {Node->getOperand(0), Node->getOperand(1)}); in PromoteNode()
4827 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
4828 {Tmp1.getValue(1), Tmp1, Node->getOperand(2)}); in PromoteNode()
4851 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4852 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
4872 {Node->getOperand(0), Node->getOperand(1)}); in PromoteNode()
4873 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
4897 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) { in PromoteNode()
4898 SDValue Op = Node->getOperand(I); in PromoteNode()
4902 SDLoc SL(Node); in PromoteNode()
4930 SDValue Idx = Node->getOperand(1); in PromoteNode()
4932 SDLoc SL(Node); in PromoteNode()
4936 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); in PromoteNode()
4976 SDValue Val = Node->getOperand(1); in PromoteNode()
4977 SDValue Idx = Node->getOperand(2); in PromoteNode()
4979 SDLoc SL(Node); in PromoteNode()
4984 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); in PromoteNode()
5014 SDValue Val = Node->getOperand(0); in PromoteNode()
5015 SDLoc SL(Node); in PromoteNode()
5031 AtomicSDNode *AM = cast<AtomicSDNode>(Node); in PromoteNode()
5032 SDLoc SL(Node); in PromoteNode()
5053 ReplaceNode(Node, Results.data()); in PromoteNode()