Lines Matching refs:Node
112 void LegalizeOp(SDNode *Node);
117 void LegalizeLoadOps(SDNode *Node);
118 void LegalizeStoreOps(SDNode *Node);
137 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
139 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
141 void ExpandFrexpLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
142 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
144 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
149 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
155 void ExpandArgFPLibCall(SDNode *Node,
160 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
161 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
167 SDValue ExpandBUILD_VECTOR(SDNode *Node);
168 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
169 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
170 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
176 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
177 SDValue ExpandFABS(SDNode *Node) const;
178 SDValue ExpandFNEG(SDNode *Node) const;
179 SDValue expandLdexp(SDNode *Node) const;
180 SDValue expandFrexp(SDNode *Node) const;
182 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
187 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
193 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
199 bool ExpandNode(SDNode *Node);
200 void ConvertNodeToLibcall(SDNode *Node);
201 void PromoteNode(SDNode *Node);
498 void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { in LegalizeStoreOps() argument
499 StoreSDNode *ST = cast<StoreSDNode>(Node); in LegalizeStoreOps()
502 SDLoc dl(Node); in LegalizeStoreOps()
534 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); in LegalizeStoreOps()
535 if (Res && Res != SDValue(Node, 0)) in LegalizeStoreOps()
536 ReplaceNode(SDValue(Node, 0), Res); in LegalizeStoreOps()
546 ReplaceNode(SDValue(Node, 0), Result); in LegalizeStoreOps()
569 ReplaceNode(SDValue(Node, 0), Result); in LegalizeStoreOps()
627 ReplaceNode(SDValue(Node, 0), Result); in LegalizeStoreOps()
643 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); in LegalizeStoreOps()
644 if (Res && Res != SDValue(Node, 0)) in LegalizeStoreOps()
645 ReplaceNode(SDValue(Node, 0), Res); in LegalizeStoreOps()
670 ReplaceNode(SDValue(Node, 0), Result); in LegalizeStoreOps()
676 void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { in LegalizeLoadOps() argument
677 LoadSDNode *LD = cast<LoadSDNode>(Node); in LegalizeLoadOps()
681 SDLoc dl(Node); in LegalizeLoadOps()
686 MVT VT = Node->getSimpleValueType(0); in LegalizeLoadOps()
687 SDValue RVal = SDValue(Node, 0); in LegalizeLoadOps()
688 SDValue RChain = SDValue(Node, 1); in LegalizeLoadOps()
690 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { in LegalizeLoadOps()
711 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); in LegalizeLoadOps()
721 if (RChain.getNode() != Node) { in LegalizeLoadOps()
722 assert(RVal.getNode() != Node && "Load must be completely replaced"); in LegalizeLoadOps()
723 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal); in LegalizeLoadOps()
724 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain); in LegalizeLoadOps()
729 ReplacedNode(Node); in LegalizeLoadOps()
749 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) == in LegalizeLoadOps()
763 SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), in LegalizeLoadOps()
803 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
811 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
827 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
832 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
840 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
856 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
862 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0), in LegalizeLoadOps()
869 Value = SDValue(Node, 0); in LegalizeLoadOps()
870 Chain = SDValue(Node, 1); in LegalizeLoadOps()
873 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { in LegalizeLoadOps()
890 EVT DestVT = Node->getValueType(0); in LegalizeLoadOps()
907 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load); in LegalizeLoadOps()
944 Node->getValueType(0), in LegalizeLoadOps()
963 if (Chain.getNode() != Node) { in LegalizeLoadOps()
964 assert(Value.getNode() != Node && "Load must be completely replaced"); in LegalizeLoadOps()
965 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value); in LegalizeLoadOps()
966 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); in LegalizeLoadOps()
971 ReplacedNode(Node); in LegalizeLoadOps()
976 void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { in LegalizeOp() argument
977 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG)); in LegalizeOp()
980 if (Node->getOpcode() == ISD::TargetConstant || in LegalizeOp()
981 Node->getOpcode() == ISD::Register) in LegalizeOp()
985 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) in LegalizeOp()
986 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == in LegalizeOp()
990 for (const SDValue &Op : Node->op_values()) in LegalizeOp()
1001 switch (Node->getOpcode()) { in LegalizeOp()
1006 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); in LegalizeOp()
1009 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1010 Node->getValueType(0)); in LegalizeOp()
1013 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1014 Node->getValueType(0)); in LegalizeOp()
1016 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); in LegalizeOp()
1020 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1021 Node->getOperand(1).getValueType()); in LegalizeOp()
1032 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1033 Node->getOperand(0).getValueType()); in LegalizeOp()
1045 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1046 Node->getOperand(1).getValueType()); in LegalizeOp()
1049 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT(); in LegalizeOp()
1050 Action = TLI.getOperationAction(Node->getOpcode(), InnerType); in LegalizeOp()
1054 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1055 Node->getOperand(1).getValueType()); in LegalizeOp()
1064 unsigned Opc = Node->getOpcode(); in LegalizeOp()
1075 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType(); in LegalizeOp()
1077 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get(); in LegalizeOp()
1080 if (Node->getOpcode() == ISD::SELECT_CC) in LegalizeOp()
1081 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1082 Node->getValueType(0)); in LegalizeOp()
1084 Action = TLI.getOperationAction(Node->getOpcode(), OpVT); in LegalizeOp()
1112 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1124 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1131 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64); in LegalizeOp()
1141 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1145 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), in LegalizeOp()
1146 Node->getOperand(0)); in LegalizeOp()
1147 ReplaceNode(Node, NewVal.getNode()); in LegalizeOp()
1153 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1157 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), in LegalizeOp()
1158 Node->getOperand(0)); in LegalizeOp()
1159 ReplaceNode(Node, NewVal.getNode()); in LegalizeOp()
1172 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1182 unsigned Scale = Node->getConstantOperandVal(2); in LegalizeOp()
1183 Action = TLI.getFixedPointOperationAction(Node->getOpcode(), in LegalizeOp()
1184 Node->getValueType(0), Scale); in LegalizeOp()
1188 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1189 cast<MaskedScatterSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1192 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1193 cast<MaskedStoreSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1197 Node->getOpcode(), in LegalizeOp()
1198 cast<VPScatterSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1202 Node->getOpcode(), in LegalizeOp()
1203 cast<VPStoreSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1207 Node->getOpcode(), in LegalizeOp()
1208 cast<VPStridedStoreSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1227 Node->getOpcode(), Node->getOperand(0).getValueType()); in LegalizeOp()
1247 Node->getOpcode(), Node->getOperand(1).getValueType()); in LegalizeOp()
1250 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { in LegalizeOp()
1251 Action = TLI.getCustomOperationAction(*Node); in LegalizeOp()
1253 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1259 SDNode *NewNode = Node; in LegalizeOp()
1260 switch (Node->getOpcode()) { in LegalizeOp()
1269 SDValue Op0 = Node->getOperand(0); in LegalizeOp()
1270 SDValue Op1 = Node->getOperand(1); in LegalizeOp()
1279 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO); in LegalizeOp()
1290 SDValue Op0 = Node->getOperand(0); in LegalizeOp()
1291 SDValue Op1 = Node->getOperand(1); in LegalizeOp()
1292 SDValue Op2 = Node->getOperand(2); in LegalizeOp()
1299 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO); in LegalizeOp()
1305 if (NewNode != Node) { in LegalizeOp()
1306 ReplaceNode(Node, NewNode); in LegalizeOp()
1307 Node = NewNode; in LegalizeOp()
1317 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { in LegalizeOp()
1318 if (!(Res.getNode() != Node || Res.getResNo() != 0)) in LegalizeOp()
1321 if (Node->getNumValues() == 1) { in LegalizeOp()
1324 assert((Res.getValueType() == Node->getValueType(0) || in LegalizeOp()
1325 Node->getValueType(0) == MVT::Glue) && in LegalizeOp()
1329 ReplaceNode(SDValue(Node, 0), Res); in LegalizeOp()
1334 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) { in LegalizeOp()
1337 assert((Res->getValueType(i) == Node->getValueType(i) || in LegalizeOp()
1338 Node->getValueType(i) == MVT::Glue) && in LegalizeOp()
1343 ReplaceNode(Node, ResultVals.data()); in LegalizeOp()
1349 if (ExpandNode(Node)) in LegalizeOp()
1353 ConvertNodeToLibcall(Node); in LegalizeOp()
1356 PromoteNode(Node); in LegalizeOp()
1361 switch (Node->getOpcode()) { in LegalizeOp()
1365 Node->dump( &DAG); in LegalizeOp()
1374 return LegalizeLoadOps(Node); in LegalizeOp()
1376 return LegalizeStoreOps(Node); in LegalizeOp()
1496 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) { in ExpandVectorBuildThroughStack() argument
1497 assert((Node->getOpcode() == ISD::BUILD_VECTOR || in ExpandVectorBuildThroughStack()
1498 Node->getOpcode() == ISD::CONCAT_VECTORS) && in ExpandVectorBuildThroughStack()
1505 EVT VT = Node->getValueType(0); in ExpandVectorBuildThroughStack()
1506 EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType() in ExpandVectorBuildThroughStack()
1507 : Node->getOperand(0).getValueType(); in ExpandVectorBuildThroughStack()
1508 SDLoc dl(Node); in ExpandVectorBuildThroughStack()
1521 bool Truncate = isa<BuildVectorSDNode>(Node) && in ExpandVectorBuildThroughStack()
1522 MemVT.bitsLT(Node->getOperand(0).getValueType()); in ExpandVectorBuildThroughStack()
1525 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { in ExpandVectorBuildThroughStack()
1527 if (Node->getOperand(i).isUndef()) continue; in ExpandVectorBuildThroughStack()
1536 Node->getOperand(i), Idx, in ExpandVectorBuildThroughStack()
1539 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i), in ExpandVectorBuildThroughStack()
1621 SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const { in ExpandFCOPYSIGN()
1622 SDLoc DL(Node); in ExpandFCOPYSIGN()
1623 SDValue Mag = Node->getOperand(0); in ExpandFCOPYSIGN()
1624 SDValue Sign = Node->getOperand(1); in ExpandFCOPYSIGN()
1679 SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const { in ExpandFNEG()
1681 SDLoc DL(Node); in ExpandFNEG()
1683 getSignAsIntValue(SignAsInt, DL, Node->getOperand(0)); in ExpandFNEG()
1695 SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const { in ExpandFABS()
1696 SDLoc DL(Node); in ExpandFABS()
1697 SDValue Value = Node->getOperand(0); in ExpandFABS()
1716 void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node, in ExpandDYNAMIC_STACKALLOC() argument
1721 SDLoc dl(Node); in ExpandDYNAMIC_STACKALLOC()
1722 EVT VT = Node->getValueType(0); in ExpandDYNAMIC_STACKALLOC()
1723 SDValue Tmp1 = SDValue(Node, 0); in ExpandDYNAMIC_STACKALLOC()
1724 SDValue Tmp2 = SDValue(Node, 1); in ExpandDYNAMIC_STACKALLOC()
1725 SDValue Tmp3 = Node->getOperand(2); in ExpandDYNAMIC_STACKALLOC()
1808 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { in ExpandSCALAR_TO_VECTOR() argument
1809 SDLoc dl(Node); in ExpandSCALAR_TO_VECTOR()
1812 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); in ExpandSCALAR_TO_VECTOR()
1818 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr, in ExpandSCALAR_TO_VECTOR()
1820 Node->getValueType(0).getVectorElementType()); in ExpandSCALAR_TO_VECTOR()
1822 Node->getValueType(0), dl, Ch, StackPtr, in ExpandSCALAR_TO_VECTOR()
1827 ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, in ExpandBVWithShuffles() argument
1829 unsigned NumElems = Node->getNumOperands(); in ExpandBVWithShuffles()
1830 SDLoc dl(Node); in ExpandBVWithShuffles()
1831 EVT VT = Node->getValueType(0); in ExpandBVWithShuffles()
1844 SDValue V = Node->getOperand(i); in ExpandBVWithShuffles()
1922 SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { in ExpandBUILD_VECTOR() argument
1923 unsigned NumElems = Node->getNumOperands(); in ExpandBUILD_VECTOR()
1925 SDLoc dl(Node); in ExpandBUILD_VECTOR()
1926 EVT VT = Node->getValueType(0); in ExpandBUILD_VECTOR()
1927 EVT OpVT = Node->getOperand(0).getValueType(); in ExpandBUILD_VECTOR()
1936 SDValue V = Node->getOperand(i); in ExpandBUILD_VECTOR()
1958 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); in ExpandBUILD_VECTOR()
1965 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { in ExpandBUILD_VECTOR()
1968 dyn_cast<ConstantSDNode>(Node->getOperand(i))) { in ExpandBUILD_VECTOR()
1980 assert(Node->getOperand(i).isUndef()); in ExpandBUILD_VECTOR()
1997 if (Node->getOperand(i).isUndef()) in ExpandBUILD_VECTOR()
1999 DefinedValues.insert(Node->getOperand(i)); in ExpandBUILD_VECTOR()
2006 SDValue V = Node->getOperand(i); in ExpandBUILD_VECTOR()
2011 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) { in ExpandBUILD_VECTOR()
2025 if (ExpandBVWithShuffles(Node, DAG, TLI, Res)) in ExpandBUILD_VECTOR()
2031 return ExpandVectorBuildThroughStack(Node); in ExpandBUILD_VECTOR()
2034 SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) { in ExpandSPLAT_VECTOR() argument
2035 SDLoc DL(Node); in ExpandSPLAT_VECTOR()
2036 EVT VT = Node->getValueType(0); in ExpandSPLAT_VECTOR()
2037 SDValue SplatVal = Node->getOperand(0); in ExpandSPLAT_VECTOR()
2047 std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, in ExpandLibCall() argument
2053 EVT RetVT = Node->getValueType(0); in ExpandLibCall()
2067 TLI.isInTailCallPosition(DAG, Node, TCChain) && in ExpandLibCall()
2074 CLI.setDebugLoc(SDLoc(Node)) in ExpandLibCall()
2095 std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, in ExpandLibCall() argument
2099 for (const SDValue &Op : Node->op_values()) { in ExpandLibCall()
2102 Entry.Node = Op; in ExpandLibCall()
2109 return ExpandLibCall(LC, Node, std::move(Args), isSigned); in ExpandLibCall()
2113 SDNode *Node, SmallVectorImpl<SDValue> &Results) { in ExpandFrexpLibCall() argument
2114 SDLoc dl(Node); in ExpandFrexpLibCall()
2115 EVT VT = Node->getValueType(0); in ExpandFrexpLibCall()
2116 EVT ExpVT = Node->getValueType(1); in ExpandFrexpLibCall()
2118 SDValue FPOp = Node->getOperand(0); in ExpandFrexpLibCall()
2124 FPArgEntry.Node = FPOp; in ExpandFrexpLibCall()
2129 PtrArgEntry.Node = StackSlot; in ExpandFrexpLibCall()
2136 auto [Call, Chain] = ExpandLibCall(LC, Node, std::move(Args), false); in ExpandFrexpLibCall()
2153 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, in ExpandFPLibCall() argument
2159 if (Node->isStrictFPOpcode()) { in ExpandFPLibCall()
2160 EVT RetVT = Node->getValueType(0); in ExpandFPLibCall()
2161 SmallVector<SDValue, 4> Ops(drop_begin(Node->ops())); in ExpandFPLibCall()
2166 SDLoc(Node), in ExpandFPLibCall()
2167 Node->getOperand(0)); in ExpandFPLibCall()
2171 SDValue Tmp = ExpandLibCall(LC, Node, false).first; in ExpandFPLibCall()
2177 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, in ExpandFPLibCall() argument
2184 RTLIB::Libcall LC = RTLIB::getFPLibCall(Node->getSimpleValueType(0), in ExpandFPLibCall()
2187 ExpandFPLibCall(Node, LC, Results); in ExpandFPLibCall()
2190 SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, in ExpandIntLibCall() argument
2197 switch (Node->getSimpleValueType(0).SimpleTy) { in ExpandIntLibCall()
2205 return ExpandLibCall(LC, Node, isSigned).first; in ExpandIntLibCall()
2210 void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node, in ExpandArgFPLibCall() argument
2217 EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType(); in ExpandArgFPLibCall()
2221 ExpandFPLibCall(Node, LC, Results); in ExpandArgFPLibCall()
2226 SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, in ExpandDivRemLibCall() argument
2228 unsigned Opcode = Node->getOpcode(); in ExpandDivRemLibCall()
2232 switch (Node->getSimpleValueType(0).SimpleTy) { in ExpandDivRemLibCall()
2246 EVT RetVT = Node->getValueType(0); in ExpandDivRemLibCall()
2251 for (const SDValue &Op : Node->op_values()) { in ExpandDivRemLibCall()
2254 Entry.Node = Op; in ExpandDivRemLibCall()
2263 Entry.Node = FIPtr; in ExpandDivRemLibCall()
2272 SDLoc dl(Node); in ExpandDivRemLibCall()
2291 static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) { in isSinCosLibcallAvailable() argument
2293 switch (Node->getSimpleValueType(0).SimpleTy) { in isSinCosLibcallAvailable()
2305 static bool useSinCos(SDNode *Node) { in useSinCos() argument
2306 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN in useSinCos()
2309 SDValue Op0 = Node->getOperand(0); in useSinCos()
2311 if (User == Node) in useSinCos()
2322 SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node, in ExpandSinCosLibCall() argument
2325 switch (Node->getSimpleValueType(0).SimpleTy) { in ExpandSinCosLibCall()
2339 EVT RetVT = Node->getValueType(0); in ExpandSinCosLibCall()
2346 Entry.Node = Node->getOperand(0); in ExpandSinCosLibCall()
2354 Entry.Node = SinPtr; in ExpandSinCosLibCall()
2362 Entry.Node = CosPtr; in ExpandSinCosLibCall()
2371 SDLoc dl(Node); in ExpandSinCosLibCall()
2385 SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const { in expandLdexp()
2386 SDLoc dl(Node); in expandLdexp()
2387 EVT VT = Node->getValueType(0); in expandLdexp()
2388 SDValue X = Node->getOperand(0); in expandLdexp()
2389 SDValue N = Node->getOperand(1); in expandLdexp()
2395 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO in expandLdexp()
2497 SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const { in expandFrexp()
2498 SDLoc dl(Node); in expandFrexp()
2499 SDValue Val = Node->getOperand(0); in expandFrexp()
2501 EVT ExpVT = Node->getValueType(1); in expandFrexp()
2620 SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node, in ExpandLegalINT_TO_FP() argument
2622 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP || in ExpandLegalINT_TO_FP()
2623 Node->getOpcode() == ISD::SINT_TO_FP); in ExpandLegalINT_TO_FP()
2624 EVT DestVT = Node->getValueType(0); in ExpandLegalINT_TO_FP()
2625 SDLoc dl(Node); in ExpandLegalINT_TO_FP()
2626 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0; in ExpandLegalINT_TO_FP()
2627 SDValue Op0 = Node->getOperand(OpNo); in ExpandLegalINT_TO_FP()
2634 TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND in ExpandLegalINT_TO_FP()
2680 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2682 {Node->getOperand(0), Load, Bias}); in ExpandLegalINT_TO_FP()
2734 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2739 { Node->getOperand(0), InCvt }); in ExpandLegalINT_TO_FP()
2747 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept()); in ExpandLegalINT_TO_FP()
2762 Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT)) in ExpandLegalINT_TO_FP()
2773 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2775 { Node->getOperand(0), Op0 }); in ExpandLegalINT_TO_FP()
2824 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2950 SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node, in PromoteLegalFP_TO_INT_SAT() argument
2952 unsigned Opcode = Node->getOpcode(); in PromoteLegalFP_TO_INT_SAT()
2955 EVT NewOutTy = Node->getValueType(0); in PromoteLegalFP_TO_INT_SAT()
2966 SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0), in PromoteLegalFP_TO_INT_SAT()
2967 Node->getOperand(1)); in PromoteLegalFP_TO_INT_SAT()
2968 return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result); in PromoteLegalFP_TO_INT_SAT()
2993 bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { in ExpandNode() argument
2996 SDLoc dl(Node); in ExpandNode()
2999 switch (Node->getOpcode()) { in ExpandNode()
3001 if ((Tmp1 = TLI.expandABS(Node, DAG))) in ExpandNode()
3006 if ((Tmp1 = TLI.expandABD(Node, DAG))) in ExpandNode()
3010 if ((Tmp1 = TLI.expandCTPOP(Node, DAG))) in ExpandNode()
3015 if ((Tmp1 = TLI.expandCTLZ(Node, DAG))) in ExpandNode()
3020 if ((Tmp1 = TLI.expandCTTZ(Node, DAG))) in ExpandNode()
3024 if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG))) in ExpandNode()
3028 if ((Tmp1 = TLI.expandBSWAP(Node, DAG))) in ExpandNode()
3032 Results.push_back(ExpandPARITY(Node->getOperand(0), dl)); in ExpandNode()
3037 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); in ExpandNode()
3040 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl, in ExpandNode()
3055 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0))); in ExpandNode()
3056 Results.push_back(Node->getOperand(0)); in ExpandNode()
3065 Results.push_back(Node->getOperand(0)); in ExpandNode()
3070 Results.append(Node->getNumValues() - 1, in ExpandNode()
3071 DAG.getConstant(0, dl, Node->getValueType(0))); in ExpandNode()
3072 Results.push_back(Node->getOperand(0)); in ExpandNode()
3078 Results.push_back(Node->getOperand(0)); in ExpandNode()
3082 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0)); in ExpandNode()
3083 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); in ExpandNode()
3085 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, in ExpandNode()
3086 Node->getOperand(0), Node->getOperand(1), Zero, Zero, in ExpandNode()
3087 cast<AtomicSDNode>(Node)->getMemOperand()); in ExpandNode()
3095 ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), in ExpandNode()
3096 Node->getOperand(0), Node->getOperand(2), Node->getOperand(1), in ExpandNode()
3097 cast<AtomicSDNode>(Node)->getMemOperand()); in ExpandNode()
3105 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); in ExpandNode()
3107 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, in ExpandNode()
3108 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), in ExpandNode()
3109 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand()); in ExpandNode()
3113 SDValue RHS = Node->getOperand(1); in ExpandNode()
3115 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT(); in ExpandNode()
3116 EVT OuterType = Node->getValueType(0); in ExpandNode()
3122 Node->getOperand(2), DAG.getValueType(AtomicType)); in ExpandNode()
3128 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); in ExpandNode()
3133 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); in ExpandNode()
3140 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ); in ExpandNode()
3148 SDLoc DL(Node); in ExpandNode()
3149 EVT VT = Node->getValueType(0); in ExpandNode()
3150 SDValue RHS = Node->getOperand(2); in ExpandNode()
3151 AtomicSDNode *AN = cast<AtomicSDNode>(Node); in ExpandNode()
3158 Node->getOperand(0), Node->getOperand(1), in ExpandNode()
3165 ExpandDYNAMIC_STACKALLOC(Node, Results); in ExpandNode()
3168 for (unsigned i = 0; i < Node->getNumValues(); i++) in ExpandNode()
3169 Results.push_back(Node->getOperand(i)); in ExpandNode()
3172 EVT VT = Node->getValueType(0); in ExpandNode()
3188 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
3189 Node->getValueType(0)) in ExpandNode()
3194 if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0), in ExpandNode()
3195 Node->getValueType(0), dl, in ExpandNode()
3196 Node->getOperand(0)))) { in ExpandNode()
3197 ReplaceNode(Node, Tmp1.getNode()); in ExpandNode()
3203 EVT VT = Node->getValueType(0); in ExpandNode()
3206 DAG.getNode(ISD::FP_TO_BF16, SDLoc(Node), VT, Node->getOperand(0))); in ExpandNode()
3213 if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), in ExpandNode()
3214 Node->getValueType(0), dl))) in ExpandNode()
3224 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
3225 Node->getValueType(0)) in ExpandNode()
3231 Node->getOperand(1), Node->getOperand(1).getValueType(), in ExpandNode()
3232 Node->getValueType(0), dl, Node->getOperand(0)))) { in ExpandNode()
3233 ReplaceNode(Node, Tmp1.getNode()); in ExpandNode()
3239 SDValue Op = Node->getOperand(0); in ExpandNode()
3241 EVT DstVT = Node->getValueType(0); in ExpandNode()
3243 Results.push_back(DAG.getNode(ISD::BF16_TO_FP, SDLoc(Node), DstVT, Op)); in ExpandNode()
3256 SDValue Op = Node->getOperand(0); in ExpandNode()
3269 if (Node->getValueType(0) != MVT::f32) in ExpandNode()
3270 Op = DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Op); in ExpandNode()
3275 SDValue Op = Node->getOperand(0); in ExpandNode()
3285 if (Node->getValueType(0) == MVT::bf16) { in ExpandNode()
3289 Op = DAG.getAnyExtOrTrunc(Op, dl, Node->getValueType(0)); in ExpandNode()
3295 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); in ExpandNode()
3296 EVT VT = Node->getValueType(0); in ExpandNode()
3307 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One); in ExpandNode()
3320 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), in ExpandNode()
3321 Node->getOperand(0), ShiftCst); in ExpandNode()
3322 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); in ExpandNode()
3328 if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) { in ExpandNode()
3330 if (Node->isStrictFPOpcode()) in ExpandNode()
3337 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) { in ExpandNode()
3339 if (Node->isStrictFPOpcode()) in ExpandNode()
3344 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) in ExpandNode()
3348 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) { in ExpandNode()
3349 ReplaceNode(Node, Tmp1.getNode()); in ExpandNode()
3355 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) in ExpandNode()
3359 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) { in ExpandNode()
3361 DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2); in ExpandNode()
3363 ReplaceNodeWithValue(SDValue(Node, 0), Tmp1); in ExpandNode()
3370 Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG)); in ExpandNode()
3373 Results.push_back(DAG.expandVAArg(Node)); in ExpandNode()
3377 Results.push_back(DAG.expandVACopy(Node)); in ExpandNode()
3380 if (Node->getOperand(0).getValueType().getVectorElementCount().isScalar()) in ExpandNode()
3382 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), in ExpandNode()
3383 Node->getOperand(0)); in ExpandNode()
3385 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0)); in ExpandNode()
3389 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0))); in ExpandNode()
3392 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0))); in ExpandNode()
3395 Results.push_back(ExpandVectorBuildThroughStack(Node)); in ExpandNode()
3398 Results.push_back(ExpandSCALAR_TO_VECTOR(Node)); in ExpandNode()
3401 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0), in ExpandNode()
3402 Node->getOperand(1), in ExpandNode()
3403 Node->getOperand(2), dl)); in ExpandNode()
3407 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); in ExpandNode()
3409 EVT VT = Node->getValueType(0); in ExpandNode()
3411 SDValue Op0 = Node->getOperand(0); in ExpandNode()
3412 SDValue Op1 = Node->getOperand(1); in ExpandNode()
3475 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3480 Results.push_back(TLI.expandVectorSplice(Node, DAG)); in ExpandNode()
3484 EVT OpTy = Node->getOperand(0).getValueType(); in ExpandNode()
3485 if (Node->getConstantOperandVal(1)) { in ExpandNode()
3487 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), in ExpandNode()
3490 Node->getOperand(0).getValueType(), in ExpandNode()
3492 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3495 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), in ExpandNode()
3496 Node->getOperand(0)); in ExpandNode()
3505 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP, in ExpandNode()
3506 Node->getValueType(0))); in ExpandNode()
3509 Results.push_back(DAG.getUNDEF(Node->getValueType(0))); in ExpandNode()
3510 Results.push_back(Node->getOperand(0)); in ExpandNode()
3517 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP, in ExpandNode()
3518 Node->getOperand(1))); in ExpandNode()
3520 Results.push_back(Node->getOperand(0)); in ExpandNode()
3524 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); in ExpandNode()
3528 Results.push_back(ExpandFCOPYSIGN(Node)); in ExpandNode()
3531 Results.push_back(ExpandFNEG(Node)); in ExpandNode()
3534 Results.push_back(ExpandFABS(Node)); in ExpandNode()
3537 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(1)); in ExpandNode()
3539 TLI.expandIS_FPCLASS(Node->getValueType(0), Node->getOperand(0), in ExpandNode()
3540 Test, Node->getFlags(), SDLoc(Node), DAG)) in ExpandNode()
3550 switch (Node->getOpcode()) { in ExpandNode()
3557 Tmp1 = Node->getOperand(0); in ExpandNode()
3558 Tmp2 = Node->getOperand(1); in ExpandNode()
3565 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) in ExpandNode()
3571 EVT VT = Node->getValueType(0); in ExpandNode()
3575 isSinCosLibcallAvailable(Node, TLI)) in ExpandNode()
3576 && useSinCos(Node)) { in ExpandNode()
3578 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); in ExpandNode()
3579 if (Node->getOpcode() == ISD::FCOS) in ExpandNode()
3587 EVT VT = Node->getValueType(0); in ExpandNode()
3594 if (SDValue Expanded = expandLdexp(Node)) { in ExpandNode()
3596 if (Node->getOpcode() == ISD::STRICT_FLDEXP) in ExpandNode()
3603 RTLIB::Libcall LC = RTLIB::getFREXP(Node->getValueType(0)); in ExpandNode()
3609 if (SDValue Expanded = expandFrexp(Node)) { in ExpandNode()
3619 if (Node->getValueType(0) != MVT::f32) { in ExpandNode()
3624 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); in ExpandNode()
3626 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res)); in ExpandNode()
3630 if (Node->getValueType(0) != MVT::f32) { in ExpandNode()
3636 {Node->getOperand(0), Node->getOperand(1)}); in ExpandNode()
3638 {Node->getValueType(0), MVT::Other}, in ExpandNode()
3647 SDValue Op = Node->getOperand(0); in ExpandNode()
3657 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal)); in ExpandNode()
3662 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); in ExpandNode()
3665 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0), in ExpandNode()
3671 ConstantSDNode *CP = cast<ConstantSDNode>(Node); in ExpandNode()
3676 EVT VT = Node->getValueType(0); in ExpandNode()
3679 const SDNodeFlags Flags = Node->getFlags(); in ExpandNode()
3680 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); in ExpandNode()
3681 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags); in ExpandNode()
3687 EVT VT = Node->getValueType(0); in ExpandNode()
3691 Tmp1 = DAG.getNOT(dl, Node->getOperand(1), VT); in ExpandNode()
3693 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); in ExpandNode()
3698 if (TLI.expandREM(Node, Tmp1, DAG)) in ExpandNode()
3703 bool isSigned = Node->getOpcode() == ISD::SDIV; in ExpandNode()
3705 EVT VT = Node->getValueType(0); in ExpandNode()
3708 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), in ExpandNode()
3709 Node->getOperand(1)); in ExpandNode()
3717 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI; in ExpandNode()
3718 EVT VT = Node->getValueType(0); in ExpandNode()
3721 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), in ExpandNode()
3722 Node->getOperand(1)); in ExpandNode()
3728 SDValue LHS = Node->getOperand(0); in ExpandNode()
3729 SDValue RHS = Node->getOperand(1); in ExpandNode()
3732 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS; in ExpandNode()
3743 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves, in ExpandNode()
3760 EVT VT = Node->getValueType(0); in ExpandNode()
3782 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0), in ExpandNode()
3783 Node->getOperand(1))); in ExpandNode()
3793 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG, in ExpandNode()
3807 if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG)) in ExpandNode()
3812 if (SDValue Expanded = TLI.expandROT(Node, true /*AllowVectorOps*/, DAG)) in ExpandNode()
3819 Results.push_back(TLI.expandAddSubSat(Node, DAG)); in ExpandNode()
3823 Results.push_back(TLI.expandShlSat(Node, DAG)); in ExpandNode()
3829 Results.push_back(TLI.expandFixedPointMul(Node, DAG)); in ExpandNode()
3835 if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node), in ExpandNode()
3836 Node->getOperand(0), in ExpandNode()
3837 Node->getOperand(1), in ExpandNode()
3838 Node->getConstantOperandVal(2), in ExpandNode()
3853 SDValue LHS = Node->getOperand(0); in ExpandNode()
3854 SDValue RHS = Node->getOperand(1); in ExpandNode()
3855 SDValue Carry = Node->getOperand(2); in ExpandNode()
3857 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY; in ExpandNode()
3865 EVT CarryType = Node->getValueType(1); in ExpandNode()
3866 EVT SetCCType = getSetCCResultType(Node->getValueType(0)); in ExpandNode()
3897 TLI.expandSADDSUBO(Node, Result, Overflow, DAG); in ExpandNode()
3905 TLI.expandUADDSUBO(Node, Result, Overflow, DAG); in ExpandNode()
3913 if (TLI.expandMULO(Node, Result, Overflow, DAG)) { in ExpandNode()
3920 EVT PairTy = Node->getValueType(0); in ExpandNode()
3921 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); in ExpandNode()
3922 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); in ExpandNode()
3931 Tmp1 = Node->getOperand(0); in ExpandNode()
3932 Tmp2 = Node->getOperand(1); in ExpandNode()
3933 Tmp3 = Node->getOperand(2); in ExpandNode()
3943 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
3947 SDValue Chain = Node->getOperand(0); in ExpandNode()
3948 SDValue Table = Node->getOperand(1); in ExpandNode()
3949 SDValue Index = Node->getOperand(2); in ExpandNode()
3992 Tmp1 = Node->getOperand(0); in ExpandNode()
3993 Tmp2 = Node->getOperand(1); in ExpandNode()
3999 Node->getOperand(2)); in ExpandNode()
4011 Node->getOperand(2)); in ExpandNode()
4019 bool IsVP = Node->getOpcode() == ISD::VP_SETCC; in ExpandNode()
4020 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC || in ExpandNode()
4021 Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandNode()
4022 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandNode()
4023 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ExpandNode()
4025 Tmp1 = Node->getOperand(0 + Offset); in ExpandNode()
4026 Tmp2 = Node->getOperand(1 + Offset); in ExpandNode()
4027 Tmp3 = Node->getOperand(2 + Offset); in ExpandNode()
4030 Mask = Node->getOperand(3 + Offset); in ExpandNode()
4031 EVL = Node->getOperand(4 + Offset); in ExpandNode()
4034 DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl, in ExpandNode()
4042 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(), in ExpandNode()
4043 {Chain, Tmp1, Tmp2, Tmp3}, Node->getFlags()); in ExpandNode()
4046 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), in ExpandNode()
4047 {Tmp1, Tmp2, Tmp3, Mask, EVL}, Node->getFlags()); in ExpandNode()
4049 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Tmp1, in ExpandNode()
4050 Tmp2, Tmp3, Node->getFlags()); in ExpandNode()
4078 EVT VT = Node->getValueType(0); in ExpandNode()
4083 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
4089 Tmp1 = Node->getOperand(0); // LHS in ExpandNode()
4090 Tmp2 = Node->getOperand(1); // RHS in ExpandNode()
4091 Tmp3 = Node->getOperand(2); // True in ExpandNode()
4092 Tmp4 = Node->getOperand(3); // False in ExpandNode()
4093 EVT VT = Node->getValueType(0); in ExpandNode()
4095 SDValue CC = Node->getOperand(4); in ExpandNode()
4106 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags()); in ExpandNode()
4121 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
4131 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
4150 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), in ExpandNode()
4155 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
4158 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
4166 Tmp1 = Node->getOperand(0); // Chain in ExpandNode()
4167 Tmp2 = Node->getOperand(2); // LHS in ExpandNode()
4168 Tmp3 = Node->getOperand(3); // RHS in ExpandNode()
4169 Tmp4 = Node->getOperand(1); // CC in ExpandNode()
4182 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
4183 Tmp4, Tmp2, Tmp3, Node->getOperand(4)); in ExpandNode()
4187 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, in ExpandNode()
4188 Tmp2, Tmp3, Node->getOperand(4)); in ExpandNode()
4194 Results.push_back(ExpandBUILD_VECTOR(Node)); in ExpandNode()
4197 Results.push_back(ExpandSPLAT_VECTOR(Node)); in ExpandNode()
4203 EVT VT = Node->getValueType(0); in ExpandNode()
4212 Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl)); in ExpandNode()
4215 Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl)); in ExpandNode()
4216 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, in ExpandNode()
4220 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars); in ExpandNode()
4239 Results.push_back(TLI.expandVecReduce(Node, DAG)); in ExpandNode()
4256 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) { in ExpandNode()
4262 switch (Node->getOpcode()) { in ExpandNode()
4264 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
4265 Node->getValueType(0)) in ExpandNode()
4271 ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal) in ExpandNode()
4274 ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal) in ExpandNode()
4277 EVT VT = Node->getValueType(0); in ExpandNode()
4278 const SDNodeFlags Flags = Node->getFlags(); in ExpandNode()
4279 SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags); in ExpandNode()
4280 SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(), in ExpandNode()
4281 {Node->getOperand(0), Node->getOperand(1), Neg}, in ExpandNode()
4296 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
4297 Node->getOperand(1).getValueType()) in ExpandNode()
4311 ReplaceNode(Node, Results.data()); in ExpandNode()
4315 void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) { in ConvertNodeToLibcall() argument
4318 SDLoc dl(Node); in ConvertNodeToLibcall()
4320 unsigned Opc = Node->getOpcode(); in ConvertNodeToLibcall()
4329 .setChain(Node->getOperand(0)) in ConvertNodeToLibcall()
4357 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); in ConvertNodeToLibcall()
4358 AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering(); in ConvertNodeToLibcall()
4360 EVT RetVT = Node->getValueType(0); in ConvertNodeToLibcall()
4365 Ops.append(Node->op_begin() + 2, Node->op_end()); in ConvertNodeToLibcall()
4366 Ops.push_back(Node->getOperand(1)); in ConvertNodeToLibcall()
4373 Ops.append(Node->op_begin() + 1, Node->op_end()); in ConvertNodeToLibcall()
4377 SDLoc(Node), in ConvertNodeToLibcall()
4378 Node->getOperand(0)); in ConvertNodeToLibcall()
4388 .setChain(Node->getOperand(0)) in ConvertNodeToLibcall()
4400 ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64, in ConvertNodeToLibcall()
4409 ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64, in ConvertNodeToLibcall()
4415 ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64, in ConvertNodeToLibcall()
4420 ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64, in ConvertNodeToLibcall()
4426 ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64, in ConvertNodeToLibcall()
4432 ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64, in ConvertNodeToLibcall()
4438 ExpandSinCosLibCall(Node, Results); in ConvertNodeToLibcall()
4442 ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80, in ConvertNodeToLibcall()
4447 ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80, in ConvertNodeToLibcall()
4452 ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80, in ConvertNodeToLibcall()
4457 ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80, in ConvertNodeToLibcall()
4462 ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80, in ConvertNodeToLibcall()
4466 ExpandFPLibCall(Node, RTLIB::EXP10_F32, RTLIB::EXP10_F64, RTLIB::EXP10_F80, in ConvertNodeToLibcall()
4471 ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, in ConvertNodeToLibcall()
4477 ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, in ConvertNodeToLibcall()
4483 ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64, in ConvertNodeToLibcall()
4489 ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64, in ConvertNodeToLibcall()
4495 ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32, in ConvertNodeToLibcall()
4503 ExpandFPLibCall(Node, RTLIB::ROUND_F32, in ConvertNodeToLibcall()
4511 ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32, in ConvertNodeToLibcall()
4519 ExpandFPLibCall(Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80, in ConvertNodeToLibcall()
4523 ExpandFrexpLibCall(Node, Results); in ConvertNodeToLibcall()
4528 RTLIB::Libcall LC = RTLIB::getPOWI(Node->getSimpleValueType(0)); in ConvertNodeToLibcall()
4532 if (Node->isStrictFPOpcode()) { in ConvertNodeToLibcall()
4534 DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(Node), in ConvertNodeToLibcall()
4535 {Node->getValueType(0), Node->getValueType(1)}, in ConvertNodeToLibcall()
4536 {Node->getOperand(0), Node->getOperand(2)}); in ConvertNodeToLibcall()
4538 DAG.getNode(ISD::STRICT_FPOW, SDLoc(Node), in ConvertNodeToLibcall()
4539 {Node->getValueType(0), Node->getValueType(1)}, in ConvertNodeToLibcall()
4540 {Exponent.getValue(1), Node->getOperand(1), Exponent}); in ConvertNodeToLibcall()
4545 DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), Node->getValueType(0), in ConvertNodeToLibcall()
4546 Node->getOperand(1)); in ConvertNodeToLibcall()
4547 Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node), in ConvertNodeToLibcall()
4548 Node->getValueType(0), in ConvertNodeToLibcall()
4549 Node->getOperand(0), Exponent)); in ConvertNodeToLibcall()
4553 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0; in ConvertNodeToLibcall()
4556 Node->getOperand(1 + Offset).getValueType().getSizeInBits(); in ConvertNodeToLibcall()
4561 Results.push_back(DAG.getUNDEF(Node->getValueType(0))); in ConvertNodeToLibcall()
4564 ExpandFPLibCall(Node, LC, Results); in ConvertNodeToLibcall()
4569 ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80, in ConvertNodeToLibcall()
4574 ExpandArgFPLibCall(Node, RTLIB::LROUND_F32, in ConvertNodeToLibcall()
4581 ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32, in ConvertNodeToLibcall()
4588 ExpandArgFPLibCall(Node, RTLIB::LRINT_F32, in ConvertNodeToLibcall()
4595 ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32, in ConvertNodeToLibcall()
4602 ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64, in ConvertNodeToLibcall()
4608 ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, in ConvertNodeToLibcall()
4614 ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64, in ConvertNodeToLibcall()
4620 ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64, in ConvertNodeToLibcall()
4626 ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64, in ConvertNodeToLibcall()
4631 if (Node->getValueType(0) == MVT::f32) { in ConvertNodeToLibcall()
4632 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false).first); in ConvertNodeToLibcall()
4636 if (Node->getValueType(0) == MVT::f32) { in ConvertNodeToLibcall()
4639 DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions, in ConvertNodeToLibcall()
4640 SDLoc(Node), Node->getOperand(0)); in ConvertNodeToLibcall()
4648 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16); in ConvertNodeToLibcall()
4650 Results.push_back(ExpandLibCall(LC, Node, false).first); in ConvertNodeToLibcall()
4655 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::bf16); in ConvertNodeToLibcall()
4657 Results.push_back(ExpandLibCall(LC, Node, false).first); in ConvertNodeToLibcall()
4665 bool IsStrict = Node->isStrictFPOpcode(); in ConvertNodeToLibcall()
4666 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP || in ConvertNodeToLibcall()
4667 Node->getOpcode() == ISD::STRICT_SINT_TO_FP; in ConvertNodeToLibcall()
4668 EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType(); in ConvertNodeToLibcall()
4669 EVT RVT = Node->getValueType(0); in ConvertNodeToLibcall()
4671 SDLoc dl(Node); in ConvertNodeToLibcall()
4688 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ConvertNodeToLibcall()
4691 NVT, Node->getOperand(IsStrict ? 1 : 0)); in ConvertNodeToLibcall()
4706 bool IsStrict = Node->isStrictFPOpcode(); in ConvertNodeToLibcall()
4707 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT || in ConvertNodeToLibcall()
4708 Node->getOpcode() == ISD::STRICT_FP_TO_SINT; in ConvertNodeToLibcall()
4710 SDValue Op = Node->getOperand(IsStrict ? 1 : 0); in ConvertNodeToLibcall()
4712 EVT RVT = Node->getValueType(0); in ConvertNodeToLibcall()
4714 SDLoc dl(Node); in ConvertNodeToLibcall()
4731 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ConvertNodeToLibcall()
4750 bool IsStrict = Node->isStrictFPOpcode(); in ConvertNodeToLibcall()
4751 SDValue Op = Node->getOperand(IsStrict ? 1 : 0); in ConvertNodeToLibcall()
4752 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ConvertNodeToLibcall()
4753 EVT VT = Node->getValueType(0); in ConvertNodeToLibcall()
4754 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() && in ConvertNodeToLibcall()
4762 TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, SDLoc(Node), Chain); in ConvertNodeToLibcall()
4770 ExpandLibCall(RTLIB::getFPEXT(Node->getOperand(0).getValueType(), in ConvertNodeToLibcall()
4771 Node->getValueType(0)), in ConvertNodeToLibcall()
4772 Node, false).first); in ConvertNodeToLibcall()
4778 Node->getOpcode() == ISD::STRICT_FP_TO_FP16 in ConvertNodeToLibcall()
4779 ? RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16) in ConvertNodeToLibcall()
4780 : RTLIB::getFPEXT(Node->getOperand(1).getValueType(), in ConvertNodeToLibcall()
4781 Node->getValueType(0)); in ConvertNodeToLibcall()
4786 TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1), in ConvertNodeToLibcall()
4787 CallOptions, SDLoc(Node), Node->getOperand(0)); in ConvertNodeToLibcall()
4794 ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64, in ConvertNodeToLibcall()
4799 Results.push_back(ExpandIntLibCall(Node, true, in ConvertNodeToLibcall()
4805 Results.push_back(ExpandIntLibCall(Node, false, in ConvertNodeToLibcall()
4811 Results.push_back(ExpandIntLibCall(Node, true, in ConvertNodeToLibcall()
4817 Results.push_back(ExpandIntLibCall(Node, false, in ConvertNodeToLibcall()
4825 ExpandDivRemLibCall(Node, Results); in ConvertNodeToLibcall()
4828 Results.push_back(ExpandIntLibCall(Node, false, in ConvertNodeToLibcall()
4834 switch (Node->getSimpleValueType(0).SimpleTy) { in ConvertNodeToLibcall()
4838 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false).first); in ConvertNodeToLibcall()
4841 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false).first); in ConvertNodeToLibcall()
4844 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false).first); in ConvertNodeToLibcall()
4852 SDValue Chain = Node->getOperand(0); in ConvertNodeToLibcall()
4858 SDValue Chain = Node->getOperand(0); in ConvertNodeToLibcall()
4859 SDValue EnvPtr = Node->getOperand(1); in ConvertNodeToLibcall()
4865 SDValue Chain = Node->getOperand(0); in ConvertNodeToLibcall()
4866 SDValue EnvPtr = Node->getOperand(1); in ConvertNodeToLibcall()
4874 EVT ModeVT = Node->getValueType(0); in ConvertNodeToLibcall()
4878 Node->getOperand(0), dl); in ConvertNodeToLibcall()
4889 SDValue Mode = Node->getOperand(1); in ConvertNodeToLibcall()
4894 Node->getOperand(0), dl, Mode, StackPtr, in ConvertNodeToLibcall()
4908 Node->getOperand(0), dl)); in ConvertNodeToLibcall()
4916 ReplaceNode(Node, Results.data()); in ConvertNodeToLibcall()
4933 void SelectionDAGLegalize::PromoteNode(SDNode *Node) { in PromoteNode() argument
4936 MVT OVT = Node->getSimpleValueType(0); in PromoteNode()
4937 if (Node->getOpcode() == ISD::UINT_TO_FP || in PromoteNode()
4938 Node->getOpcode() == ISD::SINT_TO_FP || in PromoteNode()
4939 Node->getOpcode() == ISD::SETCC || in PromoteNode()
4940 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT || in PromoteNode()
4941 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) { in PromoteNode()
4942 OVT = Node->getOperand(0).getSimpleValueType(); in PromoteNode()
4944 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP || in PromoteNode()
4945 Node->getOpcode() == ISD::STRICT_SINT_TO_FP || in PromoteNode()
4946 Node->getOpcode() == ISD::STRICT_FSETCC || in PromoteNode()
4947 Node->getOpcode() == ISD::STRICT_FSETCCS) in PromoteNode()
4948 OVT = Node->getOperand(1).getSimpleValueType(); in PromoteNode()
4949 if (Node->getOpcode() == ISD::BR_CC || in PromoteNode()
4950 Node->getOpcode() == ISD::SELECT_CC) in PromoteNode()
4951 OVT = Node->getOperand(2).getSimpleValueType(); in PromoteNode()
4952 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); in PromoteNode()
4953 SDLoc dl(Node); in PromoteNode()
4955 switch (Node->getOpcode()) { in PromoteNode()
4962 if (Node->getOpcode() == ISD::CTTZ || in PromoteNode()
4963 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF) in PromoteNode()
4964 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4966 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4968 if (Node->getOpcode() == ISD::CTTZ) { in PromoteNode()
4979 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
4980 if (Node->getOpcode() == ISD::CTLZ || in PromoteNode()
4981 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) { in PromoteNode()
4992 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4993 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
5006 PromoteLegalFP_TO_INT(Node, dl, Results); in PromoteNode()
5010 Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl)); in PromoteNode()
5016 PromoteLegalINT_TO_FP(Node, dl, Results); in PromoteNode()
5019 SDValue Chain = Node->getOperand(0); // Get the chain. in PromoteNode()
5020 SDValue Ptr = Node->getOperand(1); // Get the pointer. in PromoteNode()
5032 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2), in PromoteNode()
5033 Node->getConstantOperandVal(3)); in PromoteNode()
5040 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2); in PromoteNode()
5041 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); in PromoteNode()
5046 ReplacedNode(Node); in PromoteNode()
5068 switch (Node->getOpcode()) { in PromoteNode()
5093 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
5094 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5096 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); in PromoteNode()
5103 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND in PromoteNode()
5105 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
5106 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5120 if (Node->getValueType(0).isVector() || in PromoteNode()
5121 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) { in PromoteNode()
5124 } else if (Node->getValueType(0).isInteger()) { in PromoteNode()
5131 Tmp1 = Node->getOperand(0); in PromoteNode()
5133 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5134 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); in PromoteNode()
5137 Tmp1->setFlags(Node->getFlags()); in PromoteNode()
5139 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); in PromoteNode()
5141 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1, in PromoteNode()
5147 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); in PromoteNode()
5150 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); in PromoteNode()
5151 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); in PromoteNode()
5160 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5161 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
5163 Node->getOperand(2)); in PromoteNode()
5168 SDValue Cond = Node->getOperand(4); in PromoteNode()
5171 MVT CVT = Node->getSimpleValueType(0); in PromoteNode()
5181 Tmp1 = Node->getOperand(0); in PromoteNode()
5182 Tmp2 = Node->getOperand(1); in PromoteNode()
5184 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
5185 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5188 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); in PromoteNode()
5189 Tmp4 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); in PromoteNode()
5192 Node->getFlags()); in PromoteNode()
5209 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); in PromoteNode()
5211 TLI.isSExtCheaperThanZExt(Node->getOperand(0).getValueType(), NVT)) in PromoteNode()
5216 if (Node->isStrictFPOpcode()) { in PromoteNode()
5217 SDValue InChain = Node->getOperand(0); in PromoteNode()
5219 DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT); in PromoteNode()
5221 DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT); in PromoteNode()
5224 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); in PromoteNode()
5225 Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs, in PromoteNode()
5226 {OutChain, Tmp1, Tmp2, Node->getOperand(3)}, in PromoteNode()
5227 Node->getFlags())); in PromoteNode()
5231 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
5232 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
5233 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1, in PromoteNode()
5234 Tmp2, Node->getOperand(2), Node->getFlags())); in PromoteNode()
5241 cast<CondCodeSDNode>(Node->getOperand(1))->get(); in PromoteNode()
5244 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); in PromoteNode()
5245 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); in PromoteNode()
5246 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), in PromoteNode()
5247 Node->getOperand(0), Node->getOperand(1), in PromoteNode()
5248 Tmp1, Tmp2, Node->getOperand(4))); in PromoteNode()
5261 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5262 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
5263 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, in PromoteNode()
5264 Node->getFlags()); in PromoteNode()
5278 {Node->getOperand(0), Node->getOperand(1)}); in PromoteNode()
5280 {Node->getOperand(0), Node->getOperand(2)}); in PromoteNode()
5283 Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
5291 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5292 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
5293 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2)); in PromoteNode()
5296 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3), in PromoteNode()
5301 {Node->getOperand(0), Node->getOperand(1)}); in PromoteNode()
5303 {Node->getOperand(0), Node->getOperand(2)}); in PromoteNode()
5305 {Node->getOperand(0), Node->getOperand(3)}); in PromoteNode()
5308 Tmp4 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
5318 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5319 Tmp2 = Node->getOperand(1); in PromoteNode()
5320 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); in PromoteNode()
5327 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN); in PromoteNode()
5335 {Node->getOperand(0), Node->getOperand(1)}); in PromoteNode()
5336 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
5337 {Tmp1.getValue(1), Tmp1, Node->getOperand(2)}); in PromoteNode()
5344 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5345 Tmp2 = DAG.getNode(ISD::FFREXP, dl, {NVT, Node->getValueType(1)}, Tmp1); in PromoteNode()
5373 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
5374 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
5395 {Node->getOperand(0), Node->getOperand(1)}); in PromoteNode()
5396 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
5420 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) { in PromoteNode()
5421 SDValue Op = Node->getOperand(I); in PromoteNode()
5425 SDLoc SL(Node); in PromoteNode()
5455 SDValue Idx = Node->getOperand(1); in PromoteNode()
5457 SDLoc SL(Node); in PromoteNode()
5461 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); in PromoteNode()
5501 SDValue Val = Node->getOperand(1); in PromoteNode()
5502 SDValue Idx = Node->getOperand(2); in PromoteNode()
5504 SDLoc SL(Node); in PromoteNode()
5509 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); in PromoteNode()
5539 SDValue Val = Node->getOperand(0); in PromoteNode()
5540 SDLoc SL(Node); in PromoteNode()
5556 AtomicSDNode *AM = cast<AtomicSDNode>(Node); in PromoteNode()
5557 SDLoc SL(Node); in PromoteNode()
5574 SDValue Scalar = Node->getOperand(0); in PromoteNode()
5579 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
5584 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
5595 ReplaceNode(Node, Results.data()); in PromoteNode()