Lines Matching refs:Node
111 void LegalizeOp(SDNode *Node);
116 void LegalizeLoadOps(SDNode *Node);
117 void LegalizeStoreOps(SDNode *Node);
136 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
138 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
140 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
145 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
151 void ExpandArgFPLibCall(SDNode *Node,
156 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
163 SDValue ExpandBUILD_VECTOR(SDNode *Node);
164 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
165 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
166 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
172 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
173 SDValue ExpandFABS(SDNode *Node) const;
174 SDValue ExpandFNEG(SDNode *Node) const;
175 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
180 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
186 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
192 bool ExpandNode(SDNode *Node);
193 void ConvertNodeToLibcall(SDNode *Node);
194 void PromoteNode(SDNode *Node);
489 void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { in LegalizeStoreOps() argument
490 StoreSDNode *ST = cast<StoreSDNode>(Node); in LegalizeStoreOps()
493 SDLoc dl(Node); in LegalizeStoreOps()
525 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); in LegalizeStoreOps()
526 if (Res && Res != SDValue(Node, 0)) in LegalizeStoreOps()
527 ReplaceNode(SDValue(Node, 0), Res); in LegalizeStoreOps()
537 ReplaceNode(SDValue(Node, 0), Result); in LegalizeStoreOps()
560 ReplaceNode(SDValue(Node, 0), Result); in LegalizeStoreOps()
617 ReplaceNode(SDValue(Node, 0), Result); in LegalizeStoreOps()
633 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); in LegalizeStoreOps()
634 if (Res && Res != SDValue(Node, 0)) in LegalizeStoreOps()
635 ReplaceNode(SDValue(Node, 0), Res); in LegalizeStoreOps()
660 ReplaceNode(SDValue(Node, 0), Result); in LegalizeStoreOps()
666 void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { in LegalizeLoadOps() argument
667 LoadSDNode *LD = cast<LoadSDNode>(Node); in LegalizeLoadOps()
671 SDLoc dl(Node); in LegalizeLoadOps()
676 MVT VT = Node->getSimpleValueType(0); in LegalizeLoadOps()
677 SDValue RVal = SDValue(Node, 0); in LegalizeLoadOps()
678 SDValue RChain = SDValue(Node, 1); in LegalizeLoadOps()
680 switch (TLI.getOperationAction(Node->getOpcode(), VT)) { in LegalizeLoadOps()
701 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); in LegalizeLoadOps()
711 if (RChain.getNode() != Node) { in LegalizeLoadOps()
712 assert(RVal.getNode() != Node && "Load must be completely replaced"); in LegalizeLoadOps()
713 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal); in LegalizeLoadOps()
714 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain); in LegalizeLoadOps()
719 ReplacedNode(Node); in LegalizeLoadOps()
739 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) == in LegalizeLoadOps()
753 SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), in LegalizeLoadOps()
793 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
800 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
816 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
821 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
828 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, in LegalizeLoadOps()
844 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
850 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0), in LegalizeLoadOps()
857 Value = SDValue(Node, 0); in LegalizeLoadOps()
858 Chain = SDValue(Node, 1); in LegalizeLoadOps()
861 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { in LegalizeLoadOps()
878 EVT DestVT = Node->getValueType(0); in LegalizeLoadOps()
894 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load); in LegalizeLoadOps()
928 Node->getValueType(0), in LegalizeLoadOps()
947 if (Chain.getNode() != Node) { in LegalizeLoadOps()
948 assert(Value.getNode() != Node && "Load must be completely replaced"); in LegalizeLoadOps()
949 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value); in LegalizeLoadOps()
950 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); in LegalizeLoadOps()
955 ReplacedNode(Node); in LegalizeLoadOps()
960 void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { in LegalizeOp() argument
961 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG)); in LegalizeOp()
964 if (Node->getOpcode() == ISD::TargetConstant || in LegalizeOp()
965 Node->getOpcode() == ISD::Register) in LegalizeOp()
969 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) in LegalizeOp()
970 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == in LegalizeOp()
974 for (const SDValue &Op : Node->op_values()) in LegalizeOp()
985 switch (Node->getOpcode()) { in LegalizeOp()
990 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); in LegalizeOp()
993 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
994 Node->getValueType(0)); in LegalizeOp()
997 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
998 Node->getValueType(0)); in LegalizeOp()
1000 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); in LegalizeOp()
1010 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1011 Node->getOperand(0).getValueType()); in LegalizeOp()
1023 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1024 Node->getOperand(1).getValueType()); in LegalizeOp()
1027 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT(); in LegalizeOp()
1028 Action = TLI.getOperationAction(Node->getOpcode(), InnerType); in LegalizeOp()
1032 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1033 Node->getOperand(2).getValueType()); in LegalizeOp()
1040 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 : in LegalizeOp()
1041 Node->getOpcode() == ISD::STRICT_FSETCC ? 3 : in LegalizeOp()
1042 Node->getOpcode() == ISD::STRICT_FSETCCS ? 3 : in LegalizeOp()
1043 Node->getOpcode() == ISD::SETCC ? 2 : 1; in LegalizeOp()
1044 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : in LegalizeOp()
1045 Node->getOpcode() == ISD::STRICT_FSETCC ? 1 : in LegalizeOp()
1046 Node->getOpcode() == ISD::STRICT_FSETCCS ? 1 : 0; in LegalizeOp()
1047 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType(); in LegalizeOp()
1049 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get(); in LegalizeOp()
1052 if (Node->getOpcode() == ISD::SELECT_CC) in LegalizeOp()
1053 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1054 Node->getValueType(0)); in LegalizeOp()
1056 Action = TLI.getOperationAction(Node->getOpcode(), OpVT); in LegalizeOp()
1084 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1096 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1103 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64); in LegalizeOp()
1113 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1117 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), in LegalizeOp()
1118 Node->getOperand(0)); in LegalizeOp()
1119 ReplaceNode(Node, NewVal.getNode()); in LegalizeOp()
1125 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1129 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), in LegalizeOp()
1130 Node->getOperand(0)); in LegalizeOp()
1131 ReplaceNode(Node, NewVal.getNode()); in LegalizeOp()
1144 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1154 unsigned Scale = Node->getConstantOperandVal(2); in LegalizeOp()
1155 Action = TLI.getFixedPointOperationAction(Node->getOpcode(), in LegalizeOp()
1156 Node->getValueType(0), Scale); in LegalizeOp()
1160 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1161 cast<MaskedScatterSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1164 Action = TLI.getOperationAction(Node->getOpcode(), in LegalizeOp()
1165 cast<MaskedStoreSDNode>(Node)->getValue().getValueType()); in LegalizeOp()
1181 Node->getOpcode(), Node->getOperand(0).getValueType()); in LegalizeOp()
1185 Node->getOpcode(), Node->getOperand(1).getValueType()); in LegalizeOp()
1188 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { in LegalizeOp()
1191 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); in LegalizeOp()
1197 SDNode *NewNode = Node; in LegalizeOp()
1198 switch (Node->getOpcode()) { in LegalizeOp()
1207 SDValue Op0 = Node->getOperand(0); in LegalizeOp()
1208 SDValue Op1 = Node->getOperand(1); in LegalizeOp()
1217 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO); in LegalizeOp()
1228 SDValue Op0 = Node->getOperand(0); in LegalizeOp()
1229 SDValue Op1 = Node->getOperand(1); in LegalizeOp()
1230 SDValue Op2 = Node->getOperand(2); in LegalizeOp()
1237 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO); in LegalizeOp()
1243 if (NewNode != Node) { in LegalizeOp()
1244 ReplaceNode(Node, NewNode); in LegalizeOp()
1245 Node = NewNode; in LegalizeOp()
1255 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { in LegalizeOp()
1256 if (!(Res.getNode() != Node || Res.getResNo() != 0)) in LegalizeOp()
1259 if (Node->getNumValues() == 1) { in LegalizeOp()
1262 assert((Res.getValueType() == Node->getValueType(0) || in LegalizeOp()
1263 Node->getValueType(0) == MVT::Glue) && in LegalizeOp()
1267 ReplaceNode(SDValue(Node, 0), Res); in LegalizeOp()
1272 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) { in LegalizeOp()
1275 assert((Res->getValueType(i) == Node->getValueType(i) || in LegalizeOp()
1276 Node->getValueType(i) == MVT::Glue) && in LegalizeOp()
1281 ReplaceNode(Node, ResultVals.data()); in LegalizeOp()
1287 if (ExpandNode(Node)) in LegalizeOp()
1291 ConvertNodeToLibcall(Node); in LegalizeOp()
1294 PromoteNode(Node); in LegalizeOp()
1299 switch (Node->getOpcode()) { in LegalizeOp()
1303 Node->dump( &DAG); in LegalizeOp()
1312 return LegalizeLoadOps(Node); in LegalizeOp()
1314 return LegalizeStoreOps(Node); in LegalizeOp()
1432 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) { in ExpandVectorBuildThroughStack() argument
1433 assert((Node->getOpcode() == ISD::BUILD_VECTOR || in ExpandVectorBuildThroughStack()
1434 Node->getOpcode() == ISD::CONCAT_VECTORS) && in ExpandVectorBuildThroughStack()
1441 EVT VT = Node->getValueType(0); in ExpandVectorBuildThroughStack()
1442 EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType() in ExpandVectorBuildThroughStack()
1443 : Node->getOperand(0).getValueType(); in ExpandVectorBuildThroughStack()
1444 SDLoc dl(Node); in ExpandVectorBuildThroughStack()
1457 bool Truncate = isa<BuildVectorSDNode>(Node) && in ExpandVectorBuildThroughStack()
1458 MemVT.bitsLT(Node->getOperand(0).getValueType()); in ExpandVectorBuildThroughStack()
1461 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { in ExpandVectorBuildThroughStack()
1463 if (Node->getOperand(i).isUndef()) continue; in ExpandVectorBuildThroughStack()
1471 Node->getOperand(i), Idx, in ExpandVectorBuildThroughStack()
1474 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i), in ExpandVectorBuildThroughStack()
1556 SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const { in ExpandFCOPYSIGN()
1557 SDLoc DL(Node); in ExpandFCOPYSIGN()
1558 SDValue Mag = Node->getOperand(0); in ExpandFCOPYSIGN()
1559 SDValue Sign = Node->getOperand(1); in ExpandFCOPYSIGN()
1614 SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const { in ExpandFNEG()
1616 SDLoc DL(Node); in ExpandFNEG()
1618 getSignAsIntValue(SignAsInt, DL, Node->getOperand(0)); in ExpandFNEG()
1630 SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const { in ExpandFABS()
1631 SDLoc DL(Node); in ExpandFABS()
1632 SDValue Value = Node->getOperand(0); in ExpandFABS()
1651 void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node, in ExpandDYNAMIC_STACKALLOC() argument
1656 SDLoc dl(Node); in ExpandDYNAMIC_STACKALLOC()
1657 EVT VT = Node->getValueType(0); in ExpandDYNAMIC_STACKALLOC()
1658 SDValue Tmp1 = SDValue(Node, 0); in ExpandDYNAMIC_STACKALLOC()
1659 SDValue Tmp2 = SDValue(Node, 1); in ExpandDYNAMIC_STACKALLOC()
1660 SDValue Tmp3 = Node->getOperand(2); in ExpandDYNAMIC_STACKALLOC()
1747 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { in ExpandSCALAR_TO_VECTOR() argument
1748 SDLoc dl(Node); in ExpandSCALAR_TO_VECTOR()
1751 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); in ExpandSCALAR_TO_VECTOR()
1757 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr, in ExpandSCALAR_TO_VECTOR()
1759 Node->getValueType(0).getVectorElementType()); in ExpandSCALAR_TO_VECTOR()
1761 Node->getValueType(0), dl, Ch, StackPtr, in ExpandSCALAR_TO_VECTOR()
1766 ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, in ExpandBVWithShuffles() argument
1768 unsigned NumElems = Node->getNumOperands(); in ExpandBVWithShuffles()
1769 SDLoc dl(Node); in ExpandBVWithShuffles()
1770 EVT VT = Node->getValueType(0); in ExpandBVWithShuffles()
1783 SDValue V = Node->getOperand(i); in ExpandBVWithShuffles()
1861 SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { in ExpandBUILD_VECTOR() argument
1862 unsigned NumElems = Node->getNumOperands(); in ExpandBUILD_VECTOR()
1864 SDLoc dl(Node); in ExpandBUILD_VECTOR()
1865 EVT VT = Node->getValueType(0); in ExpandBUILD_VECTOR()
1866 EVT OpVT = Node->getOperand(0).getValueType(); in ExpandBUILD_VECTOR()
1875 SDValue V = Node->getOperand(i); in ExpandBUILD_VECTOR()
1897 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); in ExpandBUILD_VECTOR()
1904 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { in ExpandBUILD_VECTOR()
1907 dyn_cast<ConstantSDNode>(Node->getOperand(i))) { in ExpandBUILD_VECTOR()
1919 assert(Node->getOperand(i).isUndef()); in ExpandBUILD_VECTOR()
1936 if (Node->getOperand(i).isUndef()) in ExpandBUILD_VECTOR()
1938 DefinedValues.insert(Node->getOperand(i)); in ExpandBUILD_VECTOR()
1945 SDValue V = Node->getOperand(i); in ExpandBUILD_VECTOR()
1950 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) { in ExpandBUILD_VECTOR()
1964 if (ExpandBVWithShuffles(Node, DAG, TLI, Res)) in ExpandBUILD_VECTOR()
1970 return ExpandVectorBuildThroughStack(Node); in ExpandBUILD_VECTOR()
1973 SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) { in ExpandSPLAT_VECTOR() argument
1974 SDLoc DL(Node); in ExpandSPLAT_VECTOR()
1975 EVT VT = Node->getValueType(0); in ExpandSPLAT_VECTOR()
1976 SDValue SplatVal = Node->getOperand(0); in ExpandSPLAT_VECTOR()
1985 SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, in ExpandLibCall() argument
1989 for (const SDValue &Op : Node->op_values()) { in ExpandLibCall()
1992 Entry.Node = Op; in ExpandLibCall()
2001 EVT RetVT = Node->getValueType(0); in ExpandLibCall()
2015 TLI.isInTailCallPosition(DAG, Node, TCChain) && in ExpandLibCall()
2022 CLI.setDebugLoc(SDLoc(Node)) in ExpandLibCall()
2043 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, in ExpandFPLibCall() argument
2049 if (Node->isStrictFPOpcode()) { in ExpandFPLibCall()
2050 EVT RetVT = Node->getValueType(0); in ExpandFPLibCall()
2051 SmallVector<SDValue, 4> Ops(drop_begin(Node->ops())); in ExpandFPLibCall()
2056 SDLoc(Node), in ExpandFPLibCall()
2057 Node->getOperand(0)); in ExpandFPLibCall()
2061 SDValue Tmp = ExpandLibCall(LC, Node, false); in ExpandFPLibCall()
2067 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, in ExpandFPLibCall() argument
2074 RTLIB::Libcall LC = RTLIB::getFPLibCall(Node->getSimpleValueType(0), in ExpandFPLibCall()
2077 ExpandFPLibCall(Node, LC, Results); in ExpandFPLibCall()
2080 SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, in ExpandIntLibCall() argument
2087 switch (Node->getSimpleValueType(0).SimpleTy) { in ExpandIntLibCall()
2095 return ExpandLibCall(LC, Node, isSigned); in ExpandIntLibCall()
2100 void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node, in ExpandArgFPLibCall() argument
2107 EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType(); in ExpandArgFPLibCall()
2111 ExpandFPLibCall(Node, LC, Results); in ExpandArgFPLibCall()
2116 SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, in ExpandDivRemLibCall() argument
2118 unsigned Opcode = Node->getOpcode(); in ExpandDivRemLibCall()
2122 switch (Node->getSimpleValueType(0).SimpleTy) { in ExpandDivRemLibCall()
2136 EVT RetVT = Node->getValueType(0); in ExpandDivRemLibCall()
2141 for (const SDValue &Op : Node->op_values()) { in ExpandDivRemLibCall()
2144 Entry.Node = Op; in ExpandDivRemLibCall()
2153 Entry.Node = FIPtr; in ExpandDivRemLibCall()
2162 SDLoc dl(Node); in ExpandDivRemLibCall()
2181 static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) { in isSinCosLibcallAvailable() argument
2183 switch (Node->getSimpleValueType(0).SimpleTy) { in isSinCosLibcallAvailable()
2195 static bool useSinCos(SDNode *Node) { in useSinCos() argument
2196 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN in useSinCos()
2199 SDValue Op0 = Node->getOperand(0); in useSinCos()
2203 if (User == Node) in useSinCos()
2214 SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node, in ExpandSinCosLibCall() argument
2217 switch (Node->getSimpleValueType(0).SimpleTy) { in ExpandSinCosLibCall()
2231 EVT RetVT = Node->getValueType(0); in ExpandSinCosLibCall()
2238 Entry.Node = Node->getOperand(0); in ExpandSinCosLibCall()
2246 Entry.Node = SinPtr; in ExpandSinCosLibCall()
2254 Entry.Node = CosPtr; in ExpandSinCosLibCall()
2263 SDLoc dl(Node); in ExpandSinCosLibCall()
2281 SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node, in ExpandLegalINT_TO_FP() argument
2283 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP || in ExpandLegalINT_TO_FP()
2284 Node->getOpcode() == ISD::SINT_TO_FP); in ExpandLegalINT_TO_FP()
2285 EVT DestVT = Node->getValueType(0); in ExpandLegalINT_TO_FP()
2286 SDLoc dl(Node); in ExpandLegalINT_TO_FP()
2287 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0; in ExpandLegalINT_TO_FP()
2288 SDValue Op0 = Node->getOperand(OpNo); in ExpandLegalINT_TO_FP()
2295 TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND in ExpandLegalINT_TO_FP()
2340 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2342 {Node->getOperand(0), Load, Bias}); in ExpandLegalINT_TO_FP()
2394 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2399 { Node->getOperand(0), InCvt }); in ExpandLegalINT_TO_FP()
2407 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept()); in ExpandLegalINT_TO_FP()
2422 Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT)) in ExpandLegalINT_TO_FP()
2433 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2435 { Node->getOperand(0), Op0 }); in ExpandLegalINT_TO_FP()
2484 if (Node->isStrictFPOpcode()) { in ExpandLegalINT_TO_FP()
2610 SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node, in PromoteLegalFP_TO_INT_SAT() argument
2612 unsigned Opcode = Node->getOpcode(); in PromoteLegalFP_TO_INT_SAT()
2615 EVT NewOutTy = Node->getValueType(0); in PromoteLegalFP_TO_INT_SAT()
2626 SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0), in PromoteLegalFP_TO_INT_SAT()
2627 Node->getOperand(1)); in PromoteLegalFP_TO_INT_SAT()
2628 return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result); in PromoteLegalFP_TO_INT_SAT()
2653 bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { in ExpandNode() argument
2656 SDLoc dl(Node); in ExpandNode()
2659 switch (Node->getOpcode()) { in ExpandNode()
2661 if (TLI.expandABS(Node, Tmp1, DAG)) in ExpandNode()
2665 if (TLI.expandCTPOP(Node, Tmp1, DAG)) in ExpandNode()
2670 if (TLI.expandCTLZ(Node, Tmp1, DAG)) in ExpandNode()
2675 if (TLI.expandCTTZ(Node, Tmp1, DAG)) in ExpandNode()
2679 if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG))) in ExpandNode()
2683 if ((Tmp1 = TLI.expandBSWAP(Node, DAG))) in ExpandNode()
2687 Results.push_back(ExpandPARITY(Node->getOperand(0), dl)); in ExpandNode()
2692 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); in ExpandNode()
2695 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl, in ExpandNode()
2710 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0))); in ExpandNode()
2711 Results.push_back(Node->getOperand(0)); in ExpandNode()
2720 Results.push_back(Node->getOperand(0)); in ExpandNode()
2725 Results.append(Node->getNumValues() - 1, in ExpandNode()
2726 DAG.getConstant(0, dl, Node->getValueType(0))); in ExpandNode()
2727 Results.push_back(Node->getOperand(0)); in ExpandNode()
2733 Results.push_back(Node->getOperand(0)); in ExpandNode()
2737 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0)); in ExpandNode()
2738 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); in ExpandNode()
2740 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, in ExpandNode()
2741 Node->getOperand(0), Node->getOperand(1), Zero, Zero, in ExpandNode()
2742 cast<AtomicSDNode>(Node)->getMemOperand()); in ExpandNode()
2750 cast<AtomicSDNode>(Node)->getMemoryVT(), in ExpandNode()
2751 Node->getOperand(0), in ExpandNode()
2752 Node->getOperand(1), Node->getOperand(2), in ExpandNode()
2753 cast<AtomicSDNode>(Node)->getMemOperand()); in ExpandNode()
2761 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); in ExpandNode()
2763 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, in ExpandNode()
2764 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), in ExpandNode()
2765 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand()); in ExpandNode()
2769 SDValue RHS = Node->getOperand(1); in ExpandNode()
2771 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT(); in ExpandNode()
2772 EVT OuterType = Node->getValueType(0); in ExpandNode()
2778 Node->getOperand(2), DAG.getValueType(AtomicType)); in ExpandNode()
2784 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); in ExpandNode()
2789 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); in ExpandNode()
2796 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ); in ExpandNode()
2804 ExpandDYNAMIC_STACKALLOC(Node, Results); in ExpandNode()
2807 for (unsigned i = 0; i < Node->getNumValues(); i++) in ExpandNode()
2808 Results.push_back(Node->getOperand(i)); in ExpandNode()
2811 EVT VT = Node->getValueType(0); in ExpandNode()
2827 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
2828 Node->getValueType(0)) in ExpandNode()
2833 if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0), in ExpandNode()
2834 Node->getValueType(0), dl, in ExpandNode()
2835 Node->getOperand(0)))) { in ExpandNode()
2836 ReplaceNode(Node, Tmp1.getNode()); in ExpandNode()
2843 if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), in ExpandNode()
2844 Node->getValueType(0), dl))) in ExpandNode()
2854 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
2855 Node->getValueType(0)) in ExpandNode()
2861 Node->getOperand(1), Node->getOperand(1).getValueType(), in ExpandNode()
2862 Node->getValueType(0), dl, Node->getOperand(0)))) { in ExpandNode()
2863 ReplaceNode(Node, Tmp1.getNode()); in ExpandNode()
2869 if ((Tmp1 = EmitStackConvert(Node->getOperand(0), in ExpandNode()
2870 Node->getOperand(0).getValueType(), in ExpandNode()
2871 Node->getValueType(0), dl))) in ExpandNode()
2875 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); in ExpandNode()
2876 EVT VT = Node->getValueType(0); in ExpandNode()
2887 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One); in ExpandNode()
2900 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), in ExpandNode()
2901 Node->getOperand(0), ShiftCst); in ExpandNode()
2902 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); in ExpandNode()
2908 if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) { in ExpandNode()
2910 if (Node->isStrictFPOpcode()) in ExpandNode()
2917 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) { in ExpandNode()
2919 if (Node->isStrictFPOpcode()) in ExpandNode()
2924 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) in ExpandNode()
2928 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) { in ExpandNode()
2929 ReplaceNode(Node, Tmp1.getNode()); in ExpandNode()
2935 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) in ExpandNode()
2939 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) { in ExpandNode()
2941 DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2); in ExpandNode()
2943 ReplaceNodeWithValue(SDValue(Node, 0), Tmp1); in ExpandNode()
2950 Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG)); in ExpandNode()
2953 Results.push_back(DAG.expandVAArg(Node)); in ExpandNode()
2957 Results.push_back(DAG.expandVACopy(Node)); in ExpandNode()
2960 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1) in ExpandNode()
2962 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), in ExpandNode()
2963 Node->getOperand(0)); in ExpandNode()
2965 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0)); in ExpandNode()
2969 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0))); in ExpandNode()
2972 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0))); in ExpandNode()
2975 Results.push_back(ExpandVectorBuildThroughStack(Node)); in ExpandNode()
2978 Results.push_back(ExpandSCALAR_TO_VECTOR(Node)); in ExpandNode()
2981 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0), in ExpandNode()
2982 Node->getOperand(1), in ExpandNode()
2983 Node->getOperand(2), dl)); in ExpandNode()
2987 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); in ExpandNode()
2989 EVT VT = Node->getValueType(0); in ExpandNode()
2991 SDValue Op0 = Node->getOperand(0); in ExpandNode()
2992 SDValue Op1 = Node->getOperand(1); in ExpandNode()
3055 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3060 Results.push_back(TLI.expandVectorSplice(Node, DAG)); in ExpandNode()
3064 EVT OpTy = Node->getOperand(0).getValueType(); in ExpandNode()
3065 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) { in ExpandNode()
3067 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), in ExpandNode()
3070 Node->getOperand(0).getValueType(), in ExpandNode()
3072 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3075 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), in ExpandNode()
3076 Node->getOperand(0)); in ExpandNode()
3085 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP, in ExpandNode()
3086 Node->getValueType(0))); in ExpandNode()
3089 Results.push_back(DAG.getUNDEF(Node->getValueType(0))); in ExpandNode()
3090 Results.push_back(Node->getOperand(0)); in ExpandNode()
3097 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP, in ExpandNode()
3098 Node->getOperand(1))); in ExpandNode()
3100 Results.push_back(Node->getOperand(0)); in ExpandNode()
3104 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); in ExpandNode()
3108 Results.push_back(ExpandFCOPYSIGN(Node)); in ExpandNode()
3111 Results.push_back(ExpandFNEG(Node)); in ExpandNode()
3114 Results.push_back(ExpandFABS(Node)); in ExpandNode()
3122 switch (Node->getOpcode()) { in ExpandNode()
3129 Tmp1 = Node->getOperand(0); in ExpandNode()
3130 Tmp2 = Node->getOperand(1); in ExpandNode()
3137 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) in ExpandNode()
3143 EVT VT = Node->getValueType(0); in ExpandNode()
3147 isSinCosLibcallAvailable(Node, TLI)) in ExpandNode()
3148 && useSinCos(Node)) { in ExpandNode()
3150 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); in ExpandNode()
3151 if (Node->getOpcode() == ISD::FCOS) in ExpandNode()
3161 if (Node->getValueType(0) != MVT::f32) { in ExpandNode()
3166 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); in ExpandNode()
3168 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res)); in ExpandNode()
3172 if (Node->getValueType(0) != MVT::f32) { in ExpandNode()
3178 {Node->getOperand(0), Node->getOperand(1)}); in ExpandNode()
3180 {Node->getValueType(0), MVT::Other}, in ExpandNode()
3189 SDValue Op = Node->getOperand(0); in ExpandNode()
3198 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal)); in ExpandNode()
3203 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); in ExpandNode()
3206 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0), in ExpandNode()
3212 ConstantSDNode *CP = cast<ConstantSDNode>(Node); in ExpandNode()
3217 EVT VT = Node->getValueType(0); in ExpandNode()
3220 const SDNodeFlags Flags = Node->getFlags(); in ExpandNode()
3221 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); in ExpandNode()
3222 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags); in ExpandNode()
3228 EVT VT = Node->getValueType(0); in ExpandNode()
3232 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1), in ExpandNode()
3236 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); in ExpandNode()
3241 if (TLI.expandREM(Node, Tmp1, DAG)) in ExpandNode()
3246 bool isSigned = Node->getOpcode() == ISD::SDIV; in ExpandNode()
3248 EVT VT = Node->getValueType(0); in ExpandNode()
3251 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), in ExpandNode()
3252 Node->getOperand(1)); in ExpandNode()
3260 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI; in ExpandNode()
3261 EVT VT = Node->getValueType(0); in ExpandNode()
3264 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), in ExpandNode()
3265 Node->getOperand(1)); in ExpandNode()
3271 SDValue LHS = Node->getOperand(0); in ExpandNode()
3272 SDValue RHS = Node->getOperand(1); in ExpandNode()
3275 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS; in ExpandNode()
3286 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves, in ExpandNode()
3303 EVT VT = Node->getValueType(0); in ExpandNode()
3325 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0), in ExpandNode()
3326 Node->getOperand(1))); in ExpandNode()
3336 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG, in ExpandNode()
3350 if (TLI.expandFunnelShift(Node, Tmp1, DAG)) in ExpandNode()
3355 if (TLI.expandROT(Node, true /*AllowVectorOps*/, Tmp1, DAG)) in ExpandNode()
3362 Results.push_back(TLI.expandAddSubSat(Node, DAG)); in ExpandNode()
3366 Results.push_back(TLI.expandShlSat(Node, DAG)); in ExpandNode()
3372 Results.push_back(TLI.expandFixedPointMul(Node, DAG)); in ExpandNode()
3378 if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node), in ExpandNode()
3379 Node->getOperand(0), in ExpandNode()
3380 Node->getOperand(1), in ExpandNode()
3381 Node->getConstantOperandVal(2), in ExpandNode()
3396 SDValue LHS = Node->getOperand(0); in ExpandNode()
3397 SDValue RHS = Node->getOperand(1); in ExpandNode()
3398 SDValue Carry = Node->getOperand(2); in ExpandNode()
3400 bool IsAdd = Node->getOpcode() == ISD::ADDCARRY; in ExpandNode()
3408 EVT CarryType = Node->getValueType(1); in ExpandNode()
3409 EVT SetCCType = getSetCCResultType(Node->getValueType(0)); in ExpandNode()
3440 TLI.expandSADDSUBO(Node, Result, Overflow, DAG); in ExpandNode()
3448 TLI.expandUADDSUBO(Node, Result, Overflow, DAG); in ExpandNode()
3456 if (TLI.expandMULO(Node, Result, Overflow, DAG)) { in ExpandNode()
3463 EVT PairTy = Node->getValueType(0); in ExpandNode()
3464 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); in ExpandNode()
3465 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); in ExpandNode()
3474 Tmp1 = Node->getOperand(0); in ExpandNode()
3475 Tmp2 = Node->getOperand(1); in ExpandNode()
3476 Tmp3 = Node->getOperand(2); in ExpandNode()
3486 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
3490 SDValue Chain = Node->getOperand(0); in ExpandNode()
3491 SDValue Table = Node->getOperand(1); in ExpandNode()
3492 SDValue Index = Node->getOperand(2); in ExpandNode()
3534 Tmp1 = Node->getOperand(0); in ExpandNode()
3535 Tmp2 = Node->getOperand(1); in ExpandNode()
3540 Node->getOperand(2)); in ExpandNode()
3554 Node->getOperand(2)); in ExpandNode()
3561 bool IsStrict = Node->getOpcode() != ISD::SETCC; in ExpandNode()
3562 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS; in ExpandNode()
3563 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ExpandNode()
3565 Tmp1 = Node->getOperand(0 + Offset); in ExpandNode()
3566 Tmp2 = Node->getOperand(1 + Offset); in ExpandNode()
3567 Tmp3 = Node->getOperand(2 + Offset); in ExpandNode()
3569 TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3, in ExpandNode()
3576 Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), in ExpandNode()
3577 Tmp1, Tmp2, Tmp3, Node->getFlags()); in ExpandNode()
3597 EVT VT = Node->getValueType(0); in ExpandNode()
3612 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
3618 Tmp1 = Node->getOperand(0); // LHS in ExpandNode()
3619 Tmp2 = Node->getOperand(1); // RHS in ExpandNode()
3620 Tmp3 = Node->getOperand(2); // True in ExpandNode()
3621 Tmp4 = Node->getOperand(3); // False in ExpandNode()
3622 EVT VT = Node->getValueType(0); in ExpandNode()
3624 SDValue CC = Node->getOperand(4); in ExpandNode()
3635 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags()); in ExpandNode()
3650 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
3660 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
3679 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), in ExpandNode()
3684 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
3687 Tmp1->setFlags(Node->getFlags()); in ExpandNode()
3695 Tmp1 = Node->getOperand(0); // Chain in ExpandNode()
3696 Tmp2 = Node->getOperand(2); // LHS in ExpandNode()
3697 Tmp3 = Node->getOperand(3); // RHS in ExpandNode()
3698 Tmp4 = Node->getOperand(1); // CC in ExpandNode()
3711 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
3712 Tmp4, Tmp2, Tmp3, Node->getOperand(4)); in ExpandNode()
3716 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, in ExpandNode()
3717 Tmp2, Tmp3, Node->getOperand(4)); in ExpandNode()
3723 Results.push_back(ExpandBUILD_VECTOR(Node)); in ExpandNode()
3726 Results.push_back(ExpandSPLAT_VECTOR(Node)); in ExpandNode()
3732 EVT VT = Node->getValueType(0); in ExpandNode()
3741 Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl)); in ExpandNode()
3744 Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl)); in ExpandNode()
3745 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, in ExpandNode()
3749 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars); in ExpandNode()
3766 Results.push_back(TLI.expandVecReduce(Node, DAG)); in ExpandNode()
3783 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) { in ExpandNode()
3789 switch (Node->getOpcode()) { in ExpandNode()
3791 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
3792 Node->getValueType(0)) in ExpandNode()
3798 ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal) in ExpandNode()
3801 ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal) in ExpandNode()
3804 EVT VT = Node->getValueType(0); in ExpandNode()
3805 const SDNodeFlags Flags = Node->getFlags(); in ExpandNode()
3806 SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags); in ExpandNode()
3807 SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(), in ExpandNode()
3808 {Node->getOperand(0), Node->getOperand(1), Neg}, in ExpandNode()
3823 if (TLI.getStrictFPOperationAction(Node->getOpcode(), in ExpandNode()
3824 Node->getOperand(1).getValueType()) in ExpandNode()
3838 ReplaceNode(Node, Results.data()); in ExpandNode()
3842 void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) { in ConvertNodeToLibcall() argument
3845 SDLoc dl(Node); in ConvertNodeToLibcall()
3847 unsigned Opc = Node->getOpcode(); in ConvertNodeToLibcall()
3856 .setChain(Node->getOperand(0)) in ConvertNodeToLibcall()
3884 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); in ConvertNodeToLibcall()
3885 AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering(); in ConvertNodeToLibcall()
3887 EVT RetVT = Node->getValueType(0); in ConvertNodeToLibcall()
3892 Ops.append(Node->op_begin() + 2, Node->op_end()); in ConvertNodeToLibcall()
3893 Ops.push_back(Node->getOperand(1)); in ConvertNodeToLibcall()
3900 Ops.append(Node->op_begin() + 1, Node->op_end()); in ConvertNodeToLibcall()
3904 SDLoc(Node), in ConvertNodeToLibcall()
3905 Node->getOperand(0)); in ConvertNodeToLibcall()
3915 .setChain(Node->getOperand(0)) in ConvertNodeToLibcall()
3927 ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64, in ConvertNodeToLibcall()
3933 ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64, in ConvertNodeToLibcall()
3939 ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64, in ConvertNodeToLibcall()
3944 ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64, in ConvertNodeToLibcall()
3950 ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64, in ConvertNodeToLibcall()
3956 ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64, in ConvertNodeToLibcall()
3962 ExpandSinCosLibCall(Node, Results); in ConvertNodeToLibcall()
3966 ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80, in ConvertNodeToLibcall()
3971 ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80, in ConvertNodeToLibcall()
3976 ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80, in ConvertNodeToLibcall()
3981 ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80, in ConvertNodeToLibcall()
3986 ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80, in ConvertNodeToLibcall()
3991 ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, in ConvertNodeToLibcall()
3997 ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, in ConvertNodeToLibcall()
4003 ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64, in ConvertNodeToLibcall()
4009 ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64, in ConvertNodeToLibcall()
4015 ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32, in ConvertNodeToLibcall()
4023 ExpandFPLibCall(Node, RTLIB::ROUND_F32, in ConvertNodeToLibcall()
4031 ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32, in ConvertNodeToLibcall()
4039 RTLIB::Libcall LC = RTLIB::getPOWI(Node->getSimpleValueType(0)); in ConvertNodeToLibcall()
4043 SDValue Exponent = DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), in ConvertNodeToLibcall()
4044 Node->getValueType(0), in ConvertNodeToLibcall()
4045 Node->getOperand(1)); in ConvertNodeToLibcall()
4046 Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node), in ConvertNodeToLibcall()
4047 Node->getValueType(0), Node->getOperand(0), in ConvertNodeToLibcall()
4051 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0; in ConvertNodeToLibcall()
4054 Node->getOperand(1 + Offset).getValueType().getSizeInBits(); in ConvertNodeToLibcall()
4059 Results.push_back(DAG.getUNDEF(Node->getValueType(0))); in ConvertNodeToLibcall()
4062 ExpandFPLibCall(Node, LC, Results); in ConvertNodeToLibcall()
4067 ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80, in ConvertNodeToLibcall()
4072 ExpandArgFPLibCall(Node, RTLIB::LROUND_F32, in ConvertNodeToLibcall()
4079 ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32, in ConvertNodeToLibcall()
4086 ExpandArgFPLibCall(Node, RTLIB::LRINT_F32, in ConvertNodeToLibcall()
4093 ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32, in ConvertNodeToLibcall()
4100 ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64, in ConvertNodeToLibcall()
4106 ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, in ConvertNodeToLibcall()
4112 ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64, in ConvertNodeToLibcall()
4118 ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64, in ConvertNodeToLibcall()
4124 ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64, in ConvertNodeToLibcall()
4129 if (Node->getValueType(0) == MVT::f32) { in ConvertNodeToLibcall()
4130 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false)); in ConvertNodeToLibcall()
4134 if (Node->getValueType(0) == MVT::f32) { in ConvertNodeToLibcall()
4137 DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions, in ConvertNodeToLibcall()
4138 SDLoc(Node), Node->getOperand(0)); in ConvertNodeToLibcall()
4146 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16); in ConvertNodeToLibcall()
4148 Results.push_back(ExpandLibCall(LC, Node, false)); in ConvertNodeToLibcall()
4156 bool IsStrict = Node->isStrictFPOpcode(); in ConvertNodeToLibcall()
4157 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP || in ConvertNodeToLibcall()
4158 Node->getOpcode() == ISD::STRICT_SINT_TO_FP; in ConvertNodeToLibcall()
4159 EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType(); in ConvertNodeToLibcall()
4160 EVT RVT = Node->getValueType(0); in ConvertNodeToLibcall()
4162 SDLoc dl(Node); in ConvertNodeToLibcall()
4179 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ConvertNodeToLibcall()
4182 NVT, Node->getOperand(IsStrict ? 1 : 0)); in ConvertNodeToLibcall()
4197 bool IsStrict = Node->isStrictFPOpcode(); in ConvertNodeToLibcall()
4198 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT || in ConvertNodeToLibcall()
4199 Node->getOpcode() == ISD::STRICT_FP_TO_SINT; in ConvertNodeToLibcall()
4201 SDValue Op = Node->getOperand(IsStrict ? 1 : 0); in ConvertNodeToLibcall()
4203 EVT RVT = Node->getValueType(0); in ConvertNodeToLibcall()
4205 SDLoc dl(Node); in ConvertNodeToLibcall()
4222 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ConvertNodeToLibcall()
4241 bool IsStrict = Node->isStrictFPOpcode(); in ConvertNodeToLibcall()
4242 SDValue Op = Node->getOperand(IsStrict ? 1 : 0); in ConvertNodeToLibcall()
4243 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue(); in ConvertNodeToLibcall()
4244 EVT VT = Node->getValueType(0); in ConvertNodeToLibcall()
4245 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1)) in ConvertNodeToLibcall()
4254 TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, SDLoc(Node), Chain); in ConvertNodeToLibcall()
4262 ExpandLibCall(RTLIB::getFPEXT(Node->getOperand(0).getValueType(), in ConvertNodeToLibcall()
4263 Node->getValueType(0)), in ConvertNodeToLibcall()
4264 Node, false)); in ConvertNodeToLibcall()
4270 Node->getOpcode() == ISD::STRICT_FP_TO_FP16 in ConvertNodeToLibcall()
4271 ? RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16) in ConvertNodeToLibcall()
4272 : RTLIB::getFPEXT(Node->getOperand(1).getValueType(), in ConvertNodeToLibcall()
4273 Node->getValueType(0)); in ConvertNodeToLibcall()
4278 TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1), in ConvertNodeToLibcall()
4279 CallOptions, SDLoc(Node), Node->getOperand(0)); in ConvertNodeToLibcall()
4286 ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64, in ConvertNodeToLibcall()
4291 Results.push_back(ExpandIntLibCall(Node, true, in ConvertNodeToLibcall()
4297 Results.push_back(ExpandIntLibCall(Node, false, in ConvertNodeToLibcall()
4303 Results.push_back(ExpandIntLibCall(Node, true, in ConvertNodeToLibcall()
4309 Results.push_back(ExpandIntLibCall(Node, false, in ConvertNodeToLibcall()
4317 ExpandDivRemLibCall(Node, Results); in ConvertNodeToLibcall()
4320 Results.push_back(ExpandIntLibCall(Node, false, in ConvertNodeToLibcall()
4326 switch (Node->getSimpleValueType(0).SimpleTy) { in ConvertNodeToLibcall()
4330 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false)); in ConvertNodeToLibcall()
4333 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false)); in ConvertNodeToLibcall()
4336 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false)); in ConvertNodeToLibcall()
4345 ReplaceNode(Node, Results.data()); in ConvertNodeToLibcall()
4360 void SelectionDAGLegalize::PromoteNode(SDNode *Node) { in PromoteNode() argument
4363 MVT OVT = Node->getSimpleValueType(0); in PromoteNode()
4364 if (Node->getOpcode() == ISD::UINT_TO_FP || in PromoteNode()
4365 Node->getOpcode() == ISD::SINT_TO_FP || in PromoteNode()
4366 Node->getOpcode() == ISD::SETCC || in PromoteNode()
4367 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT || in PromoteNode()
4368 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) { in PromoteNode()
4369 OVT = Node->getOperand(0).getSimpleValueType(); in PromoteNode()
4371 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP || in PromoteNode()
4372 Node->getOpcode() == ISD::STRICT_SINT_TO_FP || in PromoteNode()
4373 Node->getOpcode() == ISD::STRICT_FSETCC || in PromoteNode()
4374 Node->getOpcode() == ISD::STRICT_FSETCCS) in PromoteNode()
4375 OVT = Node->getOperand(1).getSimpleValueType(); in PromoteNode()
4376 if (Node->getOpcode() == ISD::BR_CC || in PromoteNode()
4377 Node->getOpcode() == ISD::SELECT_CC) in PromoteNode()
4378 OVT = Node->getOperand(2).getSimpleValueType(); in PromoteNode()
4379 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); in PromoteNode()
4380 SDLoc dl(Node); in PromoteNode()
4382 switch (Node->getOpcode()) { in PromoteNode()
4389 if (Node->getOpcode() == ISD::CTTZ || in PromoteNode()
4390 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF) in PromoteNode()
4391 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4393 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4395 if (Node->getOpcode() == ISD::CTTZ) { in PromoteNode()
4406 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
4407 if (Node->getOpcode() == ISD::CTLZ || in PromoteNode()
4408 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) { in PromoteNode()
4419 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4420 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
4433 PromoteLegalFP_TO_INT(Node, dl, Results); in PromoteNode()
4437 Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl)); in PromoteNode()
4443 PromoteLegalINT_TO_FP(Node, dl, Results); in PromoteNode()
4446 SDValue Chain = Node->getOperand(0); // Get the chain. in PromoteNode()
4447 SDValue Ptr = Node->getOperand(1); // Get the pointer. in PromoteNode()
4459 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2), in PromoteNode()
4460 Node->getConstantOperandVal(3)); in PromoteNode()
4467 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2); in PromoteNode()
4468 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); in PromoteNode()
4473 ReplacedNode(Node); in PromoteNode()
4491 switch (Node->getOpcode()) { in PromoteNode()
4507 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
4508 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
4510 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); in PromoteNode()
4517 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND in PromoteNode()
4519 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
4520 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
4534 if (Node->getValueType(0).isVector() || in PromoteNode()
4535 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) { in PromoteNode()
4538 } else if (Node->getValueType(0).isInteger()) { in PromoteNode()
4545 Tmp1 = Node->getOperand(0); in PromoteNode()
4547 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
4548 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); in PromoteNode()
4551 Tmp1->setFlags(Node->getFlags()); in PromoteNode()
4553 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); in PromoteNode()
4555 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1, in PromoteNode()
4561 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); in PromoteNode()
4564 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); in PromoteNode()
4565 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); in PromoteNode()
4574 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4575 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
4577 Node->getOperand(2)); in PromoteNode()
4582 SDValue Cond = Node->getOperand(4); in PromoteNode()
4585 MVT CVT = Node->getSimpleValueType(0); in PromoteNode()
4595 Tmp1 = Node->getOperand(0); in PromoteNode()
4596 Tmp2 = Node->getOperand(1); in PromoteNode()
4598 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
4599 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
4602 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); in PromoteNode()
4603 Tmp4 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); in PromoteNode()
4606 Node->getFlags()); in PromoteNode()
4623 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); in PromoteNode()
4626 if (Node->isStrictFPOpcode()) { in PromoteNode()
4627 SDValue InChain = Node->getOperand(0); in PromoteNode()
4629 DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT); in PromoteNode()
4631 DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT); in PromoteNode()
4634 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); in PromoteNode()
4635 Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs, in PromoteNode()
4636 {OutChain, Tmp1, Tmp2, Node->getOperand(3)}, in PromoteNode()
4637 Node->getFlags())); in PromoteNode()
4641 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); in PromoteNode()
4642 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); in PromoteNode()
4643 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1, in PromoteNode()
4644 Tmp2, Node->getOperand(2), Node->getFlags())); in PromoteNode()
4651 cast<CondCodeSDNode>(Node->getOperand(1))->get(); in PromoteNode()
4654 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); in PromoteNode()
4655 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); in PromoteNode()
4656 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), in PromoteNode()
4657 Node->getOperand(0), Node->getOperand(1), in PromoteNode()
4658 Tmp1, Tmp2, Node->getOperand(4))); in PromoteNode()
4669 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4670 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
4671 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, in PromoteNode()
4672 Node->getFlags()); in PromoteNode()
4679 {Node->getOperand(0), Node->getOperand(1)}); in PromoteNode()
4681 {Node->getOperand(0), Node->getOperand(2)}); in PromoteNode()
4684 Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
4692 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4693 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
4694 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2)); in PromoteNode()
4697 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3), in PromoteNode()
4702 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4703 Tmp2 = Node->getOperand(1); in PromoteNode()
4704 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); in PromoteNode()
4711 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN); in PromoteNode()
4733 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4734 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); in PromoteNode()
4746 {Node->getOperand(0), Node->getOperand(1)}); in PromoteNode()
4747 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other}, in PromoteNode()
4771 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) { in PromoteNode()
4772 SDValue Op = Node->getOperand(I); in PromoteNode()
4776 SDLoc SL(Node); in PromoteNode()
4804 SDValue Idx = Node->getOperand(1); in PromoteNode()
4806 SDLoc SL(Node); in PromoteNode()
4810 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); in PromoteNode()
4850 SDValue Val = Node->getOperand(1); in PromoteNode()
4851 SDValue Idx = Node->getOperand(2); in PromoteNode()
4853 SDLoc SL(Node); in PromoteNode()
4858 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); in PromoteNode()
4888 SDValue Val = Node->getOperand(0); in PromoteNode()
4889 SDLoc SL(Node); in PromoteNode()
4905 AtomicSDNode *AM = cast<AtomicSDNode>(Node); in PromoteNode()
4906 SDLoc SL(Node); in PromoteNode()
4927 ReplaceNode(Node, Results.data()); in PromoteNode()