Lines Matching refs:ISD
63 setOperationAction(ISD::LOAD, MVT::f32, Promote); in AMDGPUTargetLowering()
64 AddPromotedToType(ISD::LOAD, MVT::f32, MVT::i32); in AMDGPUTargetLowering()
66 setOperationAction(ISD::LOAD, MVT::v2f32, Promote); in AMDGPUTargetLowering()
67 AddPromotedToType(ISD::LOAD, MVT::v2f32, MVT::v2i32); in AMDGPUTargetLowering()
69 setOperationAction(ISD::LOAD, MVT::v3f32, Promote); in AMDGPUTargetLowering()
70 AddPromotedToType(ISD::LOAD, MVT::v3f32, MVT::v3i32); in AMDGPUTargetLowering()
72 setOperationAction(ISD::LOAD, MVT::v4f32, Promote); in AMDGPUTargetLowering()
73 AddPromotedToType(ISD::LOAD, MVT::v4f32, MVT::v4i32); in AMDGPUTargetLowering()
75 setOperationAction(ISD::LOAD, MVT::v5f32, Promote); in AMDGPUTargetLowering()
76 AddPromotedToType(ISD::LOAD, MVT::v5f32, MVT::v5i32); in AMDGPUTargetLowering()
78 setOperationAction(ISD::LOAD, MVT::v6f32, Promote); in AMDGPUTargetLowering()
79 AddPromotedToType(ISD::LOAD, MVT::v6f32, MVT::v6i32); in AMDGPUTargetLowering()
81 setOperationAction(ISD::LOAD, MVT::v7f32, Promote); in AMDGPUTargetLowering()
82 AddPromotedToType(ISD::LOAD, MVT::v7f32, MVT::v7i32); in AMDGPUTargetLowering()
84 setOperationAction(ISD::LOAD, MVT::v8f32, Promote); in AMDGPUTargetLowering()
85 AddPromotedToType(ISD::LOAD, MVT::v8f32, MVT::v8i32); in AMDGPUTargetLowering()
87 setOperationAction(ISD::LOAD, MVT::v16f32, Promote); in AMDGPUTargetLowering()
88 AddPromotedToType(ISD::LOAD, MVT::v16f32, MVT::v16i32); in AMDGPUTargetLowering()
90 setOperationAction(ISD::LOAD, MVT::v32f32, Promote); in AMDGPUTargetLowering()
91 AddPromotedToType(ISD::LOAD, MVT::v32f32, MVT::v32i32); in AMDGPUTargetLowering()
93 setOperationAction(ISD::LOAD, MVT::i64, Promote); in AMDGPUTargetLowering()
94 AddPromotedToType(ISD::LOAD, MVT::i64, MVT::v2i32); in AMDGPUTargetLowering()
96 setOperationAction(ISD::LOAD, MVT::v2i64, Promote); in AMDGPUTargetLowering()
97 AddPromotedToType(ISD::LOAD, MVT::v2i64, MVT::v4i32); in AMDGPUTargetLowering()
99 setOperationAction(ISD::LOAD, MVT::f64, Promote); in AMDGPUTargetLowering()
100 AddPromotedToType(ISD::LOAD, MVT::f64, MVT::v2i32); in AMDGPUTargetLowering()
102 setOperationAction(ISD::LOAD, MVT::v2f64, Promote); in AMDGPUTargetLowering()
103 AddPromotedToType(ISD::LOAD, MVT::v2f64, MVT::v4i32); in AMDGPUTargetLowering()
105 setOperationAction(ISD::LOAD, MVT::v3i64, Promote); in AMDGPUTargetLowering()
106 AddPromotedToType(ISD::LOAD, MVT::v3i64, MVT::v6i32); in AMDGPUTargetLowering()
108 setOperationAction(ISD::LOAD, MVT::v4i64, Promote); in AMDGPUTargetLowering()
109 AddPromotedToType(ISD::LOAD, MVT::v4i64, MVT::v8i32); in AMDGPUTargetLowering()
111 setOperationAction(ISD::LOAD, MVT::v3f64, Promote); in AMDGPUTargetLowering()
112 AddPromotedToType(ISD::LOAD, MVT::v3f64, MVT::v6i32); in AMDGPUTargetLowering()
114 setOperationAction(ISD::LOAD, MVT::v4f64, Promote); in AMDGPUTargetLowering()
115 AddPromotedToType(ISD::LOAD, MVT::v4f64, MVT::v8i32); in AMDGPUTargetLowering()
117 setOperationAction(ISD::LOAD, MVT::v8i64, Promote); in AMDGPUTargetLowering()
118 AddPromotedToType(ISD::LOAD, MVT::v8i64, MVT::v16i32); in AMDGPUTargetLowering()
120 setOperationAction(ISD::LOAD, MVT::v8f64, Promote); in AMDGPUTargetLowering()
121 AddPromotedToType(ISD::LOAD, MVT::v8f64, MVT::v16i32); in AMDGPUTargetLowering()
123 setOperationAction(ISD::LOAD, MVT::v16i64, Promote); in AMDGPUTargetLowering()
124 AddPromotedToType(ISD::LOAD, MVT::v16i64, MVT::v32i32); in AMDGPUTargetLowering()
126 setOperationAction(ISD::LOAD, MVT::v16f64, Promote); in AMDGPUTargetLowering()
127 AddPromotedToType(ISD::LOAD, MVT::v16f64, MVT::v32i32); in AMDGPUTargetLowering()
132 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::i64, VT, in AMDGPUTargetLowering()
139 for (auto Op : {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}) { in AMDGPUTargetLowering()
150 setLoadExtAction({ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}, VT, MemVT, in AMDGPUTargetLowering()
153 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand); in AMDGPUTargetLowering()
154 setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2f16, Expand); in AMDGPUTargetLowering()
155 setLoadExtAction(ISD::EXTLOAD, MVT::v3f32, MVT::v3f16, Expand); in AMDGPUTargetLowering()
156 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Expand); in AMDGPUTargetLowering()
157 setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Expand); in AMDGPUTargetLowering()
158 setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16f16, Expand); in AMDGPUTargetLowering()
159 setLoadExtAction(ISD::EXTLOAD, MVT::v32f32, MVT::v32f16, Expand); in AMDGPUTargetLowering()
161 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand); in AMDGPUTargetLowering()
162 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Expand); in AMDGPUTargetLowering()
163 setLoadExtAction(ISD::EXTLOAD, MVT::v3f64, MVT::v3f32, Expand); in AMDGPUTargetLowering()
164 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Expand); in AMDGPUTargetLowering()
165 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f32, Expand); in AMDGPUTargetLowering()
166 setLoadExtAction(ISD::EXTLOAD, MVT::v16f64, MVT::v16f32, Expand); in AMDGPUTargetLowering()
168 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand); in AMDGPUTargetLowering()
169 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Expand); in AMDGPUTargetLowering()
170 setLoadExtAction(ISD::EXTLOAD, MVT::v3f64, MVT::v3f16, Expand); in AMDGPUTargetLowering()
171 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Expand); in AMDGPUTargetLowering()
172 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Expand); in AMDGPUTargetLowering()
173 setLoadExtAction(ISD::EXTLOAD, MVT::v16f64, MVT::v16f16, Expand); in AMDGPUTargetLowering()
175 setOperationAction(ISD::STORE, MVT::f32, Promote); in AMDGPUTargetLowering()
176 AddPromotedToType(ISD::STORE, MVT::f32, MVT::i32); in AMDGPUTargetLowering()
178 setOperationAction(ISD::STORE, MVT::v2f32, Promote); in AMDGPUTargetLowering()
179 AddPromotedToType(ISD::STORE, MVT::v2f32, MVT::v2i32); in AMDGPUTargetLowering()
181 setOperationAction(ISD::STORE, MVT::v3f32, Promote); in AMDGPUTargetLowering()
182 AddPromotedToType(ISD::STORE, MVT::v3f32, MVT::v3i32); in AMDGPUTargetLowering()
184 setOperationAction(ISD::STORE, MVT::v4f32, Promote); in AMDGPUTargetLowering()
185 AddPromotedToType(ISD::STORE, MVT::v4f32, MVT::v4i32); in AMDGPUTargetLowering()
187 setOperationAction(ISD::STORE, MVT::v5f32, Promote); in AMDGPUTargetLowering()
188 AddPromotedToType(ISD::STORE, MVT::v5f32, MVT::v5i32); in AMDGPUTargetLowering()
190 setOperationAction(ISD::STORE, MVT::v6f32, Promote); in AMDGPUTargetLowering()
191 AddPromotedToType(ISD::STORE, MVT::v6f32, MVT::v6i32); in AMDGPUTargetLowering()
193 setOperationAction(ISD::STORE, MVT::v7f32, Promote); in AMDGPUTargetLowering()
194 AddPromotedToType(ISD::STORE, MVT::v7f32, MVT::v7i32); in AMDGPUTargetLowering()
196 setOperationAction(ISD::STORE, MVT::v8f32, Promote); in AMDGPUTargetLowering()
197 AddPromotedToType(ISD::STORE, MVT::v8f32, MVT::v8i32); in AMDGPUTargetLowering()
199 setOperationAction(ISD::STORE, MVT::v16f32, Promote); in AMDGPUTargetLowering()
200 AddPromotedToType(ISD::STORE, MVT::v16f32, MVT::v16i32); in AMDGPUTargetLowering()
202 setOperationAction(ISD::STORE, MVT::v32f32, Promote); in AMDGPUTargetLowering()
203 AddPromotedToType(ISD::STORE, MVT::v32f32, MVT::v32i32); in AMDGPUTargetLowering()
205 setOperationAction(ISD::STORE, MVT::i64, Promote); in AMDGPUTargetLowering()
206 AddPromotedToType(ISD::STORE, MVT::i64, MVT::v2i32); in AMDGPUTargetLowering()
208 setOperationAction(ISD::STORE, MVT::v2i64, Promote); in AMDGPUTargetLowering()
209 AddPromotedToType(ISD::STORE, MVT::v2i64, MVT::v4i32); in AMDGPUTargetLowering()
211 setOperationAction(ISD::STORE, MVT::f64, Promote); in AMDGPUTargetLowering()
212 AddPromotedToType(ISD::STORE, MVT::f64, MVT::v2i32); in AMDGPUTargetLowering()
214 setOperationAction(ISD::STORE, MVT::v2f64, Promote); in AMDGPUTargetLowering()
215 AddPromotedToType(ISD::STORE, MVT::v2f64, MVT::v4i32); in AMDGPUTargetLowering()
217 setOperationAction(ISD::STORE, MVT::v3i64, Promote); in AMDGPUTargetLowering()
218 AddPromotedToType(ISD::STORE, MVT::v3i64, MVT::v6i32); in AMDGPUTargetLowering()
220 setOperationAction(ISD::STORE, MVT::v3f64, Promote); in AMDGPUTargetLowering()
221 AddPromotedToType(ISD::STORE, MVT::v3f64, MVT::v6i32); in AMDGPUTargetLowering()
223 setOperationAction(ISD::STORE, MVT::v4i64, Promote); in AMDGPUTargetLowering()
224 AddPromotedToType(ISD::STORE, MVT::v4i64, MVT::v8i32); in AMDGPUTargetLowering()
226 setOperationAction(ISD::STORE, MVT::v4f64, Promote); in AMDGPUTargetLowering()
227 AddPromotedToType(ISD::STORE, MVT::v4f64, MVT::v8i32); in AMDGPUTargetLowering()
229 setOperationAction(ISD::STORE, MVT::v8i64, Promote); in AMDGPUTargetLowering()
230 AddPromotedToType(ISD::STORE, MVT::v8i64, MVT::v16i32); in AMDGPUTargetLowering()
232 setOperationAction(ISD::STORE, MVT::v8f64, Promote); in AMDGPUTargetLowering()
233 AddPromotedToType(ISD::STORE, MVT::v8f64, MVT::v16i32); in AMDGPUTargetLowering()
235 setOperationAction(ISD::STORE, MVT::v16i64, Promote); in AMDGPUTargetLowering()
236 AddPromotedToType(ISD::STORE, MVT::v16i64, MVT::v32i32); in AMDGPUTargetLowering()
238 setOperationAction(ISD::STORE, MVT::v16f64, Promote); in AMDGPUTargetLowering()
239 AddPromotedToType(ISD::STORE, MVT::v16f64, MVT::v32i32); in AMDGPUTargetLowering()
286 setOperationAction(ISD::Constant, {MVT::i32, MVT::i64}, Legal); in AMDGPUTargetLowering()
287 setOperationAction(ISD::ConstantFP, {MVT::f32, MVT::f64}, Legal); in AMDGPUTargetLowering()
289 setOperationAction({ISD::BR_JT, ISD::BRIND}, MVT::Other, Expand); in AMDGPUTargetLowering()
292 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); in AMDGPUTargetLowering()
296 setOperationAction({ISD::FCEIL, ISD::FEXP2, ISD::FPOW, ISD::FLOG2, ISD::FABS, in AMDGPUTargetLowering()
297 ISD::FFLOOR, ISD::FRINT, ISD::FTRUNC, ISD::FMINNUM, in AMDGPUTargetLowering()
298 ISD::FMAXNUM}, in AMDGPUTargetLowering()
301 setOperationAction(ISD::FROUND, {MVT::f32, MVT::f64}, Custom); in AMDGPUTargetLowering()
303 setOperationAction({ISD::FLOG, ISD::FLOG10, ISD::FEXP}, MVT::f32, Custom); in AMDGPUTargetLowering()
305 setOperationAction(ISD::FNEARBYINT, {MVT::f32, MVT::f64}, Custom); in AMDGPUTargetLowering()
307 setOperationAction(ISD::FREM, {MVT::f16, MVT::f32, MVT::f64}, Custom); in AMDGPUTargetLowering()
310 setOperationAction(ISD::FSUB, MVT::f64, Expand); in AMDGPUTargetLowering()
312 setOperationAction(ISD::CONCAT_VECTORS, in AMDGPUTargetLowering()
318 ISD::EXTRACT_SUBVECTOR, in AMDGPUTargetLowering()
328 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); in AMDGPUTargetLowering()
329 setOperationAction(ISD::FP_TO_FP16, {MVT::f64, MVT::f32}, Custom); in AMDGPUTargetLowering()
334 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, VT, in AMDGPUTargetLowering()
338 setOperationAction({ISD::SDIVREM, ISD::UDIVREM}, VT, Custom); in AMDGPUTargetLowering()
341 setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Expand); in AMDGPUTargetLowering()
343 setOperationAction({ISD::BSWAP, ISD::CTTZ, ISD::CTLZ}, VT, Expand); in AMDGPUTargetLowering()
346 setOperationAction({ISD::ADDC, ISD::SUBC, ISD::ADDE, ISD::SUBE}, VT, Legal); in AMDGPUTargetLowering()
350 setOperationAction(ISD::FSHR, MVT::i32, Legal); in AMDGPUTargetLowering()
353 setOperationAction(ISD::ROTL, {MVT::i32, MVT::i64}, Expand); in AMDGPUTargetLowering()
354 setOperationAction(ISD::ROTR, MVT::i64, Expand); in AMDGPUTargetLowering()
356 setOperationAction({ISD::MULHU, ISD::MULHS}, MVT::i16, Expand); in AMDGPUTargetLowering()
358 setOperationAction({ISD::MUL, ISD::MULHU, ISD::MULHS}, MVT::i64, Expand); in AMDGPUTargetLowering()
360 {ISD::UINT_TO_FP, ISD::SINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT}, in AMDGPUTargetLowering()
362 setOperationAction(ISD::SELECT_CC, MVT::i64, Expand); in AMDGPUTargetLowering()
364 setOperationAction({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, MVT::i32, in AMDGPUTargetLowering()
368 {ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF, ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, in AMDGPUTargetLowering()
376 setOperationAction({ISD::ADD, ISD::AND, ISD::FP_TO_SINT, in AMDGPUTargetLowering()
377 ISD::FP_TO_UINT, ISD::MUL, ISD::MULHU, in AMDGPUTargetLowering()
378 ISD::MULHS, ISD::OR, ISD::SHL, in AMDGPUTargetLowering()
379 ISD::SRA, ISD::SRL, ISD::ROTL, in AMDGPUTargetLowering()
380 ISD::ROTR, ISD::SUB, ISD::SINT_TO_FP, in AMDGPUTargetLowering()
381 ISD::UINT_TO_FP, ISD::SDIV, ISD::UDIV, in AMDGPUTargetLowering()
382 ISD::SREM, ISD::UREM, ISD::SMUL_LOHI, in AMDGPUTargetLowering()
383 ISD::UMUL_LOHI, ISD::SDIVREM, ISD::UDIVREM, in AMDGPUTargetLowering()
384 ISD::SELECT, ISD::VSELECT, ISD::SELECT_CC, in AMDGPUTargetLowering()
385 ISD::XOR, ISD::BSWAP, ISD::CTPOP, in AMDGPUTargetLowering()
386 ISD::CTTZ, ISD::CTLZ, ISD::VECTOR_SHUFFLE, in AMDGPUTargetLowering()
387 ISD::SETCC}, in AMDGPUTargetLowering()
396 {ISD::FABS, ISD::FMINNUM, ISD::FMAXNUM, ISD::FADD, in AMDGPUTargetLowering()
397 ISD::FCEIL, ISD::FCOS, ISD::FDIV, ISD::FEXP2, in AMDGPUTargetLowering()
398 ISD::FEXP, ISD::FLOG2, ISD::FREM, ISD::FLOG, in AMDGPUTargetLowering()
399 ISD::FLOG10, ISD::FPOW, ISD::FFLOOR, ISD::FTRUNC, in AMDGPUTargetLowering()
400 ISD::FMUL, ISD::FMA, ISD::FRINT, ISD::FNEARBYINT, in AMDGPUTargetLowering()
401 ISD::FSQRT, ISD::FSIN, ISD::FSUB, ISD::FNEG, in AMDGPUTargetLowering()
402 ISD::VSELECT, ISD::SELECT_CC, ISD::FCOPYSIGN, ISD::VECTOR_SHUFFLE, in AMDGPUTargetLowering()
403 ISD::SETCC, ISD::FCANONICALIZE}, in AMDGPUTargetLowering()
410 setOperationAction(ISD::SELECT, MVT::v2f32, Promote); in AMDGPUTargetLowering()
411 AddPromotedToType(ISD::SELECT, MVT::v2f32, MVT::v2i32); in AMDGPUTargetLowering()
413 setOperationAction(ISD::SELECT, MVT::v3f32, Promote); in AMDGPUTargetLowering()
414 AddPromotedToType(ISD::SELECT, MVT::v3f32, MVT::v3i32); in AMDGPUTargetLowering()
416 setOperationAction(ISD::SELECT, MVT::v4f32, Promote); in AMDGPUTargetLowering()
417 AddPromotedToType(ISD::SELECT, MVT::v4f32, MVT::v4i32); in AMDGPUTargetLowering()
419 setOperationAction(ISD::SELECT, MVT::v5f32, Promote); in AMDGPUTargetLowering()
420 AddPromotedToType(ISD::SELECT, MVT::v5f32, MVT::v5i32); in AMDGPUTargetLowering()
422 setOperationAction(ISD::SELECT, MVT::v6f32, Promote); in AMDGPUTargetLowering()
423 AddPromotedToType(ISD::SELECT, MVT::v6f32, MVT::v6i32); in AMDGPUTargetLowering()
425 setOperationAction(ISD::SELECT, MVT::v7f32, Promote); in AMDGPUTargetLowering()
426 AddPromotedToType(ISD::SELECT, MVT::v7f32, MVT::v7i32); in AMDGPUTargetLowering()
468 setTargetDAGCombine({ISD::BITCAST, ISD::SHL, in AMDGPUTargetLowering()
469 ISD::SRA, ISD::SRL, in AMDGPUTargetLowering()
470 ISD::TRUNCATE, ISD::MUL, in AMDGPUTargetLowering()
471 ISD::SMUL_LOHI, ISD::UMUL_LOHI, in AMDGPUTargetLowering()
472 ISD::MULHU, ISD::MULHS, in AMDGPUTargetLowering()
473 ISD::SELECT, ISD::SELECT_CC, in AMDGPUTargetLowering()
474 ISD::STORE, ISD::FADD, in AMDGPUTargetLowering()
475 ISD::FSUB, ISD::FNEG, in AMDGPUTargetLowering()
476 ISD::FABS, ISD::AssertZext, in AMDGPUTargetLowering()
477 ISD::AssertSext, ISD::INTRINSIC_WO_CHAIN}); in AMDGPUTargetLowering()
498 case ISD::FADD: in fnegFoldsIntoOp()
499 case ISD::FSUB: in fnegFoldsIntoOp()
500 case ISD::FMUL: in fnegFoldsIntoOp()
501 case ISD::FMA: in fnegFoldsIntoOp()
502 case ISD::FMAD: in fnegFoldsIntoOp()
503 case ISD::FMINNUM: in fnegFoldsIntoOp()
504 case ISD::FMAXNUM: in fnegFoldsIntoOp()
505 case ISD::FMINNUM_IEEE: in fnegFoldsIntoOp()
506 case ISD::FMAXNUM_IEEE: in fnegFoldsIntoOp()
507 case ISD::FSIN: in fnegFoldsIntoOp()
508 case ISD::FTRUNC: in fnegFoldsIntoOp()
509 case ISD::FRINT: in fnegFoldsIntoOp()
510 case ISD::FNEARBYINT: in fnegFoldsIntoOp()
511 case ISD::FCANONICALIZE: in fnegFoldsIntoOp()
543 case ISD::CopyToReg: in hasSourceMods()
544 case ISD::SELECT: in hasSourceMods()
545 case ISD::FDIV: in hasSourceMods()
546 case ISD::FREM: in hasSourceMods()
547 case ISD::INLINEASM: in hasSourceMods()
548 case ISD::INLINEASM_BR: in hasSourceMods()
550 case ISD::INTRINSIC_W_CHAIN: in hasSourceMods()
555 case ISD::BITCAST: in hasSourceMods()
557 case ISD::INTRINSIC_WO_CHAIN: { in hasSourceMods()
599 ISD::NodeType ExtendKind) const { in getTypeForExtReturn()
633 ISD::LoadExtType ExtTy, in shouldReduceLoadWidth()
706 case ISD::EntryToken: in isSDNodeAlwaysUniform()
707 case ISD::TokenFactor: in isSDNodeAlwaysUniform()
709 case ISD::INTRINSIC_WO_CHAIN: { in isSDNodeAlwaysUniform()
718 case ISD::LOAD: in isSDNodeAlwaysUniform()
734 case ISD::FMA: in getNegatedExpression()
735 case ISD::FMAD: { in getNegatedExpression()
924 const SmallVectorImpl<ISD::InputArg> &Ins) const { in analyzeFormalArgumentsCompute()
1041 const SmallVectorImpl<ISD::OutputArg> &Outs, in LowerReturn()
1096 return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); in addTokenForArgument()
1150 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG); in LowerOperation()
1151 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG); in LowerOperation()
1152 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG); in LowerOperation()
1153 case ISD::UDIVREM: return LowerUDIVREM(Op, DAG); in LowerOperation()
1154 case ISD::SDIVREM: return LowerSDIVREM(Op, DAG); in LowerOperation()
1155 case ISD::FREM: return LowerFREM(Op, DAG); in LowerOperation()
1156 case ISD::FCEIL: return LowerFCEIL(Op, DAG); in LowerOperation()
1157 case ISD::FTRUNC: return LowerFTRUNC(Op, DAG); in LowerOperation()
1158 case ISD::FRINT: return LowerFRINT(Op, DAG); in LowerOperation()
1159 case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG); in LowerOperation()
1160 case ISD::FROUND: return LowerFROUND(Op, DAG); in LowerOperation()
1161 case ISD::FFLOOR: return LowerFFLOOR(Op, DAG); in LowerOperation()
1162 case ISD::FLOG: in LowerOperation()
1164 case ISD::FLOG10: in LowerOperation()
1166 case ISD::FEXP: in LowerOperation()
1168 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); in LowerOperation()
1169 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG); in LowerOperation()
1170 case ISD::FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG); in LowerOperation()
1171 case ISD::FP_TO_SINT: in LowerOperation()
1172 case ISD::FP_TO_UINT: in LowerOperation()
1174 case ISD::CTTZ: in LowerOperation()
1175 case ISD::CTTZ_ZERO_UNDEF: in LowerOperation()
1176 case ISD::CTLZ: in LowerOperation()
1177 case ISD::CTLZ_ZERO_UNDEF: in LowerOperation()
1179 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); in LowerOperation()
1188 case ISD::SIGN_EXTEND_INREG: in ReplaceNodeResults()
1225 SDValue Trap = DAG.getNode(ISD::TRAP, DL, MVT::Other, DAG.getEntryNode()); in LowerGlobalAddress()
1226 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in LowerGlobalAddress()
1252 SDValue Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Op.getOperand(0)); in LowerCONCAT_VECTORS()
1253 SDValue Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Op.getOperand(1)); in LowerCONCAT_VECTORS()
1256 return DAG.getNode(ISD::BITCAST, SL, VT, BV); in LowerCONCAT_VECTORS()
1305 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); in combineFMinMaxLegacy()
1307 case ISD::SETOEQ: in combineFMinMaxLegacy()
1308 case ISD::SETONE: in combineFMinMaxLegacy()
1309 case ISD::SETUNE: in combineFMinMaxLegacy()
1310 case ISD::SETNE: in combineFMinMaxLegacy()
1311 case ISD::SETUEQ: in combineFMinMaxLegacy()
1312 case ISD::SETEQ: in combineFMinMaxLegacy()
1313 case ISD::SETFALSE: in combineFMinMaxLegacy()
1314 case ISD::SETFALSE2: in combineFMinMaxLegacy()
1315 case ISD::SETTRUE: in combineFMinMaxLegacy()
1316 case ISD::SETTRUE2: in combineFMinMaxLegacy()
1317 case ISD::SETUO: in combineFMinMaxLegacy()
1318 case ISD::SETO: in combineFMinMaxLegacy()
1320 case ISD::SETULE: in combineFMinMaxLegacy()
1321 case ISD::SETULT: { in combineFMinMaxLegacy()
1326 case ISD::SETOLE: in combineFMinMaxLegacy()
1327 case ISD::SETOLT: in combineFMinMaxLegacy()
1328 case ISD::SETLE: in combineFMinMaxLegacy()
1329 case ISD::SETLT: { in combineFMinMaxLegacy()
1345 case ISD::SETUGE: in combineFMinMaxLegacy()
1346 case ISD::SETUGT: { in combineFMinMaxLegacy()
1351 case ISD::SETGT: in combineFMinMaxLegacy()
1352 case ISD::SETGE: in combineFMinMaxLegacy()
1353 case ISD::SETOGE: in combineFMinMaxLegacy()
1354 case ISD::SETOGT: { in combineFMinMaxLegacy()
1363 case ISD::SETCC_INVALID: in combineFMinMaxLegacy()
1373 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in split64BitValue()
1378 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero); in split64BitValue()
1379 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One); in split64BitValue()
1387 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in getLoHalf64()
1389 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero); in getLoHalf64()
1395 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in getHiHalf64()
1397 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One); in getHiHalf64()
1426 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, in splitVector()
1429 HiVT.isVector() ? ISD::EXTRACT_SUBVECTOR : ISD::EXTRACT_VECTOR_ELT, DL, in splitVector()
1478 Join = DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, LoLoad, HiLoad); in SplitVectorLoad()
1480 Join = DAG.getNode(ISD::INSERT_SUBVECTOR, SL, VT, DAG.getUNDEF(VT), LoLoad, in SplitVectorLoad()
1483 HiVT.isVector() ? ISD::INSERT_SUBVECTOR : ISD::INSERT_VECTOR_ELT, SL, in SplitVectorLoad()
1488 SDValue Ops[] = {Join, DAG.getNode(ISD::TokenFactor, SL, MVT::Other, in SplitVectorLoad()
1522 {DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, VT, WideLoad, in WidenOrSplitVectorLoad()
1566 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoStore, HiStore); in SplitVectorStore()
1595 ISD::NodeType ToFp = Sign ? ISD::SINT_TO_FP : ISD::UINT_TO_FP; in LowerDIVREM24()
1596 ISD::NodeType ToInt = Sign ? ISD::FP_TO_SINT : ISD::FP_TO_UINT; in LowerDIVREM24()
1602 jq = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS); in LowerDIVREM24()
1605 jq = DAG.getNode(ISD::SRA, DL, VT, jq, in LowerDIVREM24()
1609 jq = DAG.getNode(ISD::OR, DL, VT, jq, DAG.getConstant(1, DL, VT)); in LowerDIVREM24()
1624 SDValue fq = DAG.getNode(ISD::FMUL, DL, FltVT, in LowerDIVREM24()
1628 fq = DAG.getNode(ISD::FTRUNC, DL, FltVT, fq); in LowerDIVREM24()
1631 SDValue fqneg = DAG.getNode(ISD::FNEG, DL, FltVT, fq); in LowerDIVREM24()
1638 (unsigned)ISD::FMA : in LowerDIVREM24()
1640 (unsigned)ISD::FMAD : in LowerDIVREM24()
1648 fr = DAG.getNode(ISD::FABS, DL, FltVT, fr); in LowerDIVREM24()
1651 fb = DAG.getNode(ISD::FABS, DL, FltVT, fb); in LowerDIVREM24()
1656 SDValue cv = DAG.getSetCC(DL, SetCCVT, fr, fb, ISD::SETOGE); in LowerDIVREM24()
1659 jq = DAG.getNode(ISD::SELECT, DL, VT, cv, jq, DAG.getConstant(0, DL, VT)); in LowerDIVREM24()
1662 SDValue Div = DAG.getNode(ISD::ADD, DL, VT, iq, jq); in LowerDIVREM24()
1665 SDValue Rem = DAG.getNode(ISD::MUL, DL, VT, Div, RHS); in LowerDIVREM24()
1666 Rem = DAG.getNode(ISD::SUB, DL, VT, LHS, Rem); in LowerDIVREM24()
1672 Div = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Div, InRegSize); in LowerDIVREM24()
1673 Rem = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Rem, InRegSize); in LowerDIVREM24()
1676 Div = DAG.getNode(ISD::AND, DL, VT, Div, TruncMask); in LowerDIVREM24()
1677 Rem = DAG.getNode(ISD::AND, DL, VT, Rem, TruncMask); in LowerDIVREM24()
1698 SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero); in LowerUDIVREM64()
1699 SDValue LHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, One); in LowerUDIVREM64()
1702 SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero); in LowerUDIVREM64()
1703 SDValue RHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, One); in LowerUDIVREM64()
1708 SDValue Res = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(HalfVT, HalfVT), in LowerUDIVREM64()
1714 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV)); in LowerUDIVREM64()
1715 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM)); in LowerUDIVREM64()
1728 (unsigned)ISD::FMA : in LowerUDIVREM64()
1730 (unsigned)ISD::FMAD : in LowerUDIVREM64()
1733 SDValue Cvt_Lo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Lo); in LowerUDIVREM64()
1734 SDValue Cvt_Hi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Hi); in LowerUDIVREM64()
1739 SDValue Mul1 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Rcp, in LowerUDIVREM64()
1741 SDValue Mul2 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Mul1, in LowerUDIVREM64()
1743 SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, MVT::f32, Mul2); in LowerUDIVREM64()
1747 SDValue Rcp_Lo = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Mad2); in LowerUDIVREM64()
1748 SDValue Rcp_Hi = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Trunc); in LowerUDIVREM64()
1758 SDValue Neg_RHS = DAG.getNode(ISD::SUB, DL, VT, Zero64, RHS); in LowerUDIVREM64()
1759 SDValue Mullo1 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Rcp64); in LowerUDIVREM64()
1760 SDValue Mulhi1 = DAG.getNode(ISD::MULHU, DL, VT, Rcp64, Mullo1); in LowerUDIVREM64()
1761 SDValue Mulhi1_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi1, in LowerUDIVREM64()
1764 DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi1, One); in LowerUDIVREM64()
1765 SDValue Add1_Lo = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Rcp_Lo, in LowerUDIVREM64()
1767 SDValue Add1_Hi = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Rcp_Hi, in LowerUDIVREM64()
1773 SDValue Mullo2 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Add1); in LowerUDIVREM64()
1774 SDValue Mulhi2 = DAG.getNode(ISD::MULHU, DL, VT, Add1, Mullo2); in LowerUDIVREM64()
1775 SDValue Mulhi2_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi2, in LowerUDIVREM64()
1778 DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi2, One); in LowerUDIVREM64()
1779 SDValue Add2_Lo = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add1_Lo, in LowerUDIVREM64()
1781 SDValue Add2_Hi = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add1_Hi, in LowerUDIVREM64()
1786 SDValue Mulhi3 = DAG.getNode(ISD::MULHU, DL, VT, LHS, Add2); in LowerUDIVREM64()
1788 SDValue Mul3 = DAG.getNode(ISD::MUL, DL, VT, RHS, Mulhi3); in LowerUDIVREM64()
1790 SDValue Mul3_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mul3, Zero); in LowerUDIVREM64()
1791 SDValue Mul3_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mul3, One); in LowerUDIVREM64()
1792 SDValue Sub1_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, LHS_Lo, in LowerUDIVREM64()
1794 SDValue Sub1_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, LHS_Hi, in LowerUDIVREM64()
1796 SDValue Sub1_Mi = DAG.getNode(ISD::SUB, DL, HalfVT, LHS_Hi, Mul3_Hi); in LowerUDIVREM64()
1802 ISD::SETUGE); in LowerUDIVREM64()
1804 ISD::SETUGE); in LowerUDIVREM64()
1805 SDValue C3 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, C2, C1, ISD::SETEQ); in LowerUDIVREM64()
1812 SDValue Sub2_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub1_Lo, in LowerUDIVREM64()
1814 SDValue Sub2_Mi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub1_Mi, in LowerUDIVREM64()
1816 SDValue Sub2_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Mi, in LowerUDIVREM64()
1821 SDValue Add3 = DAG.getNode(ISD::ADD, DL, VT, Mulhi3, One64); in LowerUDIVREM64()
1824 ISD::SETUGE); in LowerUDIVREM64()
1826 ISD::SETUGE); in LowerUDIVREM64()
1827 SDValue C6 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, C5, C4, ISD::SETEQ); in LowerUDIVREM64()
1830 SDValue Add4 = DAG.getNode(ISD::ADD, DL, VT, Add3, One64); in LowerUDIVREM64()
1832 SDValue Sub3_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Lo, in LowerUDIVREM64()
1834 SDValue Sub3_Mi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Mi, in LowerUDIVREM64()
1836 SDValue Sub3_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub3_Mi, in LowerUDIVREM64()
1844 SDValue Sel1 = DAG.getSelectCC(DL, C6, Zero, Add4, Add3, ISD::SETNE); in LowerUDIVREM64()
1845 SDValue Div = DAG.getSelectCC(DL, C3, Zero, Sel1, Mulhi3, ISD::SETNE); in LowerUDIVREM64()
1847 SDValue Sel2 = DAG.getSelectCC(DL, C6, Zero, Sub3, Sub2, ISD::SETNE); in LowerUDIVREM64()
1848 SDValue Rem = DAG.getSelectCC(DL, C3, Zero, Sel2, Sub1, ISD::SETNE); in LowerUDIVREM64()
1858 SDValue DIV_Part = DAG.getNode(ISD::UDIV, DL, HalfVT, LHS_Hi, RHS_Lo); in LowerUDIVREM64()
1859 SDValue REM_Part = DAG.getNode(ISD::UREM, DL, HalfVT, LHS_Hi, RHS_Lo); in LowerUDIVREM64()
1861 SDValue REM_Lo = DAG.getSelectCC(DL, RHS_Hi, Zero, REM_Part, LHS_Hi, ISD::SETEQ); in LowerUDIVREM64()
1863 REM = DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM); in LowerUDIVREM64()
1865 SDValue DIV_Hi = DAG.getSelectCC(DL, RHS_Hi, Zero, DIV_Part, Zero, ISD::SETEQ); in LowerUDIVREM64()
1874 SDValue HBit = DAG.getNode(ISD::SRL, DL, HalfVT, LHS_Lo, POS); in LowerUDIVREM64()
1875 HBit = DAG.getNode(ISD::AND, DL, HalfVT, HBit, One); in LowerUDIVREM64()
1876 HBit = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, HBit); in LowerUDIVREM64()
1879 REM = DAG.getNode(ISD::SHL, DL, VT, REM, DAG.getConstant(1, DL, VT)); in LowerUDIVREM64()
1881 REM = DAG.getNode(ISD::OR, DL, VT, REM, HBit); in LowerUDIVREM64()
1884 SDValue realBIT = DAG.getSelectCC(DL, REM, RHS, BIT, Zero, ISD::SETUGE); in LowerUDIVREM64()
1886 DIV_Lo = DAG.getNode(ISD::OR, DL, HalfVT, DIV_Lo, realBIT); in LowerUDIVREM64()
1889 SDValue REM_sub = DAG.getNode(ISD::SUB, DL, VT, REM, RHS); in LowerUDIVREM64()
1890 REM = DAG.getSelectCC(DL, REM, RHS, REM_sub, REM, ISD::SETUGE); in LowerUDIVREM64()
1894 DIV = DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV); in LowerUDIVREM64()
1925 SDValue NegY = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Y); in LowerUDIVREM()
1926 SDValue NegYZ = DAG.getNode(ISD::MUL, DL, VT, NegY, Z); in LowerUDIVREM()
1927 Z = DAG.getNode(ISD::ADD, DL, VT, Z, in LowerUDIVREM()
1928 DAG.getNode(ISD::MULHU, DL, VT, Z, NegYZ)); in LowerUDIVREM()
1931 SDValue Q = DAG.getNode(ISD::MULHU, DL, VT, X, Z); in LowerUDIVREM()
1933 DAG.getNode(ISD::SUB, DL, VT, X, DAG.getNode(ISD::MUL, DL, VT, Q, Y)); in LowerUDIVREM()
1938 SDValue Cond = DAG.getSetCC(DL, CCVT, R, Y, ISD::SETUGE); in LowerUDIVREM()
1939 Q = DAG.getNode(ISD::SELECT, DL, VT, Cond, in LowerUDIVREM()
1940 DAG.getNode(ISD::ADD, DL, VT, Q, One), Q); in LowerUDIVREM()
1941 R = DAG.getNode(ISD::SELECT, DL, VT, Cond, in LowerUDIVREM()
1942 DAG.getNode(ISD::SUB, DL, VT, R, Y), R); in LowerUDIVREM()
1945 Cond = DAG.getSetCC(DL, CCVT, R, Y, ISD::SETUGE); in LowerUDIVREM()
1946 Q = DAG.getNode(ISD::SELECT, DL, VT, Cond, in LowerUDIVREM()
1947 DAG.getNode(ISD::ADD, DL, VT, Q, One), Q); in LowerUDIVREM()
1948 R = DAG.getNode(ISD::SELECT, DL, VT, Cond, in LowerUDIVREM()
1949 DAG.getNode(ISD::SUB, DL, VT, R, Y), R); in LowerUDIVREM()
1976 SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero); in LowerSDIVREM()
1977 SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero); in LowerSDIVREM()
1978 SDValue DIVREM = DAG.getNode(ISD::SDIVREM, DL, DAG.getVTList(HalfVT, HalfVT), in LowerSDIVREM()
1981 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(0)), in LowerSDIVREM()
1982 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(1)) in LowerSDIVREM()
1987 SDValue LHSign = DAG.getSelectCC(DL, LHS, Zero, NegOne, Zero, ISD::SETLT); in LowerSDIVREM()
1988 SDValue RHSign = DAG.getSelectCC(DL, RHS, Zero, NegOne, Zero, ISD::SETLT); in LowerSDIVREM()
1989 SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign); in LowerSDIVREM()
1992 LHS = DAG.getNode(ISD::ADD, DL, VT, LHS, LHSign); in LowerSDIVREM()
1993 RHS = DAG.getNode(ISD::ADD, DL, VT, RHS, RHSign); in LowerSDIVREM()
1995 LHS = DAG.getNode(ISD::XOR, DL, VT, LHS, LHSign); in LowerSDIVREM()
1996 RHS = DAG.getNode(ISD::XOR, DL, VT, RHS, RHSign); in LowerSDIVREM()
1998 SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS); in LowerSDIVREM()
2001 Div = DAG.getNode(ISD::XOR, DL, VT, Div, DSign); in LowerSDIVREM()
2002 Rem = DAG.getNode(ISD::XOR, DL, VT, Rem, RSign); in LowerSDIVREM()
2004 Div = DAG.getNode(ISD::SUB, DL, VT, Div, DSign); in LowerSDIVREM()
2005 Rem = DAG.getNode(ISD::SUB, DL, VT, Rem, RSign); in LowerSDIVREM()
2022 SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y, Flags); in LowerFREM()
2023 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, VT, Div, Flags); in LowerFREM()
2024 SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Trunc, Flags); in LowerFREM()
2026 return DAG.getNode(ISD::FMA, SL, VT, Neg, Y, X, Flags); in LowerFREM()
2037 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); in LowerFCEIL()
2045 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOGT); in LowerFCEIL()
2046 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE); in LowerFCEIL()
2047 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc); in LowerFCEIL()
2049 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero); in LowerFCEIL()
2051 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); in LowerFCEIL()
2063 SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart, in extractF64Exponent()
2087 SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask); in LowerFTRUNC()
2091 SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64); in LowerFTRUNC()
2093 SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src); in LowerFTRUNC()
2097 SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp); in LowerFTRUNC()
2099 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not); in LowerFTRUNC()
2106 SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT); in LowerFTRUNC()
2107 SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT); in LowerFTRUNC()
2109 SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0); in LowerFTRUNC()
2110 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1); in LowerFTRUNC()
2112 return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2); in LowerFTRUNC()
2123 SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src); in LowerFRINT()
2127 SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign); in LowerFRINT()
2128 SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign); in LowerFRINT()
2130 SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src); in LowerFRINT()
2137 SDValue Cond = DAG.getSetCC(SL, SetCCVT, Fabs, C2, ISD::SETOGT); in LowerFRINT()
2146 return DAG.getNode(ISD::FRINT, SDLoc(Op), Op.getValueType(), Op.getOperand(0)); in LowerFNEARBYINT()
2159 SDValue T = DAG.getNode(ISD::FTRUNC, SL, VT, X); in LowerFROUND()
2163 SDValue Diff = DAG.getNode(ISD::FSUB, SL, VT, X, T); in LowerFROUND()
2165 SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, VT, Diff); in LowerFROUND()
2171 SDValue SignOne = DAG.getNode(ISD::FCOPYSIGN, SL, VT, One, X); in LowerFROUND()
2176 SDValue Cmp = DAG.getSetCC(SL, SetCCVT, AbsDiff, Half, ISD::SETOGE); in LowerFROUND()
2178 SDValue Sel = DAG.getNode(ISD::SELECT, SL, VT, Cmp, SignOne, Zero); in LowerFROUND()
2180 return DAG.getNode(ISD::FADD, SL, VT, T, Sel); in LowerFROUND()
2191 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); in LowerFFLOOR()
2199 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOLT); in LowerFFLOOR()
2200 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE); in LowerFFLOOR()
2201 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc); in LowerFFLOOR()
2203 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero); in LowerFFLOOR()
2205 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); in LowerFFLOOR()
2214 SDValue Log2Operand = DAG.getNode(ISD::FLOG2, SL, VT, Operand); in LowerFLOG()
2217 return DAG.getNode(ISD::FMUL, SL, VT, Log2Operand, Log2BaseInvertedOperand); in LowerFLOG()
2227 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, Src, K, Op->getFlags()); in lowerFEXP()
2228 return DAG.getNode(ISD::FEXP2, SL, VT, Mul, Op->getFlags()); in lowerFEXP()
2232 return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_UNDEF; in isCtlzOpc()
2236 return Opc == ISD::CTTZ || Opc == ISD::CTTZ_ZERO_UNDEF; in isCttzOpc()
2247 bool ZeroUndef = Op.getOpcode() == ISD::CTLZ_ZERO_UNDEF || in LowerCTLZ_CTTZ()
2248 Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF; in LowerCTLZ_CTTZ()
2258 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, Const32); in LowerCTLZ_CTTZ()
2274 unsigned AddOpc = ZeroUndef ? ISD::ADD : ISD::UADDSAT; in LowerCTLZ_CTTZ()
2282 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, OprLo, OprHi); in LowerCTLZ_CTTZ()
2285 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, Const64); in LowerCTLZ_CTTZ()
2288 return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i64, NewOpr); in LowerCTLZ_CTTZ()
2349 ISD::SRA, SL, MVT::i32, DAG.getNode(ISD::XOR, SL, MVT::i32, Lo, Hi), in LowerINT_TO_FP32()
2352 DAG.getNode(ISD::ADD, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32), in LowerINT_TO_FP32()
2358 ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, ShAmt, in LowerINT_TO_FP32()
2360 ShAmt = DAG.getNode(ISD::UMIN, SL, MVT::i32, ShAmt, MaxShAmt); in LowerINT_TO_FP32()
2365 Sign = DAG.getNode(ISD::SRA, SL, MVT::i64, Src, in LowerINT_TO_FP32()
2368 DAG.getNode(ISD::XOR, SL, MVT::i64, in LowerINT_TO_FP32()
2369 DAG.getNode(ISD::ADD, SL, MVT::i64, Src, Sign), Sign); in LowerINT_TO_FP32()
2373 ShAmt = DAG.getNode(ISD::CTLZ, SL, MVT::i32, Hi); in LowerINT_TO_FP32()
2377 SDValue Norm = DAG.getNode(ISD::SHL, SL, MVT::i64, Src, ShAmt); in LowerINT_TO_FP32()
2382 SDValue Adjust = DAG.getNode(ISD::UMIN, SL, MVT::i32, in LowerINT_TO_FP32()
2385 Norm = DAG.getNode(ISD::OR, SL, MVT::i32, Hi, Adjust); in LowerINT_TO_FP32()
2388 (Signed && Subtarget->isGCN()) ? ISD::SINT_TO_FP : ISD::UINT_TO_FP; in LowerINT_TO_FP32()
2393 ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32), in LowerINT_TO_FP32()
2402 SDValue Exp = DAG.getNode(ISD::SHL, SL, MVT::i32, ShAmt, in LowerINT_TO_FP32()
2405 DAG.getNode(ISD::ADD, SL, MVT::i32, in LowerINT_TO_FP32()
2406 DAG.getNode(ISD::BITCAST, SL, MVT::i32, FVal), Exp); in LowerINT_TO_FP32()
2409 Sign = DAG.getNode(ISD::SHL, SL, MVT::i32, in LowerINT_TO_FP32()
2410 DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Sign), in LowerINT_TO_FP32()
2412 IVal = DAG.getNode(ISD::OR, SL, MVT::i32, IVal, Sign); in LowerINT_TO_FP32()
2414 return DAG.getNode(ISD::BITCAST, SL, MVT::f32, IVal); in LowerINT_TO_FP32()
2425 SDValue CvtHi = DAG.getNode(Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, in LowerINT_TO_FP64()
2428 SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo); in LowerINT_TO_FP64()
2433 return DAG.getNode(ISD::FADD, SL, MVT::f64, LdExp, CvtLo); in LowerINT_TO_FP64()
2449 SDValue Ext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Src); in LowerUINT_TO_FP()
2450 return DAG.getNode(ISD::UINT_TO_FP, DL, DestVT, Ext); in LowerUINT_TO_FP()
2461 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag); in LowerUINT_TO_FP()
2486 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32, Src); in LowerSINT_TO_FP()
2487 return DAG.getNode(ISD::SINT_TO_FP, DL, DestVT, Ext); in LowerSINT_TO_FP()
2501 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag); in LowerSINT_TO_FP()
2531 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, SrcVT, Src); in LowerFP_TO_INT64()
2539 Sign = DAG.getNode(ISD::SRA, SL, MVT::i32, in LowerFP_TO_INT64()
2540 DAG.getNode(ISD::BITCAST, SL, MVT::i32, Trunc), in LowerFP_TO_INT64()
2542 Trunc = DAG.getNode(ISD::FABS, SL, SrcVT, Trunc); in LowerFP_TO_INT64()
2558 SDValue Mul = DAG.getNode(ISD::FMUL, SL, SrcVT, Trunc, K0); in LowerFP_TO_INT64()
2560 SDValue FloorMul = DAG.getNode(ISD::FFLOOR, SL, SrcVT, Mul); in LowerFP_TO_INT64()
2562 SDValue Fma = DAG.getNode(ISD::FMA, SL, SrcVT, FloorMul, K1, Trunc); in LowerFP_TO_INT64()
2564 SDValue Hi = DAG.getNode((Signed && SrcVT == MVT::f64) ? ISD::FP_TO_SINT in LowerFP_TO_INT64()
2565 : ISD::FP_TO_UINT, in LowerFP_TO_INT64()
2567 SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, SL, MVT::i32, Fma); in LowerFP_TO_INT64()
2569 SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i64, in LowerFP_TO_INT64()
2575 Sign = DAG.getNode(ISD::BITCAST, SL, MVT::i64, in LowerFP_TO_INT64()
2579 DAG.getNode(ISD::SUB, SL, MVT::i64, in LowerFP_TO_INT64()
2580 DAG.getNode(ISD::XOR, SL, MVT::i64, Result, Sign), Sign); in LowerFP_TO_INT64()
2607 SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, N0); in LowerFP_TO_FP16()
2608 SDValue UH = DAG.getNode(ISD::SRL, DL, MVT::i64, U, in LowerFP_TO_FP16()
2612 SDValue E = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, in LowerFP_TO_FP16()
2614 E = DAG.getNode(ISD::AND, DL, MVT::i32, E, in LowerFP_TO_FP16()
2618 E = DAG.getNode(ISD::ADD, DL, MVT::i32, E, in LowerFP_TO_FP16()
2621 SDValue M = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, in LowerFP_TO_FP16()
2623 M = DAG.getNode(ISD::AND, DL, MVT::i32, M, in LowerFP_TO_FP16()
2626 SDValue MaskedSig = DAG.getNode(ISD::AND, DL, MVT::i32, UH, in LowerFP_TO_FP16()
2628 MaskedSig = DAG.getNode(ISD::OR, DL, MVT::i32, MaskedSig, U); in LowerFP_TO_FP16()
2630 SDValue Lo40Set = DAG.getSelectCC(DL, MaskedSig, Zero, Zero, One, ISD::SETEQ); in LowerFP_TO_FP16()
2631 M = DAG.getNode(ISD::OR, DL, MVT::i32, M, Lo40Set); in LowerFP_TO_FP16()
2634 SDValue I = DAG.getNode(ISD::OR, DL, MVT::i32, in LowerFP_TO_FP16()
2636 Zero, ISD::SETNE), DAG.getConstant(0x7c00, DL, MVT::i32)); in LowerFP_TO_FP16()
2639 SDValue N = DAG.getNode(ISD::OR, DL, MVT::i32, M, in LowerFP_TO_FP16()
2640 DAG.getNode(ISD::SHL, DL, MVT::i32, E, in LowerFP_TO_FP16()
2644 SDValue OneSubExp = DAG.getNode(ISD::SUB, DL, MVT::i32, in LowerFP_TO_FP16()
2646 SDValue B = DAG.getNode(ISD::SMAX, DL, MVT::i32, OneSubExp, Zero); in LowerFP_TO_FP16()
2647 B = DAG.getNode(ISD::SMIN, DL, MVT::i32, B, in LowerFP_TO_FP16()
2650 SDValue SigSetHigh = DAG.getNode(ISD::OR, DL, MVT::i32, M, in LowerFP_TO_FP16()
2653 SDValue D = DAG.getNode(ISD::SRL, DL, MVT::i32, SigSetHigh, B); in LowerFP_TO_FP16()
2654 SDValue D0 = DAG.getNode(ISD::SHL, DL, MVT::i32, D, B); in LowerFP_TO_FP16()
2655 SDValue D1 = DAG.getSelectCC(DL, D0, SigSetHigh, One, Zero, ISD::SETNE); in LowerFP_TO_FP16()
2656 D = DAG.getNode(ISD::OR, DL, MVT::i32, D, D1); in LowerFP_TO_FP16()
2658 SDValue V = DAG.getSelectCC(DL, E, One, D, N, ISD::SETLT); in LowerFP_TO_FP16()
2659 SDValue VLow3 = DAG.getNode(ISD::AND, DL, MVT::i32, V, in LowerFP_TO_FP16()
2661 V = DAG.getNode(ISD::SRL, DL, MVT::i32, V, in LowerFP_TO_FP16()
2664 One, Zero, ISD::SETEQ); in LowerFP_TO_FP16()
2666 One, Zero, ISD::SETGT); in LowerFP_TO_FP16()
2667 V1 = DAG.getNode(ISD::OR, DL, MVT::i32, V0, V1); in LowerFP_TO_FP16()
2668 V = DAG.getNode(ISD::ADD, DL, MVT::i32, V, V1); in LowerFP_TO_FP16()
2671 DAG.getConstant(0x7c00, DL, MVT::i32), V, ISD::SETGT); in LowerFP_TO_FP16()
2673 I, V, ISD::SETEQ); in LowerFP_TO_FP16()
2676 SDValue Sign = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, in LowerFP_TO_FP16()
2678 Sign = DAG.getNode(ISD::AND, DL, MVT::i32, Sign, in LowerFP_TO_FP16()
2681 V = DAG.getNode(ISD::OR, DL, MVT::i32, Sign, V); in LowerFP_TO_FP16()
2701 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToInt32); in LowerFP_TO_INT()
2705 (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP)) { in LowerFP_TO_INT()
2710 OpOpcode == ISD::FP_TO_SINT ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in LowerFP_TO_INT()
2715 return LowerFP_TO_INT64(Op, DAG, OpOpcode == ISD::FP_TO_SINT); in LowerFP_TO_INT()
2738 Args[I] = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ScalarVT, Args[I], VTOp); in LowerSIGN_EXTEND_INREG()
2762 bool IsIntrin = Node24->getOpcode() == ISD::INTRINSIC_WO_CHAIN; in simplifyMul24()
2859 if (!LN->isSimple() || !ISD::isNormalLoad(LN) || hasVolatileUser(LN)) in performLoadCombine()
2899 SDValue BC = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad); in performLoadCombine()
2912 if (!SN->isSimple() || !ISD::isNormalStore(SN)) in performStoreCombine()
2950 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NewVT, Val); in performStoreCombine()
2952 SDValue CastBack = DAG.getNode(ISD::BITCAST, SL, VT, CastVal); in performStoreCombine()
2970 if (N0.getOpcode() == ISD::TRUNCATE) { in performAssertSZExtCombine()
2979 return DAG.getNode(ISD::TRUNCATE, SL, N->getValueType(0), NewInReg); in performAssertSZExtCombine()
3032 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in splitBinaryBitConstantOpImpl()
3054 case ISD::ZERO_EXTEND: in performShlCombine()
3055 case ISD::SIGN_EXTEND: in performShlCombine()
3056 case ISD::ANY_EXTEND: { in performShlCombine()
3060 isOperationLegal(ISD::BUILD_VECTOR, MVT::v2i16)) { in performShlCombine()
3065 return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Vec); in performShlCombine()
3076 SDValue Shl = DAG.getNode(ISD::SHL, SL, XVT, X, SDValue(RHS, 0)); in performShlCombine()
3094 SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS); in performShlCombine()
3095 SDValue NewShift = DAG.getNode(ISD::SHL, SL, MVT::i32, Lo, ShiftAmt); in performShlCombine()
3100 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in performShlCombine()
3119 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi, in performSraCombine()
3123 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec); in performSraCombine()
3129 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi, in performSraCombine()
3132 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec); in performSraCombine()
3152 if (LHS.getOpcode() == ISD::AND) { in performSrlCombine()
3158 ISD::AND, SL, VT, in performSrlCombine()
3159 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(0), N->getOperand(1)), in performSrlCombine()
3160 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(1), N->getOperand(1))); in performSrlCombine()
3179 SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst); in performSrlCombine()
3183 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair); in performSrlCombine()
3194 if (Src.getOpcode() == ISD::BITCAST && !VT.isVector()) { in performTruncateCombine()
3196 if (Vec.getOpcode() == ISD::BUILD_VECTOR) { in performTruncateCombine()
3201 Elt0 = DAG.getNode(ISD::BITCAST, SL, in performTruncateCombine()
3205 return DAG.getNode(ISD::TRUNCATE, SL, VT, Elt0); in performTruncateCombine()
3213 if (Src.getOpcode() == ISD::SRL && !VT.isVector()) { in performTruncateCombine()
3217 if (BV.getOpcode() == ISD::BUILD_VECTOR && in performTruncateCombine()
3222 SrcElt = DAG.getNode(ISD::BITCAST, SL, in performTruncateCombine()
3226 return DAG.getNode(ISD::TRUNCATE, SL, VT, SrcElt); in performTruncateCombine()
3239 (Src.getOpcode() == ISD::SRL || in performTruncateCombine()
3240 Src.getOpcode() == ISD::SRA || in performTruncateCombine()
3241 Src.getOpcode() == ISD::SHL)) { in performTruncateCombine()
3252 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MidVT, in performTruncateCombine()
3263 return DAG.getNode(ISD::TRUNCATE, SL, VT, ShrunkShift); in performTruncateCombine()
3288 return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, MulLo, MulHi); in getMul24()
3320 if (N0.getOpcode() == ISD::ANY_EXTEND) in performMulCombine()
3323 if (N1.getOpcode() == ISD::ANY_EXTEND) in performMulCombine()
3361 if (N0.getOpcode() == ISD::ANY_EXTEND) in performMulLoHiCombine()
3363 if (N1.getOpcode() == ISD::ANY_EXTEND) in performMulLoHiCombine()
3472 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Op); in getFFBX_U32()
3476 FFBX = DAG.getNode(ISD::TRUNCATE, DL, VT, FFBX); in getFFBX_U32()
3496 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in performCtlz_CttzCombine()
3501 if (CCOpcode == ISD::SETEQ && in performCtlz_CttzCombine()
3511 if (CCOpcode == ISD::SETNE && in performCtlz_CttzCombine()
3532 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, Cond, in distributeOpThroughSelect()
3553 if ((LHS.getOpcode() == ISD::FABS && RHS.getOpcode() == ISD::FABS) || in foldFreeOpFromSelect()
3554 (LHS.getOpcode() == ISD::FNEG && RHS.getOpcode() == ISD::FNEG)) { in foldFreeOpFromSelect()
3560 if (RHS.getOpcode() == ISD::FABS || RHS.getOpcode() == ISD::FNEG) { in foldFreeOpFromSelect()
3567 if ((LHS.getOpcode() == ISD::FNEG || LHS.getOpcode() == ISD::FABS) && CRHS) { in foldFreeOpFromSelect()
3579 if (LHS.getOpcode() == ISD::FNEG && fnegFoldsIntoOp(Opc)) in foldFreeOpFromSelect()
3581 if (LHS.getOpcode() == ISD::FABS && Opc == ISD::FMUL) in foldFreeOpFromSelect()
3586 if (LHS.getOpcode() == ISD::FNEG) in foldFreeOpFromSelect()
3587 NewRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in foldFreeOpFromSelect()
3594 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, in foldFreeOpFromSelect()
3611 if (Cond.getOpcode() != ISD::SETCC) in performSelectCombine()
3631 ISD::CondCode NewCC = in performSelectCombine()
3635 return DAG.getNode(ISD::SELECT, SL, VT, NewCond, False, True); in performSelectCombine()
3677 case ISD::FMAXNUM: in inverseMinMax()
3678 return ISD::FMINNUM; in inverseMinMax()
3679 case ISD::FMINNUM: in inverseMinMax()
3680 return ISD::FMAXNUM; in inverseMinMax()
3681 case ISD::FMAXNUM_IEEE: in inverseMinMax()
3682 return ISD::FMINNUM_IEEE; in inverseMinMax()
3683 case ISD::FMINNUM_IEEE: in inverseMinMax()
3684 return ISD::FMAXNUM_IEEE; in inverseMinMax()
3719 case ISD::FADD: { in performFNegCombine()
3727 if (LHS.getOpcode() != ISD::FNEG) in performFNegCombine()
3728 LHS = DAG.getNode(ISD::FNEG, SL, VT, LHS); in performFNegCombine()
3732 if (RHS.getOpcode() != ISD::FNEG) in performFNegCombine()
3733 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
3737 SDValue Res = DAG.getNode(ISD::FADD, SL, VT, LHS, RHS, N0->getFlags()); in performFNegCombine()
3738 if (Res.getOpcode() != ISD::FADD) in performFNegCombine()
3741 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
3744 case ISD::FMUL: in performFNegCombine()
3751 if (LHS.getOpcode() == ISD::FNEG) in performFNegCombine()
3753 else if (RHS.getOpcode() == ISD::FNEG) in performFNegCombine()
3756 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
3762 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
3765 case ISD::FMA: in performFNegCombine()
3766 case ISD::FMAD: { in performFNegCombine()
3776 if (LHS.getOpcode() == ISD::FNEG) in performFNegCombine()
3778 else if (MHS.getOpcode() == ISD::FNEG) in performFNegCombine()
3781 MHS = DAG.getNode(ISD::FNEG, SL, VT, MHS); in performFNegCombine()
3783 if (RHS.getOpcode() != ISD::FNEG) in performFNegCombine()
3784 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
3792 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
3795 case ISD::FMAXNUM: in performFNegCombine()
3796 case ISD::FMINNUM: in performFNegCombine()
3797 case ISD::FMAXNUM_IEEE: in performFNegCombine()
3798 case ISD::FMINNUM_IEEE: in performFNegCombine()
3814 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, VT, LHS); in performFNegCombine()
3815 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS); in performFNegCombine()
3822 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res)); in performFNegCombine()
3828 Ops[I] = DAG.getNode(ISD::FNEG, SL, VT, N0->getOperand(I), N0->getFlags()); in performFNegCombine()
3835 SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Res); in performFNegCombine()
3844 case ISD::FP_EXTEND: in performFNegCombine()
3845 case ISD::FTRUNC: in performFNegCombine()
3846 case ISD::FRINT: in performFNegCombine()
3847 case ISD::FNEARBYINT: // XXX - Should fround be handled? in performFNegCombine()
3848 case ISD::FSIN: in performFNegCombine()
3849 case ISD::FCANONICALIZE: in performFNegCombine()
3855 if (CvtSrc.getOpcode() == ISD::FNEG) { in performFNegCombine()
3866 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc); in performFNegCombine()
3869 case ISD::FP_ROUND: { in performFNegCombine()
3872 if (CvtSrc.getOpcode() == ISD::FNEG) { in performFNegCombine()
3874 return DAG.getNode(ISD::FP_ROUND, SL, VT, in performFNegCombine()
3882 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc); in performFNegCombine()
3883 return DAG.getNode(ISD::FP_ROUND, SL, VT, Neg, N0.getOperand(1)); in performFNegCombine()
3885 case ISD::FP16_TO_FP: { in performFNegCombine()
3895 SDValue IntFNeg = DAG.getNode(ISD::XOR, SL, SrcVT, Src, in performFNegCombine()
3897 return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFNeg); in performFNegCombine()
3913 case ISD::FP16_TO_FP: { in performFAbsCombine()
3920 SDValue IntFAbs = DAG.getNode(ISD::AND, SL, SrcVT, Src, in performFAbsCombine()
3922 return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFAbs); in performFAbsCombine()
3949 case ISD::BITCAST: { in PerformDAGCombine()
3959 if (Src.getOpcode() == ISD::BUILD_VECTOR) { in PerformDAGCombine()
3970 CastedElts.push_back(DAG.getNode(ISD::BITCAST, DL, DestEltVT, Elt)); in PerformDAGCombine()
3989 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in PerformDAGCombine()
3992 return DAG.getNode(ISD::BITCAST, SL, DestVT, BV); in PerformDAGCombine()
3999 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in PerformDAGCombine()
4003 return DAG.getNode(ISD::BITCAST, SL, DestVT, Vec); in PerformDAGCombine()
4008 case ISD::SHL: { in PerformDAGCombine()
4014 case ISD::SRL: { in PerformDAGCombine()
4020 case ISD::SRA: { in PerformDAGCombine()
4026 case ISD::TRUNCATE: in PerformDAGCombine()
4028 case ISD::MUL: in PerformDAGCombine()
4030 case ISD::SMUL_LOHI: in PerformDAGCombine()
4031 case ISD::UMUL_LOHI: in PerformDAGCombine()
4033 case ISD::MULHS: in PerformDAGCombine()
4035 case ISD::MULHU: in PerformDAGCombine()
4042 case ISD::SELECT: in PerformDAGCombine()
4044 case ISD::FNEG: in PerformDAGCombine()
4046 case ISD::FABS: in PerformDAGCombine()
4085 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, BitsFrom, in PerformDAGCombine()
4111 return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32, in PerformDAGCombine()
4132 case ISD::LOAD: in PerformDAGCombine()
4134 case ISD::STORE: in PerformDAGCombine()
4139 case ISD::AssertZext: in PerformDAGCombine()
4140 case ISD::AssertSext: in PerformDAGCombine()
4142 case ISD::INTRINSIC_WO_CHAIN: in PerformDAGCombine()
4217 Ptr = DAG.getNode(ISD::ADD, SL, MVT::i32, SP, Ptr); in storeStackInputValue()
4238 V = DAG.getNode(ISD::SRL, SL, VT, V, in loadInputValue()
4240 return DAG.getNode(ISD::AND, SL, VT, V, in loadInputValue()
4596 case ISD::INTRINSIC_WO_CHAIN: { in computeKnownBitsForTargetNode()
4758 case ISD::INTRINSIC_WO_CHAIN: { in isKnownNeverNaNForTargetNode()