Lines Matching refs:MVT
88 addRegisterClass(MVT::i1, &AMDGPU::VReg_1RegClass); in SITargetLowering()
89 addRegisterClass(MVT::i64, &AMDGPU::SReg_64RegClass); in SITargetLowering()
91 addRegisterClass(MVT::i32, &AMDGPU::SReg_32RegClass); in SITargetLowering()
92 addRegisterClass(MVT::f32, &AMDGPU::VGPR_32RegClass); in SITargetLowering()
94 addRegisterClass(MVT::v2i32, &AMDGPU::SReg_64RegClass); in SITargetLowering()
99 addRegisterClass(MVT::f64, V64RegClass); in SITargetLowering()
100 addRegisterClass(MVT::v2f32, V64RegClass); in SITargetLowering()
102 addRegisterClass(MVT::v3i32, &AMDGPU::SGPR_96RegClass); in SITargetLowering()
103 addRegisterClass(MVT::v3f32, TRI->getVGPRClassForBitWidth(96)); in SITargetLowering()
105 addRegisterClass(MVT::v2i64, &AMDGPU::SGPR_128RegClass); in SITargetLowering()
106 addRegisterClass(MVT::v2f64, &AMDGPU::SGPR_128RegClass); in SITargetLowering()
108 addRegisterClass(MVT::v4i32, &AMDGPU::SGPR_128RegClass); in SITargetLowering()
109 addRegisterClass(MVT::v4f32, TRI->getVGPRClassForBitWidth(128)); in SITargetLowering()
111 addRegisterClass(MVT::v5i32, &AMDGPU::SGPR_160RegClass); in SITargetLowering()
112 addRegisterClass(MVT::v5f32, TRI->getVGPRClassForBitWidth(160)); in SITargetLowering()
114 addRegisterClass(MVT::v6i32, &AMDGPU::SGPR_192RegClass); in SITargetLowering()
115 addRegisterClass(MVT::v6f32, TRI->getVGPRClassForBitWidth(192)); in SITargetLowering()
117 addRegisterClass(MVT::v3i64, &AMDGPU::SGPR_192RegClass); in SITargetLowering()
118 addRegisterClass(MVT::v3f64, TRI->getVGPRClassForBitWidth(192)); in SITargetLowering()
120 addRegisterClass(MVT::v7i32, &AMDGPU::SGPR_224RegClass); in SITargetLowering()
121 addRegisterClass(MVT::v7f32, TRI->getVGPRClassForBitWidth(224)); in SITargetLowering()
123 addRegisterClass(MVT::v8i32, &AMDGPU::SGPR_256RegClass); in SITargetLowering()
124 addRegisterClass(MVT::v8f32, TRI->getVGPRClassForBitWidth(256)); in SITargetLowering()
126 addRegisterClass(MVT::v4i64, &AMDGPU::SGPR_256RegClass); in SITargetLowering()
127 addRegisterClass(MVT::v4f64, TRI->getVGPRClassForBitWidth(256)); in SITargetLowering()
129 addRegisterClass(MVT::v9i32, &AMDGPU::SGPR_288RegClass); in SITargetLowering()
130 addRegisterClass(MVT::v9f32, TRI->getVGPRClassForBitWidth(288)); in SITargetLowering()
132 addRegisterClass(MVT::v10i32, &AMDGPU::SGPR_320RegClass); in SITargetLowering()
133 addRegisterClass(MVT::v10f32, TRI->getVGPRClassForBitWidth(320)); in SITargetLowering()
135 addRegisterClass(MVT::v11i32, &AMDGPU::SGPR_352RegClass); in SITargetLowering()
136 addRegisterClass(MVT::v11f32, TRI->getVGPRClassForBitWidth(352)); in SITargetLowering()
138 addRegisterClass(MVT::v12i32, &AMDGPU::SGPR_384RegClass); in SITargetLowering()
139 addRegisterClass(MVT::v12f32, TRI->getVGPRClassForBitWidth(384)); in SITargetLowering()
141 addRegisterClass(MVT::v16i32, &AMDGPU::SGPR_512RegClass); in SITargetLowering()
142 addRegisterClass(MVT::v16f32, TRI->getVGPRClassForBitWidth(512)); in SITargetLowering()
144 addRegisterClass(MVT::v8i64, &AMDGPU::SGPR_512RegClass); in SITargetLowering()
145 addRegisterClass(MVT::v8f64, TRI->getVGPRClassForBitWidth(512)); in SITargetLowering()
147 addRegisterClass(MVT::v16i64, &AMDGPU::SGPR_1024RegClass); in SITargetLowering()
148 addRegisterClass(MVT::v16f64, TRI->getVGPRClassForBitWidth(1024)); in SITargetLowering()
152 addRegisterClass(MVT::i16, &AMDGPU::VGPR_16RegClass); in SITargetLowering()
153 addRegisterClass(MVT::f16, &AMDGPU::VGPR_16RegClass); in SITargetLowering()
154 addRegisterClass(MVT::bf16, &AMDGPU::VGPR_16RegClass); in SITargetLowering()
156 addRegisterClass(MVT::i16, &AMDGPU::SReg_32RegClass); in SITargetLowering()
157 addRegisterClass(MVT::f16, &AMDGPU::SReg_32RegClass); in SITargetLowering()
158 addRegisterClass(MVT::bf16, &AMDGPU::SReg_32RegClass); in SITargetLowering()
162 addRegisterClass(MVT::v2i16, &AMDGPU::SReg_32RegClass); in SITargetLowering()
163 addRegisterClass(MVT::v2f16, &AMDGPU::SReg_32RegClass); in SITargetLowering()
164 addRegisterClass(MVT::v2bf16, &AMDGPU::SReg_32RegClass); in SITargetLowering()
165 addRegisterClass(MVT::v4i16, &AMDGPU::SReg_64RegClass); in SITargetLowering()
166 addRegisterClass(MVT::v4f16, &AMDGPU::SReg_64RegClass); in SITargetLowering()
167 addRegisterClass(MVT::v4bf16, &AMDGPU::SReg_64RegClass); in SITargetLowering()
168 addRegisterClass(MVT::v8i16, &AMDGPU::SGPR_128RegClass); in SITargetLowering()
169 addRegisterClass(MVT::v8f16, &AMDGPU::SGPR_128RegClass); in SITargetLowering()
170 addRegisterClass(MVT::v8bf16, &AMDGPU::SGPR_128RegClass); in SITargetLowering()
171 addRegisterClass(MVT::v16i16, &AMDGPU::SGPR_256RegClass); in SITargetLowering()
172 addRegisterClass(MVT::v16f16, &AMDGPU::SGPR_256RegClass); in SITargetLowering()
173 addRegisterClass(MVT::v16bf16, &AMDGPU::SGPR_256RegClass); in SITargetLowering()
174 addRegisterClass(MVT::v32i16, &AMDGPU::SGPR_512RegClass); in SITargetLowering()
175 addRegisterClass(MVT::v32f16, &AMDGPU::SGPR_512RegClass); in SITargetLowering()
176 addRegisterClass(MVT::v32bf16, &AMDGPU::SGPR_512RegClass); in SITargetLowering()
179 addRegisterClass(MVT::v32i32, &AMDGPU::VReg_1024RegClass); in SITargetLowering()
180 addRegisterClass(MVT::v32f32, TRI->getVGPRClassForBitWidth(1024)); in SITargetLowering()
193 {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, in SITargetLowering()
194 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32, in SITargetLowering()
195 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32, in SITargetLowering()
196 MVT::i1, MVT::v32i32}, in SITargetLowering()
200 {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, in SITargetLowering()
201 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32, in SITargetLowering()
202 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32, in SITargetLowering()
203 MVT::i1, MVT::v32i32}, in SITargetLowering()
206 if (isTypeLegal(MVT::bf16)) { in SITargetLowering()
218 setOperationAction(Opc, MVT::bf16, Promote); in SITargetLowering()
219 AddPromotedToType(Opc, MVT::bf16, MVT::f32); in SITargetLowering()
222 setOperationAction(ISD::FP_ROUND, MVT::bf16, Expand); in SITargetLowering()
224 setOperationAction(ISD::SELECT, MVT::bf16, Promote); in SITargetLowering()
225 AddPromotedToType(ISD::SELECT, MVT::bf16, MVT::i16); in SITargetLowering()
228 setOperationAction(ISD::FABS, MVT::bf16, Expand); in SITargetLowering()
229 setOperationAction(ISD::FNEG, MVT::bf16, Expand); in SITargetLowering()
230 setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Expand); in SITargetLowering()
234 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); in SITargetLowering()
235 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); in SITargetLowering()
237 setOperationAction(ISD::BUILD_VECTOR, MVT::v2bf16, Promote); in SITargetLowering()
238 AddPromotedToType(ISD::BUILD_VECTOR, MVT::v2bf16, MVT::v2i16); in SITargetLowering()
241 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand); in SITargetLowering()
242 setTruncStoreAction(MVT::v3i32, MVT::v3i16, Expand); in SITargetLowering()
243 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Expand); in SITargetLowering()
244 setTruncStoreAction(MVT::v8i32, MVT::v8i16, Expand); in SITargetLowering()
245 setTruncStoreAction(MVT::v16i32, MVT::v16i16, Expand); in SITargetLowering()
246 setTruncStoreAction(MVT::v32i32, MVT::v32i16, Expand); in SITargetLowering()
247 setTruncStoreAction(MVT::v2i32, MVT::v2i8, Expand); in SITargetLowering()
248 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Expand); in SITargetLowering()
249 setTruncStoreAction(MVT::v8i32, MVT::v8i8, Expand); in SITargetLowering()
250 setTruncStoreAction(MVT::v16i32, MVT::v16i8, Expand); in SITargetLowering()
251 setTruncStoreAction(MVT::v32i32, MVT::v32i8, Expand); in SITargetLowering()
252 setTruncStoreAction(MVT::v2i16, MVT::v2i8, Expand); in SITargetLowering()
253 setTruncStoreAction(MVT::v4i16, MVT::v4i8, Expand); in SITargetLowering()
254 setTruncStoreAction(MVT::v8i16, MVT::v8i8, Expand); in SITargetLowering()
255 setTruncStoreAction(MVT::v16i16, MVT::v16i8, Expand); in SITargetLowering()
256 setTruncStoreAction(MVT::v32i16, MVT::v32i8, Expand); in SITargetLowering()
258 setTruncStoreAction(MVT::v3i64, MVT::v3i16, Expand); in SITargetLowering()
259 setTruncStoreAction(MVT::v3i64, MVT::v3i32, Expand); in SITargetLowering()
260 setTruncStoreAction(MVT::v4i64, MVT::v4i8, Expand); in SITargetLowering()
261 setTruncStoreAction(MVT::v8i64, MVT::v8i8, Expand); in SITargetLowering()
262 setTruncStoreAction(MVT::v8i64, MVT::v8i16, Expand); in SITargetLowering()
263 setTruncStoreAction(MVT::v8i64, MVT::v8i32, Expand); in SITargetLowering()
264 setTruncStoreAction(MVT::v16i64, MVT::v16i32, Expand); in SITargetLowering()
266 setOperationAction(ISD::GlobalAddress, {MVT::i32, MVT::i64}, Custom); in SITargetLowering()
268 setOperationAction(ISD::SELECT, MVT::i1, Promote); in SITargetLowering()
269 setOperationAction(ISD::SELECT, MVT::i64, Custom); in SITargetLowering()
270 setOperationAction(ISD::SELECT, MVT::f64, Promote); in SITargetLowering()
271 AddPromotedToType(ISD::SELECT, MVT::f64, MVT::i64); in SITargetLowering()
273 setOperationAction(ISD::FSQRT, {MVT::f32, MVT::f64}, Custom); in SITargetLowering()
276 {MVT::f32, MVT::i32, MVT::i64, MVT::f64, MVT::i1}, Expand); in SITargetLowering()
278 setOperationAction(ISD::SETCC, MVT::i1, Promote); in SITargetLowering()
279 setOperationAction(ISD::SETCC, {MVT::v2i1, MVT::v4i1}, Expand); in SITargetLowering()
280 AddPromotedToType(ISD::SETCC, MVT::i1, MVT::i32); in SITargetLowering()
283 {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, in SITargetLowering()
284 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32, in SITargetLowering()
285 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32}, in SITargetLowering()
288 {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32, in SITargetLowering()
289 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v9f32, in SITargetLowering()
290 MVT::v10f32, MVT::v11f32, MVT::v12f32, MVT::v16f32}, in SITargetLowering()
294 {MVT::v2i1, MVT::v4i1, MVT::v2i8, MVT::v4i8, MVT::v2i16, in SITargetLowering()
295 MVT::v3i16, MVT::v4i16, MVT::Other}, in SITargetLowering()
298 setOperationAction(ISD::BRCOND, MVT::Other, Custom); in SITargetLowering()
300 {MVT::i1, MVT::i32, MVT::i64, MVT::f32, MVT::f64}, Expand); in SITargetLowering()
302 setOperationAction({ISD::UADDO, ISD::USUBO}, MVT::i32, Legal); in SITargetLowering()
304 setOperationAction({ISD::UADDO_CARRY, ISD::USUBO_CARRY}, MVT::i32, Legal); in SITargetLowering()
306 setOperationAction({ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS}, MVT::i64, in SITargetLowering()
310 setOperationAction({ISD::UADDO_CARRY, ISD::USUBO_CARRY}, MVT::i64, Legal); in SITargetLowering()
315 for (MVT VT : in SITargetLowering()
316 {MVT::v8i32, MVT::v8f32, MVT::v9i32, MVT::v9f32, MVT::v10i32, in SITargetLowering()
317 MVT::v10f32, MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32, in SITargetLowering()
318 MVT::v16i32, MVT::v16f32, MVT::v2i64, MVT::v2f64, MVT::v4i16, in SITargetLowering()
319 MVT::v4f16, MVT::v4bf16, MVT::v3i64, MVT::v3f64, MVT::v6i32, in SITargetLowering()
320 MVT::v6f32, MVT::v4i64, MVT::v4f64, MVT::v8i64, MVT::v8f64, in SITargetLowering()
321 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16, in SITargetLowering()
322 MVT::v16bf16, MVT::v16i64, MVT::v16f64, MVT::v32i32, MVT::v32f32, in SITargetLowering()
323 MVT::v32i16, MVT::v32f16, MVT::v32bf16}) { in SITargetLowering()
348 setOperationAction(ISD::FP_EXTEND, MVT::v4f32, Expand); in SITargetLowering()
355 for (MVT Vec64 : { MVT::v2i64, MVT::v2f64 }) { in SITargetLowering()
357 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v4i32); in SITargetLowering()
360 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v4i32); in SITargetLowering()
363 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v4i32); in SITargetLowering()
366 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v4i32); in SITargetLowering()
369 for (MVT Vec64 : { MVT::v3i64, MVT::v3f64 }) { in SITargetLowering()
371 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v6i32); in SITargetLowering()
374 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v6i32); in SITargetLowering()
377 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v6i32); in SITargetLowering()
380 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v6i32); in SITargetLowering()
383 for (MVT Vec64 : { MVT::v4i64, MVT::v4f64 }) { in SITargetLowering()
385 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v8i32); in SITargetLowering()
388 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v8i32); in SITargetLowering()
391 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v8i32); in SITargetLowering()
394 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v8i32); in SITargetLowering()
397 for (MVT Vec64 : { MVT::v8i64, MVT::v8f64 }) { in SITargetLowering()
399 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v16i32); in SITargetLowering()
402 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v16i32); in SITargetLowering()
405 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v16i32); in SITargetLowering()
408 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v16i32); in SITargetLowering()
411 for (MVT Vec64 : { MVT::v16i64, MVT::v16f64 }) { in SITargetLowering()
413 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v32i32); in SITargetLowering()
416 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v32i32); in SITargetLowering()
419 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v32i32); in SITargetLowering()
422 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v32i32); in SITargetLowering()
426 {MVT::v8i32, MVT::v8f32, MVT::v16i32, MVT::v16f32}, in SITargetLowering()
429 setOperationAction(ISD::BUILD_VECTOR, {MVT::v4f16, MVT::v4i16, MVT::v4bf16}, in SITargetLowering()
435 {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v2i8, MVT::v4i8, in SITargetLowering()
436 MVT::v8i8, MVT::v4i16, MVT::v4f16, MVT::v4bf16}, in SITargetLowering()
441 {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32}, Custom); in SITargetLowering()
445 {MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32, in SITargetLowering()
446 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32, in SITargetLowering()
447 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32, in SITargetLowering()
448 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32}, in SITargetLowering()
453 setOperationAction(ISD::ATOMIC_CMP_SWAP, {MVT::i32, MVT::i64}, Custom); in SITargetLowering()
457 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, {MVT::i32, MVT::i64}, in SITargetLowering()
460 setOperationAction(ISD::ADDRSPACECAST, {MVT::i32, MVT::i64}, Custom); in SITargetLowering()
462 setOperationAction(ISD::BITREVERSE, {MVT::i32, MVT::i64}, Legal); in SITargetLowering()
467 setOperationAction(ISD::BSWAP, {MVT::i64, MVT::i32}, Legal); in SITargetLowering()
470 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal); in SITargetLowering()
471 setOperationAction({ISD::TRAP, ISD::DEBUGTRAP}, MVT::Other, Custom); in SITargetLowering()
474 setOperationAction({ISD::FPOW, ISD::FPOWI}, MVT::f16, Promote); in SITargetLowering()
475 setOperationAction({ISD::FLOG, ISD::FEXP, ISD::FLOG10}, MVT::f16, Custom); in SITargetLowering()
477 setOperationAction(ISD::FSQRT, MVT::f16, Custom); in SITargetLowering()
481 setOperationAction(ISD::FMAD, MVT::f32, Legal); in SITargetLowering()
485 setOperationAction(ISD::FCOPYSIGN, {MVT::f32, MVT::f64}, Expand); in SITargetLowering()
488 setOperationAction(ISD::CTPOP, MVT::i32, Expand); in SITargetLowering()
491 setOperationAction(ISD::CTPOP, MVT::i64, Expand); in SITargetLowering()
494 setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, MVT::i32, Custom); in SITargetLowering()
497 setOperationAction({ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF}, MVT::i32, Custom); in SITargetLowering()
512 setOperationAction({ISD::UADDSAT, ISD::USUBSAT}, MVT::i32, Legal); in SITargetLowering()
515 setOperationAction({ISD::SADDSAT, ISD::SSUBSAT}, {MVT::i16, MVT::i32}, in SITargetLowering()
518 setOperationAction({ISD::FMINNUM, ISD::FMAXNUM}, {MVT::f32, MVT::f64}, in SITargetLowering()
525 {MVT::f32, MVT::f64}, Legal); in SITargetLowering()
528 setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FROUNDEVEN}, MVT::f64, in SITargetLowering()
532 MVT::f64, Custom); in SITargetLowering()
534 setOperationAction(ISD::FFLOOR, MVT::f64, Legal); in SITargetLowering()
535 setOperationAction({ISD::FLDEXP, ISD::STRICT_FLDEXP}, {MVT::f32, MVT::f64}, in SITargetLowering()
537 setOperationAction(ISD::FFREXP, {MVT::f32, MVT::f64}, Custom); in SITargetLowering()
539 setOperationAction({ISD::FSIN, ISD::FCOS, ISD::FDIV}, MVT::f32, Custom); in SITargetLowering()
540 setOperationAction(ISD::FDIV, MVT::f64, Custom); in SITargetLowering()
542 setOperationAction(ISD::BF16_TO_FP, {MVT::i16, MVT::f32, MVT::f64}, Expand); in SITargetLowering()
543 setOperationAction(ISD::FP_TO_BF16, {MVT::i16, MVT::f32, MVT::f64}, Expand); in SITargetLowering()
547 setOperationAction({ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, MVT::f32, Custom); in SITargetLowering()
548 setOperationAction({ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, MVT::f64, Custom); in SITargetLowering()
553 MVT::i16, Legal); in SITargetLowering()
555 AddPromotedToType(ISD::SIGN_EXTEND, MVT::i16, MVT::i32); in SITargetLowering()
558 MVT::i16, Expand); in SITargetLowering()
564 MVT::i16, Promote); in SITargetLowering()
566 setOperationAction(ISD::LOAD, MVT::i16, Custom); in SITargetLowering()
568 setTruncStoreAction(MVT::i64, MVT::i16, Expand); in SITargetLowering()
570 setOperationAction(ISD::FP16_TO_FP, MVT::i16, Promote); in SITargetLowering()
571 AddPromotedToType(ISD::FP16_TO_FP, MVT::i16, MVT::i32); in SITargetLowering()
572 setOperationAction(ISD::FP_TO_FP16, MVT::i16, Promote); in SITargetLowering()
573 AddPromotedToType(ISD::FP_TO_FP16, MVT::i16, MVT::i32); in SITargetLowering()
575 setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, MVT::i16, Custom); in SITargetLowering()
576 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i16, Custom); in SITargetLowering()
577 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i16, Custom); in SITargetLowering()
579 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i32, Custom); in SITargetLowering()
582 setOperationAction(ISD::ConstantFP, MVT::f16, Legal); in SITargetLowering()
583 setOperationAction(ISD::ConstantFP, MVT::bf16, Legal); in SITargetLowering()
586 setOperationAction(ISD::LOAD, MVT::f16, Promote); in SITargetLowering()
587 AddPromotedToType(ISD::LOAD, MVT::f16, MVT::i16); in SITargetLowering()
588 setOperationAction(ISD::STORE, MVT::f16, Promote); in SITargetLowering()
589 AddPromotedToType(ISD::STORE, MVT::f16, MVT::i16); in SITargetLowering()
592 setOperationAction(ISD::LOAD, MVT::bf16, Promote); in SITargetLowering()
593 AddPromotedToType(ISD::LOAD, MVT::bf16, MVT::i16); in SITargetLowering()
594 setOperationAction(ISD::STORE, MVT::bf16, Promote); in SITargetLowering()
595 AddPromotedToType(ISD::STORE, MVT::bf16, MVT::i16); in SITargetLowering()
600 MVT::f16, Custom); in SITargetLowering()
602 setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, MVT::f16, Promote); in SITargetLowering()
603 setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, MVT::bf16, Promote); in SITargetLowering()
606 setOperationAction({ISD::BR_CC, ISD::SELECT_CC}, {MVT::f16, MVT::bf16}, in SITargetLowering()
608 setOperationAction({ISD::FLDEXP, ISD::STRICT_FLDEXP}, MVT::f16, Custom); in SITargetLowering()
609 setOperationAction(ISD::FFREXP, MVT::f16, Custom); in SITargetLowering()
610 setOperationAction(ISD::FDIV, MVT::f16, Custom); in SITargetLowering()
613 setOperationAction(ISD::FMA, MVT::f16, Legal); in SITargetLowering()
615 setOperationAction(ISD::FMAD, MVT::f16, Legal); in SITargetLowering()
617 for (MVT VT : in SITargetLowering()
618 {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v4i16, MVT::v4f16, in SITargetLowering()
619 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, in SITargetLowering()
620 MVT::v16f16, MVT::v16bf16, MVT::v32i16, MVT::v32f16}) { in SITargetLowering()
646 setOperationAction(ISD::BSWAP, {MVT::i16, MVT::v2i16}, Legal); in SITargetLowering()
647 setOperationAction(ISD::BSWAP, MVT::v4i16, Custom); in SITargetLowering()
650 setOperationAction(ISD::Constant, {MVT::v2i16, MVT::v2f16}, Legal); in SITargetLowering()
652 setOperationAction(ISD::UNDEF, {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, in SITargetLowering()
655 setOperationAction(ISD::STORE, MVT::v2i16, Promote); in SITargetLowering()
656 AddPromotedToType(ISD::STORE, MVT::v2i16, MVT::i32); in SITargetLowering()
657 setOperationAction(ISD::STORE, MVT::v2f16, Promote); in SITargetLowering()
658 AddPromotedToType(ISD::STORE, MVT::v2f16, MVT::i32); in SITargetLowering()
660 setOperationAction(ISD::LOAD, MVT::v2i16, Promote); in SITargetLowering()
661 AddPromotedToType(ISD::LOAD, MVT::v2i16, MVT::i32); in SITargetLowering()
662 setOperationAction(ISD::LOAD, MVT::v2f16, Promote); in SITargetLowering()
663 AddPromotedToType(ISD::LOAD, MVT::v2f16, MVT::i32); in SITargetLowering()
665 setOperationAction(ISD::AND, MVT::v2i16, Promote); in SITargetLowering()
666 AddPromotedToType(ISD::AND, MVT::v2i16, MVT::i32); in SITargetLowering()
667 setOperationAction(ISD::OR, MVT::v2i16, Promote); in SITargetLowering()
668 AddPromotedToType(ISD::OR, MVT::v2i16, MVT::i32); in SITargetLowering()
669 setOperationAction(ISD::XOR, MVT::v2i16, Promote); in SITargetLowering()
670 AddPromotedToType(ISD::XOR, MVT::v2i16, MVT::i32); in SITargetLowering()
672 setOperationAction(ISD::LOAD, MVT::v4i16, Promote); in SITargetLowering()
673 AddPromotedToType(ISD::LOAD, MVT::v4i16, MVT::v2i32); in SITargetLowering()
674 setOperationAction(ISD::LOAD, MVT::v4f16, Promote); in SITargetLowering()
675 AddPromotedToType(ISD::LOAD, MVT::v4f16, MVT::v2i32); in SITargetLowering()
676 setOperationAction(ISD::LOAD, MVT::v4bf16, Promote); in SITargetLowering()
677 AddPromotedToType(ISD::LOAD, MVT::v4bf16, MVT::v2i32); in SITargetLowering()
679 setOperationAction(ISD::STORE, MVT::v4i16, Promote); in SITargetLowering()
680 AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32); in SITargetLowering()
681 setOperationAction(ISD::STORE, MVT::v4f16, Promote); in SITargetLowering()
682 AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32); in SITargetLowering()
683 setOperationAction(ISD::STORE, MVT::v4bf16, Promote); in SITargetLowering()
684 AddPromotedToType(ISD::STORE, MVT::v4bf16, MVT::v2i32); in SITargetLowering()
686 setOperationAction(ISD::LOAD, MVT::v8i16, Promote); in SITargetLowering()
687 AddPromotedToType(ISD::LOAD, MVT::v8i16, MVT::v4i32); in SITargetLowering()
688 setOperationAction(ISD::LOAD, MVT::v8f16, Promote); in SITargetLowering()
689 AddPromotedToType(ISD::LOAD, MVT::v8f16, MVT::v4i32); in SITargetLowering()
690 setOperationAction(ISD::LOAD, MVT::v8bf16, Promote); in SITargetLowering()
691 AddPromotedToType(ISD::LOAD, MVT::v8bf16, MVT::v4i32); in SITargetLowering()
693 setOperationAction(ISD::STORE, MVT::v4i16, Promote); in SITargetLowering()
694 AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32); in SITargetLowering()
695 setOperationAction(ISD::STORE, MVT::v4f16, Promote); in SITargetLowering()
696 AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32); in SITargetLowering()
698 setOperationAction(ISD::STORE, MVT::v8i16, Promote); in SITargetLowering()
699 AddPromotedToType(ISD::STORE, MVT::v8i16, MVT::v4i32); in SITargetLowering()
700 setOperationAction(ISD::STORE, MVT::v8f16, Promote); in SITargetLowering()
701 AddPromotedToType(ISD::STORE, MVT::v8f16, MVT::v4i32); in SITargetLowering()
702 setOperationAction(ISD::STORE, MVT::v8bf16, Promote); in SITargetLowering()
703 AddPromotedToType(ISD::STORE, MVT::v8bf16, MVT::v4i32); in SITargetLowering()
705 setOperationAction(ISD::LOAD, MVT::v16i16, Promote); in SITargetLowering()
706 AddPromotedToType(ISD::LOAD, MVT::v16i16, MVT::v8i32); in SITargetLowering()
707 setOperationAction(ISD::LOAD, MVT::v16f16, Promote); in SITargetLowering()
708 AddPromotedToType(ISD::LOAD, MVT::v16f16, MVT::v8i32); in SITargetLowering()
709 setOperationAction(ISD::LOAD, MVT::v16bf16, Promote); in SITargetLowering()
710 AddPromotedToType(ISD::LOAD, MVT::v16bf16, MVT::v8i32); in SITargetLowering()
712 setOperationAction(ISD::STORE, MVT::v16i16, Promote); in SITargetLowering()
713 AddPromotedToType(ISD::STORE, MVT::v16i16, MVT::v8i32); in SITargetLowering()
714 setOperationAction(ISD::STORE, MVT::v16f16, Promote); in SITargetLowering()
715 AddPromotedToType(ISD::STORE, MVT::v16f16, MVT::v8i32); in SITargetLowering()
716 setOperationAction(ISD::STORE, MVT::v16bf16, Promote); in SITargetLowering()
717 AddPromotedToType(ISD::STORE, MVT::v16bf16, MVT::v8i32); in SITargetLowering()
719 setOperationAction(ISD::LOAD, MVT::v32i16, Promote); in SITargetLowering()
720 AddPromotedToType(ISD::LOAD, MVT::v32i16, MVT::v16i32); in SITargetLowering()
721 setOperationAction(ISD::LOAD, MVT::v32f16, Promote); in SITargetLowering()
722 AddPromotedToType(ISD::LOAD, MVT::v32f16, MVT::v16i32); in SITargetLowering()
723 setOperationAction(ISD::LOAD, MVT::v32bf16, Promote); in SITargetLowering()
724 AddPromotedToType(ISD::LOAD, MVT::v32bf16, MVT::v16i32); in SITargetLowering()
726 setOperationAction(ISD::STORE, MVT::v32i16, Promote); in SITargetLowering()
727 AddPromotedToType(ISD::STORE, MVT::v32i16, MVT::v16i32); in SITargetLowering()
728 setOperationAction(ISD::STORE, MVT::v32f16, Promote); in SITargetLowering()
729 AddPromotedToType(ISD::STORE, MVT::v32f16, MVT::v16i32); in SITargetLowering()
730 setOperationAction(ISD::STORE, MVT::v32bf16, Promote); in SITargetLowering()
731 AddPromotedToType(ISD::STORE, MVT::v32bf16, MVT::v16i32); in SITargetLowering()
734 MVT::v2i32, Expand); in SITargetLowering()
735 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Expand); in SITargetLowering()
738 MVT::v4i32, Expand); in SITargetLowering()
741 MVT::v8i32, Expand); in SITargetLowering()
745 {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Custom); in SITargetLowering()
747 setOperationAction(ISD::FNEG, MVT::v2f16, Legal); in SITargetLowering()
750 setOperationAction(ISD::FABS, MVT::v2f16, Legal); in SITargetLowering()
752 setOperationAction({ISD::FMAXNUM, ISD::FMINNUM}, MVT::f16, Custom); in SITargetLowering()
753 setOperationAction({ISD::FMAXNUM_IEEE, ISD::FMINNUM_IEEE}, MVT::f16, Legal); in SITargetLowering()
756 {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16}, in SITargetLowering()
760 {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16}, in SITargetLowering()
763 for (MVT Vec16 : in SITargetLowering()
764 {MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16, in SITargetLowering()
765 MVT::v16bf16, MVT::v32i16, MVT::v32f16, MVT::v32bf16}) { in SITargetLowering()
777 MVT::v2i16, Legal); in SITargetLowering()
781 MVT::v2f16, Legal); in SITargetLowering()
783 setOperationAction(ISD::EXTRACT_VECTOR_ELT, {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, in SITargetLowering()
787 {MVT::v4f16, MVT::v4i16, MVT::v8f16, MVT::v8i16, in SITargetLowering()
788 MVT::v16f16, MVT::v16i16, MVT::v32f16, MVT::v32i16}, in SITargetLowering()
791 for (MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16}) in SITargetLowering()
799 for (MVT VT : {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16}) in SITargetLowering()
804 setOperationAction({ISD::FMAXNUM, ISD::FMINNUM}, {MVT::v2f16, MVT::v4f16}, in SITargetLowering()
807 setOperationAction(ISD::FEXP, MVT::v2f16, Custom); in SITargetLowering()
808 setOperationAction(ISD::SELECT, {MVT::v4i16, MVT::v4f16, MVT::v4bf16}, in SITargetLowering()
813 MVT::v2f32, Legal); in SITargetLowering()
815 {MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32}, in SITargetLowering()
820 setOperationAction({ISD::FNEG, ISD::FABS}, MVT::v4f16, Custom); in SITargetLowering()
823 setOperationAction(ISD::SELECT, MVT::v2i16, Promote); in SITargetLowering()
824 AddPromotedToType(ISD::SELECT, MVT::v2i16, MVT::i32); in SITargetLowering()
825 setOperationAction(ISD::SELECT, MVT::v2f16, Promote); in SITargetLowering()
826 AddPromotedToType(ISD::SELECT, MVT::v2f16, MVT::i32); in SITargetLowering()
829 setOperationAction(ISD::SELECT, {MVT::v2i16, MVT::v2f16}, Custom); in SITargetLowering()
831 setOperationAction({ISD::FNEG, ISD::FABS}, MVT::v2f16, Custom); in SITargetLowering()
835 {MVT::v4i16, MVT::v4f16, MVT::v4bf16, MVT::v2i8, MVT::v4i8, in SITargetLowering()
836 MVT::v8i8, MVT::v8i16, MVT::v8f16, MVT::v8bf16, in SITargetLowering()
837 MVT::v16i16, MVT::v16f16, MVT::v16bf16, MVT::v32i16, in SITargetLowering()
838 MVT::v32f16, MVT::v32bf16}, in SITargetLowering()
841 setOperationAction({ISD::SMULO, ISD::UMULO}, MVT::i64, Custom); in SITargetLowering()
844 setOperationAction(ISD::MUL, MVT::i64, Custom); in SITargetLowering()
847 setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, MVT::i32, Custom); in SITargetLowering()
850 setOperationAction(ISD::PREFETCH, MVT::Other, Custom); in SITargetLowering()
854 {MVT::f16, MVT::f32, MVT::f64, MVT::v2f16}, Legal); in SITargetLowering()
857 {MVT::Other, MVT::f32, MVT::v4f32, MVT::i16, MVT::f16, in SITargetLowering()
858 MVT::v2i16, MVT::v2f16, MVT::i128, MVT::i8}, in SITargetLowering()
862 {MVT::v2f16, MVT::v2i16, MVT::v3f16, MVT::v3i16, in SITargetLowering()
863 MVT::v4f16, MVT::v4i16, MVT::v8f16, MVT::Other, MVT::f16, in SITargetLowering()
864 MVT::i16, MVT::i8, MVT::i128}, in SITargetLowering()
868 {MVT::Other, MVT::v2i16, MVT::v2f16, MVT::v3i16, in SITargetLowering()
869 MVT::v3f16, MVT::v4f16, MVT::v4i16, MVT::f16, MVT::i16, in SITargetLowering()
870 MVT::i8, MVT::i128}, in SITargetLowering()
873 setOperationAction(ISD::STACKSAVE, MVT::Other, Custom); in SITargetLowering()
874 setOperationAction(ISD::GET_ROUNDING, MVT::i32, Custom); in SITargetLowering()
878 setOperationAction(ISD::GET_FPMODE, MVT::i32, Legal); in SITargetLowering()
880 setOperationAction(ISD::MUL, MVT::i1, Promote); in SITargetLowering()
965 DestVT.getScalarType() == MVT::f32 && in isFPExtFoldable()
966 SrcVT.getScalarType() == MVT::f16 && in isFPExtFoldable()
987 MVT SITargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context, in getRegisterTypeForCallingConv()
999 return MVT::v2i16; in getRegisterTypeForCallingConv()
1000 return (ScalarVT == MVT::bf16 ? MVT::i32 : MVT::v2f16); in getRegisterTypeForCallingConv()
1002 return VT.isInteger() ? MVT::i32 : MVT::f32; in getRegisterTypeForCallingConv()
1006 return Subtarget->has16BitInsts() ? MVT::i16 : MVT::i32; in getRegisterTypeForCallingConv()
1007 return Size == 32 ? ScalarVT.getSimpleVT() : MVT::i32; in getRegisterTypeForCallingConv()
1011 return MVT::i32; in getRegisterTypeForCallingConv()
1045 unsigned &NumIntermediates, MVT &RegisterVT) const { in getVectorTypeBreakdownForCallingConv()
1054 if (ScalarVT == MVT::bf16) { in getVectorTypeBreakdownForCallingConv()
1055 RegisterVT = MVT::i32; in getVectorTypeBreakdownForCallingConv()
1056 IntermediateVT = MVT::v2bf16; in getVectorTypeBreakdownForCallingConv()
1058 RegisterVT = VT.isInteger() ? MVT::v2i16 : MVT::v2f16; in getVectorTypeBreakdownForCallingConv()
1074 RegisterVT = MVT::i16; in getVectorTypeBreakdownForCallingConv()
1082 RegisterVT = MVT::i32; in getVectorTypeBreakdownForCallingConv()
1089 RegisterVT = MVT::i32; in getVectorTypeBreakdownForCallingConv()
1132 MVT SITargetLowering::getPointerTy(const DataLayout &DL, unsigned AS) const { in getPointerTy()
1134 return MVT::v5i32; in getPointerTy()
1137 return MVT::v6i32; in getPointerTy()
1144 MVT SITargetLowering::getPointerMemTy(const DataLayout &DL, unsigned AS) const { in getPointerMemTy()
1149 return MVT::v8i32; in getPointerMemTy()
1229 Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType()); in getTgtMemIntrinsic()
1260 Info.memVT = MVT::getVT(CI.getType()); in getTgtMemIntrinsic()
1273 Info.memVT = MVT::getVT(CI.getOperand(0)->getType()); in getTgtMemIntrinsic()
1287 Info.memVT = MVT::getVT(CI.getOperand(0)->getType()); in getTgtMemIntrinsic()
1296 Info.memVT = MVT::getVT(CI.getType()); in getTgtMemIntrinsic()
1309 Info.memVT = MVT::getVT(CI.getType()); in getTgtMemIntrinsic()
1319 Info.memVT = MVT::getVT(CI.getType()); // XXX: what is correct VT? in getTgtMemIntrinsic()
1342 Info.memVT = MVT::getVT(CI.getType()); in getTgtMemIntrinsic()
1353 Info.memVT = MVT::getVT(CI.getType()); in getTgtMemIntrinsic()
1374 Info.memVT = MVT::i32; in getTgtMemIntrinsic()
1402 Info.memVT = MVT::i32; in getTgtMemIntrinsic()
1816 return MVT::v4i32; in getOptimalMemOpType()
1819 return MVT::v2i32; in getOptimalMemOpType()
1822 return MVT::Other; in getOptimalMemOpType()
1854 SITargetLowering::getPreferredVectorAction(MVT VT) const { in getPreferredVectorAction()
1856 VT.getScalarType().bitsLE(MVT::i16)) in getPreferredVectorAction()
1877 if (Subtarget->has16BitInsts() && VT == MVT::i16) { in isTypeDesirableForOp()
1896 if (VT == MVT::i1 && Op == ISD::SETCC) in isTypeDesirableForOp()
1913 MVT PtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS); in lowerKernArgParameterPtr()
1944 return DAG.getConstant(*KnownSize, SL, MVT::i32); in getLDSKernelId()
1959 DAG.getConstant(0, SL, MVT::i32)); in convertArgType()
1998 SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr, PtrInfo, Align(4), in lowerKernargMemParameter()
2002 SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32); in lowerKernargMemParameter()
2003 SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt); in lowerKernargMemParameter()
2031 return DAG.getFrameIndex(FrameIdx, MVT::i32); in lowerStackParameter()
2040 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32); in lowerStackParameter()
2045 MVT MemVT = VA.getValVT(); in lowerStackParameter()
2856 MVT VT = VA.getLocVT(); in LowerFormalArguments()
2896 SDValue Copy = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i32); in LowerFormalArguments()
2898 SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, DL, MVT::i32); in LowerFormalArguments()
2899 SDValue Extract = DAG.getNode(ISD::SRL, DL, MVT::i32, Copy, ShiftAmt); in LowerFormalArguments()
2931 Copy = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i32); in LowerFormalArguments()
2935 DAG.getBuildVector(EVT::getVectorVT(*DAG.getContext(), MVT::i32, in LowerFormalArguments()
2965 DAG.getValueType(MVT::i16)); in LowerFormalArguments()
3046 DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); in LowerFormalArguments()
3152 RetOps.push_back(DAG.getRegister(*I, MVT::i64)); in LowerReturn()
3154 RetOps.push_back(DAG.getRegister(*I, MVT::i32)); in LowerReturn()
3169 return DAG.getNode(Opc, DL, MVT::Other, RetOps); in LowerReturn()
3296 EVT ArgVT = TRI->getSpillSize(*ArgRC) == 8 ? MVT::i64 : MVT::i32; in passSpecialInputs()
3367 InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgX); in passSpecialInputs()
3369 InputReg = DAG.getConstant(0, DL, MVT::i32); in passSpecialInputs()
3375 SDValue Y = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgY); in passSpecialInputs()
3376 Y = DAG.getNode(ISD::SHL, SL, MVT::i32, Y, in passSpecialInputs()
3377 DAG.getShiftAmountConstant(10, MVT::i32, SL)); in passSpecialInputs()
3379 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Y) : Y; in passSpecialInputs()
3384 SDValue Z = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgZ); in passSpecialInputs()
3385 Z = DAG.getNode(ISD::SHL, SL, MVT::i32, Z, in passSpecialInputs()
3386 DAG.getShiftAmountConstant(20, MVT::i32, SL)); in passSpecialInputs()
3388 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Z) : Z; in passSpecialInputs()
3397 InputReg = DAG.getUNDEF(MVT::i32); in passSpecialInputs()
3405 InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, IncomingArg); in passSpecialInputs()
3659 = DAG.getCopyFromReg(Chain, DL, Info->getScratchRSrcReg(), MVT::v4i32); in LowerCall()
3669 MVT PtrVT = MVT::i32; in LowerCall()
3741 MVT::i32); in LowerCall()
3742 DstAddr = DAG.getNode(ISD::ADD, DL, MVT::i32, SP, PtrOff); in LowerCall()
3750 DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i32); in LowerCall()
3768 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); in LowerCall()
3796 Ops.push_back(DAG.getTargetGlobalAddress(GV, DL, MVT::i64)); in LowerCall()
3798 Ops.push_back(DAG.getTargetConstant(0, DL, MVT::i64)); in LowerCall()
3805 Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32)); in LowerCall()
3827 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerCall()
3895 DAG.getConstant(Subtarget->getWavefrontSizeLog2(), dl, MVT::i32)); in lowerDYNAMIC_STACKALLOCImpl()
3927 if (Op.getValueType() != MVT::i32) in LowerSTACKSAVE()
3933 SDValue CopyFromSP = DAG.getCopyFromReg(Op->getOperand(0), SL, SP, MVT::i32); in LowerSTACKSAVE()
3939 DAG.getNode(AMDGPUISD::WAVE_ADDRESS, SL, MVT::i32, CopyFromSP); in LowerSTACKSAVE()
3946 assert(Op.getValueType() == MVT::i32); in lowerGET_ROUNDING()
3950 SDValue GetRoundBothImm = DAG.getTargetConstant(BothRoundHwReg, SL, MVT::i32); in lowerGET_ROUNDING()
3953 DAG.getTargetConstant(Intrinsic::amdgcn_s_getreg, SL, MVT::i32); in lowerGET_ROUNDING()
3986 DAG.getConstant(AMDGPU::FltRoundConversionTable, SL, MVT::i64); in lowerGET_ROUNDING()
3988 SDValue Two = DAG.getConstant(2, SL, MVT::i32); in lowerGET_ROUNDING()
3990 DAG.getNode(ISD::SHL, SL, MVT::i32, GetReg, Two); in lowerGET_ROUNDING()
3995 DAG.getNode(ISD::SRL, SL, MVT::i64, BitTable, RoundModeTimesNumBits); in lowerGET_ROUNDING()
3996 SDValue TruncTable = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, TableValue); in lowerGET_ROUNDING()
3998 SDValue EntryMask = DAG.getConstant(0xf, SL, MVT::i32); in lowerGET_ROUNDING()
4000 DAG.getNode(ISD::AND, SL, MVT::i32, TruncTable, EntryMask); in lowerGET_ROUNDING()
4004 SDValue Four = DAG.getConstant(4, SL, MVT::i32); in lowerGET_ROUNDING()
4006 DAG.getSetCC(SL, MVT::i1, TableEntry, Four, ISD::SETULT); in lowerGET_ROUNDING()
4007 SDValue EnumOffset = DAG.getNode(ISD::ADD, SL, MVT::i32, TableEntry, Four); in lowerGET_ROUNDING()
4008 SDValue Result = DAG.getNode(ISD::SELECT, SL, MVT::i32, IsStandardValue, in lowerGET_ROUNDING()
4037 if (SrcVT.getScalarType() != MVT::bf16) in lowerFP_EXTEND()
5303 case MVT::f32: in hasAtomicFaddRtnForTy()
5305 case MVT::v2f16: in hasAtomicFaddRtnForTy()
5306 case MVT::f64: in hasAtomicFaddRtnForTy()
5329 return MVT::i1; in getSetCCResultType()
5331 return EVT::getVectorVT(Ctx, MVT::i1, VT.getVectorNumElements()); in getSetCCResultType()
5334 MVT SITargetLowering::getScalarShiftAmountTy(const DataLayout &, EVT VT) const { in getScalarShiftAmountTy()
5337 return (VT == MVT::i16) ? MVT::i16 : MVT::i32; in getScalarShiftAmountTy()
5366 case MVT::f32: { in isFMAFasterThanFMulAndFAdd()
5380 case MVT::f64: in isFMAFasterThanFMulAndFAdd()
5382 case MVT::f16: in isFMAFasterThanFMulAndFAdd()
5395 return isFMAFasterThanFMulAndFAdd(MF, MVT::f16); in isFMAFasterThanFMulAndFAdd()
5397 return isFMAFasterThanFMulAndFAdd(MF, MVT::f32); in isFMAFasterThanFMulAndFAdd()
5399 return isFMAFasterThanFMulAndFAdd(MF, MVT::f64); in isFMAFasterThanFMulAndFAdd()
5425 if (VT == MVT::f32) in isFMADLegal()
5428 if (VT == MVT::f16) { in isFMADLegal()
5446 assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4f32 || in splitUnaryVectorOp()
5447 VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i16 || in splitUnaryVectorOp()
5448 VT == MVT::v16f16 || VT == MVT::v8f32 || VT == MVT::v16f32 || in splitUnaryVectorOp()
5449 VT == MVT::v32f32 || VT == MVT::v32i16 || VT == MVT::v32f16); in splitUnaryVectorOp()
5469 assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4f32 || in splitBinaryVectorOp()
5470 VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i16 || in splitBinaryVectorOp()
5471 VT == MVT::v16f16 || VT == MVT::v8f32 || VT == MVT::v16f32 || in splitBinaryVectorOp()
5472 VT == MVT::v32f32 || VT == MVT::v32i16 || VT == MVT::v32f16); in splitBinaryVectorOp()
5493 assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v8i16 || in splitTernaryVectorOp()
5494 VT == MVT::v8f16 || VT == MVT::v4f32 || VT == MVT::v16i16 || in splitTernaryVectorOp()
5495 VT == MVT::v16f16 || VT == MVT::v8f32 || VT == MVT::v16f32 || in splitTernaryVectorOp()
5496 VT == MVT::v32f32 || VT == MVT::v32f16 || VT == MVT::v32i16 || in splitTernaryVectorOp()
5497 VT == MVT::v4bf16 || VT == MVT::v8bf16 || VT == MVT::v16bf16 || in splitTernaryVectorOp()
5498 VT == MVT::v32bf16); in splitTernaryVectorOp()
5536 if (VT == MVT::f32) in LowerOperation()
5538 if (VT == MVT::f64) in LowerOperation()
5578 if (Op.getOperand(0)->getValueType(0) != MVT::f32) in LowerOperation()
5680 Elt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Elt); in adjustLoadValueTypeImpl()
5684 Elts.push_back(DAG.getUNDEF(MVT::i16)); in adjustLoadValueTypeImpl()
5709 EquivLoadVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, in adjustLoadValueType()
5720 SDVTList VTList = DAG.getVTList(EquivLoadVT, MVT::Other); in adjustLoadValueType()
5771 SDVTList VTList = DAG.getVTList(CastVT, MVT::Other); in lowerIntrinsicLoad()
5794 if (CmpVT == MVT::i16 && !TLI.isTypeLegal(MVT::i16)) { in lowerICMPIntrinsic()
5797 LHS = DAG.getNode(PromoteOp, DL, MVT::i32, LHS); in lowerICMPIntrinsic()
5798 RHS = DAG.getNode(PromoteOp, DL, MVT::i32, RHS); in lowerICMPIntrinsic()
5826 if (CmpVT == MVT::f16 && !TLI.isTypeLegal(CmpVT)) { in lowerFCMPIntrinsic()
5827 Src0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0); in lowerFCMPIntrinsic()
5828 Src1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1); in lowerFCMPIntrinsic()
5875 AMDGPUISD::SETCC, SL, VT, DAG.getZExtOrTrunc(Src, SL, MVT::i32), in lowerBALLOTIntrinsic()
5876 DAG.getConstant(0, SL, MVT::i32), DAG.getCondCode(ISD::SETNE)); in lowerBALLOTIntrinsic()
5903 SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_PKRTZ_F16_F32, SL, MVT::i32, in ReplaceNodeResults()
5905 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Cvt)); in ReplaceNodeResults()
5930 SDValue Cvt = DAG.getNode(Opcode, SL, MVT::i32, Src0, Src1); in ReplaceNodeResults()
5931 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, Cvt)); in ReplaceNodeResults()
5948 assert(VT == MVT::i8 && "Expected 8-bit s_buffer_load intrinsics.\n"); in ReplaceNodeResults()
5965 DAG.getVTList(MVT::i32), Ops, VT, MMO); in ReplaceNodeResults()
5971 DAG.getConstant(0, DL, MVT::i32), // vindex in ReplaceNodeResults()
5976 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in ReplaceNodeResults()
6011 if (NewVT.bitsLT(MVT::i32)) { in ReplaceNodeResults()
6012 LHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, LHS); in ReplaceNodeResults()
6013 RHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, RHS); in ReplaceNodeResults()
6014 SelectVT = MVT::i32; in ReplaceNodeResults()
6026 if (N->getValueType(0) != MVT::v2f16) in ReplaceNodeResults()
6030 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0)); in ReplaceNodeResults()
6032 SDValue Op = DAG.getNode(ISD::XOR, SL, MVT::i32, in ReplaceNodeResults()
6034 DAG.getConstant(0x80008000, SL, MVT::i32)); in ReplaceNodeResults()
6035 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op)); in ReplaceNodeResults()
6039 if (N->getValueType(0) != MVT::v2f16) in ReplaceNodeResults()
6043 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0)); in ReplaceNodeResults()
6045 SDValue Op = DAG.getNode(ISD::AND, SL, MVT::i32, in ReplaceNodeResults()
6047 DAG.getConstant(0x7fff7fff, SL, MVT::i32)); in ReplaceNodeResults()
6048 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op)); in ReplaceNodeResults()
6052 if (N->getValueType(0) != MVT::f16) in ReplaceNodeResults()
6227 MVT VT = Op.getSimpleValueType(); in LowerRETURNADDR()
6257 DAG.getTargetConstant(0, DL, MVT::i32)); in getFPExtOrFPRound()
6261 assert(Op.getValueType() == MVT::f16 && in lowerFP_ROUND()
6266 if (SrcVT != MVT::f64) in lowerFP_ROUND()
6275 SDValue FpToFp16 = DAG.getNode(ISD::FP_TO_FP16, DL, MVT::i32, Src); in lowerFP_ROUND()
6276 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToFp16); in lowerFP_ROUND()
6277 return DAG.getNode(ISD::BITCAST, DL, MVT::f16, Trunc); in lowerFP_ROUND()
6294 if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 || in lowerFMINNUM_FMAXNUM()
6295 VT == MVT::v16f16) in lowerFMINNUM_FMAXNUM()
6303 assert(VT == MVT::f16); in lowerFLDEXP()
6307 if (ExpVT == MVT::i16) in lowerFLDEXP()
6324 SDValue TruncExp = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Clamp); in lowerFLDEXP()
6327 return DAG.getNode(ISD::STRICT_FLDEXP, DL, {VT, MVT::Other}, in lowerFLDEXP()
6342 assert(VT == MVT::i64 && "The following code is a special for s_mul_u64"); in lowerMUL()
6407 SDValue ShiftAmt = DAG.getConstant(C.logBase2(), SL, MVT::i32); in lowerXMULO()
6409 SDValue Overflow = DAG.getSetCC(SL, MVT::i1, in lowerXMULO()
6423 DAG.getConstant(VT.getScalarSizeInBits() - 1, SL, MVT::i32)) in lowerXMULO()
6425 SDValue Overflow = DAG.getSetCC(SL, MVT::i1, Top, Sign, ISD::SETNE); in lowerXMULO()
6458 return DAG.getNode(AMDGPUISD::ENDPGM_TRAP, SL, MVT::Other, Chain); in lowerTrapEndpgm()
6461 SDValue SITargetLowering::loadImplicitKernelArgument(SelectionDAG &DAG, MVT VT, in loadImplicitKernelArgument()
6482 loadImplicitKernelArgument(DAG, MVT::i64, SL, Align(8), QUEUE_PTR); in lowerTrapHsaQueuePtr()
6492 QueuePtr = DAG.getConstant(0, SL, MVT::i64); in lowerTrapHsaQueuePtr()
6495 MVT::i64); in lowerTrapHsaQueuePtr()
6499 SDValue SGPR01 = DAG.getRegister(AMDGPU::SGPR0_SGPR1, MVT::i64); in lowerTrapHsaQueuePtr()
6506 DAG.getTargetConstant(TrapID, SL, MVT::i16), in lowerTrapHsaQueuePtr()
6510 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); in lowerTrapHsaQueuePtr()
6521 DAG.getTargetConstant(TrapID, SL, MVT::i16) in lowerTrapHsa()
6523 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); in lowerTrapHsa()
6545 DAG.getTargetConstant(TrapID, SL, MVT::i16) in lowerDEBUGTRAP()
6547 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); in lowerDEBUGTRAP()
6573 SDNode *Mov = DAG.getMachineNode(AMDGPU::S_MOV_B64, DL, MVT::i64, in getSegmentAperture()
6574 DAG.getRegister(ApertureRegNo, MVT::i64)); in getSegmentAperture()
6576 ISD::TRUNCATE, DL, MVT::i32, in getSegmentAperture()
6577 DAG.getNode(ISD::SRL, DL, MVT::i64, in getSegmentAperture()
6578 {SDValue(Mov, 0), DAG.getConstant(32, DL, MVT::i64)})); in getSegmentAperture()
6587 return loadImplicitKernelArgument(DAG, MVT::i32, DL, Align(4), Param); in getSegmentAperture()
6596 return DAG.getUNDEF(MVT::i32); in getSegmentAperture()
6600 DAG, &AMDGPU::SReg_64RegClass, UserSGPR, MVT::i64); in getSegmentAperture()
6613 return DAG.getLoad(MVT::i32, DL, QueuePtr.getValue(1), Ptr, PtrInfo, in getSegmentAperture()
6641 SDValue FlatNullPtr = DAG.getConstant(0, SL, MVT::i64); in lowerADDRSPACECAST()
6653 SDValue Ptr = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src); in lowerADDRSPACECAST()
6659 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32); in lowerADDRSPACECAST()
6660 SDValue NonNull = DAG.getSetCC(SL, MVT::i1, Src, FlatNullPtr, ISD::SETNE); in lowerADDRSPACECAST()
6662 return DAG.getNode(ISD::SELECT, SL, MVT::i32, NonNull, Ptr, in lowerADDRSPACECAST()
6674 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Aperture); in lowerADDRSPACECAST()
6675 CvtPtr = DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr); in lowerADDRSPACECAST()
6681 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32); in lowerADDRSPACECAST()
6684 = DAG.getSetCC(SL, MVT::i1, Src, SegmentNullPtr, ISD::SETNE); in lowerADDRSPACECAST()
6686 return DAG.getNode(ISD::SELECT, SL, MVT::i64, NonNull, CvtPtr, in lowerADDRSPACECAST()
6692 Op.getValueType() == MVT::i64) { in lowerADDRSPACECAST()
6695 SDValue Hi = DAG.getConstant(Info->get32BitAddressHighBits(), SL, MVT::i32); in lowerADDRSPACECAST()
6696 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Hi); in lowerADDRSPACECAST()
6697 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in lowerADDRSPACECAST()
6701 Src.getValueType() == MVT::i64) in lowerADDRSPACECAST()
6702 return DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src); in lowerADDRSPACECAST()
6737 EVT::getVectorVT(*DAG.getContext(), MVT::i32, VecNumElts / 2); in lowerINSERT_SUBVECTOR()
6738 EVT NewInsVT = InsNumElts == 2 ? MVT::i32 in lowerINSERT_SUBVECTOR()
6740 MVT::i32, InsNumElts / 2); in lowerINSERT_SUBVECTOR()
6750 Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Ins, in lowerINSERT_SUBVECTOR()
6751 DAG.getConstant(I, SL, MVT::i32)); in lowerINSERT_SUBVECTOR()
6754 DAG.getConstant(IdxVal / 2 + I, SL, MVT::i32)); in lowerINSERT_SUBVECTOR()
6762 DAG.getConstant(I, SL, MVT::i32)); in lowerINSERT_SUBVECTOR()
6764 DAG.getConstant(IdxVal + I, SL, MVT::i32)); in lowerINSERT_SUBVECTOR()
6784 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Vec); in lowerINSERT_VECTOR_ELT()
6786 SDValue LoHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec, in lowerINSERT_VECTOR_ELT()
6787 DAG.getConstant(0, SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
6788 SDValue HiHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec, in lowerINSERT_VECTOR_ELT()
6789 DAG.getConstant(1, SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
6791 SDValue LoVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, LoHalf); in lowerINSERT_VECTOR_ELT()
6792 SDValue HiVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, HiHalf); in lowerINSERT_VECTOR_ELT()
6796 SDValue InsHalf = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, MVT::v2i16, in lowerINSERT_VECTOR_ELT()
6798 DAG.getNode(ISD::BITCAST, SL, MVT::i16, InsVal), in lowerINSERT_VECTOR_ELT()
6799 DAG.getConstant(InsertLo ? Idx : (Idx - 2), SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
6801 InsHalf = DAG.getNode(ISD::BITCAST, SL, MVT::i32, InsHalf); in lowerINSERT_VECTOR_ELT()
6804 DAG.getBuildVector(MVT::v2i32, SL, { InsHalf, HiHalf }) : in lowerINSERT_VECTOR_ELT()
6805 DAG.getBuildVector(MVT::v2i32, SL, { LoHalf, InsHalf }); in lowerINSERT_VECTOR_ELT()
6820 MVT IntVT = MVT::getIntegerVT(VecSize); in lowerINSERT_VECTOR_ELT()
6825 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32); in lowerINSERT_VECTOR_ELT()
6826 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor); in lowerINSERT_VECTOR_ELT()
6874 SDValue V2 = DAG.getBitcast(MVT::v2i64, Vec); in lowerEXTRACT_VECTOR_ELT()
6876 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2, in lowerEXTRACT_VECTOR_ELT()
6877 DAG.getConstant(0, SL, MVT::i32))); in lowerEXTRACT_VECTOR_ELT()
6879 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2, in lowerEXTRACT_VECTOR_ELT()
6880 DAG.getConstant(1, SL, MVT::i32))); in lowerEXTRACT_VECTOR_ELT()
6882 SDValue V2 = DAG.getBitcast(MVT::v4i64, Vec); in lowerEXTRACT_VECTOR_ELT()
6885 Parts[P] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2, in lowerEXTRACT_VECTOR_ELT()
6886 DAG.getConstant(P, SL, MVT::i32)); in lowerEXTRACT_VECTOR_ELT()
6889 Lo = DAG.getBitcast(LoVT, DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i64, in lowerEXTRACT_VECTOR_ELT()
6891 Hi = DAG.getBitcast(HiVT, DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i64, in lowerEXTRACT_VECTOR_ELT()
6896 SDValue V2 = DAG.getBitcast(MVT::v8i64, Vec); in lowerEXTRACT_VECTOR_ELT()
6899 Parts[P] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i64, V2, in lowerEXTRACT_VECTOR_ELT()
6900 DAG.getConstant(P, SL, MVT::i32)); in lowerEXTRACT_VECTOR_ELT()
6904 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v4i64, in lowerEXTRACT_VECTOR_ELT()
6907 DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v4i64, in lowerEXTRACT_VECTOR_ELT()
6922 MVT IntVT = MVT::getIntegerVT(VecSize); in lowerEXTRACT_VECTOR_ELT()
6935 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32); in lowerEXTRACT_VECTOR_ELT()
6938 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor); in lowerEXTRACT_VECTOR_ELT()
6943 if (ResultVT == MVT::f16 || ResultVT == MVT::bf16) { in lowerEXTRACT_VECTOR_ELT()
6944 SDValue Result = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, Elt); in lowerEXTRACT_VECTOR_ELT()
6962 EVT PackVT = ResultVT.isInteger() ? MVT::v2i16 : MVT::v2f16; in lowerVECTOR_SHUFFLE()
6984 DAG.getConstant(EltIdx, SL, MVT::i32)); in lowerVECTOR_SHUFFLE()
6996 Vec0, DAG.getConstant(EltIdx0, SL, MVT::i32)); in lowerVECTOR_SHUFFLE()
7000 Vec1, DAG.getConstant(EltIdx1, SL, MVT::i32)); in lowerVECTOR_SHUFFLE()
7029 if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v8i16 || in lowerBUILD_VECTOR()
7030 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) { in lowerBUILD_VECTOR()
7031 EVT HalfVT = MVT::getVectorVT(VT.getVectorElementType().getSimpleVT(), in lowerBUILD_VECTOR()
7033 MVT HalfIntVT = MVT::getIntegerVT(HalfVT.getSizeInBits()); in lowerBUILD_VECTOR()
7048 SDValue Blend = DAG.getBuildVector(MVT::getVectorVT(HalfIntVT, 2), SL, in lowerBUILD_VECTOR()
7053 if (VT == MVT::v16i16 || VT == MVT::v16f16 || VT == MVT::v16bf16) { in lowerBUILD_VECTOR()
7054 EVT QuarterVT = MVT::getVectorVT(VT.getVectorElementType().getSimpleVT(), in lowerBUILD_VECTOR()
7056 MVT QuarterIntVT = MVT::getIntegerVT(QuarterVT.getSizeInBits()); in lowerBUILD_VECTOR()
7070 DAG.getBuildVector(MVT::getVectorVT(QuarterIntVT, 4), SL, Casts); in lowerBUILD_VECTOR()
7074 if (VT == MVT::v32i16 || VT == MVT::v32f16 || VT == MVT::v32bf16) { in lowerBUILD_VECTOR()
7075 EVT QuarterVT = MVT::getVectorVT(VT.getVectorElementType().getSimpleVT(), in lowerBUILD_VECTOR()
7077 MVT QuarterIntVT = MVT::getIntegerVT(QuarterVT.getSizeInBits()); in lowerBUILD_VECTOR()
7091 DAG.getBuildVector(MVT::getVectorVT(QuarterIntVT, 8), SL, Casts); in lowerBUILD_VECTOR()
7095 assert(VT == MVT::v2f16 || VT == MVT::v2i16 || VT == MVT::v2bf16); in lowerBUILD_VECTOR()
7103 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo); in lowerBUILD_VECTOR()
7104 SDValue ExtLo = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Lo); in lowerBUILD_VECTOR()
7108 Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Hi); in lowerBUILD_VECTOR()
7109 Hi = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Hi); in lowerBUILD_VECTOR()
7111 SDValue ShlHi = DAG.getNode(ISD::SHL, SL, MVT::i32, Hi, in lowerBUILD_VECTOR()
7112 DAG.getConstant(16, SL, MVT::i32)); in lowerBUILD_VECTOR()
7116 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo); in lowerBUILD_VECTOR()
7117 Lo = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Lo); in lowerBUILD_VECTOR()
7119 SDValue Or = DAG.getNode(ISD::OR, SL, MVT::i32, Lo, ShlHi); in lowerBUILD_VECTOR()
7160 SDValue PtrLo = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, Offset, GAFlags); in buildPCRelGlobalAddress()
7163 PtrHi = DAG.getTargetConstant(0, DL, MVT::i32); in buildPCRelGlobalAddress()
7165 PtrHi = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, Offset, GAFlags + 1); in buildPCRelGlobalAddress()
7190 assert(PtrVT == MVT::i32 && "32-bit pointer is expected."); in LowerGlobalAddress()
7203 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, GSD->getOffset(), in LowerGlobalAddress()
7205 return DAG.getNode(AMDGPUISD::LDS, DL, MVT::i32, GA); in LowerGlobalAddress()
7210 GV, DL, MVT::i32, GSD->getOffset(), SIInstrInfo::MO_ABS32_LO); in LowerGlobalAddress()
7211 AddrLo = {DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, AddrLo), 0}; in LowerGlobalAddress()
7214 GV, DL, MVT::i32, GSD->getOffset(), SIInstrInfo::MO_ABS32_HI); in LowerGlobalAddress()
7215 AddrHi = {DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, AddrHi), 0}; in LowerGlobalAddress()
7217 return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, AddrLo, AddrHi); in LowerGlobalAddress()
7253 SDNode *M0 = DAG.getMachineNode(AMDGPU::SI_INIT_M0, DL, MVT::Other, MVT::Glue, in copyToM0()
7260 MVT VT, in lowerImplicitZextParam()
7264 DAG, MVT::i32, MVT::i32, SL, DAG.getEntryNode(), Offset, Align(4), false); in lowerImplicitZextParam()
7266 return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Param, in lowerImplicitZextParam()
7291 MVT Type; in getBuildDwordsVector()
7295 Type = MVT::getVectorVT(MVT::f32, NumElts); in getBuildDwordsVector()
7298 Type = MVT::v16f32; in getBuildDwordsVector()
7305 if (Elt.getValueType() != MVT::f32) in getBuildDwordsVector()
7306 Elt = DAG.getBitcast(MVT::f32, Elt); in getBuildDwordsVector()
7310 VecElts[i] = DAG.getUNDEF(MVT::f32); in getBuildDwordsVector()
7353 MVT DataDwordVT = NumDataDwords == 1 ? in constructRetValue()
7354 MVT::i32 : MVT::getVectorVT(MVT::i32, NumDataDwords); in constructRetValue()
7356 MVT MaskPopVT = MaskPopDwords == 1 ? in constructRetValue()
7357 MVT::i32 : MVT::getVectorVT(MVT::i32, MaskPopDwords); in constructRetValue()
7363 SDValue ZeroIdx = DAG.getConstant(0, DL, MVT::i32); in constructRetValue()
7399 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, SDValue(Result, 0), in constructRetValue()
7400 DAG.getConstant(MaskPopDwords, DL, MVT::i32)); in constructRetValue()
7421 *TFE = DAG.getTargetConstant((Value & 0x1) ? 1 : 0, DL, MVT::i32); in parseTexFail()
7423 *LWE = DAG.getTargetConstant((Value & 0x2) ? 1 : 0, DL, MVT::i32); in parseTexFail()
7430 MVT PackVectorVT, in packImage16bitOpsToDwords()
7446 if (Addr.getValueType() != MVT::i16) in packImage16bitOpsToDwords()
7447 Addr = DAG.getBitcast(MVT::i16, Addr); in packImage16bitOpsToDwords()
7448 Addr = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Addr); in packImage16bitOpsToDwords()
7453 Addr = DAG.getBitcast(MVT::f32, Addr); in packImage16bitOpsToDwords()
7498 VData = DAG.getBuildVector(Is64Bit ? MVT::v2i64 : MVT::v2i32, DL, in lowerImage()
7501 VData = DAG.getBitcast(MVT::v4i32, VData); in lowerImage()
7503 ResultTypes[0] = Is64Bit ? MVT::v2i64 : MVT::v2i32; in lowerImage()
7517 MVT StoreVT = VData.getSimpleValueType(); in lowerImage()
7518 if (StoreVT.getScalarType() == MVT::f16) { in lowerImage()
7530 MVT LoadVT = ResultTypes[0].getSimpleVT(); in lowerImage()
7531 if (LoadVT.getScalarType() == MVT::f16) { in lowerImage()
7560 MVT VAddrVT = in lowerImage()
7562 MVT VAddrScalarVT = VAddrVT.getScalarType(); in lowerImage()
7563 MVT GradPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16; in lowerImage()
7564 IsG16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16; in lowerImage()
7568 MVT AddrPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16; in lowerImage()
7569 IsA16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16; in lowerImage()
7573 if (IsA16 && (Op.getOperand(ArgOffset + I).getValueType() == MVT::f16)) { in lowerImage()
7578 MVT::v2f16, DL, in lowerImage()
7579 {Op.getOperand(ArgOffset + I), DAG.getUNDEF(MVT::f16)}); in lowerImage()
7672 SDValue True = DAG.getTargetConstant(1, DL, MVT::i1); in lowerImage()
7673 SDValue False = DAG.getTargetConstant(0, DL, MVT::i1); in lowerImage()
7716 EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumVDataDwords) in lowerImage()
7717 : MVT::i32; in lowerImage()
7750 Ops.push_back(DAG.getTargetConstant(DMask, DL, MVT::i32)); in lowerImage()
7752 Ops.push_back(DAG.getTargetConstant(DimInfo->Encoding, DL, MVT::i32)); in lowerImage()
7755 Ops.push_back(DAG.getTargetConstant(CPol, DL, MVT::i32)); in lowerImage()
7850 if (VT == MVT::i16 && Subtarget->hasScalarSubwordLoads()) { in lowerSBuffer()
7853 DAG.getVTList(MVT::i32), Ops, VT, MMO); in lowerSBuffer()
7879 DAG.getConstant(0, DL, MVT::i32), // vindex in lowerSBuffer()
7884 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in lowerSBuffer()
7886 if (VT == MVT::i16 && Subtarget->hasScalarSubwordLoads()) { in lowerSBuffer()
7893 MVT LoadVT = VT.getSimpleVT(); in lowerSBuffer()
7895 assert((LoadVT.getScalarType() == MVT::i32 || in lowerSBuffer()
7896 LoadVT.getScalarType() == MVT::f32)); in lowerSBuffer()
7900 LoadVT = MVT::getVectorVT(LoadVT.getScalarType(), 4); in lowerSBuffer()
7903 SDVTList VTList = DAG.getVTList({LoadVT, MVT::Glue}); in lowerSBuffer()
7912 Ops[5] = DAG.getTargetConstant(InstOffset + 16 * i, DL, MVT::i32); in lowerSBuffer()
7928 MVT VT = MVT::i32; in lowerWaveID()
7941 return DAG.getConstant(0, SL, MVT::i32); in lowerWorkitemID()
7943 SDValue Val = loadInputValue(DAG, &AMDGPU::VGPR_32RegClass, MVT::i32, in lowerWorkitemID()
7955 return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Val, in lowerWorkitemID()
8081 return lowerImplicitZextParam(DAG, Op, MVT::i16, in LowerINTRINSIC_WO_CHAIN()
8087 return lowerImplicitZextParam(DAG, Op, MVT::i16, in LowerINTRINSIC_WO_CHAIN()
8093 return lowerImplicitZextParam(DAG, Op, MVT::i16, in LowerINTRINSIC_WO_CHAIN()
8120 SDLoc(Op), MVT::i32); in LowerINTRINSIC_WO_CHAIN()
8186 if (Op.getOperand(1).getValueType() == MVT::i1 && in LowerINTRINSIC_WO_CHAIN()
8238 SDValue Node = DAG.getNode(Opcode, DL, MVT::i32, in LowerINTRINSIC_WO_CHAIN()
8258 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, 0, in LowerINTRINSIC_WO_CHAIN()
8260 return {DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, GA), 0}; in LowerINTRINSIC_WO_CHAIN()
8268 SDValue SrcVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, in LowerINTRINSIC_WO_CHAIN()
8271 SDValue SrcHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, SrcVec, in LowerINTRINSIC_WO_CHAIN()
8272 DAG.getConstant(1, SL, MVT::i32)); in LowerINTRINSIC_WO_CHAIN()
8273 return DAG.getSetCC(SL, MVT::i1, SrcHi, Aperture, ISD::SETEQ); in LowerINTRINSIC_WO_CHAIN()
8276 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, Op.getOperand(1), in LowerINTRINSIC_WO_CHAIN()
8284 SDValue GA = DAG.getTargetGlobalAddress(RelocSymbol, DL, MVT::i32, 0, in LowerINTRINSIC_WO_CHAIN()
8286 return {DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, GA), 0}; in LowerINTRINSIC_WO_CHAIN()
8296 if (Op.getOperand(4).getValueType() == MVT::i32) in LowerINTRINSIC_WO_CHAIN()
8300 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op.getOperand(4), SL, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
8308 if (Op.getOperand(6).getValueType() == MVT::i32) in LowerINTRINSIC_WO_CHAIN()
8312 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op.getOperand(6), SL, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
8332 return DAG.getRegister(AMDGPU::SGPR_NULL, MVT::i32); in selectSOffset()
8349 DAG.getConstant(0, DL, MVT::i32), // vindex in lowerRawBufferAtomicIntrin()
8354 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in lowerRawBufferAtomicIntrin()
8388 DAG.getTargetConstant(1, DL, MVT::i1), // idxen in lowerStructBufferAtomicIntrin()
8451 DAG.getTargetConstant(Offset, DL, MVT::i16), in LowerINTRINSIC_W_CHAIN()
8506 DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy in LowerINTRINSIC_W_CHAIN()
8507 DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
8519 if (LoadVT.getScalarType() == MVT::f16) in LowerINTRINSIC_W_CHAIN()
8524 if (LoadVT.getScalarType() == MVT::i8 || LoadVT.getScalarType() == MVT::i16) in LowerINTRINSIC_W_CHAIN()
8545 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_W_CHAIN()
8550 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
8575 DAG.getTargetConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
8597 DAG.getTargetConstant(Dfmt | (Nfmt << 4), DL, MVT::i32), // format in LowerINTRINSIC_W_CHAIN()
8598 DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy in LowerINTRINSIC_W_CHAIN()
8599 DAG.getTargetConstant(IdxEn, DL, MVT::i1) // idxen in LowerINTRINSIC_W_CHAIN()
8602 if (LoadVT.getScalarType() == MVT::f16) in LowerINTRINSIC_W_CHAIN()
8620 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_W_CHAIN()
8626 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
8629 if (LoadVT.getScalarType() == MVT::f16) in LowerINTRINSIC_W_CHAIN()
8653 DAG.getTargetConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
8656 if (LoadVT.getScalarType() == MVT::f16) in LowerINTRINSIC_W_CHAIN()
8685 DAG.getTargetConstant(Slc << 1, DL, MVT::i32), // cachepolicy in LowerINTRINSIC_W_CHAIN()
8686 DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
8859 DAG.getTargetConstant(Slc << 1, DL, MVT::i32), // cachepolicy in LowerINTRINSIC_W_CHAIN()
8860 DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
8880 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_W_CHAIN()
8885 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
8908 DAG.getTargetConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
8925 assert(NodePtr.getValueType() == MVT::i32 || in LowerINTRINSIC_W_CHAIN()
8926 NodePtr.getValueType() == MVT::i64); in LowerINTRINSIC_W_CHAIN()
8927 assert(RayDir.getValueType() == MVT::v3f16 || in LowerINTRINSIC_W_CHAIN()
8928 RayDir.getValueType() == MVT::v3f32); in LowerINTRINSIC_W_CHAIN()
8938 const bool IsA16 = RayDir.getValueType().getVectorElementType() == MVT::f16; in LowerINTRINSIC_W_CHAIN()
8939 const bool Is64 = NodePtr.getValueType() == MVT::i64; in LowerINTRINSIC_W_CHAIN()
8973 Ops.push_back(DAG.getBitcast(MVT::i32, Lanes[I])); in LowerINTRINSIC_W_CHAIN()
8977 DAG.getBitcast(MVT::i32, in LowerINTRINSIC_W_CHAIN()
8978 DAG.getBuildVector(MVT::v2f16, DL, in LowerINTRINSIC_W_CHAIN()
8984 DAG.getBitcast(MVT::i32, in LowerINTRINSIC_W_CHAIN()
8985 DAG.getBuildVector(MVT::v2f16, DL, in LowerINTRINSIC_W_CHAIN()
8988 DAG.getBitcast(MVT::i32, in LowerINTRINSIC_W_CHAIN()
8989 DAG.getBuildVector(MVT::v2f16, DL, in LowerINTRINSIC_W_CHAIN()
8997 Ops.push_back(DAG.getBitcast(MVT::i32, RayExtent)); in LowerINTRINSIC_W_CHAIN()
9005 MVT::i32, DAG.getBuildVector(MVT::v2f16, DL, in LowerINTRINSIC_W_CHAIN()
9008 Ops.push_back(DAG.getBuildVector(MVT::v3i32, DL, MergedLanes)); in LowerINTRINSIC_W_CHAIN()
9015 DAG.ExtractVectorElements(DAG.getBitcast(MVT::v2i32, NodePtr), Ops, 0, in LowerINTRINSIC_W_CHAIN()
9020 Ops.push_back(DAG.getBitcast(MVT::i32, RayExtent)); in LowerINTRINSIC_W_CHAIN()
9029 SDValue Undef = DAG.getUNDEF(MVT::i32); in LowerINTRINSIC_W_CHAIN()
9034 MVT::getVectorVT(MVT::i32, Ops.size()), DL, Ops); in LowerINTRINSIC_W_CHAIN()
9040 Ops.push_back(DAG.getTargetConstant(IsA16, DL, MVT::i1)); in LowerINTRINSIC_W_CHAIN()
9099 SDValue K = DAG.getTargetConstant(BarID, DL, MVT::i32); in LowerINTRINSIC_W_CHAIN()
9136 EVT OpDWordsVT = EVT::getVectorVT(C, MVT::i32, NumOpDWords); in getMemIntrinsicNode()
9142 SDValue Status = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Op, in getMemIntrinsicNode()
9147 ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Op, ZeroIdx) in getMemIntrinsicNode()
9149 EVT::getVectorVT(C, MVT::i32, NumValueDWords), Op, in getMemIntrinsicNode()
9156 (VT == MVT::v3i32 || VT == MVT::v3f32)) { in getMemIntrinsicNode()
9188 EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElements); in handleD16VData()
9209 DAG.getBuildVector(MVT::v2i16, DL, {Elts[I * 2], Elts[I * 2 + 1]}); in handleD16VData()
9210 SDValue IntPair = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Pair); in handleD16VData()
9216 SDValue Pair = DAG.getBuildVector(MVT::v2i16, DL, in handleD16VData()
9217 {Elts[I * 2], DAG.getUNDEF(MVT::i16)}); in handleD16VData()
9218 SDValue IntPair = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Pair); in handleD16VData()
9223 PackedElts.resize(Elts.size(), DAG.getUNDEF(MVT::i32)); in handleD16VData()
9227 EVT::getVectorVT(*DAG.getContext(), MVT::i32, PackedElts.size()); in handleD16VData()
9270 SDValue Undef = DAG.getUNDEF(MVT::f32); in LowerINTRINSIC_VOID()
9273 DAG.getNode(ISD::BITCAST, DL, MVT::f32, Src0), // src0 in LowerINTRINSIC_VOID()
9274 DAG.getNode(ISD::BITCAST, DL, MVT::f32, Src1), // src1 in LowerINTRINSIC_VOID()
9278 DAG.getTargetConstant(1, DL, MVT::i1), // compr in LowerINTRINSIC_VOID()
9291 return SDValue(DAG.getMachineNode(AMDGPU::WAVE_BARRIER, DL, MVT::Other, in LowerINTRINSIC_VOID()
9298 DAG.getTargetConstant(AMDGPU::Barrier::WORKGROUP, DL, MVT::i32); in LowerINTRINSIC_VOID()
9301 MVT::Other, K, Op.getOperand(0)), in LowerINTRINSIC_VOID()
9304 SDValue(DAG.getMachineNode(AMDGPU::S_BARRIER_WAIT, DL, MVT::Other, K, in LowerINTRINSIC_VOID()
9314 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); in LowerINTRINSIC_VOID()
9330 DAG.getTargetConstant(Dfmt | (Nfmt << 4), DL, MVT::i32), // format in LowerINTRINSIC_VOID()
9331 DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy in LowerINTRINSIC_VOID()
9332 DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen in LowerINTRINSIC_VOID()
9344 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); in LowerINTRINSIC_VOID()
9360 DAG.getTargetConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_VOID()
9372 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); in LowerINTRINSIC_VOID()
9382 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_VOID()
9388 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_VOID()
9400 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); in LowerINTRINSIC_VOID()
9414 DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy in LowerINTRINSIC_VOID()
9415 DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen in LowerINTRINSIC_VOID()
9426 if (VDataType == MVT::i8 || VDataType == MVT::i16) in LowerINTRINSIC_VOID()
9463 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_VOID()
9468 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_VOID()
9519 DAG.getTargetConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_VOID()
9576 Ops.push_back(DAG.getBuildVector(MVT::v2i32, DL, in LowerINTRINSIC_VOID()
9590 DAG.getTargetConstant(Aux & AMDGPU::CPol::ALL, DL, MVT::i8)); // cpol in LowerINTRINSIC_VOID()
9592 Aux & AMDGPU::CPol::SWZ_pregfx12 ? 1 : 0, DL, MVT::i8)); // swz in LowerINTRINSIC_VOID()
9657 RHS.getOperand(0).getValueType() == MVT::i32) { in LowerINTRINSIC_VOID()
9669 DAG.getMachineNode(AMDGPU::V_MOV_B32_e32, DL, MVT::i32, in LowerINTRINSIC_VOID()
9670 DAG.getTargetConstant(0, DL, MVT::i32)), 0); in LowerINTRINSIC_VOID()
9702 return SDValue(DAG.getMachineNode(AMDGPU::SI_END_CF, DL, MVT::Other, in LowerINTRINSIC_VOID()
9734 SDValue K = DAG.getTargetConstant(BarVal, DL, MVT::i32); in LowerINTRINSIC_VOID()
9755 M0Val = DAG.getNode(ISD::SHL, DL, MVT::i32, Op.getOperand(3), in LowerINTRINSIC_VOID()
9756 DAG.getShiftAmountConstant(16, MVT::i32, DL)); in LowerINTRINSIC_VOID()
9762 M0Val = SDValue(DAG.getMachineNode(AMDGPU::S_OR_B32, DL, MVT::i32, in LowerINTRINSIC_VOID()
9820 C1 = cast<ConstantSDNode>(DAG.getTargetConstant(ImmOffset, DL, MVT::i32)); in splitBufferOffsets()
9822 auto OverflowVal = DAG.getConstant(Overflow, DL, MVT::i32); in splitBufferOffsets()
9827 N0 = DAG.getNode(ISD::ADD, DL, MVT::i32, Ops); in splitBufferOffsets()
9832 N0 = DAG.getConstant(0, DL, MVT::i32); in splitBufferOffsets()
9834 C1 = cast<ConstantSDNode>(DAG.getTargetConstant(0, DL, MVT::i32)); in splitBufferOffsets()
9850 Offsets[0] = DAG.getConstant(0, DL, MVT::i32); in setBufferOffsets()
9851 Offsets[1] = DAG.getConstant(SOffset, DL, MVT::i32); in setBufferOffsets()
9852 Offsets[2] = DAG.getTargetConstant(ImmOffset, DL, MVT::i32); in setBufferOffsets()
9864 Offsets[1] = DAG.getConstant(SOffset, DL, MVT::i32); in setBufferOffsets()
9865 Offsets[2] = DAG.getTargetConstant(ImmOffset, DL, MVT::i32); in setBufferOffsets()
9871 ? DAG.getRegister(AMDGPU::SGPR_NULL, MVT::i32) in setBufferOffsets()
9872 : DAG.getConstant(0, DL, MVT::i32); in setBufferOffsets()
9876 Offsets[2] = DAG.getTargetConstant(0, DL, MVT::i32); in setBufferOffsets()
9886 SDValue Rsrc = DAG.getBitcast(MVT::v4i32, MaybePointer); in bufferRsrcPtrToVector()
9901 auto [LowHalf, HighHalf] = DAG.SplitScalar(Pointer, Loc, MVT::i32, MVT::i32); in lowerPointerAsRsrcIntrin()
9902 SDValue Mask = DAG.getConstant(0x0000ffff, Loc, MVT::i32); in lowerPointerAsRsrcIntrin()
9903 SDValue Masked = DAG.getNode(ISD::AND, Loc, MVT::i32, HighHalf, Mask); in lowerPointerAsRsrcIntrin()
9912 ShiftedStride = DAG.getConstant(*ConstStride << 16, Loc, MVT::i32); in lowerPointerAsRsrcIntrin()
9914 SDValue ExtStride = DAG.getAnyExtOrTrunc(Stride, Loc, MVT::i32); in lowerPointerAsRsrcIntrin()
9916 DAG.getNode(ISD::SHL, Loc, MVT::i32, ExtStride, in lowerPointerAsRsrcIntrin()
9917 DAG.getShiftAmountConstant(16, MVT::i32, Loc)); in lowerPointerAsRsrcIntrin()
9919 NewHighHalf = DAG.getNode(ISD::OR, Loc, MVT::i32, Masked, ShiftedStride); in lowerPointerAsRsrcIntrin()
9922 SDValue Rsrc = DAG.getNode(ISD::BUILD_VECTOR, Loc, MVT::v4i32, LowHalf, in lowerPointerAsRsrcIntrin()
9924 SDValue RsrcPtr = DAG.getNode(ISD::BITCAST, Loc, MVT::i128, Rsrc); in lowerPointerAsRsrcIntrin()
9934 unsigned Opc = (LoadVT.getScalarType() == MVT::i8) ? in handleByteShortBufferLoads()
9937 SDVTList ResList = DAG.getVTList(MVT::i32, MVT::Other); in handleByteShortBufferLoads()
9951 if (VDataType == MVT::f16) in handleByteShortBufferStores()
9952 Ops[1] = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Ops[1]); in handleByteShortBufferStores()
9954 SDValue BufferStoreExt = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Ops[1]); in handleByteShortBufferStores()
9956 unsigned Opc = (VDataType == MVT::i8) ? AMDGPUISD::BUFFER_STORE_BYTE : in handleByteShortBufferStores()
10013 ISD::UNINDEXED, ISD::NON_EXTLOAD, MVT::i32, SL, Ld->getChain(), Ptr, in widenLoad()
10014 Ld->getOffset(), Ld->getPointerInfo(), MVT::i32, Ld->getAlign(), in widenLoad()
10027 Cvt = DAG.getNode(ISD::SIGN_EXTEND_INREG, SL, MVT::i32, NewLoad, in widenLoad()
10067 if (MemVT == MVT::i16 && isTypeLegal(MVT::i16)) in LowerLOAD()
10077 EVT RealMemVT = (MemVT == MVT::i1) ? MVT::i8 : MVT::i16; in LowerLOAD()
10079 SDValue NewLD = DAG.getExtLoad(ISD::EXTLOAD, DL, MVT::i32, Chain, in LowerLOAD()
10093 SDValue Elt = DAG.getNode(ISD::SRL, DL, MVT::i32, NewLD, in LowerLOAD()
10094 DAG.getConstant(I, DL, MVT::i32)); in LowerLOAD()
10096 Elts.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Elt)); in LowerLOAD()
10110 assert(Op.getValueType().getVectorElementType() == MVT::i32 && in LowerLOAD()
10233 SDValue Zero = DAG.getConstant(0, DL, MVT::i32); in LowerSELECT()
10234 SDValue One = DAG.getConstant(1, DL, MVT::i32); in LowerSELECT()
10236 SDValue LHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(1)); in LowerSELECT()
10237 SDValue RHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(2)); in LowerSELECT()
10239 SDValue Lo0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, LHS, Zero); in LowerSELECT()
10240 SDValue Lo1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, RHS, Zero); in LowerSELECT()
10242 SDValue Lo = DAG.getSelect(DL, MVT::i32, Cond, Lo0, Lo1); in LowerSELECT()
10244 SDValue Hi0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, LHS, One); in LowerSELECT()
10245 SDValue Hi1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, RHS, One); in LowerSELECT()
10247 SDValue Hi = DAG.getSelect(DL, MVT::i32, Cond, Hi0, Hi1); in LowerSELECT()
10249 SDValue Res = DAG.getBuildVector(MVT::v2i32, DL, {Lo, Hi}); in LowerSELECT()
10270 if (!AllowInaccurateRcp && VT != MVT::f16) in lowerFastUnsafeFDIV()
10299 if (!AllowInaccurateRcp && (VT != MVT::f16 || !Flags.hasAllowReciprocal())) in lowerFastUnsafeFDIV()
10344 SDVTList VTList = DAG.getVTList(VT, MVT::Other, MVT::Glue); in getFPBinOp()
10366 SDVTList VTList = DAG.getVTList(VT, MVT::Other, MVT::Glue); in getFPTernOp()
10387 SDValue CvtSrc0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0); in LowerFDIV16()
10388 SDValue CvtSrc1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1); in LowerFDIV16()
10390 SDValue RcpSrc1 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, CvtSrc1); in LowerFDIV16()
10391 SDValue Quot = DAG.getNode(ISD::FMUL, SL, MVT::f32, CvtSrc0, RcpSrc1); in LowerFDIV16()
10393 SDValue FPRoundFlag = DAG.getTargetConstant(0, SL, MVT::i32); in LowerFDIV16()
10394 SDValue BestQuot = DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Quot, FPRoundFlag); in LowerFDIV16()
10396 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f16, BestQuot, Src1, Src0); in LowerFDIV16()
10406 SDValue r1 = DAG.getNode(ISD::FABS, SL, MVT::f32, RHS, Flags); in lowerFDIV_FAST()
10409 const SDValue K0 = DAG.getConstantFP(K0Val, SL, MVT::f32); in lowerFDIV_FAST()
10412 const SDValue K1 = DAG.getConstantFP(K1Val, SL, MVT::f32); in lowerFDIV_FAST()
10414 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32); in lowerFDIV_FAST()
10417 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f32); in lowerFDIV_FAST()
10421 SDValue r3 = DAG.getNode(ISD::SELECT, SL, MVT::f32, r2, K1, One, Flags); in lowerFDIV_FAST()
10423 r1 = DAG.getNode(ISD::FMUL, SL, MVT::f32, RHS, r3, Flags); in lowerFDIV_FAST()
10426 SDValue r0 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, r1, Flags); in lowerFDIV_FAST()
10428 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f32, LHS, r0, Flags); in lowerFDIV_FAST()
10430 return DAG.getNode(ISD::FMUL, SL, MVT::f32, r3, Mul, Flags); in lowerFDIV_FAST()
10441 return DAG.getTargetConstant(Mode, SDLoc(), MVT::i32); in getSPDenormModeValue()
10459 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32); in LowerFDIV32()
10461 SDVTList ScaleVT = DAG.getVTList(MVT::f32, MVT::i1); in LowerFDIV32()
10469 SDValue ApproxRcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, in LowerFDIV32()
10471 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f32, in LowerFDIV32()
10477 const SDValue BitField = DAG.getTargetConstant(Denorm32Reg, SL, MVT::i32); in LowerFDIV32()
10495 SDVTList BindParamVTs = DAG.getVTList(MVT::Other, MVT::Glue); in LowerFDIV32()
10500 DAG.getVTList(MVT::i32, MVT::Glue), in LowerFDIV32()
10518 SL, MVT::i32); in LowerFDIV32()
10532 SDValue Fma0 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, in LowerFDIV32()
10535 SDValue Fma1 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, Fma0, ApproxRcp, in LowerFDIV32()
10538 SDValue Mul = getFPBinOp(DAG, ISD::FMUL, SL, MVT::f32, NumeratorScaled, in LowerFDIV32()
10541 SDValue Fma2 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Mul, in LowerFDIV32()
10544 SDValue Fma3 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, in LowerFDIV32()
10547 SDValue Fma4 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Fma3, in LowerFDIV32()
10556 DisableDenorm = DAG.getNode(AMDGPUISD::DENORM_MODE, SL, MVT::Other, in LowerFDIV32()
10564 : DAG.getConstant(FP_DENORM_FLUSH_IN_FLUSH_OUT, SL, MVT::i32); in LowerFDIV32()
10567 AMDGPU::S_SETREG_B32, SL, MVT::Other, in LowerFDIV32()
10571 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, in LowerFDIV32()
10577 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f32, in LowerFDIV32()
10580 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f32, Fmas, RHS, LHS, Flags); in LowerFDIV32()
10591 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64); in LowerFDIV64()
10593 SDVTList ScaleVT = DAG.getVTList(MVT::f64, MVT::i1); in LowerFDIV64()
10597 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f64, DivScale0); in LowerFDIV64()
10599 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f64, DivScale0); in LowerFDIV64()
10601 SDValue Fma0 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Rcp, One); in LowerFDIV64()
10603 SDValue Fma1 = DAG.getNode(ISD::FMA, SL, MVT::f64, Rcp, Fma0, Rcp); in LowerFDIV64()
10605 SDValue Fma2 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Fma1, One); in LowerFDIV64()
10609 SDValue Fma3 = DAG.getNode(ISD::FMA, SL, MVT::f64, Fma1, Fma2, Fma1); in LowerFDIV64()
10610 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, DivScale1, Fma3); in LowerFDIV64()
10612 SDValue Fma4 = DAG.getNode(ISD::FMA, SL, MVT::f64, in LowerFDIV64()
10621 const SDValue Hi = DAG.getConstant(1, SL, MVT::i32); in LowerFDIV64()
10624 SDValue NumBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X); in LowerFDIV64()
10625 SDValue DenBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Y); in LowerFDIV64()
10626 SDValue Scale0BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale0); in LowerFDIV64()
10627 SDValue Scale1BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale1); in LowerFDIV64()
10629 SDValue NumHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, NumBC, Hi); in LowerFDIV64()
10630 SDValue DenHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, DenBC, Hi); in LowerFDIV64()
10633 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale0BC, Hi); in LowerFDIV64()
10635 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale1BC, Hi); in LowerFDIV64()
10637 SDValue CmpDen = DAG.getSetCC(SL, MVT::i1, DenHi, Scale0Hi, ISD::SETEQ); in LowerFDIV64()
10638 SDValue CmpNum = DAG.getSetCC(SL, MVT::i1, NumHi, Scale1Hi, ISD::SETEQ); in LowerFDIV64()
10639 Scale = DAG.getNode(ISD::XOR, SL, MVT::i1, CmpNum, CmpDen); in LowerFDIV64()
10644 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f64, in LowerFDIV64()
10647 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f64, Fmas, Y, X); in LowerFDIV64()
10653 if (VT == MVT::f32) in LowerFDIV()
10656 if (VT == MVT::f64) in LowerFDIV()
10659 if (VT == MVT::f16) in LowerFDIV()
10670 EVT InstrExpVT = VT == MVT::f16 ? MVT::i16 : MVT::i32; in LowerFFREXP()
10674 DAG.getTargetConstant(Intrinsic::amdgcn_frexp_mant, dl, MVT::i32), Val); in LowerFFREXP()
10678 DAG.getTargetConstant(Intrinsic::amdgcn_frexp_exp, dl, MVT::i32), Val); in LowerFFREXP()
10685 SDValue IsFinite = DAG.getSetCC(dl, MVT::i1, Fabs, Inf, ISD::SETOLT); in LowerFFREXP()
10700 if (VT == MVT::i1) { in LowerSTORE()
10702 DAG.getSExtOrTrunc(Store->getValue(), DL, MVT::i32), in LowerSTORE()
10703 Store->getBasePtr(), MVT::i1, Store->getMemOperand()); in LowerSTORE()
10707 Store->getValue().getValueType().getScalarType() == MVT::i32); in LowerSTORE()
10779 DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Op.getOperand(0), Flags); in lowerFSQRTF16()
10781 SDValue SqrtID = DAG.getTargetConstant(Intrinsic::amdgcn_sqrt, SL, MVT::i32); in lowerFSQRTF16()
10783 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::f32, SqrtID, Ext, Flags); in lowerFSQRTF16()
10785 return DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Sqrt, in lowerFSQRTF16()
10786 DAG.getTargetConstant(0, SL, MVT::i32), Flags); in lowerFSQRTF16()
10792 MVT VT = Op.getValueType().getSimpleVT(); in lowerFSQRTF32()
10799 DAG.getTargetConstant(Intrinsic::amdgcn_sqrt, DL, MVT::i32), X, Flags); in lowerFSQRTF32()
10803 SDValue NeedScale = DAG.getSetCC(DL, MVT::i1, X, ScaleThreshold, ISD::SETOLT); in lowerFSQRTF32()
10815 DAG.getTargetConstant(Intrinsic::amdgcn_sqrt, DL, MVT::i32); in lowerFSQRTF32()
10818 SDValue SqrtSAsInt = DAG.getNode(ISD::BITCAST, DL, MVT::i32, SqrtS); in lowerFSQRTF32()
10819 SDValue SqrtSNextDownInt = DAG.getNode(ISD::ADD, DL, MVT::i32, SqrtSAsInt, in lowerFSQRTF32()
10820 DAG.getConstant(-1, DL, MVT::i32)); in lowerFSQRTF32()
10829 SDValue SqrtSNextUpInt = DAG.getNode(ISD::ADD, DL, MVT::i32, SqrtSAsInt, in lowerFSQRTF32()
10830 DAG.getConstant(1, DL, MVT::i32)); in lowerFSQRTF32()
10838 SDValue SqrtVPLE0 = DAG.getSetCC(DL, MVT::i1, SqrtVP, Zero, ISD::SETOLE); in lowerFSQRTF32()
10843 SDValue SqrtVPVSGT0 = DAG.getSetCC(DL, MVT::i1, SqrtVS, Zero, ISD::SETOGT); in lowerFSQRTF32()
10872 DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, SqrtX, in lowerFSQRTF32()
10873 DAG.getTargetConstant(fcZero | fcPosInf, DL, MVT::i32)); in lowerFSQRTF32()
10904 SDValue ScaleConstant = DAG.getConstantFP(0x1.0p-767, DL, MVT::f64); in lowerFSQRTF64()
10906 SDValue Scaling = DAG.getSetCC(DL, MVT::i1, X, ScaleConstant, ISD::SETOLT); in lowerFSQRTF64()
10908 SDValue ZeroInt = DAG.getConstant(0, DL, MVT::i32); in lowerFSQRTF64()
10911 SDValue ScaleUpFactor = DAG.getConstant(256, DL, MVT::i32); in lowerFSQRTF64()
10913 DAG.getNode(ISD::SELECT, DL, MVT::i32, Scaling, ScaleUpFactor, ZeroInt); in lowerFSQRTF64()
10914 SDValue SqrtX = DAG.getNode(ISD::FLDEXP, DL, MVT::f64, X, ScaleUp, Flags); in lowerFSQRTF64()
10916 SDValue SqrtY = DAG.getNode(AMDGPUISD::RSQ, DL, MVT::f64, SqrtX); in lowerFSQRTF64()
10918 SDValue SqrtS0 = DAG.getNode(ISD::FMUL, DL, MVT::f64, SqrtX, SqrtY); in lowerFSQRTF64()
10920 SDValue Half = DAG.getConstantFP(0.5, DL, MVT::f64); in lowerFSQRTF64()
10921 SDValue SqrtH0 = DAG.getNode(ISD::FMUL, DL, MVT::f64, SqrtY, Half); in lowerFSQRTF64()
10923 SDValue NegSqrtH0 = DAG.getNode(ISD::FNEG, DL, MVT::f64, SqrtH0); in lowerFSQRTF64()
10924 SDValue SqrtR0 = DAG.getNode(ISD::FMA, DL, MVT::f64, NegSqrtH0, SqrtS0, Half); in lowerFSQRTF64()
10926 SDValue SqrtH1 = DAG.getNode(ISD::FMA, DL, MVT::f64, SqrtH0, SqrtR0, SqrtH0); in lowerFSQRTF64()
10928 SDValue SqrtS1 = DAG.getNode(ISD::FMA, DL, MVT::f64, SqrtS0, SqrtR0, SqrtS0); in lowerFSQRTF64()
10930 SDValue NegSqrtS1 = DAG.getNode(ISD::FNEG, DL, MVT::f64, SqrtS1); in lowerFSQRTF64()
10931 SDValue SqrtD0 = DAG.getNode(ISD::FMA, DL, MVT::f64, NegSqrtS1, SqrtS1, SqrtX); in lowerFSQRTF64()
10933 SDValue SqrtS2 = DAG.getNode(ISD::FMA, DL, MVT::f64, SqrtD0, SqrtH1, SqrtS1); in lowerFSQRTF64()
10935 SDValue NegSqrtS2 = DAG.getNode(ISD::FNEG, DL, MVT::f64, SqrtS2); in lowerFSQRTF64()
10937 DAG.getNode(ISD::FMA, DL, MVT::f64, NegSqrtS2, SqrtS2, SqrtX); in lowerFSQRTF64()
10939 SDValue SqrtRet = DAG.getNode(ISD::FMA, DL, MVT::f64, SqrtD1, SqrtH1, SqrtS2); in lowerFSQRTF64()
10941 SDValue ScaleDownFactor = DAG.getConstant(-128, DL, MVT::i32); in lowerFSQRTF64()
10943 DAG.getNode(ISD::SELECT, DL, MVT::i32, Scaling, ScaleDownFactor, ZeroInt); in lowerFSQRTF64()
10944 SqrtRet = DAG.getNode(ISD::FLDEXP, DL, MVT::f64, SqrtRet, ScaleDown, Flags); in lowerFSQRTF64()
10951 DAG.getNode(ISD::IS_FPCLASS, DL, MVT::i1, SqrtX, in lowerFSQRTF64()
10952 DAG.getTargetConstant(fcZero | fcPosInf, DL, MVT::i32)); in lowerFSQRTF64()
10955 return DAG.getNode(ISD::SELECT, DL, MVT::f64, IsZeroOrInf, SqrtX, SqrtRet, in lowerFSQRTF64()
11005 MVT SimpleVT = VT.getSimpleVT(); in LowerATOMIC_CMP_SWAP()
11006 MVT VecType = MVT::getVectorVT(SimpleVT, 2); in LowerATOMIC_CMP_SWAP()
11023 if (ScalarVT != MVT::f32 && ScalarVT != MVT::f16) in performUCharToFloatCombine()
11036 if (DCI.isAfterLegalizeDAG() && SrcVT == MVT::i32) { in performUCharToFloatCombine()
11038 SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0, DL, MVT::f32, Src); in performUCharToFloatCombine()
11042 if (ScalarVT != MVT::f32) { in performUCharToFloatCombine()
11044 DAG.getTargetConstant(0, DL, MVT::i32)); in performUCharToFloatCombine()
11063 if (MagnitudeOp.getValueType() == MVT::f64) { in performFCopySignCombine()
11064 SDValue MagAsVector = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32, MagnitudeOp); in performFCopySignCombine()
11066 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, MagAsVector, in performFCopySignCombine()
11067 DAG.getConstant(0, DL, MVT::i32)); in performFCopySignCombine()
11069 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, MagAsVector, in performFCopySignCombine()
11070 DAG.getConstant(1, DL, MVT::i32)); in performFCopySignCombine()
11073 DAG.getNode(ISD::FCOPYSIGN, DL, MVT::f32, MagHi, SignOp); in performFCopySignCombine()
11075 SDValue Vector = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2f32, MagLo, HiOp); in performFCopySignCombine()
11077 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, Vector); in performFCopySignCombine()
11080 if (SignOp.getValueType() != MVT::f64) in performFCopySignCombine()
11088 SDValue SignAsVector = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32, SignOp); in performFCopySignCombine()
11090 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, SignAsVector, in performFCopySignCombine()
11091 DAG.getConstant(1, DL, MVT::i32)); in performFCopySignCombine()
11236 if (V.getValueType() != MVT::i1) in isBoolSGPR()
11327 if (VT == MVT::i64 && CRHS) { in performAndCombine()
11333 if (CRHS && VT == MVT::i32) { in performAndCombine()
11348 SDValue BFE = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32, in performAndCombine()
11350 DAG.getConstant(Offset, SL, MVT::i32), in performAndCombine()
11351 DAG.getConstant(Bits, SL, MVT::i32)); in performAndCombine()
11356 DAG.getConstant(NB, SDLoc(CRHS), MVT::i32)); in performAndCombine()
11372 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, LHS.getOperand(0), in performAndCombine()
11373 LHS.getOperand(1), DAG.getConstant(Sel, DL, MVT::i32)); in performAndCombine()
11412 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1, in performAndCombine()
11413 X, DAG.getConstant(Mask, DL, MVT::i32)); in performAndCombine()
11436 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1, RHS.getOperand(0), in performAndCombine()
11437 DAG.getConstant(NewMask, DL, MVT::i32)); in performAndCombine()
11441 if (VT == MVT::i32 && in performAndCombine()
11447 return DAG.getSelect(SDLoc(N), MVT::i32, RHS.getOperand(0), in performAndCombine()
11448 LHS, DAG.getConstant(0, SDLoc(N), MVT::i32)); in performAndCombine()
11453 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() && in performAndCombine()
11492 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, in performAndCombine()
11494 DAG.getConstant(Sel, DL, MVT::i32)); in performAndCombine()
11919 if (VT != MVT::i32) in matchPERM()
11981 return DAG.getBitcast(MVT::getIntegerVT(32), Op); in matchPERM()
11994 Op = DAG.getBitcastedAnyExtOrTrunc(Op, DL, MVT::i32); in matchPERM()
11995 OtherOp = DAG.getBitcastedAnyExtOrTrunc(OtherOp, DL, MVT::i32); in matchPERM()
11997 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, Op, OtherOp, in matchPERM()
11998 DAG.getConstant(PermMask, DL, MVT::i32)); in matchPERM()
12011 if (VT == MVT::i1) { in performOrCombine()
12029 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1, in performOrCombine()
12030 Src, DAG.getConstant(NewMask, DL, MVT::i32)); in performOrCombine()
12046 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, LHS.getOperand(0), in performOrCombine()
12047 LHS.getOperand(1), DAG.getConstant(Sel, DL, MVT::i32)); in performOrCombine()
12052 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() && in performOrCombine()
12111 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, in performOrCombine()
12113 DAG.getConstant(Sel, DL, MVT::i32)); in performOrCombine()
12122 if (VT != MVT::i64 || DCI.isBeforeLegalizeOps()) in performOrCombine()
12137 if (SrcVT == MVT::i32) { in performOrCombine()
12141 SDValue LowOr = DAG.getNode(ISD::OR, SL, MVT::i32, LowLHS, ExtSrc); in performOrCombine()
12146 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in performOrCombine()
12148 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in performOrCombine()
12175 if (CRHS && VT == MVT::i64) { in performXorCombine()
12183 if (LHS.getOpcode() == ISD::SELECT && VT == MVT::i32) { in performXorCombine()
12191 DAG.getNode(ISD::BITCAST, DL, MVT::f32, LHS->getOperand(1)); in performXorCombine()
12193 DAG.getNode(ISD::BITCAST, DL, MVT::f32, LHS->getOperand(2)); in performXorCombine()
12194 SDValue FNegLHS = DAG.getNode(ISD::FNEG, DL, MVT::f32, CastLHS); in performXorCombine()
12195 SDValue FNegRHS = DAG.getNode(ISD::FNEG, DL, MVT::f32, CastRHS); in performXorCombine()
12196 SDValue NewSelect = DAG.getNode(ISD::SELECT, DL, MVT::f32, in performXorCombine()
12212 if (VT != MVT::i32) in performZeroExtendCombine()
12216 if (Src.getValueType() != MVT::i16) in performZeroExtendCombine()
12231 VTSign->getVT() == MVT::i8) || in performSignExtendInRegCombine()
12233 VTSign->getVT() == MVT::i16))) { in performSignExtendInRegCombine()
12242 SDVTList ResList = DCI.DAG.getVTList(MVT::i32); in performSignExtendInRegCombine()
12254 VTSign->getVT() == MVT::i8) || in performSignExtendInRegCombine()
12256 VTSign->getVT() == MVT::i16)) && in performSignExtendInRegCombine()
12270 SDVTList ResList = DCI.DAG.getVTList(MVT::i32, in performSignExtendInRegCombine()
12291 return DAG.getConstant(0, SDLoc(N), MVT::i1); in performClassCombine()
12294 return DAG.getUNDEF(MVT::i1); in performClassCombine()
12310 if (VT == MVT::f32 && (N0.getOpcode() == ISD::UINT_TO_FP || in performRcpCombine()
12317 if ((VT == MVT::f16 && N0.getOpcode() == ISD::FSQRT) && in performRcpCombine()
12394 return Op.getValueType().getScalarType() != MVT::f16; in isCanonicalized()
12459 if (Op.getValueType() == MVT::i16) { in isCanonicalized()
12461 if (TruncSrc.getValueType() == MVT::i32 && in isCanonicalized()
12463 TruncSrc.getOperand(0).getValueType() == MVT::v2f16) { in isCanonicalized()
12682 if (N0.getOpcode() == ISD::BUILD_VECTOR && VT == MVT::v2f16 && in performFCanonicalizeCombine()
12683 isTypeLegal(MVT::v2f16)) { in performFCanonicalizeCombine()
12798 if (VT == MVT::i32 || (VT == MVT::i16 && Subtarget->hasMed3_16())) in performIntMed3ImmCombine()
12850 if (VT == MVT::f32 || (VT == MVT::f16 && Subtarget->hasMed3_16())) { in performFPMed3ImmCombine()
12887 (VT == MVT::i32 || VT == MVT::f32 || in performMinMaxCombine()
12888 ((VT == MVT::f16 || VT == MVT::i16) && Subtarget->hasMin3Max3_16()))) { in performMinMaxCombine()
12944 (VT == MVT::f32 || VT == MVT::f64 || in performMinMaxCombine()
12945 (VT == MVT::f16 && Subtarget->has16BitInsts()) || in performMinMaxCombine()
12946 (VT == MVT::v2f16 && Subtarget->hasVOP3PInsts())) && in performMinMaxCombine()
13170 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast, in performExtractVectorEltCombine()
13171 DAG.getConstant(EltIdx, SL, MVT::i32)); in performExtractVectorEltCombine()
13173 SDValue Srl = DAG.getNode(ISD::SRL, SL, MVT::i32, Elt, in performExtractVectorEltCombine()
13174 DAG.getConstant(LeftoverBitIdx, SL, MVT::i32)); in performExtractVectorEltCombine()
13225 Src.getOperand(0).getValueType() == MVT::f16) { in strictFPExtFromF16()
13234 return DAG.getConstantFP(Val, SDLoc(Src), MVT::f16); in strictFPExtFromF16()
13247 if (VT != MVT::f16) in performFPRoundCombine()
13251 TruncSrc.getValueType() != MVT::f32 || !TruncSrc.hasOneUse()) in performFPRoundCombine()
13292 if (((VT == MVT::f32 && in getFusedOpcode()
13294 (VT == MVT::f16 && Subtarget->hasMadF16() && in getFusedOpcode()
13315 if (VT != MVT::i32 && VT != MVT::i64) in reassociateScalarOps()
13352 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i1); in getMad64_32()
13431 if (VT != MVT::i64) { in tryFoldToMad64_32()
13432 MulLHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, MulLHS); in tryFoldToMad64_32()
13433 MulRHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, MulRHS); in tryFoldToMad64_32()
13434 AddRHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i64, AddRHS); in tryFoldToMad64_32()
13449 SDValue One = DAG.getConstant(1, SL, MVT::i32); in tryFoldToMad64_32()
13451 auto MulLHSLo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, MulLHS); in tryFoldToMad64_32()
13452 auto MulRHSLo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, MulRHS); in tryFoldToMad64_32()
13454 getMad64_32(DAG, SL, MVT::i64, MulLHSLo, MulRHSLo, AddRHS, MulSignedLo); in tryFoldToMad64_32()
13458 std::tie(AccumLo, AccumHi) = DAG.SplitScalar(Accum, SL, MVT::i32, MVT::i32); in tryFoldToMad64_32()
13462 DAG.getNode(ISD::EXTRACT_ELEMENT, SL, MVT::i32, MulLHS, One); in tryFoldToMad64_32()
13463 SDValue MulHi = DAG.getNode(ISD::MUL, SL, MVT::i32, MulLHSHi, MulRHSLo); in tryFoldToMad64_32()
13464 AccumHi = DAG.getNode(ISD::ADD, SL, MVT::i32, MulHi, AccumHi); in tryFoldToMad64_32()
13469 DAG.getNode(ISD::EXTRACT_ELEMENT, SL, MVT::i32, MulRHS, One); in tryFoldToMad64_32()
13470 SDValue MulHi = DAG.getNode(ISD::MUL, SL, MVT::i32, MulLHSLo, MulRHSHi); in tryFoldToMad64_32()
13471 AccumHi = DAG.getNode(ISD::ADD, SL, MVT::i32, MulHi, AccumHi); in tryFoldToMad64_32()
13474 Accum = DAG.getBuildVector(MVT::v2i32, SL, {AccumLo, AccumHi}); in tryFoldToMad64_32()
13475 Accum = DAG.getBitcast(MVT::i64, Accum); in tryFoldToMad64_32()
13478 if (VT != MVT::i64) in tryFoldToMad64_32()
13593 auto EltVal = DAG.getBitcastedAnyExtOrTrunc(Elt->first, SL, MVT::i32); in resolveSources()
13599 return DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, EltVal, EltVal, in resolveSources()
13600 DAG.getConstant(Elt->second, SL, MVT::i32)); in resolveSources()
13622 DAG.getBitcastedAnyExtOrTrunc(FirstElt->first, SL, MVT::i32); in resolveSources()
13624 DAG.getBitcastedAnyExtOrTrunc(SecondElt->first, SL, MVT::i32); in resolveSources()
13626 Perms.push_back(DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, FirstVal, in resolveSources()
13628 DAG.getConstant(PermMask, SL, MVT::i32))); in resolveSources()
13639 DAG.getBitcastedAnyExtOrTrunc(FirstElt->first, SL, MVT::i32); in resolveSources()
13642 DAG.getNode(AMDGPUISD::PERM, SL, MVT::i32, EltVal, EltVal, in resolveSources()
13643 DAG.getConstant(FirstElt->second, SL, MVT::i32))); in resolveSources()
13650 ? DAG.getNode(ISD::OR, SL, MVT::i32, Perms[0], Perms[1]) in resolveSources()
13807 Src2s.push_back(DAG.getConstant(0, SL, MVT::i32)); in performAddCombine()
13861 Src0 = DAG.getBitcast(MVT::getIntegerVT(32), Src0s.begin()->first); in performAddCombine()
13862 Src1 = DAG.getBitcast(MVT::getIntegerVT(32), Src1s.begin()->first); in performAddCombine()
13873 DAG.getExtOrTrunc(*IsSigned, Src2s[ChainLength - 1], SL, MVT::i32); in performAddCombine()
13877 SL, MVT::i64); in performAddCombine()
13880 auto Dot = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32, IID, Src0, in performAddCombine()
13881 Src1, Src2, DAG.getTargetConstant(0, SL, MVT::i1)); in performAddCombine()
13886 if (VT != MVT::i32 || !DCI.isAfterLegalizeDAG()) in performAddCombine()
13907 SDVTList VTList = DAG.getVTList(MVT::i32, MVT::i1); in performAddCombine()
13908 SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond }; in performAddCombine()
13928 if (VT != MVT::i32) in performSubCombine()
13948 SDVTList VTList = DAG.getVTList(MVT::i32, MVT::i1); in performSubCombine()
13949 SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond }; in performSubCombine()
13968 if (N->getValueType(0) != MVT::i32) in performAddCarrySubCarryCombine()
14083 if (VT != MVT::f16 || !Subtarget->has16BitInsts()) in performFDivCombine()
14119 if (!Subtarget->hasDot7Insts() || VT != MVT::f32) in performFMACombine()
14176 if (Vec1.getValueType() != MVT::v2f16 || Vec2.getValueType() != MVT::v2f16) in performFMACombine()
14181 return DAG.getNode(AMDGPUISD::FDOT2, SL, MVT::f32, Vec1, Vec2, FMAAcc, in performFMACombine()
14182 DAG.getTargetConstant(0, SL, MVT::i1)); in performFMACombine()
14208 if (VT == MVT::i32 && LHS.getOpcode() == ISD::SIGN_EXTEND && in performSetCCCombine()
14218 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
14219 DAG.getConstant(-1, SL, MVT::i1)); in performSetCCCombine()
14244 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
14245 DAG.getConstant(-1, SL, MVT::i1)); in performSetCCCombine()
14252 if (VT != MVT::f32 && VT != MVT::f64 && in performSetCCCombine()
14253 (!Subtarget->has16BitInsts() || VT != MVT::f16)) in performSetCCCombine()
14276 return DAG.getNode(AMDGPUISD::FP_CLASS, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
14277 DAG.getConstant(Mask, SL, MVT::i32)); in performSetCCCombine()
14305 SDLoc(Shift.getOperand(0)), MVT::i32); in performCvtF32UByteNCombine()
14315 MVT::f32, Shifted); in performCvtF32UByteNCombine()
14333 return DAG.getNode(N->getOpcode(), SL, MVT::f32, DemandedSrc); in performCvtF32UByteNCombine()
14401 if (N->getValueType(0) == MVT::i32 && N->isDivergent() && in PerformDAGCombine()
14452 if (VT == MVT::v2i16 || VT == MVT::v2f16 || VT == MVT::v2f16) { in PerformDAGCombine()
14456 if (EltVT != MVT::i16) in PerformDAGCombine()
14457 Src = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Src); in PerformDAGCombine()
14459 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Src); in PerformDAGCombine()
14611 Ops.push_back(DAG.getTargetConstant(NewDmask, SDLoc(Node), MVT::i32)); in adjustWritemask()
14614 MVT SVT = Node->getValueType(0).getVectorElementType().getSimpleVT(); in adjustWritemask()
14616 MVT ResultVT = NewChannels == 1 ? in adjustWritemask()
14617 SVT : MVT::getVectorVT(SVT, NewChannels == 3 ? 4 : in adjustWritemask()
14620 DAG.getVTList(ResultVT, MVT::Other) : DAG.getVTList(ResultVT); in adjustWritemask()
14650 SDValue Op = DAG.getTargetConstant(Idx, SDLoc(User), MVT::i32); in adjustWritemask()
14689 if (SrcVal.getValueType() == MVT::i1 && DestReg->getReg().isPhysical()) { in legalizeTargetIndependentNode()
14693 MRI.createVirtualRegister(&AMDGPU::VReg_1RegClass), MVT::i1); in legalizeTargetIndependentNode()
14758 MVT VT = Src0.getValueType().getSimpleVT(); in PostISelFolding()
14964 SDValue K = DAG.getTargetConstant(Val, DL, MVT::i32); in buildSMovImm32()
14965 return SDValue(DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, K), 0); in buildSMovImm32()
14977 DAG.getTargetConstant(AMDGPU::SGPR_64RegClassID, DL, MVT::i32), in wrapAddr64Rsrc()
14979 DAG.getTargetConstant(AMDGPU::sub0, DL, MVT::i32), in wrapAddr64Rsrc()
14981 DAG.getTargetConstant(AMDGPU::sub1, DL, MVT::i32) in wrapAddr64Rsrc()
14985 MVT::v2i32, Ops0), 0); in wrapAddr64Rsrc()
14989 DAG.getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32), in wrapAddr64Rsrc()
14991 DAG.getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32), in wrapAddr64Rsrc()
14993 DAG.getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32) in wrapAddr64Rsrc()
14996 return DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL, MVT::v4i32, Ops1); in wrapAddr64Rsrc()
15006 SDValue PtrLo = DAG.getTargetExtractSubreg(AMDGPU::sub0, DL, MVT::i32, Ptr); in buildRSRC()
15007 SDValue PtrHi = DAG.getTargetExtractSubreg(AMDGPU::sub1, DL, MVT::i32, Ptr); in buildRSRC()
15009 PtrHi = SDValue(DAG.getMachineNode(AMDGPU::S_OR_B32, DL, MVT::i32, PtrHi, in buildRSRC()
15010 DAG.getConstant(RsrcDword1, DL, MVT::i32)), in buildRSRC()
15019 DAG.getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32), in buildRSRC()
15021 DAG.getTargetConstant(AMDGPU::sub0, DL, MVT::i32), in buildRSRC()
15023 DAG.getTargetConstant(AMDGPU::sub1, DL, MVT::i32), in buildRSRC()
15025 DAG.getTargetConstant(AMDGPU::sub2, DL, MVT::i32), in buildRSRC()
15027 DAG.getTargetConstant(AMDGPU::sub3, DL, MVT::i32) in buildRSRC()
15030 return DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL, MVT::v4i32, Ops); in buildRSRC()
15040 MVT VT) const { in getRegForInlineAsmConstraint()
15094 if (RC && (isTypeLegal(VT) || VT.SimpleTy == MVT::i128 || in getRegForInlineAsmConstraint()
15095 VT.SimpleTy == MVT::i16 || VT.SimpleTy == MVT::f16)) in getRegForInlineAsmConstraint()
15197 Ops.push_back(DAG.getTargetConstant(Val, SDLoc(Op), MVT::i64)); in LowerAsmOperandForConstraint()
15689 case MVT::f32: in denormalsEnabledForType()
15691 case MVT::f64: in denormalsEnabledForType()
15692 case MVT::f16: in denormalsEnabledForType()
15899 SITargetLowering::getRegClassFor(MVT VT, bool isDivergent) const { in getRegClassFor()
16043 if (User->getOperand(Op)->getValueType(ResNo) != MVT::i1) in checkForPhysRegDependency()