Lines Matching refs:MVT
118 addRegisterClass(MVT::i1, &AMDGPU::VReg_1RegClass); in SITargetLowering()
119 addRegisterClass(MVT::i64, &AMDGPU::SReg_64RegClass); in SITargetLowering()
121 addRegisterClass(MVT::i32, &AMDGPU::SReg_32_XM0RegClass); in SITargetLowering()
122 addRegisterClass(MVT::f32, &AMDGPU::VGPR_32RegClass); in SITargetLowering()
124 addRegisterClass(MVT::f64, &AMDGPU::VReg_64RegClass); in SITargetLowering()
125 addRegisterClass(MVT::v2i32, &AMDGPU::SReg_64RegClass); in SITargetLowering()
126 addRegisterClass(MVT::v2f32, &AMDGPU::VReg_64RegClass); in SITargetLowering()
128 addRegisterClass(MVT::v2i64, &AMDGPU::SReg_128RegClass); in SITargetLowering()
129 addRegisterClass(MVT::v2f64, &AMDGPU::SReg_128RegClass); in SITargetLowering()
131 addRegisterClass(MVT::v4i32, &AMDGPU::SReg_128RegClass); in SITargetLowering()
132 addRegisterClass(MVT::v4f32, &AMDGPU::VReg_128RegClass); in SITargetLowering()
134 addRegisterClass(MVT::v8i32, &AMDGPU::SReg_256RegClass); in SITargetLowering()
135 addRegisterClass(MVT::v8f32, &AMDGPU::VReg_256RegClass); in SITargetLowering()
137 addRegisterClass(MVT::v16i32, &AMDGPU::SReg_512RegClass); in SITargetLowering()
138 addRegisterClass(MVT::v16f32, &AMDGPU::VReg_512RegClass); in SITargetLowering()
141 addRegisterClass(MVT::i16, &AMDGPU::SReg_32_XM0RegClass); in SITargetLowering()
142 addRegisterClass(MVT::f16, &AMDGPU::SReg_32_XM0RegClass); in SITargetLowering()
145 addRegisterClass(MVT::v2i16, &AMDGPU::SReg_32_XM0RegClass); in SITargetLowering()
146 addRegisterClass(MVT::v2f16, &AMDGPU::SReg_32_XM0RegClass); in SITargetLowering()
147 addRegisterClass(MVT::v4i16, &AMDGPU::SReg_64RegClass); in SITargetLowering()
148 addRegisterClass(MVT::v4f16, &AMDGPU::SReg_64RegClass); in SITargetLowering()
154 setOperationAction(ISD::LOAD, MVT::v2i32, Custom); in SITargetLowering()
155 setOperationAction(ISD::LOAD, MVT::v4i32, Custom); in SITargetLowering()
156 setOperationAction(ISD::LOAD, MVT::v8i32, Custom); in SITargetLowering()
157 setOperationAction(ISD::LOAD, MVT::v16i32, Custom); in SITargetLowering()
158 setOperationAction(ISD::LOAD, MVT::i1, Custom); in SITargetLowering()
159 setOperationAction(ISD::LOAD, MVT::v32i32, Custom); in SITargetLowering()
161 setOperationAction(ISD::STORE, MVT::v2i32, Custom); in SITargetLowering()
162 setOperationAction(ISD::STORE, MVT::v4i32, Custom); in SITargetLowering()
163 setOperationAction(ISD::STORE, MVT::v8i32, Custom); in SITargetLowering()
164 setOperationAction(ISD::STORE, MVT::v16i32, Custom); in SITargetLowering()
165 setOperationAction(ISD::STORE, MVT::i1, Custom); in SITargetLowering()
166 setOperationAction(ISD::STORE, MVT::v32i32, Custom); in SITargetLowering()
168 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand); in SITargetLowering()
169 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Expand); in SITargetLowering()
170 setTruncStoreAction(MVT::v8i32, MVT::v8i16, Expand); in SITargetLowering()
171 setTruncStoreAction(MVT::v16i32, MVT::v16i16, Expand); in SITargetLowering()
172 setTruncStoreAction(MVT::v32i32, MVT::v32i16, Expand); in SITargetLowering()
173 setTruncStoreAction(MVT::v2i32, MVT::v2i8, Expand); in SITargetLowering()
174 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Expand); in SITargetLowering()
175 setTruncStoreAction(MVT::v8i32, MVT::v8i8, Expand); in SITargetLowering()
176 setTruncStoreAction(MVT::v16i32, MVT::v16i8, Expand); in SITargetLowering()
177 setTruncStoreAction(MVT::v32i32, MVT::v32i8, Expand); in SITargetLowering()
179 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); in SITargetLowering()
180 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); in SITargetLowering()
182 setOperationAction(ISD::SELECT, MVT::i1, Promote); in SITargetLowering()
183 setOperationAction(ISD::SELECT, MVT::i64, Custom); in SITargetLowering()
184 setOperationAction(ISD::SELECT, MVT::f64, Promote); in SITargetLowering()
185 AddPromotedToType(ISD::SELECT, MVT::f64, MVT::i64); in SITargetLowering()
187 setOperationAction(ISD::SELECT_CC, MVT::f32, Expand); in SITargetLowering()
188 setOperationAction(ISD::SELECT_CC, MVT::i32, Expand); in SITargetLowering()
189 setOperationAction(ISD::SELECT_CC, MVT::i64, Expand); in SITargetLowering()
190 setOperationAction(ISD::SELECT_CC, MVT::f64, Expand); in SITargetLowering()
191 setOperationAction(ISD::SELECT_CC, MVT::i1, Expand); in SITargetLowering()
193 setOperationAction(ISD::SETCC, MVT::i1, Promote); in SITargetLowering()
194 setOperationAction(ISD::SETCC, MVT::v2i1, Expand); in SITargetLowering()
195 setOperationAction(ISD::SETCC, MVT::v4i1, Expand); in SITargetLowering()
196 AddPromotedToType(ISD::SETCC, MVT::i1, MVT::i32); in SITargetLowering()
198 setOperationAction(ISD::TRUNCATE, MVT::v2i32, Expand); in SITargetLowering()
199 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Expand); in SITargetLowering()
201 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i1, Custom); in SITargetLowering()
202 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i1, Custom); in SITargetLowering()
203 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Custom); in SITargetLowering()
204 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8, Custom); in SITargetLowering()
205 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Custom); in SITargetLowering()
206 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Custom); in SITargetLowering()
207 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::Other, Custom); in SITargetLowering()
209 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); in SITargetLowering()
210 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::f32, Custom); in SITargetLowering()
211 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v4f32, Custom); in SITargetLowering()
212 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom); in SITargetLowering()
213 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::f16, Custom); in SITargetLowering()
214 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v2i16, Custom); in SITargetLowering()
215 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v2f16, Custom); in SITargetLowering()
217 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v2f16, Custom); in SITargetLowering()
218 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v4f16, Custom); in SITargetLowering()
219 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v8f16, Custom); in SITargetLowering()
220 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); in SITargetLowering()
222 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); in SITargetLowering()
223 setOperationAction(ISD::INTRINSIC_VOID, MVT::v2i16, Custom); in SITargetLowering()
224 setOperationAction(ISD::INTRINSIC_VOID, MVT::v2f16, Custom); in SITargetLowering()
225 setOperationAction(ISD::INTRINSIC_VOID, MVT::v4f16, Custom); in SITargetLowering()
227 setOperationAction(ISD::BRCOND, MVT::Other, Custom); in SITargetLowering()
228 setOperationAction(ISD::BR_CC, MVT::i1, Expand); in SITargetLowering()
229 setOperationAction(ISD::BR_CC, MVT::i32, Expand); in SITargetLowering()
230 setOperationAction(ISD::BR_CC, MVT::i64, Expand); in SITargetLowering()
231 setOperationAction(ISD::BR_CC, MVT::f32, Expand); in SITargetLowering()
232 setOperationAction(ISD::BR_CC, MVT::f64, Expand); in SITargetLowering()
234 setOperationAction(ISD::UADDO, MVT::i32, Legal); in SITargetLowering()
235 setOperationAction(ISD::USUBO, MVT::i32, Legal); in SITargetLowering()
237 setOperationAction(ISD::ADDCARRY, MVT::i32, Legal); in SITargetLowering()
238 setOperationAction(ISD::SUBCARRY, MVT::i32, Legal); in SITargetLowering()
240 setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand); in SITargetLowering()
241 setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand); in SITargetLowering()
242 setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand); in SITargetLowering()
245 setOperationAction(ISD::ADDCARRY, MVT::i64, Legal); in SITargetLowering()
246 setOperationAction(ISD::SUBCARRY, MVT::i64, Legal); in SITargetLowering()
251 for (MVT VT : {MVT::v8i32, MVT::v8f32, MVT::v16i32, MVT::v16f32, in SITargetLowering()
252 MVT::v2i64, MVT::v2f64, MVT::v4i16, MVT::v4f16, MVT::v32i32 }) { in SITargetLowering()
275 setOperationAction(ISD::FP_EXTEND, MVT::v4f32, Expand); in SITargetLowering()
282 for (MVT Vec64 : { MVT::v2i64, MVT::v2f64 }) { in SITargetLowering()
284 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v4i32); in SITargetLowering()
287 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v4i32); in SITargetLowering()
290 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v4i32); in SITargetLowering()
293 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v4i32); in SITargetLowering()
296 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i32, Expand); in SITargetLowering()
297 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8f32, Expand); in SITargetLowering()
298 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i32, Expand); in SITargetLowering()
299 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16f32, Expand); in SITargetLowering()
301 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f16, Custom); in SITargetLowering()
302 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i16, Custom); in SITargetLowering()
306 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i16, Custom); in SITargetLowering()
307 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f16, Custom); in SITargetLowering()
308 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i16, Custom); in SITargetLowering()
309 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f16, Custom); in SITargetLowering()
311 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i16, Custom); in SITargetLowering()
312 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f16, Custom); in SITargetLowering()
313 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i8, Custom); in SITargetLowering()
314 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i8, Custom); in SITargetLowering()
315 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i8, Custom); in SITargetLowering()
317 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i8, Custom); in SITargetLowering()
318 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i8, Custom); in SITargetLowering()
319 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i8, Custom); in SITargetLowering()
321 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i16, Custom); in SITargetLowering()
322 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f16, Custom); in SITargetLowering()
323 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i16, Custom); in SITargetLowering()
324 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f16, Custom); in SITargetLowering()
328 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); in SITargetLowering()
329 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, Custom); in SITargetLowering()
333 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i32, Expand); in SITargetLowering()
334 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i64, Expand); in SITargetLowering()
337 setOperationAction(ISD::ADDRSPACECAST, MVT::i32, Custom); in SITargetLowering()
338 setOperationAction(ISD::ADDRSPACECAST, MVT::i64, Custom); in SITargetLowering()
341 setOperationAction(ISD::BSWAP, MVT::i32, Legal); in SITargetLowering()
342 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); in SITargetLowering()
345 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal); in SITargetLowering()
346 setOperationAction(ISD::TRAP, MVT::Other, Custom); in SITargetLowering()
347 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Custom); in SITargetLowering()
350 setOperationAction(ISD::FLOG, MVT::f16, Custom); in SITargetLowering()
351 setOperationAction(ISD::FEXP, MVT::f16, Custom); in SITargetLowering()
352 setOperationAction(ISD::FLOG10, MVT::f16, Custom); in SITargetLowering()
357 setOperationAction(ISD::FMAD, MVT::f32, Legal); in SITargetLowering()
361 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); in SITargetLowering()
362 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); in SITargetLowering()
366 setOperationAction(ISD::CTPOP, MVT::i32, Expand); in SITargetLowering()
369 setOperationAction(ISD::CTPOP, MVT::i64, Expand); in SITargetLowering()
372 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Custom); in SITargetLowering()
375 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Custom); in SITargetLowering()
388 setOperationAction(ISD::FMINNUM, MVT::f32, Custom); in SITargetLowering()
389 setOperationAction(ISD::FMAXNUM, MVT::f32, Custom); in SITargetLowering()
390 setOperationAction(ISD::FMINNUM, MVT::f64, Custom); in SITargetLowering()
391 setOperationAction(ISD::FMAXNUM, MVT::f64, Custom); in SITargetLowering()
397 setOperationAction(ISD::FMINNUM_IEEE, MVT::f32, Legal); in SITargetLowering()
398 setOperationAction(ISD::FMAXNUM_IEEE, MVT::f32, Legal); in SITargetLowering()
399 setOperationAction(ISD::FMINNUM_IEEE, MVT::f64, Legal); in SITargetLowering()
400 setOperationAction(ISD::FMAXNUM_IEEE, MVT::f64, Legal); in SITargetLowering()
404 setOperationAction(ISD::FTRUNC, MVT::f64, Legal); in SITargetLowering()
405 setOperationAction(ISD::FCEIL, MVT::f64, Legal); in SITargetLowering()
406 setOperationAction(ISD::FRINT, MVT::f64, Legal); in SITargetLowering()
408 setOperationAction(ISD::FCEIL, MVT::f64, Custom); in SITargetLowering()
409 setOperationAction(ISD::FTRUNC, MVT::f64, Custom); in SITargetLowering()
410 setOperationAction(ISD::FRINT, MVT::f64, Custom); in SITargetLowering()
411 setOperationAction(ISD::FFLOOR, MVT::f64, Custom); in SITargetLowering()
414 setOperationAction(ISD::FFLOOR, MVT::f64, Legal); in SITargetLowering()
416 setOperationAction(ISD::FSIN, MVT::f32, Custom); in SITargetLowering()
417 setOperationAction(ISD::FCOS, MVT::f32, Custom); in SITargetLowering()
418 setOperationAction(ISD::FDIV, MVT::f32, Custom); in SITargetLowering()
419 setOperationAction(ISD::FDIV, MVT::f64, Custom); in SITargetLowering()
422 setOperationAction(ISD::Constant, MVT::i16, Legal); in SITargetLowering()
424 setOperationAction(ISD::SMIN, MVT::i16, Legal); in SITargetLowering()
425 setOperationAction(ISD::SMAX, MVT::i16, Legal); in SITargetLowering()
427 setOperationAction(ISD::UMIN, MVT::i16, Legal); in SITargetLowering()
428 setOperationAction(ISD::UMAX, MVT::i16, Legal); in SITargetLowering()
430 setOperationAction(ISD::SIGN_EXTEND, MVT::i16, Promote); in SITargetLowering()
431 AddPromotedToType(ISD::SIGN_EXTEND, MVT::i16, MVT::i32); in SITargetLowering()
433 setOperationAction(ISD::ROTR, MVT::i16, Promote); in SITargetLowering()
434 setOperationAction(ISD::ROTL, MVT::i16, Promote); in SITargetLowering()
436 setOperationAction(ISD::SDIV, MVT::i16, Promote); in SITargetLowering()
437 setOperationAction(ISD::UDIV, MVT::i16, Promote); in SITargetLowering()
438 setOperationAction(ISD::SREM, MVT::i16, Promote); in SITargetLowering()
439 setOperationAction(ISD::UREM, MVT::i16, Promote); in SITargetLowering()
441 setOperationAction(ISD::BSWAP, MVT::i16, Promote); in SITargetLowering()
442 setOperationAction(ISD::BITREVERSE, MVT::i16, Promote); in SITargetLowering()
444 setOperationAction(ISD::CTTZ, MVT::i16, Promote); in SITargetLowering()
445 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16, Promote); in SITargetLowering()
446 setOperationAction(ISD::CTLZ, MVT::i16, Promote); in SITargetLowering()
447 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16, Promote); in SITargetLowering()
448 setOperationAction(ISD::CTPOP, MVT::i16, Promote); in SITargetLowering()
450 setOperationAction(ISD::SELECT_CC, MVT::i16, Expand); in SITargetLowering()
452 setOperationAction(ISD::BR_CC, MVT::i16, Expand); in SITargetLowering()
454 setOperationAction(ISD::LOAD, MVT::i16, Custom); in SITargetLowering()
456 setTruncStoreAction(MVT::i64, MVT::i16, Expand); in SITargetLowering()
458 setOperationAction(ISD::FP16_TO_FP, MVT::i16, Promote); in SITargetLowering()
459 AddPromotedToType(ISD::FP16_TO_FP, MVT::i16, MVT::i32); in SITargetLowering()
460 setOperationAction(ISD::FP_TO_FP16, MVT::i16, Promote); in SITargetLowering()
461 AddPromotedToType(ISD::FP_TO_FP16, MVT::i16, MVT::i32); in SITargetLowering()
463 setOperationAction(ISD::FP_TO_SINT, MVT::i16, Promote); in SITargetLowering()
464 setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote); in SITargetLowering()
465 setOperationAction(ISD::SINT_TO_FP, MVT::i16, Promote); in SITargetLowering()
466 setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote); in SITargetLowering()
469 setOperationAction(ISD::ConstantFP, MVT::f16, Legal); in SITargetLowering()
472 setOperationAction(ISD::LOAD, MVT::f16, Promote); in SITargetLowering()
473 AddPromotedToType(ISD::LOAD, MVT::f16, MVT::i16); in SITargetLowering()
474 setOperationAction(ISD::STORE, MVT::f16, Promote); in SITargetLowering()
475 AddPromotedToType(ISD::STORE, MVT::f16, MVT::i16); in SITargetLowering()
478 setOperationAction(ISD::FP_ROUND, MVT::f16, Custom); in SITargetLowering()
479 setOperationAction(ISD::FCOS, MVT::f16, Promote); in SITargetLowering()
480 setOperationAction(ISD::FSIN, MVT::f16, Promote); in SITargetLowering()
481 setOperationAction(ISD::FP_TO_SINT, MVT::f16, Promote); in SITargetLowering()
482 setOperationAction(ISD::FP_TO_UINT, MVT::f16, Promote); in SITargetLowering()
483 setOperationAction(ISD::SINT_TO_FP, MVT::f16, Promote); in SITargetLowering()
484 setOperationAction(ISD::UINT_TO_FP, MVT::f16, Promote); in SITargetLowering()
485 setOperationAction(ISD::FROUND, MVT::f16, Custom); in SITargetLowering()
488 setOperationAction(ISD::BR_CC, MVT::f16, Expand); in SITargetLowering()
489 setOperationAction(ISD::SELECT_CC, MVT::f16, Expand); in SITargetLowering()
491 setOperationAction(ISD::FDIV, MVT::f16, Custom); in SITargetLowering()
494 setOperationAction(ISD::FMA, MVT::f16, Legal); in SITargetLowering()
496 setOperationAction(ISD::FMAD, MVT::f16, Legal); in SITargetLowering()
498 for (MVT VT : {MVT::v2i16, MVT::v2f16, MVT::v4i16, MVT::v4f16}) { in SITargetLowering()
522 setOperationAction(ISD::Constant, MVT::v2i16, Legal); in SITargetLowering()
523 setOperationAction(ISD::ConstantFP, MVT::v2f16, Legal); in SITargetLowering()
525 setOperationAction(ISD::UNDEF, MVT::v2i16, Legal); in SITargetLowering()
526 setOperationAction(ISD::UNDEF, MVT::v2f16, Legal); in SITargetLowering()
528 setOperationAction(ISD::STORE, MVT::v2i16, Promote); in SITargetLowering()
529 AddPromotedToType(ISD::STORE, MVT::v2i16, MVT::i32); in SITargetLowering()
530 setOperationAction(ISD::STORE, MVT::v2f16, Promote); in SITargetLowering()
531 AddPromotedToType(ISD::STORE, MVT::v2f16, MVT::i32); in SITargetLowering()
533 setOperationAction(ISD::LOAD, MVT::v2i16, Promote); in SITargetLowering()
534 AddPromotedToType(ISD::LOAD, MVT::v2i16, MVT::i32); in SITargetLowering()
535 setOperationAction(ISD::LOAD, MVT::v2f16, Promote); in SITargetLowering()
536 AddPromotedToType(ISD::LOAD, MVT::v2f16, MVT::i32); in SITargetLowering()
538 setOperationAction(ISD::AND, MVT::v2i16, Promote); in SITargetLowering()
539 AddPromotedToType(ISD::AND, MVT::v2i16, MVT::i32); in SITargetLowering()
540 setOperationAction(ISD::OR, MVT::v2i16, Promote); in SITargetLowering()
541 AddPromotedToType(ISD::OR, MVT::v2i16, MVT::i32); in SITargetLowering()
542 setOperationAction(ISD::XOR, MVT::v2i16, Promote); in SITargetLowering()
543 AddPromotedToType(ISD::XOR, MVT::v2i16, MVT::i32); in SITargetLowering()
545 setOperationAction(ISD::LOAD, MVT::v4i16, Promote); in SITargetLowering()
546 AddPromotedToType(ISD::LOAD, MVT::v4i16, MVT::v2i32); in SITargetLowering()
547 setOperationAction(ISD::LOAD, MVT::v4f16, Promote); in SITargetLowering()
548 AddPromotedToType(ISD::LOAD, MVT::v4f16, MVT::v2i32); in SITargetLowering()
550 setOperationAction(ISD::STORE, MVT::v4i16, Promote); in SITargetLowering()
551 AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32); in SITargetLowering()
552 setOperationAction(ISD::STORE, MVT::v4f16, Promote); in SITargetLowering()
553 AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32); in SITargetLowering()
555 setOperationAction(ISD::ANY_EXTEND, MVT::v2i32, Expand); in SITargetLowering()
556 setOperationAction(ISD::ZERO_EXTEND, MVT::v2i32, Expand); in SITargetLowering()
557 setOperationAction(ISD::SIGN_EXTEND, MVT::v2i32, Expand); in SITargetLowering()
558 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Expand); in SITargetLowering()
560 setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Expand); in SITargetLowering()
561 setOperationAction(ISD::ZERO_EXTEND, MVT::v4i32, Expand); in SITargetLowering()
562 setOperationAction(ISD::SIGN_EXTEND, MVT::v4i32, Expand); in SITargetLowering()
565 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i16, Custom); in SITargetLowering()
566 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f16, Custom); in SITargetLowering()
569 setOperationAction(ISD::FNEG, MVT::v2f16, Legal); in SITargetLowering()
572 setOperationAction(ISD::FABS, MVT::v2f16, Legal); in SITargetLowering()
574 setOperationAction(ISD::FMAXNUM, MVT::f16, Custom); in SITargetLowering()
575 setOperationAction(ISD::FMINNUM, MVT::f16, Custom); in SITargetLowering()
576 setOperationAction(ISD::FMAXNUM_IEEE, MVT::f16, Legal); in SITargetLowering()
577 setOperationAction(ISD::FMINNUM_IEEE, MVT::f16, Legal); in SITargetLowering()
579 setOperationAction(ISD::FMINNUM_IEEE, MVT::v4f16, Custom); in SITargetLowering()
580 setOperationAction(ISD::FMAXNUM_IEEE, MVT::v4f16, Custom); in SITargetLowering()
582 setOperationAction(ISD::FMINNUM, MVT::v4f16, Expand); in SITargetLowering()
583 setOperationAction(ISD::FMAXNUM, MVT::v4f16, Expand); in SITargetLowering()
587 setOperationAction(ISD::ADD, MVT::v2i16, Legal); in SITargetLowering()
588 setOperationAction(ISD::SUB, MVT::v2i16, Legal); in SITargetLowering()
589 setOperationAction(ISD::MUL, MVT::v2i16, Legal); in SITargetLowering()
590 setOperationAction(ISD::SHL, MVT::v2i16, Legal); in SITargetLowering()
591 setOperationAction(ISD::SRL, MVT::v2i16, Legal); in SITargetLowering()
592 setOperationAction(ISD::SRA, MVT::v2i16, Legal); in SITargetLowering()
593 setOperationAction(ISD::SMIN, MVT::v2i16, Legal); in SITargetLowering()
594 setOperationAction(ISD::UMIN, MVT::v2i16, Legal); in SITargetLowering()
595 setOperationAction(ISD::SMAX, MVT::v2i16, Legal); in SITargetLowering()
596 setOperationAction(ISD::UMAX, MVT::v2i16, Legal); in SITargetLowering()
598 setOperationAction(ISD::FADD, MVT::v2f16, Legal); in SITargetLowering()
599 setOperationAction(ISD::FMUL, MVT::v2f16, Legal); in SITargetLowering()
600 setOperationAction(ISD::FMA, MVT::v2f16, Legal); in SITargetLowering()
602 setOperationAction(ISD::FMINNUM_IEEE, MVT::v2f16, Legal); in SITargetLowering()
603 setOperationAction(ISD::FMAXNUM_IEEE, MVT::v2f16, Legal); in SITargetLowering()
605 setOperationAction(ISD::FCANONICALIZE, MVT::v2f16, Legal); in SITargetLowering()
607 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i16, Custom); in SITargetLowering()
608 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f16, Custom); in SITargetLowering()
610 setOperationAction(ISD::SHL, MVT::v4i16, Custom); in SITargetLowering()
611 setOperationAction(ISD::SRA, MVT::v4i16, Custom); in SITargetLowering()
612 setOperationAction(ISD::SRL, MVT::v4i16, Custom); in SITargetLowering()
613 setOperationAction(ISD::ADD, MVT::v4i16, Custom); in SITargetLowering()
614 setOperationAction(ISD::SUB, MVT::v4i16, Custom); in SITargetLowering()
615 setOperationAction(ISD::MUL, MVT::v4i16, Custom); in SITargetLowering()
617 setOperationAction(ISD::SMIN, MVT::v4i16, Custom); in SITargetLowering()
618 setOperationAction(ISD::SMAX, MVT::v4i16, Custom); in SITargetLowering()
619 setOperationAction(ISD::UMIN, MVT::v4i16, Custom); in SITargetLowering()
620 setOperationAction(ISD::UMAX, MVT::v4i16, Custom); in SITargetLowering()
622 setOperationAction(ISD::FADD, MVT::v4f16, Custom); in SITargetLowering()
623 setOperationAction(ISD::FMUL, MVT::v4f16, Custom); in SITargetLowering()
625 setOperationAction(ISD::FMAXNUM, MVT::v2f16, Custom); in SITargetLowering()
626 setOperationAction(ISD::FMINNUM, MVT::v2f16, Custom); in SITargetLowering()
628 setOperationAction(ISD::FMINNUM, MVT::v4f16, Custom); in SITargetLowering()
629 setOperationAction(ISD::FMAXNUM, MVT::v4f16, Custom); in SITargetLowering()
630 setOperationAction(ISD::FCANONICALIZE, MVT::v4f16, Custom); in SITargetLowering()
632 setOperationAction(ISD::FEXP, MVT::v2f16, Custom); in SITargetLowering()
633 setOperationAction(ISD::SELECT, MVT::v4i16, Custom); in SITargetLowering()
634 setOperationAction(ISD::SELECT, MVT::v4f16, Custom); in SITargetLowering()
637 setOperationAction(ISD::FNEG, MVT::v4f16, Custom); in SITargetLowering()
638 setOperationAction(ISD::FABS, MVT::v4f16, Custom); in SITargetLowering()
641 setOperationAction(ISD::SELECT, MVT::v2i16, Promote); in SITargetLowering()
642 AddPromotedToType(ISD::SELECT, MVT::v2i16, MVT::i32); in SITargetLowering()
643 setOperationAction(ISD::SELECT, MVT::v2f16, Promote); in SITargetLowering()
644 AddPromotedToType(ISD::SELECT, MVT::v2f16, MVT::i32); in SITargetLowering()
647 setOperationAction(ISD::SELECT, MVT::v2i16, Custom); in SITargetLowering()
648 setOperationAction(ISD::SELECT, MVT::v2f16, Custom); in SITargetLowering()
650 setOperationAction(ISD::FNEG, MVT::v2f16, Custom); in SITargetLowering()
651 setOperationAction(ISD::FABS, MVT::v2f16, Custom); in SITargetLowering()
654 for (MVT VT : { MVT::v4i16, MVT::v4f16, MVT::v2i8, MVT::v4i8, MVT::v8i8 }) { in SITargetLowering()
729 DestVT.getScalarType() == MVT::f32 && !Subtarget->hasFP32Denormals() && in isFPExtFoldable()
730 SrcVT.getScalarType() == MVT::f16; in isFPExtFoldable()
739 MVT SITargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context, in getRegisterTypeForCallingConv()
750 return MVT::i32; in getRegisterTypeForCallingConv()
753 return VT.isInteger() ? MVT::v2i16 : MVT::v2f16; in getRegisterTypeForCallingConv()
783 unsigned &NumIntermediates, MVT &RegisterVT) const { in getVectorTypeBreakdownForCallingConv()
796 RegisterVT = MVT::i32; in getVectorTypeBreakdownForCallingConv()
806 RegisterVT = VT.isInteger() ? MVT::v2i16 : MVT::v2f16; in getVectorTypeBreakdownForCallingConv()
817 static MVT memVTFromAggregate(Type *Ty) { in memVTFromAggregate()
854 return MVT::getVectorVT(MVT::getVT(ElementType, false), in memVTFromAggregate()
885 Info.memVT = MVT::getVT(CI.getType(), true); in getTgtMemIntrinsic()
886 if (Info.memVT == MVT::Other) { in getTgtMemIntrinsic()
894 Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType()); in getTgtMemIntrinsic()
899 Info.memVT = MVT::getVT(CI.getType()); in getTgtMemIntrinsic()
919 Info.memVT = MVT::getVT(CI.getType()); in getTgtMemIntrinsic()
1130 if (VT == MVT::Other || (VT != MVT::Other && VT.getSizeInBits() > 1024 && in allowsMisalignedMemoryAccesses()
1173 if (VT.bitsLT(MVT::i32)) in allowsMisalignedMemoryAccesses()
1182 return VT.bitsGT(MVT::i32) && Align % 4 == 0; in allowsMisalignedMemoryAccesses()
1196 return MVT::v4i32; in getOptimalMemOpType()
1199 return MVT::v2i32; in getOptimalMemOpType()
1202 return MVT::Other; in getOptimalMemOpType()
1240 SITargetLowering::getPreferredVectorAction(MVT VT) const { in getPreferredVectorAction()
1241 if (VT.getVectorNumElements() != 1 && VT.getScalarType().bitsLE(MVT::i16)) in getPreferredVectorAction()
1254 if (Subtarget->has16BitInsts() && VT == MVT::i16) { in isTypeDesirableForOp()
1273 if (VT == MVT::i1 && Op == ISD::SETCC) in isTypeDesirableForOp()
1294 MVT PtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS); in lowerKernArgParameterPtr()
1350 SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr, PtrInfo, 4, in lowerKernargMemParameter()
1354 SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32); in lowerKernargMemParameter()
1355 SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt); in lowerKernargMemParameter()
1383 return DAG.getFrameIndex(FrameIdx, MVT::i32); in lowerStackParameter()
1392 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32); in lowerStackParameter()
1397 MVT MemVT = VA.getValVT(); in lowerStackParameter()
1966 MVT VT = VA.getLocVT(); in LowerFormalArguments()
1987 DAG.getValueType(MVT::i16)); in LowerFormalArguments()
2072 DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); in LowerFormalArguments()
2131 DAG, &AMDGPU::SReg_64RegClass, TRI->getReturnAddressReg(MF), MVT::i64); in LowerReturn()
2137 MVT::i64); in LowerReturn()
2186 RetOps.push_back(DAG.getRegister(*I, MVT::i64)); in LowerReturn()
2188 RetOps.push_back(DAG.getRegister(*I, MVT::i32)); in LowerReturn()
2203 return DAG.getNode(Opc, DL, MVT::Other, RetOps); in LowerReturn()
2323 EVT ArgVT = TRI->getSpillSize(*ArgRC) == 8 ? MVT::i64 : MVT::i32; in passSpecialInputs()
2546 = DAG.getCopyFromReg(Chain, DL, Info->getScratchRSrcReg(), MVT::v4i32); in LowerCall()
2551 = DAG.getCopyFromReg(Chain, DL, OffsetReg, MVT::i32); in LowerCall()
2558 Info->getFrameOffsetReg(), MVT::i32); in LowerCall()
2563 MVT PtrVT = MVT::i32; in LowerCall()
2639 DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i32); in LowerCall()
2659 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); in LowerCall()
2678 DAG, &AMDGPU::SReg_64RegClass, TRI->getReturnAddressReg(MF), MVT::i64); in LowerCall()
2681 MVT::i64); in LowerCall()
2692 DAG.getTargetConstant(NumBytes, DL, MVT::i32), in LowerCall()
2693 DAG.getTargetConstant(0, DL, MVT::i32), in LowerCall()
2706 Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32)); in LowerCall()
2728 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerCall()
2743 SDValue FPReg = DAG.getRegister(Info->getFrameOffsetReg(), MVT::i32); in LowerCall()
2749 Chain = DAG.getCALLSEQ_END(Chain, DAG.getTargetConstant(0, DL, MVT::i32), in LowerCall()
2750 DAG.getTargetConstant(CalleePopBytes, DL, MVT::i32), in LowerCall()
3521 return MVT::i1; in getSetCCResultType()
3523 return EVT::getVectorVT(Ctx, MVT::i1, VT.getVectorNumElements()); in getSetCCResultType()
3526 MVT SITargetLowering::getScalarShiftAmountTy(const DataLayout &, EVT VT) const { in getScalarShiftAmountTy()
3529 return (VT == MVT::i16) ? MVT::i16 : MVT::i32; in getScalarShiftAmountTy()
3551 case MVT::f32: { in isFMAFasterThanFMulAndFAdd()
3562 case MVT::f64: in isFMAFasterThanFMulAndFAdd()
3564 case MVT::f16: in isFMAFasterThanFMulAndFAdd()
3583 assert(VT == MVT::v4f16); in splitUnaryVectorOp()
3603 assert(VT == MVT::v4i16 || VT == MVT::v4f16); in splitBinaryVectorOp()
3701 Elt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Elt); in adjustLoadValueTypeImpl()
3726 EVT::getVectorVT(*DAG.getContext(), MVT::i32, in adjustLoadValueType()
3731 SDVTList VTList = DAG.getVTList(EquivLoadVT, MVT::Other); in adjustLoadValueType()
3767 if (CmpVT == MVT::i16 && !TLI.isTypeLegal(MVT::i16)) { in lowerICMPIntrinsic()
3770 LHS = DAG.getNode(PromoteOp, DL, MVT::i32, LHS); in lowerICMPIntrinsic()
3771 RHS = DAG.getNode(PromoteOp, DL, MVT::i32, RHS); in lowerICMPIntrinsic()
3798 if (CmpVT == MVT::f16 && !TLI.isTypeLegal(CmpVT)) { in lowerFCMPIntrinsic()
3799 Src0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0); in lowerFCMPIntrinsic()
3800 Src1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1); in lowerFCMPIntrinsic()
3830 SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_PKRTZ_F16_F32, SL, MVT::i32, in ReplaceNodeResults()
3832 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Cvt)); in ReplaceNodeResults()
3857 SDValue Cvt = DAG.getNode(Opcode, SL, MVT::i32, Src0, Src1); in ReplaceNodeResults()
3858 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, Cvt)); in ReplaceNodeResults()
3882 if (NewVT.bitsLT(MVT::i32)) { in ReplaceNodeResults()
3883 LHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, LHS); in ReplaceNodeResults()
3884 RHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, RHS); in ReplaceNodeResults()
3885 SelectVT = MVT::i32; in ReplaceNodeResults()
3897 if (N->getValueType(0) != MVT::v2f16) in ReplaceNodeResults()
3901 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0)); in ReplaceNodeResults()
3903 SDValue Op = DAG.getNode(ISD::XOR, SL, MVT::i32, in ReplaceNodeResults()
3905 DAG.getConstant(0x80008000, SL, MVT::i32)); in ReplaceNodeResults()
3906 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op)); in ReplaceNodeResults()
3910 if (N->getValueType(0) != MVT::v2f16) in ReplaceNodeResults()
3914 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0)); in ReplaceNodeResults()
3916 SDValue Op = DAG.getNode(ISD::AND, SL, MVT::i32, in ReplaceNodeResults()
3918 DAG.getConstant(0x7fff7fff, SL, MVT::i32)); in ReplaceNodeResults()
3919 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op)); in ReplaceNodeResults()
4120 assert(Op.getValueType() == MVT::f16 && in lowerFP_ROUND()
4125 if (SrcVT != MVT::f64) in lowerFP_ROUND()
4130 SDValue FpToFp16 = DAG.getNode(ISD::FP_TO_FP16, DL, MVT::i32, Src); in lowerFP_ROUND()
4131 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToFp16); in lowerFP_ROUND()
4132 return DAG.getNode(ISD::BITCAST, DL, MVT::f16, Trunc); in lowerFP_ROUND()
4147 if (VT == MVT::v4f16) in lowerFMINNUM_FMAXNUM()
4158 return DAG.getNode(AMDGPUISD::ENDPGM, SL, MVT::Other, Chain); in lowerTRAP()
4165 DAG, &AMDGPU::SReg_64RegClass, UserSGPR, MVT::i64); in lowerTRAP()
4166 SDValue SGPR01 = DAG.getRegister(AMDGPU::SGPR0_SGPR1, MVT::i64); in lowerTRAP()
4171 DAG.getTargetConstant(GCNSubtarget::TrapIDLLVMTrap, SL, MVT::i16), in lowerTRAP()
4175 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); in lowerTRAP()
4196 DAG.getTargetConstant(GCNSubtarget::TrapIDLLVMDebugTrap, SL, MVT::i16) in lowerDEBUGTRAP()
4198 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); in lowerDEBUGTRAP()
4216 SDValue EncodingImm = DAG.getTargetConstant(Encoding, DL, MVT::i16); in getSegmentAperture()
4218 DAG.getMachineNode(AMDGPU::S_GETREG_B32, DL, MVT::i32, EncodingImm), 0); in getSegmentAperture()
4219 SDValue ShiftAmount = DAG.getTargetConstant(WidthM1 + 1, DL, MVT::i32); in getSegmentAperture()
4220 return DAG.getNode(ISD::SHL, DL, MVT::i32, ApertureReg, ShiftAmount); in getSegmentAperture()
4229 DAG, &AMDGPU::SReg_64RegClass, UserSGPR, MVT::i64); in getSegmentAperture()
4244 return DAG.getLoad(MVT::i32, DL, QueuePtr.getValue(1), Ptr, PtrInfo, in getSegmentAperture()
4256 SDValue FlatNullPtr = DAG.getConstant(0, SL, MVT::i64); in lowerADDRSPACECAST()
4268 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32); in lowerADDRSPACECAST()
4269 SDValue NonNull = DAG.getSetCC(SL, MVT::i1, Src, FlatNullPtr, ISD::SETNE); in lowerADDRSPACECAST()
4270 SDValue Ptr = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src); in lowerADDRSPACECAST()
4272 return DAG.getNode(ISD::SELECT, SL, MVT::i32, in lowerADDRSPACECAST()
4284 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32); in lowerADDRSPACECAST()
4287 = DAG.getSetCC(SL, MVT::i1, Src, SegmentNullPtr, ISD::SETNE); in lowerADDRSPACECAST()
4291 = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Aperture); in lowerADDRSPACECAST()
4293 return DAG.getNode(ISD::SELECT, SL, MVT::i64, NonNull, in lowerADDRSPACECAST()
4294 DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr), in lowerADDRSPACECAST()
4327 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Vec); in lowerINSERT_VECTOR_ELT()
4329 SDValue LoHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec, in lowerINSERT_VECTOR_ELT()
4330 DAG.getConstant(0, SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
4331 SDValue HiHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec, in lowerINSERT_VECTOR_ELT()
4332 DAG.getConstant(1, SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
4334 SDValue LoVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, LoHalf); in lowerINSERT_VECTOR_ELT()
4335 SDValue HiVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, HiHalf); in lowerINSERT_VECTOR_ELT()
4339 SDValue InsHalf = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, MVT::v2i16, in lowerINSERT_VECTOR_ELT()
4341 DAG.getNode(ISD::BITCAST, SL, MVT::i16, InsVal), in lowerINSERT_VECTOR_ELT()
4342 DAG.getConstant(InsertLo ? Idx : (Idx - 2), SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
4344 InsHalf = DAG.getNode(ISD::BITCAST, SL, MVT::i32, InsHalf); in lowerINSERT_VECTOR_ELT()
4347 DAG.getBuildVector(MVT::v2i32, SL, { InsHalf, HiHalf }) : in lowerINSERT_VECTOR_ELT()
4348 DAG.getBuildVector(MVT::v2i32, SL, { LoHalf, InsHalf }); in lowerINSERT_VECTOR_ELT()
4356 MVT IntVT = MVT::getIntegerVT(VecSize); in lowerINSERT_VECTOR_ELT()
4360 if (InsVal.getValueType() == MVT::f16) in lowerINSERT_VECTOR_ELT()
4361 Val = DAG.getNode(ISD::BITCAST, SL, MVT::i16, InsVal); in lowerINSERT_VECTOR_ELT()
4367 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32); in lowerINSERT_VECTOR_ELT()
4370 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor); in lowerINSERT_VECTOR_ELT()
4409 MVT IntVT = MVT::getIntegerVT(VecSize); in lowerEXTRACT_VECTOR_ELT()
4410 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32); in lowerEXTRACT_VECTOR_ELT()
4413 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor); in lowerEXTRACT_VECTOR_ELT()
4418 if (ResultVT == MVT::f16) { in lowerEXTRACT_VECTOR_ELT()
4419 SDValue Result = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, Elt); in lowerEXTRACT_VECTOR_ELT()
4431 if (VT == MVT::v4i16 || VT == MVT::v4f16) { in lowerBUILD_VECTOR()
4432 EVT HalfVT = MVT::getVectorVT(VT.getVectorElementType().getSimpleVT(), 2); in lowerBUILD_VECTOR()
4441 SDValue CastLo = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Lo); in lowerBUILD_VECTOR()
4442 SDValue CastHi = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Hi); in lowerBUILD_VECTOR()
4444 SDValue Blend = DAG.getBuildVector(MVT::v2i32, SL, { CastLo, CastHi }); in lowerBUILD_VECTOR()
4448 assert(VT == MVT::v2f16 || VT == MVT::v2i16); in lowerBUILD_VECTOR()
4456 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo); in lowerBUILD_VECTOR()
4457 SDValue ExtLo = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Lo); in lowerBUILD_VECTOR()
4461 Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Hi); in lowerBUILD_VECTOR()
4462 Hi = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Hi); in lowerBUILD_VECTOR()
4464 SDValue ShlHi = DAG.getNode(ISD::SHL, SL, MVT::i32, Hi, in lowerBUILD_VECTOR()
4465 DAG.getConstant(16, SL, MVT::i32)); in lowerBUILD_VECTOR()
4469 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo); in lowerBUILD_VECTOR()
4470 Lo = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Lo); in lowerBUILD_VECTOR()
4472 SDValue Or = DAG.getNode(ISD::OR, SL, MVT::i32, Lo, ShlHi); in lowerBUILD_VECTOR()
4519 SDValue PtrLo = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, Offset + 4, in buildPCRelGlobalAddress()
4521 SDValue PtrHi = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, Offset + 4, in buildPCRelGlobalAddress()
4573 SDNode *M0 = DAG.getMachineNode(AMDGPU::SI_INIT_M0, DL, MVT::Other, MVT::Glue, in copyToM0()
4580 MVT VT, in lowerImplicitZextParam()
4583 SDValue Param = lowerKernargMemParameter(DAG, MVT::i32, MVT::i32, SL, in lowerImplicitZextParam()
4586 return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Param, in lowerImplicitZextParam()
4611 MVT Type; in getBuildDwordsVector()
4615 Type = MVT::f32; in getBuildDwordsVector()
4618 Type = MVT::v2f32; in getBuildDwordsVector()
4621 Type = MVT::v4f32; in getBuildDwordsVector()
4624 Type = MVT::v8f32; in getBuildDwordsVector()
4628 Type = MVT::v16f32; in getBuildDwordsVector()
4635 if (Elt.getValueType() != MVT::f32) in getBuildDwordsVector()
4636 Elt = DAG.getBitcast(MVT::f32, Elt); in getBuildDwordsVector()
4640 VecElts[i] = DAG.getUNDEF(MVT::f32); in getBuildDwordsVector()
4656 *GLC = DAG.getTargetConstant((Value & 0x1) ? 1 : 0, DL, MVT::i32); in parseCachePolicy()
4660 *SLC = DAG.getTargetConstant((Value & 0x2) ? 1 : 0, DL, MVT::i32); in parseCachePolicy()
4680 EVT AdjEltVT = Unpacked && IsD16 ? MVT::i32 : ReqRetEltVT; in constructRetValue()
4699 if (CastVT == MVT::v8f16) { in constructRetValue()
4700 CastVT = MVT::v4i32; in constructRetValue()
4704 AdjVT = MVT::v2i32; in constructRetValue()
4734 PreTFCRes = DAG.getNode(ISD::BITCAST, DL, MVT::v4f16, PreTFCRes); in constructRetValue()
4762 *TFE = DAG.getTargetConstant((Value & 0x1) ? 1 : 0, DL, MVT::i32); in parseTexFail()
4764 *LWE = DAG.getTargetConstant((Value & 0x2) ? 1 : 0, DL, MVT::i32); in parseTexFail()
4798 bool Is64Bit = VData.getValueType() == MVT::i64; in lowerImage()
4801 VData = DAG.getBuildVector(Is64Bit ? MVT::v2i64 : MVT::v2i32, DL, in lowerImage()
4804 VData = DAG.getBitcast(MVT::v4i32, VData); in lowerImage()
4806 ResultTypes[0] = Is64Bit ? MVT::v2i64 : MVT::v2i32; in lowerImage()
4826 MVT StoreVT = VData.getSimpleValueType(); in lowerImage()
4827 if (StoreVT.getScalarType() == MVT::f16) { in lowerImage()
4840 MVT LoadVT = ResultTypes[0].getSimpleVT(); in lowerImage()
4841 if (LoadVT.getScalarType() == MVT::f16) { in lowerImage()
4887 MVT VAddrVT = Op.getOperand(DimIdx).getSimpleValueType(); in lowerImage()
4888 const MVT VAddrScalarVT = VAddrVT.getScalarType(); in lowerImage()
4889 if (((VAddrScalarVT == MVT::f16) || (VAddrScalarVT == MVT::i16)) && in lowerImage()
4892 const MVT VectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16; in lowerImage()
4906 AddrHi = DAG.getUNDEF(MVT::f16); in lowerImage()
4913 AddrLo = DAG.getBitcast(MVT::i32, AddrLo); in lowerImage()
4924 SDValue True = DAG.getTargetConstant(1, DL, MVT::i1); in lowerImage()
4925 SDValue False = DAG.getTargetConstant(0, DL, MVT::i1); in lowerImage()
4976 EVT::getVectorVT(*DAG.getContext(), MVT::f32, NumVDataDwords) in lowerImage()
4977 : MVT::f32; in lowerImage()
5006 Ops.push_back(DAG.getTargetConstant(DMask, DL, MVT::i32)); in lowerImage()
5076 MVT LoadVT = VT.getSimpleVT(); in lowerSBuffer()
5078 assert((LoadVT.getScalarType() == MVT::i32 || in lowerSBuffer()
5079 LoadVT.getScalarType() == MVT::f32) && in lowerSBuffer()
5084 LoadVT = MVT::v4i32; in lowerSBuffer()
5087 SDVTList VTList = DAG.getVTList({LoadVT, MVT::Glue}); in lowerSBuffer()
5092 DAG.getConstant(0, DL, MVT::i32), // vindex in lowerSBuffer()
5096 DAG.getConstant(CachePolicy, DL, MVT::i32), // cachepolicy in lowerSBuffer()
5097 DAG.getConstant(0, DL, MVT::i1), // idxen in lowerSBuffer()
5106 Ops[5] = DAG.getConstant(InstOffset + 16 * i, DL, MVT::i32); in lowerSBuffer()
5111 if (VT == MVT::v8i32 || VT == MVT::v16i32) in lowerSBuffer()
5229 return lowerImplicitZextParam(DAG, Op, MVT::i16, in LowerINTRINSIC_WO_CHAIN()
5235 return lowerImplicitZextParam(DAG, Op, MVT::i16, in LowerINTRINSIC_WO_CHAIN()
5241 return lowerImplicitZextParam(DAG, Op, MVT::i16, in LowerINTRINSIC_WO_CHAIN()
5257 return loadInputValue(DAG, &AMDGPU::VGPR_32RegClass, MVT::i32, in LowerINTRINSIC_WO_CHAIN()
5262 return loadInputValue(DAG, &AMDGPU::VGPR_32RegClass, MVT::i32, in LowerINTRINSIC_WO_CHAIN()
5267 return loadInputValue(DAG, &AMDGPU::VGPR_32RegClass, MVT::i32, in LowerINTRINSIC_WO_CHAIN()
5272 lowerSBuffer(MVT::i32, DL, Op.getOperand(1), Op.getOperand(2), in LowerINTRINSIC_WO_CHAIN()
5273 DAG.getTargetConstant(0, DL, MVT::i1), DAG); in LowerINTRINSIC_WO_CHAIN()
5274 return DAG.getNode(ISD::BITCAST, DL, MVT::f32, Load); in LowerINTRINSIC_WO_CHAIN()
5279 DAG.getTargetConstant(Cache & 1, DL, MVT::i1), DAG); in LowerINTRINSIC_WO_CHAIN()
5286 return DAG.getNode(AMDGPUISD::INTERP_MOV, DL, MVT::f32, Op.getOperand(1), in LowerINTRINSIC_WO_CHAIN()
5292 return DAG.getNode(AMDGPUISD::INTERP_P1, DL, MVT::f32, Op.getOperand(1), in LowerINTRINSIC_WO_CHAIN()
5298 return DAG.getNode(AMDGPUISD::INTERP_P2, DL, MVT::f32, Op.getOperand(1), in LowerINTRINSIC_WO_CHAIN()
5344 return DAG.getMergeValues({ DAG.getUNDEF(VT), DAG.getUNDEF(MVT::i1) }, DL); in LowerINTRINSIC_WO_CHAIN()
5363 if (Op.getOperand(1).getValueType() == MVT::i1 && in LowerINTRINSIC_WO_CHAIN()
5413 SDValue Node = DAG.getNode(Opcode, DL, MVT::i32, in LowerINTRINSIC_WO_CHAIN()
5495 DAG.getTargetConstant(Offset, DL, MVT::i16), in LowerINTRINSIC_W_CHAIN()
5551 DAG.getConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy in LowerINTRINSIC_W_CHAIN()
5552 DAG.getConstant(IdxEn, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
5564 if (LoadVT.getScalarType() == MVT::f16) in LowerINTRINSIC_W_CHAIN()
5576 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_W_CHAIN()
5581 DAG.getConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
5592 if (LoadVT.getScalarType() == MVT::f16) in LowerINTRINSIC_W_CHAIN()
5609 DAG.getConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
5620 if (LoadVT.getScalarType() == MVT::f16) in LowerINTRINSIC_W_CHAIN()
5644 DAG.getConstant(Dfmt | (Nfmt << 4), DL, MVT::i32), // format in LowerINTRINSIC_W_CHAIN()
5645 DAG.getConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy in LowerINTRINSIC_W_CHAIN()
5646 DAG.getConstant(IdxEn, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
5649 if (LoadVT.getScalarType() == MVT::f16) in LowerINTRINSIC_W_CHAIN()
5664 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_W_CHAIN()
5670 DAG.getConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
5673 if (LoadVT.getScalarType() == MVT::f16) in LowerINTRINSIC_W_CHAIN()
5694 DAG.getConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
5697 if (LoadVT.getScalarType() == MVT::f16) in LowerINTRINSIC_W_CHAIN()
5726 DAG.getConstant(Slc << 1, DL, MVT::i32), // cachepolicy in LowerINTRINSIC_W_CHAIN()
5727 DAG.getConstant(IdxEn, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
5788 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_W_CHAIN()
5793 DAG.getConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
5858 DAG.getConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
5917 DAG.getConstant(Slc << 1, DL, MVT::i32), // cachepolicy in LowerINTRINSIC_W_CHAIN()
5918 DAG.getConstant(IdxEn, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
5934 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_W_CHAIN()
5939 DAG.getConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
5959 DAG.getConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
5993 EVT EquivStoreVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, in handleD16VData()
6019 DAG.getTargetConstant(Tgt->getZExtValue(), DL, MVT::i8), // tgt in LowerINTRINSIC_VOID()
6020 DAG.getTargetConstant(En->getZExtValue(), DL, MVT::i8), // en in LowerINTRINSIC_VOID()
6025 DAG.getTargetConstant(0, DL, MVT::i1), // compr in LowerINTRINSIC_VOID()
6026 DAG.getTargetConstant(VM->getZExtValue(), DL, MVT::i1) in LowerINTRINSIC_VOID()
6041 SDValue Undef = DAG.getUNDEF(MVT::f32); in LowerINTRINSIC_VOID()
6044 DAG.getTargetConstant(Tgt->getZExtValue(), DL, MVT::i8), // tgt in LowerINTRINSIC_VOID()
6045 DAG.getTargetConstant(En->getZExtValue(), DL, MVT::i8), // en in LowerINTRINSIC_VOID()
6046 DAG.getNode(ISD::BITCAST, DL, MVT::f32, Src0), in LowerINTRINSIC_VOID()
6047 DAG.getNode(ISD::BITCAST, DL, MVT::f32, Src1), in LowerINTRINSIC_VOID()
6050 DAG.getTargetConstant(1, DL, MVT::i1), // compr in LowerINTRINSIC_VOID()
6051 DAG.getTargetConstant(VM->getZExtValue(), DL, MVT::i1) in LowerINTRINSIC_VOID()
6064 return DAG.getNode(NodeOp, DL, MVT::Other, Chain, in LowerINTRINSIC_VOID()
6068 return DAG.getNode(AMDGPUISD::INIT_EXEC, DL, MVT::Other, Chain, in LowerINTRINSIC_VOID()
6072 return DAG.getNode(AMDGPUISD::INIT_EXEC_FROM_INPUT, DL, MVT::Other, Chain, in LowerINTRINSIC_VOID()
6080 return SDValue(DAG.getMachineNode(AMDGPU::WAVE_BARRIER, DL, MVT::Other, in LowerINTRINSIC_VOID()
6087 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); in LowerINTRINSIC_VOID()
6105 DAG.getConstant(Dfmt | (Nfmt << 4), DL, MVT::i32), // format in LowerINTRINSIC_VOID()
6106 DAG.getConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy in LowerINTRINSIC_VOID()
6107 DAG.getConstant(IdxEn, DL, MVT::i1), // idexen in LowerINTRINSIC_VOID()
6118 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); in LowerINTRINSIC_VOID()
6132 DAG.getConstant(1, DL, MVT::i1), // idexen in LowerINTRINSIC_VOID()
6143 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); in LowerINTRINSIC_VOID()
6151 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_VOID()
6157 DAG.getConstant(0, DL, MVT::i1), // idexen in LowerINTRINSIC_VOID()
6169 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); in LowerINTRINSIC_VOID()
6185 DAG.getConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy in LowerINTRINSIC_VOID()
6186 DAG.getConstant(IdxEn, DL, MVT::i1), // idxen in LowerINTRINSIC_VOID()
6200 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); in LowerINTRINSIC_VOID()
6208 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_VOID()
6213 DAG.getConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_VOID()
6226 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); in LowerINTRINSIC_VOID()
6239 DAG.getConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_VOID()
6294 C1 = cast<ConstantSDNode>(DAG.getConstant(ImmOffset, DL, MVT::i32)); in splitBufferOffsets()
6296 auto OverflowVal = DAG.getConstant(Overflow, DL, MVT::i32); in splitBufferOffsets()
6301 N0 = DAG.getNode(ISD::ADD, DL, MVT::i32, Ops); in splitBufferOffsets()
6306 N0 = DAG.getConstant(0, DL, MVT::i32); in splitBufferOffsets()
6308 C1 = cast<ConstantSDNode>(DAG.getConstant(0, DL, MVT::i32)); in splitBufferOffsets()
6323 Offsets[0] = DAG.getConstant(0, DL, MVT::i32); in setBufferOffsets()
6324 Offsets[1] = DAG.getConstant(SOffset, DL, MVT::i32); in setBufferOffsets()
6325 Offsets[2] = DAG.getConstant(ImmOffset, DL, MVT::i32); in setBufferOffsets()
6337 Offsets[1] = DAG.getConstant(SOffset, DL, MVT::i32); in setBufferOffsets()
6338 Offsets[2] = DAG.getConstant(ImmOffset, DL, MVT::i32); in setBufferOffsets()
6343 Offsets[1] = DAG.getConstant(0, DL, MVT::i32); in setBufferOffsets()
6344 Offsets[2] = DAG.getConstant(0, DL, MVT::i32); in setBufferOffsets()
6395 MVT::i32, SL, Ld->getChain(), Ptr, in widenLoad()
6397 Ld->getPointerInfo(), MVT::i32, in widenLoad()
6412 Cvt = DAG.getNode(ISD::SIGN_EXTEND_INREG, SL, MVT::i32, NewLoad, in widenLoad()
6444 if (MemVT == MVT::i16 && isTypeLegal(MVT::i16)) in LowerLOAD()
6454 EVT RealMemVT = (MemVT == MVT::i1) ? MVT::i8 : MVT::i16; in LowerLOAD()
6456 SDValue NewLD = DAG.getExtLoad(ISD::EXTLOAD, DL, MVT::i32, Chain, in LowerLOAD()
6470 assert(Op.getValueType().getVectorElementType() == MVT::i32 && in LowerLOAD()
6572 SDValue Zero = DAG.getConstant(0, DL, MVT::i32); in LowerSELECT()
6573 SDValue One = DAG.getConstant(1, DL, MVT::i32); in LowerSELECT()
6575 SDValue LHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(1)); in LowerSELECT()
6576 SDValue RHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(2)); in LowerSELECT()
6578 SDValue Lo0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, LHS, Zero); in LowerSELECT()
6579 SDValue Lo1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, RHS, Zero); in LowerSELECT()
6581 SDValue Lo = DAG.getSelect(DL, MVT::i32, Cond, Lo0, Lo1); in LowerSELECT()
6583 SDValue Hi0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, LHS, One); in LowerSELECT()
6584 SDValue Hi1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, RHS, One); in LowerSELECT()
6586 SDValue Hi = DAG.getSelect(DL, MVT::i32, Cond, Hi0, Hi1); in LowerSELECT()
6588 SDValue Res = DAG.getBuildVector(MVT::v2i32, DL, {Lo, Hi}); in LowerSELECT()
6603 if (!Unsafe && VT == MVT::f32 && Subtarget->hasFP32Denormals()) in lowerFastUnsafeFDIV()
6607 if (Unsafe || VT == MVT::f32 || VT == MVT::f16) { in lowerFastUnsafeFDIV()
6654 SDVTList VTList = DAG.getVTList(VT, MVT::Other, MVT::Glue); in getFPBinOp()
6675 SDVTList VTList = DAG.getVTList(VT, MVT::Other, MVT::Glue); in getFPTernOp()
6695 SDValue CvtSrc0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0); in LowerFDIV16()
6696 SDValue CvtSrc1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1); in LowerFDIV16()
6698 SDValue RcpSrc1 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, CvtSrc1); in LowerFDIV16()
6699 SDValue Quot = DAG.getNode(ISD::FMUL, SL, MVT::f32, CvtSrc0, RcpSrc1); in LowerFDIV16()
6701 SDValue FPRoundFlag = DAG.getTargetConstant(0, SL, MVT::i32); in LowerFDIV16()
6702 SDValue BestQuot = DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Quot, FPRoundFlag); in LowerFDIV16()
6704 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f16, BestQuot, Src1, Src0); in LowerFDIV16()
6713 SDValue r1 = DAG.getNode(ISD::FABS, SL, MVT::f32, RHS); in lowerFDIV_FAST()
6716 const SDValue K0 = DAG.getConstantFP(K0Val, SL, MVT::f32); in lowerFDIV_FAST()
6719 const SDValue K1 = DAG.getConstantFP(K1Val, SL, MVT::f32); in lowerFDIV_FAST()
6721 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32); in lowerFDIV_FAST()
6724 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f32); in lowerFDIV_FAST()
6728 SDValue r3 = DAG.getNode(ISD::SELECT, SL, MVT::f32, r2, K1, One); in lowerFDIV_FAST()
6731 r1 = DAG.getNode(ISD::FMUL, SL, MVT::f32, RHS, r3); in lowerFDIV_FAST()
6734 SDValue r0 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, r1); in lowerFDIV_FAST()
6736 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f32, LHS, r0); in lowerFDIV_FAST()
6738 return DAG.getNode(ISD::FMUL, SL, MVT::f32, r3, Mul); in lowerFDIV_FAST()
6749 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32); in LowerFDIV32()
6751 SDVTList ScaleVT = DAG.getVTList(MVT::f32, MVT::i1); in LowerFDIV32()
6759 SDValue ApproxRcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, in LowerFDIV32()
6761 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f32, in LowerFDIV32()
6768 const SDValue BitField = DAG.getTargetConstant(Denorm32Reg, SL, MVT::i16); in LowerFDIV32()
6771 SDVTList BindParamVTs = DAG.getVTList(MVT::Other, MVT::Glue); in LowerFDIV32()
6773 SL, MVT::i32); in LowerFDIV32()
6786 SDValue Fma0 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, in LowerFDIV32()
6789 SDValue Fma1 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, Fma0, ApproxRcp, in LowerFDIV32()
6792 SDValue Mul = getFPBinOp(DAG, ISD::FMUL, SL, MVT::f32, NumeratorScaled, in LowerFDIV32()
6795 SDValue Fma2 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Mul, in LowerFDIV32()
6798 SDValue Fma3 = getFPTernOp(DAG, ISD::FMA,SL, MVT::f32, Fma2, Fma1, Mul, Fma2); in LowerFDIV32()
6800 SDValue Fma4 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Fma3, in LowerFDIV32()
6805 DAG.getConstant(FP_DENORM_FLUSH_IN_FLUSH_OUT, SL, MVT::i32); in LowerFDIV32()
6806 SDValue DisableDenorm = DAG.getNode(AMDGPUISD::SETREG, SL, MVT::Other, in LowerFDIV32()
6812 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, in LowerFDIV32()
6818 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f32, in LowerFDIV32()
6821 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f32, Fmas, RHS, LHS); in LowerFDIV32()
6832 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64); in LowerFDIV64()
6834 SDVTList ScaleVT = DAG.getVTList(MVT::f64, MVT::i1); in LowerFDIV64()
6838 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f64, DivScale0); in LowerFDIV64()
6840 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f64, DivScale0); in LowerFDIV64()
6842 SDValue Fma0 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Rcp, One); in LowerFDIV64()
6844 SDValue Fma1 = DAG.getNode(ISD::FMA, SL, MVT::f64, Rcp, Fma0, Rcp); in LowerFDIV64()
6846 SDValue Fma2 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Fma1, One); in LowerFDIV64()
6850 SDValue Fma3 = DAG.getNode(ISD::FMA, SL, MVT::f64, Fma1, Fma2, Fma1); in LowerFDIV64()
6851 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, DivScale1, Fma3); in LowerFDIV64()
6853 SDValue Fma4 = DAG.getNode(ISD::FMA, SL, MVT::f64, in LowerFDIV64()
6862 const SDValue Hi = DAG.getConstant(1, SL, MVT::i32); in LowerFDIV64()
6865 SDValue NumBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X); in LowerFDIV64()
6866 SDValue DenBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Y); in LowerFDIV64()
6867 SDValue Scale0BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale0); in LowerFDIV64()
6868 SDValue Scale1BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale1); in LowerFDIV64()
6870 SDValue NumHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, NumBC, Hi); in LowerFDIV64()
6871 SDValue DenHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, DenBC, Hi); in LowerFDIV64()
6874 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale0BC, Hi); in LowerFDIV64()
6876 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale1BC, Hi); in LowerFDIV64()
6878 SDValue CmpDen = DAG.getSetCC(SL, MVT::i1, DenHi, Scale0Hi, ISD::SETEQ); in LowerFDIV64()
6879 SDValue CmpNum = DAG.getSetCC(SL, MVT::i1, NumHi, Scale1Hi, ISD::SETEQ); in LowerFDIV64()
6880 Scale = DAG.getNode(ISD::XOR, SL, MVT::i1, CmpNum, CmpDen); in LowerFDIV64()
6885 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f64, in LowerFDIV64()
6888 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f64, Fmas, Y, X); in LowerFDIV64()
6894 if (VT == MVT::f32) in LowerFDIV()
6897 if (VT == MVT::f64) in LowerFDIV()
6900 if (VT == MVT::f16) in LowerFDIV()
6911 if (VT == MVT::i1) { in LowerSTORE()
6913 DAG.getSExtOrTrunc(Store->getValue(), DL, MVT::i32), in LowerSTORE()
6914 Store->getBasePtr(), MVT::i1, Store->getMemOperand()); in LowerSTORE()
6918 Store->getValue().getValueType().getScalarType() == MVT::i32); in LowerSTORE()
7025 MVT SimpleVT = VT.getSimpleVT(); in LowerATOMIC_CMP_SWAP()
7026 MVT VecType = MVT::getVectorVT(SimpleVT, 2); in LowerATOMIC_CMP_SWAP()
7043 if (ScalarVT != MVT::f32) in performUCharToFloatCombine()
7056 if (DCI.isAfterLegalizeDAG() && SrcVT == MVT::i32) { in performUCharToFloatCombine()
7118 SDValue COffset = DAG.getConstant(Offset, SL, MVT::i32); in performSHLPtrCombine()
7185 if (V.getValueType() != MVT::i1) in isBoolSGPR()
7276 if (VT == MVT::i64 && CRHS) { in performAndCombine()
7282 if (CRHS && VT == MVT::i32) { in performAndCombine()
7297 SDValue BFE = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32, in performAndCombine()
7299 DAG.getConstant(Offset, SL, MVT::i32), in performAndCombine()
7300 DAG.getConstant(Bits, SL, MVT::i32)); in performAndCombine()
7305 DAG.getConstant(NB, SDLoc(CRHS), MVT::i32)); in performAndCombine()
7321 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, LHS.getOperand(0), in performAndCombine()
7322 LHS.getOperand(1), DAG.getConstant(Sel, DL, MVT::i32)); in performAndCombine()
7360 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1, in performAndCombine()
7361 X, DAG.getConstant(Mask, DL, MVT::i32)); in performAndCombine()
7384 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1, RHS.getOperand(0), in performAndCombine()
7385 DAG.getConstant(NewMask, DL, MVT::i32)); in performAndCombine()
7389 if (VT == MVT::i32 && in performAndCombine()
7395 return DAG.getSelect(SDLoc(N), MVT::i32, RHS.getOperand(0), in performAndCombine()
7396 LHS, DAG.getConstant(0, SDLoc(N), MVT::i32)); in performAndCombine()
7401 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() && in performAndCombine()
7440 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, in performAndCombine()
7442 DAG.getConstant(Sel, DL, MVT::i32)); in performAndCombine()
7457 if (VT == MVT::i1) { in performOrCombine()
7475 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1, in performOrCombine()
7476 Src, DAG.getConstant(NewMask, DL, MVT::i32)); in performOrCombine()
7492 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, LHS.getOperand(0), in performOrCombine()
7493 LHS.getOperand(1), DAG.getConstant(Sel, DL, MVT::i32)); in performOrCombine()
7498 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() && in performOrCombine()
7529 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, in performOrCombine()
7531 DAG.getConstant(Sel, DL, MVT::i32)); in performOrCombine()
7536 if (VT != MVT::i64) in performOrCombine()
7551 if (SrcVT == MVT::i32) { in performOrCombine()
7555 SDValue LowOr = DAG.getNode(ISD::OR, SL, MVT::i32, LowLHS, ExtSrc); in performOrCombine()
7560 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in performOrCombine()
7562 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in performOrCombine()
7579 if (VT != MVT::i64) in performXorCombine()
7659 if (VT != MVT::i32) in performZeroExtendCombine()
7663 if (Src.getValueType() != MVT::i16) in performZeroExtendCombine()
7670 if (BCSrc.getValueType() == MVT::f16 && in performZeroExtendCombine()
7686 return DAG.getConstant(0, SDLoc(N), MVT::i1); in performClassCombine()
7690 return DAG.getUNDEF(MVT::i1); in performClassCombine()
7703 if (VT == MVT::f32 && (N0.getOpcode() == ISD::UINT_TO_FP || in performRcpCombine()
7775 return Op.getValueType().getScalarType() != MVT::f16; in isCanonicalized()
7834 if (Src.getValueType() == MVT::i16 && in isCanonicalized()
7837 if (TruncSrc.getValueType() == MVT::i32 && in isCanonicalized()
7839 TruncSrc.getOperand(0).getValueType() == MVT::v2f16) { in isCanonicalized()
7926 if (N0.getOpcode() == ISD::BUILD_VECTOR && VT == MVT::v2f16 && in performFCanonicalizeCombine()
7927 isTypeLegal(MVT::v2f16)) { in performFCanonicalizeCombine()
8031 if (VT == MVT::i32 || (VT == MVT::i16 && Subtarget->hasMed3_16())) { in performIntMed3ImmCombine()
8037 MVT NVT = MVT::i32; in performIntMed3ImmCombine()
8088 if (VT == MVT::f32 || (VT == MVT::f16 && Subtarget->hasMed3_16())) { in performFPMed3ImmCombine()
8125 !VT.isVector() && VT != MVT::f64 && in performMinMaxCombine()
8126 ((VT != MVT::f16 && VT != MVT::i16) || Subtarget->hasMin3Max3_16())) { in performMinMaxCombine()
8169 (VT == MVT::f32 || VT == MVT::f64 || in performMinMaxCombine()
8170 (VT == MVT::f16 && Subtarget->has16BitInsts()) || in performMinMaxCombine()
8171 (VT == MVT::v2f16 && Subtarget->hasVOP3PInsts())) && in performMinMaxCombine()
8348 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast, in performExtractVectorEltCombine()
8349 DAG.getConstant(EltIdx, SL, MVT::i32)); in performExtractVectorEltCombine()
8351 SDValue Srl = DAG.getNode(ISD::SRL, SL, MVT::i32, Elt, in performExtractVectorEltCombine()
8352 DAG.getConstant(LeftoverBitIdx, SL, MVT::i32)); in performExtractVectorEltCombine()
8406 if ((VT == MVT::f32 && !Subtarget->hasFP32Denormals()) || in getFusedOpcode()
8407 (VT == MVT::f16 && !Subtarget->hasFP16Denormals())) in getFusedOpcode()
8426 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i1); in getMad64_32()
8453 MulLHS = DAG.getZExtOrTrunc(MulLHS, SL, MVT::i32); in performAddCombine()
8454 MulRHS = DAG.getZExtOrTrunc(MulRHS, SL, MVT::i32); in performAddCombine()
8455 AddRHS = DAG.getZExtOrTrunc(AddRHS, SL, MVT::i64); in performAddCombine()
8460 MulLHS = DAG.getSExtOrTrunc(MulLHS, SL, MVT::i32); in performAddCombine()
8461 MulRHS = DAG.getSExtOrTrunc(MulRHS, SL, MVT::i32); in performAddCombine()
8462 AddRHS = DAG.getSExtOrTrunc(AddRHS, SL, MVT::i64); in performAddCombine()
8469 if (VT != MVT::i32 || !DCI.isAfterLegalizeDAG()) in performAddCombine()
8488 SDVTList VTList = DAG.getVTList(MVT::i32, MVT::i1); in performAddCombine()
8489 SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond }; in performAddCombine()
8509 if (VT != MVT::i32) in performSubCombine()
8534 if (N->getValueType(0) != MVT::i32) in performAddCarrySubCarryCombine()
8651 if (!Subtarget->hasDotInsts() || VT != MVT::f32) in performFMACombine()
8708 if (Vec1.getValueType() != MVT::v2f16 || Vec2.getValueType() != MVT::v2f16) in performFMACombine()
8713 return DAG.getNode(AMDGPUISD::FDOT2, SL, MVT::f32, Vec1, Vec2, FMAAcc, in performFMACombine()
8714 DAG.getTargetConstant(0, SL, MVT::i1)); in performFMACombine()
8740 if (VT == MVT::i32 && LHS.getOpcode() == ISD::SIGN_EXTEND && in performSetCCCombine()
8750 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
8751 DAG.getConstant(-1, SL, MVT::i1)); in performSetCCCombine()
8776 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
8777 DAG.getConstant(-1, SL, MVT::i1)); in performSetCCCombine()
8784 if (VT != MVT::f32 && VT != MVT::f64 && (Subtarget->has16BitInsts() && in performSetCCCombine()
8785 VT != MVT::f16)) in performSetCCCombine()
8808 return DAG.getNode(AMDGPUISD::FP_CLASS, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
8809 DAG.getConstant(Mask, SL, MVT::i32)); in performSetCCCombine()
8836 EVT(MVT::i32)); in performCvtF32UByteNCombine()
8841 MVT::f32, Srl); in performCvtF32UByteNCombine()
8990 if (VT == MVT::v2i16 || VT == MVT::v2f16) { in PerformDAGCombine()
8994 if (EltVT == MVT::f16) in PerformDAGCombine()
8995 Src = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Src); in PerformDAGCombine()
8997 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Src); in PerformDAGCombine()
9125 Ops.push_back(DAG.getTargetConstant(NewDmask, SDLoc(Node), MVT::i32)); in adjustWritemask()
9128 MVT SVT = Node->getValueType(0).getVectorElementType().getSimpleVT(); in adjustWritemask()
9130 MVT ResultVT = NewChannels == 1 ? in adjustWritemask()
9131 SVT : MVT::getVectorVT(SVT, NewChannels == 3 ? 4 : in adjustWritemask()
9134 DAG.getVTList(ResultVT, MVT::Other) : DAG.getVTList(ResultVT); in adjustWritemask()
9164 SDValue Op = DAG.getTargetConstant(Idx, SDLoc(User), MVT::i32); in adjustWritemask()
9199 if (SrcVal.getValueType() == MVT::i1 && in legalizeTargetIndependentNode()
9204 MRI.createVirtualRegister(&AMDGPU::VReg_1RegClass), MVT::i1); in legalizeTargetIndependentNode()
9268 MVT VT = Src0.getValueType().getSimpleVT(); in PostISelFolding()
9361 SDValue K = DAG.getTargetConstant(Val, DL, MVT::i32); in buildSMovImm32()
9362 return SDValue(DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, K), 0); in buildSMovImm32()
9374 DAG.getTargetConstant(AMDGPU::SGPR_64RegClassID, DL, MVT::i32), in wrapAddr64Rsrc()
9376 DAG.getTargetConstant(AMDGPU::sub0, DL, MVT::i32), in wrapAddr64Rsrc()
9378 DAG.getTargetConstant(AMDGPU::sub1, DL, MVT::i32) in wrapAddr64Rsrc()
9382 MVT::v2i32, Ops0), 0); in wrapAddr64Rsrc()
9386 DAG.getTargetConstant(AMDGPU::SReg_128RegClassID, DL, MVT::i32), in wrapAddr64Rsrc()
9388 DAG.getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32), in wrapAddr64Rsrc()
9390 DAG.getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32) in wrapAddr64Rsrc()
9393 return DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL, MVT::v4i32, Ops1); in wrapAddr64Rsrc()
9403 SDValue PtrLo = DAG.getTargetExtractSubreg(AMDGPU::sub0, DL, MVT::i32, Ptr); in buildRSRC()
9404 SDValue PtrHi = DAG.getTargetExtractSubreg(AMDGPU::sub1, DL, MVT::i32, Ptr); in buildRSRC()
9406 PtrHi = SDValue(DAG.getMachineNode(AMDGPU::S_OR_B32, DL, MVT::i32, PtrHi, in buildRSRC()
9407 DAG.getConstant(RsrcDword1, DL, MVT::i32)), in buildRSRC()
9416 DAG.getTargetConstant(AMDGPU::SReg_128RegClassID, DL, MVT::i32), in buildRSRC()
9418 DAG.getTargetConstant(AMDGPU::sub0, DL, MVT::i32), in buildRSRC()
9420 DAG.getTargetConstant(AMDGPU::sub1, DL, MVT::i32), in buildRSRC()
9422 DAG.getTargetConstant(AMDGPU::sub2, DL, MVT::i32), in buildRSRC()
9424 DAG.getTargetConstant(AMDGPU::sub3, DL, MVT::i32) in buildRSRC()
9427 return DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL, MVT::v4i32, Ops); in buildRSRC()
9437 MVT VT) const { in getRegForInlineAsmConstraint()
9494 if (RC && (isTypeLegal(VT) || VT.SimpleTy == MVT::i128 || in getRegForInlineAsmConstraint()
9495 VT.SimpleTy == MVT::i16 || VT.SimpleTy == MVT::f16)) in getRegForInlineAsmConstraint()
9664 case MVT::f32: in denormalsEnabledForType()
9666 case MVT::f64: in denormalsEnabledForType()
9668 case MVT::f16: in denormalsEnabledForType()