Lines Matching refs:MVT
79 addRegisterClass(MVT::i1, &AMDGPU::VReg_1RegClass); in SITargetLowering()
80 addRegisterClass(MVT::i64, &AMDGPU::SReg_64RegClass); in SITargetLowering()
82 addRegisterClass(MVT::i32, &AMDGPU::SReg_32RegClass); in SITargetLowering()
83 addRegisterClass(MVT::f32, &AMDGPU::VGPR_32RegClass); in SITargetLowering()
85 addRegisterClass(MVT::v2i32, &AMDGPU::SReg_64RegClass); in SITargetLowering()
90 addRegisterClass(MVT::f64, V64RegClass); in SITargetLowering()
91 addRegisterClass(MVT::v2f32, V64RegClass); in SITargetLowering()
93 addRegisterClass(MVT::v3i32, &AMDGPU::SGPR_96RegClass); in SITargetLowering()
94 addRegisterClass(MVT::v3f32, TRI->getVGPRClassForBitWidth(96)); in SITargetLowering()
96 addRegisterClass(MVT::v2i64, &AMDGPU::SGPR_128RegClass); in SITargetLowering()
97 addRegisterClass(MVT::v2f64, &AMDGPU::SGPR_128RegClass); in SITargetLowering()
99 addRegisterClass(MVT::v4i32, &AMDGPU::SGPR_128RegClass); in SITargetLowering()
100 addRegisterClass(MVT::v4f32, TRI->getVGPRClassForBitWidth(128)); in SITargetLowering()
102 addRegisterClass(MVT::v5i32, &AMDGPU::SGPR_160RegClass); in SITargetLowering()
103 addRegisterClass(MVT::v5f32, TRI->getVGPRClassForBitWidth(160)); in SITargetLowering()
105 addRegisterClass(MVT::v6i32, &AMDGPU::SGPR_192RegClass); in SITargetLowering()
106 addRegisterClass(MVT::v6f32, TRI->getVGPRClassForBitWidth(192)); in SITargetLowering()
108 addRegisterClass(MVT::v3i64, &AMDGPU::SGPR_192RegClass); in SITargetLowering()
109 addRegisterClass(MVT::v3f64, TRI->getVGPRClassForBitWidth(192)); in SITargetLowering()
111 addRegisterClass(MVT::v7i32, &AMDGPU::SGPR_224RegClass); in SITargetLowering()
112 addRegisterClass(MVT::v7f32, TRI->getVGPRClassForBitWidth(224)); in SITargetLowering()
114 addRegisterClass(MVT::v8i32, &AMDGPU::SGPR_256RegClass); in SITargetLowering()
115 addRegisterClass(MVT::v8f32, TRI->getVGPRClassForBitWidth(256)); in SITargetLowering()
117 addRegisterClass(MVT::v4i64, &AMDGPU::SGPR_256RegClass); in SITargetLowering()
118 addRegisterClass(MVT::v4f64, TRI->getVGPRClassForBitWidth(256)); in SITargetLowering()
120 addRegisterClass(MVT::v16i32, &AMDGPU::SGPR_512RegClass); in SITargetLowering()
121 addRegisterClass(MVT::v16f32, TRI->getVGPRClassForBitWidth(512)); in SITargetLowering()
123 addRegisterClass(MVT::v8i64, &AMDGPU::SGPR_512RegClass); in SITargetLowering()
124 addRegisterClass(MVT::v8f64, TRI->getVGPRClassForBitWidth(512)); in SITargetLowering()
126 addRegisterClass(MVT::v16i64, &AMDGPU::SGPR_1024RegClass); in SITargetLowering()
127 addRegisterClass(MVT::v16f64, TRI->getVGPRClassForBitWidth(1024)); in SITargetLowering()
130 addRegisterClass(MVT::i16, &AMDGPU::SReg_32RegClass); in SITargetLowering()
131 addRegisterClass(MVT::f16, &AMDGPU::SReg_32RegClass); in SITargetLowering()
134 addRegisterClass(MVT::v2i16, &AMDGPU::SReg_32RegClass); in SITargetLowering()
135 addRegisterClass(MVT::v2f16, &AMDGPU::SReg_32RegClass); in SITargetLowering()
136 addRegisterClass(MVT::v4i16, &AMDGPU::SReg_64RegClass); in SITargetLowering()
137 addRegisterClass(MVT::v4f16, &AMDGPU::SReg_64RegClass); in SITargetLowering()
140 addRegisterClass(MVT::v32i32, &AMDGPU::VReg_1024RegClass); in SITargetLowering()
141 addRegisterClass(MVT::v32f32, TRI->getVGPRClassForBitWidth(1024)); in SITargetLowering()
153 setOperationAction(ISD::LOAD, MVT::v2i32, Custom); in SITargetLowering()
154 setOperationAction(ISD::LOAD, MVT::v3i32, Custom); in SITargetLowering()
155 setOperationAction(ISD::LOAD, MVT::v4i32, Custom); in SITargetLowering()
156 setOperationAction(ISD::LOAD, MVT::v5i32, Custom); in SITargetLowering()
157 setOperationAction(ISD::LOAD, MVT::v6i32, Custom); in SITargetLowering()
158 setOperationAction(ISD::LOAD, MVT::v7i32, Custom); in SITargetLowering()
159 setOperationAction(ISD::LOAD, MVT::v8i32, Custom); in SITargetLowering()
160 setOperationAction(ISD::LOAD, MVT::v16i32, Custom); in SITargetLowering()
161 setOperationAction(ISD::LOAD, MVT::i1, Custom); in SITargetLowering()
162 setOperationAction(ISD::LOAD, MVT::v32i32, Custom); in SITargetLowering()
164 setOperationAction(ISD::STORE, MVT::v2i32, Custom); in SITargetLowering()
165 setOperationAction(ISD::STORE, MVT::v3i32, Custom); in SITargetLowering()
166 setOperationAction(ISD::STORE, MVT::v4i32, Custom); in SITargetLowering()
167 setOperationAction(ISD::STORE, MVT::v5i32, Custom); in SITargetLowering()
168 setOperationAction(ISD::STORE, MVT::v6i32, Custom); in SITargetLowering()
169 setOperationAction(ISD::STORE, MVT::v7i32, Custom); in SITargetLowering()
170 setOperationAction(ISD::STORE, MVT::v8i32, Custom); in SITargetLowering()
171 setOperationAction(ISD::STORE, MVT::v16i32, Custom); in SITargetLowering()
172 setOperationAction(ISD::STORE, MVT::i1, Custom); in SITargetLowering()
173 setOperationAction(ISD::STORE, MVT::v32i32, Custom); in SITargetLowering()
175 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand); in SITargetLowering()
176 setTruncStoreAction(MVT::v3i32, MVT::v3i16, Expand); in SITargetLowering()
177 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Expand); in SITargetLowering()
178 setTruncStoreAction(MVT::v8i32, MVT::v8i16, Expand); in SITargetLowering()
179 setTruncStoreAction(MVT::v16i32, MVT::v16i16, Expand); in SITargetLowering()
180 setTruncStoreAction(MVT::v32i32, MVT::v32i16, Expand); in SITargetLowering()
181 setTruncStoreAction(MVT::v2i32, MVT::v2i8, Expand); in SITargetLowering()
182 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Expand); in SITargetLowering()
183 setTruncStoreAction(MVT::v8i32, MVT::v8i8, Expand); in SITargetLowering()
184 setTruncStoreAction(MVT::v16i32, MVT::v16i8, Expand); in SITargetLowering()
185 setTruncStoreAction(MVT::v32i32, MVT::v32i8, Expand); in SITargetLowering()
186 setTruncStoreAction(MVT::v2i16, MVT::v2i8, Expand); in SITargetLowering()
187 setTruncStoreAction(MVT::v4i16, MVT::v4i8, Expand); in SITargetLowering()
188 setTruncStoreAction(MVT::v8i16, MVT::v8i8, Expand); in SITargetLowering()
189 setTruncStoreAction(MVT::v16i16, MVT::v16i8, Expand); in SITargetLowering()
190 setTruncStoreAction(MVT::v32i16, MVT::v32i8, Expand); in SITargetLowering()
192 setTruncStoreAction(MVT::v3i64, MVT::v3i16, Expand); in SITargetLowering()
193 setTruncStoreAction(MVT::v3i64, MVT::v3i32, Expand); in SITargetLowering()
194 setTruncStoreAction(MVT::v4i64, MVT::v4i8, Expand); in SITargetLowering()
195 setTruncStoreAction(MVT::v8i64, MVT::v8i8, Expand); in SITargetLowering()
196 setTruncStoreAction(MVT::v8i64, MVT::v8i16, Expand); in SITargetLowering()
197 setTruncStoreAction(MVT::v8i64, MVT::v8i32, Expand); in SITargetLowering()
198 setTruncStoreAction(MVT::v16i64, MVT::v16i32, Expand); in SITargetLowering()
200 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); in SITargetLowering()
201 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); in SITargetLowering()
203 setOperationAction(ISD::SELECT, MVT::i1, Promote); in SITargetLowering()
204 setOperationAction(ISD::SELECT, MVT::i64, Custom); in SITargetLowering()
205 setOperationAction(ISD::SELECT, MVT::f64, Promote); in SITargetLowering()
206 AddPromotedToType(ISD::SELECT, MVT::f64, MVT::i64); in SITargetLowering()
208 setOperationAction(ISD::SELECT_CC, MVT::f32, Expand); in SITargetLowering()
209 setOperationAction(ISD::SELECT_CC, MVT::i32, Expand); in SITargetLowering()
210 setOperationAction(ISD::SELECT_CC, MVT::i64, Expand); in SITargetLowering()
211 setOperationAction(ISD::SELECT_CC, MVT::f64, Expand); in SITargetLowering()
212 setOperationAction(ISD::SELECT_CC, MVT::i1, Expand); in SITargetLowering()
214 setOperationAction(ISD::SETCC, MVT::i1, Promote); in SITargetLowering()
215 setOperationAction(ISD::SETCC, MVT::v2i1, Expand); in SITargetLowering()
216 setOperationAction(ISD::SETCC, MVT::v4i1, Expand); in SITargetLowering()
217 AddPromotedToType(ISD::SETCC, MVT::i1, MVT::i32); in SITargetLowering()
219 setOperationAction(ISD::TRUNCATE, MVT::v2i32, Expand); in SITargetLowering()
220 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Expand); in SITargetLowering()
221 setOperationAction(ISD::TRUNCATE, MVT::v3i32, Expand); in SITargetLowering()
222 setOperationAction(ISD::FP_ROUND, MVT::v3f32, Expand); in SITargetLowering()
223 setOperationAction(ISD::TRUNCATE, MVT::v4i32, Expand); in SITargetLowering()
224 setOperationAction(ISD::FP_ROUND, MVT::v4f32, Expand); in SITargetLowering()
225 setOperationAction(ISD::TRUNCATE, MVT::v5i32, Expand); in SITargetLowering()
226 setOperationAction(ISD::FP_ROUND, MVT::v5f32, Expand); in SITargetLowering()
227 setOperationAction(ISD::TRUNCATE, MVT::v6i32, Expand); in SITargetLowering()
228 setOperationAction(ISD::FP_ROUND, MVT::v6f32, Expand); in SITargetLowering()
229 setOperationAction(ISD::TRUNCATE, MVT::v7i32, Expand); in SITargetLowering()
230 setOperationAction(ISD::FP_ROUND, MVT::v7f32, Expand); in SITargetLowering()
231 setOperationAction(ISD::TRUNCATE, MVT::v8i32, Expand); in SITargetLowering()
232 setOperationAction(ISD::FP_ROUND, MVT::v8f32, Expand); in SITargetLowering()
233 setOperationAction(ISD::TRUNCATE, MVT::v16i32, Expand); in SITargetLowering()
234 setOperationAction(ISD::FP_ROUND, MVT::v16f32, Expand); in SITargetLowering()
236 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i1, Custom); in SITargetLowering()
237 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i1, Custom); in SITargetLowering()
238 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Custom); in SITargetLowering()
239 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8, Custom); in SITargetLowering()
240 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Custom); in SITargetLowering()
241 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v3i16, Custom); in SITargetLowering()
242 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Custom); in SITargetLowering()
243 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::Other, Custom); in SITargetLowering()
245 setOperationAction(ISD::BRCOND, MVT::Other, Custom); in SITargetLowering()
246 setOperationAction(ISD::BR_CC, MVT::i1, Expand); in SITargetLowering()
247 setOperationAction(ISD::BR_CC, MVT::i32, Expand); in SITargetLowering()
248 setOperationAction(ISD::BR_CC, MVT::i64, Expand); in SITargetLowering()
249 setOperationAction(ISD::BR_CC, MVT::f32, Expand); in SITargetLowering()
250 setOperationAction(ISD::BR_CC, MVT::f64, Expand); in SITargetLowering()
252 setOperationAction(ISD::UADDO, MVT::i32, Legal); in SITargetLowering()
253 setOperationAction(ISD::USUBO, MVT::i32, Legal); in SITargetLowering()
255 setOperationAction(ISD::ADDCARRY, MVT::i32, Legal); in SITargetLowering()
256 setOperationAction(ISD::SUBCARRY, MVT::i32, Legal); in SITargetLowering()
258 setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand); in SITargetLowering()
259 setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand); in SITargetLowering()
260 setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand); in SITargetLowering()
263 setOperationAction(ISD::ADDCARRY, MVT::i64, Legal); in SITargetLowering()
264 setOperationAction(ISD::SUBCARRY, MVT::i64, Legal); in SITargetLowering()
269 for (MVT VT : { MVT::v8i32, MVT::v8f32, MVT::v16i32, MVT::v16f32, in SITargetLowering()
270 MVT::v2i64, MVT::v2f64, MVT::v4i16, MVT::v4f16, in SITargetLowering()
271 MVT::v3i64, MVT::v3f64, MVT::v6i32, MVT::v6f32, in SITargetLowering()
272 MVT::v4i64, MVT::v4f64, MVT::v8i64, MVT::v8f64, in SITargetLowering()
273 MVT::v16i64, MVT::v16f64, MVT::v32i32, MVT::v32f32 }) { in SITargetLowering()
296 setOperationAction(ISD::FP_EXTEND, MVT::v4f32, Expand); in SITargetLowering()
303 for (MVT Vec64 : { MVT::v2i64, MVT::v2f64 }) { in SITargetLowering()
305 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v4i32); in SITargetLowering()
308 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v4i32); in SITargetLowering()
311 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v4i32); in SITargetLowering()
314 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v4i32); in SITargetLowering()
317 for (MVT Vec64 : { MVT::v3i64, MVT::v3f64 }) { in SITargetLowering()
319 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v6i32); in SITargetLowering()
322 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v6i32); in SITargetLowering()
325 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v6i32); in SITargetLowering()
328 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v6i32); in SITargetLowering()
331 for (MVT Vec64 : { MVT::v4i64, MVT::v4f64 }) { in SITargetLowering()
333 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v8i32); in SITargetLowering()
336 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v8i32); in SITargetLowering()
339 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v8i32); in SITargetLowering()
342 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v8i32); in SITargetLowering()
345 for (MVT Vec64 : { MVT::v8i64, MVT::v8f64 }) { in SITargetLowering()
347 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v16i32); in SITargetLowering()
350 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v16i32); in SITargetLowering()
353 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v16i32); in SITargetLowering()
356 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v16i32); in SITargetLowering()
359 for (MVT Vec64 : { MVT::v16i64, MVT::v16f64 }) { in SITargetLowering()
361 AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v32i32); in SITargetLowering()
364 AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v32i32); in SITargetLowering()
367 AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v32i32); in SITargetLowering()
370 AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v32i32); in SITargetLowering()
373 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i32, Expand); in SITargetLowering()
374 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8f32, Expand); in SITargetLowering()
375 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i32, Expand); in SITargetLowering()
376 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16f32, Expand); in SITargetLowering()
378 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f16, Custom); in SITargetLowering()
379 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i16, Custom); in SITargetLowering()
383 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i16, Custom); in SITargetLowering()
384 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f16, Custom); in SITargetLowering()
385 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i16, Custom); in SITargetLowering()
386 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f16, Custom); in SITargetLowering()
388 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i8, Custom); in SITargetLowering()
389 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i8, Custom); in SITargetLowering()
390 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i8, Custom); in SITargetLowering()
391 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i8, Custom); in SITargetLowering()
392 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i8, Custom); in SITargetLowering()
393 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i8, Custom); in SITargetLowering()
395 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i16, Custom); in SITargetLowering()
396 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f16, Custom); in SITargetLowering()
397 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i16, Custom); in SITargetLowering()
398 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f16, Custom); in SITargetLowering()
401 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v3i32, Custom); in SITargetLowering()
402 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v3f32, Custom); in SITargetLowering()
403 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v4i32, Custom); in SITargetLowering()
404 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v4f32, Custom); in SITargetLowering()
407 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v5i32, Custom); in SITargetLowering()
408 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v5f32, Custom); in SITargetLowering()
409 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v6i32, Custom); in SITargetLowering()
410 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v6f32, Custom); in SITargetLowering()
411 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v7i32, Custom); in SITargetLowering()
412 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v7f32, Custom); in SITargetLowering()
413 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v8i32, Custom); in SITargetLowering()
414 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v8f32, Custom); in SITargetLowering()
418 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); in SITargetLowering()
419 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, Custom); in SITargetLowering()
423 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i32, Expand); in SITargetLowering()
424 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i64, Expand); in SITargetLowering()
427 setOperationAction(ISD::ADDRSPACECAST, MVT::i32, Custom); in SITargetLowering()
428 setOperationAction(ISD::ADDRSPACECAST, MVT::i64, Custom); in SITargetLowering()
431 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); in SITargetLowering()
432 setOperationAction(ISD::BITREVERSE, MVT::i64, Legal); in SITargetLowering()
437 setOperationAction(ISD::BSWAP, MVT::i64, Legal); in SITargetLowering()
438 setOperationAction(ISD::BSWAP, MVT::i32, Legal); in SITargetLowering()
441 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal); in SITargetLowering()
442 setOperationAction(ISD::TRAP, MVT::Other, Custom); in SITargetLowering()
443 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Custom); in SITargetLowering()
446 setOperationAction(ISD::FPOW, MVT::f16, Promote); in SITargetLowering()
447 setOperationAction(ISD::FPOWI, MVT::f16, Promote); in SITargetLowering()
448 setOperationAction(ISD::FLOG, MVT::f16, Custom); in SITargetLowering()
449 setOperationAction(ISD::FEXP, MVT::f16, Custom); in SITargetLowering()
450 setOperationAction(ISD::FLOG10, MVT::f16, Custom); in SITargetLowering()
454 setOperationAction(ISD::FMAD, MVT::f32, Legal); in SITargetLowering()
458 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); in SITargetLowering()
459 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); in SITargetLowering()
463 setOperationAction(ISD::CTPOP, MVT::i32, Expand); in SITargetLowering()
466 setOperationAction(ISD::CTPOP, MVT::i64, Expand); in SITargetLowering()
469 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Custom); in SITargetLowering()
472 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Custom); in SITargetLowering()
487 setOperationAction(ISD::UADDSAT, MVT::i32, Legal); in SITargetLowering()
488 setOperationAction(ISD::USUBSAT, MVT::i32, Legal); in SITargetLowering()
492 setOperationAction(ISD::SADDSAT, MVT::i16, Legal); in SITargetLowering()
493 setOperationAction(ISD::SSUBSAT, MVT::i16, Legal); in SITargetLowering()
494 setOperationAction(ISD::SADDSAT, MVT::i32, Legal); in SITargetLowering()
495 setOperationAction(ISD::SSUBSAT, MVT::i32, Legal); in SITargetLowering()
498 setOperationAction(ISD::FMINNUM, MVT::f32, Custom); in SITargetLowering()
499 setOperationAction(ISD::FMAXNUM, MVT::f32, Custom); in SITargetLowering()
500 setOperationAction(ISD::FMINNUM, MVT::f64, Custom); in SITargetLowering()
501 setOperationAction(ISD::FMAXNUM, MVT::f64, Custom); in SITargetLowering()
507 setOperationAction(ISD::FMINNUM_IEEE, MVT::f32, Legal); in SITargetLowering()
508 setOperationAction(ISD::FMAXNUM_IEEE, MVT::f32, Legal); in SITargetLowering()
509 setOperationAction(ISD::FMINNUM_IEEE, MVT::f64, Legal); in SITargetLowering()
510 setOperationAction(ISD::FMAXNUM_IEEE, MVT::f64, Legal); in SITargetLowering()
514 setOperationAction(ISD::FTRUNC, MVT::f64, Legal); in SITargetLowering()
515 setOperationAction(ISD::FCEIL, MVT::f64, Legal); in SITargetLowering()
516 setOperationAction(ISD::FRINT, MVT::f64, Legal); in SITargetLowering()
518 setOperationAction(ISD::FCEIL, MVT::f64, Custom); in SITargetLowering()
519 setOperationAction(ISD::FTRUNC, MVT::f64, Custom); in SITargetLowering()
520 setOperationAction(ISD::FRINT, MVT::f64, Custom); in SITargetLowering()
521 setOperationAction(ISD::FFLOOR, MVT::f64, Custom); in SITargetLowering()
524 setOperationAction(ISD::FFLOOR, MVT::f64, Legal); in SITargetLowering()
526 setOperationAction(ISD::FSIN, MVT::f32, Custom); in SITargetLowering()
527 setOperationAction(ISD::FCOS, MVT::f32, Custom); in SITargetLowering()
528 setOperationAction(ISD::FDIV, MVT::f32, Custom); in SITargetLowering()
529 setOperationAction(ISD::FDIV, MVT::f64, Custom); in SITargetLowering()
532 setOperationAction(ISD::Constant, MVT::i16, Legal); in SITargetLowering()
534 setOperationAction(ISD::SMIN, MVT::i16, Legal); in SITargetLowering()
535 setOperationAction(ISD::SMAX, MVT::i16, Legal); in SITargetLowering()
537 setOperationAction(ISD::UMIN, MVT::i16, Legal); in SITargetLowering()
538 setOperationAction(ISD::UMAX, MVT::i16, Legal); in SITargetLowering()
540 setOperationAction(ISD::SIGN_EXTEND, MVT::i16, Promote); in SITargetLowering()
541 AddPromotedToType(ISD::SIGN_EXTEND, MVT::i16, MVT::i32); in SITargetLowering()
543 setOperationAction(ISD::ROTR, MVT::i16, Expand); in SITargetLowering()
544 setOperationAction(ISD::ROTL, MVT::i16, Expand); in SITargetLowering()
546 setOperationAction(ISD::SDIV, MVT::i16, Promote); in SITargetLowering()
547 setOperationAction(ISD::UDIV, MVT::i16, Promote); in SITargetLowering()
548 setOperationAction(ISD::SREM, MVT::i16, Promote); in SITargetLowering()
549 setOperationAction(ISD::UREM, MVT::i16, Promote); in SITargetLowering()
550 setOperationAction(ISD::UADDSAT, MVT::i16, Legal); in SITargetLowering()
551 setOperationAction(ISD::USUBSAT, MVT::i16, Legal); in SITargetLowering()
553 setOperationAction(ISD::BITREVERSE, MVT::i16, Promote); in SITargetLowering()
555 setOperationAction(ISD::CTTZ, MVT::i16, Promote); in SITargetLowering()
556 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16, Promote); in SITargetLowering()
557 setOperationAction(ISD::CTLZ, MVT::i16, Promote); in SITargetLowering()
558 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16, Promote); in SITargetLowering()
559 setOperationAction(ISD::CTPOP, MVT::i16, Promote); in SITargetLowering()
561 setOperationAction(ISD::SELECT_CC, MVT::i16, Expand); in SITargetLowering()
563 setOperationAction(ISD::BR_CC, MVT::i16, Expand); in SITargetLowering()
565 setOperationAction(ISD::LOAD, MVT::i16, Custom); in SITargetLowering()
567 setTruncStoreAction(MVT::i64, MVT::i16, Expand); in SITargetLowering()
569 setOperationAction(ISD::FP16_TO_FP, MVT::i16, Promote); in SITargetLowering()
570 AddPromotedToType(ISD::FP16_TO_FP, MVT::i16, MVT::i32); in SITargetLowering()
571 setOperationAction(ISD::FP_TO_FP16, MVT::i16, Promote); in SITargetLowering()
572 AddPromotedToType(ISD::FP_TO_FP16, MVT::i16, MVT::i32); in SITargetLowering()
574 setOperationAction(ISD::FP_TO_SINT, MVT::i16, Custom); in SITargetLowering()
575 setOperationAction(ISD::FP_TO_UINT, MVT::i16, Custom); in SITargetLowering()
578 setOperationAction(ISD::ConstantFP, MVT::f16, Legal); in SITargetLowering()
581 setOperationAction(ISD::LOAD, MVT::f16, Promote); in SITargetLowering()
582 AddPromotedToType(ISD::LOAD, MVT::f16, MVT::i16); in SITargetLowering()
583 setOperationAction(ISD::STORE, MVT::f16, Promote); in SITargetLowering()
584 AddPromotedToType(ISD::STORE, MVT::f16, MVT::i16); in SITargetLowering()
587 setOperationAction(ISD::FP_ROUND, MVT::f16, Custom); in SITargetLowering()
588 setOperationAction(ISD::FCOS, MVT::f16, Custom); in SITargetLowering()
589 setOperationAction(ISD::FSIN, MVT::f16, Custom); in SITargetLowering()
591 setOperationAction(ISD::SINT_TO_FP, MVT::i16, Custom); in SITargetLowering()
592 setOperationAction(ISD::UINT_TO_FP, MVT::i16, Custom); in SITargetLowering()
594 setOperationAction(ISD::FP_TO_SINT, MVT::f16, Promote); in SITargetLowering()
595 setOperationAction(ISD::FP_TO_UINT, MVT::f16, Promote); in SITargetLowering()
596 setOperationAction(ISD::SINT_TO_FP, MVT::f16, Promote); in SITargetLowering()
597 setOperationAction(ISD::UINT_TO_FP, MVT::f16, Promote); in SITargetLowering()
598 setOperationAction(ISD::FROUND, MVT::f16, Custom); in SITargetLowering()
601 setOperationAction(ISD::BR_CC, MVT::f16, Expand); in SITargetLowering()
602 setOperationAction(ISD::SELECT_CC, MVT::f16, Expand); in SITargetLowering()
604 setOperationAction(ISD::FDIV, MVT::f16, Custom); in SITargetLowering()
607 setOperationAction(ISD::FMA, MVT::f16, Legal); in SITargetLowering()
609 setOperationAction(ISD::FMAD, MVT::f16, Legal); in SITargetLowering()
611 for (MVT VT : {MVT::v2i16, MVT::v2f16, MVT::v4i16, MVT::v4f16}) { in SITargetLowering()
635 setOperationAction(ISD::BSWAP, MVT::i16, Legal); in SITargetLowering()
636 setOperationAction(ISD::BSWAP, MVT::v2i16, Legal); in SITargetLowering()
637 setOperationAction(ISD::BSWAP, MVT::v4i16, Custom); in SITargetLowering()
640 setOperationAction(ISD::Constant, MVT::v2i16, Legal); in SITargetLowering()
641 setOperationAction(ISD::ConstantFP, MVT::v2f16, Legal); in SITargetLowering()
643 setOperationAction(ISD::UNDEF, MVT::v2i16, Legal); in SITargetLowering()
644 setOperationAction(ISD::UNDEF, MVT::v2f16, Legal); in SITargetLowering()
646 setOperationAction(ISD::STORE, MVT::v2i16, Promote); in SITargetLowering()
647 AddPromotedToType(ISD::STORE, MVT::v2i16, MVT::i32); in SITargetLowering()
648 setOperationAction(ISD::STORE, MVT::v2f16, Promote); in SITargetLowering()
649 AddPromotedToType(ISD::STORE, MVT::v2f16, MVT::i32); in SITargetLowering()
651 setOperationAction(ISD::LOAD, MVT::v2i16, Promote); in SITargetLowering()
652 AddPromotedToType(ISD::LOAD, MVT::v2i16, MVT::i32); in SITargetLowering()
653 setOperationAction(ISD::LOAD, MVT::v2f16, Promote); in SITargetLowering()
654 AddPromotedToType(ISD::LOAD, MVT::v2f16, MVT::i32); in SITargetLowering()
656 setOperationAction(ISD::AND, MVT::v2i16, Promote); in SITargetLowering()
657 AddPromotedToType(ISD::AND, MVT::v2i16, MVT::i32); in SITargetLowering()
658 setOperationAction(ISD::OR, MVT::v2i16, Promote); in SITargetLowering()
659 AddPromotedToType(ISD::OR, MVT::v2i16, MVT::i32); in SITargetLowering()
660 setOperationAction(ISD::XOR, MVT::v2i16, Promote); in SITargetLowering()
661 AddPromotedToType(ISD::XOR, MVT::v2i16, MVT::i32); in SITargetLowering()
663 setOperationAction(ISD::LOAD, MVT::v4i16, Promote); in SITargetLowering()
664 AddPromotedToType(ISD::LOAD, MVT::v4i16, MVT::v2i32); in SITargetLowering()
665 setOperationAction(ISD::LOAD, MVT::v4f16, Promote); in SITargetLowering()
666 AddPromotedToType(ISD::LOAD, MVT::v4f16, MVT::v2i32); in SITargetLowering()
668 setOperationAction(ISD::STORE, MVT::v4i16, Promote); in SITargetLowering()
669 AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32); in SITargetLowering()
670 setOperationAction(ISD::STORE, MVT::v4f16, Promote); in SITargetLowering()
671 AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32); in SITargetLowering()
673 setOperationAction(ISD::ANY_EXTEND, MVT::v2i32, Expand); in SITargetLowering()
674 setOperationAction(ISD::ZERO_EXTEND, MVT::v2i32, Expand); in SITargetLowering()
675 setOperationAction(ISD::SIGN_EXTEND, MVT::v2i32, Expand); in SITargetLowering()
676 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Expand); in SITargetLowering()
678 setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Expand); in SITargetLowering()
679 setOperationAction(ISD::ZERO_EXTEND, MVT::v4i32, Expand); in SITargetLowering()
680 setOperationAction(ISD::SIGN_EXTEND, MVT::v4i32, Expand); in SITargetLowering()
683 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i16, Custom); in SITargetLowering()
684 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f16, Custom); in SITargetLowering()
687 setOperationAction(ISD::FNEG, MVT::v2f16, Legal); in SITargetLowering()
690 setOperationAction(ISD::FABS, MVT::v2f16, Legal); in SITargetLowering()
692 setOperationAction(ISD::FMAXNUM, MVT::f16, Custom); in SITargetLowering()
693 setOperationAction(ISD::FMINNUM, MVT::f16, Custom); in SITargetLowering()
694 setOperationAction(ISD::FMAXNUM_IEEE, MVT::f16, Legal); in SITargetLowering()
695 setOperationAction(ISD::FMINNUM_IEEE, MVT::f16, Legal); in SITargetLowering()
697 setOperationAction(ISD::FMINNUM_IEEE, MVT::v4f16, Custom); in SITargetLowering()
698 setOperationAction(ISD::FMAXNUM_IEEE, MVT::v4f16, Custom); in SITargetLowering()
700 setOperationAction(ISD::FMINNUM, MVT::v4f16, Expand); in SITargetLowering()
701 setOperationAction(ISD::FMAXNUM, MVT::v4f16, Expand); in SITargetLowering()
705 setOperationAction(ISD::ADD, MVT::v2i16, Legal); in SITargetLowering()
706 setOperationAction(ISD::SUB, MVT::v2i16, Legal); in SITargetLowering()
707 setOperationAction(ISD::MUL, MVT::v2i16, Legal); in SITargetLowering()
708 setOperationAction(ISD::SHL, MVT::v2i16, Legal); in SITargetLowering()
709 setOperationAction(ISD::SRL, MVT::v2i16, Legal); in SITargetLowering()
710 setOperationAction(ISD::SRA, MVT::v2i16, Legal); in SITargetLowering()
711 setOperationAction(ISD::SMIN, MVT::v2i16, Legal); in SITargetLowering()
712 setOperationAction(ISD::UMIN, MVT::v2i16, Legal); in SITargetLowering()
713 setOperationAction(ISD::SMAX, MVT::v2i16, Legal); in SITargetLowering()
714 setOperationAction(ISD::UMAX, MVT::v2i16, Legal); in SITargetLowering()
716 setOperationAction(ISD::UADDSAT, MVT::v2i16, Legal); in SITargetLowering()
717 setOperationAction(ISD::USUBSAT, MVT::v2i16, Legal); in SITargetLowering()
718 setOperationAction(ISD::SADDSAT, MVT::v2i16, Legal); in SITargetLowering()
719 setOperationAction(ISD::SSUBSAT, MVT::v2i16, Legal); in SITargetLowering()
721 setOperationAction(ISD::FADD, MVT::v2f16, Legal); in SITargetLowering()
722 setOperationAction(ISD::FMUL, MVT::v2f16, Legal); in SITargetLowering()
723 setOperationAction(ISD::FMA, MVT::v2f16, Legal); in SITargetLowering()
725 setOperationAction(ISD::FMINNUM_IEEE, MVT::v2f16, Legal); in SITargetLowering()
726 setOperationAction(ISD::FMAXNUM_IEEE, MVT::v2f16, Legal); in SITargetLowering()
728 setOperationAction(ISD::FCANONICALIZE, MVT::v2f16, Legal); in SITargetLowering()
730 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i16, Custom); in SITargetLowering()
731 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f16, Custom); in SITargetLowering()
733 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f16, Custom); in SITargetLowering()
734 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom); in SITargetLowering()
736 setOperationAction(ISD::SHL, MVT::v4i16, Custom); in SITargetLowering()
737 setOperationAction(ISD::SRA, MVT::v4i16, Custom); in SITargetLowering()
738 setOperationAction(ISD::SRL, MVT::v4i16, Custom); in SITargetLowering()
739 setOperationAction(ISD::ADD, MVT::v4i16, Custom); in SITargetLowering()
740 setOperationAction(ISD::SUB, MVT::v4i16, Custom); in SITargetLowering()
741 setOperationAction(ISD::MUL, MVT::v4i16, Custom); in SITargetLowering()
743 setOperationAction(ISD::SMIN, MVT::v4i16, Custom); in SITargetLowering()
744 setOperationAction(ISD::SMAX, MVT::v4i16, Custom); in SITargetLowering()
745 setOperationAction(ISD::UMIN, MVT::v4i16, Custom); in SITargetLowering()
746 setOperationAction(ISD::UMAX, MVT::v4i16, Custom); in SITargetLowering()
748 setOperationAction(ISD::UADDSAT, MVT::v4i16, Custom); in SITargetLowering()
749 setOperationAction(ISD::SADDSAT, MVT::v4i16, Custom); in SITargetLowering()
750 setOperationAction(ISD::USUBSAT, MVT::v4i16, Custom); in SITargetLowering()
751 setOperationAction(ISD::SSUBSAT, MVT::v4i16, Custom); in SITargetLowering()
753 setOperationAction(ISD::FADD, MVT::v4f16, Custom); in SITargetLowering()
754 setOperationAction(ISD::FMUL, MVT::v4f16, Custom); in SITargetLowering()
755 setOperationAction(ISD::FMA, MVT::v4f16, Custom); in SITargetLowering()
757 setOperationAction(ISD::FMAXNUM, MVT::v2f16, Custom); in SITargetLowering()
758 setOperationAction(ISD::FMINNUM, MVT::v2f16, Custom); in SITargetLowering()
760 setOperationAction(ISD::FMINNUM, MVT::v4f16, Custom); in SITargetLowering()
761 setOperationAction(ISD::FMAXNUM, MVT::v4f16, Custom); in SITargetLowering()
762 setOperationAction(ISD::FCANONICALIZE, MVT::v4f16, Custom); in SITargetLowering()
764 setOperationAction(ISD::FEXP, MVT::v2f16, Custom); in SITargetLowering()
765 setOperationAction(ISD::SELECT, MVT::v4i16, Custom); in SITargetLowering()
766 setOperationAction(ISD::SELECT, MVT::v4f16, Custom); in SITargetLowering()
769 setOperationAction(ISD::FADD, MVT::v2f32, Legal); in SITargetLowering()
770 setOperationAction(ISD::FMUL, MVT::v2f32, Legal); in SITargetLowering()
771 setOperationAction(ISD::FMA, MVT::v2f32, Legal); in SITargetLowering()
772 setOperationAction(ISD::FNEG, MVT::v2f32, Legal); in SITargetLowering()
774 for (MVT VT : { MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32 }) { in SITargetLowering()
782 setOperationAction(ISD::FNEG, MVT::v4f16, Custom); in SITargetLowering()
783 setOperationAction(ISD::FABS, MVT::v4f16, Custom); in SITargetLowering()
786 setOperationAction(ISD::SELECT, MVT::v2i16, Promote); in SITargetLowering()
787 AddPromotedToType(ISD::SELECT, MVT::v2i16, MVT::i32); in SITargetLowering()
788 setOperationAction(ISD::SELECT, MVT::v2f16, Promote); in SITargetLowering()
789 AddPromotedToType(ISD::SELECT, MVT::v2f16, MVT::i32); in SITargetLowering()
792 setOperationAction(ISD::SELECT, MVT::v2i16, Custom); in SITargetLowering()
793 setOperationAction(ISD::SELECT, MVT::v2f16, Custom); in SITargetLowering()
795 setOperationAction(ISD::FNEG, MVT::v2f16, Custom); in SITargetLowering()
796 setOperationAction(ISD::FABS, MVT::v2f16, Custom); in SITargetLowering()
799 for (MVT VT : { MVT::v4i16, MVT::v4f16, MVT::v2i8, MVT::v4i8, MVT::v8i8 }) { in SITargetLowering()
803 setOperationAction(ISD::SMULO, MVT::i64, Custom); in SITargetLowering()
804 setOperationAction(ISD::UMULO, MVT::i64, Custom); in SITargetLowering()
806 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); in SITargetLowering()
807 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::f32, Custom); in SITargetLowering()
808 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v4f32, Custom); in SITargetLowering()
809 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom); in SITargetLowering()
810 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::f16, Custom); in SITargetLowering()
811 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v2i16, Custom); in SITargetLowering()
812 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v2f16, Custom); in SITargetLowering()
814 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v2f16, Custom); in SITargetLowering()
815 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v2i16, Custom); in SITargetLowering()
816 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v3f16, Custom); in SITargetLowering()
817 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v3i16, Custom); in SITargetLowering()
818 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v4f16, Custom); in SITargetLowering()
819 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v4i16, Custom); in SITargetLowering()
820 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v8f16, Custom); in SITargetLowering()
821 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); in SITargetLowering()
822 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::f16, Custom); in SITargetLowering()
823 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i16, Custom); in SITargetLowering()
824 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom); in SITargetLowering()
826 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); in SITargetLowering()
827 setOperationAction(ISD::INTRINSIC_VOID, MVT::v2i16, Custom); in SITargetLowering()
828 setOperationAction(ISD::INTRINSIC_VOID, MVT::v2f16, Custom); in SITargetLowering()
829 setOperationAction(ISD::INTRINSIC_VOID, MVT::v3i16, Custom); in SITargetLowering()
830 setOperationAction(ISD::INTRINSIC_VOID, MVT::v3f16, Custom); in SITargetLowering()
831 setOperationAction(ISD::INTRINSIC_VOID, MVT::v4f16, Custom); in SITargetLowering()
832 setOperationAction(ISD::INTRINSIC_VOID, MVT::v4i16, Custom); in SITargetLowering()
833 setOperationAction(ISD::INTRINSIC_VOID, MVT::f16, Custom); in SITargetLowering()
834 setOperationAction(ISD::INTRINSIC_VOID, MVT::i16, Custom); in SITargetLowering()
835 setOperationAction(ISD::INTRINSIC_VOID, MVT::i8, Custom); in SITargetLowering()
910 DestVT.getScalarType() == MVT::f32 && in isFPExtFoldable()
911 SrcVT.getScalarType() == MVT::f16 && in isFPExtFoldable()
922 MVT SITargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context, in getRegisterTypeForCallingConv()
933 return VT.isInteger() ? MVT::v2i16 : MVT::v2f16; in getRegisterTypeForCallingConv()
934 return VT.isInteger() ? MVT::i32 : MVT::f32; in getRegisterTypeForCallingConv()
938 return Subtarget->has16BitInsts() ? MVT::i16 : MVT::i32; in getRegisterTypeForCallingConv()
939 return Size == 32 ? ScalarVT.getSimpleVT() : MVT::i32; in getRegisterTypeForCallingConv()
943 return MVT::i32; in getRegisterTypeForCallingConv()
977 unsigned &NumIntermediates, MVT &RegisterVT) const { in getVectorTypeBreakdownForCallingConv()
986 RegisterVT = VT.isInteger() ? MVT::v2i16 : MVT::v2f16; in getVectorTypeBreakdownForCallingConv()
1001 RegisterVT = MVT::i16; in getVectorTypeBreakdownForCallingConv()
1009 RegisterVT = MVT::i32; in getVectorTypeBreakdownForCallingConv()
1016 RegisterVT = MVT::i32; in getVectorTypeBreakdownForCallingConv()
1119 Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType()); in getTgtMemIntrinsic()
1139 Info.memVT = MVT::getVT(CI.getType()); in getTgtMemIntrinsic()
1154 Info.memVT = MVT::getVT(CI.getOperand(0)->getType()); in getTgtMemIntrinsic()
1169 Info.memVT = MVT::getVT(CI.getType()); in getTgtMemIntrinsic()
1182 Info.memVT = MVT::getVT(CI.getType()); in getTgtMemIntrinsic()
1193 Info.memVT = MVT::getVT(CI.getType()); // XXX: what is correct VT? in getTgtMemIntrinsic()
1208 Info.memVT = MVT::getVT(CI.getType()); in getTgtMemIntrinsic()
1230 Info.memVT = MVT::i32; in getTgtMemIntrinsic()
1552 if (VT == MVT::Other || (VT != MVT::Other && VT.getSizeInBits() > 1024 && in allowsMisalignedMemoryAccesses()
1570 return MVT::v4i32; in getOptimalMemOpType()
1573 return MVT::v2i32; in getOptimalMemOpType()
1576 return MVT::Other; in getOptimalMemOpType()
1610 SITargetLowering::getPreferredVectorAction(MVT VT) const { in getPreferredVectorAction()
1612 VT.getScalarType().bitsLE(MVT::i16)) in getPreferredVectorAction()
1624 if (Subtarget->has16BitInsts() && VT == MVT::i16) { in isTypeDesirableForOp()
1643 if (VT == MVT::i1 && Op == ISD::SETCC) in isTypeDesirableForOp()
1665 MVT PtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS); in lowerKernArgParameterPtr()
1690 DAG.getConstant(0, SL, MVT::i32)); in convertArgType()
1729 SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr, PtrInfo, Align(4), in lowerKernargMemParameter()
1733 SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32); in lowerKernargMemParameter()
1734 SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt); in lowerKernargMemParameter()
1762 return DAG.getFrameIndex(FrameIdx, MVT::i32); in lowerStackParameter()
1771 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32); in lowerStackParameter()
1776 MVT MemVT = VA.getValVT(); in lowerStackParameter()
2411 MVT VT = VA.getLocVT(); in LowerFormalArguments()
2448 DAG.getValueType(MVT::i16)); in LowerFormalArguments()
2532 DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); in LowerFormalArguments()
2591 DAG, &AMDGPU::SReg_64RegClass, TRI->getReturnAddressReg(MF), MVT::i64); in LowerReturn()
2595 MVT::i64); in LowerReturn()
2643 RetOps.push_back(DAG.getRegister(*I, MVT::i64)); in LowerReturn()
2645 RetOps.push_back(DAG.getRegister(*I, MVT::i32)); in LowerReturn()
2660 return DAG.getNode(Opc, DL, MVT::Other, RetOps); in LowerReturn()
2778 EVT ArgVT = TRI->getSpillSize(*ArgRC) == 8 ? MVT::i64 : MVT::i32; in passSpecialInputs()
2832 InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgX); in passSpecialInputs()
2835 SDValue Y = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgY); in passSpecialInputs()
2836 Y = DAG.getNode(ISD::SHL, SL, MVT::i32, Y, in passSpecialInputs()
2837 DAG.getShiftAmountConstant(10, MVT::i32, SL)); in passSpecialInputs()
2839 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Y) : Y; in passSpecialInputs()
2843 SDValue Z = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgZ); in passSpecialInputs()
2844 Z = DAG.getNode(ISD::SHL, SL, MVT::i32, Z, in passSpecialInputs()
2845 DAG.getShiftAmountConstant(20, MVT::i32, SL)); in passSpecialInputs()
2847 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Z) : Z; in passSpecialInputs()
2857 InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, IncomingArg); in passSpecialInputs()
3089 = DAG.getCopyFromReg(Chain, DL, Info->getScratchRSrcReg(), MVT::v4i32); in LowerCall()
3096 MVT PtrVT = MVT::i32; in LowerCall()
3168 MVT::i32); in LowerCall()
3169 DstAddr = DAG.getNode(ISD::ADD, DL, MVT::i32, SP, PtrOff); in LowerCall()
3177 DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i32); in LowerCall()
3201 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); in LowerCall()
3220 DAG, &AMDGPU::SReg_64RegClass, TRI->getReturnAddressReg(MF), MVT::i64); in LowerCall()
3223 MVT::i64); in LowerCall()
3234 DAG.getTargetConstant(NumBytes, DL, MVT::i32), in LowerCall()
3235 DAG.getTargetConstant(0, DL, MVT::i32), in LowerCall()
3247 Ops.push_back(DAG.getTargetGlobalAddress(GV, DL, MVT::i64)); in LowerCall()
3249 Ops.push_back(DAG.getTargetConstant(0, DL, MVT::i64)); in LowerCall()
3256 Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32)); in LowerCall()
3278 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerCall()
3293 Chain = DAG.getCALLSEQ_END(Chain, DAG.getTargetConstant(0, DL, MVT::i32), in LowerCall()
3294 DAG.getTargetConstant(CalleePopBytes, DL, MVT::i32), in LowerCall()
3338 DAG.getConstant(ST.getWavefrontSizeLog2(), dl, MVT::i32)); in lowerDYNAMIC_STACKALLOCImpl()
4391 return MVT::i1; in getSetCCResultType()
4393 return EVT::getVectorVT(Ctx, MVT::i1, VT.getVectorNumElements()); in getSetCCResultType()
4396 MVT SITargetLowering::getScalarShiftAmountTy(const DataLayout &, EVT VT) const { in getScalarShiftAmountTy()
4399 return (VT == MVT::i16) ? MVT::i16 : MVT::i32; in getScalarShiftAmountTy()
4428 case MVT::f32: { in isFMAFasterThanFMulAndFAdd()
4442 case MVT::f64: in isFMAFasterThanFMulAndFAdd()
4444 case MVT::f16: in isFMAFasterThanFMulAndFAdd()
4458 if (VT == MVT::f32) in isFMADLegal()
4461 if (VT == MVT::f16) { in isFMADLegal()
4479 assert(VT == MVT::v4f16 || VT == MVT::v4i16); in splitUnaryVectorOp()
4499 assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4f32 || in splitBinaryVectorOp()
4500 VT == MVT::v8f32 || VT == MVT::v16f32 || VT == MVT::v32f32); in splitBinaryVectorOp()
4521 assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4f32 || in splitTernaryVectorOp()
4522 VT == MVT::v8f32 || VT == MVT::v16f32 || VT == MVT::v32f32); in splitTernaryVectorOp()
4655 Elt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Elt); in adjustLoadValueTypeImpl()
4659 Elts.push_back(DAG.getUNDEF(MVT::i16)); in adjustLoadValueTypeImpl()
4684 EquivLoadVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, in adjustLoadValueType()
4695 SDVTList VTList = DAG.getVTList(EquivLoadVT, MVT::Other); in adjustLoadValueType()
4735 SDVTList VTList = DAG.getVTList(CastVT, MVT::Other); in lowerIntrinsicLoad()
4759 if (CmpVT == MVT::i16 && !TLI.isTypeLegal(MVT::i16)) { in lowerICMPIntrinsic()
4762 LHS = DAG.getNode(PromoteOp, DL, MVT::i32, LHS); in lowerICMPIntrinsic()
4763 RHS = DAG.getNode(PromoteOp, DL, MVT::i32, RHS); in lowerICMPIntrinsic()
4792 if (CmpVT == MVT::f16 && !TLI.isTypeLegal(CmpVT)) { in lowerFCMPIntrinsic()
4793 Src0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0); in lowerFCMPIntrinsic()
4794 Src1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1); in lowerFCMPIntrinsic()
4841 AMDGPUISD::SETCC, SL, VT, DAG.getZExtOrTrunc(Src, SL, MVT::i32), in lowerBALLOTIntrinsic()
4842 DAG.getConstant(0, SL, MVT::i32), DAG.getCondCode(ISD::SETNE)); in lowerBALLOTIntrinsic()
4866 SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_PKRTZ_F16_F32, SL, MVT::i32, in ReplaceNodeResults()
4868 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Cvt)); in ReplaceNodeResults()
4893 SDValue Cvt = DAG.getNode(Opcode, SL, MVT::i32, Src0, Src1); in ReplaceNodeResults()
4894 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, Cvt)); in ReplaceNodeResults()
4925 if (NewVT.bitsLT(MVT::i32)) { in ReplaceNodeResults()
4926 LHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, LHS); in ReplaceNodeResults()
4927 RHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, RHS); in ReplaceNodeResults()
4928 SelectVT = MVT::i32; in ReplaceNodeResults()
4940 if (N->getValueType(0) != MVT::v2f16) in ReplaceNodeResults()
4944 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0)); in ReplaceNodeResults()
4946 SDValue Op = DAG.getNode(ISD::XOR, SL, MVT::i32, in ReplaceNodeResults()
4948 DAG.getConstant(0x80008000, SL, MVT::i32)); in ReplaceNodeResults()
4949 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op)); in ReplaceNodeResults()
4953 if (N->getValueType(0) != MVT::v2f16) in ReplaceNodeResults()
4957 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0)); in ReplaceNodeResults()
4959 SDValue Op = DAG.getNode(ISD::AND, SL, MVT::i32, in ReplaceNodeResults()
4961 DAG.getConstant(0x7fff7fff, SL, MVT::i32)); in ReplaceNodeResults()
4962 Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op)); in ReplaceNodeResults()
5131 MVT VT = Op.getSimpleValueType(); in LowerRETURNADDR()
5161 DAG.getTargetConstant(0, DL, MVT::i32)); in getFPExtOrFPRound()
5165 assert(Op.getValueType() == MVT::f16 && in lowerFP_ROUND()
5170 if (SrcVT != MVT::f64) in lowerFP_ROUND()
5175 SDValue FpToFp16 = DAG.getNode(ISD::FP_TO_FP16, DL, MVT::i32, Src); in lowerFP_ROUND()
5176 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToFp16); in lowerFP_ROUND()
5177 return DAG.getNode(ISD::BITCAST, DL, MVT::f16, Trunc); in lowerFP_ROUND()
5194 if (VT == MVT::v4f16) in lowerFMINNUM_FMAXNUM()
5212 SDValue ShiftAmt = DAG.getConstant(C.logBase2(), SL, MVT::i32); in lowerXMULO()
5214 SDValue Overflow = DAG.getSetCC(SL, MVT::i1, in lowerXMULO()
5228 DAG.getConstant(VT.getScalarSizeInBits() - 1, SL, MVT::i32)) in lowerXMULO()
5230 SDValue Overflow = DAG.getSetCC(SL, MVT::i1, Top, Sign, ISD::SETNE); in lowerXMULO()
5258 return DAG.getNode(AMDGPUISD::ENDPGM, SL, MVT::Other, Chain); in lowerTrapEndpgm()
5271 DAG, &AMDGPU::SReg_64RegClass, UserSGPR, MVT::i64); in lowerTrapHsaQueuePtr()
5272 SDValue SGPR01 = DAG.getRegister(AMDGPU::SGPR0_SGPR1, MVT::i64); in lowerTrapHsaQueuePtr()
5279 DAG.getTargetConstant(TrapID, SL, MVT::i16), in lowerTrapHsaQueuePtr()
5283 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); in lowerTrapHsaQueuePtr()
5294 DAG.getTargetConstant(TrapID, SL, MVT::i16) in lowerTrapHsa()
5296 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); in lowerTrapHsa()
5318 DAG.getTargetConstant(TrapID, SL, MVT::i16) in lowerDEBUGTRAP()
5320 return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops); in lowerDEBUGTRAP()
5338 SDValue EncodingImm = DAG.getTargetConstant(Encoding, DL, MVT::i16); in getSegmentAperture()
5340 DAG.getMachineNode(AMDGPU::S_GETREG_B32, DL, MVT::i32, EncodingImm), 0); in getSegmentAperture()
5341 SDValue ShiftAmount = DAG.getTargetConstant(WidthM1 + 1, DL, MVT::i32); in getSegmentAperture()
5342 return DAG.getNode(ISD::SHL, DL, MVT::i32, ApertureReg, ShiftAmount); in getSegmentAperture()
5351 DAG, &AMDGPU::SReg_64RegClass, UserSGPR, MVT::i64); in getSegmentAperture()
5364 return DAG.getLoad(MVT::i32, DL, QueuePtr.getValue(1), Ptr, PtrInfo, in getSegmentAperture()
5376 SDValue FlatNullPtr = DAG.getConstant(0, SL, MVT::i64); in lowerADDRSPACECAST()
5388 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32); in lowerADDRSPACECAST()
5389 SDValue NonNull = DAG.getSetCC(SL, MVT::i1, Src, FlatNullPtr, ISD::SETNE); in lowerADDRSPACECAST()
5390 SDValue Ptr = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src); in lowerADDRSPACECAST()
5392 return DAG.getNode(ISD::SELECT, SL, MVT::i32, in lowerADDRSPACECAST()
5404 SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32); in lowerADDRSPACECAST()
5407 = DAG.getSetCC(SL, MVT::i1, Src, SegmentNullPtr, ISD::SETNE); in lowerADDRSPACECAST()
5411 = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Aperture); in lowerADDRSPACECAST()
5413 return DAG.getNode(ISD::SELECT, SL, MVT::i64, NonNull, in lowerADDRSPACECAST()
5414 DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr), in lowerADDRSPACECAST()
5420 Src.getValueType() == MVT::i64) in lowerADDRSPACECAST()
5421 return DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src); in lowerADDRSPACECAST()
5452 DAG.getConstant(I, SL, MVT::i32)); in lowerINSERT_SUBVECTOR()
5454 DAG.getConstant(IdxVal + I, SL, MVT::i32)); in lowerINSERT_SUBVECTOR()
5477 SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Vec); in lowerINSERT_VECTOR_ELT()
5479 SDValue LoHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec, in lowerINSERT_VECTOR_ELT()
5480 DAG.getConstant(0, SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
5481 SDValue HiHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec, in lowerINSERT_VECTOR_ELT()
5482 DAG.getConstant(1, SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
5484 SDValue LoVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, LoHalf); in lowerINSERT_VECTOR_ELT()
5485 SDValue HiVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, HiHalf); in lowerINSERT_VECTOR_ELT()
5489 SDValue InsHalf = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, MVT::v2i16, in lowerINSERT_VECTOR_ELT()
5491 DAG.getNode(ISD::BITCAST, SL, MVT::i16, InsVal), in lowerINSERT_VECTOR_ELT()
5492 DAG.getConstant(InsertLo ? Idx : (Idx - 2), SL, MVT::i32)); in lowerINSERT_VECTOR_ELT()
5494 InsHalf = DAG.getNode(ISD::BITCAST, SL, MVT::i32, InsHalf); in lowerINSERT_VECTOR_ELT()
5497 DAG.getBuildVector(MVT::v2i32, SL, { InsHalf, HiHalf }) : in lowerINSERT_VECTOR_ELT()
5498 DAG.getBuildVector(MVT::v2i32, SL, { LoHalf, InsHalf }); in lowerINSERT_VECTOR_ELT()
5506 MVT IntVT = MVT::getIntegerVT(VecSize); in lowerINSERT_VECTOR_ELT()
5517 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32); in lowerINSERT_VECTOR_ELT()
5520 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor); in lowerINSERT_VECTOR_ELT()
5559 MVT IntVT = MVT::getIntegerVT(VecSize); in lowerEXTRACT_VECTOR_ELT()
5560 SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32); in lowerEXTRACT_VECTOR_ELT()
5563 SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor); in lowerEXTRACT_VECTOR_ELT()
5568 if (ResultVT == MVT::f16) { in lowerEXTRACT_VECTOR_ELT()
5569 SDValue Result = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, Elt); in lowerEXTRACT_VECTOR_ELT()
5587 EVT PackVT = ResultVT.isInteger() ? MVT::v2i16 : MVT::v2f16; in lowerVECTOR_SHUFFLE()
5609 DAG.getConstant(EltIdx, SL, MVT::i32)); in lowerVECTOR_SHUFFLE()
5621 Vec0, DAG.getConstant(EltIdx0, SL, MVT::i32)); in lowerVECTOR_SHUFFLE()
5625 Vec1, DAG.getConstant(EltIdx1, SL, MVT::i32)); in lowerVECTOR_SHUFFLE()
5638 if (VT == MVT::v4i16 || VT == MVT::v4f16) { in lowerBUILD_VECTOR()
5639 EVT HalfVT = MVT::getVectorVT(VT.getVectorElementType().getSimpleVT(), 2); in lowerBUILD_VECTOR()
5648 SDValue CastLo = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Lo); in lowerBUILD_VECTOR()
5649 SDValue CastHi = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Hi); in lowerBUILD_VECTOR()
5651 SDValue Blend = DAG.getBuildVector(MVT::v2i32, SL, { CastLo, CastHi }); in lowerBUILD_VECTOR()
5655 assert(VT == MVT::v2f16 || VT == MVT::v2i16); in lowerBUILD_VECTOR()
5663 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo); in lowerBUILD_VECTOR()
5664 SDValue ExtLo = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Lo); in lowerBUILD_VECTOR()
5668 Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Hi); in lowerBUILD_VECTOR()
5669 Hi = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Hi); in lowerBUILD_VECTOR()
5671 SDValue ShlHi = DAG.getNode(ISD::SHL, SL, MVT::i32, Hi, in lowerBUILD_VECTOR()
5672 DAG.getConstant(16, SL, MVT::i32)); in lowerBUILD_VECTOR()
5676 Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo); in lowerBUILD_VECTOR()
5677 Lo = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Lo); in lowerBUILD_VECTOR()
5679 SDValue Or = DAG.getNode(ISD::OR, SL, MVT::i32, Lo, ShlHi); in lowerBUILD_VECTOR()
5730 DAG.getTargetGlobalAddress(GV, DL, MVT::i32, Offset + 4, GAFlags); in buildPCRelGlobalAddress()
5733 PtrHi = DAG.getTargetConstant(0, DL, MVT::i32); in buildPCRelGlobalAddress()
5736 DAG.getTargetGlobalAddress(GV, DL, MVT::i32, Offset + 12, GAFlags + 1); in buildPCRelGlobalAddress()
5762 assert(PtrVT == MVT::i32 && "32-bit pointer is expected."); in LowerGlobalAddress()
5773 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, GSD->getOffset(), in LowerGlobalAddress()
5775 return DAG.getNode(AMDGPUISD::LDS, DL, MVT::i32, GA); in LowerGlobalAddress()
5810 SDNode *M0 = DAG.getMachineNode(AMDGPU::SI_INIT_M0, DL, MVT::Other, MVT::Glue, in copyToM0()
5817 MVT VT, in lowerImplicitZextParam()
5821 DAG, MVT::i32, MVT::i32, SL, DAG.getEntryNode(), Offset, Align(4), false); in lowerImplicitZextParam()
5823 return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Param, in lowerImplicitZextParam()
5848 MVT Type; in getBuildDwordsVector()
5852 Type = MVT::getVectorVT(MVT::f32, NumElts); in getBuildDwordsVector()
5855 Type = MVT::v16f32; in getBuildDwordsVector()
5862 if (Elt.getValueType() != MVT::f32) in getBuildDwordsVector()
5863 Elt = DAG.getBitcast(MVT::f32, Elt); in getBuildDwordsVector()
5867 VecElts[i] = DAG.getUNDEF(MVT::f32); in getBuildDwordsVector()
5910 MVT DataDwordVT = NumDataDwords == 1 ? in constructRetValue()
5911 MVT::i32 : MVT::getVectorVT(MVT::i32, NumDataDwords); in constructRetValue()
5913 MVT MaskPopVT = MaskPopDwords == 1 ? in constructRetValue()
5914 MVT::i32 : MVT::getVectorVT(MVT::i32, MaskPopDwords); in constructRetValue()
5920 SDValue ZeroIdx = DAG.getConstant(0, DL, MVT::i32); in constructRetValue()
5953 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, SDValue(Result, 0), in constructRetValue()
5954 DAG.getConstant(MaskPopDwords, DL, MVT::i32)); in constructRetValue()
5975 *TFE = DAG.getTargetConstant((Value & 0x1) ? 1 : 0, DL, MVT::i32); in parseTexFail()
5977 *LWE = DAG.getTargetConstant((Value & 0x2) ? 1 : 0, DL, MVT::i32); in parseTexFail()
5984 MVT PackVectorVT, in packImage16bitOpsToDwords()
6000 if (Addr.getValueType() != MVT::i16) in packImage16bitOpsToDwords()
6001 Addr = DAG.getBitcast(MVT::i16, Addr); in packImage16bitOpsToDwords()
6002 Addr = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Addr); in packImage16bitOpsToDwords()
6007 Addr = DAG.getBitcast(MVT::f32, Addr); in packImage16bitOpsToDwords()
6046 bool Is64Bit = VData.getValueType() == MVT::i64; in lowerImage()
6049 VData = DAG.getBuildVector(Is64Bit ? MVT::v2i64 : MVT::v2i32, DL, in lowerImage()
6052 VData = DAG.getBitcast(MVT::v4i32, VData); in lowerImage()
6054 ResultTypes[0] = Is64Bit ? MVT::v2i64 : MVT::v2i32; in lowerImage()
6070 MVT StoreVT = VData.getSimpleValueType(); in lowerImage()
6071 if (StoreVT.getScalarType() == MVT::f16) { in lowerImage()
6083 MVT LoadVT = ResultTypes[0].getSimpleVT(); in lowerImage()
6084 if (LoadVT.getScalarType() == MVT::f16) { in lowerImage()
6139 MVT VAddrVT = in lowerImage()
6141 MVT VAddrScalarVT = VAddrVT.getScalarType(); in lowerImage()
6142 MVT GradPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16; in lowerImage()
6143 IsG16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16; in lowerImage()
6147 MVT AddrPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16; in lowerImage()
6148 IsA16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16; in lowerImage()
6221 SDValue True = DAG.getTargetConstant(1, DL, MVT::i1); in lowerImage()
6222 SDValue False = DAG.getTargetConstant(0, DL, MVT::i1); in lowerImage()
6265 EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumVDataDwords) in lowerImage()
6266 : MVT::i32; in lowerImage()
6294 Ops.push_back(DAG.getTargetConstant(DMask, DL, MVT::i32)); in lowerImage()
6296 Ops.push_back(DAG.getTargetConstant(DimInfo->Encoding, DL, MVT::i32)); in lowerImage()
6298 Ops.push_back(DAG.getTargetConstant(CPol, DL, MVT::i32)); in lowerImage()
6404 MVT LoadVT = VT.getSimpleVT(); in lowerSBuffer()
6406 assert((LoadVT.getScalarType() == MVT::i32 || in lowerSBuffer()
6407 LoadVT.getScalarType() == MVT::f32)); in lowerSBuffer()
6411 LoadVT = MVT::getVectorVT(LoadVT.getScalarType(), 4); in lowerSBuffer()
6414 SDVTList VTList = DAG.getVTList({LoadVT, MVT::Glue}); in lowerSBuffer()
6418 DAG.getConstant(0, DL, MVT::i32), // vindex in lowerSBuffer()
6423 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in lowerSBuffer()
6433 Ops[5] = DAG.getTargetConstant(InstOffset + 16 * i, DL, MVT::i32); in lowerSBuffer()
6566 return lowerImplicitZextParam(DAG, Op, MVT::i16, in LowerINTRINSIC_WO_CHAIN()
6572 return lowerImplicitZextParam(DAG, Op, MVT::i16, in LowerINTRINSIC_WO_CHAIN()
6578 return lowerImplicitZextParam(DAG, Op, MVT::i16, in LowerINTRINSIC_WO_CHAIN()
6590 return loadInputValue(DAG, &AMDGPU::VGPR_32RegClass, MVT::i32, in LowerINTRINSIC_WO_CHAIN()
6594 return loadInputValue(DAG, &AMDGPU::VGPR_32RegClass, MVT::i32, in LowerINTRINSIC_WO_CHAIN()
6598 return loadInputValue(DAG, &AMDGPU::VGPR_32RegClass, MVT::i32, in LowerINTRINSIC_WO_CHAIN()
6603 SDLoc(Op), MVT::i32); in LowerINTRINSIC_WO_CHAIN()
6669 if (Op.getOperand(1).getValueType() == MVT::i1 && in LowerINTRINSIC_WO_CHAIN()
6721 SDValue Node = DAG.getNode(Opcode, DL, MVT::i32, in LowerINTRINSIC_WO_CHAIN()
6741 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, 0, in LowerINTRINSIC_WO_CHAIN()
6743 return {DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, GA), 0}; in LowerINTRINSIC_WO_CHAIN()
6751 SDValue SrcVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, in LowerINTRINSIC_WO_CHAIN()
6754 SDValue SrcHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, SrcVec, in LowerINTRINSIC_WO_CHAIN()
6755 DAG.getConstant(1, SL, MVT::i32)); in LowerINTRINSIC_WO_CHAIN()
6756 return DAG.getSetCC(SL, MVT::i1, SrcHi, Aperture, ISD::SETEQ); in LowerINTRINSIC_WO_CHAIN()
6762 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, Op.getOperand(1), in LowerINTRINSIC_WO_CHAIN()
6770 SDValue GA = DAG.getTargetGlobalAddress(RelocSymbol, DL, MVT::i32, 0, in LowerINTRINSIC_WO_CHAIN()
6772 return {DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, GA), 0}; in LowerINTRINSIC_WO_CHAIN()
6819 DAG.getConstant(0, DL, MVT::i32), // vindex in lowerRawBufferAtomicIntrin()
6824 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in lowerRawBufferAtomicIntrin()
6859 DAG.getTargetConstant(1, DL, MVT::i1), // idxen in lowerStructBufferAtomicIntrin()
6921 DAG.getTargetConstant(Offset, DL, MVT::i16), in LowerINTRINSIC_W_CHAIN()
6984 DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy in LowerINTRINSIC_W_CHAIN()
6985 DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
6998 if (LoadVT.getScalarType() == MVT::f16) in LowerINTRINSIC_W_CHAIN()
7003 if (LoadVT.getScalarType() == MVT::i8 || in LowerINTRINSIC_W_CHAIN()
7004 LoadVT.getScalarType() == MVT::i16) in LowerINTRINSIC_W_CHAIN()
7018 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_W_CHAIN()
7023 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
7043 DAG.getTargetConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
7066 DAG.getTargetConstant(Dfmt | (Nfmt << 4), DL, MVT::i32), // format in LowerINTRINSIC_W_CHAIN()
7067 DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy in LowerINTRINSIC_W_CHAIN()
7068 DAG.getTargetConstant(IdxEn, DL, MVT::i1) // idxen in LowerINTRINSIC_W_CHAIN()
7071 if (LoadVT.getScalarType() == MVT::f16) in LowerINTRINSIC_W_CHAIN()
7086 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_W_CHAIN()
7092 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
7095 if (LoadVT.getScalarType() == MVT::f16) in LowerINTRINSIC_W_CHAIN()
7116 DAG.getTargetConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
7119 if (LoadVT.getScalarType() == MVT::f16) in LowerINTRINSIC_W_CHAIN()
7148 DAG.getTargetConstant(Slc << 1, DL, MVT::i32), // cachepolicy in LowerINTRINSIC_W_CHAIN()
7149 DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
7289 DAG.getTargetConstant(Slc << 1, DL, MVT::i32), // cachepolicy in LowerINTRINSIC_W_CHAIN()
7290 DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
7308 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_W_CHAIN()
7313 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
7334 DAG.getTargetConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_W_CHAIN()
7353 assert(NodePtr.getValueType() == MVT::i32 || in LowerINTRINSIC_W_CHAIN()
7354 NodePtr.getValueType() == MVT::i64); in LowerINTRINSIC_W_CHAIN()
7355 assert(RayDir.getValueType() == MVT::v4f16 || in LowerINTRINSIC_W_CHAIN()
7356 RayDir.getValueType() == MVT::v4f32); in LowerINTRINSIC_W_CHAIN()
7363 bool IsA16 = RayDir.getValueType().getVectorElementType() == MVT::f16; in LowerINTRINSIC_W_CHAIN()
7364 bool Is64 = NodePtr.getValueType() == MVT::i64; in LowerINTRINSIC_W_CHAIN()
7377 Ops.push_back(DAG.getBitcast(MVT::i32, Lanes[I])); in LowerINTRINSIC_W_CHAIN()
7381 DAG.getBitcast(MVT::i32, in LowerINTRINSIC_W_CHAIN()
7382 DAG.getBuildVector(MVT::v2f16, DL, in LowerINTRINSIC_W_CHAIN()
7388 DAG.getBitcast(MVT::i32, in LowerINTRINSIC_W_CHAIN()
7389 DAG.getBuildVector(MVT::v2f16, DL, in LowerINTRINSIC_W_CHAIN()
7392 DAG.getBitcast(MVT::i32, in LowerINTRINSIC_W_CHAIN()
7393 DAG.getBuildVector(MVT::v2f16, DL, in LowerINTRINSIC_W_CHAIN()
7400 DAG.ExtractVectorElements(DAG.getBitcast(MVT::v2i32, NodePtr), Ops, 0, 2); in LowerINTRINSIC_W_CHAIN()
7404 Ops.push_back(DAG.getBitcast(MVT::i32, RayExtent)); in LowerINTRINSIC_W_CHAIN()
7410 Ops.push_back(DAG.getTargetConstant(1, DL, MVT::i1)); in LowerINTRINSIC_W_CHAIN()
7445 DAG.getVTList(VT, MVT::Other), Ops, in LowerINTRINSIC_W_CHAIN()
7486 (WidenedVT == MVT::v3i32 || WidenedVT == MVT::v3f32)) { in getMemIntrinsicNode()
7524 EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElements); in handleD16VData()
7545 DAG.getBuildVector(MVT::v2i16, DL, {Elts[I * 2], Elts[I * 2 + 1]}); in handleD16VData()
7546 SDValue IntPair = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Pair); in handleD16VData()
7552 SDValue Pair = DAG.getBuildVector(MVT::v2i16, DL, in handleD16VData()
7553 {Elts[I * 2], DAG.getUNDEF(MVT::i16)}); in handleD16VData()
7554 SDValue IntPair = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Pair); in handleD16VData()
7559 PackedElts.resize(Elts.size(), DAG.getUNDEF(MVT::i32)); in handleD16VData()
7563 EVT::getVectorVT(*DAG.getContext(), MVT::i32, PackedElts.size()); in handleD16VData()
7600 SDValue Undef = DAG.getUNDEF(MVT::f32); in LowerINTRINSIC_VOID()
7603 DAG.getNode(ISD::BITCAST, DL, MVT::f32, Src0), // src0 in LowerINTRINSIC_VOID()
7604 DAG.getNode(ISD::BITCAST, DL, MVT::f32, Src1), // src1 in LowerINTRINSIC_VOID()
7608 DAG.getTargetConstant(1, DL, MVT::i1), // compr in LowerINTRINSIC_VOID()
7621 return SDValue(DAG.getMachineNode(AMDGPU::WAVE_BARRIER, DL, MVT::Other, in LowerINTRINSIC_VOID()
7628 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); in LowerINTRINSIC_VOID()
7644 DAG.getTargetConstant(Dfmt | (Nfmt << 4), DL, MVT::i32), // format in LowerINTRINSIC_VOID()
7645 DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy in LowerINTRINSIC_VOID()
7646 DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen in LowerINTRINSIC_VOID()
7657 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); in LowerINTRINSIC_VOID()
7671 DAG.getTargetConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_VOID()
7682 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); in LowerINTRINSIC_VOID()
7690 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_VOID()
7696 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_VOID()
7708 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16); in LowerINTRINSIC_VOID()
7722 DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy in LowerINTRINSIC_VOID()
7723 DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen in LowerINTRINSIC_VOID()
7735 if (VDataType == MVT::i8 || VDataType == MVT::i16) in LowerINTRINSIC_VOID()
7767 DAG.getConstant(0, DL, MVT::i32), // vindex in LowerINTRINSIC_VOID()
7772 DAG.getTargetConstant(0, DL, MVT::i1), // idxen in LowerINTRINSIC_VOID()
7819 DAG.getTargetConstant(1, DL, MVT::i1), // idxen in LowerINTRINSIC_VOID()
7836 return SDValue(DAG.getMachineNode(AMDGPU::SI_END_CF, DL, MVT::Other, in LowerINTRINSIC_VOID()
7884 C1 = cast<ConstantSDNode>(DAG.getTargetConstant(ImmOffset, DL, MVT::i32)); in splitBufferOffsets()
7886 auto OverflowVal = DAG.getConstant(Overflow, DL, MVT::i32); in splitBufferOffsets()
7891 N0 = DAG.getNode(ISD::ADD, DL, MVT::i32, Ops); in splitBufferOffsets()
7896 N0 = DAG.getConstant(0, DL, MVT::i32); in splitBufferOffsets()
7898 C1 = cast<ConstantSDNode>(DAG.getTargetConstant(0, DL, MVT::i32)); in splitBufferOffsets()
7914 Offsets[0] = DAG.getConstant(0, DL, MVT::i32); in setBufferOffsets()
7915 Offsets[1] = DAG.getConstant(SOffset, DL, MVT::i32); in setBufferOffsets()
7916 Offsets[2] = DAG.getTargetConstant(ImmOffset, DL, MVT::i32); in setBufferOffsets()
7928 Offsets[1] = DAG.getConstant(SOffset, DL, MVT::i32); in setBufferOffsets()
7929 Offsets[2] = DAG.getTargetConstant(ImmOffset, DL, MVT::i32); in setBufferOffsets()
7934 Offsets[1] = DAG.getConstant(0, DL, MVT::i32); in setBufferOffsets()
7935 Offsets[2] = DAG.getTargetConstant(0, DL, MVT::i32); in setBufferOffsets()
7944 unsigned Opc = (LoadVT.getScalarType() == MVT::i8) ? in handleByteShortBufferLoads()
7947 SDVTList ResList = DAG.getVTList(MVT::i32, MVT::Other); in handleByteShortBufferLoads()
7962 if (VDataType == MVT::f16) in handleByteShortBufferStores()
7963 Ops[1] = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Ops[1]); in handleByteShortBufferStores()
7965 SDValue BufferStoreExt = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Ops[1]); in handleByteShortBufferStores()
7967 unsigned Opc = (VDataType == MVT::i8) ? AMDGPUISD::BUFFER_STORE_BYTE : in handleByteShortBufferStores()
8022 MVT::i32, SL, Ld->getChain(), Ptr, in widenLoad()
8024 Ld->getPointerInfo(), MVT::i32, in widenLoad()
8039 Cvt = DAG.getNode(ISD::SIGN_EXTEND_INREG, SL, MVT::i32, NewLoad, in widenLoad()
8071 if (MemVT == MVT::i16 && isTypeLegal(MVT::i16)) in LowerLOAD()
8081 EVT RealMemVT = (MemVT == MVT::i1) ? MVT::i8 : MVT::i16; in LowerLOAD()
8083 SDValue NewLD = DAG.getExtLoad(ISD::EXTLOAD, DL, MVT::i32, Chain, in LowerLOAD()
8097 SDValue Elt = DAG.getNode(ISD::SRL, DL, MVT::i32, NewLD, in LowerLOAD()
8098 DAG.getConstant(I, DL, MVT::i32)); in LowerLOAD()
8100 Elts.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Elt)); in LowerLOAD()
8114 assert(Op.getValueType().getVectorElementType() == MVT::i32 && in LowerLOAD()
8244 SDValue Zero = DAG.getConstant(0, DL, MVT::i32); in LowerSELECT()
8245 SDValue One = DAG.getConstant(1, DL, MVT::i32); in LowerSELECT()
8247 SDValue LHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(1)); in LowerSELECT()
8248 SDValue RHS = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Op.getOperand(2)); in LowerSELECT()
8250 SDValue Lo0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, LHS, Zero); in LowerSELECT()
8251 SDValue Lo1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, RHS, Zero); in LowerSELECT()
8253 SDValue Lo = DAG.getSelect(DL, MVT::i32, Cond, Lo0, Lo1); in LowerSELECT()
8255 SDValue Hi0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, LHS, One); in LowerSELECT()
8256 SDValue Hi1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, RHS, One); in LowerSELECT()
8258 SDValue Hi = DAG.getSelect(DL, MVT::i32, Cond, Hi0, Hi1); in LowerSELECT()
8260 SDValue Res = DAG.getBuildVector(MVT::v2i32, DL, {Lo, Hi}); in LowerSELECT()
8351 SDVTList VTList = DAG.getVTList(VT, MVT::Other, MVT::Glue); in getFPBinOp()
8373 SDVTList VTList = DAG.getVTList(VT, MVT::Other, MVT::Glue); in getFPTernOp()
8394 SDValue CvtSrc0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0); in LowerFDIV16()
8395 SDValue CvtSrc1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1); in LowerFDIV16()
8397 SDValue RcpSrc1 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, CvtSrc1); in LowerFDIV16()
8398 SDValue Quot = DAG.getNode(ISD::FMUL, SL, MVT::f32, CvtSrc0, RcpSrc1); in LowerFDIV16()
8400 SDValue FPRoundFlag = DAG.getTargetConstant(0, SL, MVT::i32); in LowerFDIV16()
8401 SDValue BestQuot = DAG.getNode(ISD::FP_ROUND, SL, MVT::f16, Quot, FPRoundFlag); in LowerFDIV16()
8403 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f16, BestQuot, Src1, Src0); in LowerFDIV16()
8412 SDValue r1 = DAG.getNode(ISD::FABS, SL, MVT::f32, RHS); in lowerFDIV_FAST()
8415 const SDValue K0 = DAG.getConstantFP(K0Val, SL, MVT::f32); in lowerFDIV_FAST()
8418 const SDValue K1 = DAG.getConstantFP(K1Val, SL, MVT::f32); in lowerFDIV_FAST()
8420 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32); in lowerFDIV_FAST()
8423 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f32); in lowerFDIV_FAST()
8427 SDValue r3 = DAG.getNode(ISD::SELECT, SL, MVT::f32, r2, K1, One); in lowerFDIV_FAST()
8430 r1 = DAG.getNode(ISD::FMUL, SL, MVT::f32, RHS, r3); in lowerFDIV_FAST()
8433 SDValue r0 = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, r1); in lowerFDIV_FAST()
8435 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f32, LHS, r0); in lowerFDIV_FAST()
8437 return DAG.getNode(ISD::FMUL, SL, MVT::f32, r3, Mul); in lowerFDIV_FAST()
8450 return DAG.getTargetConstant(Mode, SL, MVT::i32); in getSPDenormModeValue()
8468 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32); in LowerFDIV32()
8470 SDVTList ScaleVT = DAG.getVTList(MVT::f32, MVT::i1); in LowerFDIV32()
8478 SDValue ApproxRcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, in LowerFDIV32()
8480 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f32, in LowerFDIV32()
8486 const SDValue BitField = DAG.getTargetConstant(Denorm32Reg, SL, MVT::i32); in LowerFDIV32()
8495 SDVTList BindParamVTs = DAG.getVTList(MVT::Other, MVT::Glue); in LowerFDIV32()
8506 SL, MVT::i32); in LowerFDIV32()
8521 SDValue Fma0 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, in LowerFDIV32()
8524 SDValue Fma1 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, Fma0, ApproxRcp, in LowerFDIV32()
8527 SDValue Mul = getFPBinOp(DAG, ISD::FMUL, SL, MVT::f32, NumeratorScaled, in LowerFDIV32()
8530 SDValue Fma2 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Mul, in LowerFDIV32()
8533 SDValue Fma3 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, in LowerFDIV32()
8536 SDValue Fma4 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Fma3, in LowerFDIV32()
8545 DisableDenorm = DAG.getNode(AMDGPUISD::DENORM_MODE, SL, MVT::Other, in LowerFDIV32()
8550 DAG.getConstant(FP_DENORM_FLUSH_IN_FLUSH_OUT, SL, MVT::i32); in LowerFDIV32()
8553 AMDGPU::S_SETREG_B32, SL, MVT::Other, in LowerFDIV32()
8557 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, in LowerFDIV32()
8563 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f32, in LowerFDIV32()
8566 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f32, Fmas, RHS, LHS, Flags); in LowerFDIV32()
8577 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64); in LowerFDIV64()
8579 SDVTList ScaleVT = DAG.getVTList(MVT::f64, MVT::i1); in LowerFDIV64()
8583 SDValue NegDivScale0 = DAG.getNode(ISD::FNEG, SL, MVT::f64, DivScale0); in LowerFDIV64()
8585 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, SL, MVT::f64, DivScale0); in LowerFDIV64()
8587 SDValue Fma0 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Rcp, One); in LowerFDIV64()
8589 SDValue Fma1 = DAG.getNode(ISD::FMA, SL, MVT::f64, Rcp, Fma0, Rcp); in LowerFDIV64()
8591 SDValue Fma2 = DAG.getNode(ISD::FMA, SL, MVT::f64, NegDivScale0, Fma1, One); in LowerFDIV64()
8595 SDValue Fma3 = DAG.getNode(ISD::FMA, SL, MVT::f64, Fma1, Fma2, Fma1); in LowerFDIV64()
8596 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, DivScale1, Fma3); in LowerFDIV64()
8598 SDValue Fma4 = DAG.getNode(ISD::FMA, SL, MVT::f64, in LowerFDIV64()
8607 const SDValue Hi = DAG.getConstant(1, SL, MVT::i32); in LowerFDIV64()
8610 SDValue NumBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X); in LowerFDIV64()
8611 SDValue DenBC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Y); in LowerFDIV64()
8612 SDValue Scale0BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale0); in LowerFDIV64()
8613 SDValue Scale1BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, DivScale1); in LowerFDIV64()
8615 SDValue NumHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, NumBC, Hi); in LowerFDIV64()
8616 SDValue DenHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, DenBC, Hi); in LowerFDIV64()
8619 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale0BC, Hi); in LowerFDIV64()
8621 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Scale1BC, Hi); in LowerFDIV64()
8623 SDValue CmpDen = DAG.getSetCC(SL, MVT::i1, DenHi, Scale0Hi, ISD::SETEQ); in LowerFDIV64()
8624 SDValue CmpNum = DAG.getSetCC(SL, MVT::i1, NumHi, Scale1Hi, ISD::SETEQ); in LowerFDIV64()
8625 Scale = DAG.getNode(ISD::XOR, SL, MVT::i1, CmpNum, CmpDen); in LowerFDIV64()
8630 SDValue Fmas = DAG.getNode(AMDGPUISD::DIV_FMAS, SL, MVT::f64, in LowerFDIV64()
8633 return DAG.getNode(AMDGPUISD::DIV_FIXUP, SL, MVT::f64, Fmas, Y, X); in LowerFDIV64()
8639 if (VT == MVT::f32) in LowerFDIV()
8642 if (VT == MVT::f64) in LowerFDIV()
8645 if (VT == MVT::f16) in LowerFDIV()
8656 if (VT == MVT::i1) { in LowerSTORE()
8658 DAG.getSExtOrTrunc(Store->getValue(), DL, MVT::i32), in LowerSTORE()
8659 Store->getBasePtr(), MVT::i1, Store->getMemOperand()); in LowerSTORE()
8663 Store->getValue().getValueType().getScalarType() == MVT::i32); in LowerSTORE()
8793 MVT SimpleVT = VT.getSimpleVT(); in LowerATOMIC_CMP_SWAP()
8794 MVT VecType = MVT::getVectorVT(SimpleVT, 2); in LowerATOMIC_CMP_SWAP()
8811 if (ScalarVT != MVT::f32 && ScalarVT != MVT::f16) in performUCharToFloatCombine()
8824 if (DCI.isAfterLegalizeDAG() && SrcVT == MVT::i32) { in performUCharToFloatCombine()
8826 SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0, DL, MVT::f32, Src); in performUCharToFloatCombine()
8830 if (ScalarVT != MVT::f32) { in performUCharToFloatCombine()
8832 DAG.getTargetConstant(0, DL, MVT::i32)); in performUCharToFloatCombine()
8975 if (V.getValueType() != MVT::i1) in isBoolSGPR()
9068 if (VT == MVT::i64 && CRHS) { in performAndCombine()
9074 if (CRHS && VT == MVT::i32) { in performAndCombine()
9089 SDValue BFE = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32, in performAndCombine()
9091 DAG.getConstant(Offset, SL, MVT::i32), in performAndCombine()
9092 DAG.getConstant(Bits, SL, MVT::i32)); in performAndCombine()
9097 DAG.getConstant(NB, SDLoc(CRHS), MVT::i32)); in performAndCombine()
9113 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, LHS.getOperand(0), in performAndCombine()
9114 LHS.getOperand(1), DAG.getConstant(Sel, DL, MVT::i32)); in performAndCombine()
9152 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1, in performAndCombine()
9153 X, DAG.getConstant(Mask, DL, MVT::i32)); in performAndCombine()
9176 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1, RHS.getOperand(0), in performAndCombine()
9177 DAG.getConstant(NewMask, DL, MVT::i32)); in performAndCombine()
9181 if (VT == MVT::i32 && in performAndCombine()
9187 return DAG.getSelect(SDLoc(N), MVT::i32, RHS.getOperand(0), in performAndCombine()
9188 LHS, DAG.getConstant(0, SDLoc(N), MVT::i32)); in performAndCombine()
9193 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() && in performAndCombine()
9232 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, in performAndCombine()
9234 DAG.getConstant(Sel, DL, MVT::i32)); in performAndCombine()
9249 if (VT == MVT::i1) { in performOrCombine()
9267 return DAG.getNode(AMDGPUISD::FP_CLASS, DL, MVT::i1, in performOrCombine()
9268 Src, DAG.getConstant(NewMask, DL, MVT::i32)); in performOrCombine()
9284 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, LHS.getOperand(0), in performOrCombine()
9285 LHS.getOperand(1), DAG.getConstant(Sel, DL, MVT::i32)); in performOrCombine()
9290 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() && in performOrCombine()
9321 return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, in performOrCombine()
9323 DAG.getConstant(Sel, DL, MVT::i32)); in performOrCombine()
9328 if (VT != MVT::i64 || DCI.isBeforeLegalizeOps()) in performOrCombine()
9343 if (SrcVT == MVT::i32) { in performOrCombine()
9347 SDValue LowOr = DAG.getNode(ISD::OR, SL, MVT::i32, LowLHS, ExtSrc); in performOrCombine()
9352 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in performOrCombine()
9354 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in performOrCombine()
9371 if (VT != MVT::i64) in performXorCombine()
9394 if (VT != MVT::i32) in performZeroExtendCombine()
9398 if (Src.getValueType() != MVT::i16) in performZeroExtendCombine()
9411 VTSign->getVT() == MVT::i8) || in performSignExtendInRegCombine()
9413 VTSign->getVT() == MVT::i16)) && in performSignExtendInRegCombine()
9427 SDVTList ResList = DCI.DAG.getVTList(MVT::i32, in performSignExtendInRegCombine()
9449 return DAG.getConstant(0, SDLoc(N), MVT::i1); in performClassCombine()
9453 return DAG.getUNDEF(MVT::i1); in performClassCombine()
9466 if (VT == MVT::f32 && (N0.getOpcode() == ISD::UINT_TO_FP || in performRcpCombine()
9472 if ((VT == MVT::f32 || VT == MVT::f16) && N0.getOpcode() == ISD::FSQRT) { in performRcpCombine()
9541 return Op.getValueType().getScalarType() != MVT::f16; in isCanonicalized()
9601 if (Op.getValueType() == MVT::i16) { in isCanonicalized()
9603 if (TruncSrc.getValueType() == MVT::i32 && in isCanonicalized()
9605 TruncSrc.getOperand(0).getValueType() == MVT::v2f16) { in isCanonicalized()
9736 if (N0.getOpcode() == ISD::BUILD_VECTOR && VT == MVT::v2f16 && in performFCanonicalizeCombine()
9737 isTypeLegal(MVT::v2f16)) { in performFCanonicalizeCombine()
9841 if (VT == MVT::i32 || (VT == MVT::i16 && Subtarget->hasMed3_16())) { in performIntMed3ImmCombine()
9847 if (VT == MVT::i16) { in performIntMed3ImmCombine()
9848 MVT NVT = MVT::i32; in performIntMed3ImmCombine()
9904 if (VT == MVT::f32 || (VT == MVT::f16 && Subtarget->hasMed3_16())) { in performFPMed3ImmCombine()
9941 (VT == MVT::i32 || VT == MVT::f32 || in performMinMaxCombine()
9942 ((VT == MVT::f16 || VT == MVT::i16) && Subtarget->hasMin3Max3_16()))) { in performMinMaxCombine()
9985 (VT == MVT::f32 || VT == MVT::f64 || in performMinMaxCombine()
9986 (VT == MVT::f16 && Subtarget->has16BitInsts()) || in performMinMaxCombine()
9987 (VT == MVT::v2f16 && Subtarget->hasVOP3PInsts())) && in performMinMaxCombine()
10205 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Cast, in performExtractVectorEltCombine()
10206 DAG.getConstant(EltIdx, SL, MVT::i32)); in performExtractVectorEltCombine()
10208 SDValue Srl = DAG.getNode(ISD::SRL, SL, MVT::i32, Elt, in performExtractVectorEltCombine()
10209 DAG.getConstant(LeftoverBitIdx, SL, MVT::i32)); in performExtractVectorEltCombine()
10256 if (((VT == MVT::f32 && !hasFP32Denormals(DAG.getMachineFunction())) || in getFusedOpcode()
10257 (VT == MVT::f16 && !hasFP64FP16Denormals(DAG.getMachineFunction()) && in getFusedOpcode()
10278 if (VT != MVT::i32 && VT != MVT::i64) in reassociateScalarOps()
10318 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i1); in getMad64_32()
10345 MulLHS = DAG.getZExtOrTrunc(MulLHS, SL, MVT::i32); in performAddCombine()
10346 MulRHS = DAG.getZExtOrTrunc(MulRHS, SL, MVT::i32); in performAddCombine()
10347 AddRHS = DAG.getZExtOrTrunc(AddRHS, SL, MVT::i64); in performAddCombine()
10352 MulLHS = DAG.getSExtOrTrunc(MulLHS, SL, MVT::i32); in performAddCombine()
10353 MulRHS = DAG.getSExtOrTrunc(MulRHS, SL, MVT::i32); in performAddCombine()
10354 AddRHS = DAG.getSExtOrTrunc(AddRHS, SL, MVT::i64); in performAddCombine()
10365 if (VT != MVT::i32 || !DCI.isAfterLegalizeDAG()) in performAddCombine()
10386 SDVTList VTList = DAG.getVTList(MVT::i32, MVT::i1); in performAddCombine()
10387 SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond }; in performAddCombine()
10407 if (VT != MVT::i32) in performSubCombine()
10427 SDVTList VTList = DAG.getVTList(MVT::i32, MVT::i1); in performSubCombine()
10428 SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond }; in performSubCombine()
10448 if (N->getValueType(0) != MVT::i32) in performAddCarrySubCarryCombine()
10565 if (!Subtarget->hasDot7Insts() || VT != MVT::f32) in performFMACombine()
10622 if (Vec1.getValueType() != MVT::v2f16 || Vec2.getValueType() != MVT::v2f16) in performFMACombine()
10627 return DAG.getNode(AMDGPUISD::FDOT2, SL, MVT::f32, Vec1, Vec2, FMAAcc, in performFMACombine()
10628 DAG.getTargetConstant(0, SL, MVT::i1)); in performFMACombine()
10654 if (VT == MVT::i32 && LHS.getOpcode() == ISD::SIGN_EXTEND && in performSetCCCombine()
10664 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
10665 DAG.getConstant(-1, SL, MVT::i1)); in performSetCCCombine()
10690 return DAG.getNode(ISD::XOR, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
10691 DAG.getConstant(-1, SL, MVT::i1)); in performSetCCCombine()
10698 if (VT != MVT::f32 && VT != MVT::f64 && (Subtarget->has16BitInsts() && in performSetCCCombine()
10699 VT != MVT::f16)) in performSetCCCombine()
10722 return DAG.getNode(AMDGPUISD::FP_CLASS, SL, MVT::i1, LHS.getOperand(0), in performSetCCCombine()
10723 DAG.getConstant(Mask, SL, MVT::i32)); in performSetCCCombine()
10751 SDLoc(Shift.getOperand(0)), MVT::i32); in performCvtF32UByteNCombine()
10761 MVT::f32, Shift); in performCvtF32UByteNCombine()
10779 return DAG.getNode(N->getOpcode(), SL, MVT::f32, DemandedSrc); in performCvtF32UByteNCombine()
10884 if (VT == MVT::v2i16 || VT == MVT::v2f16) { in PerformDAGCombine()
10888 if (EltVT == MVT::f16) in PerformDAGCombine()
10889 Src = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Src); in PerformDAGCombine()
10891 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Src); in PerformDAGCombine()
11039 Ops.push_back(DAG.getTargetConstant(NewDmask, SDLoc(Node), MVT::i32)); in adjustWritemask()
11042 MVT SVT = Node->getValueType(0).getVectorElementType().getSimpleVT(); in adjustWritemask()
11044 MVT ResultVT = NewChannels == 1 ? in adjustWritemask()
11045 SVT : MVT::getVectorVT(SVT, NewChannels == 3 ? 4 : in adjustWritemask()
11048 DAG.getVTList(ResultVT, MVT::Other) : DAG.getVTList(ResultVT); in adjustWritemask()
11078 SDValue Op = DAG.getTargetConstant(Idx, SDLoc(User), MVT::i32); in adjustWritemask()
11113 if (SrcVal.getValueType() == MVT::i1 && DestReg->getReg().isPhysical()) { in legalizeTargetIndependentNode()
11117 MRI.createVirtualRegister(&AMDGPU::VReg_1RegClass), MVT::i1); in legalizeTargetIndependentNode()
11182 MVT VT = Src0.getValueType().getSimpleVT(); in PostISelFolding()
11403 SDValue K = DAG.getTargetConstant(Val, DL, MVT::i32); in buildSMovImm32()
11404 return SDValue(DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, K), 0); in buildSMovImm32()
11416 DAG.getTargetConstant(AMDGPU::SGPR_64RegClassID, DL, MVT::i32), in wrapAddr64Rsrc()
11418 DAG.getTargetConstant(AMDGPU::sub0, DL, MVT::i32), in wrapAddr64Rsrc()
11420 DAG.getTargetConstant(AMDGPU::sub1, DL, MVT::i32) in wrapAddr64Rsrc()
11424 MVT::v2i32, Ops0), 0); in wrapAddr64Rsrc()
11428 DAG.getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32), in wrapAddr64Rsrc()
11430 DAG.getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32), in wrapAddr64Rsrc()
11432 DAG.getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32) in wrapAddr64Rsrc()
11435 return DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL, MVT::v4i32, Ops1); in wrapAddr64Rsrc()
11445 SDValue PtrLo = DAG.getTargetExtractSubreg(AMDGPU::sub0, DL, MVT::i32, Ptr); in buildRSRC()
11446 SDValue PtrHi = DAG.getTargetExtractSubreg(AMDGPU::sub1, DL, MVT::i32, Ptr); in buildRSRC()
11448 PtrHi = SDValue(DAG.getMachineNode(AMDGPU::S_OR_B32, DL, MVT::i32, PtrHi, in buildRSRC()
11449 DAG.getConstant(RsrcDword1, DL, MVT::i32)), in buildRSRC()
11458 DAG.getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32), in buildRSRC()
11460 DAG.getTargetConstant(AMDGPU::sub0, DL, MVT::i32), in buildRSRC()
11462 DAG.getTargetConstant(AMDGPU::sub1, DL, MVT::i32), in buildRSRC()
11464 DAG.getTargetConstant(AMDGPU::sub2, DL, MVT::i32), in buildRSRC()
11466 DAG.getTargetConstant(AMDGPU::sub3, DL, MVT::i32) in buildRSRC()
11469 return DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL, MVT::v4i32, Ops); in buildRSRC()
11479 MVT VT) const { in getRegForInlineAsmConstraint()
11533 if (RC && (isTypeLegal(VT) || VT.SimpleTy == MVT::i128 || in getRegForInlineAsmConstraint()
11534 VT.SimpleTy == MVT::i16 || VT.SimpleTy == MVT::f16)) in getRegForInlineAsmConstraint()
11610 Ops.push_back(DAG.getTargetConstant(Val, SDLoc(Op), MVT::i64)); in LowerAsmOperandForConstraint()
12037 case MVT::f32: in denormalsEnabledForType()
12039 case MVT::f64: in denormalsEnabledForType()
12040 case MVT::f16: in denormalsEnabledForType()
12158 SITargetLowering::getRegClassFor(MVT VT, bool isDivergent) const { in getRegClassFor()
12258 std::pair<InstructionCost, MVT>
12261 std::pair<InstructionCost, MVT> Cost = in getTypeLegalizationCost()