Lines Matching refs:MVT
45 return EVT::getVectorVT(Ctx, MVT::i32, StoreSize / 32); in getEquivalentMemType()
63 setOperationAction(ISD::LOAD, MVT::f32, Promote); in AMDGPUTargetLowering()
64 AddPromotedToType(ISD::LOAD, MVT::f32, MVT::i32); in AMDGPUTargetLowering()
66 setOperationAction(ISD::LOAD, MVT::v2f32, Promote); in AMDGPUTargetLowering()
67 AddPromotedToType(ISD::LOAD, MVT::v2f32, MVT::v2i32); in AMDGPUTargetLowering()
69 setOperationAction(ISD::LOAD, MVT::v3f32, Promote); in AMDGPUTargetLowering()
70 AddPromotedToType(ISD::LOAD, MVT::v3f32, MVT::v3i32); in AMDGPUTargetLowering()
72 setOperationAction(ISD::LOAD, MVT::v4f32, Promote); in AMDGPUTargetLowering()
73 AddPromotedToType(ISD::LOAD, MVT::v4f32, MVT::v4i32); in AMDGPUTargetLowering()
75 setOperationAction(ISD::LOAD, MVT::v5f32, Promote); in AMDGPUTargetLowering()
76 AddPromotedToType(ISD::LOAD, MVT::v5f32, MVT::v5i32); in AMDGPUTargetLowering()
78 setOperationAction(ISD::LOAD, MVT::v6f32, Promote); in AMDGPUTargetLowering()
79 AddPromotedToType(ISD::LOAD, MVT::v6f32, MVT::v6i32); in AMDGPUTargetLowering()
81 setOperationAction(ISD::LOAD, MVT::v7f32, Promote); in AMDGPUTargetLowering()
82 AddPromotedToType(ISD::LOAD, MVT::v7f32, MVT::v7i32); in AMDGPUTargetLowering()
84 setOperationAction(ISD::LOAD, MVT::v8f32, Promote); in AMDGPUTargetLowering()
85 AddPromotedToType(ISD::LOAD, MVT::v8f32, MVT::v8i32); in AMDGPUTargetLowering()
87 setOperationAction(ISD::LOAD, MVT::v16f32, Promote); in AMDGPUTargetLowering()
88 AddPromotedToType(ISD::LOAD, MVT::v16f32, MVT::v16i32); in AMDGPUTargetLowering()
90 setOperationAction(ISD::LOAD, MVT::v32f32, Promote); in AMDGPUTargetLowering()
91 AddPromotedToType(ISD::LOAD, MVT::v32f32, MVT::v32i32); in AMDGPUTargetLowering()
93 setOperationAction(ISD::LOAD, MVT::i64, Promote); in AMDGPUTargetLowering()
94 AddPromotedToType(ISD::LOAD, MVT::i64, MVT::v2i32); in AMDGPUTargetLowering()
96 setOperationAction(ISD::LOAD, MVT::v2i64, Promote); in AMDGPUTargetLowering()
97 AddPromotedToType(ISD::LOAD, MVT::v2i64, MVT::v4i32); in AMDGPUTargetLowering()
99 setOperationAction(ISD::LOAD, MVT::f64, Promote); in AMDGPUTargetLowering()
100 AddPromotedToType(ISD::LOAD, MVT::f64, MVT::v2i32); in AMDGPUTargetLowering()
102 setOperationAction(ISD::LOAD, MVT::v2f64, Promote); in AMDGPUTargetLowering()
103 AddPromotedToType(ISD::LOAD, MVT::v2f64, MVT::v4i32); in AMDGPUTargetLowering()
105 setOperationAction(ISD::LOAD, MVT::v3i64, Promote); in AMDGPUTargetLowering()
106 AddPromotedToType(ISD::LOAD, MVT::v3i64, MVT::v6i32); in AMDGPUTargetLowering()
108 setOperationAction(ISD::LOAD, MVT::v4i64, Promote); in AMDGPUTargetLowering()
109 AddPromotedToType(ISD::LOAD, MVT::v4i64, MVT::v8i32); in AMDGPUTargetLowering()
111 setOperationAction(ISD::LOAD, MVT::v3f64, Promote); in AMDGPUTargetLowering()
112 AddPromotedToType(ISD::LOAD, MVT::v3f64, MVT::v6i32); in AMDGPUTargetLowering()
114 setOperationAction(ISD::LOAD, MVT::v4f64, Promote); in AMDGPUTargetLowering()
115 AddPromotedToType(ISD::LOAD, MVT::v4f64, MVT::v8i32); in AMDGPUTargetLowering()
117 setOperationAction(ISD::LOAD, MVT::v8i64, Promote); in AMDGPUTargetLowering()
118 AddPromotedToType(ISD::LOAD, MVT::v8i64, MVT::v16i32); in AMDGPUTargetLowering()
120 setOperationAction(ISD::LOAD, MVT::v8f64, Promote); in AMDGPUTargetLowering()
121 AddPromotedToType(ISD::LOAD, MVT::v8f64, MVT::v16i32); in AMDGPUTargetLowering()
123 setOperationAction(ISD::LOAD, MVT::v16i64, Promote); in AMDGPUTargetLowering()
124 AddPromotedToType(ISD::LOAD, MVT::v16i64, MVT::v32i32); in AMDGPUTargetLowering()
126 setOperationAction(ISD::LOAD, MVT::v16f64, Promote); in AMDGPUTargetLowering()
127 AddPromotedToType(ISD::LOAD, MVT::v16f64, MVT::v32i32); in AMDGPUTargetLowering()
131 for (MVT VT : MVT::integer_valuetypes()) in AMDGPUTargetLowering()
132 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::i64, VT, in AMDGPUTargetLowering()
135 for (MVT VT : MVT::integer_valuetypes()) { in AMDGPUTargetLowering()
136 if (VT == MVT::i64) in AMDGPUTargetLowering()
140 setLoadExtAction(Op, VT, MVT::i1, Promote); in AMDGPUTargetLowering()
141 setLoadExtAction(Op, VT, MVT::i8, Legal); in AMDGPUTargetLowering()
142 setLoadExtAction(Op, VT, MVT::i16, Legal); in AMDGPUTargetLowering()
143 setLoadExtAction(Op, VT, MVT::i32, Expand); in AMDGPUTargetLowering()
147 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) in AMDGPUTargetLowering()
149 {MVT::v2i8, MVT::v4i8, MVT::v2i16, MVT::v3i16, MVT::v4i16}) in AMDGPUTargetLowering()
153 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand); in AMDGPUTargetLowering()
154 setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2f16, Expand); in AMDGPUTargetLowering()
155 setLoadExtAction(ISD::EXTLOAD, MVT::v3f32, MVT::v3f16, Expand); in AMDGPUTargetLowering()
156 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Expand); in AMDGPUTargetLowering()
157 setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Expand); in AMDGPUTargetLowering()
158 setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16f16, Expand); in AMDGPUTargetLowering()
159 setLoadExtAction(ISD::EXTLOAD, MVT::v32f32, MVT::v32f16, Expand); in AMDGPUTargetLowering()
161 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand); in AMDGPUTargetLowering()
162 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Expand); in AMDGPUTargetLowering()
163 setLoadExtAction(ISD::EXTLOAD, MVT::v3f64, MVT::v3f32, Expand); in AMDGPUTargetLowering()
164 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Expand); in AMDGPUTargetLowering()
165 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f32, Expand); in AMDGPUTargetLowering()
166 setLoadExtAction(ISD::EXTLOAD, MVT::v16f64, MVT::v16f32, Expand); in AMDGPUTargetLowering()
168 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand); in AMDGPUTargetLowering()
169 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Expand); in AMDGPUTargetLowering()
170 setLoadExtAction(ISD::EXTLOAD, MVT::v3f64, MVT::v3f16, Expand); in AMDGPUTargetLowering()
171 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Expand); in AMDGPUTargetLowering()
172 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Expand); in AMDGPUTargetLowering()
173 setLoadExtAction(ISD::EXTLOAD, MVT::v16f64, MVT::v16f16, Expand); in AMDGPUTargetLowering()
175 setOperationAction(ISD::STORE, MVT::f32, Promote); in AMDGPUTargetLowering()
176 AddPromotedToType(ISD::STORE, MVT::f32, MVT::i32); in AMDGPUTargetLowering()
178 setOperationAction(ISD::STORE, MVT::v2f32, Promote); in AMDGPUTargetLowering()
179 AddPromotedToType(ISD::STORE, MVT::v2f32, MVT::v2i32); in AMDGPUTargetLowering()
181 setOperationAction(ISD::STORE, MVT::v3f32, Promote); in AMDGPUTargetLowering()
182 AddPromotedToType(ISD::STORE, MVT::v3f32, MVT::v3i32); in AMDGPUTargetLowering()
184 setOperationAction(ISD::STORE, MVT::v4f32, Promote); in AMDGPUTargetLowering()
185 AddPromotedToType(ISD::STORE, MVT::v4f32, MVT::v4i32); in AMDGPUTargetLowering()
187 setOperationAction(ISD::STORE, MVT::v5f32, Promote); in AMDGPUTargetLowering()
188 AddPromotedToType(ISD::STORE, MVT::v5f32, MVT::v5i32); in AMDGPUTargetLowering()
190 setOperationAction(ISD::STORE, MVT::v6f32, Promote); in AMDGPUTargetLowering()
191 AddPromotedToType(ISD::STORE, MVT::v6f32, MVT::v6i32); in AMDGPUTargetLowering()
193 setOperationAction(ISD::STORE, MVT::v7f32, Promote); in AMDGPUTargetLowering()
194 AddPromotedToType(ISD::STORE, MVT::v7f32, MVT::v7i32); in AMDGPUTargetLowering()
196 setOperationAction(ISD::STORE, MVT::v8f32, Promote); in AMDGPUTargetLowering()
197 AddPromotedToType(ISD::STORE, MVT::v8f32, MVT::v8i32); in AMDGPUTargetLowering()
199 setOperationAction(ISD::STORE, MVT::v16f32, Promote); in AMDGPUTargetLowering()
200 AddPromotedToType(ISD::STORE, MVT::v16f32, MVT::v16i32); in AMDGPUTargetLowering()
202 setOperationAction(ISD::STORE, MVT::v32f32, Promote); in AMDGPUTargetLowering()
203 AddPromotedToType(ISD::STORE, MVT::v32f32, MVT::v32i32); in AMDGPUTargetLowering()
205 setOperationAction(ISD::STORE, MVT::i64, Promote); in AMDGPUTargetLowering()
206 AddPromotedToType(ISD::STORE, MVT::i64, MVT::v2i32); in AMDGPUTargetLowering()
208 setOperationAction(ISD::STORE, MVT::v2i64, Promote); in AMDGPUTargetLowering()
209 AddPromotedToType(ISD::STORE, MVT::v2i64, MVT::v4i32); in AMDGPUTargetLowering()
211 setOperationAction(ISD::STORE, MVT::f64, Promote); in AMDGPUTargetLowering()
212 AddPromotedToType(ISD::STORE, MVT::f64, MVT::v2i32); in AMDGPUTargetLowering()
214 setOperationAction(ISD::STORE, MVT::v2f64, Promote); in AMDGPUTargetLowering()
215 AddPromotedToType(ISD::STORE, MVT::v2f64, MVT::v4i32); in AMDGPUTargetLowering()
217 setOperationAction(ISD::STORE, MVT::v3i64, Promote); in AMDGPUTargetLowering()
218 AddPromotedToType(ISD::STORE, MVT::v3i64, MVT::v6i32); in AMDGPUTargetLowering()
220 setOperationAction(ISD::STORE, MVT::v3f64, Promote); in AMDGPUTargetLowering()
221 AddPromotedToType(ISD::STORE, MVT::v3f64, MVT::v6i32); in AMDGPUTargetLowering()
223 setOperationAction(ISD::STORE, MVT::v4i64, Promote); in AMDGPUTargetLowering()
224 AddPromotedToType(ISD::STORE, MVT::v4i64, MVT::v8i32); in AMDGPUTargetLowering()
226 setOperationAction(ISD::STORE, MVT::v4f64, Promote); in AMDGPUTargetLowering()
227 AddPromotedToType(ISD::STORE, MVT::v4f64, MVT::v8i32); in AMDGPUTargetLowering()
229 setOperationAction(ISD::STORE, MVT::v8i64, Promote); in AMDGPUTargetLowering()
230 AddPromotedToType(ISD::STORE, MVT::v8i64, MVT::v16i32); in AMDGPUTargetLowering()
232 setOperationAction(ISD::STORE, MVT::v8f64, Promote); in AMDGPUTargetLowering()
233 AddPromotedToType(ISD::STORE, MVT::v8f64, MVT::v16i32); in AMDGPUTargetLowering()
235 setOperationAction(ISD::STORE, MVT::v16i64, Promote); in AMDGPUTargetLowering()
236 AddPromotedToType(ISD::STORE, MVT::v16i64, MVT::v32i32); in AMDGPUTargetLowering()
238 setOperationAction(ISD::STORE, MVT::v16f64, Promote); in AMDGPUTargetLowering()
239 AddPromotedToType(ISD::STORE, MVT::v16f64, MVT::v32i32); in AMDGPUTargetLowering()
241 setTruncStoreAction(MVT::i64, MVT::i1, Expand); in AMDGPUTargetLowering()
242 setTruncStoreAction(MVT::i64, MVT::i8, Expand); in AMDGPUTargetLowering()
243 setTruncStoreAction(MVT::i64, MVT::i16, Expand); in AMDGPUTargetLowering()
244 setTruncStoreAction(MVT::i64, MVT::i32, Expand); in AMDGPUTargetLowering()
246 setTruncStoreAction(MVT::v2i64, MVT::v2i1, Expand); in AMDGPUTargetLowering()
247 setTruncStoreAction(MVT::v2i64, MVT::v2i8, Expand); in AMDGPUTargetLowering()
248 setTruncStoreAction(MVT::v2i64, MVT::v2i16, Expand); in AMDGPUTargetLowering()
249 setTruncStoreAction(MVT::v2i64, MVT::v2i32, Expand); in AMDGPUTargetLowering()
251 setTruncStoreAction(MVT::f32, MVT::f16, Expand); in AMDGPUTargetLowering()
252 setTruncStoreAction(MVT::v2f32, MVT::v2f16, Expand); in AMDGPUTargetLowering()
253 setTruncStoreAction(MVT::v3f32, MVT::v3f16, Expand); in AMDGPUTargetLowering()
254 setTruncStoreAction(MVT::v4f32, MVT::v4f16, Expand); in AMDGPUTargetLowering()
255 setTruncStoreAction(MVT::v8f32, MVT::v8f16, Expand); in AMDGPUTargetLowering()
256 setTruncStoreAction(MVT::v16f32, MVT::v16f16, Expand); in AMDGPUTargetLowering()
257 setTruncStoreAction(MVT::v32f32, MVT::v32f16, Expand); in AMDGPUTargetLowering()
259 setTruncStoreAction(MVT::f64, MVT::f16, Expand); in AMDGPUTargetLowering()
260 setTruncStoreAction(MVT::f64, MVT::f32, Expand); in AMDGPUTargetLowering()
262 setTruncStoreAction(MVT::v2f64, MVT::v2f32, Expand); in AMDGPUTargetLowering()
263 setTruncStoreAction(MVT::v2f64, MVT::v2f16, Expand); in AMDGPUTargetLowering()
265 setTruncStoreAction(MVT::v3i64, MVT::v3i32, Expand); in AMDGPUTargetLowering()
266 setTruncStoreAction(MVT::v3i64, MVT::v3i16, Expand); in AMDGPUTargetLowering()
267 setTruncStoreAction(MVT::v3f64, MVT::v3f32, Expand); in AMDGPUTargetLowering()
268 setTruncStoreAction(MVT::v3f64, MVT::v3f16, Expand); in AMDGPUTargetLowering()
270 setTruncStoreAction(MVT::v4i64, MVT::v4i32, Expand); in AMDGPUTargetLowering()
271 setTruncStoreAction(MVT::v4i64, MVT::v4i16, Expand); in AMDGPUTargetLowering()
272 setTruncStoreAction(MVT::v4f64, MVT::v4f32, Expand); in AMDGPUTargetLowering()
273 setTruncStoreAction(MVT::v4f64, MVT::v4f16, Expand); in AMDGPUTargetLowering()
275 setTruncStoreAction(MVT::v8f64, MVT::v8f32, Expand); in AMDGPUTargetLowering()
276 setTruncStoreAction(MVT::v8f64, MVT::v8f16, Expand); in AMDGPUTargetLowering()
278 setTruncStoreAction(MVT::v16f64, MVT::v16f32, Expand); in AMDGPUTargetLowering()
279 setTruncStoreAction(MVT::v16f64, MVT::v16f16, Expand); in AMDGPUTargetLowering()
280 setTruncStoreAction(MVT::v16i64, MVT::v16i16, Expand); in AMDGPUTargetLowering()
281 setTruncStoreAction(MVT::v16i64, MVT::v16i16, Expand); in AMDGPUTargetLowering()
282 setTruncStoreAction(MVT::v16i64, MVT::v16i8, Expand); in AMDGPUTargetLowering()
283 setTruncStoreAction(MVT::v16i64, MVT::v16i8, Expand); in AMDGPUTargetLowering()
284 setTruncStoreAction(MVT::v16i64, MVT::v16i1, Expand); in AMDGPUTargetLowering()
286 setOperationAction(ISD::Constant, {MVT::i32, MVT::i64}, Legal); in AMDGPUTargetLowering()
287 setOperationAction(ISD::ConstantFP, {MVT::f32, MVT::f64}, Legal); in AMDGPUTargetLowering()
289 setOperationAction({ISD::BR_JT, ISD::BRIND}, MVT::Other, Expand); in AMDGPUTargetLowering()
292 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); in AMDGPUTargetLowering()
299 MVT::f32, Legal); in AMDGPUTargetLowering()
301 setOperationAction(ISD::FROUND, {MVT::f32, MVT::f64}, Custom); in AMDGPUTargetLowering()
303 setOperationAction({ISD::FLOG, ISD::FLOG10, ISD::FEXP}, MVT::f32, Custom); in AMDGPUTargetLowering()
305 setOperationAction(ISD::FNEARBYINT, {MVT::f32, MVT::f64}, Custom); in AMDGPUTargetLowering()
307 setOperationAction(ISD::FREM, {MVT::f16, MVT::f32, MVT::f64}, Custom); in AMDGPUTargetLowering()
310 setOperationAction(ISD::FSUB, MVT::f64, Expand); in AMDGPUTargetLowering()
313 {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32, in AMDGPUTargetLowering()
314 MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32, in AMDGPUTargetLowering()
315 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32}, in AMDGPUTargetLowering()
319 {MVT::v2f16, MVT::v2i16, MVT::v4f16, MVT::v4i16, MVT::v2f32, in AMDGPUTargetLowering()
320 MVT::v2i32, MVT::v3f32, MVT::v3i32, MVT::v4f32, MVT::v4i32, in AMDGPUTargetLowering()
321 MVT::v5f32, MVT::v5i32, MVT::v6f32, MVT::v6i32, MVT::v7f32, in AMDGPUTargetLowering()
322 MVT::v7i32, MVT::v8f32, MVT::v8i32, MVT::v16f16, MVT::v16i16, in AMDGPUTargetLowering()
323 MVT::v16f32, MVT::v16i32, MVT::v32f32, MVT::v32i32, MVT::v2f64, in AMDGPUTargetLowering()
324 MVT::v2i64, MVT::v3f64, MVT::v3i64, MVT::v4f64, MVT::v4i64, in AMDGPUTargetLowering()
325 MVT::v8f64, MVT::v8i64, MVT::v16f64, MVT::v16i64}, in AMDGPUTargetLowering()
328 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); in AMDGPUTargetLowering()
329 setOperationAction(ISD::FP_TO_FP16, {MVT::f64, MVT::f32}, Custom); in AMDGPUTargetLowering()
331 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 }; in AMDGPUTargetLowering()
332 for (MVT VT : ScalarIntVTs) { in AMDGPUTargetLowering()
350 setOperationAction(ISD::FSHR, MVT::i32, Legal); in AMDGPUTargetLowering()
353 setOperationAction(ISD::ROTL, {MVT::i32, MVT::i64}, Expand); in AMDGPUTargetLowering()
354 setOperationAction(ISD::ROTR, MVT::i64, Expand); in AMDGPUTargetLowering()
356 setOperationAction({ISD::MULHU, ISD::MULHS}, MVT::i16, Expand); in AMDGPUTargetLowering()
358 setOperationAction({ISD::MUL, ISD::MULHU, ISD::MULHS}, MVT::i64, Expand); in AMDGPUTargetLowering()
361 MVT::i64, Custom); in AMDGPUTargetLowering()
362 setOperationAction(ISD::SELECT_CC, MVT::i64, Expand); in AMDGPUTargetLowering()
364 setOperationAction({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, MVT::i32, in AMDGPUTargetLowering()
369 MVT::i64, Custom); in AMDGPUTargetLowering()
371 static const MVT::SimpleValueType VectorIntTypes[] = { in AMDGPUTargetLowering()
372 MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, MVT::v6i32, MVT::v7i32}; in AMDGPUTargetLowering()
374 for (MVT VT : VectorIntTypes) { in AMDGPUTargetLowering()
391 static const MVT::SimpleValueType FloatVectorTypes[] = { in AMDGPUTargetLowering()
392 MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32, MVT::v6f32, MVT::v7f32}; in AMDGPUTargetLowering()
394 for (MVT VT : FloatVectorTypes) { in AMDGPUTargetLowering()
410 setOperationAction(ISD::SELECT, MVT::v2f32, Promote); in AMDGPUTargetLowering()
411 AddPromotedToType(ISD::SELECT, MVT::v2f32, MVT::v2i32); in AMDGPUTargetLowering()
413 setOperationAction(ISD::SELECT, MVT::v3f32, Promote); in AMDGPUTargetLowering()
414 AddPromotedToType(ISD::SELECT, MVT::v3f32, MVT::v3i32); in AMDGPUTargetLowering()
416 setOperationAction(ISD::SELECT, MVT::v4f32, Promote); in AMDGPUTargetLowering()
417 AddPromotedToType(ISD::SELECT, MVT::v4f32, MVT::v4i32); in AMDGPUTargetLowering()
419 setOperationAction(ISD::SELECT, MVT::v5f32, Promote); in AMDGPUTargetLowering()
420 AddPromotedToType(ISD::SELECT, MVT::v5f32, MVT::v5i32); in AMDGPUTargetLowering()
422 setOperationAction(ISD::SELECT, MVT::v6f32, Promote); in AMDGPUTargetLowering()
423 AddPromotedToType(ISD::SELECT, MVT::v6f32, MVT::v6i32); in AMDGPUTargetLowering()
425 setOperationAction(ISD::SELECT, MVT::v7f32, Promote); in AMDGPUTargetLowering()
426 AddPromotedToType(ISD::SELECT, MVT::v7f32, MVT::v7i32); in AMDGPUTargetLowering()
531 static bool opMustUseVOP3Encoding(const SDNode *N, MVT VT) { in opMustUseVOP3Encoding()
532 return N->getNumOperands() > 2 || VT == MVT::f64; in opMustUseVOP3Encoding()
582 MVT VT = N->getValueType(0).getScalarType().getSimpleVT(); in allUsesHaveSourceMods()
605 return MVT::i32; in getTypeForExtReturn()
609 MVT AMDGPUTargetLowering::getVectorIdxTy(const DataLayout &) const { in getVectorIdxTy()
610 return MVT::i32; in getVectorIdxTy()
622 return (ScalarVT == MVT::f32 || ScalarVT == MVT::f64 || in isFPImmLegal()
623 (ScalarVT == MVT::f16 && Subtarget->has16BitInsts())); in isFPImmLegal()
629 return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64); in ShouldShrinkFPConstant()
677 if (LoadTy.getScalarType() == MVT::i32) in isLoadBitCastBeneficial()
757 return VT == MVT::f32 || VT == MVT::f64 || in isFAbsFree()
758 (Subtarget->has16BitInsts() && VT == MVT::f16); in isFAbsFree()
765 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16; in isFNegFree()
823 if (Src == MVT::i16) in isZExtFree()
824 return Dest == MVT::i32 ||Dest == MVT::i64 ; in isZExtFree()
826 return Src == MVT::i32 && Dest == MVT::i64; in isZExtFree()
967 MVT RegisterVT = getRegisterTypeForCallingConv(Ctx, CC, ArgVT); in analyzeFormalArgumentsCompute()
1047 return DAG.getNode(AMDGPUISD::ENDPGM, DL, MVT::Other, Chain); in LowerReturn()
1096 return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); in addTokenForArgument()
1225 SDValue Trap = DAG.getNode(ISD::TRAP, DL, MVT::Other, DAG.getEntryNode()); in LowerGlobalAddress()
1226 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in LowerGlobalAddress()
1250 if (VT == MVT::v4i16 || VT == MVT::v4f16) { in LowerCONCAT_VECTORS()
1252 SDValue Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Op.getOperand(0)); in LowerCONCAT_VECTORS()
1253 SDValue Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Op.getOperand(1)); in LowerCONCAT_VECTORS()
1255 SDValue BV = DAG.getBuildVector(MVT::v2i32, SL, { Lo, Hi }); in LowerCONCAT_VECTORS()
1274 if (((SrcVT == MVT::v4f16 && VT == MVT::v2f16) || in LowerEXTRACT_SUBVECTOR()
1275 (SrcVT == MVT::v4i16 && VT == MVT::v2i16)) && in LowerEXTRACT_SUBVECTOR()
1279 if (((SrcVT == MVT::v8f16 && VT == MVT::v4f16) || in LowerEXTRACT_SUBVECTOR()
1280 (SrcVT == MVT::v8i16 && VT == MVT::v4i16)) && in LowerEXTRACT_SUBVECTOR()
1284 if (((SrcVT == MVT::v16f16 && VT == MVT::v8f16) || in LowerEXTRACT_SUBVECTOR()
1285 (SrcVT == MVT::v16i16 && VT == MVT::v8i16)) && in LowerEXTRACT_SUBVECTOR()
1373 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in split64BitValue()
1375 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in split64BitValue()
1376 const SDValue One = DAG.getConstant(1, SL, MVT::i32); in split64BitValue()
1378 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero); in split64BitValue()
1379 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One); in split64BitValue()
1387 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in getLoHalf64()
1388 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in getLoHalf64()
1389 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero); in getLoHalf64()
1395 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in getHiHalf64()
1396 const SDValue One = DAG.getConstant(1, SL, MVT::i32); in getHiHalf64()
1397 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One); in getHiHalf64()
1488 SDValue Ops[] = {Join, DAG.getNode(ISD::TokenFactor, SL, MVT::Other, in SplitVectorLoad()
1566 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoStore, HiStore); in SplitVectorStore()
1578 MVT IntVT = MVT::i32; in LowerDIVREM24()
1579 MVT FltVT = MVT::f32; in LowerDIVREM24()
1689 assert(VT == MVT::i64 && "LowerUDIVREM64 expects an i64"); in LowerUDIVREM64()
1711 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(0), Zero}); in LowerUDIVREM64()
1712 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(1), Zero}); in LowerUDIVREM64()
1714 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV)); in LowerUDIVREM64()
1715 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM)); in LowerUDIVREM64()
1719 if (isTypeLegal(MVT::i64)) { in LowerUDIVREM64()
1733 SDValue Cvt_Lo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Lo); in LowerUDIVREM64()
1734 SDValue Cvt_Hi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Hi); in LowerUDIVREM64()
1735 SDValue Mad1 = DAG.getNode(FMAD, DL, MVT::f32, Cvt_Hi, in LowerUDIVREM64()
1736 DAG.getConstantFP(APInt(32, 0x4f800000).bitsToFloat(), DL, MVT::f32), in LowerUDIVREM64()
1738 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, DL, MVT::f32, Mad1); in LowerUDIVREM64()
1739 SDValue Mul1 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Rcp, in LowerUDIVREM64()
1740 DAG.getConstantFP(APInt(32, 0x5f7ffffc).bitsToFloat(), DL, MVT::f32)); in LowerUDIVREM64()
1741 SDValue Mul2 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Mul1, in LowerUDIVREM64()
1742 DAG.getConstantFP(APInt(32, 0x2f800000).bitsToFloat(), DL, MVT::f32)); in LowerUDIVREM64()
1743 SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, MVT::f32, Mul2); in LowerUDIVREM64()
1744 SDValue Mad2 = DAG.getNode(FMAD, DL, MVT::f32, Trunc, in LowerUDIVREM64()
1745 DAG.getConstantFP(APInt(32, 0xcf800000).bitsToFloat(), DL, MVT::f32), in LowerUDIVREM64()
1750 DAG.getBuildVector(MVT::v2i32, DL, {Rcp_Lo, Rcp_Hi})); in LowerUDIVREM64()
1754 SDValue Zero1 = DAG.getConstant(0, DL, MVT::i1); in LowerUDIVREM64()
1755 SDVTList HalfCarryVT = DAG.getVTList(HalfVT, MVT::i1); in LowerUDIVREM64()
1770 DAG.getBuildVector(MVT::v2i32, DL, {Add1_Lo, Add1_Hi})); in LowerUDIVREM64()
1784 DAG.getBuildVector(MVT::v2i32, DL, {Add2_Lo, Add2_Hi})); in LowerUDIVREM64()
1798 DAG.getBuildVector(MVT::v2i32, DL, {Sub1_Lo, Sub1_Hi})); in LowerUDIVREM64()
1819 DAG.getBuildVector(MVT::v2i32, DL, {Sub2_Lo, Sub2_Hi})); in LowerUDIVREM64()
1839 DAG.getBuildVector(MVT::v2i32, DL, {Sub3_Lo, Sub3_Hi})); in LowerUDIVREM64()
1862 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {REM_Lo, Zero}); in LowerUDIVREM64()
1863 REM = DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM); in LowerUDIVREM64()
1893 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {DIV_Lo, DIV_Hi}); in LowerUDIVREM64()
1894 DIV = DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV); in LowerUDIVREM64()
1904 if (VT == MVT::i64) { in LowerUDIVREM()
1910 if (VT == MVT::i32) { in LowerUDIVREM()
1965 if (VT == MVT::i32) { in LowerSDIVREM()
1970 if (VT == MVT::i64 && in LowerSDIVREM()
2037 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); in LowerFCEIL()
2039 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64); in LowerFCEIL()
2040 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64); in LowerFCEIL()
2043 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64); in LowerFCEIL()
2049 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero); in LowerFCEIL()
2051 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); in LowerFCEIL()
2059 SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32, in extractF64Exponent()
2061 DAG.getConstant(FractBits - 32, SL, MVT::i32), in extractF64Exponent()
2062 DAG.getConstant(ExpBits, SL, MVT::i32)); in extractF64Exponent()
2063 SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart, in extractF64Exponent()
2064 DAG.getConstant(1023, SL, MVT::i32)); in extractF64Exponent()
2073 assert(Op.getValueType() == MVT::f64); in LowerFTRUNC()
2075 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in LowerFTRUNC()
2086 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, SL, MVT::i32); in LowerFTRUNC()
2087 SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask); in LowerFTRUNC()
2090 SDValue SignBit64 = DAG.getBuildVector(MVT::v2i32, SL, {Zero, SignBit}); in LowerFTRUNC()
2091 SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64); in LowerFTRUNC()
2093 SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src); in LowerFTRUNC()
2095 = DAG.getConstant((UINT64_C(1) << FractBits) - 1, SL, MVT::i64); in LowerFTRUNC()
2097 SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp); in LowerFTRUNC()
2098 SDValue Not = DAG.getNOT(SL, Shr, MVT::i64); in LowerFTRUNC()
2099 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not); in LowerFTRUNC()
2102 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i32); in LowerFTRUNC()
2104 const SDValue FiftyOne = DAG.getConstant(FractBits - 1, SL, MVT::i32); in LowerFTRUNC()
2109 SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0); in LowerFTRUNC()
2110 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1); in LowerFTRUNC()
2112 return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2); in LowerFTRUNC()
2119 assert(Op.getValueType() == MVT::f64); in LowerFRINT()
2122 SDValue C1 = DAG.getConstantFP(C1Val, SL, MVT::f64); in LowerFRINT()
2123 SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src); in LowerFRINT()
2127 SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign); in LowerFRINT()
2128 SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign); in LowerFRINT()
2130 SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src); in LowerFRINT()
2133 SDValue C2 = DAG.getConstantFP(C2Val, SL, MVT::f64); in LowerFRINT()
2136 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64); in LowerFRINT()
2139 return DAG.getSelect(SL, MVT::f64, Cond, Src, Tmp2); in LowerFRINT()
2191 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); in LowerFFLOOR()
2193 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64); in LowerFFLOOR()
2194 const SDValue NegOne = DAG.getConstantFP(-1.0, SL, MVT::f64); in LowerFFLOOR()
2197 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64); in LowerFFLOOR()
2203 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero); in LowerFFLOOR()
2205 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); in LowerFFLOOR()
2250 if (Src.getValueType() == MVT::i32) { in LowerCTLZ_CTTZ()
2255 SDValue NewOpr = DAG.getNode(NewOpc, SL, MVT::i32, Src); in LowerCTLZ_CTTZ()
2257 const SDValue Const32 = DAG.getConstant(32, SL, MVT::i32); in LowerCTLZ_CTTZ()
2258 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, Const32); in LowerCTLZ_CTTZ()
2266 SDValue OprLo = DAG.getNode(NewOpc, SL, MVT::i32, Lo); in LowerCTLZ_CTTZ()
2267 SDValue OprHi = DAG.getNode(NewOpc, SL, MVT::i32, Hi); in LowerCTLZ_CTTZ()
2275 const SDValue Const32 = DAG.getConstant(32, SL, MVT::i32); in LowerCTLZ_CTTZ()
2277 OprLo = DAG.getNode(AddOpc, SL, MVT::i32, OprLo, Const32); in LowerCTLZ_CTTZ()
2279 OprHi = DAG.getNode(AddOpc, SL, MVT::i32, OprHi, Const32); in LowerCTLZ_CTTZ()
2282 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, OprLo, OprHi); in LowerCTLZ_CTTZ()
2284 const SDValue Const64 = DAG.getConstant(64, SL, MVT::i32); in LowerCTLZ_CTTZ()
2285 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, Const64); in LowerCTLZ_CTTZ()
2288 return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i64, NewOpr); in LowerCTLZ_CTTZ()
2349 ISD::SRA, SL, MVT::i32, DAG.getNode(ISD::XOR, SL, MVT::i32, Lo, Hi), in LowerINT_TO_FP32()
2350 DAG.getConstant(31, SL, MVT::i32)); in LowerINT_TO_FP32()
2352 DAG.getNode(ISD::ADD, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32), in LowerINT_TO_FP32()
2355 ShAmt = DAG.getNode(AMDGPUISD::FFBH_I32, SL, MVT::i32, Hi); in LowerINT_TO_FP32()
2358 ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, ShAmt, in LowerINT_TO_FP32()
2359 DAG.getConstant(1, SL, MVT::i32)); in LowerINT_TO_FP32()
2360 ShAmt = DAG.getNode(ISD::UMIN, SL, MVT::i32, ShAmt, MaxShAmt); in LowerINT_TO_FP32()
2365 Sign = DAG.getNode(ISD::SRA, SL, MVT::i64, Src, in LowerINT_TO_FP32()
2366 DAG.getConstant(63, SL, MVT::i64)); in LowerINT_TO_FP32()
2368 DAG.getNode(ISD::XOR, SL, MVT::i64, in LowerINT_TO_FP32()
2369 DAG.getNode(ISD::ADD, SL, MVT::i64, Src, Sign), Sign); in LowerINT_TO_FP32()
2373 ShAmt = DAG.getNode(ISD::CTLZ, SL, MVT::i32, Hi); in LowerINT_TO_FP32()
2377 SDValue Norm = DAG.getNode(ISD::SHL, SL, MVT::i64, Src, ShAmt); in LowerINT_TO_FP32()
2382 SDValue Adjust = DAG.getNode(ISD::UMIN, SL, MVT::i32, in LowerINT_TO_FP32()
2383 DAG.getConstant(1, SL, MVT::i32), Lo); in LowerINT_TO_FP32()
2385 Norm = DAG.getNode(ISD::OR, SL, MVT::i32, Hi, Adjust); in LowerINT_TO_FP32()
2389 SDValue FVal = DAG.getNode(Opc, SL, MVT::f32, Norm); in LowerINT_TO_FP32()
2393 ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32), in LowerINT_TO_FP32()
2397 return DAG.getNode(AMDGPUISD::LDEXP, SL, MVT::f32, FVal, ShAmt); in LowerINT_TO_FP32()
2402 SDValue Exp = DAG.getNode(ISD::SHL, SL, MVT::i32, ShAmt, in LowerINT_TO_FP32()
2403 DAG.getConstant(23, SL, MVT::i32)); in LowerINT_TO_FP32()
2405 DAG.getNode(ISD::ADD, SL, MVT::i32, in LowerINT_TO_FP32()
2406 DAG.getNode(ISD::BITCAST, SL, MVT::i32, FVal), Exp); in LowerINT_TO_FP32()
2409 Sign = DAG.getNode(ISD::SHL, SL, MVT::i32, in LowerINT_TO_FP32()
2410 DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Sign), in LowerINT_TO_FP32()
2411 DAG.getConstant(31, SL, MVT::i32)); in LowerINT_TO_FP32()
2412 IVal = DAG.getNode(ISD::OR, SL, MVT::i32, IVal, Sign); in LowerINT_TO_FP32()
2414 return DAG.getNode(ISD::BITCAST, SL, MVT::f32, IVal); in LowerINT_TO_FP32()
2426 SL, MVT::f64, Hi); in LowerINT_TO_FP64()
2428 SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo); in LowerINT_TO_FP64()
2430 SDValue LdExp = DAG.getNode(AMDGPUISD::LDEXP, SL, MVT::f64, CvtHi, in LowerINT_TO_FP64()
2431 DAG.getConstant(32, SL, MVT::i32)); in LowerINT_TO_FP64()
2433 return DAG.getNode(ISD::FADD, SL, MVT::f64, LdExp, CvtLo); in LowerINT_TO_FP64()
2443 if (SrcVT == MVT::i16) { in LowerUINT_TO_FP()
2444 if (DestVT == MVT::f16) in LowerUINT_TO_FP()
2449 SDValue Ext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Src); in LowerUINT_TO_FP()
2453 assert(SrcVT == MVT::i64 && "operation should be legal"); in LowerUINT_TO_FP()
2455 if (Subtarget->has16BitInsts() && DestVT == MVT::f16) { in LowerUINT_TO_FP()
2458 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src); in LowerUINT_TO_FP()
2461 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag); in LowerUINT_TO_FP()
2466 if (DestVT == MVT::f32) in LowerUINT_TO_FP()
2469 assert(DestVT == MVT::f64); in LowerUINT_TO_FP()
2480 if (SrcVT == MVT::i16) { in LowerSINT_TO_FP()
2481 if (DestVT == MVT::f16) in LowerSINT_TO_FP()
2486 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32, Src); in LowerSINT_TO_FP()
2490 assert(SrcVT == MVT::i64 && "operation should be legal"); in LowerSINT_TO_FP()
2494 if (Subtarget->has16BitInsts() && DestVT == MVT::f16) { in LowerSINT_TO_FP()
2498 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src); in LowerSINT_TO_FP()
2501 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag); in LowerSINT_TO_FP()
2506 if (DestVT == MVT::f32) in LowerSINT_TO_FP()
2509 assert(DestVT == MVT::f64); in LowerSINT_TO_FP()
2520 assert(SrcVT == MVT::f32 || SrcVT == MVT::f64); in LowerFP_TO_INT64()
2533 if (Signed && SrcVT == MVT::f32) { in LowerFP_TO_INT64()
2539 Sign = DAG.getNode(ISD::SRA, SL, MVT::i32, in LowerFP_TO_INT64()
2540 DAG.getNode(ISD::BITCAST, SL, MVT::i32, Trunc), in LowerFP_TO_INT64()
2541 DAG.getConstant(31, SL, MVT::i32)); in LowerFP_TO_INT64()
2546 if (SrcVT == MVT::f64) { in LowerFP_TO_INT64()
2564 SDValue Hi = DAG.getNode((Signed && SrcVT == MVT::f64) ? ISD::FP_TO_SINT in LowerFP_TO_INT64()
2566 SL, MVT::i32, FloorMul); in LowerFP_TO_INT64()
2567 SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, SL, MVT::i32, Fma); in LowerFP_TO_INT64()
2569 SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i64, in LowerFP_TO_INT64()
2570 DAG.getBuildVector(MVT::v2i32, SL, {Lo, Hi})); in LowerFP_TO_INT64()
2572 if (Signed && SrcVT == MVT::f32) { in LowerFP_TO_INT64()
2575 Sign = DAG.getNode(ISD::BITCAST, SL, MVT::i64, in LowerFP_TO_INT64()
2576 DAG.getBuildVector(MVT::v2i32, SL, {Sign, Sign})); in LowerFP_TO_INT64()
2579 DAG.getNode(ISD::SUB, SL, MVT::i64, in LowerFP_TO_INT64()
2580 DAG.getNode(ISD::XOR, SL, MVT::i64, Result, Sign), Sign); in LowerFP_TO_INT64()
2591 if (N0.getValueType() == MVT::f32) in LowerFP_TO_FP16()
2599 assert(N0.getSimpleValueType() == MVT::f64); in LowerFP_TO_FP16()
2605 SDValue Zero = DAG.getConstant(0, DL, MVT::i32); in LowerFP_TO_FP16()
2606 SDValue One = DAG.getConstant(1, DL, MVT::i32); in LowerFP_TO_FP16()
2607 SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, N0); in LowerFP_TO_FP16()
2608 SDValue UH = DAG.getNode(ISD::SRL, DL, MVT::i64, U, in LowerFP_TO_FP16()
2609 DAG.getConstant(32, DL, MVT::i64)); in LowerFP_TO_FP16()
2610 UH = DAG.getZExtOrTrunc(UH, DL, MVT::i32); in LowerFP_TO_FP16()
2611 U = DAG.getZExtOrTrunc(U, DL, MVT::i32); in LowerFP_TO_FP16()
2612 SDValue E = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, in LowerFP_TO_FP16()
2613 DAG.getConstant(20, DL, MVT::i64)); in LowerFP_TO_FP16()
2614 E = DAG.getNode(ISD::AND, DL, MVT::i32, E, in LowerFP_TO_FP16()
2615 DAG.getConstant(ExpMask, DL, MVT::i32)); in LowerFP_TO_FP16()
2618 E = DAG.getNode(ISD::ADD, DL, MVT::i32, E, in LowerFP_TO_FP16()
2619 DAG.getConstant(-ExpBiasf64 + ExpBiasf16, DL, MVT::i32)); in LowerFP_TO_FP16()
2621 SDValue M = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, in LowerFP_TO_FP16()
2622 DAG.getConstant(8, DL, MVT::i32)); in LowerFP_TO_FP16()
2623 M = DAG.getNode(ISD::AND, DL, MVT::i32, M, in LowerFP_TO_FP16()
2624 DAG.getConstant(0xffe, DL, MVT::i32)); in LowerFP_TO_FP16()
2626 SDValue MaskedSig = DAG.getNode(ISD::AND, DL, MVT::i32, UH, in LowerFP_TO_FP16()
2627 DAG.getConstant(0x1ff, DL, MVT::i32)); in LowerFP_TO_FP16()
2628 MaskedSig = DAG.getNode(ISD::OR, DL, MVT::i32, MaskedSig, U); in LowerFP_TO_FP16()
2631 M = DAG.getNode(ISD::OR, DL, MVT::i32, M, Lo40Set); in LowerFP_TO_FP16()
2634 SDValue I = DAG.getNode(ISD::OR, DL, MVT::i32, in LowerFP_TO_FP16()
2635 DAG.getSelectCC(DL, M, Zero, DAG.getConstant(0x0200, DL, MVT::i32), in LowerFP_TO_FP16()
2636 Zero, ISD::SETNE), DAG.getConstant(0x7c00, DL, MVT::i32)); in LowerFP_TO_FP16()
2639 SDValue N = DAG.getNode(ISD::OR, DL, MVT::i32, M, in LowerFP_TO_FP16()
2640 DAG.getNode(ISD::SHL, DL, MVT::i32, E, in LowerFP_TO_FP16()
2641 DAG.getConstant(12, DL, MVT::i32))); in LowerFP_TO_FP16()
2644 SDValue OneSubExp = DAG.getNode(ISD::SUB, DL, MVT::i32, in LowerFP_TO_FP16()
2646 SDValue B = DAG.getNode(ISD::SMAX, DL, MVT::i32, OneSubExp, Zero); in LowerFP_TO_FP16()
2647 B = DAG.getNode(ISD::SMIN, DL, MVT::i32, B, in LowerFP_TO_FP16()
2648 DAG.getConstant(13, DL, MVT::i32)); in LowerFP_TO_FP16()
2650 SDValue SigSetHigh = DAG.getNode(ISD::OR, DL, MVT::i32, M, in LowerFP_TO_FP16()
2651 DAG.getConstant(0x1000, DL, MVT::i32)); in LowerFP_TO_FP16()
2653 SDValue D = DAG.getNode(ISD::SRL, DL, MVT::i32, SigSetHigh, B); in LowerFP_TO_FP16()
2654 SDValue D0 = DAG.getNode(ISD::SHL, DL, MVT::i32, D, B); in LowerFP_TO_FP16()
2656 D = DAG.getNode(ISD::OR, DL, MVT::i32, D, D1); in LowerFP_TO_FP16()
2659 SDValue VLow3 = DAG.getNode(ISD::AND, DL, MVT::i32, V, in LowerFP_TO_FP16()
2660 DAG.getConstant(0x7, DL, MVT::i32)); in LowerFP_TO_FP16()
2661 V = DAG.getNode(ISD::SRL, DL, MVT::i32, V, in LowerFP_TO_FP16()
2662 DAG.getConstant(2, DL, MVT::i32)); in LowerFP_TO_FP16()
2663 SDValue V0 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(3, DL, MVT::i32), in LowerFP_TO_FP16()
2665 SDValue V1 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(5, DL, MVT::i32), in LowerFP_TO_FP16()
2667 V1 = DAG.getNode(ISD::OR, DL, MVT::i32, V0, V1); in LowerFP_TO_FP16()
2668 V = DAG.getNode(ISD::ADD, DL, MVT::i32, V, V1); in LowerFP_TO_FP16()
2670 V = DAG.getSelectCC(DL, E, DAG.getConstant(30, DL, MVT::i32), in LowerFP_TO_FP16()
2671 DAG.getConstant(0x7c00, DL, MVT::i32), V, ISD::SETGT); in LowerFP_TO_FP16()
2672 V = DAG.getSelectCC(DL, E, DAG.getConstant(1039, DL, MVT::i32), in LowerFP_TO_FP16()
2676 SDValue Sign = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, in LowerFP_TO_FP16()
2677 DAG.getConstant(16, DL, MVT::i32)); in LowerFP_TO_FP16()
2678 Sign = DAG.getNode(ISD::AND, DL, MVT::i32, Sign, in LowerFP_TO_FP16()
2679 DAG.getConstant(0x8000, DL, MVT::i32)); in LowerFP_TO_FP16()
2681 V = DAG.getNode(ISD::OR, DL, MVT::i32, Sign, V); in LowerFP_TO_FP16()
2693 if (SrcVT == MVT::f16 && DestVT == MVT::i16) in LowerFP_TO_INT()
2697 if (DestVT == MVT::i16 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) { in LowerFP_TO_INT()
2700 SDValue FpToInt32 = DAG.getNode(OpOpcode, DL, MVT::i32, Src); in LowerFP_TO_INT()
2701 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToInt32); in LowerFP_TO_INT()
2704 if (SrcVT == MVT::f16 || in LowerFP_TO_INT()
2705 (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP)) { in LowerFP_TO_INT()
2708 SDValue FpToInt32 = DAG.getNode(OpOpcode, DL, MVT::i32, Src); in LowerFP_TO_INT()
2711 return DAG.getNode(Ext, DL, MVT::i64, FpToInt32); in LowerFP_TO_INT()
2714 if (DestVT == MVT::i64 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) in LowerFP_TO_INT()
2723 MVT VT = Op.getSimpleValueType(); in LowerSIGN_EXTEND_INREG()
2724 MVT ScalarVT = VT.getScalarType(); in LowerSIGN_EXTEND_INREG()
2815 return DAG.getConstant(Result, DL, MVT::i32); in constantFoldBFE()
2818 return DAG.getConstant(Src0 >> Offset, DL, MVT::i32); in constantFoldBFE()
2834 if (VT.getScalarType() == MVT::i32 || isTypeLegal(VT)) in shouldCombineMemoryType()
3020 SDValue LoRHS = DAG.getConstant(ValLo, SL, MVT::i32); in splitBinaryBitConstantOpImpl()
3021 SDValue HiRHS = DAG.getConstant(ValHi, SL, MVT::i32); in splitBinaryBitConstantOpImpl()
3023 SDValue LoAnd = DAG.getNode(Opc, SL, MVT::i32, Lo, LoRHS); in splitBinaryBitConstantOpImpl()
3024 SDValue HiAnd = DAG.getNode(Opc, SL, MVT::i32, Hi, HiRHS); in splitBinaryBitConstantOpImpl()
3031 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {LoAnd, HiAnd}); in splitBinaryBitConstantOpImpl()
3032 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in splitBinaryBitConstantOpImpl()
3059 if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 && in performShlCombine()
3060 isOperationLegal(ISD::BUILD_VECTOR, MVT::v2i16)) { in performShlCombine()
3063 SDValue Vec = DAG.getBuildVector(MVT::v2i16, SL, in performShlCombine()
3064 { DAG.getConstant(0, SL, MVT::i16), LHS->getOperand(0) }); in performShlCombine()
3065 return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Vec); in performShlCombine()
3069 if (VT != MVT::i64) in performShlCombine()
3081 if (VT != MVT::i64) in performShlCombine()
3092 SDValue ShiftAmt = DAG.getConstant(RHSVal - 32, SL, MVT::i32); in performShlCombine()
3094 SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS); in performShlCombine()
3095 SDValue NewShift = DAG.getNode(ISD::SHL, SL, MVT::i32, Lo, ShiftAmt); in performShlCombine()
3097 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in performShlCombine()
3099 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {Zero, NewShift}); in performShlCombine()
3100 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in performShlCombine()
3105 if (N->getValueType(0) != MVT::i64) in performSraCombine()
3119 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi, in performSraCombine()
3120 DAG.getConstant(31, SL, MVT::i32)); in performSraCombine()
3122 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {Hi, NewShift}); in performSraCombine()
3123 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec); in performSraCombine()
3129 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi, in performSraCombine()
3130 DAG.getConstant(31, SL, MVT::i32)); in performSraCombine()
3131 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, NewShift}); in performSraCombine()
3132 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec); in performSraCombine()
3165 if (VT != MVT::i64) in performSrlCombine()
3174 SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in performSrlCombine()
3178 SDValue NewConst = DAG.getConstant(ShiftAmt - 32, SL, MVT::i32); in performSrlCombine()
3179 SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst); in performSrlCombine()
3181 SDValue BuildPair = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, Zero}); in performSrlCombine()
3183 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair); in performSrlCombine()
3248 EVT::getVectorVT(*DAG.getContext(), MVT::i32, in performTruncateCombine()
3249 VT.getVectorNumElements()) : MVT::i32; in performTruncateCombine()
3279 return DAG.getNode(MulOpc, SL, MVT::i32, N0, N1); in getMul24()
3285 SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1); in getMul24()
3286 SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1); in getMul24()
3288 return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, MulLo, MulHi); in getMul24()
3307 if (Subtarget->has16BitInsts() && VT.getScalarType().bitsLE(MVT::i16)) in performMulCombine()
3329 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); in performMulCombine()
3330 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); in performMulCombine()
3333 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); in performMulCombine()
3334 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); in performMulCombine()
3348 if (N->getValueType(0) != MVT::i32) in performMulLoHiCombine()
3370 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); in performMulLoHiCombine()
3371 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); in performMulLoHiCombine()
3375 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); in performMulLoHiCombine()
3376 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); in performMulLoHiCombine()
3383 SDValue Lo = DAG.getNode(LoOpcode, DL, MVT::i32, N0, N1); in performMulLoHiCombine()
3384 SDValue Hi = DAG.getNode(HiOpcode, DL, MVT::i32, N0, N1); in performMulLoHiCombine()
3414 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); in performMulhsCombine()
3415 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); in performMulhsCombine()
3417 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_I24, DL, MVT::i32, N0, N1); in performMulhsCombine()
3447 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); in performMulhuCombine()
3448 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); in performMulhuCombine()
3450 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_U24, DL, MVT::i32, N0, N1); in performMulhuCombine()
3467 if (LegalVT != MVT::i32 && (Subtarget->has16BitInsts() && in getFFBX_U32()
3468 LegalVT != MVT::i16)) in getFFBX_U32()
3471 if (VT != MVT::i32) in getFFBX_U32()
3472 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Op); in getFFBX_U32()
3474 SDValue FFBX = DAG.getNode(Opc, DL, MVT::i32, Op); in getFFBX_U32()
3475 if (VT != MVT::i32) in getFFBX_U32()
3638 if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) { in performSelectCombine()
3989 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in PerformDAGCombine()
3990 DAG.getConstant(Lo_32(CVal), SL, MVT::i32), in PerformDAGCombine()
3991 DAG.getConstant(Hi_32(CVal), SL, MVT::i32)); in PerformDAGCombine()
3999 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in PerformDAGCombine()
4000 DAG.getConstant(Lo_32(CVal), SL, MVT::i32), in PerformDAGCombine()
4001 DAG.getConstant(Hi_32(CVal), SL, MVT::i32)); in PerformDAGCombine()
4058 return DAG.getConstant(0, DL, MVT::i32); in PerformDAGCombine()
4085 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, BitsFrom, in PerformDAGCombine()
4110 SDValue ShiftVal = DAG.getConstant(OffsetVal, DL, MVT::i32); in PerformDAGCombine()
4111 return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32, in PerformDAGCombine()
4197 SDValue Ptr = DAG.getFrameIndex(FI, MVT::i32); in loadStackInputValue()
4213 SDValue Ptr = DAG.getConstant(Offset, SL, MVT::i32); in storeStackInputValue()
4216 DAG.getCopyFromReg(Chain, SL, Info->getStackPtrOffsetReg(), MVT::i32); in storeStackInputValue()
4217 Ptr = DAG.getNode(ISD::ADD, SL, MVT::i32, SP, Ptr); in storeStackInputValue()
4427 if (VT == MVT::f32) { in getSqrtEstimate()
4443 if (VT == MVT::f32) { in getRecipEstimate()