Lines Matching refs:MVT
46 return EVT::getVectorVT(Ctx, MVT::i32, StoreSize / 32); in getEquivalentMemType()
64 setOperationAction(ISD::LOAD, MVT::f32, Promote); in AMDGPUTargetLowering()
65 AddPromotedToType(ISD::LOAD, MVT::f32, MVT::i32); in AMDGPUTargetLowering()
67 setOperationAction(ISD::LOAD, MVT::v2f32, Promote); in AMDGPUTargetLowering()
68 AddPromotedToType(ISD::LOAD, MVT::v2f32, MVT::v2i32); in AMDGPUTargetLowering()
70 setOperationAction(ISD::LOAD, MVT::v3f32, Promote); in AMDGPUTargetLowering()
71 AddPromotedToType(ISD::LOAD, MVT::v3f32, MVT::v3i32); in AMDGPUTargetLowering()
73 setOperationAction(ISD::LOAD, MVT::v4f32, Promote); in AMDGPUTargetLowering()
74 AddPromotedToType(ISD::LOAD, MVT::v4f32, MVT::v4i32); in AMDGPUTargetLowering()
76 setOperationAction(ISD::LOAD, MVT::v5f32, Promote); in AMDGPUTargetLowering()
77 AddPromotedToType(ISD::LOAD, MVT::v5f32, MVT::v5i32); in AMDGPUTargetLowering()
79 setOperationAction(ISD::LOAD, MVT::v6f32, Promote); in AMDGPUTargetLowering()
80 AddPromotedToType(ISD::LOAD, MVT::v6f32, MVT::v6i32); in AMDGPUTargetLowering()
82 setOperationAction(ISD::LOAD, MVT::v7f32, Promote); in AMDGPUTargetLowering()
83 AddPromotedToType(ISD::LOAD, MVT::v7f32, MVT::v7i32); in AMDGPUTargetLowering()
85 setOperationAction(ISD::LOAD, MVT::v8f32, Promote); in AMDGPUTargetLowering()
86 AddPromotedToType(ISD::LOAD, MVT::v8f32, MVT::v8i32); in AMDGPUTargetLowering()
88 setOperationAction(ISD::LOAD, MVT::v9f32, Promote); in AMDGPUTargetLowering()
89 AddPromotedToType(ISD::LOAD, MVT::v9f32, MVT::v9i32); in AMDGPUTargetLowering()
91 setOperationAction(ISD::LOAD, MVT::v10f32, Promote); in AMDGPUTargetLowering()
92 AddPromotedToType(ISD::LOAD, MVT::v10f32, MVT::v10i32); in AMDGPUTargetLowering()
94 setOperationAction(ISD::LOAD, MVT::v11f32, Promote); in AMDGPUTargetLowering()
95 AddPromotedToType(ISD::LOAD, MVT::v11f32, MVT::v11i32); in AMDGPUTargetLowering()
97 setOperationAction(ISD::LOAD, MVT::v12f32, Promote); in AMDGPUTargetLowering()
98 AddPromotedToType(ISD::LOAD, MVT::v12f32, MVT::v12i32); in AMDGPUTargetLowering()
100 setOperationAction(ISD::LOAD, MVT::v16f32, Promote); in AMDGPUTargetLowering()
101 AddPromotedToType(ISD::LOAD, MVT::v16f32, MVT::v16i32); in AMDGPUTargetLowering()
103 setOperationAction(ISD::LOAD, MVT::v32f32, Promote); in AMDGPUTargetLowering()
104 AddPromotedToType(ISD::LOAD, MVT::v32f32, MVT::v32i32); in AMDGPUTargetLowering()
106 setOperationAction(ISD::LOAD, MVT::i64, Promote); in AMDGPUTargetLowering()
107 AddPromotedToType(ISD::LOAD, MVT::i64, MVT::v2i32); in AMDGPUTargetLowering()
109 setOperationAction(ISD::LOAD, MVT::v2i64, Promote); in AMDGPUTargetLowering()
110 AddPromotedToType(ISD::LOAD, MVT::v2i64, MVT::v4i32); in AMDGPUTargetLowering()
112 setOperationAction(ISD::LOAD, MVT::f64, Promote); in AMDGPUTargetLowering()
113 AddPromotedToType(ISD::LOAD, MVT::f64, MVT::v2i32); in AMDGPUTargetLowering()
115 setOperationAction(ISD::LOAD, MVT::v2f64, Promote); in AMDGPUTargetLowering()
116 AddPromotedToType(ISD::LOAD, MVT::v2f64, MVT::v4i32); in AMDGPUTargetLowering()
118 setOperationAction(ISD::LOAD, MVT::v3i64, Promote); in AMDGPUTargetLowering()
119 AddPromotedToType(ISD::LOAD, MVT::v3i64, MVT::v6i32); in AMDGPUTargetLowering()
121 setOperationAction(ISD::LOAD, MVT::v4i64, Promote); in AMDGPUTargetLowering()
122 AddPromotedToType(ISD::LOAD, MVT::v4i64, MVT::v8i32); in AMDGPUTargetLowering()
124 setOperationAction(ISD::LOAD, MVT::v3f64, Promote); in AMDGPUTargetLowering()
125 AddPromotedToType(ISD::LOAD, MVT::v3f64, MVT::v6i32); in AMDGPUTargetLowering()
127 setOperationAction(ISD::LOAD, MVT::v4f64, Promote); in AMDGPUTargetLowering()
128 AddPromotedToType(ISD::LOAD, MVT::v4f64, MVT::v8i32); in AMDGPUTargetLowering()
130 setOperationAction(ISD::LOAD, MVT::v8i64, Promote); in AMDGPUTargetLowering()
131 AddPromotedToType(ISD::LOAD, MVT::v8i64, MVT::v16i32); in AMDGPUTargetLowering()
133 setOperationAction(ISD::LOAD, MVT::v8f64, Promote); in AMDGPUTargetLowering()
134 AddPromotedToType(ISD::LOAD, MVT::v8f64, MVT::v16i32); in AMDGPUTargetLowering()
136 setOperationAction(ISD::LOAD, MVT::v16i64, Promote); in AMDGPUTargetLowering()
137 AddPromotedToType(ISD::LOAD, MVT::v16i64, MVT::v32i32); in AMDGPUTargetLowering()
139 setOperationAction(ISD::LOAD, MVT::v16f64, Promote); in AMDGPUTargetLowering()
140 AddPromotedToType(ISD::LOAD, MVT::v16f64, MVT::v32i32); in AMDGPUTargetLowering()
142 setOperationAction(ISD::LOAD, MVT::i128, Promote); in AMDGPUTargetLowering()
143 AddPromotedToType(ISD::LOAD, MVT::i128, MVT::v4i32); in AMDGPUTargetLowering()
147 for (MVT VT : MVT::integer_valuetypes()) in AMDGPUTargetLowering()
148 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::i64, VT, in AMDGPUTargetLowering()
151 for (MVT VT : MVT::integer_valuetypes()) { in AMDGPUTargetLowering()
152 if (VT == MVT::i64) in AMDGPUTargetLowering()
156 setLoadExtAction(Op, VT, MVT::i1, Promote); in AMDGPUTargetLowering()
157 setLoadExtAction(Op, VT, MVT::i8, Legal); in AMDGPUTargetLowering()
158 setLoadExtAction(Op, VT, MVT::i16, Legal); in AMDGPUTargetLowering()
159 setLoadExtAction(Op, VT, MVT::i32, Expand); in AMDGPUTargetLowering()
163 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) in AMDGPUTargetLowering()
165 {MVT::v2i8, MVT::v4i8, MVT::v2i16, MVT::v3i16, MVT::v4i16}) in AMDGPUTargetLowering()
169 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand); in AMDGPUTargetLowering()
170 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::bf16, Expand); in AMDGPUTargetLowering()
171 setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2f16, Expand); in AMDGPUTargetLowering()
172 setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2bf16, Expand); in AMDGPUTargetLowering()
173 setLoadExtAction(ISD::EXTLOAD, MVT::v3f32, MVT::v3f16, Expand); in AMDGPUTargetLowering()
174 setLoadExtAction(ISD::EXTLOAD, MVT::v3f32, MVT::v3bf16, Expand); in AMDGPUTargetLowering()
175 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Expand); in AMDGPUTargetLowering()
176 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4bf16, Expand); in AMDGPUTargetLowering()
177 setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Expand); in AMDGPUTargetLowering()
178 setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8bf16, Expand); in AMDGPUTargetLowering()
179 setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16f16, Expand); in AMDGPUTargetLowering()
180 setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16bf16, Expand); in AMDGPUTargetLowering()
181 setLoadExtAction(ISD::EXTLOAD, MVT::v32f32, MVT::v32f16, Expand); in AMDGPUTargetLowering()
182 setLoadExtAction(ISD::EXTLOAD, MVT::v32f32, MVT::v32bf16, Expand); in AMDGPUTargetLowering()
184 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand); in AMDGPUTargetLowering()
185 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Expand); in AMDGPUTargetLowering()
186 setLoadExtAction(ISD::EXTLOAD, MVT::v3f64, MVT::v3f32, Expand); in AMDGPUTargetLowering()
187 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Expand); in AMDGPUTargetLowering()
188 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f32, Expand); in AMDGPUTargetLowering()
189 setLoadExtAction(ISD::EXTLOAD, MVT::v16f64, MVT::v16f32, Expand); in AMDGPUTargetLowering()
191 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand); in AMDGPUTargetLowering()
192 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::bf16, Expand); in AMDGPUTargetLowering()
193 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Expand); in AMDGPUTargetLowering()
194 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2bf16, Expand); in AMDGPUTargetLowering()
195 setLoadExtAction(ISD::EXTLOAD, MVT::v3f64, MVT::v3f16, Expand); in AMDGPUTargetLowering()
196 setLoadExtAction(ISD::EXTLOAD, MVT::v3f64, MVT::v3bf16, Expand); in AMDGPUTargetLowering()
197 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Expand); in AMDGPUTargetLowering()
198 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4bf16, Expand); in AMDGPUTargetLowering()
199 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Expand); in AMDGPUTargetLowering()
200 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8bf16, Expand); in AMDGPUTargetLowering()
201 setLoadExtAction(ISD::EXTLOAD, MVT::v16f64, MVT::v16f16, Expand); in AMDGPUTargetLowering()
202 setLoadExtAction(ISD::EXTLOAD, MVT::v16f64, MVT::v16bf16, Expand); in AMDGPUTargetLowering()
204 setOperationAction(ISD::STORE, MVT::f32, Promote); in AMDGPUTargetLowering()
205 AddPromotedToType(ISD::STORE, MVT::f32, MVT::i32); in AMDGPUTargetLowering()
207 setOperationAction(ISD::STORE, MVT::v2f32, Promote); in AMDGPUTargetLowering()
208 AddPromotedToType(ISD::STORE, MVT::v2f32, MVT::v2i32); in AMDGPUTargetLowering()
210 setOperationAction(ISD::STORE, MVT::v3f32, Promote); in AMDGPUTargetLowering()
211 AddPromotedToType(ISD::STORE, MVT::v3f32, MVT::v3i32); in AMDGPUTargetLowering()
213 setOperationAction(ISD::STORE, MVT::v4f32, Promote); in AMDGPUTargetLowering()
214 AddPromotedToType(ISD::STORE, MVT::v4f32, MVT::v4i32); in AMDGPUTargetLowering()
216 setOperationAction(ISD::STORE, MVT::v5f32, Promote); in AMDGPUTargetLowering()
217 AddPromotedToType(ISD::STORE, MVT::v5f32, MVT::v5i32); in AMDGPUTargetLowering()
219 setOperationAction(ISD::STORE, MVT::v6f32, Promote); in AMDGPUTargetLowering()
220 AddPromotedToType(ISD::STORE, MVT::v6f32, MVT::v6i32); in AMDGPUTargetLowering()
222 setOperationAction(ISD::STORE, MVT::v7f32, Promote); in AMDGPUTargetLowering()
223 AddPromotedToType(ISD::STORE, MVT::v7f32, MVT::v7i32); in AMDGPUTargetLowering()
225 setOperationAction(ISD::STORE, MVT::v8f32, Promote); in AMDGPUTargetLowering()
226 AddPromotedToType(ISD::STORE, MVT::v8f32, MVT::v8i32); in AMDGPUTargetLowering()
228 setOperationAction(ISD::STORE, MVT::v9f32, Promote); in AMDGPUTargetLowering()
229 AddPromotedToType(ISD::STORE, MVT::v9f32, MVT::v9i32); in AMDGPUTargetLowering()
231 setOperationAction(ISD::STORE, MVT::v10f32, Promote); in AMDGPUTargetLowering()
232 AddPromotedToType(ISD::STORE, MVT::v10f32, MVT::v10i32); in AMDGPUTargetLowering()
234 setOperationAction(ISD::STORE, MVT::v11f32, Promote); in AMDGPUTargetLowering()
235 AddPromotedToType(ISD::STORE, MVT::v11f32, MVT::v11i32); in AMDGPUTargetLowering()
237 setOperationAction(ISD::STORE, MVT::v12f32, Promote); in AMDGPUTargetLowering()
238 AddPromotedToType(ISD::STORE, MVT::v12f32, MVT::v12i32); in AMDGPUTargetLowering()
240 setOperationAction(ISD::STORE, MVT::v16f32, Promote); in AMDGPUTargetLowering()
241 AddPromotedToType(ISD::STORE, MVT::v16f32, MVT::v16i32); in AMDGPUTargetLowering()
243 setOperationAction(ISD::STORE, MVT::v32f32, Promote); in AMDGPUTargetLowering()
244 AddPromotedToType(ISD::STORE, MVT::v32f32, MVT::v32i32); in AMDGPUTargetLowering()
246 setOperationAction(ISD::STORE, MVT::i64, Promote); in AMDGPUTargetLowering()
247 AddPromotedToType(ISD::STORE, MVT::i64, MVT::v2i32); in AMDGPUTargetLowering()
249 setOperationAction(ISD::STORE, MVT::v2i64, Promote); in AMDGPUTargetLowering()
250 AddPromotedToType(ISD::STORE, MVT::v2i64, MVT::v4i32); in AMDGPUTargetLowering()
252 setOperationAction(ISD::STORE, MVT::f64, Promote); in AMDGPUTargetLowering()
253 AddPromotedToType(ISD::STORE, MVT::f64, MVT::v2i32); in AMDGPUTargetLowering()
255 setOperationAction(ISD::STORE, MVT::v2f64, Promote); in AMDGPUTargetLowering()
256 AddPromotedToType(ISD::STORE, MVT::v2f64, MVT::v4i32); in AMDGPUTargetLowering()
258 setOperationAction(ISD::STORE, MVT::v3i64, Promote); in AMDGPUTargetLowering()
259 AddPromotedToType(ISD::STORE, MVT::v3i64, MVT::v6i32); in AMDGPUTargetLowering()
261 setOperationAction(ISD::STORE, MVT::v3f64, Promote); in AMDGPUTargetLowering()
262 AddPromotedToType(ISD::STORE, MVT::v3f64, MVT::v6i32); in AMDGPUTargetLowering()
264 setOperationAction(ISD::STORE, MVT::v4i64, Promote); in AMDGPUTargetLowering()
265 AddPromotedToType(ISD::STORE, MVT::v4i64, MVT::v8i32); in AMDGPUTargetLowering()
267 setOperationAction(ISD::STORE, MVT::v4f64, Promote); in AMDGPUTargetLowering()
268 AddPromotedToType(ISD::STORE, MVT::v4f64, MVT::v8i32); in AMDGPUTargetLowering()
270 setOperationAction(ISD::STORE, MVT::v8i64, Promote); in AMDGPUTargetLowering()
271 AddPromotedToType(ISD::STORE, MVT::v8i64, MVT::v16i32); in AMDGPUTargetLowering()
273 setOperationAction(ISD::STORE, MVT::v8f64, Promote); in AMDGPUTargetLowering()
274 AddPromotedToType(ISD::STORE, MVT::v8f64, MVT::v16i32); in AMDGPUTargetLowering()
276 setOperationAction(ISD::STORE, MVT::v16i64, Promote); in AMDGPUTargetLowering()
277 AddPromotedToType(ISD::STORE, MVT::v16i64, MVT::v32i32); in AMDGPUTargetLowering()
279 setOperationAction(ISD::STORE, MVT::v16f64, Promote); in AMDGPUTargetLowering()
280 AddPromotedToType(ISD::STORE, MVT::v16f64, MVT::v32i32); in AMDGPUTargetLowering()
282 setOperationAction(ISD::STORE, MVT::i128, Promote); in AMDGPUTargetLowering()
283 AddPromotedToType(ISD::STORE, MVT::i128, MVT::v4i32); in AMDGPUTargetLowering()
285 setTruncStoreAction(MVT::i64, MVT::i1, Expand); in AMDGPUTargetLowering()
286 setTruncStoreAction(MVT::i64, MVT::i8, Expand); in AMDGPUTargetLowering()
287 setTruncStoreAction(MVT::i64, MVT::i16, Expand); in AMDGPUTargetLowering()
288 setTruncStoreAction(MVT::i64, MVT::i32, Expand); in AMDGPUTargetLowering()
290 setTruncStoreAction(MVT::v2i64, MVT::v2i1, Expand); in AMDGPUTargetLowering()
291 setTruncStoreAction(MVT::v2i64, MVT::v2i8, Expand); in AMDGPUTargetLowering()
292 setTruncStoreAction(MVT::v2i64, MVT::v2i16, Expand); in AMDGPUTargetLowering()
293 setTruncStoreAction(MVT::v2i64, MVT::v2i32, Expand); in AMDGPUTargetLowering()
295 setTruncStoreAction(MVT::f32, MVT::bf16, Expand); in AMDGPUTargetLowering()
296 setTruncStoreAction(MVT::f32, MVT::f16, Expand); in AMDGPUTargetLowering()
297 setTruncStoreAction(MVT::v2f32, MVT::v2f16, Expand); in AMDGPUTargetLowering()
298 setTruncStoreAction(MVT::v3f32, MVT::v3f16, Expand); in AMDGPUTargetLowering()
299 setTruncStoreAction(MVT::v4f32, MVT::v4f16, Expand); in AMDGPUTargetLowering()
300 setTruncStoreAction(MVT::v8f32, MVT::v8f16, Expand); in AMDGPUTargetLowering()
301 setTruncStoreAction(MVT::v16f32, MVT::v16f16, Expand); in AMDGPUTargetLowering()
302 setTruncStoreAction(MVT::v32f32, MVT::v32f16, Expand); in AMDGPUTargetLowering()
304 setTruncStoreAction(MVT::f64, MVT::bf16, Expand); in AMDGPUTargetLowering()
305 setTruncStoreAction(MVT::f64, MVT::f16, Expand); in AMDGPUTargetLowering()
306 setTruncStoreAction(MVT::f64, MVT::f32, Expand); in AMDGPUTargetLowering()
308 setTruncStoreAction(MVT::v2f64, MVT::v2f32, Expand); in AMDGPUTargetLowering()
309 setTruncStoreAction(MVT::v2f64, MVT::v2f16, Expand); in AMDGPUTargetLowering()
311 setTruncStoreAction(MVT::v3i64, MVT::v3i32, Expand); in AMDGPUTargetLowering()
312 setTruncStoreAction(MVT::v3i64, MVT::v3i16, Expand); in AMDGPUTargetLowering()
313 setTruncStoreAction(MVT::v3f64, MVT::v3f32, Expand); in AMDGPUTargetLowering()
314 setTruncStoreAction(MVT::v3f64, MVT::v3f16, Expand); in AMDGPUTargetLowering()
316 setTruncStoreAction(MVT::v4i64, MVT::v4i32, Expand); in AMDGPUTargetLowering()
317 setTruncStoreAction(MVT::v4i64, MVT::v4i16, Expand); in AMDGPUTargetLowering()
318 setTruncStoreAction(MVT::v4f64, MVT::v4f32, Expand); in AMDGPUTargetLowering()
319 setTruncStoreAction(MVT::v4f64, MVT::v4f16, Expand); in AMDGPUTargetLowering()
321 setTruncStoreAction(MVT::v8f64, MVT::v8f32, Expand); in AMDGPUTargetLowering()
322 setTruncStoreAction(MVT::v8f64, MVT::v8f16, Expand); in AMDGPUTargetLowering()
324 setTruncStoreAction(MVT::v16f64, MVT::v16f32, Expand); in AMDGPUTargetLowering()
325 setTruncStoreAction(MVT::v16f64, MVT::v16f16, Expand); in AMDGPUTargetLowering()
326 setTruncStoreAction(MVT::v16i64, MVT::v16i16, Expand); in AMDGPUTargetLowering()
327 setTruncStoreAction(MVT::v16i64, MVT::v16i16, Expand); in AMDGPUTargetLowering()
328 setTruncStoreAction(MVT::v16i64, MVT::v16i8, Expand); in AMDGPUTargetLowering()
329 setTruncStoreAction(MVT::v16i64, MVT::v16i8, Expand); in AMDGPUTargetLowering()
330 setTruncStoreAction(MVT::v16i64, MVT::v16i1, Expand); in AMDGPUTargetLowering()
332 setOperationAction(ISD::Constant, {MVT::i32, MVT::i64}, Legal); in AMDGPUTargetLowering()
333 setOperationAction(ISD::ConstantFP, {MVT::f32, MVT::f64}, Legal); in AMDGPUTargetLowering()
335 setOperationAction({ISD::BR_JT, ISD::BRIND}, MVT::Other, Expand); in AMDGPUTargetLowering()
339 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); in AMDGPUTargetLowering()
345 MVT::f32, Legal); in AMDGPUTargetLowering()
347 setOperationAction(ISD::FLOG2, MVT::f32, Custom); in AMDGPUTargetLowering()
348 setOperationAction(ISD::FROUND, {MVT::f32, MVT::f64}, Custom); in AMDGPUTargetLowering()
351 {ISD::FLOG, ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FEXP10}, MVT::f32, in AMDGPUTargetLowering()
354 setOperationAction(ISD::FNEARBYINT, {MVT::f16, MVT::f32, MVT::f64}, Custom); in AMDGPUTargetLowering()
356 setOperationAction(ISD::FRINT, {MVT::f16, MVT::f32, MVT::f64}, Custom); in AMDGPUTargetLowering()
358 setOperationAction(ISD::FREM, {MVT::f16, MVT::f32, MVT::f64}, Custom); in AMDGPUTargetLowering()
361 setOperationAction(ISD::IS_FPCLASS, {MVT::f16, MVT::f32, MVT::f64}, Legal); in AMDGPUTargetLowering()
363 setOperationAction(ISD::IS_FPCLASS, {MVT::f32, MVT::f64}, Legal); in AMDGPUTargetLowering()
364 setOperationAction({ISD::FLOG2, ISD::FEXP2}, MVT::f16, Custom); in AMDGPUTargetLowering()
367 setOperationAction({ISD::FLOG10, ISD::FLOG, ISD::FEXP, ISD::FEXP10}, MVT::f16, in AMDGPUTargetLowering()
375 {MVT::v2f16, MVT::v3f16, MVT::v4f16, MVT::v16f16, MVT::v2f32, MVT::v3f32, in AMDGPUTargetLowering()
376 MVT::v4f32, MVT::v5f32, MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v16f32, in AMDGPUTargetLowering()
377 MVT::v2f64, MVT::v3f64, MVT::v4f64, MVT::v8f64, MVT::v16f64}, in AMDGPUTargetLowering()
381 setOperationAction(ISD::FSUB, MVT::f64, Expand); in AMDGPUTargetLowering()
384 {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32, in AMDGPUTargetLowering()
385 MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32, in AMDGPUTargetLowering()
386 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32, in AMDGPUTargetLowering()
387 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32, in AMDGPUTargetLowering()
388 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32}, in AMDGPUTargetLowering()
394 {MVT::v2f16, MVT::v2bf16, MVT::v2i16, MVT::v4f16, MVT::v4bf16, in AMDGPUTargetLowering()
395 MVT::v4i16, MVT::v2f32, MVT::v2i32, MVT::v3f32, MVT::v3i32, in AMDGPUTargetLowering()
396 MVT::v4f32, MVT::v4i32, MVT::v5f32, MVT::v5i32, MVT::v6f32, in AMDGPUTargetLowering()
397 MVT::v6i32, MVT::v7f32, MVT::v7i32, MVT::v8f32, MVT::v8i32, in AMDGPUTargetLowering()
398 MVT::v9f32, MVT::v9i32, MVT::v10i32, MVT::v10f32, MVT::v11i32, in AMDGPUTargetLowering()
399 MVT::v11f32, MVT::v12i32, MVT::v12f32, MVT::v16f16, MVT::v16bf16, in AMDGPUTargetLowering()
400 MVT::v16i16, MVT::v16f32, MVT::v16i32, MVT::v32f32, MVT::v32i32, in AMDGPUTargetLowering()
401 MVT::v2f64, MVT::v2i64, MVT::v3f64, MVT::v3i64, MVT::v4f64, in AMDGPUTargetLowering()
402 MVT::v4i64, MVT::v8f64, MVT::v8i64, MVT::v16f64, MVT::v16i64, in AMDGPUTargetLowering()
403 MVT::v32i16, MVT::v32f16, MVT::v32bf16}, in AMDGPUTargetLowering()
406 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); in AMDGPUTargetLowering()
407 setOperationAction(ISD::FP_TO_FP16, {MVT::f64, MVT::f32}, Custom); in AMDGPUTargetLowering()
409 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 }; in AMDGPUTargetLowering()
410 for (MVT VT : ScalarIntVTs) { in AMDGPUTargetLowering()
428 setOperationAction(ISD::FSHR, MVT::i32, Legal); in AMDGPUTargetLowering()
431 setOperationAction(ISD::ROTL, {MVT::i32, MVT::i64}, Expand); in AMDGPUTargetLowering()
432 setOperationAction(ISD::ROTR, MVT::i64, Expand); in AMDGPUTargetLowering()
434 setOperationAction({ISD::MULHU, ISD::MULHS}, MVT::i16, Expand); in AMDGPUTargetLowering()
436 setOperationAction({ISD::MUL, ISD::MULHU, ISD::MULHS}, MVT::i64, Expand); in AMDGPUTargetLowering()
439 MVT::i64, Custom); in AMDGPUTargetLowering()
440 setOperationAction(ISD::SELECT_CC, MVT::i64, Expand); in AMDGPUTargetLowering()
442 setOperationAction({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, MVT::i32, in AMDGPUTargetLowering()
447 MVT::i64, Custom); in AMDGPUTargetLowering()
449 for (auto VT : {MVT::i8, MVT::i16}) in AMDGPUTargetLowering()
452 static const MVT::SimpleValueType VectorIntTypes[] = { in AMDGPUTargetLowering()
453 MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, MVT::v6i32, MVT::v7i32, in AMDGPUTargetLowering()
454 MVT::v9i32, MVT::v10i32, MVT::v11i32, MVT::v12i32}; in AMDGPUTargetLowering()
456 for (MVT VT : VectorIntTypes) { in AMDGPUTargetLowering()
473 static const MVT::SimpleValueType FloatVectorTypes[] = { in AMDGPUTargetLowering()
474 MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32, MVT::v6f32, MVT::v7f32, in AMDGPUTargetLowering()
475 MVT::v9f32, MVT::v10f32, MVT::v11f32, MVT::v12f32}; in AMDGPUTargetLowering()
477 for (MVT VT : FloatVectorTypes) { in AMDGPUTargetLowering()
496 setOperationAction(ISD::SELECT, MVT::v2f32, Promote); in AMDGPUTargetLowering()
497 AddPromotedToType(ISD::SELECT, MVT::v2f32, MVT::v2i32); in AMDGPUTargetLowering()
499 setOperationAction(ISD::SELECT, MVT::v3f32, Promote); in AMDGPUTargetLowering()
500 AddPromotedToType(ISD::SELECT, MVT::v3f32, MVT::v3i32); in AMDGPUTargetLowering()
502 setOperationAction(ISD::SELECT, MVT::v4f32, Promote); in AMDGPUTargetLowering()
503 AddPromotedToType(ISD::SELECT, MVT::v4f32, MVT::v4i32); in AMDGPUTargetLowering()
505 setOperationAction(ISD::SELECT, MVT::v5f32, Promote); in AMDGPUTargetLowering()
506 AddPromotedToType(ISD::SELECT, MVT::v5f32, MVT::v5i32); in AMDGPUTargetLowering()
508 setOperationAction(ISD::SELECT, MVT::v6f32, Promote); in AMDGPUTargetLowering()
509 AddPromotedToType(ISD::SELECT, MVT::v6f32, MVT::v6i32); in AMDGPUTargetLowering()
511 setOperationAction(ISD::SELECT, MVT::v7f32, Promote); in AMDGPUTargetLowering()
512 AddPromotedToType(ISD::SELECT, MVT::v7f32, MVT::v7i32); in AMDGPUTargetLowering()
514 setOperationAction(ISD::SELECT, MVT::v9f32, Promote); in AMDGPUTargetLowering()
515 AddPromotedToType(ISD::SELECT, MVT::v9f32, MVT::v9i32); in AMDGPUTargetLowering()
517 setOperationAction(ISD::SELECT, MVT::v10f32, Promote); in AMDGPUTargetLowering()
518 AddPromotedToType(ISD::SELECT, MVT::v10f32, MVT::v10i32); in AMDGPUTargetLowering()
520 setOperationAction(ISD::SELECT, MVT::v11f32, Promote); in AMDGPUTargetLowering()
521 AddPromotedToType(ISD::SELECT, MVT::v11f32, MVT::v11i32); in AMDGPUTargetLowering()
523 setOperationAction(ISD::SELECT, MVT::v12f32, Promote); in AMDGPUTargetLowering()
524 AddPromotedToType(ISD::SELECT, MVT::v12f32, MVT::v12i32); in AMDGPUTargetLowering()
647 return BCSrc.getOpcode() == ISD::SELECT && BCSrc.getValueType() == MVT::f32; in fnegFoldsIntoOp()
657 static bool opMustUseVOP3Encoding(const SDNode *N, MVT VT) { in opMustUseVOP3Encoding()
659 VT == MVT::f64; in opMustUseVOP3Encoding()
667 return N->getValueType(0) == MVT::f32; in selectSupportsSourceMods()
718 MVT VT = N->getValueType(0).getScalarType().getSimpleVT(); in allUsesHaveSourceMods()
743 return MVT::i32; in getTypeForExtReturn()
747 MVT AMDGPUTargetLowering::getVectorIdxTy(const DataLayout &) const { in getVectorIdxTy()
748 return MVT::i32; in getVectorIdxTy()
760 return (ScalarVT == MVT::f32 || ScalarVT == MVT::f64 || in isFPImmLegal()
761 (ScalarVT == MVT::f16 && Subtarget->has16BitInsts())); in isFPImmLegal()
767 return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64); in ShouldShrinkFPConstant()
816 if (LoadTy.getScalarType() == MVT::i32) in isLoadBitCastBeneficial()
906 return VT == MVT::f32 || VT == MVT::f64 || in isFAbsFree()
907 (Subtarget->has16BitInsts() && VT == MVT::f16); in isFAbsFree()
914 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16; in isFNegFree()
972 if (Src == MVT::i16) in isZExtFree()
973 return Dest == MVT::i32 ||Dest == MVT::i64 ; in isZExtFree()
975 return Src == MVT::i32 && Dest == MVT::i64; in isZExtFree()
1000 if (N->getValueType(0) == MVT::i32 && N->use_size() == 1 && in isDesirableToCommuteWithShift()
1150 MVT RegisterVT = getRegisterTypeForCallingConv(Ctx, CC, ArgVT); in analyzeFormalArgumentsCompute()
1232 return DAG.getNode(AMDGPUISD::ENDPGM, DL, MVT::Other, Chain); in LowerReturn()
1281 return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); in addTokenForArgument()
1446 SDValue Trap = DAG.getNode(ISD::TRAP, DL, MVT::Other, DAG.getEntryNode()); in LowerGlobalAddress()
1447 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in LowerGlobalAddress()
1476 EVT NewEltVT = (NewNumElt == 1) ? MVT::i32 in LowerCONCAT_VECTORS()
1478 MVT::i32, NewNumElt); in LowerCONCAT_VECTORS()
1488 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, in LowerCONCAT_VECTORS()
1515 EVT NewSrcVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumSrcElt / 2); in LowerEXTRACT_SUBVECTOR()
1517 ? MVT::i32 in LowerEXTRACT_SUBVECTOR()
1518 : EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElt / 2); in LowerEXTRACT_SUBVECTOR()
1665 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in split64BitValue()
1667 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in split64BitValue()
1668 const SDValue One = DAG.getConstant(1, SL, MVT::i32); in split64BitValue()
1670 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero); in split64BitValue()
1671 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One); in split64BitValue()
1679 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in getLoHalf64()
1680 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in getLoHalf64()
1681 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero); in getLoHalf64()
1687 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in getHiHalf64()
1688 const SDValue One = DAG.getConstant(1, SL, MVT::i32); in getHiHalf64()
1689 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One); in getHiHalf64()
1780 SDValue Ops[] = {Join, DAG.getNode(ISD::TokenFactor, SL, MVT::Other, in SplitVectorLoad()
1858 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoStore, HiStore); in SplitVectorStore()
1870 MVT IntVT = MVT::i32; in LowerDIVREM24()
1871 MVT FltVT = MVT::f32; in LowerDIVREM24()
1985 assert(VT == MVT::i64 && "LowerUDIVREM64 expects an i64"); in LowerUDIVREM64()
2007 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(0), Zero}); in LowerUDIVREM64()
2008 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(1), Zero}); in LowerUDIVREM64()
2010 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV)); in LowerUDIVREM64()
2011 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM)); in LowerUDIVREM64()
2015 if (isTypeLegal(MVT::i64)) { in LowerUDIVREM64()
2029 SDValue Cvt_Lo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Lo); in LowerUDIVREM64()
2030 SDValue Cvt_Hi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Hi); in LowerUDIVREM64()
2031 SDValue Mad1 = DAG.getNode(FMAD, DL, MVT::f32, Cvt_Hi, in LowerUDIVREM64()
2032 DAG.getConstantFP(APInt(32, 0x4f800000).bitsToFloat(), DL, MVT::f32), in LowerUDIVREM64()
2034 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, DL, MVT::f32, Mad1); in LowerUDIVREM64()
2035 SDValue Mul1 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Rcp, in LowerUDIVREM64()
2036 DAG.getConstantFP(APInt(32, 0x5f7ffffc).bitsToFloat(), DL, MVT::f32)); in LowerUDIVREM64()
2037 SDValue Mul2 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Mul1, in LowerUDIVREM64()
2038 DAG.getConstantFP(APInt(32, 0x2f800000).bitsToFloat(), DL, MVT::f32)); in LowerUDIVREM64()
2039 SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, MVT::f32, Mul2); in LowerUDIVREM64()
2040 SDValue Mad2 = DAG.getNode(FMAD, DL, MVT::f32, Trunc, in LowerUDIVREM64()
2041 DAG.getConstantFP(APInt(32, 0xcf800000).bitsToFloat(), DL, MVT::f32), in LowerUDIVREM64()
2046 DAG.getBuildVector(MVT::v2i32, DL, {Rcp_Lo, Rcp_Hi})); in LowerUDIVREM64()
2050 SDValue Zero1 = DAG.getConstant(0, DL, MVT::i1); in LowerUDIVREM64()
2051 SDVTList HalfCarryVT = DAG.getVTList(HalfVT, MVT::i1); in LowerUDIVREM64()
2065 DAG.getBuildVector(MVT::v2i32, DL, {Add1_Lo, Add1_Hi})); in LowerUDIVREM64()
2078 DAG.getBuildVector(MVT::v2i32, DL, {Add2_Lo, Add2_Hi})); in LowerUDIVREM64()
2092 DAG.getBuildVector(MVT::v2i32, DL, {Sub1_Lo, Sub1_Hi})); in LowerUDIVREM64()
2113 DAG.getBuildVector(MVT::v2i32, DL, {Sub2_Lo, Sub2_Hi})); in LowerUDIVREM64()
2133 DAG.getBuildVector(MVT::v2i32, DL, {Sub3_Lo, Sub3_Hi})); in LowerUDIVREM64()
2156 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {REM_Lo, Zero}); in LowerUDIVREM64()
2157 REM = DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM); in LowerUDIVREM64()
2187 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {DIV_Lo, DIV_Hi}); in LowerUDIVREM64()
2188 DIV = DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV); in LowerUDIVREM64()
2198 if (VT == MVT::i64) { in LowerUDIVREM()
2204 if (VT == MVT::i32) { in LowerUDIVREM()
2259 if (VT == MVT::i32) { in LowerSDIVREM()
2264 if (VT == MVT::i64 && in LowerSDIVREM()
2331 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); in LowerFCEIL()
2333 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64); in LowerFCEIL()
2334 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64); in LowerFCEIL()
2337 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64); in LowerFCEIL()
2343 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero); in LowerFCEIL()
2345 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); in LowerFCEIL()
2353 SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32, in extractF64Exponent()
2355 DAG.getConstant(FractBits - 32, SL, MVT::i32), in extractF64Exponent()
2356 DAG.getConstant(ExpBits, SL, MVT::i32)); in extractF64Exponent()
2357 SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart, in extractF64Exponent()
2358 DAG.getConstant(1023, SL, MVT::i32)); in extractF64Exponent()
2367 assert(Op.getValueType() == MVT::f64); in LowerFTRUNC()
2369 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in LowerFTRUNC()
2380 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, SL, MVT::i32); in LowerFTRUNC()
2381 SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask); in LowerFTRUNC()
2384 SDValue SignBit64 = DAG.getBuildVector(MVT::v2i32, SL, {Zero, SignBit}); in LowerFTRUNC()
2385 SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64); in LowerFTRUNC()
2387 SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src); in LowerFTRUNC()
2389 = DAG.getConstant((UINT64_C(1) << FractBits) - 1, SL, MVT::i64); in LowerFTRUNC()
2391 SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp); in LowerFTRUNC()
2392 SDValue Not = DAG.getNOT(SL, Shr, MVT::i64); in LowerFTRUNC()
2393 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not); in LowerFTRUNC()
2396 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i32); in LowerFTRUNC()
2398 const SDValue FiftyOne = DAG.getConstant(FractBits - 1, SL, MVT::i32); in LowerFTRUNC()
2403 SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0); in LowerFTRUNC()
2404 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1); in LowerFTRUNC()
2406 return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2); in LowerFTRUNC()
2414 assert(Op.getValueType() == MVT::f64); in LowerFROUNDEVEN()
2417 SDValue C1 = DAG.getConstantFP(C1Val, SL, MVT::f64); in LowerFROUNDEVEN()
2418 SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src); in LowerFROUNDEVEN()
2422 SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign); in LowerFROUNDEVEN()
2423 SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign); in LowerFROUNDEVEN()
2425 SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src); in LowerFROUNDEVEN()
2428 SDValue C2 = DAG.getConstantFP(C2Val, SL, MVT::f64); in LowerFROUNDEVEN()
2431 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64); in LowerFROUNDEVEN()
2434 return DAG.getSelect(SL, MVT::f64, Cond, Src, Tmp2); in LowerFROUNDEVEN()
2492 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); in LowerFFLOOR()
2494 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64); in LowerFFLOOR()
2495 const SDValue NegOne = DAG.getConstantFP(-1.0, SL, MVT::f64); in LowerFFLOOR()
2498 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64); in LowerFFLOOR()
2504 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero); in LowerFFLOOR()
2506 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); in LowerFFLOOR()
2513 return Src.getOperand(0).getValueType() == MVT::f16; in valueIsKnownNeverF32Denorm()
2590 MVT VT = MVT::f32; in getScaledLogInput()
2620 if (VT == MVT::f16) { in LowerFLOG2()
2623 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src, Flags); in LowerFLOG2()
2624 SDValue Log = DAG.getNode(AMDGPUISD::LOG, SL, MVT::f32, Ext, Flags); in LowerFLOG2()
2626 DAG.getTargetConstant(0, SL, MVT::i32), Flags); in LowerFLOG2()
2660 if (VT == MVT::f16 || Flags.hasApproximateFuncs() || in LowerFLOGCommon()
2663 if (VT == MVT::f16 && !Subtarget->has16BitInsts()) { in LowerFLOGCommon()
2665 X = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, X, Flags); in LowerFLOGCommon()
2669 if (VT == MVT::f16 && !Subtarget->has16BitInsts()) { in LowerFLOGCommon()
2671 DAG.getTargetConstant(0, DL, MVT::i32), Flags); in LowerFLOGCommon()
2713 SDValue YAsInt = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Y); in LowerFLOGCommon()
2714 SDValue MaskConst = DAG.getConstant(0xfffff000, DL, MVT::i32); in LowerFLOGCommon()
2715 SDValue YHInt = DAG.getNode(ISD::AND, DL, MVT::i32, YAsInt, MaskConst); in LowerFLOGCommon()
2716 SDValue YH = DAG.getNode(ISD::BITCAST, DL, MVT::f32, YHInt); in LowerFLOGCommon()
2757 VT == MVT::f32 ? (unsigned)AMDGPUISD::LOG : (unsigned)ISD::FLOG2; in LowerFLOGUnsafe()
2762 if (VT == MVT::f32) { in LowerFLOGUnsafe()
2800 if (VT == MVT::f16) { in lowerFEXP2()
2803 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src, Flags); in lowerFEXP2()
2804 SDValue Log = DAG.getNode(AMDGPUISD::EXP, SL, MVT::f32, Ext, Flags); in lowerFEXP2()
2806 DAG.getTargetConstant(0, SL, MVT::i32), Flags); in lowerFEXP2()
2809 assert(VT == MVT::f32); in lowerFEXP2()
2812 return DAG.getNode(AMDGPUISD::EXP, SL, MVT::f32, Src, Flags); in lowerFEXP2()
2848 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, X, Flags)) { in lowerFEXPUnsafe()
2851 return DAG.getNode(VT == MVT::f32 ? (unsigned)AMDGPUISD::EXP in lowerFEXPUnsafe()
2886 const unsigned Exp2Op = VT == MVT::f32 ? AMDGPUISD::EXP : ISD::FEXP2; in lowerFEXP10Unsafe()
2888 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, X, Flags)) { in lowerFEXP10Unsafe()
2941 if (VT.getScalarType() == MVT::f16) { in lowerFEXP()
2953 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, X, Flags); in lowerFEXP()
2956 DAG.getTargetConstant(0, SL, MVT::i32), Flags); in lowerFEXP()
2959 assert(VT == MVT::f32); in lowerFEXP()
3019 SDValue XAsInt = DAG.getNode(ISD::BITCAST, SL, MVT::i32, X); in lowerFEXP()
3020 SDValue MaskConst = DAG.getConstant(0xfffff000, SL, MVT::i32); in lowerFEXP()
3021 SDValue XHAsInt = DAG.getNode(ISD::AND, SL, MVT::i32, XAsInt, MaskConst); in lowerFEXP()
3038 SDValue IntE = DAG.getNode(ISD::FP_TO_SINT, SL, MVT::i32, E); in lowerFEXP()
3081 if (ResultVT != MVT::i8 && ResultVT != MVT::i16) in lowerCTLZResults()
3088 auto NewOp = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Arg); in lowerCTLZResults()
3089 auto ShiftVal = DAG.getConstant(LeadingZeroes, SL, MVT::i32); in lowerCTLZResults()
3090 NewOp = DAG.getNode(ISD::SHL, SL, MVT::i32, NewOp, ShiftVal); in lowerCTLZResults()
3091 NewOp = DAG.getNode(Op.getOpcode(), SL, MVT::i32, NewOp); in lowerCTLZResults()
3105 bool Is64BitScalar = !Src->isDivergent() && Src.getValueType() == MVT::i64; in LowerCTLZ_CTTZ()
3107 if (Src.getValueType() == MVT::i32 || Is64BitScalar) { in LowerCTLZ_CTTZ()
3118 SDValue NewOpr = DAG.getNode(NewOpc, SL, MVT::i32, Src); in LowerCTLZ_CTTZ()
3121 Op.getValueType().getScalarSizeInBits(), SL, MVT::i32); in LowerCTLZ_CTTZ()
3122 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, ConstVal); in LowerCTLZ_CTTZ()
3130 SDValue OprLo = DAG.getNode(NewOpc, SL, MVT::i32, Lo); in LowerCTLZ_CTTZ()
3131 SDValue OprHi = DAG.getNode(NewOpc, SL, MVT::i32, Hi); in LowerCTLZ_CTTZ()
3139 const SDValue Const32 = DAG.getConstant(32, SL, MVT::i32); in LowerCTLZ_CTTZ()
3141 OprLo = DAG.getNode(AddOpc, SL, MVT::i32, OprLo, Const32); in LowerCTLZ_CTTZ()
3143 OprHi = DAG.getNode(AddOpc, SL, MVT::i32, OprHi, Const32); in LowerCTLZ_CTTZ()
3146 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, OprLo, OprHi); in LowerCTLZ_CTTZ()
3148 const SDValue Const64 = DAG.getConstant(64, SL, MVT::i32); in LowerCTLZ_CTTZ()
3149 NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, Const64); in LowerCTLZ_CTTZ()
3152 return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i64, NewOpr); in LowerCTLZ_CTTZ()
3213 ISD::SRA, SL, MVT::i32, DAG.getNode(ISD::XOR, SL, MVT::i32, Lo, Hi), in LowerINT_TO_FP32()
3214 DAG.getConstant(31, SL, MVT::i32)); in LowerINT_TO_FP32()
3216 DAG.getNode(ISD::ADD, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32), in LowerINT_TO_FP32()
3219 ShAmt = DAG.getNode(AMDGPUISD::FFBH_I32, SL, MVT::i32, Hi); in LowerINT_TO_FP32()
3222 ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, ShAmt, in LowerINT_TO_FP32()
3223 DAG.getConstant(1, SL, MVT::i32)); in LowerINT_TO_FP32()
3224 ShAmt = DAG.getNode(ISD::UMIN, SL, MVT::i32, ShAmt, MaxShAmt); in LowerINT_TO_FP32()
3229 Sign = DAG.getNode(ISD::SRA, SL, MVT::i64, Src, in LowerINT_TO_FP32()
3230 DAG.getConstant(63, SL, MVT::i64)); in LowerINT_TO_FP32()
3232 DAG.getNode(ISD::XOR, SL, MVT::i64, in LowerINT_TO_FP32()
3233 DAG.getNode(ISD::ADD, SL, MVT::i64, Src, Sign), Sign); in LowerINT_TO_FP32()
3237 ShAmt = DAG.getNode(ISD::CTLZ, SL, MVT::i32, Hi); in LowerINT_TO_FP32()
3241 SDValue Norm = DAG.getNode(ISD::SHL, SL, MVT::i64, Src, ShAmt); in LowerINT_TO_FP32()
3246 SDValue Adjust = DAG.getNode(ISD::UMIN, SL, MVT::i32, in LowerINT_TO_FP32()
3247 DAG.getConstant(1, SL, MVT::i32), Lo); in LowerINT_TO_FP32()
3249 Norm = DAG.getNode(ISD::OR, SL, MVT::i32, Hi, Adjust); in LowerINT_TO_FP32()
3253 SDValue FVal = DAG.getNode(Opc, SL, MVT::f32, Norm); in LowerINT_TO_FP32()
3257 ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32), in LowerINT_TO_FP32()
3261 return DAG.getNode(ISD::FLDEXP, SL, MVT::f32, FVal, ShAmt); in LowerINT_TO_FP32()
3266 SDValue Exp = DAG.getNode(ISD::SHL, SL, MVT::i32, ShAmt, in LowerINT_TO_FP32()
3267 DAG.getConstant(23, SL, MVT::i32)); in LowerINT_TO_FP32()
3269 DAG.getNode(ISD::ADD, SL, MVT::i32, in LowerINT_TO_FP32()
3270 DAG.getNode(ISD::BITCAST, SL, MVT::i32, FVal), Exp); in LowerINT_TO_FP32()
3273 Sign = DAG.getNode(ISD::SHL, SL, MVT::i32, in LowerINT_TO_FP32()
3274 DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Sign), in LowerINT_TO_FP32()
3275 DAG.getConstant(31, SL, MVT::i32)); in LowerINT_TO_FP32()
3276 IVal = DAG.getNode(ISD::OR, SL, MVT::i32, IVal, Sign); in LowerINT_TO_FP32()
3278 return DAG.getNode(ISD::BITCAST, SL, MVT::f32, IVal); in LowerINT_TO_FP32()
3290 SL, MVT::f64, Hi); in LowerINT_TO_FP64()
3292 SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo); in LowerINT_TO_FP64()
3294 SDValue LdExp = DAG.getNode(ISD::FLDEXP, SL, MVT::f64, CvtHi, in LowerINT_TO_FP64()
3295 DAG.getConstant(32, SL, MVT::i32)); in LowerINT_TO_FP64()
3297 return DAG.getNode(ISD::FADD, SL, MVT::f64, LdExp, CvtLo); in LowerINT_TO_FP64()
3307 if (SrcVT == MVT::i16) { in LowerUINT_TO_FP()
3308 if (DestVT == MVT::f16) in LowerUINT_TO_FP()
3313 SDValue Ext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Src); in LowerUINT_TO_FP()
3317 if (DestVT == MVT::bf16) { in LowerUINT_TO_FP()
3319 SDValue ToF32 = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f32, Src); in LowerUINT_TO_FP()
3321 return DAG.getNode(ISD::FP_ROUND, SL, MVT::bf16, ToF32, FPRoundFlag); in LowerUINT_TO_FP()
3324 if (SrcVT != MVT::i64) in LowerUINT_TO_FP()
3327 if (Subtarget->has16BitInsts() && DestVT == MVT::f16) { in LowerUINT_TO_FP()
3330 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src); in LowerUINT_TO_FP()
3334 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag); in LowerUINT_TO_FP()
3339 if (DestVT == MVT::f32) in LowerUINT_TO_FP()
3342 assert(DestVT == MVT::f64); in LowerUINT_TO_FP()
3353 if (SrcVT == MVT::i16) { in LowerSINT_TO_FP()
3354 if (DestVT == MVT::f16) in LowerSINT_TO_FP()
3359 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32, Src); in LowerSINT_TO_FP()
3363 if (DestVT == MVT::bf16) { in LowerSINT_TO_FP()
3365 SDValue ToF32 = DAG.getNode(ISD::SINT_TO_FP, SL, MVT::f32, Src); in LowerSINT_TO_FP()
3367 return DAG.getNode(ISD::FP_ROUND, SL, MVT::bf16, ToF32, FPRoundFlag); in LowerSINT_TO_FP()
3370 if (SrcVT != MVT::i64) in LowerSINT_TO_FP()
3375 if (Subtarget->has16BitInsts() && DestVT == MVT::f16) { in LowerSINT_TO_FP()
3379 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src); in LowerSINT_TO_FP()
3383 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag); in LowerSINT_TO_FP()
3388 if (DestVT == MVT::f32) in LowerSINT_TO_FP()
3391 assert(DestVT == MVT::f64); in LowerSINT_TO_FP()
3402 assert(SrcVT == MVT::f32 || SrcVT == MVT::f64); in LowerFP_TO_INT64()
3415 if (Signed && SrcVT == MVT::f32) { in LowerFP_TO_INT64()
3421 Sign = DAG.getNode(ISD::SRA, SL, MVT::i32, in LowerFP_TO_INT64()
3422 DAG.getNode(ISD::BITCAST, SL, MVT::i32, Trunc), in LowerFP_TO_INT64()
3423 DAG.getConstant(31, SL, MVT::i32)); in LowerFP_TO_INT64()
3428 if (SrcVT == MVT::f64) { in LowerFP_TO_INT64()
3448 SDValue Hi = DAG.getNode((Signed && SrcVT == MVT::f64) ? ISD::FP_TO_SINT in LowerFP_TO_INT64()
3450 SL, MVT::i32, FloorMul); in LowerFP_TO_INT64()
3451 SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, SL, MVT::i32, Fma); in LowerFP_TO_INT64()
3453 SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i64, in LowerFP_TO_INT64()
3454 DAG.getBuildVector(MVT::v2i32, SL, {Lo, Hi})); in LowerFP_TO_INT64()
3456 if (Signed && SrcVT == MVT::f32) { in LowerFP_TO_INT64()
3459 Sign = DAG.getNode(ISD::BITCAST, SL, MVT::i64, in LowerFP_TO_INT64()
3460 DAG.getBuildVector(MVT::v2i32, SL, {Sign, Sign})); in LowerFP_TO_INT64()
3463 DAG.getNode(ISD::SUB, SL, MVT::i64, in LowerFP_TO_INT64()
3464 DAG.getNode(ISD::XOR, SL, MVT::i64, Result, Sign), Sign); in LowerFP_TO_INT64()
3475 if (N0.getValueType() == MVT::f32) in LowerFP_TO_FP16()
3483 assert(N0.getSimpleValueType() == MVT::f64); in LowerFP_TO_FP16()
3489 SDValue Zero = DAG.getConstant(0, DL, MVT::i32); in LowerFP_TO_FP16()
3490 SDValue One = DAG.getConstant(1, DL, MVT::i32); in LowerFP_TO_FP16()
3491 SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, N0); in LowerFP_TO_FP16()
3492 SDValue UH = DAG.getNode(ISD::SRL, DL, MVT::i64, U, in LowerFP_TO_FP16()
3493 DAG.getConstant(32, DL, MVT::i64)); in LowerFP_TO_FP16()
3494 UH = DAG.getZExtOrTrunc(UH, DL, MVT::i32); in LowerFP_TO_FP16()
3495 U = DAG.getZExtOrTrunc(U, DL, MVT::i32); in LowerFP_TO_FP16()
3496 SDValue E = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, in LowerFP_TO_FP16()
3497 DAG.getConstant(20, DL, MVT::i64)); in LowerFP_TO_FP16()
3498 E = DAG.getNode(ISD::AND, DL, MVT::i32, E, in LowerFP_TO_FP16()
3499 DAG.getConstant(ExpMask, DL, MVT::i32)); in LowerFP_TO_FP16()
3502 E = DAG.getNode(ISD::ADD, DL, MVT::i32, E, in LowerFP_TO_FP16()
3503 DAG.getConstant(-ExpBiasf64 + ExpBiasf16, DL, MVT::i32)); in LowerFP_TO_FP16()
3505 SDValue M = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, in LowerFP_TO_FP16()
3506 DAG.getConstant(8, DL, MVT::i32)); in LowerFP_TO_FP16()
3507 M = DAG.getNode(ISD::AND, DL, MVT::i32, M, in LowerFP_TO_FP16()
3508 DAG.getConstant(0xffe, DL, MVT::i32)); in LowerFP_TO_FP16()
3510 SDValue MaskedSig = DAG.getNode(ISD::AND, DL, MVT::i32, UH, in LowerFP_TO_FP16()
3511 DAG.getConstant(0x1ff, DL, MVT::i32)); in LowerFP_TO_FP16()
3512 MaskedSig = DAG.getNode(ISD::OR, DL, MVT::i32, MaskedSig, U); in LowerFP_TO_FP16()
3515 M = DAG.getNode(ISD::OR, DL, MVT::i32, M, Lo40Set); in LowerFP_TO_FP16()
3518 SDValue I = DAG.getNode(ISD::OR, DL, MVT::i32, in LowerFP_TO_FP16()
3519 DAG.getSelectCC(DL, M, Zero, DAG.getConstant(0x0200, DL, MVT::i32), in LowerFP_TO_FP16()
3520 Zero, ISD::SETNE), DAG.getConstant(0x7c00, DL, MVT::i32)); in LowerFP_TO_FP16()
3523 SDValue N = DAG.getNode(ISD::OR, DL, MVT::i32, M, in LowerFP_TO_FP16()
3524 DAG.getNode(ISD::SHL, DL, MVT::i32, E, in LowerFP_TO_FP16()
3525 DAG.getConstant(12, DL, MVT::i32))); in LowerFP_TO_FP16()
3528 SDValue OneSubExp = DAG.getNode(ISD::SUB, DL, MVT::i32, in LowerFP_TO_FP16()
3530 SDValue B = DAG.getNode(ISD::SMAX, DL, MVT::i32, OneSubExp, Zero); in LowerFP_TO_FP16()
3531 B = DAG.getNode(ISD::SMIN, DL, MVT::i32, B, in LowerFP_TO_FP16()
3532 DAG.getConstant(13, DL, MVT::i32)); in LowerFP_TO_FP16()
3534 SDValue SigSetHigh = DAG.getNode(ISD::OR, DL, MVT::i32, M, in LowerFP_TO_FP16()
3535 DAG.getConstant(0x1000, DL, MVT::i32)); in LowerFP_TO_FP16()
3537 SDValue D = DAG.getNode(ISD::SRL, DL, MVT::i32, SigSetHigh, B); in LowerFP_TO_FP16()
3538 SDValue D0 = DAG.getNode(ISD::SHL, DL, MVT::i32, D, B); in LowerFP_TO_FP16()
3540 D = DAG.getNode(ISD::OR, DL, MVT::i32, D, D1); in LowerFP_TO_FP16()
3543 SDValue VLow3 = DAG.getNode(ISD::AND, DL, MVT::i32, V, in LowerFP_TO_FP16()
3544 DAG.getConstant(0x7, DL, MVT::i32)); in LowerFP_TO_FP16()
3545 V = DAG.getNode(ISD::SRL, DL, MVT::i32, V, in LowerFP_TO_FP16()
3546 DAG.getConstant(2, DL, MVT::i32)); in LowerFP_TO_FP16()
3547 SDValue V0 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(3, DL, MVT::i32), in LowerFP_TO_FP16()
3549 SDValue V1 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(5, DL, MVT::i32), in LowerFP_TO_FP16()
3551 V1 = DAG.getNode(ISD::OR, DL, MVT::i32, V0, V1); in LowerFP_TO_FP16()
3552 V = DAG.getNode(ISD::ADD, DL, MVT::i32, V, V1); in LowerFP_TO_FP16()
3554 V = DAG.getSelectCC(DL, E, DAG.getConstant(30, DL, MVT::i32), in LowerFP_TO_FP16()
3555 DAG.getConstant(0x7c00, DL, MVT::i32), V, ISD::SETGT); in LowerFP_TO_FP16()
3556 V = DAG.getSelectCC(DL, E, DAG.getConstant(1039, DL, MVT::i32), in LowerFP_TO_FP16()
3560 SDValue Sign = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, in LowerFP_TO_FP16()
3561 DAG.getConstant(16, DL, MVT::i32)); in LowerFP_TO_FP16()
3562 Sign = DAG.getNode(ISD::AND, DL, MVT::i32, Sign, in LowerFP_TO_FP16()
3563 DAG.getConstant(0x8000, DL, MVT::i32)); in LowerFP_TO_FP16()
3565 V = DAG.getNode(ISD::OR, DL, MVT::i32, Sign, V); in LowerFP_TO_FP16()
3577 if (SrcVT == MVT::f16 && DestVT == MVT::i16) in LowerFP_TO_INT()
3580 if (SrcVT == MVT::bf16) { in LowerFP_TO_INT()
3582 SDValue PromotedSrc = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src); in LowerFP_TO_INT()
3587 if (DestVT == MVT::i16 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) { in LowerFP_TO_INT()
3590 SDValue FpToInt32 = DAG.getNode(OpOpcode, DL, MVT::i32, Src); in LowerFP_TO_INT()
3591 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToInt32); in LowerFP_TO_INT()
3594 if (DestVT != MVT::i64) in LowerFP_TO_INT()
3597 if (SrcVT == MVT::f16 || in LowerFP_TO_INT()
3598 (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP)) { in LowerFP_TO_INT()
3601 SDValue FpToInt32 = DAG.getNode(OpOpcode, DL, MVT::i32, Src); in LowerFP_TO_INT()
3604 return DAG.getNode(Ext, DL, MVT::i64, FpToInt32); in LowerFP_TO_INT()
3607 if (SrcVT == MVT::f32 || SrcVT == MVT::f64) in LowerFP_TO_INT()
3616 MVT VT = Op.getSimpleValueType(); in LowerSIGN_EXTEND_INREG()
3617 MVT ScalarVT = VT.getScalarType(); in LowerSIGN_EXTEND_INREG()
3708 return DAG.getConstant(Result, DL, MVT::i32); in constantFoldBFE()
3711 return DAG.getConstant(Src0 >> Offset, DL, MVT::i32); in constantFoldBFE()
3727 if (VT.getScalarType() == MVT::i32 || isTypeLegal(VT)) in shouldCombineMemoryType()
3923 SDValue LoRHS = DAG.getConstant(ValLo, SL, MVT::i32); in splitBinaryBitConstantOpImpl()
3924 SDValue HiRHS = DAG.getConstant(ValHi, SL, MVT::i32); in splitBinaryBitConstantOpImpl()
3926 SDValue LoAnd = DAG.getNode(Opc, SL, MVT::i32, Lo, LoRHS); in splitBinaryBitConstantOpImpl()
3927 SDValue HiAnd = DAG.getNode(Opc, SL, MVT::i32, Hi, HiRHS); in splitBinaryBitConstantOpImpl()
3934 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {LoAnd, HiAnd}); in splitBinaryBitConstantOpImpl()
3935 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in splitBinaryBitConstantOpImpl()
3962 if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 && in performShlCombine()
3963 isOperationLegal(ISD::BUILD_VECTOR, MVT::v2i16)) { in performShlCombine()
3966 SDValue Vec = DAG.getBuildVector(MVT::v2i16, SL, in performShlCombine()
3967 { DAG.getConstant(0, SL, MVT::i16), LHS->getOperand(0) }); in performShlCombine()
3968 return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Vec); in performShlCombine()
3972 if (VT != MVT::i64) in performShlCombine()
3984 if (VT != MVT::i64) in performShlCombine()
3995 SDValue ShiftAmt = DAG.getConstant(RHSVal - 32, SL, MVT::i32); in performShlCombine()
3997 SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS); in performShlCombine()
3998 SDValue NewShift = DAG.getNode(ISD::SHL, SL, MVT::i32, Lo, ShiftAmt); in performShlCombine()
4000 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in performShlCombine()
4002 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {Zero, NewShift}); in performShlCombine()
4003 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in performShlCombine()
4008 if (N->getValueType(0) != MVT::i64) in performSraCombine()
4022 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi, in performSraCombine()
4023 DAG.getConstant(31, SL, MVT::i32)); in performSraCombine()
4025 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {Hi, NewShift}); in performSraCombine()
4026 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec); in performSraCombine()
4032 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi, in performSraCombine()
4033 DAG.getConstant(31, SL, MVT::i32)); in performSraCombine()
4034 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, NewShift}); in performSraCombine()
4035 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec); in performSraCombine()
4068 if (VT != MVT::i64) in performSrlCombine()
4077 SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in performSrlCombine()
4081 SDValue NewConst = DAG.getConstant(ShiftAmt - 32, SL, MVT::i32); in performSrlCombine()
4082 SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst); in performSrlCombine()
4084 SDValue BuildPair = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, Zero}); in performSrlCombine()
4086 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair); in performSrlCombine()
4156 EVT::getVectorVT(*DAG.getContext(), MVT::i32, in performTruncateCombine()
4157 VT.getVectorNumElements()) : MVT::i32; in performTruncateCombine()
4187 return DAG.getNode(MulOpc, SL, MVT::i32, N0, N1); in getMul24()
4193 SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1); in getMul24()
4194 SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1); in getMul24()
4196 return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, MulLo, MulHi); in getMul24()
4263 if (Subtarget->has16BitInsts() && VT.getScalarType().bitsLE(MVT::i16)) in performMulCombine()
4279 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); in performMulCombine()
4280 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); in performMulCombine()
4283 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); in performMulCombine()
4284 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); in performMulCombine()
4298 if (N->getValueType(0) != MVT::i32) in performMulLoHiCombine()
4320 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); in performMulLoHiCombine()
4321 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); in performMulLoHiCombine()
4325 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); in performMulLoHiCombine()
4326 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); in performMulLoHiCombine()
4333 SDValue Lo = DAG.getNode(LoOpcode, DL, MVT::i32, N0, N1); in performMulLoHiCombine()
4334 SDValue Hi = DAG.getNode(HiOpcode, DL, MVT::i32, N0, N1); in performMulLoHiCombine()
4364 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); in performMulhsCombine()
4365 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); in performMulhsCombine()
4367 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_I24, DL, MVT::i32, N0, N1); in performMulhsCombine()
4397 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); in performMulhuCombine()
4398 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); in performMulhuCombine()
4400 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_U24, DL, MVT::i32, N0, N1); in performMulhuCombine()
4411 if (LegalVT != MVT::i32 && (Subtarget->has16BitInsts() && in getFFBX_U32()
4412 LegalVT != MVT::i16)) in getFFBX_U32()
4415 if (VT != MVT::i32) in getFFBX_U32()
4416 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Op); in getFFBX_U32()
4418 SDValue FFBX = DAG.getNode(Opc, DL, MVT::i32, Op); in getFFBX_U32()
4419 if (VT != MVT::i32) in getFFBX_U32()
4599 if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) { in performSelectCombine()
4910 SDValue CastHi = DAG.getNode(ISD::BITCAST, SL, MVT::f32, HighBits); in performFNegCombine()
4911 SDValue NegHi = DAG.getNode(ISD::FNEG, SL, MVT::f32, CastHi); in performFNegCombine()
4927 if (BCSrc.getOpcode() == ISD::SELECT && VT == MVT::f32 && in performFNegCombine()
4935 DAG.getNode(ISD::BITCAST, SL, MVT::f32, BCSrc.getOperand(1)); in performFNegCombine()
4937 DAG.getNode(ISD::BITCAST, SL, MVT::f32, BCSrc.getOperand(2)); in performFNegCombine()
4939 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, MVT::f32, LHS); in performFNegCombine()
4940 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, MVT::f32, RHS); in performFNegCombine()
4942 return DAG.getNode(ISD::SELECT, SL, MVT::f32, BCSrc.getOperand(0), NegLHS, in performFNegCombine()
5040 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in PerformDAGCombine()
5041 DAG.getConstant(Lo_32(CVal), SL, MVT::i32), in PerformDAGCombine()
5042 DAG.getConstant(Hi_32(CVal), SL, MVT::i32)); in PerformDAGCombine()
5050 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in PerformDAGCombine()
5051 DAG.getConstant(Lo_32(CVal), SL, MVT::i32), in PerformDAGCombine()
5052 DAG.getConstant(Hi_32(CVal), SL, MVT::i32)); in PerformDAGCombine()
5113 return DAG.getConstant(0, DL, MVT::i32); in PerformDAGCombine()
5140 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, BitsFrom, in PerformDAGCombine()
5165 SDValue ShiftVal = DAG.getConstant(OffsetVal, DL, MVT::i32); in PerformDAGCombine()
5166 return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32, in PerformDAGCombine()
5282 SDValue Ptr = DAG.getFrameIndex(FI, MVT::i32); in loadStackInputValue()
5298 SDValue Ptr = DAG.getConstant(Offset, SL, MVT::i32); in storeStackInputValue()
5301 DAG.getCopyFromReg(Chain, SL, Info->getStackPtrOffsetReg(), MVT::i32); in storeStackInputValue()
5302 Ptr = DAG.getNode(ISD::ADD, SL, MVT::i32, SP, Ptr); in storeStackInputValue()
5527 if (VT == MVT::f32) { in getSqrtEstimate()
5543 if (VT == MVT::f32) { in getRecipEstimate()