Lines Matching refs:MVT
44 static bool allocateCCRegs(unsigned ValNo, MVT ValVT, MVT LocVT, in allocateCCRegs()
58 static bool allocateSGPRTuple(unsigned ValNo, MVT ValVT, MVT LocVT, in allocateSGPRTuple()
62 case MVT::i64: in allocateSGPRTuple()
63 case MVT::f64: in allocateSGPRTuple()
64 case MVT::v2i32: in allocateSGPRTuple()
65 case MVT::v2f32: in allocateSGPRTuple()
66 case MVT::v4i16: in allocateSGPRTuple()
67 case MVT::v4f16: { in allocateSGPRTuple()
81 static bool allocateVGPRTuple(unsigned ValNo, MVT ValVT, MVT LocVT, in allocateVGPRTuple()
85 case MVT::i64: in allocateVGPRTuple()
86 case MVT::f64: in allocateVGPRTuple()
87 case MVT::v2i32: in allocateVGPRTuple()
88 case MVT::v2f32: in allocateVGPRTuple()
89 case MVT::v4i16: in allocateVGPRTuple()
90 case MVT::v4f16: { in allocateVGPRTuple()
94 case MVT::v4i32: in allocateVGPRTuple()
95 case MVT::v4f32: in allocateVGPRTuple()
96 case MVT::v2i64: in allocateVGPRTuple()
97 case MVT::v2f64: { in allocateVGPRTuple()
101 case MVT::v8i32: in allocateVGPRTuple()
102 case MVT::v8f32: { in allocateVGPRTuple()
107 case MVT::v16i32: in allocateVGPRTuple()
108 case MVT::v16f32: { in allocateVGPRTuple()
127 return EVT::getVectorVT(Ctx, MVT::i32, StoreSize / 32); in getEquivalentMemType()
149 setOperationAction(ISD::LOAD, MVT::f32, Promote); in AMDGPUTargetLowering()
150 AddPromotedToType(ISD::LOAD, MVT::f32, MVT::i32); in AMDGPUTargetLowering()
152 setOperationAction(ISD::LOAD, MVT::v2f32, Promote); in AMDGPUTargetLowering()
153 AddPromotedToType(ISD::LOAD, MVT::v2f32, MVT::v2i32); in AMDGPUTargetLowering()
155 setOperationAction(ISD::LOAD, MVT::v4f32, Promote); in AMDGPUTargetLowering()
156 AddPromotedToType(ISD::LOAD, MVT::v4f32, MVT::v4i32); in AMDGPUTargetLowering()
158 setOperationAction(ISD::LOAD, MVT::v8f32, Promote); in AMDGPUTargetLowering()
159 AddPromotedToType(ISD::LOAD, MVT::v8f32, MVT::v8i32); in AMDGPUTargetLowering()
161 setOperationAction(ISD::LOAD, MVT::v16f32, Promote); in AMDGPUTargetLowering()
162 AddPromotedToType(ISD::LOAD, MVT::v16f32, MVT::v16i32); in AMDGPUTargetLowering()
164 setOperationAction(ISD::LOAD, MVT::i64, Promote); in AMDGPUTargetLowering()
165 AddPromotedToType(ISD::LOAD, MVT::i64, MVT::v2i32); in AMDGPUTargetLowering()
167 setOperationAction(ISD::LOAD, MVT::v2i64, Promote); in AMDGPUTargetLowering()
168 AddPromotedToType(ISD::LOAD, MVT::v2i64, MVT::v4i32); in AMDGPUTargetLowering()
170 setOperationAction(ISD::LOAD, MVT::f64, Promote); in AMDGPUTargetLowering()
171 AddPromotedToType(ISD::LOAD, MVT::f64, MVT::v2i32); in AMDGPUTargetLowering()
173 setOperationAction(ISD::LOAD, MVT::v2f64, Promote); in AMDGPUTargetLowering()
174 AddPromotedToType(ISD::LOAD, MVT::v2f64, MVT::v4i32); in AMDGPUTargetLowering()
178 for (MVT VT : MVT::integer_valuetypes()) { in AMDGPUTargetLowering()
179 setLoadExtAction(ISD::EXTLOAD, MVT::i64, VT, Expand); in AMDGPUTargetLowering()
180 setLoadExtAction(ISD::SEXTLOAD, MVT::i64, VT, Expand); in AMDGPUTargetLowering()
181 setLoadExtAction(ISD::ZEXTLOAD, MVT::i64, VT, Expand); in AMDGPUTargetLowering()
184 for (MVT VT : MVT::integer_valuetypes()) { in AMDGPUTargetLowering()
185 if (VT == MVT::i64) in AMDGPUTargetLowering()
188 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); in AMDGPUTargetLowering()
189 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Legal); in AMDGPUTargetLowering()
190 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i16, Legal); in AMDGPUTargetLowering()
191 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i32, Expand); in AMDGPUTargetLowering()
193 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote); in AMDGPUTargetLowering()
194 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i8, Legal); in AMDGPUTargetLowering()
195 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i16, Legal); in AMDGPUTargetLowering()
196 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i32, Expand); in AMDGPUTargetLowering()
198 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote); in AMDGPUTargetLowering()
199 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i8, Legal); in AMDGPUTargetLowering()
200 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i16, Legal); in AMDGPUTargetLowering()
201 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i32, Expand); in AMDGPUTargetLowering()
204 for (MVT VT : MVT::integer_vector_valuetypes()) { in AMDGPUTargetLowering()
205 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Expand); in AMDGPUTargetLowering()
206 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Expand); in AMDGPUTargetLowering()
207 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v2i8, Expand); in AMDGPUTargetLowering()
208 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i8, Expand); in AMDGPUTargetLowering()
209 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i8, Expand); in AMDGPUTargetLowering()
210 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v4i8, Expand); in AMDGPUTargetLowering()
211 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i16, Expand); in AMDGPUTargetLowering()
212 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i16, Expand); in AMDGPUTargetLowering()
213 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v2i16, Expand); in AMDGPUTargetLowering()
214 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i16, Expand); in AMDGPUTargetLowering()
215 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i16, Expand); in AMDGPUTargetLowering()
216 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v4i16, Expand); in AMDGPUTargetLowering()
219 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand); in AMDGPUTargetLowering()
220 setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2f16, Expand); in AMDGPUTargetLowering()
221 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Expand); in AMDGPUTargetLowering()
222 setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Expand); in AMDGPUTargetLowering()
224 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand); in AMDGPUTargetLowering()
225 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Expand); in AMDGPUTargetLowering()
226 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Expand); in AMDGPUTargetLowering()
227 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f32, Expand); in AMDGPUTargetLowering()
229 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand); in AMDGPUTargetLowering()
230 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Expand); in AMDGPUTargetLowering()
231 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Expand); in AMDGPUTargetLowering()
232 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Expand); in AMDGPUTargetLowering()
234 setOperationAction(ISD::STORE, MVT::f32, Promote); in AMDGPUTargetLowering()
235 AddPromotedToType(ISD::STORE, MVT::f32, MVT::i32); in AMDGPUTargetLowering()
237 setOperationAction(ISD::STORE, MVT::v2f32, Promote); in AMDGPUTargetLowering()
238 AddPromotedToType(ISD::STORE, MVT::v2f32, MVT::v2i32); in AMDGPUTargetLowering()
240 setOperationAction(ISD::STORE, MVT::v4f32, Promote); in AMDGPUTargetLowering()
241 AddPromotedToType(ISD::STORE, MVT::v4f32, MVT::v4i32); in AMDGPUTargetLowering()
243 setOperationAction(ISD::STORE, MVT::v8f32, Promote); in AMDGPUTargetLowering()
244 AddPromotedToType(ISD::STORE, MVT::v8f32, MVT::v8i32); in AMDGPUTargetLowering()
246 setOperationAction(ISD::STORE, MVT::v16f32, Promote); in AMDGPUTargetLowering()
247 AddPromotedToType(ISD::STORE, MVT::v16f32, MVT::v16i32); in AMDGPUTargetLowering()
249 setOperationAction(ISD::STORE, MVT::i64, Promote); in AMDGPUTargetLowering()
250 AddPromotedToType(ISD::STORE, MVT::i64, MVT::v2i32); in AMDGPUTargetLowering()
252 setOperationAction(ISD::STORE, MVT::v2i64, Promote); in AMDGPUTargetLowering()
253 AddPromotedToType(ISD::STORE, MVT::v2i64, MVT::v4i32); in AMDGPUTargetLowering()
255 setOperationAction(ISD::STORE, MVT::f64, Promote); in AMDGPUTargetLowering()
256 AddPromotedToType(ISD::STORE, MVT::f64, MVT::v2i32); in AMDGPUTargetLowering()
258 setOperationAction(ISD::STORE, MVT::v2f64, Promote); in AMDGPUTargetLowering()
259 AddPromotedToType(ISD::STORE, MVT::v2f64, MVT::v4i32); in AMDGPUTargetLowering()
261 setTruncStoreAction(MVT::i64, MVT::i1, Expand); in AMDGPUTargetLowering()
262 setTruncStoreAction(MVT::i64, MVT::i8, Expand); in AMDGPUTargetLowering()
263 setTruncStoreAction(MVT::i64, MVT::i16, Expand); in AMDGPUTargetLowering()
264 setTruncStoreAction(MVT::i64, MVT::i32, Expand); in AMDGPUTargetLowering()
266 setTruncStoreAction(MVT::v2i64, MVT::v2i1, Expand); in AMDGPUTargetLowering()
267 setTruncStoreAction(MVT::v2i64, MVT::v2i8, Expand); in AMDGPUTargetLowering()
268 setTruncStoreAction(MVT::v2i64, MVT::v2i16, Expand); in AMDGPUTargetLowering()
269 setTruncStoreAction(MVT::v2i64, MVT::v2i32, Expand); in AMDGPUTargetLowering()
271 setTruncStoreAction(MVT::f32, MVT::f16, Expand); in AMDGPUTargetLowering()
272 setTruncStoreAction(MVT::v2f32, MVT::v2f16, Expand); in AMDGPUTargetLowering()
273 setTruncStoreAction(MVT::v4f32, MVT::v4f16, Expand); in AMDGPUTargetLowering()
274 setTruncStoreAction(MVT::v8f32, MVT::v8f16, Expand); in AMDGPUTargetLowering()
276 setTruncStoreAction(MVT::f64, MVT::f16, Expand); in AMDGPUTargetLowering()
277 setTruncStoreAction(MVT::f64, MVT::f32, Expand); in AMDGPUTargetLowering()
279 setTruncStoreAction(MVT::v2f64, MVT::v2f32, Expand); in AMDGPUTargetLowering()
280 setTruncStoreAction(MVT::v2f64, MVT::v2f16, Expand); in AMDGPUTargetLowering()
282 setTruncStoreAction(MVT::v4f64, MVT::v4f32, Expand); in AMDGPUTargetLowering()
283 setTruncStoreAction(MVT::v4f64, MVT::v4f16, Expand); in AMDGPUTargetLowering()
285 setTruncStoreAction(MVT::v8f64, MVT::v8f32, Expand); in AMDGPUTargetLowering()
286 setTruncStoreAction(MVT::v8f64, MVT::v8f16, Expand); in AMDGPUTargetLowering()
289 setOperationAction(ISD::Constant, MVT::i32, Legal); in AMDGPUTargetLowering()
290 setOperationAction(ISD::Constant, MVT::i64, Legal); in AMDGPUTargetLowering()
291 setOperationAction(ISD::ConstantFP, MVT::f32, Legal); in AMDGPUTargetLowering()
292 setOperationAction(ISD::ConstantFP, MVT::f64, Legal); in AMDGPUTargetLowering()
294 setOperationAction(ISD::BR_JT, MVT::Other, Expand); in AMDGPUTargetLowering()
295 setOperationAction(ISD::BRIND, MVT::Other, Expand); in AMDGPUTargetLowering()
298 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); in AMDGPUTargetLowering()
302 setOperationAction(ISD::FCEIL, MVT::f32, Legal); in AMDGPUTargetLowering()
303 setOperationAction(ISD::FEXP2, MVT::f32, Legal); in AMDGPUTargetLowering()
304 setOperationAction(ISD::FPOW, MVT::f32, Legal); in AMDGPUTargetLowering()
305 setOperationAction(ISD::FLOG2, MVT::f32, Legal); in AMDGPUTargetLowering()
306 setOperationAction(ISD::FABS, MVT::f32, Legal); in AMDGPUTargetLowering()
307 setOperationAction(ISD::FFLOOR, MVT::f32, Legal); in AMDGPUTargetLowering()
308 setOperationAction(ISD::FRINT, MVT::f32, Legal); in AMDGPUTargetLowering()
309 setOperationAction(ISD::FTRUNC, MVT::f32, Legal); in AMDGPUTargetLowering()
310 setOperationAction(ISD::FMINNUM, MVT::f32, Legal); in AMDGPUTargetLowering()
311 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal); in AMDGPUTargetLowering()
313 setOperationAction(ISD::FROUND, MVT::f32, Custom); in AMDGPUTargetLowering()
314 setOperationAction(ISD::FROUND, MVT::f64, Custom); in AMDGPUTargetLowering()
316 setOperationAction(ISD::FLOG, MVT::f32, Custom); in AMDGPUTargetLowering()
317 setOperationAction(ISD::FLOG10, MVT::f32, Custom); in AMDGPUTargetLowering()
318 setOperationAction(ISD::FEXP, MVT::f32, Custom); in AMDGPUTargetLowering()
321 setOperationAction(ISD::FNEARBYINT, MVT::f32, Custom); in AMDGPUTargetLowering()
322 setOperationAction(ISD::FNEARBYINT, MVT::f64, Custom); in AMDGPUTargetLowering()
324 setOperationAction(ISD::FREM, MVT::f32, Custom); in AMDGPUTargetLowering()
325 setOperationAction(ISD::FREM, MVT::f64, Custom); in AMDGPUTargetLowering()
328 setOperationAction(ISD::FSUB, MVT::f64, Expand); in AMDGPUTargetLowering()
330 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i32, Custom); in AMDGPUTargetLowering()
331 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4f32, Custom); in AMDGPUTargetLowering()
332 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i32, Custom); in AMDGPUTargetLowering()
333 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f32, Custom); in AMDGPUTargetLowering()
334 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2f32, Custom); in AMDGPUTargetLowering()
335 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2i32, Custom); in AMDGPUTargetLowering()
336 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4f32, Custom); in AMDGPUTargetLowering()
337 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4i32, Custom); in AMDGPUTargetLowering()
338 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8f32, Custom); in AMDGPUTargetLowering()
339 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8i32, Custom); in AMDGPUTargetLowering()
341 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); in AMDGPUTargetLowering()
342 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Custom); in AMDGPUTargetLowering()
343 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Custom); in AMDGPUTargetLowering()
345 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 }; in AMDGPUTargetLowering()
346 for (MVT VT : ScalarIntVTs) { in AMDGPUTargetLowering()
373 setOperationAction(ISD::ROTL, MVT::i32, Expand); in AMDGPUTargetLowering()
374 setOperationAction(ISD::ROTL, MVT::i64, Expand); in AMDGPUTargetLowering()
375 setOperationAction(ISD::ROTR, MVT::i64, Expand); in AMDGPUTargetLowering()
377 setOperationAction(ISD::MUL, MVT::i64, Expand); in AMDGPUTargetLowering()
378 setOperationAction(ISD::MULHU, MVT::i64, Expand); in AMDGPUTargetLowering()
379 setOperationAction(ISD::MULHS, MVT::i64, Expand); in AMDGPUTargetLowering()
380 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom); in AMDGPUTargetLowering()
381 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); in AMDGPUTargetLowering()
382 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); in AMDGPUTargetLowering()
383 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); in AMDGPUTargetLowering()
384 setOperationAction(ISD::SELECT_CC, MVT::i64, Expand); in AMDGPUTargetLowering()
386 setOperationAction(ISD::SMIN, MVT::i32, Legal); in AMDGPUTargetLowering()
387 setOperationAction(ISD::UMIN, MVT::i32, Legal); in AMDGPUTargetLowering()
388 setOperationAction(ISD::SMAX, MVT::i32, Legal); in AMDGPUTargetLowering()
389 setOperationAction(ISD::UMAX, MVT::i32, Legal); in AMDGPUTargetLowering()
391 setOperationAction(ISD::CTTZ, MVT::i64, Custom); in AMDGPUTargetLowering()
392 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Custom); in AMDGPUTargetLowering()
393 setOperationAction(ISD::CTLZ, MVT::i64, Custom); in AMDGPUTargetLowering()
394 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Custom); in AMDGPUTargetLowering()
396 static const MVT::SimpleValueType VectorIntTypes[] = { in AMDGPUTargetLowering()
397 MVT::v2i32, MVT::v4i32 in AMDGPUTargetLowering()
400 for (MVT VT : VectorIntTypes) { in AMDGPUTargetLowering()
438 static const MVT::SimpleValueType FloatVectorTypes[] = { in AMDGPUTargetLowering()
439 MVT::v2f32, MVT::v4f32 in AMDGPUTargetLowering()
442 for (MVT VT : FloatVectorTypes) { in AMDGPUTargetLowering()
478 setOperationAction(ISD::SELECT, MVT::v2f32, Promote); in AMDGPUTargetLowering()
479 AddPromotedToType(ISD::SELECT, MVT::v2f32, MVT::v2i32); in AMDGPUTargetLowering()
481 setOperationAction(ISD::SELECT, MVT::v4f32, Promote); in AMDGPUTargetLowering()
482 AddPromotedToType(ISD::SELECT, MVT::v4f32, MVT::v4i32); in AMDGPUTargetLowering()
578 static bool opMustUseVOP3Encoding(const SDNode *N, MVT VT) { in opMustUseVOP3Encoding()
579 return N->getNumOperands() > 2 || VT == MVT::f64; in opMustUseVOP3Encoding()
617 MVT VT = N->getValueType(0).getScalarType().getSimpleVT(); in allUsesHaveSourceMods()
633 MVT AMDGPUTargetLowering::getVectorIdxTy(const DataLayout &) const { in getVectorIdxTy()
634 return MVT::i32; in getVectorIdxTy()
645 return (ScalarVT == MVT::f32 || ScalarVT == MVT::f64 || in isFPImmLegal()
646 (ScalarVT == MVT::f16 && Subtarget->has16BitInsts())); in isFPImmLegal()
652 return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64); in ShouldShrinkFPConstant()
698 if (LoadTy.getScalarType() == MVT::i32) in isLoadBitCastBeneficial()
759 return VT == MVT::f32 || VT == MVT::f64 || in isFAbsFree()
760 (Subtarget->has16BitInsts() && VT == MVT::f16); in isFAbsFree()
765 return VT == MVT::f32 || VT == MVT::f64 || in isFNegFree()
766 (Subtarget->has16BitInsts() && VT == MVT::f16) || in isFNegFree()
767 (Subtarget->hasVOP3PInsts() && VT == MVT::v2f16); in isFNegFree()
825 if (Src == MVT::i16) in isZExtFree()
826 return Dest == MVT::i32 ||Dest == MVT::i64 ; in isZExtFree()
828 return Src == MVT::i32 && Dest == MVT::i64; in isZExtFree()
963 MVT RegisterVT = getRegisterTypeForCallingConv(Ctx, CC, ArgVT); in analyzeFormalArgumentsCompute()
1040 return DAG.getNode(AMDGPUISD::ENDPGM, DL, MVT::Other, Chain); in LowerReturn()
1091 return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); in addTokenForArgument()
1243 if (VT == MVT::v4i16 || VT == MVT::v4f16) { in LowerCONCAT_VECTORS()
1245 SDValue Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Op.getOperand(0)); in LowerCONCAT_VECTORS()
1246 SDValue Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Op.getOperand(1)); in LowerCONCAT_VECTORS()
1248 SDValue BV = DAG.getBuildVector(MVT::v2i32, SL, { Lo, Hi }); in LowerCONCAT_VECTORS()
1348 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in split64BitValue()
1350 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in split64BitValue()
1351 const SDValue One = DAG.getConstant(1, SL, MVT::i32); in split64BitValue()
1353 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero); in split64BitValue()
1354 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One); in split64BitValue()
1362 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in getLoHalf64()
1363 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in getLoHalf64()
1364 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero); in getLoHalf64()
1370 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op); in getHiHalf64()
1371 const SDValue One = DAG.getConstant(1, SL, MVT::i32); in getHiHalf64()
1372 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One); in getHiHalf64()
1415 DAG.getNode(ISD::TokenFactor, SL, MVT::Other, in SplitVectorLoad()
1460 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoStore, HiStore); in SplitVectorStore()
1472 MVT IntVT = MVT::i32; in LowerDIVREM24()
1473 MVT FltVT = MVT::f32; in LowerDIVREM24()
1578 assert(VT == MVT::i64 && "LowerUDIVREM64 expects an i64"); in LowerUDIVREM64()
1600 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(0), Zero}); in LowerUDIVREM64()
1601 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(1), Zero}); in LowerUDIVREM64()
1603 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV)); in LowerUDIVREM64()
1604 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM)); in LowerUDIVREM64()
1608 if (isTypeLegal(MVT::i64)) { in LowerUDIVREM64()
1614 SDValue Cvt_Lo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Lo); in LowerUDIVREM64()
1615 SDValue Cvt_Hi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Hi); in LowerUDIVREM64()
1616 SDValue Mad1 = DAG.getNode(FMAD, DL, MVT::f32, Cvt_Hi, in LowerUDIVREM64()
1617 DAG.getConstantFP(APInt(32, 0x4f800000).bitsToFloat(), DL, MVT::f32), in LowerUDIVREM64()
1619 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, DL, MVT::f32, Mad1); in LowerUDIVREM64()
1620 SDValue Mul1 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Rcp, in LowerUDIVREM64()
1621 DAG.getConstantFP(APInt(32, 0x5f7ffffc).bitsToFloat(), DL, MVT::f32)); in LowerUDIVREM64()
1622 SDValue Mul2 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Mul1, in LowerUDIVREM64()
1623 DAG.getConstantFP(APInt(32, 0x2f800000).bitsToFloat(), DL, MVT::f32)); in LowerUDIVREM64()
1624 SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, MVT::f32, Mul2); in LowerUDIVREM64()
1625 SDValue Mad2 = DAG.getNode(FMAD, DL, MVT::f32, Trunc, in LowerUDIVREM64()
1626 DAG.getConstantFP(APInt(32, 0xcf800000).bitsToFloat(), DL, MVT::f32), in LowerUDIVREM64()
1631 DAG.getBuildVector(MVT::v2i32, DL, {Rcp_Lo, Rcp_Hi})); in LowerUDIVREM64()
1635 SDValue Zero1 = DAG.getConstant(0, DL, MVT::i1); in LowerUDIVREM64()
1636 SDVTList HalfCarryVT = DAG.getVTList(HalfVT, MVT::i1); in LowerUDIVREM64()
1652 DAG.getBuildVector(MVT::v2i32, DL, {Add1_Lo, Add1_Hi})); in LowerUDIVREM64()
1668 DAG.getBuildVector(MVT::v2i32, DL, {Add2_Lo, Add2_Hi})); in LowerUDIVREM64()
1681 DAG.getBuildVector(MVT::v2i32, DL, {Sub1_Lo, Sub1_Hi})); in LowerUDIVREM64()
1702 DAG.getBuildVector(MVT::v2i32, DL, {Sub2_Lo, Sub2_Hi})); in LowerUDIVREM64()
1722 DAG.getBuildVector(MVT::v2i32, DL, {Sub3_Lo, Sub3_Hi})); in LowerUDIVREM64()
1745 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {REM_Lo, Zero}); in LowerUDIVREM64()
1746 REM = DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM); in LowerUDIVREM64()
1776 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {DIV_Lo, DIV_Hi}); in LowerUDIVREM64()
1777 DIV = DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV); in LowerUDIVREM64()
1787 if (VT == MVT::i64) { in LowerUDIVREM()
1793 if (VT == MVT::i32) { in LowerUDIVREM()
1908 if (VT == MVT::i32) { in LowerSDIVREM()
1913 if (VT == MVT::i64 && in LowerSDIVREM()
1981 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); in LowerFCEIL()
1983 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64); in LowerFCEIL()
1984 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64); in LowerFCEIL()
1987 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64); in LowerFCEIL()
1993 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero); in LowerFCEIL()
1995 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); in LowerFCEIL()
2003 SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32, in extractF64Exponent()
2005 DAG.getConstant(FractBits - 32, SL, MVT::i32), in extractF64Exponent()
2006 DAG.getConstant(ExpBits, SL, MVT::i32)); in extractF64Exponent()
2007 SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart, in extractF64Exponent()
2008 DAG.getConstant(1023, SL, MVT::i32)); in extractF64Exponent()
2017 assert(Op.getValueType() == MVT::f64); in LowerFTRUNC()
2019 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in LowerFTRUNC()
2020 const SDValue One = DAG.getConstant(1, SL, MVT::i32); in LowerFTRUNC()
2022 SDValue VecSrc = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src); in LowerFTRUNC()
2026 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, VecSrc, One); in LowerFTRUNC()
2033 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, SL, MVT::i32); in LowerFTRUNC()
2034 SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask); in LowerFTRUNC()
2037 SDValue SignBit64 = DAG.getBuildVector(MVT::v2i32, SL, {Zero, SignBit}); in LowerFTRUNC()
2038 SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64); in LowerFTRUNC()
2040 SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src); in LowerFTRUNC()
2042 = DAG.getConstant((UINT64_C(1) << FractBits) - 1, SL, MVT::i64); in LowerFTRUNC()
2044 SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp); in LowerFTRUNC()
2045 SDValue Not = DAG.getNOT(SL, Shr, MVT::i64); in LowerFTRUNC()
2046 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not); in LowerFTRUNC()
2049 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i32); in LowerFTRUNC()
2051 const SDValue FiftyOne = DAG.getConstant(FractBits - 1, SL, MVT::i32); in LowerFTRUNC()
2056 SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0); in LowerFTRUNC()
2057 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1); in LowerFTRUNC()
2059 return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2); in LowerFTRUNC()
2066 assert(Op.getValueType() == MVT::f64); in LowerFRINT()
2069 SDValue C1 = DAG.getConstantFP(C1Val, SL, MVT::f64); in LowerFRINT()
2070 SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src); in LowerFRINT()
2074 SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign); in LowerFRINT()
2075 SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign); in LowerFRINT()
2077 SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src); in LowerFRINT()
2080 SDValue C2 = DAG.getConstantFP(C2Val, SL, MVT::f64); in LowerFRINT()
2083 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64); in LowerFRINT()
2086 return DAG.getSelect(SL, MVT::f64, Cond, Src, Tmp2); in LowerFRINT()
2134 SDValue L = DAG.getNode(ISD::BITCAST, SL, MVT::i64, X); in LowerFROUND64()
2136 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in LowerFROUND64()
2137 const SDValue One = DAG.getConstant(1, SL, MVT::i32); in LowerFROUND64()
2138 const SDValue NegOne = DAG.getConstant(-1, SL, MVT::i32); in LowerFROUND64()
2139 const SDValue FiftyOne = DAG.getConstant(51, SL, MVT::i32); in LowerFROUND64()
2141 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i32); in LowerFROUND64()
2143 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X); in LowerFROUND64()
2145 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC, One); in LowerFROUND64()
2150 MVT::i64); in LowerFROUND64()
2152 SDValue M = DAG.getNode(ISD::SRA, SL, MVT::i64, Mask, Exp); in LowerFROUND64()
2153 SDValue D = DAG.getNode(ISD::SRA, SL, MVT::i64, in LowerFROUND64()
2155 MVT::i64), in LowerFROUND64()
2158 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, L, M); in LowerFROUND64()
2160 DAG.getConstant(0, SL, MVT::i64), Tmp0, in LowerFROUND64()
2163 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, Tmp1, in LowerFROUND64()
2164 D, DAG.getConstant(0, SL, MVT::i64)); in LowerFROUND64()
2165 SDValue K = DAG.getNode(ISD::ADD, SL, MVT::i64, L, Tmp2); in LowerFROUND64()
2167 K = DAG.getNode(ISD::AND, SL, MVT::i64, K, DAG.getNOT(SL, M, MVT::i64)); in LowerFROUND64()
2168 K = DAG.getNode(ISD::BITCAST, SL, MVT::f64, K); in LowerFROUND64()
2174 SDValue Mag = DAG.getNode(ISD::SELECT, SL, MVT::f64, in LowerFROUND64()
2176 DAG.getConstantFP(1.0, SL, MVT::f64), in LowerFROUND64()
2177 DAG.getConstantFP(0.0, SL, MVT::f64)); in LowerFROUND64()
2179 SDValue S = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, Mag, X); in LowerFROUND64()
2181 K = DAG.getNode(ISD::SELECT, SL, MVT::f64, ExpLt0, S, K); in LowerFROUND64()
2182 K = DAG.getNode(ISD::SELECT, SL, MVT::f64, ExpGt51, X, K); in LowerFROUND64()
2190 if (VT == MVT::f32 || VT == MVT::f16) in LowerFROUND()
2193 if (VT == MVT::f64) in LowerFROUND()
2207 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); in LowerFFLOOR()
2209 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64); in LowerFFLOOR()
2210 const SDValue NegOne = DAG.getConstantFP(-1.0, SL, MVT::f64); in LowerFFLOOR()
2213 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64); in LowerFFLOOR()
2219 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero); in LowerFFLOOR()
2221 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); in LowerFFLOOR()
2239 case MVT::f32: in getLog2EVal()
2241 case MVT::f16: in getLog2EVal()
2245 case MVT::f64: in getLog2EVal()
2289 if (ZeroUndef && Src.getValueType() == MVT::i32) in LowerCTLZ_CTTZ()
2290 return DAG.getNode(NewOpc, SL, MVT::i32, Src); in LowerCTLZ_CTTZ()
2292 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src); in LowerCTLZ_CTTZ()
2294 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in LowerCTLZ_CTTZ()
2295 const SDValue One = DAG.getConstant(1, SL, MVT::i32); in LowerCTLZ_CTTZ()
2297 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero); in LowerCTLZ_CTTZ()
2298 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One); in LowerCTLZ_CTTZ()
2301 *DAG.getContext(), MVT::i32); in LowerCTLZ_CTTZ()
2306 SDValue OprLo = DAG.getNode(ISDOpc, SL, MVT::i32, Lo); in LowerCTLZ_CTTZ()
2307 SDValue OprHi = DAG.getNode(ISDOpc, SL, MVT::i32, Hi); in LowerCTLZ_CTTZ()
2309 const SDValue Bits32 = DAG.getConstant(32, SL, MVT::i32); in LowerCTLZ_CTTZ()
2312 Add = DAG.getNode(ISD::ADD, SL, MVT::i32, OprLo, Bits32); in LowerCTLZ_CTTZ()
2314 NewOpr = DAG.getNode(ISD::SELECT, SL, MVT::i32, Hi0orLo0, Add, OprHi); in LowerCTLZ_CTTZ()
2316 Add = DAG.getNode(ISD::ADD, SL, MVT::i32, OprHi, Bits32); in LowerCTLZ_CTTZ()
2318 NewOpr = DAG.getNode(ISD::SELECT, SL, MVT::i32, Hi0orLo0, Add, OprLo); in LowerCTLZ_CTTZ()
2335 const SDValue Bits32 = DAG.getConstant(64, SL, MVT::i32); in LowerCTLZ_CTTZ()
2339 NewOpr = DAG.getNode(ISD::SELECT, SL, MVT::i32, in LowerCTLZ_CTTZ()
2343 return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i64, NewOpr); in LowerCTLZ_CTTZ()
2373 const SDValue SignBit = DAG.getConstant(63, SL, MVT::i64); in LowerINT_TO_FP32()
2374 S = DAG.getNode(ISD::SRA, SL, MVT::i64, L, SignBit); in LowerINT_TO_FP32()
2376 SDValue LPlusS = DAG.getNode(ISD::ADD, SL, MVT::i64, L, S); in LowerINT_TO_FP32()
2377 L = DAG.getNode(ISD::XOR, SL, MVT::i64, LPlusS, S); in LowerINT_TO_FP32()
2381 *DAG.getContext(), MVT::f32); in LowerINT_TO_FP32()
2384 SDValue ZeroI32 = DAG.getConstant(0, SL, MVT::i32); in LowerINT_TO_FP32()
2385 SDValue ZeroI64 = DAG.getConstant(0, SL, MVT::i64); in LowerINT_TO_FP32()
2386 SDValue LZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SL, MVT::i64, L); in LowerINT_TO_FP32()
2387 LZ = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LZ); in LowerINT_TO_FP32()
2389 SDValue K = DAG.getConstant(127U + 63U, SL, MVT::i32); in LowerINT_TO_FP32()
2390 SDValue E = DAG.getSelect(SL, MVT::i32, in LowerINT_TO_FP32()
2392 DAG.getNode(ISD::SUB, SL, MVT::i32, K, LZ), in LowerINT_TO_FP32()
2395 SDValue U = DAG.getNode(ISD::AND, SL, MVT::i64, in LowerINT_TO_FP32()
2396 DAG.getNode(ISD::SHL, SL, MVT::i64, L, LZ), in LowerINT_TO_FP32()
2397 DAG.getConstant((-1ULL) >> 1, SL, MVT::i64)); in LowerINT_TO_FP32()
2399 SDValue T = DAG.getNode(ISD::AND, SL, MVT::i64, U, in LowerINT_TO_FP32()
2400 DAG.getConstant(0xffffffffffULL, SL, MVT::i64)); in LowerINT_TO_FP32()
2402 SDValue UShl = DAG.getNode(ISD::SRL, SL, MVT::i64, in LowerINT_TO_FP32()
2403 U, DAG.getConstant(40, SL, MVT::i64)); in LowerINT_TO_FP32()
2405 SDValue V = DAG.getNode(ISD::OR, SL, MVT::i32, in LowerINT_TO_FP32()
2406 DAG.getNode(ISD::SHL, SL, MVT::i32, E, DAG.getConstant(23, SL, MVT::i32)), in LowerINT_TO_FP32()
2407 DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, UShl)); in LowerINT_TO_FP32()
2409 SDValue C = DAG.getConstant(0x8000000000ULL, SL, MVT::i64); in LowerINT_TO_FP32()
2413 SDValue One = DAG.getConstant(1, SL, MVT::i32); in LowerINT_TO_FP32()
2415 SDValue VTrunc1 = DAG.getNode(ISD::AND, SL, MVT::i32, V, One); in LowerINT_TO_FP32()
2417 SDValue R = DAG.getSelect(SL, MVT::i32, in LowerINT_TO_FP32()
2420 DAG.getSelect(SL, MVT::i32, TCmp, VTrunc1, ZeroI32)); in LowerINT_TO_FP32()
2421 R = DAG.getNode(ISD::ADD, SL, MVT::i32, V, R); in LowerINT_TO_FP32()
2422 R = DAG.getNode(ISD::BITCAST, SL, MVT::f32, R); in LowerINT_TO_FP32()
2427 SDValue RNeg = DAG.getNode(ISD::FNEG, SL, MVT::f32, R); in LowerINT_TO_FP32()
2428 return DAG.getSelect(SL, MVT::f32, DAG.getSExtOrTrunc(S, SL, SetCCVT), RNeg, R); in LowerINT_TO_FP32()
2436 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src); in LowerINT_TO_FP64()
2438 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC, in LowerINT_TO_FP64()
2439 DAG.getConstant(0, SL, MVT::i32)); in LowerINT_TO_FP64()
2440 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC, in LowerINT_TO_FP64()
2441 DAG.getConstant(1, SL, MVT::i32)); in LowerINT_TO_FP64()
2444 SL, MVT::f64, Hi); in LowerINT_TO_FP64()
2446 SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo); in LowerINT_TO_FP64()
2448 SDValue LdExp = DAG.getNode(AMDGPUISD::LDEXP, SL, MVT::f64, CvtHi, in LowerINT_TO_FP64()
2449 DAG.getConstant(32, SL, MVT::i32)); in LowerINT_TO_FP64()
2451 return DAG.getNode(ISD::FADD, SL, MVT::f64, LdExp, CvtLo); in LowerINT_TO_FP64()
2456 assert(Op.getOperand(0).getValueType() == MVT::i64 && in LowerUINT_TO_FP()
2462 if (Subtarget->has16BitInsts() && DestVT == MVT::f16) { in LowerUINT_TO_FP()
2466 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src); in LowerUINT_TO_FP()
2469 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag); in LowerUINT_TO_FP()
2474 if (DestVT == MVT::f32) in LowerUINT_TO_FP()
2477 assert(DestVT == MVT::f64); in LowerUINT_TO_FP()
2483 assert(Op.getOperand(0).getValueType() == MVT::i64 && in LowerSINT_TO_FP()
2489 if (Subtarget->has16BitInsts() && DestVT == MVT::f16) { in LowerSINT_TO_FP()
2493 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src); in LowerSINT_TO_FP()
2496 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag); in LowerSINT_TO_FP()
2501 if (DestVT == MVT::f32) in LowerSINT_TO_FP()
2504 assert(DestVT == MVT::f64); in LowerSINT_TO_FP()
2514 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); in LowerFP64_TO_INT()
2517 MVT::f64); in LowerFP64_TO_INT()
2519 MVT::f64); in LowerFP64_TO_INT()
2521 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, Trunc, K0); in LowerFP64_TO_INT()
2523 SDValue FloorMul = DAG.getNode(ISD::FFLOOR, SL, MVT::f64, Mul); in LowerFP64_TO_INT()
2526 SDValue Fma = DAG.getNode(ISD::FMA, SL, MVT::f64, FloorMul, K1, Trunc); in LowerFP64_TO_INT()
2529 MVT::i32, FloorMul); in LowerFP64_TO_INT()
2530 SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, SL, MVT::i32, Fma); in LowerFP64_TO_INT()
2532 SDValue Result = DAG.getBuildVector(MVT::v2i32, SL, {Lo, Hi}); in LowerFP64_TO_INT()
2534 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Result); in LowerFP64_TO_INT()
2542 if (N0.getValueType() == MVT::f32) in LowerFP_TO_FP16()
2550 assert(N0.getSimpleValueType() == MVT::f64); in LowerFP_TO_FP16()
2556 SDValue Zero = DAG.getConstant(0, DL, MVT::i32); in LowerFP_TO_FP16()
2557 SDValue One = DAG.getConstant(1, DL, MVT::i32); in LowerFP_TO_FP16()
2558 SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, N0); in LowerFP_TO_FP16()
2559 SDValue UH = DAG.getNode(ISD::SRL, DL, MVT::i64, U, in LowerFP_TO_FP16()
2560 DAG.getConstant(32, DL, MVT::i64)); in LowerFP_TO_FP16()
2561 UH = DAG.getZExtOrTrunc(UH, DL, MVT::i32); in LowerFP_TO_FP16()
2562 U = DAG.getZExtOrTrunc(U, DL, MVT::i32); in LowerFP_TO_FP16()
2563 SDValue E = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, in LowerFP_TO_FP16()
2564 DAG.getConstant(20, DL, MVT::i64)); in LowerFP_TO_FP16()
2565 E = DAG.getNode(ISD::AND, DL, MVT::i32, E, in LowerFP_TO_FP16()
2566 DAG.getConstant(ExpMask, DL, MVT::i32)); in LowerFP_TO_FP16()
2569 E = DAG.getNode(ISD::ADD, DL, MVT::i32, E, in LowerFP_TO_FP16()
2570 DAG.getConstant(-ExpBiasf64 + ExpBiasf16, DL, MVT::i32)); in LowerFP_TO_FP16()
2572 SDValue M = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, in LowerFP_TO_FP16()
2573 DAG.getConstant(8, DL, MVT::i32)); in LowerFP_TO_FP16()
2574 M = DAG.getNode(ISD::AND, DL, MVT::i32, M, in LowerFP_TO_FP16()
2575 DAG.getConstant(0xffe, DL, MVT::i32)); in LowerFP_TO_FP16()
2577 SDValue MaskedSig = DAG.getNode(ISD::AND, DL, MVT::i32, UH, in LowerFP_TO_FP16()
2578 DAG.getConstant(0x1ff, DL, MVT::i32)); in LowerFP_TO_FP16()
2579 MaskedSig = DAG.getNode(ISD::OR, DL, MVT::i32, MaskedSig, U); in LowerFP_TO_FP16()
2582 M = DAG.getNode(ISD::OR, DL, MVT::i32, M, Lo40Set); in LowerFP_TO_FP16()
2585 SDValue I = DAG.getNode(ISD::OR, DL, MVT::i32, in LowerFP_TO_FP16()
2586 DAG.getSelectCC(DL, M, Zero, DAG.getConstant(0x0200, DL, MVT::i32), in LowerFP_TO_FP16()
2587 Zero, ISD::SETNE), DAG.getConstant(0x7c00, DL, MVT::i32)); in LowerFP_TO_FP16()
2590 SDValue N = DAG.getNode(ISD::OR, DL, MVT::i32, M, in LowerFP_TO_FP16()
2591 DAG.getNode(ISD::SHL, DL, MVT::i32, E, in LowerFP_TO_FP16()
2592 DAG.getConstant(12, DL, MVT::i32))); in LowerFP_TO_FP16()
2595 SDValue OneSubExp = DAG.getNode(ISD::SUB, DL, MVT::i32, in LowerFP_TO_FP16()
2597 SDValue B = DAG.getNode(ISD::SMAX, DL, MVT::i32, OneSubExp, Zero); in LowerFP_TO_FP16()
2598 B = DAG.getNode(ISD::SMIN, DL, MVT::i32, B, in LowerFP_TO_FP16()
2599 DAG.getConstant(13, DL, MVT::i32)); in LowerFP_TO_FP16()
2601 SDValue SigSetHigh = DAG.getNode(ISD::OR, DL, MVT::i32, M, in LowerFP_TO_FP16()
2602 DAG.getConstant(0x1000, DL, MVT::i32)); in LowerFP_TO_FP16()
2604 SDValue D = DAG.getNode(ISD::SRL, DL, MVT::i32, SigSetHigh, B); in LowerFP_TO_FP16()
2605 SDValue D0 = DAG.getNode(ISD::SHL, DL, MVT::i32, D, B); in LowerFP_TO_FP16()
2607 D = DAG.getNode(ISD::OR, DL, MVT::i32, D, D1); in LowerFP_TO_FP16()
2610 SDValue VLow3 = DAG.getNode(ISD::AND, DL, MVT::i32, V, in LowerFP_TO_FP16()
2611 DAG.getConstant(0x7, DL, MVT::i32)); in LowerFP_TO_FP16()
2612 V = DAG.getNode(ISD::SRL, DL, MVT::i32, V, in LowerFP_TO_FP16()
2613 DAG.getConstant(2, DL, MVT::i32)); in LowerFP_TO_FP16()
2614 SDValue V0 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(3, DL, MVT::i32), in LowerFP_TO_FP16()
2616 SDValue V1 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(5, DL, MVT::i32), in LowerFP_TO_FP16()
2618 V1 = DAG.getNode(ISD::OR, DL, MVT::i32, V0, V1); in LowerFP_TO_FP16()
2619 V = DAG.getNode(ISD::ADD, DL, MVT::i32, V, V1); in LowerFP_TO_FP16()
2621 V = DAG.getSelectCC(DL, E, DAG.getConstant(30, DL, MVT::i32), in LowerFP_TO_FP16()
2622 DAG.getConstant(0x7c00, DL, MVT::i32), V, ISD::SETGT); in LowerFP_TO_FP16()
2623 V = DAG.getSelectCC(DL, E, DAG.getConstant(1039, DL, MVT::i32), in LowerFP_TO_FP16()
2627 SDValue Sign = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, in LowerFP_TO_FP16()
2628 DAG.getConstant(16, DL, MVT::i32)); in LowerFP_TO_FP16()
2629 Sign = DAG.getNode(ISD::AND, DL, MVT::i32, Sign, in LowerFP_TO_FP16()
2630 DAG.getConstant(0x8000, DL, MVT::i32)); in LowerFP_TO_FP16()
2632 V = DAG.getNode(ISD::OR, DL, MVT::i32, Sign, V); in LowerFP_TO_FP16()
2643 if (Subtarget->has16BitInsts() && SrcVT == MVT::f16) { in LowerFP_TO_SINT()
2646 SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src); in LowerFP_TO_SINT()
2648 DAG.getNode(Op.getOpcode(), DL, MVT::i64, FPExtend); in LowerFP_TO_SINT()
2653 if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64) in LowerFP_TO_SINT()
2666 if (Subtarget->has16BitInsts() && SrcVT == MVT::f16) { in LowerFP_TO_UINT()
2669 SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src); in LowerFP_TO_UINT()
2671 DAG.getNode(Op.getOpcode(), DL, MVT::i64, FPExtend); in LowerFP_TO_UINT()
2676 if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64) in LowerFP_TO_UINT()
2685 MVT VT = Op.getSimpleValueType(); in LowerSIGN_EXTEND_INREG()
2686 MVT ScalarVT = VT.getScalarType(); in LowerSIGN_EXTEND_INREG()
2755 return DAG.getConstant(Result, DL, MVT::i32); in constantFoldBFE()
2758 return DAG.getConstant(Src0 >> Offset, DL, MVT::i32); in constantFoldBFE()
2774 if (VT.getScalarType() == MVT::i32 || isTypeLegal(VT)) in shouldCombineMemoryType()
2932 SDValue LoRHS = DAG.getConstant(ValLo, SL, MVT::i32); in splitBinaryBitConstantOpImpl()
2933 SDValue HiRHS = DAG.getConstant(ValHi, SL, MVT::i32); in splitBinaryBitConstantOpImpl()
2935 SDValue LoAnd = DAG.getNode(Opc, SL, MVT::i32, Lo, LoRHS); in splitBinaryBitConstantOpImpl()
2936 SDValue HiAnd = DAG.getNode(Opc, SL, MVT::i32, Hi, HiRHS); in splitBinaryBitConstantOpImpl()
2943 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {LoAnd, HiAnd}); in splitBinaryBitConstantOpImpl()
2944 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in splitBinaryBitConstantOpImpl()
2971 if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 && in performShlCombine()
2972 isOperationLegal(ISD::BUILD_VECTOR, MVT::v2i16)) { in performShlCombine()
2975 SDValue Vec = DAG.getBuildVector(MVT::v2i16, SL, in performShlCombine()
2976 { DAG.getConstant(0, SL, MVT::i16), LHS->getOperand(0) }); in performShlCombine()
2977 return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Vec); in performShlCombine()
2981 if (VT != MVT::i64) in performShlCombine()
2993 if (VT != MVT::i64) in performShlCombine()
3004 SDValue ShiftAmt = DAG.getConstant(RHSVal - 32, SL, MVT::i32); in performShlCombine()
3006 SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS); in performShlCombine()
3007 SDValue NewShift = DAG.getNode(ISD::SHL, SL, MVT::i32, Lo, ShiftAmt); in performShlCombine()
3009 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in performShlCombine()
3011 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {Zero, NewShift}); in performShlCombine()
3012 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec); in performShlCombine()
3017 if (N->getValueType(0) != MVT::i64) in performSraCombine()
3031 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi, in performSraCombine()
3032 DAG.getConstant(31, SL, MVT::i32)); in performSraCombine()
3034 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {Hi, NewShift}); in performSraCombine()
3035 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec); in performSraCombine()
3041 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi, in performSraCombine()
3042 DAG.getConstant(31, SL, MVT::i32)); in performSraCombine()
3043 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, NewShift}); in performSraCombine()
3044 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec); in performSraCombine()
3052 if (N->getValueType(0) != MVT::i64) in performSrlCombine()
3070 SDValue One = DAG.getConstant(1, SL, MVT::i32); in performSrlCombine()
3071 SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in performSrlCombine()
3073 SDValue VecOp = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, N->getOperand(0)); in performSrlCombine()
3074 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, in performSrlCombine()
3077 SDValue NewConst = DAG.getConstant(ShiftAmt - 32, SL, MVT::i32); in performSrlCombine()
3078 SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst); in performSrlCombine()
3080 SDValue BuildPair = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, Zero}); in performSrlCombine()
3082 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair); in performSrlCombine()
3147 EVT::getVectorVT(*DAG.getContext(), MVT::i32, in performTruncateCombine()
3148 VT.getVectorNumElements()) : MVT::i32; in performTruncateCombine()
3178 return DAG.getNode(MulOpc, SL, MVT::i32, N0, N1); in getMul24()
3188 DAG.getVTList(MVT::i32, MVT::i32), N0, N1); in getMul24()
3190 return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, in getMul24()
3203 if (Subtarget->has16BitInsts() && VT.getScalarType().bitsLE(MVT::i16)) in performMulCombine()
3225 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); in performMulCombine()
3226 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); in performMulCombine()
3229 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); in performMulCombine()
3230 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); in performMulCombine()
3257 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); in performMulhsCombine()
3258 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); in performMulhsCombine()
3260 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_I24, DL, MVT::i32, N0, N1); in performMulhsCombine()
3281 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); in performMulhuCombine()
3282 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); in performMulhuCombine()
3284 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_U24, DL, MVT::i32, N0, N1); in performMulhuCombine()
3307 SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1); in performMulLoHi24Combine()
3308 SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1); in performMulLoHi24Combine()
3324 if (LegalVT != MVT::i32 && (Subtarget->has16BitInsts() && in getFFBX_U32()
3325 LegalVT != MVT::i16)) in getFFBX_U32()
3328 if (VT != MVT::i32) in getFFBX_U32()
3329 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Op); in getFFBX_U32()
3331 SDValue FFBX = DAG.getNode(Opc, DL, MVT::i32, Op); in getFFBX_U32()
3332 if (VT != MVT::i32) in getFFBX_U32()
3497 if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) { in performSelectCombine()
3828 if (Src.getValueType() == MVT::i64) { in PerformDAGCombine()
3831 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in PerformDAGCombine()
3832 DAG.getConstant(Lo_32(CVal), SL, MVT::i32), in PerformDAGCombine()
3833 DAG.getConstant(Hi_32(CVal), SL, MVT::i32)); in PerformDAGCombine()
3842 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in PerformDAGCombine()
3843 DAG.getConstant(Lo_32(CVal), SL, MVT::i32), in PerformDAGCombine()
3844 DAG.getConstant(Hi_32(CVal), SL, MVT::i32)); in PerformDAGCombine()
3904 return DAG.getConstant(0, DL, MVT::i32); in PerformDAGCombine()
3931 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, BitsFrom, in PerformDAGCombine()
3956 SDValue ShiftVal = DAG.getConstant(OffsetVal, DL, MVT::i32); in PerformDAGCombine()
3957 return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32, in PerformDAGCombine()
4027 SDValue Ptr = DAG.getFrameIndex(FI, MVT::i32); in loadStackInputValue()
4042 SDValue Ptr = DAG.getConstant(Offset, SL, MVT::i32); in storeStackInputValue()
4233 if (VT == MVT::f32) { in getSqrtEstimate()
4249 if (VT == MVT::f32) { in getRecipEstimate()