Lines Matching refs:VT
159 void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT) { in addTypeForNEON() argument
160 if (VT != PromotedLdStVT) { in addTypeForNEON()
161 setOperationAction(ISD::LOAD, VT, Promote); in addTypeForNEON()
162 AddPromotedToType (ISD::LOAD, VT, PromotedLdStVT); in addTypeForNEON()
164 setOperationAction(ISD::STORE, VT, Promote); in addTypeForNEON()
165 AddPromotedToType (ISD::STORE, VT, PromotedLdStVT); in addTypeForNEON()
168 MVT ElemTy = VT.getVectorElementType(); in addTypeForNEON()
170 setOperationAction(ISD::SETCC, VT, Custom); in addTypeForNEON()
171 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in addTypeForNEON()
172 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in addTypeForNEON()
174 setOperationAction(ISD::SINT_TO_FP, VT, Custom); in addTypeForNEON()
175 setOperationAction(ISD::UINT_TO_FP, VT, Custom); in addTypeForNEON()
176 setOperationAction(ISD::FP_TO_SINT, VT, Custom); in addTypeForNEON()
177 setOperationAction(ISD::FP_TO_UINT, VT, Custom); in addTypeForNEON()
179 setOperationAction(ISD::SINT_TO_FP, VT, Expand); in addTypeForNEON()
180 setOperationAction(ISD::UINT_TO_FP, VT, Expand); in addTypeForNEON()
181 setOperationAction(ISD::FP_TO_SINT, VT, Expand); in addTypeForNEON()
182 setOperationAction(ISD::FP_TO_UINT, VT, Expand); in addTypeForNEON()
184 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in addTypeForNEON()
185 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in addTypeForNEON()
186 setOperationAction(ISD::CONCAT_VECTORS, VT, Legal); in addTypeForNEON()
187 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal); in addTypeForNEON()
188 setOperationAction(ISD::SELECT, VT, Expand); in addTypeForNEON()
189 setOperationAction(ISD::SELECT_CC, VT, Expand); in addTypeForNEON()
190 setOperationAction(ISD::VSELECT, VT, Expand); in addTypeForNEON()
191 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); in addTypeForNEON()
192 if (VT.isInteger()) { in addTypeForNEON()
193 setOperationAction(ISD::SHL, VT, Custom); in addTypeForNEON()
194 setOperationAction(ISD::SRA, VT, Custom); in addTypeForNEON()
195 setOperationAction(ISD::SRL, VT, Custom); in addTypeForNEON()
199 setOperationAction(ISD::SDIV, VT, Expand); in addTypeForNEON()
200 setOperationAction(ISD::UDIV, VT, Expand); in addTypeForNEON()
201 setOperationAction(ISD::FDIV, VT, Expand); in addTypeForNEON()
202 setOperationAction(ISD::SREM, VT, Expand); in addTypeForNEON()
203 setOperationAction(ISD::UREM, VT, Expand); in addTypeForNEON()
204 setOperationAction(ISD::FREM, VT, Expand); in addTypeForNEON()
205 setOperationAction(ISD::SDIVREM, VT, Expand); in addTypeForNEON()
206 setOperationAction(ISD::UDIVREM, VT, Expand); in addTypeForNEON()
208 if (!VT.isFloatingPoint() && in addTypeForNEON()
209 VT != MVT::v2i64 && VT != MVT::v1i64) in addTypeForNEON()
211 setOperationAction(Opcode, VT, Legal); in addTypeForNEON()
212 if (!VT.isFloatingPoint()) in addTypeForNEON()
214 setOperationAction(Opcode, VT, Legal); in addTypeForNEON()
217 void ARMTargetLowering::addDRTypeForNEON(MVT VT) { in addDRTypeForNEON() argument
218 addRegisterClass(VT, &ARM::DPRRegClass); in addDRTypeForNEON()
219 addTypeForNEON(VT, MVT::f64); in addDRTypeForNEON()
222 void ARMTargetLowering::addQRTypeForNEON(MVT VT) { in addQRTypeForNEON() argument
223 addRegisterClass(VT, &ARM::DPairRegClass); in addQRTypeForNEON()
224 addTypeForNEON(VT, MVT::v2f64); in addQRTypeForNEON()
227 void ARMTargetLowering::setAllExpand(MVT VT) { in setAllExpand() argument
229 setOperationAction(Opc, VT, Expand); in setAllExpand()
234 setOperationAction(ISD::BITCAST, VT, Legal); in setAllExpand()
235 setOperationAction(ISD::LOAD, VT, Legal); in setAllExpand()
236 setOperationAction(ISD::STORE, VT, Legal); in setAllExpand()
237 setOperationAction(ISD::UNDEF, VT, Legal); in setAllExpand()
250 for (auto VT : IntTypes) { in addMVEVectorTypes() local
251 addRegisterClass(VT, &ARM::MQPRRegClass); in addMVEVectorTypes()
252 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in addMVEVectorTypes()
253 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in addMVEVectorTypes()
254 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in addMVEVectorTypes()
255 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in addMVEVectorTypes()
256 setOperationAction(ISD::SHL, VT, Custom); in addMVEVectorTypes()
257 setOperationAction(ISD::SRA, VT, Custom); in addMVEVectorTypes()
258 setOperationAction(ISD::SRL, VT, Custom); in addMVEVectorTypes()
259 setOperationAction(ISD::SMIN, VT, Legal); in addMVEVectorTypes()
260 setOperationAction(ISD::SMAX, VT, Legal); in addMVEVectorTypes()
261 setOperationAction(ISD::UMIN, VT, Legal); in addMVEVectorTypes()
262 setOperationAction(ISD::UMAX, VT, Legal); in addMVEVectorTypes()
263 setOperationAction(ISD::ABS, VT, Legal); in addMVEVectorTypes()
264 setOperationAction(ISD::SETCC, VT, Custom); in addMVEVectorTypes()
265 setOperationAction(ISD::MLOAD, VT, Custom); in addMVEVectorTypes()
266 setOperationAction(ISD::MSTORE, VT, Legal); in addMVEVectorTypes()
267 setOperationAction(ISD::CTLZ, VT, Legal); in addMVEVectorTypes()
268 setOperationAction(ISD::CTTZ, VT, Custom); in addMVEVectorTypes()
269 setOperationAction(ISD::BITREVERSE, VT, Legal); in addMVEVectorTypes()
270 setOperationAction(ISD::BSWAP, VT, Legal); in addMVEVectorTypes()
271 setOperationAction(ISD::SADDSAT, VT, Legal); in addMVEVectorTypes()
272 setOperationAction(ISD::UADDSAT, VT, Legal); in addMVEVectorTypes()
273 setOperationAction(ISD::SSUBSAT, VT, Legal); in addMVEVectorTypes()
274 setOperationAction(ISD::USUBSAT, VT, Legal); in addMVEVectorTypes()
275 setOperationAction(ISD::ABDS, VT, Legal); in addMVEVectorTypes()
276 setOperationAction(ISD::ABDU, VT, Legal); in addMVEVectorTypes()
277 setOperationAction(ISD::AVGFLOORS, VT, Legal); in addMVEVectorTypes()
278 setOperationAction(ISD::AVGFLOORU, VT, Legal); in addMVEVectorTypes()
279 setOperationAction(ISD::AVGCEILS, VT, Legal); in addMVEVectorTypes()
280 setOperationAction(ISD::AVGCEILU, VT, Legal); in addMVEVectorTypes()
283 setOperationAction(ISD::UDIV, VT, Expand); in addMVEVectorTypes()
284 setOperationAction(ISD::SDIV, VT, Expand); in addMVEVectorTypes()
285 setOperationAction(ISD::UREM, VT, Expand); in addMVEVectorTypes()
286 setOperationAction(ISD::SREM, VT, Expand); in addMVEVectorTypes()
287 setOperationAction(ISD::UDIVREM, VT, Expand); in addMVEVectorTypes()
288 setOperationAction(ISD::SDIVREM, VT, Expand); in addMVEVectorTypes()
289 setOperationAction(ISD::CTPOP, VT, Expand); in addMVEVectorTypes()
290 setOperationAction(ISD::SELECT, VT, Expand); in addMVEVectorTypes()
291 setOperationAction(ISD::SELECT_CC, VT, Expand); in addMVEVectorTypes()
294 setOperationAction(ISD::VECREDUCE_ADD, VT, Legal); in addMVEVectorTypes()
295 setOperationAction(ISD::VECREDUCE_SMAX, VT, Legal); in addMVEVectorTypes()
296 setOperationAction(ISD::VECREDUCE_UMAX, VT, Legal); in addMVEVectorTypes()
297 setOperationAction(ISD::VECREDUCE_SMIN, VT, Legal); in addMVEVectorTypes()
298 setOperationAction(ISD::VECREDUCE_UMIN, VT, Legal); in addMVEVectorTypes()
299 setOperationAction(ISD::VECREDUCE_MUL, VT, Custom); in addMVEVectorTypes()
300 setOperationAction(ISD::VECREDUCE_AND, VT, Custom); in addMVEVectorTypes()
301 setOperationAction(ISD::VECREDUCE_OR, VT, Custom); in addMVEVectorTypes()
302 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom); in addMVEVectorTypes()
305 setOperationAction(ISD::SINT_TO_FP, VT, Expand); in addMVEVectorTypes()
306 setOperationAction(ISD::UINT_TO_FP, VT, Expand); in addMVEVectorTypes()
307 setOperationAction(ISD::FP_TO_SINT, VT, Expand); in addMVEVectorTypes()
308 setOperationAction(ISD::FP_TO_UINT, VT, Expand); in addMVEVectorTypes()
310 setOperationAction(ISD::FP_TO_SINT_SAT, VT, Custom); in addMVEVectorTypes()
311 setOperationAction(ISD::FP_TO_UINT_SAT, VT, Custom); in addMVEVectorTypes()
317 setIndexedLoadAction(im, VT, Legal); in addMVEVectorTypes()
318 setIndexedStoreAction(im, VT, Legal); in addMVEVectorTypes()
319 setIndexedMaskedLoadAction(im, VT, Legal); in addMVEVectorTypes()
320 setIndexedMaskedStoreAction(im, VT, Legal); in addMVEVectorTypes()
325 for (auto VT : FloatTypes) { in addMVEVectorTypes() local
326 addRegisterClass(VT, &ARM::MQPRRegClass); in addMVEVectorTypes()
328 setAllExpand(VT); in addMVEVectorTypes()
331 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in addMVEVectorTypes()
332 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in addMVEVectorTypes()
333 setOperationAction(ISD::INSERT_VECTOR_ELT, VT.getVectorElementType(), Custom); in addMVEVectorTypes()
334 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in addMVEVectorTypes()
335 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in addMVEVectorTypes()
336 setOperationAction(ISD::BUILD_VECTOR, VT.getVectorElementType(), Custom); in addMVEVectorTypes()
337 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Legal); in addMVEVectorTypes()
338 setOperationAction(ISD::SETCC, VT, Custom); in addMVEVectorTypes()
339 setOperationAction(ISD::MLOAD, VT, Custom); in addMVEVectorTypes()
340 setOperationAction(ISD::MSTORE, VT, Legal); in addMVEVectorTypes()
341 setOperationAction(ISD::SELECT, VT, Expand); in addMVEVectorTypes()
342 setOperationAction(ISD::SELECT_CC, VT, Expand); in addMVEVectorTypes()
347 setIndexedLoadAction(im, VT, Legal); in addMVEVectorTypes()
348 setIndexedStoreAction(im, VT, Legal); in addMVEVectorTypes()
349 setIndexedMaskedLoadAction(im, VT, Legal); in addMVEVectorTypes()
350 setIndexedMaskedStoreAction(im, VT, Legal); in addMVEVectorTypes()
354 setOperationAction(ISD::FMINNUM, VT, Legal); in addMVEVectorTypes()
355 setOperationAction(ISD::FMAXNUM, VT, Legal); in addMVEVectorTypes()
356 setOperationAction(ISD::FROUND, VT, Legal); in addMVEVectorTypes()
357 setOperationAction(ISD::VECREDUCE_FADD, VT, Custom); in addMVEVectorTypes()
358 setOperationAction(ISD::VECREDUCE_FMUL, VT, Custom); in addMVEVectorTypes()
359 setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); in addMVEVectorTypes()
360 setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); in addMVEVectorTypes()
363 setOperationAction(ISD::FDIV, VT, Expand); in addMVEVectorTypes()
364 setOperationAction(ISD::FREM, VT, Expand); in addMVEVectorTypes()
365 setOperationAction(ISD::FSQRT, VT, Expand); in addMVEVectorTypes()
366 setOperationAction(ISD::FSIN, VT, Expand); in addMVEVectorTypes()
367 setOperationAction(ISD::FCOS, VT, Expand); in addMVEVectorTypes()
368 setOperationAction(ISD::FPOW, VT, Expand); in addMVEVectorTypes()
369 setOperationAction(ISD::FLOG, VT, Expand); in addMVEVectorTypes()
370 setOperationAction(ISD::FLOG2, VT, Expand); in addMVEVectorTypes()
371 setOperationAction(ISD::FLOG10, VT, Expand); in addMVEVectorTypes()
372 setOperationAction(ISD::FEXP, VT, Expand); in addMVEVectorTypes()
373 setOperationAction(ISD::FEXP2, VT, Expand); in addMVEVectorTypes()
374 setOperationAction(ISD::FEXP10, VT, Expand); in addMVEVectorTypes()
375 setOperationAction(ISD::FNEARBYINT, VT, Expand); in addMVEVectorTypes()
394 for (auto VT : LongTypes) { in addMVEVectorTypes() local
395 addRegisterClass(VT, &ARM::MQPRRegClass); in addMVEVectorTypes()
396 setAllExpand(VT); in addMVEVectorTypes()
397 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in addMVEVectorTypes()
398 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in addMVEVectorTypes()
399 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in addMVEVectorTypes()
400 setOperationAction(ISD::VSELECT, VT, Legal); in addMVEVectorTypes()
401 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in addMVEVectorTypes()
430 for (auto VT : {MVT::v8i8, MVT::v4i8, MVT::v4i16}) { in addMVEVectorTypes()
431 setIndexedLoadAction(im, VT, Legal); in addMVEVectorTypes()
432 setIndexedStoreAction(im, VT, Legal); in addMVEVectorTypes()
433 setIndexedMaskedLoadAction(im, VT, Legal); in addMVEVectorTypes()
434 setIndexedMaskedStoreAction(im, VT, Legal); in addMVEVectorTypes()
440 for (auto VT : pTypes) { in addMVEVectorTypes() local
441 addRegisterClass(VT, &ARM::VCCRRegClass); in addMVEVectorTypes()
442 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in addMVEVectorTypes()
443 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in addMVEVectorTypes()
444 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); in addMVEVectorTypes()
445 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in addMVEVectorTypes()
446 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in addMVEVectorTypes()
447 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in addMVEVectorTypes()
448 setOperationAction(ISD::SETCC, VT, Custom); in addMVEVectorTypes()
449 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Expand); in addMVEVectorTypes()
450 setOperationAction(ISD::LOAD, VT, Custom); in addMVEVectorTypes()
451 setOperationAction(ISD::STORE, VT, Custom); in addMVEVectorTypes()
452 setOperationAction(ISD::TRUNCATE, VT, Custom); in addMVEVectorTypes()
453 setOperationAction(ISD::VSELECT, VT, Expand); in addMVEVectorTypes()
454 setOperationAction(ISD::SELECT, VT, Expand); in addMVEVectorTypes()
455 setOperationAction(ISD::SELECT_CC, VT, Expand); in addMVEVectorTypes()
458 setOperationAction(ISD::SINT_TO_FP, VT, Expand); in addMVEVectorTypes()
459 setOperationAction(ISD::UINT_TO_FP, VT, Expand); in addMVEVectorTypes()
460 setOperationAction(ISD::FP_TO_SINT, VT, Expand); in addMVEVectorTypes()
461 setOperationAction(ISD::FP_TO_UINT, VT, Expand); in addMVEVectorTypes()
804 for (MVT VT : MVT::fixedlen_vector_valuetypes()) { in ARMTargetLowering() local
806 setTruncStoreAction(VT, InnerVT, Expand); in ARMTargetLowering()
807 addAllExtLoads(VT, InnerVT, Expand); in ARMTargetLowering()
810 setOperationAction(ISD::SMUL_LOHI, VT, Expand); in ARMTargetLowering()
811 setOperationAction(ISD::UMUL_LOHI, VT, Expand); in ARMTargetLowering()
813 setOperationAction(ISD::BSWAP, VT, Expand); in ARMTargetLowering()
991 for (MVT VT : MVT::fixedlen_vector_valuetypes()) { in ARMTargetLowering() local
992 setOperationAction(ISD::MULHS, VT, Expand); in ARMTargetLowering()
993 setOperationAction(ISD::MULHU, VT, Expand); in ARMTargetLowering()
1008 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) { in ARMTargetLowering() local
1009 setLoadExtAction(ISD::EXTLOAD, VT, Ty, Legal); in ARMTargetLowering()
1010 setLoadExtAction(ISD::ZEXTLOAD, VT, Ty, Legal); in ARMTargetLowering()
1011 setLoadExtAction(ISD::SEXTLOAD, VT, Ty, Legal); in ARMTargetLowering()
1015 for (auto VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16, in ARMTargetLowering()
1017 setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom); in ARMTargetLowering()
1018 setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom); in ARMTargetLowering()
1019 setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom); in ARMTargetLowering()
1020 setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom); in ARMTargetLowering()
1102 for (MVT VT : MVT::fp_valuetypes()) { in ARMTargetLowering() local
1103 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand); in ARMTargetLowering()
1104 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand); in ARMTargetLowering()
1113 for (MVT VT : MVT::integer_valuetypes()) in ARMTargetLowering() local
1114 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); in ARMTargetLowering()
1195 for (MVT VT : MVT::fixedlen_vector_valuetypes()) { in ARMTargetLowering() local
1196 setOperationAction(ISD::ROTL, VT, Expand); in ARMTargetLowering()
1197 setOperationAction(ISD::ROTR, VT, Expand); in ARMTargetLowering()
1651 MVT VT) const { in findRepresentativeClass()
1654 switch (VT.SimpleTy) { in findRepresentativeClass()
1656 return TargetLowering::findRepresentativeClass(TRI, VT); in findRepresentativeClass()
1903 EVT VT) const { in getSetCCResultType()
1904 if (!VT.isVector()) in getSetCCResultType()
1909 (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 || in getSetCCResultType()
1910 VT == MVT::v16i8)) || in getSetCCResultType()
1912 (VT == MVT::v2f64 || VT == MVT::v4f32 || VT == MVT::v8f16))) in getSetCCResultType()
1913 return MVT::getVectorVT(MVT::i1, VT.getVectorElementCount()); in getSetCCResultType()
1914 return VT.changeVectorElementTypeToInteger(); in getSetCCResultType()
1920 ARMTargetLowering::getRegClassFor(MVT VT, bool isDivergent) const { in getRegClassFor() argument
1927 if (VT == MVT::v4i64) in getRegClassFor()
1929 if (VT == MVT::v8i64) in getRegClassFor()
1933 if (VT == MVT::v4i64) in getRegClassFor()
1935 if (VT == MVT::v8i64) in getRegClassFor()
1938 return TargetLowering::getRegClassFor(VT); in getRegClassFor()
1969 EVT VT = N->getValueType(i); in getSchedulingPreference() local
1970 if (VT == MVT::Glue || VT == MVT::Other) in getSchedulingPreference()
1972 if (VT.isFloatingPoint() || VT.isVector()) in getSchedulingPreference()
2565 Outs[0].VT == MVT::i32) { in LowerCall()
2568 assert(!Ins.empty() && Ins[0].VT == MVT::i32 && in LowerCall()
4992 EVT VT = Op.getValueType(); in LowerSignedALUO() local
4994 SDValue Overflow = DAG.getNode(ARMISD::CMOV, dl, VT, TVal, FVal, in LowerSignedALUO()
5014 static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT, in ConvertCarryFlagToBooleanCarry() argument
5020 return DAG.getNode(ARMISD::ADDE, DL, DAG.getVTList(VT, MVT::i32), in ConvertCarryFlagToBooleanCarry()
5035 EVT VT = Op.getValueType(); in LowerUnsignedALUO() local
5036 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerUnsignedALUO()
5045 Overflow = ConvertCarryFlagToBooleanCarry(Value.getValue(1), VT, DAG); in LowerUnsignedALUO()
5050 Overflow = ConvertCarryFlagToBooleanCarry(Value.getValue(1), VT, DAG); in LowerUnsignedALUO()
5064 EVT VT = Op.getValueType(); in LowerADDSUBSAT() local
5067 if (!VT.isSimple()) in LowerADDSUBSAT()
5071 switch (VT.getSimpleVT().SimpleTy) { in LowerADDSUBSAT()
5113 return DAG.getNode(ISD::TRUNCATE, dl, VT, Add); in LowerADDSUBSAT()
5133 EVT VT = Op.getValueType(); in LowerSELECT() local
5135 return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, CCR, in LowerSELECT()
5165 EVT VT = Op.getValueType(); in LowerSELECT() local
5169 assert(True.getValueType() == VT); in LowerSELECT()
5170 return getCMOV(dl, VT, True, False, ARMcc, CCR, Cmp, DAG); in LowerSELECT()
5235 SDValue ARMTargetLowering::getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal, in getCMOV() argument
5238 if (!Subtarget->hasFP64() && VT == MVT::f64) { in getCMOV()
5256 return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMcc, CCR, in getCMOV()
5302 EVT VT = Op.getValueType(); in LowerSaturatingConditional() local
5350 return DAG.getNode(ARMISD::SSAT, dl, VT, V2Tmp, in LowerSaturatingConditional()
5351 DAG.getConstant(llvm::countr_one(K), dl, VT)); in LowerSaturatingConditional()
5353 return DAG.getNode(ARMISD::USAT, dl, VT, V2Tmp, in LowerSaturatingConditional()
5354 DAG.getConstant(llvm::countr_one(K), dl, VT)); in LowerSaturatingConditional()
5401 bool ARMTargetLowering::isUnsupportedFloatingType(EVT VT) const { in isUnsupportedFloatingType()
5402 if (VT == MVT::f32) in isUnsupportedFloatingType()
5404 if (VT == MVT::f64) in isUnsupportedFloatingType()
5406 if (VT == MVT::f16) in isUnsupportedFloatingType()
5412 EVT VT = Op.getValueType(); in LowerSELECT_CC() local
5428 if (VT == MVT::i32 && in LowerSELECT_CC()
5430 SDValue ShiftV = DAG.getNode(ISD::SRA, dl, VT, SatValue, in LowerSELECT_CC()
5431 DAG.getConstant(31, dl, VT)); in LowerSELECT_CC()
5433 SDValue NotShiftV = DAG.getNode(ISD::XOR, dl, VT, ShiftV, in LowerSELECT_CC()
5434 DAG.getAllOnesConstant(dl, VT)); in LowerSELECT_CC()
5435 return DAG.getNode(ISD::AND, dl, VT, SatValue, NotShiftV); in LowerSELECT_CC()
5437 return DAG.getNode(ISD::OR, dl, VT, SatValue, ShiftV); in LowerSELECT_CC()
5491 EVT VT = TrueVal.getValueType(); in LowerSELECT_CC() local
5492 return DAG.getNode(Opcode, dl, VT, TrueVal, FalseVal, ARMcc, Cmp); in LowerSELECT_CC()
5536 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG); in LowerSELECT_CC()
5567 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG); in LowerSELECT_CC()
5572 Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, CCR, Cmp2, DAG); in LowerSELECT_CC()
5587 EVT VT = Op.getValueType(); in canChangeToInt() local
5588 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow()) in canChangeToInt()
5851 EVT VT = Op.getValueType(); in LowerVectorFP_TO_INT() local
5873 if (VT != MVT::v4i16 && VT != MVT::v8i16) in LowerVectorFP_TO_INT()
5877 return DAG.getNode(ISD::TRUNCATE, dl, VT, Op); in LowerVectorFP_TO_INT()
5881 EVT VT = Op.getValueType(); in LowerFP_TO_INT() local
5882 if (VT.isVector()) in LowerFP_TO_INT()
5921 EVT VT = Op.getValueType(); in LowerFP_TO_INT_SAT() local
5925 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f32) in LowerFP_TO_INT_SAT()
5927 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f64 && in LowerFP_TO_INT_SAT()
5930 if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f16 && in LowerFP_TO_INT_SAT()
5933 if (VT == MVT::v4i32 && ToVT == MVT::i32 && FromVT == MVT::v4f32 && in LowerFP_TO_INT_SAT()
5936 if (VT == MVT::v8i16 && ToVT == MVT::i16 && FromVT == MVT::v8f16 && in LowerFP_TO_INT_SAT()
5946 SDValue CVT = DAG.getNode(Op.getOpcode(), DL, VT, Op.getOperand(0), in LowerFP_TO_INT_SAT()
5947 DAG.getValueType(VT.getScalarType())); in LowerFP_TO_INT_SAT()
5948 SDValue Max = DAG.getNode(IsSigned ? ISD::SMIN : ISD::UMIN, DL, VT, CVT, in LowerFP_TO_INT_SAT()
5949 DAG.getConstant((1 << BW) - 1, DL, VT)); in LowerFP_TO_INT_SAT()
5951 Max = DAG.getNode(ISD::SMAX, DL, VT, Max, in LowerFP_TO_INT_SAT()
5952 DAG.getConstant(-(1 << BW), DL, VT)); in LowerFP_TO_INT_SAT()
5957 EVT VT = Op.getValueType(); in LowerVectorINT_TO_FP() local
5961 if (VT.getVectorElementType() == MVT::f32) in LowerVectorINT_TO_FP()
5973 if (VT == MVT::v4f32) in LowerVectorINT_TO_FP()
5975 else if (VT == MVT::v4f16 && HasFullFP16) in LowerVectorINT_TO_FP()
5977 else if (VT == MVT::v8f16 && HasFullFP16) in LowerVectorINT_TO_FP()
5997 return DAG.getNode(Opc, dl, VT, Op); in LowerVectorINT_TO_FP()
6001 EVT VT = Op.getValueType(); in LowerINT_TO_FP() local
6002 if (VT.isVector()) in LowerINT_TO_FP()
6004 if (isUnsupportedFloatingType(VT)) { in LowerINT_TO_FP()
6025 EVT VT = Op.getValueType(); in LowerFCOPYSIGN() local
6036 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64; in LowerFCOPYSIGN()
6037 if (VT == MVT::f64) in LowerFCOPYSIGN()
6045 if (VT == MVT::f64) in LowerFCOPYSIGN()
6049 } else if (VT == MVT::f32) in LowerFCOPYSIGN()
6065 if (VT == MVT::f32) { in LowerFCOPYSIGN()
6086 if (VT == MVT::f32) { in LowerFCOPYSIGN()
6110 EVT VT = Op.getValueType(); in LowerRETURNADDR() local
6116 return DAG.getLoad(VT, dl, DAG.getEntryNode(), in LowerRETURNADDR()
6117 DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset), in LowerRETURNADDR()
6123 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT); in LowerRETURNADDR()
6133 EVT VT = Op.getValueType(); in LowerFRAMEADDR() local
6137 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT); in LowerFRAMEADDR()
6139 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr, in LowerFRAMEADDR()
6146 Register ARMTargetLowering::getRegisterByName(const char* RegName, LLT VT, in getRegisterByName() argument
6293 static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) { in getZeroVector() argument
6294 assert(VT.isVector() && "Expected a vector type"); in getZeroVector()
6297 EVT VmovVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32; in getZeroVector()
6299 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov); in getZeroVector()
6307 EVT VT = Op.getValueType(); in LowerShiftRightParts() local
6308 unsigned VTBits = VT.getSizeInBits(); in LowerShiftRightParts()
6321 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt); in LowerShiftRightParts()
6324 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt); in LowerShiftRightParts()
6325 SDValue LoSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); in LowerShiftRightParts()
6326 SDValue LoBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt); in LowerShiftRightParts()
6329 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift, LoBigShift, in LowerShiftRightParts()
6332 SDValue HiSmallShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt); in LowerShiftRightParts()
6334 ? DAG.getNode(Opc, dl, VT, ShOpHi, in LowerShiftRightParts()
6335 DAG.getConstant(VTBits - 1, dl, VT)) in LowerShiftRightParts()
6336 : DAG.getConstant(0, dl, VT); in LowerShiftRightParts()
6339 SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift, in LowerShiftRightParts()
6351 EVT VT = Op.getValueType(); in LowerShiftLeftParts() local
6352 unsigned VTBits = VT.getSizeInBits(); in LowerShiftLeftParts()
6363 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt); in LowerShiftLeftParts()
6364 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt); in LowerShiftLeftParts()
6365 SDValue HiSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); in LowerShiftLeftParts()
6369 SDValue HiBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt); in LowerShiftLeftParts()
6372 SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift, in LowerShiftLeftParts()
6377 SDValue LoSmallShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt); in LowerShiftLeftParts()
6378 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift, in LowerShiftLeftParts()
6379 DAG.getConstant(0, dl, VT), ARMcc, CCR, CmpLo); in LowerShiftLeftParts()
6504 EVT VT = N->getValueType(0); in LowerCTTZ() local
6505 if (VT.isVector() && ST->hasNEON()) { in LowerCTTZ()
6509 SDValue NX = DAG.getNode(ISD::SUB, dl, VT, getZeroVector(VT, DAG, dl), X); in LowerCTTZ()
6510 SDValue LSB = DAG.getNode(ISD::AND, dl, VT, X, NX); in LowerCTTZ()
6512 EVT ElemTy = VT.getVectorElementType(); in LowerCTTZ()
6516 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT, in LowerCTTZ()
6518 SDValue Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One); in LowerCTTZ()
6519 return DAG.getNode(ISD::CTPOP, dl, VT, Bits); in LowerCTTZ()
6527 DAG.getNode(ARMISD::VMOVIMM, dl, VT, in LowerCTTZ()
6529 SDValue CTLZ = DAG.getNode(ISD::CTLZ, dl, VT, LSB); in LowerCTTZ()
6530 return DAG.getNode(ISD::SUB, dl, VT, WidthMinus1, CTLZ); in LowerCTTZ()
6539 SDValue FF = DAG.getNode(ARMISD::VMOVIMM, dl, VT, in LowerCTTZ()
6541 Bits = DAG.getNode(ISD::ADD, dl, VT, LSB, FF); in LowerCTTZ()
6543 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT, in LowerCTTZ()
6545 Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One); in LowerCTTZ()
6547 return DAG.getNode(ISD::CTPOP, dl, VT, Bits); in LowerCTTZ()
6553 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, VT, N->getOperand(0)); in LowerCTTZ()
6554 return DAG.getNode(ISD::CTLZ, dl, VT, rbit); in LowerCTTZ()
6559 EVT VT = N->getValueType(0); in LowerCTPOP() local
6563 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 || in LowerCTPOP()
6564 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) && in LowerCTPOP()
6568 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8; in LowerCTPOP()
6574 unsigned NumElts = VT.is64BitVector() ? 8 : 16; in LowerCTPOP()
6575 while (EltSize != VT.getScalarSizeInBits()) { in LowerCTPOP()
6614 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) { in isVShiftLImm() argument
6615 assert(VT.isVector() && "vector shift count is not a vector type"); in isVShiftLImm()
6616 int64_t ElementBits = VT.getScalarSizeInBits(); in isVShiftLImm()
6628 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, bool isIntrinsic, in isVShiftRImm() argument
6630 assert(VT.isVector() && "vector shift count is not a vector type"); in isVShiftRImm()
6631 int64_t ElementBits = VT.getScalarSizeInBits(); in isVShiftRImm()
6645 EVT VT = N->getValueType(0); in LowerShift() local
6649 if (!VT.isVector()) in LowerShift()
6659 if (isVShiftLImm(N->getOperand(1), VT, false, Cnt)) in LowerShift()
6660 return DAG.getNode(ARMISD::VSHLIMM, dl, VT, N->getOperand(0), in LowerShift()
6662 return DAG.getNode(ARMISD::VSHLu, dl, VT, N->getOperand(0), in LowerShift()
6669 if (isVShiftRImm(N->getOperand(1), VT, false, false, Cnt)) { in LowerShift()
6672 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0), in LowerShift()
6683 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0), NegatedCount); in LowerShift()
6688 EVT VT = N->getValueType(0); in Expand64BitShift() local
6692 if (VT != MVT::i64) in Expand64BitShift()
6772 EVT VT = Op.getValueType(); in LowerVSETCC() local
6791 CmpVT = VT; in LowerVSETCC()
6809 Merged = DAG.getSExtOrTrunc(Merged, dl, VT); in LowerVSETCC()
6850 Result = DAG.getNOT(dl, Result, VT); in LowerVSETCC()
6862 Result = DAG.getNOT(dl, Result, VT); in LowerVSETCC()
6904 Result = DAG.getNOT(dl, Result, VT); in LowerVSETCC()
6935 Result = DAG.getSExtOrTrunc(Result, dl, VT); in LowerVSETCC()
6938 Result = DAG.getNOT(dl, Result, VT); in LowerVSETCC()
6978 const SDLoc &dl, EVT &VT, EVT VectorVT, in isVMOVModifiedImm() argument
6999 VT = is128Bits ? MVT::v16i8 : MVT::v8i8; in isVMOVModifiedImm()
7004 VT = is128Bits ? MVT::v8i16 : MVT::v4i16; in isVMOVModifiedImm()
7024 VT = is128Bits ? MVT::v4i32 : MVT::v2i32; in isVMOVModifiedImm()
7112 VT = is128Bits ? MVT::v2i64 : MVT::v1i64; in isVMOVModifiedImm()
7126 EVT VT = Op.getValueType(); in LowerConstantFP() local
7127 bool IsDouble = (VT == MVT::f64); in LowerConstantFP()
7139 if (isFPImmLegal(FPVal, VT)) in LowerConstantFP()
7144 switch (VT.getSimpleVT().SimpleTy) { in LowerConstantFP()
7154 return DAG.getNode(ARMISD::VMOVSR, DL, VT, in LowerConstantFP()
7203 VMovVT, VT, VMOVModImm); in LowerConstantFP()
7220 VT, VMVNModImm); in LowerConstantFP()
7240 static bool isSingletonVEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) { in isSingletonVEXTMask() argument
7241 unsigned NumElts = VT.getVectorNumElements(); in isSingletonVEXTMask()
7268 static bool isVEXTMask(ArrayRef<int> M, EVT VT, in isVEXTMask() argument
7270 unsigned NumElts = VT.getVectorNumElements(); in isVEXTMask()
7304 static bool isVTBLMask(ArrayRef<int> M, EVT VT) { in isVTBLMask() argument
7308 return VT == MVT::v8i8 && M.size() == 8; in isVTBLMask()
7339 static bool isVTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isVTRNMask() argument
7340 unsigned EltSz = VT.getScalarSizeInBits(); in isVTRNMask()
7344 unsigned NumElts = VT.getVectorNumElements(); in isVTRNMask()
7371 static bool isVTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){ in isVTRN_v_undef_Mask() argument
7372 unsigned EltSz = VT.getScalarSizeInBits(); in isVTRN_v_undef_Mask()
7376 unsigned NumElts = VT.getVectorNumElements(); in isVTRN_v_undef_Mask()
7403 static bool isVUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isVUZPMask() argument
7404 unsigned EltSz = VT.getScalarSizeInBits(); in isVUZPMask()
7408 unsigned NumElts = VT.getVectorNumElements(); in isVUZPMask()
7424 if (VT.is64BitVector() && EltSz == 32) in isVUZPMask()
7433 static bool isVUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){ in isVUZP_v_undef_Mask() argument
7434 unsigned EltSz = VT.getScalarSizeInBits(); in isVUZP_v_undef_Mask()
7438 unsigned NumElts = VT.getVectorNumElements(); in isVUZP_v_undef_Mask()
7460 if (VT.is64BitVector() && EltSz == 32) in isVUZP_v_undef_Mask()
7474 static bool isVZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isVZIPMask() argument
7475 unsigned EltSz = VT.getScalarSizeInBits(); in isVZIPMask()
7479 unsigned NumElts = VT.getVectorNumElements(); in isVZIPMask()
7498 if (VT.is64BitVector() && EltSz == 32) in isVZIPMask()
7507 static bool isVZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){ in isVZIP_v_undef_Mask() argument
7508 unsigned EltSz = VT.getScalarSizeInBits(); in isVZIP_v_undef_Mask()
7512 unsigned NumElts = VT.getVectorNumElements(); in isVZIP_v_undef_Mask()
7531 if (VT.is64BitVector() && EltSz == 32) in isVZIP_v_undef_Mask()
7539 static unsigned isNEONTwoResultShuffleMask(ArrayRef<int> ShuffleMask, EVT VT, in isNEONTwoResultShuffleMask() argument
7543 if (isVTRNMask(ShuffleMask, VT, WhichResult)) in isNEONTwoResultShuffleMask()
7545 if (isVUZPMask(ShuffleMask, VT, WhichResult)) in isNEONTwoResultShuffleMask()
7547 if (isVZIPMask(ShuffleMask, VT, WhichResult)) in isNEONTwoResultShuffleMask()
7551 if (isVTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) in isNEONTwoResultShuffleMask()
7553 if (isVUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) in isNEONTwoResultShuffleMask()
7555 if (isVZIP_v_undef_Mask(ShuffleMask, VT, WhichResult)) in isNEONTwoResultShuffleMask()
7562 static bool isReverseMask(ArrayRef<int> M, EVT VT) { in isReverseMask() argument
7563 unsigned NumElts = VT.getVectorNumElements(); in isReverseMask()
7576 static bool isTruncMask(ArrayRef<int> M, EVT VT, bool Top, bool SingleSource) { in isTruncMask() argument
7577 unsigned NumElts = VT.getVectorNumElements(); in isTruncMask()
7579 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8)) in isTruncMask()
7598 static bool isVMOVNMask(ArrayRef<int> M, EVT VT, bool Top, bool SingleSource) { in isVMOVNMask() argument
7599 unsigned NumElts = VT.getVectorNumElements(); in isVMOVNMask()
7601 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8)) in isVMOVNMask()
7657 EVT VT = BV.getValueType(); in LowerBuildVectorOfFPTrunc() local
7658 if (VT != MVT::v8f16) in LowerBuildVectorOfFPTrunc()
7692 SDValue N1 = DAG.getNode(ARMISD::VCVTN, dl, VT, DAG.getUNDEF(VT), Op0, in LowerBuildVectorOfFPTrunc()
7694 return DAG.getNode(ARMISD::VCVTN, dl, VT, N1, Op1, in LowerBuildVectorOfFPTrunc()
7710 EVT VT = BV.getValueType(); in LowerBuildVectorOfFPExt() local
7711 if (VT != MVT::v4f32) in LowerBuildVectorOfFPExt()
7738 return DAG.getNode(ARMISD::VCVTL, dl, VT, Op0, in LowerBuildVectorOfFPExt()
7765 EVT VT = Op.getValueType(); in LowerBUILD_VECTOR_i1() local
7769 unsigned NumElts = VT.getVectorNumElements(); in LowerBUILD_VECTOR_i1()
7811 SDValue Base = DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, in LowerBUILD_VECTOR_i1()
7817 Base = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Base, V, in LowerBUILD_VECTOR_i1()
7830 EVT VT = Op.getValueType(); in LowerBUILD_VECTORToVIDUP() local
7832 unsigned NumElts = VT.getVectorNumElements(); in LowerBUILD_VECTORToVIDUP()
7853 return DAG.getNode(ARMISD::VIDUP, DL, DAG.getVTList(VT, MVT::i32), Op0, in LowerBUILD_VECTORToVIDUP()
7903 EVT VT = Op.getValueType(); in LowerBUILD_VECTOR() local
7905 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1) in LowerBUILD_VECTOR()
7916 return DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
7920 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == SplatBitSize && in LowerBUILD_VECTOR()
7929 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, VDup); in LowerBUILD_VECTOR()
7938 SplatBitSize, DAG, dl, VmovVT, VT, VMOVModImm); in LowerBUILD_VECTOR()
7942 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov); in LowerBUILD_VECTOR()
7949 VT, ST->hasMVEIntegerOps() ? MVEVMVNModImm : VMVNModImm); in LowerBUILD_VECTOR()
7952 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov); in LowerBUILD_VECTOR()
7956 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) { in LowerBUILD_VECTOR()
7960 return DAG.getNode(ARMISD::VMOVFPIMM, dl, VT, Val); in LowerBUILD_VECTOR()
7973 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, VDup); in LowerBUILD_VECTOR()
7985 unsigned NumElts = VT.getVectorNumElements(); in LowerBUILD_VECTOR()
8019 return DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
8024 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value); in LowerBUILD_VECTOR()
8026 unsigned EltSize = VT.getScalarSizeInBits(); in LowerBUILD_VECTOR()
8046 if (VT != Value->getOperand(0).getValueType()) { in LowerBUILD_VECTOR()
8048 VT.getVectorNumElements(); in LowerBUILD_VECTOR()
8049 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT, in LowerBUILD_VECTOR()
8050 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DAG.getUNDEF(VT), in LowerBUILD_VECTOR()
8054 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT, in LowerBUILD_VECTOR()
8057 N = DAG.getNode(ARMISD::VDUP, dl, VT, Value); in LowerBUILD_VECTOR()
8069 N = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ops); in LowerBUILD_VECTOR()
8074 if (VT.getVectorElementType().isFloatingPoint()) { in LowerBUILD_VECTOR()
8076 MVT FVT = VT.getVectorElementType().getSimpleVT(); in LowerBUILD_VECTOR()
8086 return DAG.getNode(ISD::BITCAST, dl, VT, Val); in LowerBUILD_VECTOR()
8091 return DAG.getNode(ARMISD::VDUP, dl, VT, Val); in LowerBUILD_VECTOR()
8115 if (ST->hasNEON() && VT.is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) { in LowerBUILD_VECTOR()
8119 EVT ExtVT = VT.getVectorElementType(); in LowerBUILD_VECTOR()
8129 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lower, Upper); in LowerBUILD_VECTOR()
8144 return DAG.getNode(ISD::BITCAST, dl, VT, Val); in LowerBUILD_VECTOR()
8154 SDValue Vec = DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
8160 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx); in LowerBUILD_VECTOR()
8174 EVT VT = Op.getValueType(); in ReconstructShuffle() local
8175 unsigned NumElts = VT.getVectorNumElements(); in ReconstructShuffle()
8233 EVT SmallestEltTy = VT.getVectorElementType(); in ReconstructShuffle()
8240 VT.getScalarSizeInBits() / SmallestEltTy.getSizeInBits(); in ReconstructShuffle()
8241 NumElts = VT.getSizeInBits() / SmallestEltTy.getSizeInBits(); in ReconstructShuffle()
8251 uint64_t VTSize = VT.getFixedSizeInBits(); in ReconstructShuffle()
8328 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { in ReconstructShuffle()
8341 VT.getScalarSizeInBits()); in ReconstructShuffle()
8367 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Shuffle); in ReconstructShuffle()
8406 bool ARMTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const { in isShuffleMaskLegal()
8407 if (VT.getVectorNumElements() == 4 && in isShuffleMaskLegal()
8408 (VT.is128BitVector() || VT.is64BitVector())) { in isShuffleMaskLegal()
8430 unsigned EltSize = VT.getScalarSizeInBits(); in isShuffleMaskLegal()
8432 ShuffleVectorSDNode::isSplatMask(&M[0], VT) || in isShuffleMaskLegal()
8434 isVREVMask(M, VT, 64) || in isShuffleMaskLegal()
8435 isVREVMask(M, VT, 32) || in isShuffleMaskLegal()
8436 isVREVMask(M, VT, 16)) in isShuffleMaskLegal()
8439 (isVEXTMask(M, VT, ReverseVEXT, Imm) || in isShuffleMaskLegal()
8440 isVTBLMask(M, VT) || in isShuffleMaskLegal()
8441 isNEONTwoResultShuffleMask(M, VT, WhichResult, isV_UNDEF))) in isShuffleMaskLegal()
8443 else if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) && in isShuffleMaskLegal()
8444 isReverseMask(M, VT)) in isShuffleMaskLegal()
8447 (isVMOVNMask(M, VT, true, false) || in isShuffleMaskLegal()
8448 isVMOVNMask(M, VT, false, false) || isVMOVNMask(M, VT, true, true))) in isShuffleMaskLegal()
8451 (isTruncMask(M, VT, false, false) || in isShuffleMaskLegal()
8452 isTruncMask(M, VT, false, true) || in isShuffleMaskLegal()
8453 isTruncMask(M, VT, true, false) || isTruncMask(M, VT, true, true))) in isShuffleMaskLegal()
8477 EVT VT = OpLHS.getValueType(); in GeneratePerfectShuffle() local
8483 if (VT.getScalarSizeInBits() == 32) in GeneratePerfectShuffle()
8484 return DAG.getNode(ARMISD::VREV64, dl, VT, OpLHS); in GeneratePerfectShuffle()
8486 if (VT.getScalarSizeInBits() == 16) in GeneratePerfectShuffle()
8487 return DAG.getNode(ARMISD::VREV32, dl, VT, OpLHS); in GeneratePerfectShuffle()
8489 assert(VT.getScalarSizeInBits() == 8); in GeneratePerfectShuffle()
8490 return DAG.getNode(ARMISD::VREV16, dl, VT, OpLHS); in GeneratePerfectShuffle()
8495 return DAG.getNode(ARMISD::VDUPLANE, dl, VT, in GeneratePerfectShuffle()
8500 return DAG.getNode(ARMISD::VEXT, dl, VT, in GeneratePerfectShuffle()
8505 return DAG.getNode(ARMISD::VUZP, dl, DAG.getVTList(VT, VT), in GeneratePerfectShuffle()
8509 return DAG.getNode(ARMISD::VZIP, dl, DAG.getVTList(VT, VT), in GeneratePerfectShuffle()
8513 return DAG.getNode(ARMISD::VTRN, dl, DAG.getVTList(VT, VT), in GeneratePerfectShuffle()
8540 EVT VT = Op.getValueType(); in LowerReverse_VECTOR_SHUFFLE() local
8542 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) && in LowerReverse_VECTOR_SHUFFLE()
8544 SDValue OpLHS = DAG.getNode(ARMISD::VREV64, DL, VT, Op.getOperand(0)); in LowerReverse_VECTOR_SHUFFLE()
8550 for (unsigned i = 0; i < VT.getVectorNumElements() / 2; i++) in LowerReverse_VECTOR_SHUFFLE()
8551 NewMask.push_back(VT.getVectorNumElements() / 2 + i); in LowerReverse_VECTOR_SHUFFLE()
8552 for (unsigned i = 0; i < VT.getVectorNumElements() / 2; i++) in LowerReverse_VECTOR_SHUFFLE()
8554 return DAG.getVectorShuffle(VT, DL, OpLHS, OpLHS, NewMask); in LowerReverse_VECTOR_SHUFFLE()
8557 static EVT getVectorTyFromPredicateVector(EVT VT) { in getVectorTyFromPredicateVector() argument
8558 switch (VT.getSimpleVT().SimpleTy) { in getVectorTyFromPredicateVector()
8572 static SDValue PromoteMVEPredVector(SDLoc dl, SDValue Pred, EVT VT, in PromoteMVEPredVector() argument
8586 EVT NewVT = getVectorTyFromPredicateVector(VT); in PromoteMVEPredVector()
8593 if (VT != MVT::v16i1) in PromoteMVEPredVector()
8609 EVT VT = Op.getValueType(); in LowerVECTOR_SHUFFLE_i1() local
8619 if (isReverseMask(ShuffleMask, VT)) { in LowerVECTOR_SHUFFLE_i1()
8624 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, srl); in LowerVECTOR_SHUFFLE_i1()
8635 SDValue PredAsVector1 = PromoteMVEPredVector(dl, V1, VT, DAG); in LowerVECTOR_SHUFFLE_i1()
8638 : PromoteMVEPredVector(dl, V2, VT, DAG); in LowerVECTOR_SHUFFLE_i1()
8649 if (VT == MVT::v2i1) { in LowerVECTOR_SHUFFLE_i1()
8655 return DAG.getNode(ARMISD::VCMPZ, dl, VT, Shuffled, in LowerVECTOR_SHUFFLE_i1()
8666 EVT VT = Op.getValueType(); in LowerVECTOR_SHUFFLEUsingMovs() local
8667 if (VT.getScalarSizeInBits() >= 32) in LowerVECTOR_SHUFFLEUsingMovs()
8670 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) && in LowerVECTOR_SHUFFLEUsingMovs()
8672 int NumElts = VT.getVectorNumElements(); in LowerVECTOR_SHUFFLEUsingMovs()
8731 VT, dl, Op->getOperand(0), Op->getOperand(1), NewShuffleMask); in LowerVECTOR_SHUFFLEUsingMovs()
8742 return DAG.getBitcast(VT, NewVec); in LowerVECTOR_SHUFFLEUsingMovs()
8750 EVT VT = Op.getValueType(); in LowerVECTOR_SHUFFLEUsingOneOff() local
8751 unsigned NumElts = VT.getVectorNumElements(); in LowerVECTOR_SHUFFLEUsingOneOff()
8758 auto isOneOffIdentityMask = [](ArrayRef<int> Mask, EVT VT, int BaseOffset, in LowerVECTOR_SHUFFLEUsingOneOff()
8777 if (isOneOffIdentityMask(ShuffleMask, VT, 0, OffElement)) in LowerVECTOR_SHUFFLEUsingOneOff()
8779 else if (isOneOffIdentityMask(ShuffleMask, VT, NumElts, OffElement)) in LowerVECTOR_SHUFFLEUsingOneOff()
8785 EVT SVT = VT.getScalarType() == MVT::i8 || VT.getScalarType() == MVT::i16 in LowerVECTOR_SHUFFLEUsingOneOff()
8787 : VT.getScalarType(); in LowerVECTOR_SHUFFLEUsingOneOff()
8792 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, VInput, Elt, in LowerVECTOR_SHUFFLEUsingOneOff()
8801 EVT VT = Op.getValueType(); in LowerVECTOR_SHUFFLE() local
8803 unsigned EltSize = VT.getScalarSizeInBits(); in LowerVECTOR_SHUFFLE()
8824 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0)); in LowerVECTOR_SHUFFLE()
8838 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0)); in LowerVECTOR_SHUFFLE()
8840 return DAG.getNode(ARMISD::VDUPLANE, dl, VT, V1, in LowerVECTOR_SHUFFLE()
8846 if (ST->hasNEON() && isVEXTMask(ShuffleMask, VT, ReverseVEXT, Imm)) { in LowerVECTOR_SHUFFLE()
8849 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V2, in LowerVECTOR_SHUFFLE()
8853 if (isVREVMask(ShuffleMask, VT, 64)) in LowerVECTOR_SHUFFLE()
8854 return DAG.getNode(ARMISD::VREV64, dl, VT, V1); in LowerVECTOR_SHUFFLE()
8855 if (isVREVMask(ShuffleMask, VT, 32)) in LowerVECTOR_SHUFFLE()
8856 return DAG.getNode(ARMISD::VREV32, dl, VT, V1); in LowerVECTOR_SHUFFLE()
8857 if (isVREVMask(ShuffleMask, VT, 16)) in LowerVECTOR_SHUFFLE()
8858 return DAG.getNode(ARMISD::VREV16, dl, VT, V1); in LowerVECTOR_SHUFFLE()
8860 if (ST->hasNEON() && V2->isUndef() && isSingletonVEXTMask(ShuffleMask, VT, Imm)) { in LowerVECTOR_SHUFFLE()
8861 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V1, in LowerVECTOR_SHUFFLE()
8874 ShuffleMask, VT, WhichResult, isV_UNDEF)) { in LowerVECTOR_SHUFFLE()
8877 return DAG.getNode(ShuffleOpc, dl, DAG.getVTList(VT, VT), V1, V2) in LowerVECTOR_SHUFFLE()
8882 if (isVMOVNMask(ShuffleMask, VT, false, false)) in LowerVECTOR_SHUFFLE()
8883 return DAG.getNode(ARMISD::VMOVN, dl, VT, V2, V1, in LowerVECTOR_SHUFFLE()
8885 if (isVMOVNMask(ShuffleMask, VT, true, false)) in LowerVECTOR_SHUFFLE()
8886 return DAG.getNode(ARMISD::VMOVN, dl, VT, V1, V2, in LowerVECTOR_SHUFFLE()
8888 if (isVMOVNMask(ShuffleMask, VT, true, true)) in LowerVECTOR_SHUFFLE()
8889 return DAG.getNode(ARMISD::VMOVN, dl, VT, V1, V1, in LowerVECTOR_SHUFFLE()
8915 return i < (int)VT.getVectorNumElements(); in LowerVECTOR_SHUFFLE()
8926 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Res.getValue(0), in LowerVECTOR_SHUFFLE()
8938 if (isTruncMask(ShuffleMask, VT, Top, SingleSource)) { in LowerVECTOR_SHUFFLE()
8949 return DAG.getNode(ARMISD::MVETRUNC, dl, VT, Lo, Hi); in LowerVECTOR_SHUFFLE()
8957 unsigned NumElts = VT.getVectorNumElements(); in LowerVECTOR_SHUFFLE()
9006 return DAG.getNode(ISD::BITCAST, dl, VT, Val); in LowerVECTOR_SHUFFLE()
9009 if ((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) && in LowerVECTOR_SHUFFLE()
9010 isReverseMask(ShuffleMask, VT)) in LowerVECTOR_SHUFFLE()
9013 if (ST->hasNEON() && VT == MVT::v8i8) in LowerVECTOR_SHUFFLE()
9113 EVT VT = Vec.getValueType(); in LowerEXTRACT_VECTOR_ELT() local
9115 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1) in LowerEXTRACT_VECTOR_ELT()
9142 EVT VT = Op1VT.getDoubleNumVectorElementsVT(*DAG.getContext()); in LowerCONCAT_VECTORS_i1() local
9150 getVectorTyFromPredicateVector(VT).getScalarType().getSimpleVT(); in LowerCONCAT_VECTORS_i1()
9159 return DAG.getNode(ARMISD::VCMPZ, dl, VT, ConVec, in LowerCONCAT_VECTORS_i1()
9191 return DAG.getNode(ARMISD::VCMPZ, dl, VT, ConVec, in LowerCONCAT_VECTORS_i1()
9210 EVT VT = Op->getValueType(0); in LowerCONCAT_VECTORS() local
9211 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1) in LowerCONCAT_VECTORS()
9238 EVT VT = Op.getValueType(); in LowerEXTRACT_SUBVECTOR() local
9240 unsigned NumElts = VT.getVectorNumElements(); in LowerEXTRACT_SUBVECTOR()
9243 assert(VT.getScalarSizeInBits() == 1 && in LowerEXTRACT_SUBVECTOR()
9253 MVT ElType = getVectorTyFromPredicateVector(VT).getScalarType().getSimpleVT(); in LowerEXTRACT_SUBVECTOR()
9282 return DAG.getNode(ARMISD::VCMPZ, dl, VT, SubVec, in LowerEXTRACT_SUBVECTOR()
9290 EVT VT = N->getValueType(0); in LowerTruncatei1() local
9291 assert((VT == MVT::v16i1 || VT == MVT::v8i1 || VT == MVT::v4i1) && in LowerTruncatei1()
9299 return DAG.getNode(ISD::SETCC, DL, VT, And, DAG.getConstant(0, DL, FromVT), in LowerTruncatei1()
9402 EVT VT = N->getValueType(0); in isExtendedBUILD_VECTOR() local
9403 if (VT == MVT::v2i64 && N->getOpcode() == ISD::BITCAST) { in isExtendedBUILD_VECTOR()
9433 unsigned EltSize = VT.getScalarSizeInBits(); in isExtendedBUILD_VECTOR()
9572 EVT VT = N->getValueType(0); in SkipExtensionForVMULL() local
9573 unsigned EltSize = VT.getScalarSizeInBits() / 2; in SkipExtensionForVMULL()
9574 unsigned NumElts = VT.getVectorNumElements(); in SkipExtensionForVMULL()
9612 EVT VT = Op.getValueType(); in LowerMUL() local
9613 assert(VT.is128BitVector() && VT.isInteger() && in LowerMUL()
9645 if (VT == MVT::v2i64) in LowerMUL()
9663 return DAG.getNode(NewOpc, DL, VT, Op0, Op1); in LowerMUL()
9677 return DAG.getNode(N0->getOpcode(), DL, VT, in LowerMUL()
9678 DAG.getNode(NewOpc, DL, VT, in LowerMUL()
9680 DAG.getNode(NewOpc, DL, VT, in LowerMUL()
9756 EVT VT = Op.getValueType(); in LowerSDIV() local
9757 assert((VT == MVT::v4i16 || VT == MVT::v8i8) && in LowerSDIV()
9765 if (VT == MVT::v8i8) { in LowerSDIV()
9793 EVT VT = Op.getValueType(); in LowerUDIV() local
9794 assert((VT == MVT::v4i16 || VT == MVT::v8i8) && in LowerUDIV()
9802 if (VT == MVT::v8i8) { in LowerUDIV()
9869 EVT VT = N->getValueType(0); in LowerUADDSUBO_CARRY() local
9870 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerUADDSUBO_CARRY()
9886 Carry = ConvertCarryFlagToBooleanCarry(Result.getValue(1), VT, DAG); in LowerUADDSUBO_CARRY()
9900 Carry = ConvertCarryFlagToBooleanCarry(Result.getValue(1), VT, DAG); in LowerUADDSUBO_CARRY()
9989 EVT VT = Op.getValueType(); in LowerWindowsDIVLibCall() local
9990 assert((VT == MVT::i32 || VT == MVT::i64) && in LowerWindowsDIVLibCall()
9999 Name = (VT == MVT::i32) ? "__rt_sdiv" : "__rt_sdiv64"; in LowerWindowsDIVLibCall()
10001 Name = (VT == MVT::i32) ? "__rt_udiv" : "__rt_udiv64"; in LowerWindowsDIVLibCall()
10017 .setCallee(CallingConv::ARM_AAPCS_VFP, VT.getTypeForEVT(*DAG.getContext()), in LowerWindowsDIVLibCall()
10248 MVT VT = Op.getSimpleValueType(); in LowerMLOAD() local
10258 SDValue ZeroVec = DAG.getNode(ARMISD::VMOVIMM, dl, VT, in LowerMLOAD()
10261 VT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask, ZeroVec, in LowerMLOAD()
10269 Combo = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru); in LowerMLOAD()
10293 EVT VT = Op0.getValueType(); in LowerVecReduce() local
10294 EVT EltVT = VT.getVectorElementType(); in LowerVecReduce()
10295 unsigned NumElts = VT.getVectorNumElements(); in LowerVecReduce()
10306 SDValue Rev = DAG.getNode(RevOpcode, dl, VT, Op0); in LowerVecReduce()
10307 Op0 = DAG.getNode(BaseOpcode, dl, VT, Op0, Rev); in LowerVecReduce()
10353 EVT VT = Op0.getValueType(); in LowerVecReduceMinMax() local
10354 EVT EltVT = VT.getVectorElementType(); in LowerVecReduceMinMax()
10375 unsigned NumElts = VT.getVectorNumElements(); in LowerVecReduceMinMax()
10383 if (VT.is128BitVector()) { in LowerVecReduceMinMax()
10386 VT = Lo.getValueType(); in LowerVecReduceMinMax()
10387 Op0 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, {PairwiseOp, Lo, Hi}); in LowerVecReduceMinMax()
10393 Op0 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, {PairwiseOp, Op0, Op0}); in LowerVecReduceMinMax()
10498 EVT VT = Op.getValueType(); in LowerFSETCC() local
10514 SDValue Result = DAG.getNode(ISD::SETCC, dl, VT, LHS, RHS, in LowerFSETCC()
10527 SDValue True = DAG.getConstant(1, dl, VT); in LowerFSETCC()
10528 SDValue False = DAG.getConstant(0, dl, VT); in LowerFSETCC()
10532 SDValue Result = getCMOV(dl, VT, False, True, ARMcc, CCR, Cmp, DAG); in LowerFSETCC()
10536 Result = getCMOV(dl, VT, Result, True, ARMcc, CCR, Cmp, DAG); in LowerFSETCC()
10544 EVT VT = getPointerTy(DAG.getDataLayout()); in LowerSPONENTRY() local
10547 return DAG.getFrameIndex(FI, VT); in LowerSPONENTRY()
12530 EVT VT = N->getValueType(0); in isConditionalZeroOrAllOnes() local
12538 OtherOp = DAG.getConstant(0, dl, VT); in isConditionalZeroOrAllOnes()
12541 OtherOp = DAG.getConstant(1, dl, VT); in isConditionalZeroOrAllOnes()
12543 OtherOp = DAG.getAllOnesConstant(dl, VT); in isConditionalZeroOrAllOnes()
12578 EVT VT = N->getValueType(0); in combineSelectAndUse() local
12588 SDValue FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT, in combineSelectAndUse()
12594 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, in combineSelectAndUse()
12642 EVT VT = N->getValueType(0); in AddCombineToVPADD() local
12650 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops); in AddCombineToVPADD()
12681 EVT VT = N->getValueType(0); in AddCombineVUZPToVPADDL() local
12693 unsigned NumElts = VT.getVectorNumElements(); in AddCombineVUZPToVPADDL()
12699 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops); in AddCombineVUZPToVPADDL()
12717 EVT VT = N->getValueType(0); in AddCombineBUILD_VECTORToVPADDL() local
12718 if (!VT.isInteger() || VT.getVectorElementType() == MVT::i64) in AddCombineBUILD_VECTORToVPADDL()
12768 Vec.getValueType().getVectorElementType() == VT.getVectorElementType()) in AddCombineBUILD_VECTORToVPADDL()
12787 unsigned numElem = VT.getVectorNumElements(); in AddCombineBUILD_VECTORToVPADDL()
12799 unsigned ExtOp = VT.bitsGT(tmp.getValueType()) ? ISD::ANY_EXTEND : ISD::TRUNCATE; in AddCombineBUILD_VECTORToVPADDL()
12800 return DAG.getNode(ExtOp, dl, VT, tmp); in AddCombineBUILD_VECTORToVPADDL()
13303 EVT VT = N->getValueType(0); in PerformVQDMULHCombine() local
13307 if (!VT.isVector() || VT.getScalarSizeInBits() > 64) in PerformVQDMULHCombine()
13368 VT.getScalarSizeInBits() < ScalarType.getScalarSizeInBits() * 2) in PerformVQDMULHCombine()
13389 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Trunc); in PerformVQDMULHCombine()
13406 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, in PerformVQDMULHCombine()
13456 EVT VT = N->getValueType(0); in PerformVSetCCToVCTPCombine() local
13459 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT)) in PerformVSetCCToVCTPCombine()
13467 if (CC != ISD::SETULT || VT.getScalarSizeInBits() != 1 || in PerformVSetCCToVCTPCombine()
13472 for (unsigned I = 0; I < VT.getVectorNumElements(); I++) { in PerformVSetCCToVCTPCombine()
13485 switch (VT.getVectorNumElements()) { in PerformVSetCCToVCTPCombine()
13503 return DCI.DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in PerformVSetCCToVCTPCombine()
13562 EVT VT = N->getValueType(0); in TryDistrubutionADDVecReduce() local
13583 if (VT == MVT::i32 && N1.getOpcode() == ISD::ADD && !IsVecReduce(N0) && in TryDistrubutionADDVecReduce()
13586 SDValue Add0 = DAG.getNode(ISD::ADD, dl, VT, N0, N1.getOperand(0)); in TryDistrubutionADDVecReduce()
13587 return DAG.getNode(ISD::ADD, dl, VT, Add0, N1.getOperand(1)); in TryDistrubutionADDVecReduce()
13591 if (VT == MVT::i32 && N0.getOpcode() == ISD::ADD && in TryDistrubutionADDVecReduce()
13606 SDValue Add0 = DAG.getNode(ISD::ADD, dl, VT, N0.getOperand(1 - N0RedOp), in TryDistrubutionADDVecReduce()
13609 DAG.getNode(ISD::ADD, dl, VT, Add0, N0.getOperand(N0RedOp)); in TryDistrubutionADDVecReduce()
13610 return DAG.getNode(ISD::ADD, dl, VT, Add1, N1.getOperand(N1RedOp)); in TryDistrubutionADDVecReduce()
13685 return DAG.getNode(ISD::ADD, dl, VT, N1, N0); in TryDistrubutionADDVecReduce()
13696 SDValue Add0 = DAG.getNode(ISD::ADD, dl, VT, X, N1); in TryDistrubutionADDVecReduce()
13697 return DAG.getNode(ISD::ADD, dl, VT, Add0, N0); in TryDistrubutionADDVecReduce()
13714 EVT VT = N->getValueType(0); in PerformADDVecReduce() local
13719 if (VT != MVT::i64) in PerformADDVecReduce()
13876 EVT VT) const { in shouldFoldSelectWithIdentityConstant()
13877 return Subtarget->hasMVEIntegerOps() && isTypeLegal(VT); in shouldFoldSelectWithIdentityConstant()
13880 bool ARMTargetLowering::preferIncOfAddToSubOfNot(EVT VT) const { in preferIncOfAddToSubOfNot()
13883 return VT.getScalarSizeInBits() <= 32; in preferIncOfAddToSubOfNot()
13886 return VT.isScalarInteger(); in preferIncOfAddToSubOfNot()
13890 EVT VT) const { in shouldConvertFpToSat()
13891 if (!isOperationLegalOrCustom(Op, VT) || !FPVT.isSimple()) in shouldConvertFpToSat()
14135 EVT VT = N->getValueType(0); in PerformVMULCombine() local
14139 return DAG.getNode(Opcode, DL, VT, in PerformVMULCombine()
14140 DAG.getNode(ISD::MUL, DL, VT, N00, N1), in PerformVMULCombine()
14141 DAG.getNode(ISD::MUL, DL, VT, N01, N1)); in PerformVMULCombine()
14146 EVT VT = N->getValueType(0); in PerformMVEVMULLCombine() local
14147 if (VT != MVT::v2i64) in PerformMVEVMULLCombine()
14156 EVT VT = cast<VTSDNode>(Op->getOperand(1))->getVT(); in PerformMVEVMULLCombine() local
14157 if (VT.getScalarSizeInBits() == 32) in PerformMVEVMULLCombine()
14195 return DAG.getNode(ARMISD::VMULLs, dl, VT, New0a, New1a); in PerformMVEVMULLCombine()
14202 return DAG.getNode(ARMISD::VMULLu, dl, VT, New0a, New1a); in PerformMVEVMULLCombine()
14214 EVT VT = N->getValueType(0); in PerformMULCombine() local
14215 if (Subtarget->hasMVEIntegerOps() && VT == MVT::v2i64) in PerformMULCombine()
14224 if (VT.is64BitVector() || VT.is128BitVector()) in PerformMULCombine()
14226 if (VT != MVT::i32) in PerformMULCombine()
14246 Res = DAG.getNode(ISD::ADD, DL, VT, in PerformMULCombine()
14248 DAG.getNode(ISD::SHL, DL, VT, in PerformMULCombine()
14254 Res = DAG.getNode(ISD::SUB, DL, VT, in PerformMULCombine()
14255 DAG.getNode(ISD::SHL, DL, VT, in PerformMULCombine()
14266 Res = DAG.getNode(ISD::SUB, DL, VT, in PerformMULCombine()
14268 DAG.getNode(ISD::SHL, DL, VT, in PerformMULCombine()
14274 Res = DAG.getNode(ISD::ADD, DL, VT, in PerformMULCombine()
14276 DAG.getNode(ISD::SHL, DL, VT, in PerformMULCombine()
14280 Res = DAG.getNode(ISD::SUB, DL, VT, in PerformMULCombine()
14287 Res = DAG.getNode(ISD::SHL, DL, VT, in PerformMULCombine()
14405 EVT VT = N->getValueType(0); in PerformANDCombine() local
14408 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) || VT == MVT::v2i1 || in PerformANDCombine()
14409 VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1) in PerformANDCombine()
14422 DAG, dl, VbicVT, VT, OtherModImm); in PerformANDCombine()
14427 return DAG.getNode(ISD::BITCAST, dl, VT, Vbic); in PerformANDCombine()
14516 EVT VT = N->getValueType(0); in PerformORCombineToBFI() local
14531 if (VT != MVT::i32) in PerformORCombineToBFI()
14557 Res = DAG.getNode(ARMISD::BFI, DL, VT, N00, in PerformORCombineToBFI()
14584 Res = DAG.getNode(ISD::SRL, DL, VT, N1.getOperand(0), in PerformORCombineToBFI()
14586 Res = DAG.getNode(ARMISD::BFI, DL, VT, N00, Res, in PerformORCombineToBFI()
14601 Res = DAG.getNode(ISD::SRL, DL, VT, N00, in PerformORCombineToBFI()
14603 Res = DAG.getNode(ARMISD::BFI, DL, VT, N1.getOperand(0), Res, in PerformORCombineToBFI()
14623 Res = DAG.getNode(ARMISD::BFI, DL, VT, N1, N00.getOperand(0), in PerformORCombineToBFI()
14670 EVT VT = N->getValueType(0); in PerformORCombine_i1() local
14685 SDValue NewN0 = DAG.getLogicalNOT(DL, N0, VT); in PerformORCombine_i1()
14686 SDValue NewN1 = DAG.getLogicalNOT(DL, N1, VT); in PerformORCombine_i1()
14687 SDValue And = DAG.getNode(ISD::AND, DL, VT, NewN0, NewN1); in PerformORCombine_i1()
14688 return DAG.getLogicalNOT(DL, And, VT); in PerformORCombine_i1()
14698 EVT VT = N->getValueType(0); in PerformORCombine() local
14701 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in PerformORCombine()
14704 if (Subtarget->hasMVEIntegerOps() && (VT == MVT::v2i1 || VT == MVT::v4i1 || in PerformORCombine()
14705 VT == MVT::v8i1 || VT == MVT::v16i1)) in PerformORCombine()
14718 SplatBitSize, DAG, dl, VorrVT, VT, OtherModImm); in PerformORCombine()
14723 return DAG.getNode(ISD::BITCAST, dl, VT, Vorr); in PerformORCombine()
14740 if (Subtarget->hasNEON() && N1.getOpcode() == ISD::AND && VT.isVector() && in PerformORCombine()
14741 DAG.getTargetLoweringInfo().isTypeLegal(VT)) { in PerformORCombine()
14768 EVT CanonicalVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32; in PerformORCombine()
14773 return DAG.getNode(ISD::BITCAST, dl, VT, Result); in PerformORCombine()
14795 EVT VT = N->getValueType(0); in PerformXORCombine() local
14798 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in PerformXORCombine()
14936 EVT VT = N->getValueType(0); in PerformBFICombine() local
14940 From1 = DAG.getNode(ISD::SRL, dl, VT, From1, in PerformBFICombine()
14941 DAG.getConstant(NewFromMask.countr_zero(), dl, VT)); in PerformBFICombine()
14942 return DAG.getNode(ARMISD::BFI, dl, VT, CombineBFI.getOperand(0), From1, in PerformBFICombine()
14943 DAG.getConstant(~NewToMask, dl, VT)); in PerformBFICombine()
14958 EVT VT = N->getValueType(0); in PerformBFICombine() local
14960 SDValue BFI1 = DAG.getNode(ARMISD::BFI, dl, VT, N0.getOperand(0), in PerformBFICombine()
14962 return DAG.getNode(ARMISD::BFI, dl, VT, BFI1, N0.getOperand(1), in PerformBFICombine()
15212 EVT VT = N->getValueType(0); in PerformVMOVrhCombine() local
15217 return DAG.getConstant(V.bitcastToAPInt().getZExtValue(), SDLoc(N), VT); in PerformVMOVrhCombine()
15225 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N), VT, LN0->getChain(), in PerformVMOVrhCombine()
15235 return DAG.getNode(ARMISD::VGETLANEu, SDLoc(N), VT, N0->getOperand(0), in PerformVMOVrhCombine()
15271 EVT VT = N->getValueType(0); in PerformBUILD_VECTORCombine() local
15272 if (VT.getVectorElementType() != MVT::i64 || !hasNormalLoadOperand(N)) in PerformBUILD_VECTORCombine()
15276 unsigned NumElts = VT.getVectorNumElements(); in PerformBUILD_VECTORCombine()
15285 return DAG.getNode(ISD::BITCAST, dl, VT, BV); in PerformBUILD_VECTORCombine()
15303 EVT VT = N->getValueType(0); in PerformARMBUILD_VECTORCombine() local
15304 EVT EltVT = VT.getVectorElementType(); in PerformARMBUILD_VECTORCombine()
15325 unsigned NumElts = VT.getVectorNumElements(); in PerformARMBUILD_VECTORCombine()
15374 Vec = DAG.getNode(ISD::BITCAST, dl, VT, Vec); in PerformARMBUILD_VECTORCombine()
15382 EVT VT = N->getValueType(0); in PerformPREDICATE_CASTCombine() local
15389 if (Op->getOperand(0).getValueType() == VT) in PerformPREDICATE_CASTCombine()
15391 return DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, Op->getOperand(0)); in PerformPREDICATE_CASTCombine()
15398 DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, Op->getOperand(0)); in PerformPREDICATE_CASTCombine()
15399 SDValue C = DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, in PerformPREDICATE_CASTCombine()
15401 return DCI.DAG.getNode(ISD::XOR, dl, VT, X, C); in PerformPREDICATE_CASTCombine()
15416 EVT VT = N->getValueType(0); in PerformVECTOR_REG_CASTCombine() local
15422 return DAG.getNode(ISD::BITCAST, dl, VT, Op); in PerformVECTOR_REG_CASTCombine()
15426 return DAG.getUNDEF(VT); in PerformVECTOR_REG_CASTCombine()
15431 if (Op->getOperand(0).getValueType() == VT) in PerformVECTOR_REG_CASTCombine()
15433 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Op->getOperand(0)); in PerformVECTOR_REG_CASTCombine()
15444 EVT VT = N->getValueType(0); in PerformVCMPCombine() local
15452 return DAG.getNode(ARMISD::VCMPZ, dl, VT, Op0, N->getOperand(2)); in PerformVCMPCombine()
15455 if (isValidMVECond(SwappedCond, VT.isFloatingPoint())) { in PerformVCMPCombine()
15458 return DAG.getNode(ARMISD::VCMPZ, dl, VT, Op1, in PerformVCMPCombine()
15462 return DAG.getNode(ARMISD::VCMP, dl, VT, Op1, Op0, in PerformVCMPCombine()
15475 EVT VT = N->getValueType(0); in PerformInsertEltCombine() local
15477 if (VT.getVectorElementType() != MVT::i64 || in PerformInsertEltCombine()
15484 VT.getVectorNumElements()); in PerformInsertEltCombine()
15492 return DAG.getNode(ISD::BITCAST, dl, VT, InsElt); in PerformInsertEltCombine()
15501 EVT VT = N->getValueType(0); in PerformExtractEltToVMOVRRD() local
15504 if (!DCI.isAfterLegalizeDAG() || VT != MVT::i32 || in PerformExtractEltToVMOVRRD()
15565 EVT VT = N->getValueType(0); in PerformExtractEltCombine() local
15571 if (VT == MVT::f16 && X.getValueType() == MVT::i32) in PerformExtractEltCombine()
15572 return DCI.DAG.getNode(ARMISD::VMOVhr, dl, VT, X); in PerformExtractEltCombine()
15573 if (VT == MVT::i32 && X.getValueType() == MVT::f16) in PerformExtractEltCombine()
15574 return DCI.DAG.getNode(ARMISD::VMOVrh, dl, VT, X); in PerformExtractEltCombine()
15575 if (VT == MVT::f32 && X.getValueType() == MVT::i32) in PerformExtractEltCombine()
15576 return DCI.DAG.getNode(ISD::BITCAST, dl, VT, X); in PerformExtractEltCombine()
15578 while (X.getValueType() != VT && X->getOpcode() == ISD::BITCAST) in PerformExtractEltCombine()
15580 if (X.getValueType() == VT) in PerformExtractEltCombine()
15615 return DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Op0.getOperand(Vec), in PerformExtractEltCombine()
15624 EVT VT = N->getValueType(0); in PerformSignExtendInregCombine() local
15630 return DAG.getNode(ARMISD::VGETLANEs, SDLoc(N), VT, Op.getOperand(0), in PerformSignExtendInregCombine()
15681 EVT VT = Trunc.getValueType(); in PerformShuffleVMOVNCombine() local
15686 if (isVMOVNTruncMask(N->getMask(), VT, false)) in PerformShuffleVMOVNCombine()
15688 ARMISD::VMOVN, DL, VT, in PerformShuffleVMOVNCombine()
15689 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(0)), in PerformShuffleVMOVNCombine()
15690 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(1)), in PerformShuffleVMOVNCombine()
15692 else if (isVMOVNTruncMask(N->getMask(), VT, true)) in PerformShuffleVMOVNCombine()
15694 ARMISD::VMOVN, DL, VT, in PerformShuffleVMOVNCombine()
15695 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(1)), in PerformShuffleVMOVNCombine()
15696 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, Trunc.getOperand(0)), in PerformShuffleVMOVNCombine()
15730 EVT VT = N->getValueType(0); in PerformVECTOR_SHUFFLECombine() local
15731 if (!TLI.isTypeLegal(VT) || in PerformVECTOR_SHUFFLECombine()
15736 SDValue NewConcat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, in PerformVECTOR_SHUFFLECombine()
15740 unsigned NumElts = VT.getVectorNumElements(); in PerformVECTOR_SHUFFLECombine()
15752 return DAG.getVectorShuffle(VT, SDLoc(N), NewConcat, in PerformVECTOR_SHUFFLECombine()
15753 DAG.getUNDEF(VT), NewMask); in PerformVECTOR_SHUFFLECombine()
16366 EVT VT = N->getValueType(0); in CombineVLDDUP() local
16368 if (!VT.is64BitVector()) in CombineVLDDUP()
16409 Tys[n] = VT; in CombineVLDDUP()
16446 EVT VT = N->getValueType(0); in PerformVDUPLANECombine() local
16450 EVT ExtractVT = VT.getVectorElementType(); in PerformVDUPLANECombine()
16456 return DCI.DAG.getNode(ARMISD::VDUP, SDLoc(N), VT, Extract); in PerformVDUPLANECombine()
16478 if (EltSize > VT.getScalarSizeInBits()) in PerformVDUPLANECombine()
16481 return DCI.DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op); in PerformVDUPLANECombine()
16526 EVT VT = N->getValueType(0); in PerformLOADCombine() local
16529 if (Subtarget->hasNEON() && ISD::isNormalLoad(N) && VT.isVector() && in PerformLOADCombine()
16530 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT)) in PerformLOADCombine()
16542 EVT VT = StVal.getValueType(); in PerformTruncatingStoreCombine() local
16543 if (!St->isTruncatingStore() || !VT.isVector()) in PerformTruncatingStoreCombine()
16547 unsigned NumElems = VT.getVectorNumElements(); in PerformTruncatingStoreCombine()
16548 assert(StVT != VT && "Cannot truncate to the same type"); in PerformTruncatingStoreCombine()
16549 unsigned FromEltSz = VT.getScalarSizeInBits(); in PerformTruncatingStoreCombine()
16562 assert(SizeRatio * NumElems * ToEltSz == VT.getSizeInBits()); in PerformTruncatingStoreCombine()
16567 assert(WideVecVT.getSizeInBits() == VT.getSizeInBits()); in PerformTruncatingStoreCombine()
16598 VT.getSizeInBits() / EVT(StoreType).getSizeInBits()); in PerformTruncatingStoreCombine()
16599 assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits()); in PerformTruncatingStoreCombine()
16763 EVT VT = Extract.getValueType(); in PerformExtractFpToIntStores() local
16766 if (VT != MVT::f16 || Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT) in PerformExtractFpToIntStores()
16783 EVT NewToVT = EVT::getIntegerVT(C, VT.getSizeInBits()); in PerformExtractFpToIntStores()
16800 EVT VT = StVal.getValueType(); in PerformSTORECombine() local
16866 if (Subtarget->hasNEON() && ISD::isNormalStore(N) && VT.isVector() && in PerformSTORECombine()
16867 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT)) in PerformSTORECombine()
16940 EVT VT = N->getValueType(0); in PerformFAddVSelectCombine() local
16950 if (VT == MVT::v4f32 && (ImmVal == 1664 || (ImmVal == 0 && NSZ))) in PerformFAddVSelectCombine()
16952 if (VT == MVT::v8f16 && (ImmVal == 2688 || (ImmVal == 0 && NSZ))) in PerformFAddVSelectCombine()
16969 DAG.getNode(ISD::FADD, DL, VT, Op0, Op1.getOperand(1), FaddFlags); in PerformFAddVSelectCombine()
16970 return DAG.getNode(ISD::VSELECT, DL, VT, Op1.getOperand(0), FAdd, Op0, FaddFlags); in PerformFAddVSelectCombine()
16976 EVT VT = N->getValueType(0); in PerformFADDVCMLACombine() local
16990 ISD::INTRINSIC_WO_CHAIN, DL, VT, A.getOperand(0), A.getOperand(1), in PerformFADDVCMLACombine()
16991 DAG.getNode(ISD::FADD, DL, VT, A.getOperand(2), B, N->getFlags()), in PerformFADDVCMLACombine()
17213 EVT VT = Ops[0].getValueType(); in PerformVECREDUCE_ADDCombine() local
17214 if (VT == MVT::v16i8) { in PerformVECREDUCE_ADDCombine()
17423 EVT VT = N->getValueType(0); in PerformVQDMULHCombine() local
17434 SDValue NewBinOp = DCI.DAG.getNode(N->getOpcode(), DL, VT, in PerformVQDMULHCombine()
17437 return DCI.DAG.getVectorShuffle(VT, DL, NewBinOp, UndefV, Shuf0->getMask()); in PerformVQDMULHCombine()
17499 EVT VT = N->getOperand(1).getValueType(); in PerformIntrinsicCombine() local
17506 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) { in PerformIntrinsicCombine()
17510 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt)) { in PerformIntrinsicCombine()
17519 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt)) in PerformIntrinsicCombine()
17525 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) in PerformIntrinsicCombine()
17530 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) in PerformIntrinsicCombine()
17542 if (isVShiftRImm(N->getOperand(2), VT, true, true, Cnt)) in PerformIntrinsicCombine()
17600 EVT VT = N->getOperand(1).getValueType(); in PerformIntrinsicCombine() local
17604 if (isVShiftLImm(N->getOperand(3), VT, false, Cnt)) in PerformIntrinsicCombine()
17606 else if (isVShiftRImm(N->getOperand(3), VT, false, true, Cnt)) in PerformIntrinsicCombine()
17702 EVT VT = N->getValueType(0); in PerformShiftCombine() local
17704 if (ST->isThumb1Only() && N->getOpcode() == ISD::SHL && VT == MVT::i32 && in PerformShiftCombine()
17740 if (!VT.isVector() || !TLI.isTypeLegal(VT)) in PerformShiftCombine()
17751 if (isVShiftLImm(N->getOperand(1), VT, false, Cnt)) { in PerformShiftCombine()
17753 return DAG.getNode(ARMISD::VSHLIMM, dl, VT, N->getOperand(0), in PerformShiftCombine()
17760 if (isVShiftRImm(N->getOperand(1), VT, false, false, Cnt)) { in PerformShiftCombine()
17764 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0), in PerformShiftCombine()
17868 EVT VT = N->getValueType(0); in PerformExtendCombine() local
17872 if (VT == MVT::i32 && in PerformExtendCombine()
17888 return DAG.getNode(Opc, SDLoc(N), VT, Vec, Lane); in PerformExtendCombine()
17916 EVT VT = Op.getValueType(); in PerformMinMaxToSatCombine() local
17919 if (VT != MVT::i32 || in PerformMinMaxToSatCombine()
17940 return DAG.getNode(ARMISD::SSAT, DL, VT, Input, in PerformMinMaxToSatCombine()
17941 DAG.getConstant(MinC.countr_one(), DL, VT)); in PerformMinMaxToSatCombine()
17943 return DAG.getNode(ARMISD::USAT, DL, VT, Input, in PerformMinMaxToSatCombine()
17944 DAG.getConstant(MinC.countr_one(), DL, VT)); in PerformMinMaxToSatCombine()
17953 EVT VT = N->getValueType(0); in PerformMinMaxCombine() local
17956 if (VT == MVT::i32) in PerformMinMaxCombine()
17965 if (VT != MVT::v4i32 && VT != MVT::v8i16) in PerformMinMaxCombine()
17976 if (VT == MVT::v4i32) in PerformMinMaxCombine()
17994 if (VT == MVT::v4i32) { in PerformMinMaxCombine()
18008 SDValue Bitcast = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, VQMOVN); in PerformMinMaxCombine()
18009 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Bitcast, in PerformMinMaxCombine()
18019 if (VT == MVT::v4i32) in PerformMinMaxCombine()
18035 if (VT == MVT::v4i32) { in PerformMinMaxCombine()
18049 SDValue Bitcast = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, VQMOVN); in PerformMinMaxCombine()
18050 return DAG.getNode(ISD::AND, DL, VT, Bitcast, in PerformMinMaxCombine()
18051 DAG.getConstant(ExtConst, DL, VT)); in PerformMinMaxCombine()
18131 EVT VT = X.getValueType(); in PerformCMOVToBFICombine() local
18136 X = DAG.getNode(ISD::SRL, dl, VT, X, in PerformCMOVToBFICombine()
18137 DAG.getConstant(BitInX, dl, VT)); in PerformCMOVToBFICombine()
18144 APInt Mask(VT.getSizeInBits(), 0); in PerformCMOVToBFICombine()
18146 V = DAG.getNode(ARMISD::BFI, dl, VT, V, X, in PerformCMOVToBFICombine()
18148 DAG.getConstant(~Mask, dl, VT)); in PerformCMOVToBFICombine()
18331 EVT VT = N->getValueType(0); in PerformBRCONDCombine() local
18349 ARMISD::BRCOND, dl, VT, Chain, BB, LHS->getOperand(0)->getOperand(2), in PerformBRCONDCombine()
18364 EVT VT = N->getValueType(0); in PerformCMOVCombine() local
18399 Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, TrueVal, ARMcc, in PerformCMOVCombine()
18404 Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, FalseVal, ARMcc, in PerformCMOVCombine()
18413 return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, in PerformCMOVCombine()
18418 if (!VT.isInteger()) in PerformCMOVCombine()
18445 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS); in PerformCMOVCombine()
18446 Res = DAG.getNode(ISD::SRL, dl, VT, DAG.getNode(ISD::CTLZ, dl, VT, Sub), in PerformCMOVCombine()
18458 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS); in PerformCMOVCombine()
18459 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in PerformCMOVCombine()
18473 DAG.getNode(ARMISD::SUBS, dl, DAG.getVTList(VT, MVT::i32), LHS, RHS); in PerformCMOVCombine()
18476 Res = DAG.getNode(ARMISD::CMOV, dl, VT, Sub, TrueVal, ARMcc, in PerformCMOVCombine()
18487 DAG.getNode(ARMISD::SUBS, dl, DAG.getVTList(VT, MVT::i32), LHS, RHS); in PerformCMOVCombine()
18490 Res = DAG.getNode(ARMISD::CMOV, dl, VT, Sub, FalseVal, in PerformCMOVCombine()
18516 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in PerformCMOVCombine()
18519 TrueVal = DAG.getConstant(1, dl, VT); in PerformCMOVCombine()
18525 Res = DAG.getNode(ISD::SHL, dl, VT, Res, in PerformCMOVCombine()
18587 EVT VT = N->getValueType(0); in PerformMVETruncCombine() local
18592 return DAG.getUNDEF(VT); in PerformMVETruncCombine()
18598 return DAG.getNode(ARMISD::MVETRUNC, DL, VT, N->getOperand(0).getOperand(0), in PerformMVETruncCombine()
18616 if (isVMOVNTruncMask(Mask, VT, false)) in PerformMVETruncCombine()
18618 ARMISD::VMOVN, DL, VT, in PerformMVETruncCombine()
18619 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(0)), in PerformMVETruncCombine()
18620 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(1)), in PerformMVETruncCombine()
18622 if (isVMOVNTruncMask(Mask, VT, true)) in PerformMVETruncCombine()
18624 ARMISD::VMOVN, DL, VT, in PerformMVETruncCombine()
18625 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(1)), in PerformMVETruncCombine()
18626 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, S0->getOperand(0)), in PerformMVETruncCombine()
18648 return DAG.getBuildVector(VT, DL, Extracts); in PerformMVETruncCombine()
18663 EVT StoreVT = VT.getHalfNumVectorElementsVT(*DAG.getContext()); in PerformMVETruncCombine()
18682 return DAG.getLoad(VT, DL, Chain, StackPtr, MPI, Align(4)); in PerformMVETruncCombine()
18756 EVT VT = N->getValueType(0); in PerformMVEExtCombine() local
18759 assert((VT == MVT::v4i32 || VT == MVT::v8i16) && "Unexpected MVEEXT type"); in PerformMVEExtCombine()
18764 SDValue VVT = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, V); in PerformMVEExtCombine()
18766 ? DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, VVT, in PerformMVEExtCombine()
18780 assert(Mask.size() == 2 * VT.getVectorNumElements()); in PerformMVEExtCombine()
18782 unsigned Rev = VT == MVT::v4i32 ? ARMISD::VREV32 : ARMISD::VREV16; in PerformMVEExtCombine()
18787 for (int Idx = 0, E = VT.getVectorNumElements(); Idx < E; ++Idx) in PerformMVEExtCombine()
18803 if (CheckInregMask(VT.getVectorNumElements(), Mask.size())) in PerformMVEExtCombine()
18805 else if (CheckInregMask(VT.getVectorNumElements(), Mask.size() + 1)) in PerformMVEExtCombine()
18807 else if (CheckInregMask(VT.getVectorNumElements(), 0)) in PerformMVEExtCombine()
18809 else if (CheckInregMask(VT.getVectorNumElements(), 1)) in PerformMVEExtCombine()
18850 VT, Chain, Ptr, MPI, LoadVT, Align(4)); in PerformMVEExtCombine()
19081 EVT VT) const { in isDesirableToTransformToIntegerOp()
19082 return (VT == MVT::f32) && (Opc == ISD::LOAD || Opc == ISD::STORE); in isDesirableToTransformToIntegerOp()
19085 bool ARMTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned, in allowsMisalignedMemoryAccesses() argument
19090 if (!VT.isSimple()) in allowsMisalignedMemoryAccesses()
19095 auto Ty = VT.getSimpleVT().SimpleTy; in allowsMisalignedMemoryAccesses()
19131 Alignment >= VT.getScalarSizeInBits() / 8) { in allowsMisalignedMemoryAccesses()
19224 bool ARMTargetLowering::isFNegFree(EVT VT) const { in isFNegFree()
19225 if (!VT.isSimple()) in isFNegFree()
19233 switch (VT.getSimpleVT().SimpleTy) { in isFNegFree()
19399 EVT VT = ExtVal.getValueType(); in isVectorLoadExtDesirable() local
19401 if (!isTypeLegal(VT)) in isVectorLoadExtDesirable()
19455 EVT VT) const { in isFMAFasterThanFMulAndFAdd()
19456 if (!VT.isSimple()) in isFMAFasterThanFMulAndFAdd()
19459 switch (VT.getSimpleVT().SimpleTy) { in isFMAFasterThanFMulAndFAdd()
19476 static bool isLegalT1AddressImmediate(int64_t V, EVT VT) { in isLegalT1AddressImmediate() argument
19481 switch (VT.getSimpleVT().SimpleTy) { in isLegalT1AddressImmediate()
19502 static bool isLegalT2AddressImmediate(int64_t V, EVT VT, in isLegalT2AddressImmediate() argument
19504 if (!VT.isInteger() && !VT.isFloatingPoint()) in isLegalT2AddressImmediate()
19506 if (VT.isVector() && Subtarget->hasNEON()) in isLegalT2AddressImmediate()
19508 if (VT.isVector() && VT.isFloatingPoint() && Subtarget->hasMVEIntegerOps() && in isLegalT2AddressImmediate()
19518 unsigned NumBytes = std::max((unsigned)VT.getSizeInBits() / 8, 1U); in isLegalT2AddressImmediate()
19521 if (VT.isVector() && Subtarget->hasMVEIntegerOps()) { in isLegalT2AddressImmediate()
19522 switch (VT.getSimpleVT().getVectorElementType().SimpleTy) { in isLegalT2AddressImmediate()
19537 if (VT.isFloatingPoint() && NumBytes == 2 && Subtarget->hasFPRegs16()) in isLegalT2AddressImmediate()
19540 if ((VT.isFloatingPoint() && Subtarget->hasVFP2Base()) || NumBytes == 8) in isLegalT2AddressImmediate()
19556 static bool isLegalAddressImmediate(int64_t V, EVT VT, in isLegalAddressImmediate() argument
19561 if (!VT.isSimple()) in isLegalAddressImmediate()
19565 return isLegalT1AddressImmediate(V, VT); in isLegalAddressImmediate()
19567 return isLegalT2AddressImmediate(V, VT, Subtarget); in isLegalAddressImmediate()
19572 switch (VT.getSimpleVT().SimpleTy) { in isLegalAddressImmediate()
19591 EVT VT) const { in isLegalT2ScaledAddressingMode()
19596 switch (VT.getSimpleVT().SimpleTy) { in isLegalT2ScaledAddressingMode()
19629 EVT VT) const { in isLegalT1ScaledAddressingMode()
19648 EVT VT = getValueType(DL, Ty, true); in isLegalAddressingMode() local
19649 if (!isLegalAddressImmediate(AM.BaseOffs, VT, Subtarget)) in isLegalAddressingMode()
19664 if (!VT.isSimple()) in isLegalAddressingMode()
19668 return isLegalT1ScaledAddressingMode(AM, VT); in isLegalAddressingMode()
19671 return isLegalT2ScaledAddressingMode(AM, VT); in isLegalAddressingMode()
19674 switch (VT.getSimpleVT().SimpleTy) { in isLegalAddressingMode()
19744 const EVT VT = AddNode.getValueType(); in isMulAddWithConstProfitable() local
19745 if (VT.isVector() || VT.getScalarSizeInBits() > 32) in isMulAddWithConstProfitable()
19763 static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT, in getARMIndexedAddressParts() argument
19770 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) { in getARMIndexedAddressParts()
19785 } else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) { in getARMIndexedAddressParts()
19822 static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT, in getT2IndexedAddressParts() argument
19847 static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, Align Alignment, in getMVEIndexedAddressParts() argument
19881 if (VT == MVT::v4i16) { in getMVEIndexedAddressParts()
19884 } else if (VT == MVT::v4i8 || VT == MVT::v8i8) { in getMVEIndexedAddressParts()
19888 (CanChangeType || VT == MVT::v4i32 || VT == MVT::v4f32) && in getMVEIndexedAddressParts()
19892 (CanChangeType || VT == MVT::v8i16 || VT == MVT::v8f16) && in getMVEIndexedAddressParts()
19895 else if ((CanChangeType || VT == MVT::v16i8) && IsInRange(RHSC, 0x80, 1)) in getMVEIndexedAddressParts()
19911 EVT VT; in getPreIndexedAddressParts() local
19918 VT = LD->getMemoryVT(); in getPreIndexedAddressParts()
19923 VT = ST->getMemoryVT(); in getPreIndexedAddressParts()
19927 VT = LD->getMemoryVT(); in getPreIndexedAddressParts()
19933 VT = ST->getMemoryVT(); in getPreIndexedAddressParts()
19941 if (VT.isVector()) in getPreIndexedAddressParts()
19944 Ptr.getNode(), VT, Alignment, isSEXTLoad, IsMasked, in getPreIndexedAddressParts()
19948 isLegal = getT2IndexedAddressParts(Ptr.getNode(), VT, isSEXTLoad, Base, in getPreIndexedAddressParts()
19951 isLegal = getARMIndexedAddressParts(Ptr.getNode(), VT, isSEXTLoad, Base, in getPreIndexedAddressParts()
19969 EVT VT; in getPostIndexedAddressParts() local
19975 VT = LD->getMemoryVT(); in getPostIndexedAddressParts()
19981 VT = ST->getMemoryVT(); in getPostIndexedAddressParts()
19986 VT = LD->getMemoryVT(); in getPostIndexedAddressParts()
19993 VT = ST->getMemoryVT(); in getPostIndexedAddressParts()
20021 if (VT.isVector()) in getPostIndexedAddressParts()
20023 getMVEIndexedAddressParts(Op, VT, Alignment, isSEXTLoad, IsMasked, in getPostIndexedAddressParts()
20028 isLegal = getT2IndexedAddressParts(Op, VT, isSEXTLoad, Base, Offset, in getPostIndexedAddressParts()
20031 isLegal = getARMIndexedAddressParts(Op, VT, isSEXTLoad, Base, Offset, in getPostIndexedAddressParts()
20095 EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT(); in computeKnownBitsForTargetNode() local
20096 unsigned MemBits = VT.getScalarSizeInBits(); in computeKnownBitsForTargetNode()
20127 EVT VT = Op.getValueType(); in computeKnownBitsForTargetNode() local
20128 const unsigned DstSz = VT.getScalarSizeInBits(); in computeKnownBitsForTargetNode()
20184 EVT VT = Op.getValueType(); in targetShrinkDemandedConstant() local
20187 if (VT.isVector()) in targetShrinkDemandedConstant()
20190 assert(VT == MVT::i32 && "Unexpected integer type"); in targetShrinkDemandedConstant()
20217 auto UseMask = [Mask, Op, VT, &TLO](unsigned NewMask) -> bool { in targetShrinkDemandedConstant()
20221 SDValue NewC = TLO.DAG.getConstant(NewMask, DL, VT); in targetShrinkDemandedConstant()
20222 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC); in targetShrinkDemandedConstant()
20413 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const { in getRegForInlineAsmConstraint()
20431 if (VT == MVT::Other) in getRegForInlineAsmConstraint()
20433 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16) in getRegForInlineAsmConstraint()
20435 if (VT.getSizeInBits() == 64) in getRegForInlineAsmConstraint()
20437 if (VT.getSizeInBits() == 128) in getRegForInlineAsmConstraint()
20441 if (VT == MVT::Other) in getRegForInlineAsmConstraint()
20443 if (VT == MVT::f32 || VT == MVT::f16 || VT == MVT::bf16) in getRegForInlineAsmConstraint()
20445 if (VT.getSizeInBits() == 64) in getRegForInlineAsmConstraint()
20447 if (VT.getSizeInBits() == 128) in getRegForInlineAsmConstraint()
20451 if (VT == MVT::Other) in getRegForInlineAsmConstraint()
20453 if (VT == MVT::f32 || VT == MVT::i32 || VT == MVT::f16 || VT == MVT::bf16) in getRegForInlineAsmConstraint()
20455 if (VT.getSizeInBits() == 64) in getRegForInlineAsmConstraint()
20457 if (VT.getSizeInBits() == 128) in getRegForInlineAsmConstraint()
20483 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); in getRegForInlineAsmConstraint()
20704 EVT VT = Op->getValueType(0); in LowerDivRem() local
20707 if (VT == MVT::i64 && isa<ConstantSDNode>(Op.getOperand(1))) { in LowerDivRem()
20711 DAG.getNode(ISD::BUILD_PAIR, dl, VT, Result[0], Result[1]); in LowerDivRem()
20713 DAG.getNode(ISD::BUILD_PAIR, dl, VT, Result[2], Result[3]); in LowerDivRem()
20719 Type *Ty = VT.getTypeForEVT(*DAG.getContext()); in LowerDivRem()
20733 SDValue Div = DAG.getNode(DivOpcode, dl, VT, Dividend, Divisor); in LowerDivRem()
20734 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Div, Divisor); in LowerDivRem()
20735 SDValue Rem = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul); in LowerDivRem()
20738 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(VT, VT), Values); in LowerDivRem()
20742 VT.getSimpleVT().SimpleTy); in LowerDivRem()
20769 EVT VT = N->getValueType(0); in LowerREM() local
20771 if (VT == MVT::i64 && isa<ConstantSDNode>(N->getOperand(1))) { in LowerREM()
20782 switch (VT.getSimpleVT().SimpleTy) { in LowerREM()
20972 bool ARMTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, in isFPImmLegal() argument
20976 if (VT == MVT::f16 && Subtarget->hasFullFP16()) in isFPImmLegal()
20978 if (VT == MVT::f32 && Subtarget->hasFullFP16() && in isFPImmLegal()
20981 if (VT == MVT::f32) in isFPImmLegal()
20983 if (VT == MVT::f64 && Subtarget->hasFP64()) in isFPImmLegal()
21964 } else if (auto *VT = dyn_cast<VectorType>(Ty)) { in isHomogeneousAggregate() local
21971 return VT->getPrimitiveSizeInBits().getFixedValue() == 64; in isHomogeneousAggregate()
21973 return VT->getPrimitiveSizeInBits().getFixedValue() == 128; in isHomogeneousAggregate()
21975 switch (VT->getPrimitiveSizeInBits().getFixedValue()) { in isHomogeneousAggregate()