Lines Matching refs:VT
149 void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT, in addTypeForNEON() argument
151 if (VT != PromotedLdStVT) { in addTypeForNEON()
152 setOperationAction(ISD::LOAD, VT, Promote); in addTypeForNEON()
153 AddPromotedToType (ISD::LOAD, VT, PromotedLdStVT); in addTypeForNEON()
155 setOperationAction(ISD::STORE, VT, Promote); in addTypeForNEON()
156 AddPromotedToType (ISD::STORE, VT, PromotedLdStVT); in addTypeForNEON()
159 MVT ElemTy = VT.getVectorElementType(); in addTypeForNEON()
161 setOperationAction(ISD::SETCC, VT, Custom); in addTypeForNEON()
162 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in addTypeForNEON()
163 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in addTypeForNEON()
165 setOperationAction(ISD::SINT_TO_FP, VT, Custom); in addTypeForNEON()
166 setOperationAction(ISD::UINT_TO_FP, VT, Custom); in addTypeForNEON()
167 setOperationAction(ISD::FP_TO_SINT, VT, Custom); in addTypeForNEON()
168 setOperationAction(ISD::FP_TO_UINT, VT, Custom); in addTypeForNEON()
170 setOperationAction(ISD::SINT_TO_FP, VT, Expand); in addTypeForNEON()
171 setOperationAction(ISD::UINT_TO_FP, VT, Expand); in addTypeForNEON()
172 setOperationAction(ISD::FP_TO_SINT, VT, Expand); in addTypeForNEON()
173 setOperationAction(ISD::FP_TO_UINT, VT, Expand); in addTypeForNEON()
175 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in addTypeForNEON()
176 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in addTypeForNEON()
177 setOperationAction(ISD::CONCAT_VECTORS, VT, Legal); in addTypeForNEON()
178 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal); in addTypeForNEON()
179 setOperationAction(ISD::SELECT, VT, Expand); in addTypeForNEON()
180 setOperationAction(ISD::SELECT_CC, VT, Expand); in addTypeForNEON()
181 setOperationAction(ISD::VSELECT, VT, Expand); in addTypeForNEON()
182 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); in addTypeForNEON()
183 if (VT.isInteger()) { in addTypeForNEON()
184 setOperationAction(ISD::SHL, VT, Custom); in addTypeForNEON()
185 setOperationAction(ISD::SRA, VT, Custom); in addTypeForNEON()
186 setOperationAction(ISD::SRL, VT, Custom); in addTypeForNEON()
190 if (VT.isInteger() && VT != PromotedBitwiseVT) { in addTypeForNEON()
191 setOperationAction(ISD::AND, VT, Promote); in addTypeForNEON()
192 AddPromotedToType (ISD::AND, VT, PromotedBitwiseVT); in addTypeForNEON()
193 setOperationAction(ISD::OR, VT, Promote); in addTypeForNEON()
194 AddPromotedToType (ISD::OR, VT, PromotedBitwiseVT); in addTypeForNEON()
195 setOperationAction(ISD::XOR, VT, Promote); in addTypeForNEON()
196 AddPromotedToType (ISD::XOR, VT, PromotedBitwiseVT); in addTypeForNEON()
200 setOperationAction(ISD::SDIV, VT, Expand); in addTypeForNEON()
201 setOperationAction(ISD::UDIV, VT, Expand); in addTypeForNEON()
202 setOperationAction(ISD::FDIV, VT, Expand); in addTypeForNEON()
203 setOperationAction(ISD::SREM, VT, Expand); in addTypeForNEON()
204 setOperationAction(ISD::UREM, VT, Expand); in addTypeForNEON()
205 setOperationAction(ISD::FREM, VT, Expand); in addTypeForNEON()
207 if (!VT.isFloatingPoint() && in addTypeForNEON()
208 VT != MVT::v2i64 && VT != MVT::v1i64) in addTypeForNEON()
210 setOperationAction(Opcode, VT, Legal); in addTypeForNEON()
213 void ARMTargetLowering::addDRTypeForNEON(MVT VT) { in addDRTypeForNEON() argument
214 addRegisterClass(VT, &ARM::DPRRegClass); in addDRTypeForNEON()
215 addTypeForNEON(VT, MVT::f64, MVT::v2i32); in addDRTypeForNEON()
218 void ARMTargetLowering::addQRTypeForNEON(MVT VT) { in addQRTypeForNEON() argument
219 addRegisterClass(VT, &ARM::DPairRegClass); in addQRTypeForNEON()
220 addTypeForNEON(VT, MVT::v2f64, MVT::v4i32); in addQRTypeForNEON()
528 for (MVT VT : MVT::vector_valuetypes()) { in ARMTargetLowering() local
530 setTruncStoreAction(VT, InnerVT, Expand); in ARMTargetLowering()
531 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); in ARMTargetLowering()
532 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); in ARMTargetLowering()
533 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); in ARMTargetLowering()
536 setOperationAction(ISD::MULHS, VT, Expand); in ARMTargetLowering()
537 setOperationAction(ISD::SMUL_LOHI, VT, Expand); in ARMTargetLowering()
538 setOperationAction(ISD::MULHU, VT, Expand); in ARMTargetLowering()
539 setOperationAction(ISD::UMUL_LOHI, VT, Expand); in ARMTargetLowering()
541 setOperationAction(ISD::BSWAP, VT, Expand); in ARMTargetLowering()
726 for (MVT VT : MVT::integer_vector_valuetypes()) { in ARMTargetLowering() local
727 setLoadExtAction(ISD::EXTLOAD, VT, Ty, Legal); in ARMTargetLowering()
728 setLoadExtAction(ISD::ZEXTLOAD, VT, Ty, Legal); in ARMTargetLowering()
729 setLoadExtAction(ISD::SEXTLOAD, VT, Ty, Legal); in ARMTargetLowering()
776 for (MVT VT : MVT::fp_valuetypes()) { in ARMTargetLowering() local
777 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand); in ARMTargetLowering()
778 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand); in ARMTargetLowering()
787 for (MVT VT : MVT::integer_valuetypes()) in ARMTargetLowering() local
788 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); in ARMTargetLowering()
847 for (MVT VT : MVT::vector_valuetypes()) { in ARMTargetLowering() local
848 setOperationAction(ISD::ROTL, VT, Expand); in ARMTargetLowering()
849 setOperationAction(ISD::ROTR, VT, Expand); in ARMTargetLowering()
950 for (auto &VT : {MVT::f32, MVT::f64}) in ARMTargetLowering()
951 setOperationAction(ISD::FPOWI, VT, Custom); in ARMTargetLowering()
1225 MVT VT) const { in findRepresentativeClass()
1228 switch (VT.SimpleTy) { in findRepresentativeClass()
1230 return TargetLowering::findRepresentativeClass(TRI, VT); in findRepresentativeClass()
1418 EVT VT) const { in getSetCCResultType()
1419 if (!VT.isVector()) in getSetCCResultType()
1421 return VT.changeVectorElementTypeToInteger(); in getSetCCResultType()
1426 const TargetRegisterClass *ARMTargetLowering::getRegClassFor(MVT VT) const { in getRegClassFor()
1431 if (VT == MVT::v4i64) in getRegClassFor()
1433 if (VT == MVT::v8i64) in getRegClassFor()
1436 return TargetLowering::getRegClassFor(VT); in getRegClassFor()
1466 EVT VT = N->getValueType(i); in getSchedulingPreference() local
1467 if (VT == MVT::Glue || VT == MVT::Other) in getSchedulingPreference()
1469 if (VT.isFloatingPoint() || VT.isVector()) in getSchedulingPreference()
1914 Outs[0].VT == MVT::i32) { in LowerCall()
1917 assert(!Ins.empty() && Ins[0].VT == MVT::i32 && in LowerCall()
4012 EVT VT = Op.getValueType(); in LowerSignedALUO() local
4014 SDValue Overflow = DAG.getNode(ARMISD::CMOV, dl, VT, TVal, FVal, in LowerSignedALUO()
4034 static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT, in ConvertCarryFlagToBooleanCarry() argument
4040 return DAG.getNode(ARMISD::ADDE, DL, DAG.getVTList(VT, MVT::i32), in ConvertCarryFlagToBooleanCarry()
4055 EVT VT = Op.getValueType(); in LowerUnsignedALUO() local
4056 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerUnsignedALUO()
4065 Overflow = ConvertCarryFlagToBooleanCarry(Value.getValue(1), VT, DAG); in LowerUnsignedALUO()
4070 Overflow = ConvertCarryFlagToBooleanCarry(Value.getValue(1), VT, DAG); in LowerUnsignedALUO()
4099 EVT VT = Op.getValueType(); in LowerSELECT() local
4101 return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, CCR, in LowerSELECT()
4131 EVT VT = Op.getValueType(); in LowerSELECT() local
4135 assert(True.getValueType() == VT); in LowerSELECT()
4136 return getCMOV(dl, VT, True, False, ARMcc, CCR, Cmp, DAG); in LowerSELECT()
4200 SDValue ARMTargetLowering::getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal, in getCMOV() argument
4203 if (Subtarget->isFPOnlySP() && VT == MVT::f64) { in getCMOV()
4221 return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMcc, CCR, in getCMOV()
4410 EVT VT = Op.getValueType(); in LowerSELECT_CC() local
4420 return DAG.getNode(ARMISD::USAT, dl, VT, SatValue, in LowerSELECT_CC()
4421 DAG.getConstant(countTrailingOnes(SatConstant), dl, VT)); in LowerSELECT_CC()
4423 return DAG.getNode(ARMISD::SSAT, dl, VT, SatValue, in LowerSELECT_CC()
4424 DAG.getConstant(countTrailingOnes(SatConstant), dl, VT)); in LowerSELECT_CC()
4434 if (VT == MVT::i32 && in LowerSELECT_CC()
4436 SDValue ShiftV = DAG.getNode(ISD::SRA, dl, VT, SatValue, in LowerSELECT_CC()
4437 DAG.getConstant(31, dl, VT)); in LowerSELECT_CC()
4439 SDValue NotShiftV = DAG.getNode(ISD::XOR, dl, VT, ShiftV, in LowerSELECT_CC()
4440 DAG.getAllOnesConstant(dl, VT)); in LowerSELECT_CC()
4441 return DAG.getNode(ISD::AND, dl, VT, SatValue, NotShiftV); in LowerSELECT_CC()
4443 return DAG.getNode(ISD::OR, dl, VT, SatValue, ShiftV); in LowerSELECT_CC()
4488 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG); in LowerSELECT_CC()
4517 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG); in LowerSELECT_CC()
4522 Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, CCR, Cmp2, DAG); in LowerSELECT_CC()
4537 EVT VT = Op.getValueType(); in canChangeToInt() local
4538 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow()) in canChangeToInt()
4802 EVT VT = Op.getValueType(); in LowerVectorFP_TO_INT() local
4825 if (VT != MVT::v4i16 && VT != MVT::v8i16) in LowerVectorFP_TO_INT()
4829 return DAG.getNode(ISD::TRUNCATE, dl, VT, Op); in LowerVectorFP_TO_INT()
4833 EVT VT = Op.getValueType(); in LowerFP_TO_INT() local
4834 if (VT.isVector()) in LowerFP_TO_INT()
4852 EVT VT = Op.getValueType(); in LowerVectorINT_TO_FP() local
4856 if (VT.getVectorElementType() == MVT::f32) in LowerVectorINT_TO_FP()
4869 if (VT == MVT::v4f32) in LowerVectorINT_TO_FP()
4871 else if (VT == MVT::v4f16 && HasFullFP16) in LowerVectorINT_TO_FP()
4873 else if (VT == MVT::v8f16 && HasFullFP16) in LowerVectorINT_TO_FP()
4893 return DAG.getNode(Opc, dl, VT, Op); in LowerVectorINT_TO_FP()
4897 EVT VT = Op.getValueType(); in LowerINT_TO_FP() local
4898 if (VT.isVector()) in LowerINT_TO_FP()
4920 EVT VT = Op.getValueType(); in LowerFCOPYSIGN() local
4931 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64; in LowerFCOPYSIGN()
4932 if (VT == MVT::f64) in LowerFCOPYSIGN()
4940 if (VT == MVT::f64) in LowerFCOPYSIGN()
4944 } else if (VT == MVT::f32) in LowerFCOPYSIGN()
4960 if (VT == MVT::f32) { in LowerFCOPYSIGN()
4981 if (VT == MVT::f32) { in LowerFCOPYSIGN()
5005 EVT VT = Op.getValueType(); in LowerRETURNADDR() local
5011 return DAG.getLoad(VT, dl, DAG.getEntryNode(), in LowerRETURNADDR()
5012 DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset), in LowerRETURNADDR()
5018 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT); in LowerRETURNADDR()
5028 EVT VT = Op.getValueType(); in LowerFRAMEADDR() local
5032 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT); in LowerFRAMEADDR()
5034 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr, in LowerFRAMEADDR()
5041 unsigned ARMTargetLowering::getRegisterByName(const char* RegName, EVT VT, in getRegisterByName() argument
5249 static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) { in getZeroVector() argument
5250 assert(VT.isVector() && "Expected a vector type"); in getZeroVector()
5253 EVT VmovVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32; in getZeroVector()
5255 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov); in getZeroVector()
5263 EVT VT = Op.getValueType(); in LowerShiftRightParts() local
5264 unsigned VTBits = VT.getSizeInBits(); in LowerShiftRightParts()
5277 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt); in LowerShiftRightParts()
5280 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt); in LowerShiftRightParts()
5281 SDValue LoSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); in LowerShiftRightParts()
5282 SDValue LoBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt); in LowerShiftRightParts()
5285 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift, LoBigShift, in LowerShiftRightParts()
5288 SDValue HiSmallShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt); in LowerShiftRightParts()
5290 ? DAG.getNode(Opc, dl, VT, ShOpHi, in LowerShiftRightParts()
5291 DAG.getConstant(VTBits - 1, dl, VT)) in LowerShiftRightParts()
5292 : DAG.getConstant(0, dl, VT); in LowerShiftRightParts()
5295 SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift, in LowerShiftRightParts()
5307 EVT VT = Op.getValueType(); in LowerShiftLeftParts() local
5308 unsigned VTBits = VT.getSizeInBits(); in LowerShiftLeftParts()
5319 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt); in LowerShiftLeftParts()
5320 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt); in LowerShiftLeftParts()
5321 SDValue HiSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); in LowerShiftLeftParts()
5325 SDValue HiBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt); in LowerShiftLeftParts()
5328 SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift, in LowerShiftLeftParts()
5333 SDValue LoSmallShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt); in LowerShiftLeftParts()
5334 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift, in LowerShiftLeftParts()
5335 DAG.getConstant(0, dl, VT), ARMcc, CCR, CmpLo); in LowerShiftLeftParts()
5363 EVT VT = N->getValueType(0); in LowerCTTZ() local
5364 if (VT.isVector()) { in LowerCTTZ()
5369 SDValue NX = DAG.getNode(ISD::SUB, dl, VT, getZeroVector(VT, DAG, dl), X); in LowerCTTZ()
5370 SDValue LSB = DAG.getNode(ISD::AND, dl, VT, X, NX); in LowerCTTZ()
5372 EVT ElemTy = VT.getVectorElementType(); in LowerCTTZ()
5376 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT, in LowerCTTZ()
5378 SDValue Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One); in LowerCTTZ()
5379 return DAG.getNode(ISD::CTPOP, dl, VT, Bits); in LowerCTTZ()
5387 DAG.getNode(ARMISD::VMOVIMM, dl, VT, in LowerCTTZ()
5389 SDValue CTLZ = DAG.getNode(ISD::CTLZ, dl, VT, LSB); in LowerCTTZ()
5390 return DAG.getNode(ISD::SUB, dl, VT, WidthMinus1, CTLZ); in LowerCTTZ()
5399 SDValue FF = DAG.getNode(ARMISD::VMOVIMM, dl, VT, in LowerCTTZ()
5401 Bits = DAG.getNode(ISD::ADD, dl, VT, LSB, FF); in LowerCTTZ()
5403 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT, in LowerCTTZ()
5405 Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One); in LowerCTTZ()
5407 return DAG.getNode(ISD::CTPOP, dl, VT, Bits); in LowerCTTZ()
5413 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, VT, N->getOperand(0)); in LowerCTTZ()
5414 return DAG.getNode(ISD::CTLZ, dl, VT, rbit); in LowerCTTZ()
5419 EVT VT = N->getValueType(0); in LowerCTPOP() local
5423 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 || in LowerCTPOP()
5424 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) && in LowerCTPOP()
5428 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8; in LowerCTPOP()
5434 unsigned NumElts = VT.is64BitVector() ? 8 : 16; in LowerCTPOP()
5435 while (EltSize != VT.getScalarSizeInBits()) { in LowerCTPOP()
5452 EVT VT = N->getValueType(0); in LowerShift() local
5455 if (!VT.isVector()) in LowerShift()
5463 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerShift()
5481 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, in LowerShift()
5488 EVT VT = N->getValueType(0); in Expand64BitShift() local
5492 if (VT != MVT::i64) in Expand64BitShift()
5533 EVT VT = Op.getValueType(); in LowerVSETCC() local
5552 Merged = DAG.getSExtOrTrunc(Merged, dl, VT); in LowerVSETCC()
5673 Result = DAG.getSExtOrTrunc(Result, dl, VT); in LowerVSETCC()
5676 Result = DAG.getNOT(dl, Result, VT); in LowerVSETCC()
5716 const SDLoc &dl, EVT &VT, bool is128Bits, in isNEONModifiedImm() argument
5736 VT = is128Bits ? MVT::v16i8 : MVT::v8i8; in isNEONModifiedImm()
5741 VT = is128Bits ? MVT::v8i16 : MVT::v4i16; in isNEONModifiedImm()
5761 VT = is128Bits ? MVT::v4i32 : MVT::v2i32; in isNEONModifiedImm()
5838 VT = is128Bits ? MVT::v2i64 : MVT::v1i64; in isNEONModifiedImm()
5852 EVT VT = Op.getValueType(); in LowerConstantFP() local
5853 bool IsDouble = (VT == MVT::f64); in LowerConstantFP()
5861 if (isFPImmLegal(FPVal, VT)) in LowerConstantFP()
5866 switch (VT.getSimpleVT().SimpleTy) { in LowerConstantFP()
5878 return DAG.getNode(ARMISD::VMOVSR, DL, VT, in LowerConstantFP()
5964 static bool isSingletonVEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) { in isSingletonVEXTMask() argument
5965 unsigned NumElts = VT.getVectorNumElements(); in isSingletonVEXTMask()
5992 static bool isVEXTMask(ArrayRef<int> M, EVT VT, in isVEXTMask() argument
5994 unsigned NumElts = VT.getVectorNumElements(); in isVEXTMask()
6031 static bool isVREVMask(ArrayRef<int> M, EVT VT, unsigned BlockSize) { in isVREVMask() argument
6035 unsigned EltSz = VT.getScalarSizeInBits(); in isVREVMask()
6039 unsigned NumElts = VT.getVectorNumElements(); in isVREVMask()
6057 static bool isVTBLMask(ArrayRef<int> M, EVT VT) { in isVTBLMask() argument
6061 return VT == MVT::v8i8 && M.size() == 8; in isVTBLMask()
6092 static bool isVTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isVTRNMask() argument
6093 unsigned EltSz = VT.getScalarSizeInBits(); in isVTRNMask()
6097 unsigned NumElts = VT.getVectorNumElements(); in isVTRNMask()
6124 static bool isVTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){ in isVTRN_v_undef_Mask() argument
6125 unsigned EltSz = VT.getScalarSizeInBits(); in isVTRN_v_undef_Mask()
6129 unsigned NumElts = VT.getVectorNumElements(); in isVTRN_v_undef_Mask()
6156 static bool isVUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isVUZPMask() argument
6157 unsigned EltSz = VT.getScalarSizeInBits(); in isVUZPMask()
6161 unsigned NumElts = VT.getVectorNumElements(); in isVUZPMask()
6177 if (VT.is64BitVector() && EltSz == 32) in isVUZPMask()
6186 static bool isVUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){ in isVUZP_v_undef_Mask() argument
6187 unsigned EltSz = VT.getScalarSizeInBits(); in isVUZP_v_undef_Mask()
6191 unsigned NumElts = VT.getVectorNumElements(); in isVUZP_v_undef_Mask()
6213 if (VT.is64BitVector() && EltSz == 32) in isVUZP_v_undef_Mask()
6227 static bool isVZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) { in isVZIPMask() argument
6228 unsigned EltSz = VT.getScalarSizeInBits(); in isVZIPMask()
6232 unsigned NumElts = VT.getVectorNumElements(); in isVZIPMask()
6251 if (VT.is64BitVector() && EltSz == 32) in isVZIPMask()
6260 static bool isVZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){ in isVZIP_v_undef_Mask() argument
6261 unsigned EltSz = VT.getScalarSizeInBits(); in isVZIP_v_undef_Mask()
6265 unsigned NumElts = VT.getVectorNumElements(); in isVZIP_v_undef_Mask()
6284 if (VT.is64BitVector() && EltSz == 32) in isVZIP_v_undef_Mask()
6292 static unsigned isNEONTwoResultShuffleMask(ArrayRef<int> ShuffleMask, EVT VT, in isNEONTwoResultShuffleMask() argument
6296 if (isVTRNMask(ShuffleMask, VT, WhichResult)) in isNEONTwoResultShuffleMask()
6298 if (isVUZPMask(ShuffleMask, VT, WhichResult)) in isNEONTwoResultShuffleMask()
6300 if (isVZIPMask(ShuffleMask, VT, WhichResult)) in isNEONTwoResultShuffleMask()
6304 if (isVTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) in isNEONTwoResultShuffleMask()
6306 if (isVUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) in isNEONTwoResultShuffleMask()
6308 if (isVZIP_v_undef_Mask(ShuffleMask, VT, WhichResult)) in isNEONTwoResultShuffleMask()
6315 static bool isReverseMask(ArrayRef<int> M, EVT VT) { in isReverseMask() argument
6316 unsigned NumElts = VT.getVectorNumElements(); in isReverseMask()
6355 EVT VT = Op.getValueType(); in LowerBUILD_VECTOR() local
6362 return DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
6369 DAG, dl, VmovVT, VT.is128BitVector(), in LowerBUILD_VECTOR()
6373 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov); in LowerBUILD_VECTOR()
6380 DAG, dl, VmovVT, VT.is128BitVector(), in LowerBUILD_VECTOR()
6384 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov); in LowerBUILD_VECTOR()
6388 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) { in LowerBUILD_VECTOR()
6392 return DAG.getNode(ARMISD::VMOVFPIMM, dl, VT, Val); in LowerBUILD_VECTOR()
6405 unsigned NumElts = VT.getVectorNumElements(); in LowerBUILD_VECTOR()
6439 return DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
6444 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value); in LowerBUILD_VECTOR()
6446 unsigned EltSize = VT.getScalarSizeInBits(); in LowerBUILD_VECTOR()
6466 if (VT != Value->getOperand(0).getValueType()) { in LowerBUILD_VECTOR()
6468 VT.getVectorNumElements(); in LowerBUILD_VECTOR()
6469 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT, in LowerBUILD_VECTOR()
6470 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DAG.getUNDEF(VT), in LowerBUILD_VECTOR()
6474 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT, in LowerBUILD_VECTOR()
6477 N = DAG.getNode(ARMISD::VDUP, dl, VT, Value); in LowerBUILD_VECTOR()
6489 N = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ops); in LowerBUILD_VECTOR()
6494 if (VT.getVectorElementType().isFloatingPoint()) { in LowerBUILD_VECTOR()
6503 return DAG.getNode(ISD::BITCAST, dl, VT, Val); in LowerBUILD_VECTOR()
6508 return DAG.getNode(ARMISD::VDUP, dl, VT, Val); in LowerBUILD_VECTOR()
6525 if (VT.is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) { in LowerBUILD_VECTOR()
6529 EVT ExtVT = VT.getVectorElementType(); in LowerBUILD_VECTOR()
6540 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lower, Upper); in LowerBUILD_VECTOR()
6555 return DAG.getNode(ISD::BITCAST, dl, VT, Val); in LowerBUILD_VECTOR()
6565 SDValue Vec = DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
6571 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx); in LowerBUILD_VECTOR()
6585 EVT VT = Op.getValueType(); in ReconstructShuffle() local
6586 unsigned NumElts = VT.getVectorNumElements(); in ReconstructShuffle()
6644 EVT SmallestEltTy = VT.getVectorElementType(); in ReconstructShuffle()
6651 VT.getScalarSizeInBits() / SmallestEltTy.getSizeInBits(); in ReconstructShuffle()
6652 NumElts = VT.getSizeInBits() / SmallestEltTy.getSizeInBits(); in ReconstructShuffle()
6661 if (SrcVT.getSizeInBits() == VT.getSizeInBits()) in ReconstructShuffle()
6667 unsigned NumSrcElts = VT.getSizeInBits() / EltVT.getSizeInBits(); in ReconstructShuffle()
6670 if (SrcVT.getSizeInBits() < VT.getSizeInBits()) { in ReconstructShuffle()
6671 if (2 * SrcVT.getSizeInBits() != VT.getSizeInBits()) in ReconstructShuffle()
6681 if (SrcVT.getSizeInBits() != 2 * VT.getSizeInBits()) in ReconstructShuffle()
6737 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { in ReconstructShuffle()
6750 VT.getScalarSizeInBits()); in ReconstructShuffle()
6777 return DAG.getNode(ISD::BITCAST, dl, VT, Shuffle); in ReconstructShuffle()
6784 bool ARMTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const { in isShuffleMaskLegal()
6785 if (VT.getVectorNumElements() == 4 && in isShuffleMaskLegal()
6786 (VT.is128BitVector() || VT.is64BitVector())) { in isShuffleMaskLegal()
6808 unsigned EltSize = VT.getScalarSizeInBits(); in isShuffleMaskLegal()
6810 ShuffleVectorSDNode::isSplatMask(&M[0], VT) || in isShuffleMaskLegal()
6811 isVREVMask(M, VT, 64) || in isShuffleMaskLegal()
6812 isVREVMask(M, VT, 32) || in isShuffleMaskLegal()
6813 isVREVMask(M, VT, 16) || in isShuffleMaskLegal()
6814 isVEXTMask(M, VT, ReverseVEXT, Imm) || in isShuffleMaskLegal()
6815 isVTBLMask(M, VT) || in isShuffleMaskLegal()
6816 isNEONTwoResultShuffleMask(M, VT, WhichResult, isV_UNDEF) || in isShuffleMaskLegal()
6817 ((VT == MVT::v8i16 || VT == MVT::v16i8) && isReverseMask(M, VT))); in isShuffleMaskLegal()
6856 EVT VT = OpLHS.getValueType(); in GeneratePerfectShuffle() local
6862 if (VT.getVectorElementType() == MVT::i32 || in GeneratePerfectShuffle()
6863 VT.getVectorElementType() == MVT::f32) in GeneratePerfectShuffle()
6864 return DAG.getNode(ARMISD::VREV64, dl, VT, OpLHS); in GeneratePerfectShuffle()
6866 if (VT.getVectorElementType() == MVT::i16) in GeneratePerfectShuffle()
6867 return DAG.getNode(ARMISD::VREV32, dl, VT, OpLHS); in GeneratePerfectShuffle()
6869 assert(VT.getVectorElementType() == MVT::i8); in GeneratePerfectShuffle()
6870 return DAG.getNode(ARMISD::VREV16, dl, VT, OpLHS); in GeneratePerfectShuffle()
6875 return DAG.getNode(ARMISD::VDUPLANE, dl, VT, in GeneratePerfectShuffle()
6880 return DAG.getNode(ARMISD::VEXT, dl, VT, in GeneratePerfectShuffle()
6885 return DAG.getNode(ARMISD::VUZP, dl, DAG.getVTList(VT, VT), in GeneratePerfectShuffle()
6889 return DAG.getNode(ARMISD::VZIP, dl, DAG.getVTList(VT, VT), in GeneratePerfectShuffle()
6893 return DAG.getNode(ARMISD::VTRN, dl, DAG.getVTList(VT, VT), in GeneratePerfectShuffle()
6923 EVT VT = OpLHS.getValueType(); in LowerReverse_VECTOR_SHUFFLEv16i8_v8i16() local
6925 assert((VT == MVT::v8i16 || VT == MVT::v16i8) && in LowerReverse_VECTOR_SHUFFLEv16i8_v8i16()
6927 OpLHS = DAG.getNode(ARMISD::VREV64, DL, VT, OpLHS); in LowerReverse_VECTOR_SHUFFLEv16i8_v8i16()
6931 unsigned ExtractNum = (VT == MVT::v16i8) ? 8 : 4; in LowerReverse_VECTOR_SHUFFLEv16i8_v8i16()
6932 return DAG.getNode(ARMISD::VEXT, DL, VT, OpLHS, OpLHS, in LowerReverse_VECTOR_SHUFFLEv16i8_v8i16()
6940 EVT VT = Op.getValueType(); in LowerVECTOR_SHUFFLE() local
6951 unsigned EltSize = VT.getScalarSizeInBits(); in LowerVECTOR_SHUFFLE()
6960 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0)); in LowerVECTOR_SHUFFLE()
6974 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0)); in LowerVECTOR_SHUFFLE()
6976 return DAG.getNode(ARMISD::VDUPLANE, dl, VT, V1, in LowerVECTOR_SHUFFLE()
6982 if (isVEXTMask(ShuffleMask, VT, ReverseVEXT, Imm)) { in LowerVECTOR_SHUFFLE()
6985 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V2, in LowerVECTOR_SHUFFLE()
6989 if (isVREVMask(ShuffleMask, VT, 64)) in LowerVECTOR_SHUFFLE()
6990 return DAG.getNode(ARMISD::VREV64, dl, VT, V1); in LowerVECTOR_SHUFFLE()
6991 if (isVREVMask(ShuffleMask, VT, 32)) in LowerVECTOR_SHUFFLE()
6992 return DAG.getNode(ARMISD::VREV32, dl, VT, V1); in LowerVECTOR_SHUFFLE()
6993 if (isVREVMask(ShuffleMask, VT, 16)) in LowerVECTOR_SHUFFLE()
6994 return DAG.getNode(ARMISD::VREV16, dl, VT, V1); in LowerVECTOR_SHUFFLE()
6996 if (V2->isUndef() && isSingletonVEXTMask(ShuffleMask, VT, Imm)) { in LowerVECTOR_SHUFFLE()
6997 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V1, in LowerVECTOR_SHUFFLE()
7009 ShuffleMask, VT, WhichResult, isV_UNDEF)) { in LowerVECTOR_SHUFFLE()
7012 return DAG.getNode(ShuffleOpc, dl, DAG.getVTList(VT, VT), V1, V2) in LowerVECTOR_SHUFFLE()
7038 return i < (int)VT.getVectorNumElements(); in LowerVECTOR_SHUFFLE()
7049 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Res.getValue(0), in LowerVECTOR_SHUFFLE()
7057 unsigned NumElts = VT.getVectorNumElements(); in LowerVECTOR_SHUFFLE()
7096 return DAG.getNode(ISD::BITCAST, dl, VT, Val); in LowerVECTOR_SHUFFLE()
7099 if ((VT == MVT::v8i16 || VT == MVT::v16i8) && isReverseMask(ShuffleMask, VT)) in LowerVECTOR_SHUFFLE()
7102 if (VT == MVT::v8i8) in LowerVECTOR_SHUFFLE()
7159 EVT VT = N->getValueType(0); in isExtendedBUILD_VECTOR() local
7160 if (VT == MVT::v2i64 && N->getOpcode() == ISD::BITCAST) { in isExtendedBUILD_VECTOR()
7190 unsigned EltSize = VT.getScalarSizeInBits(); in isExtendedBUILD_VECTOR()
7327 EVT VT = N->getValueType(0); in SkipExtensionForVMULL() local
7328 unsigned EltSize = VT.getScalarSizeInBits() / 2; in SkipExtensionForVMULL()
7329 unsigned NumElts = VT.getVectorNumElements(); in SkipExtensionForVMULL()
7368 EVT VT = Op.getValueType(); in LowerMUL() local
7369 assert(VT.is128BitVector() && VT.isInteger() && in LowerMUL()
7401 if (VT == MVT::v2i64) in LowerMUL()
7419 return DAG.getNode(NewOpc, DL, VT, Op0, Op1); in LowerMUL()
7433 return DAG.getNode(N0->getOpcode(), DL, VT, in LowerMUL()
7434 DAG.getNode(NewOpc, DL, VT, in LowerMUL()
7436 DAG.getNode(NewOpc, DL, VT, in LowerMUL()
7511 EVT VT = Op.getValueType(); in LowerSDIV() local
7512 assert((VT == MVT::v4i16 || VT == MVT::v8i8) && in LowerSDIV()
7520 if (VT == MVT::v8i8) { in LowerSDIV()
7547 EVT VT = Op.getValueType(); in LowerUDIV() local
7548 assert((VT == MVT::v4i16 || VT == MVT::v8i8) && in LowerUDIV()
7556 if (VT == MVT::v8i8) { in LowerUDIV()
7623 EVT VT = N->getValueType(0); in LowerADDSUBCARRY() local
7624 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerADDSUBCARRY()
7640 Carry = ConvertCarryFlagToBooleanCarry(Result.getValue(1), VT, DAG); in LowerADDSUBCARRY()
7654 Carry = ConvertCarryFlagToBooleanCarry(Result.getValue(1), VT, DAG); in LowerADDSUBCARRY()
7743 EVT VT = Op.getValueType(); in LowerWindowsDIVLibCall() local
7744 assert((VT == MVT::i32 || VT == MVT::i64) && in LowerWindowsDIVLibCall()
7753 Name = (VT == MVT::i32) ? "__rt_sdiv" : "__rt_sdiv64"; in LowerWindowsDIVLibCall()
7755 Name = (VT == MVT::i32) ? "__rt_udiv" : "__rt_udiv64"; in LowerWindowsDIVLibCall()
7771 .setCallee(CallingConv::ARM_AAPCS_VFP, VT.getTypeForEVT(*DAG.getContext()), in LowerWindowsDIVLibCall()
9688 EVT VT = N->getValueType(0); in isConditionalZeroOrAllOnes() local
9696 OtherOp = DAG.getConstant(0, dl, VT); in isConditionalZeroOrAllOnes()
9699 OtherOp = DAG.getConstant(1, dl, VT); in isConditionalZeroOrAllOnes()
9701 OtherOp = DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl, in isConditionalZeroOrAllOnes()
9702 VT); in isConditionalZeroOrAllOnes()
9737 EVT VT = N->getValueType(0); in combineSelectAndUse() local
9747 SDValue FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT, in combineSelectAndUse()
9753 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, in combineSelectAndUse()
9801 EVT VT = N->getValueType(0); in AddCombineToVPADD() local
9809 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops); in AddCombineToVPADD()
9840 EVT VT = N->getValueType(0); in AddCombineVUZPToVPADDL() local
9852 unsigned NumElts = VT.getVectorNumElements(); in AddCombineVUZPToVPADDL()
9858 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops); in AddCombineVUZPToVPADDL()
9876 EVT VT = N->getValueType(0); in AddCombineBUILD_VECTORToVPADDL() local
9877 if (!VT.isInteger() || VT.getVectorElementType() == MVT::i64) in AddCombineBUILD_VECTORToVPADDL()
9927 Vec.getValueType().getVectorElementType() == VT.getVectorElementType()) in AddCombineBUILD_VECTORToVPADDL()
9946 unsigned numElem = VT.getVectorNumElements(); in AddCombineBUILD_VECTORToVPADDL()
9958 unsigned ExtOp = VT.bitsGT(tmp.getValueType()) ? ISD::ANY_EXTEND : ISD::TRUNCATE; in AddCombineBUILD_VECTORToVPADDL()
9959 return DAG.getNode(ExtOp, dl, VT, tmp); in AddCombineBUILD_VECTORToVPADDL()
10600 EVT VT = N->getValueType(0); in PerformVMULCombine() local
10604 return DAG.getNode(Opcode, DL, VT, in PerformVMULCombine()
10605 DAG.getNode(ISD::MUL, DL, VT, N00, N1), in PerformVMULCombine()
10606 DAG.getNode(ISD::MUL, DL, VT, N01, N1)); in PerformVMULCombine()
10620 EVT VT = N->getValueType(0); in PerformMULCombine() local
10621 if (VT.is64BitVector() || VT.is128BitVector()) in PerformMULCombine()
10623 if (VT != MVT::i32) in PerformMULCombine()
10643 Res = DAG.getNode(ISD::ADD, DL, VT, in PerformMULCombine()
10645 DAG.getNode(ISD::SHL, DL, VT, in PerformMULCombine()
10651 Res = DAG.getNode(ISD::SUB, DL, VT, in PerformMULCombine()
10652 DAG.getNode(ISD::SHL, DL, VT, in PerformMULCombine()
10663 Res = DAG.getNode(ISD::SUB, DL, VT, in PerformMULCombine()
10665 DAG.getNode(ISD::SHL, DL, VT, in PerformMULCombine()
10671 Res = DAG.getNode(ISD::ADD, DL, VT, in PerformMULCombine()
10673 DAG.getNode(ISD::SHL, DL, VT, in PerformMULCombine()
10677 Res = DAG.getNode(ISD::SUB, DL, VT, in PerformMULCombine()
10684 Res = DAG.getNode(ISD::SHL, DL, VT, in PerformMULCombine()
10802 EVT VT = N->getValueType(0); in PerformANDCombine() local
10805 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in PerformANDCombine()
10817 DAG, dl, VbicVT, VT.is128BitVector(), in PerformANDCombine()
10823 return DAG.getNode(ISD::BITCAST, dl, VT, Vbic); in PerformANDCombine()
10912 EVT VT = N->getValueType(0); in PerformORCombineToBFI() local
10927 if (VT != MVT::i32) in PerformORCombineToBFI()
10953 Res = DAG.getNode(ARMISD::BFI, DL, VT, N00, in PerformORCombineToBFI()
10980 Res = DAG.getNode(ISD::SRL, DL, VT, N1.getOperand(0), in PerformORCombineToBFI()
10982 Res = DAG.getNode(ARMISD::BFI, DL, VT, N00, Res, in PerformORCombineToBFI()
10997 Res = DAG.getNode(ISD::SRL, DL, VT, N00, in PerformORCombineToBFI()
10999 Res = DAG.getNode(ARMISD::BFI, DL, VT, N1.getOperand(0), Res, in PerformORCombineToBFI()
11019 Res = DAG.getNode(ARMISD::BFI, DL, VT, N1, N00.getOperand(0), in PerformORCombineToBFI()
11038 EVT VT = N->getValueType(0); in PerformORCombine() local
11041 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in PerformORCombine()
11053 DAG, dl, VorrVT, VT.is128BitVector(), in PerformORCombine()
11059 return DAG.getNode(ISD::BITCAST, dl, VT, Vorr); in PerformORCombine()
11076 if (Subtarget->hasNEON() && N1.getOpcode() == ISD::AND && VT.isVector() && in PerformORCombine()
11077 DAG.getTargetLoweringInfo().isTypeLegal(VT)) { in PerformORCombine()
11104 EVT CanonicalVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32; in PerformORCombine()
11109 return DAG.getNode(ISD::BITCAST, dl, VT, Result); in PerformORCombine()
11131 EVT VT = N->getValueType(0); in PerformXORCombine() local
11134 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in PerformXORCombine()
11269 EVT VT = N->getValueType(0); in PerformBFICombine() local
11274 ISD::SRL, dl, VT, From1, in PerformBFICombine()
11275 DCI.DAG.getConstant(NewFromMask.countTrailingZeros(), dl, VT)); in PerformBFICombine()
11276 return DCI.DAG.getNode(ARMISD::BFI, dl, VT, N->getOperand(0), From1, in PerformBFICombine()
11277 DCI.DAG.getConstant(~NewToMask, dl, VT)); in PerformBFICombine()
11372 EVT VT = N->getValueType(0); in PerformBUILD_VECTORCombine() local
11373 if (VT.getVectorElementType() != MVT::i64 || !hasNormalLoadOperand(N)) in PerformBUILD_VECTORCombine()
11377 unsigned NumElts = VT.getVectorNumElements(); in PerformBUILD_VECTORCombine()
11386 return DAG.getNode(ISD::BITCAST, dl, VT, BV); in PerformBUILD_VECTORCombine()
11404 EVT VT = N->getValueType(0); in PerformARMBUILD_VECTORCombine() local
11405 EVT EltVT = VT.getVectorElementType(); in PerformARMBUILD_VECTORCombine()
11426 unsigned NumElts = VT.getVectorNumElements(); in PerformARMBUILD_VECTORCombine()
11475 Vec = DAG.getNode(ISD::BITCAST, dl, VT, Vec); in PerformARMBUILD_VECTORCombine()
11487 EVT VT = N->getValueType(0); in PerformInsertEltCombine() local
11489 if (VT.getVectorElementType() != MVT::i64 || in PerformInsertEltCombine()
11496 VT.getVectorNumElements()); in PerformInsertEltCombine()
11504 return DAG.getNode(ISD::BITCAST, dl, VT, InsElt); in PerformInsertEltCombine()
11533 EVT VT = N->getValueType(0); in PerformVECTOR_SHUFFLECombine() local
11534 if (!TLI.isTypeLegal(VT) || in PerformVECTOR_SHUFFLECombine()
11539 SDValue NewConcat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, in PerformVECTOR_SHUFFLECombine()
11543 unsigned NumElts = VT.getVectorNumElements(); in PerformVECTOR_SHUFFLECombine()
11555 return DAG.getVectorShuffle(VT, SDLoc(N), NewConcat, in PerformVECTOR_SHUFFLECombine()
11556 DAG.getUNDEF(VT), NewMask); in PerformVECTOR_SHUFFLECombine()
11794 EVT VT = N->getValueType(0); in CombineVLDDUP() local
11796 if (!VT.is64BitVector()) in CombineVLDDUP()
11838 Tys[n] = VT; in CombineVLDDUP()
11894 EVT VT = N->getValueType(0); in PerformVDUPLANECombine() local
11895 if (EltSize > VT.getScalarSizeInBits()) in PerformVDUPLANECombine()
11898 return DCI.DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op); in PerformVDUPLANECombine()
11928 EVT VT = N->getValueType(0); in PerformLOADCombine() local
11931 if (ISD::isNormalLoad(N) && VT.isVector() && in PerformLOADCombine()
11932 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT)) in PerformLOADCombine()
11950 EVT VT = StVal.getValueType(); in PerformSTORECombine() local
11951 if (St->isTruncatingStore() && VT.isVector()) { in PerformSTORECombine()
11955 unsigned NumElems = VT.getVectorNumElements(); in PerformSTORECombine()
11956 assert(StVT != VT && "Cannot truncate to the same type"); in PerformSTORECombine()
11957 unsigned FromEltSz = VT.getScalarSizeInBits(); in PerformSTORECombine()
11968 assert(SizeRatio * NumElems * ToEltSz == VT.getSizeInBits()); in PerformSTORECombine()
11973 assert(WideVecVT.getSizeInBits() == VT.getSizeInBits()); in PerformSTORECombine()
12004 StoreType, VT.getSizeInBits()/EVT(StoreType).getSizeInBits()); in PerformSTORECombine()
12005 assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits()); in PerformSTORECombine()
12078 if (ISD::isNormalStore(N) && VT.isVector() && in PerformSTORECombine()
12079 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT)) in PerformSTORECombine()
12224 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) { in isVShiftLImm() argument
12225 assert(VT.isVector() && "vector shift count is not a vector type"); in isVShiftLImm()
12226 int64_t ElementBits = VT.getScalarSizeInBits(); in isVShiftLImm()
12238 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, bool isIntrinsic, in isVShiftRImm() argument
12240 assert(VT.isVector() && "vector shift count is not a vector type"); in isVShiftRImm()
12241 int64_t ElementBits = VT.getScalarSizeInBits(); in isVShiftRImm()
12280 EVT VT = N->getOperand(1).getValueType(); in PerformIntrinsicCombine() local
12287 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) { in PerformIntrinsicCombine()
12291 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt)) { in PerformIntrinsicCombine()
12300 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt)) in PerformIntrinsicCombine()
12306 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) in PerformIntrinsicCombine()
12311 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) in PerformIntrinsicCombine()
12323 if (isVShiftRImm(N->getOperand(2), VT, true, true, Cnt)) in PerformIntrinsicCombine()
12369 EVT VT = N->getOperand(1).getValueType(); in PerformIntrinsicCombine() local
12373 if (isVShiftLImm(N->getOperand(3), VT, false, Cnt)) in PerformIntrinsicCombine()
12375 else if (isVShiftRImm(N->getOperand(3), VT, false, true, Cnt)) in PerformIntrinsicCombine()
12403 EVT VT = N->getValueType(0); in PerformShiftCombine() local
12404 if (N->getOpcode() == ISD::SRL && VT == MVT::i32 && ST->hasV6Ops()) { in PerformShiftCombine()
12413 return DAG.getNode(ISD::ROTR, SDLoc(N), VT, N0, N1); in PerformShiftCombine()
12419 if (!VT.isVector() || !TLI.isTypeLegal(VT)) in PerformShiftCombine()
12429 if (isVShiftLImm(N->getOperand(1), VT, false, Cnt)) { in PerformShiftCombine()
12431 return DAG.getNode(ARMISD::VSHL, dl, VT, N->getOperand(0), in PerformShiftCombine()
12438 if (isVShiftRImm(N->getOperand(1), VT, false, false, Cnt)) { in PerformShiftCombine()
12442 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0), in PerformShiftCombine()
12462 EVT VT = N->getValueType(0); in PerformExtendCombine() local
12466 if (VT == MVT::i32 && in PerformExtendCombine()
12482 return DAG.getNode(Opc, SDLoc(N), VT, Vec, Lane); in PerformExtendCombine()
12564 EVT VT = X.getValueType(); in PerformCMOVToBFICombine() local
12569 X = DAG.getNode(ISD::SRL, dl, VT, X, in PerformCMOVToBFICombine()
12570 DAG.getConstant(BitInX, dl, VT)); in PerformCMOVToBFICombine()
12577 APInt Mask(VT.getSizeInBits(), 0); in PerformCMOVToBFICombine()
12579 V = DAG.getNode(ARMISD::BFI, dl, VT, V, X, in PerformCMOVToBFICombine()
12581 DAG.getConstant(~Mask, dl, VT)); in PerformCMOVToBFICombine()
12595 EVT VT = N->getValueType(0); in PerformBRCONDCombine() local
12619 ARMISD::BRCOND, dl, VT, Chain, BB, LHS->getOperand(0)->getOperand(2), in PerformBRCONDCombine()
12635 EVT VT = N->getValueType(0); in PerformCMOVCombine() local
12671 Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, TrueVal, ARMcc, in PerformCMOVCombine()
12676 Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, FalseVal, ARMcc, in PerformCMOVCombine()
12689 return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, in PerformCMOVCombine()
12695 if (!VT.isInteger()) in PerformCMOVCombine()
12705 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS); in PerformCMOVCombine()
12706 Res = DAG.getNode(ISD::SRL, dl, VT, DAG.getNode(ISD::CTLZ, dl, VT, Sub), in PerformCMOVCombine()
12718 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS); in PerformCMOVCombine()
12719 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in PerformCMOVCombine()
12733 DAG.getNode(ARMISD::SUBS, dl, DAG.getVTList(VT, MVT::i32), LHS, RHS); in PerformCMOVCombine()
12736 Res = DAG.getNode(ARMISD::CMOV, dl, VT, Sub, TrueVal, ARMcc, in PerformCMOVCombine()
12747 DAG.getNode(ARMISD::SUBS, dl, DAG.getVTList(VT, MVT::i32), LHS, RHS); in PerformCMOVCombine()
12750 Res = DAG.getNode(ARMISD::CMOV, dl, VT, Sub, FalseVal, in PerformCMOVCombine()
12770 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in PerformCMOVCombine()
12773 TrueVal = DAG.getConstant(1, dl, VT); in PerformCMOVCombine()
12778 ISD::SUB, dl, VT, DAG.getConstant(1, dl, MVT::i32), Res.getValue(1)); in PerformCMOVCombine()
12782 Res = DAG.getNode(ISD::SHL, dl, VT, Res, in PerformCMOVCombine()
12934 EVT VT) const { in isDesirableToTransformToIntegerOp()
12935 return (VT == MVT::f32) && (Opc == ISD::LOAD || Opc == ISD::STORE); in isDesirableToTransformToIntegerOp()
12938 bool ARMTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, in allowsMisalignedMemoryAccesses() argument
12943 if (!VT.isSimple()) in allowsMisalignedMemoryAccesses()
12949 switch (VT.getSimpleVT().SimpleTy) { in allowsMisalignedMemoryAccesses()
13052 bool ARMTargetLowering::isFNegFree(EVT VT) const { in isFNegFree()
13053 if (!VT.isSimple()) in isFNegFree()
13061 switch (VT.getSimpleVT().SimpleTy) { in isFNegFree()
13071 EVT VT = ExtVal.getValueType(); in isVectorLoadExtDesirable() local
13073 if (!isTypeLegal(VT)) in isVectorLoadExtDesirable()
13117 static bool isLegalT1AddressImmediate(int64_t V, EVT VT) { in isLegalT1AddressImmediate() argument
13122 switch (VT.getSimpleVT().SimpleTy) { in isLegalT1AddressImmediate()
13144 static bool isLegalT2AddressImmediate(int64_t V, EVT VT, in isLegalT2AddressImmediate() argument
13152 switch (VT.getSimpleVT().SimpleTy) { in isLegalT2AddressImmediate()
13177 static bool isLegalAddressImmediate(int64_t V, EVT VT, in isLegalAddressImmediate() argument
13182 if (!VT.isSimple()) in isLegalAddressImmediate()
13186 return isLegalT1AddressImmediate(V, VT); in isLegalAddressImmediate()
13188 return isLegalT2AddressImmediate(V, VT, Subtarget); in isLegalAddressImmediate()
13193 switch (VT.getSimpleVT().SimpleTy) { in isLegalAddressImmediate()
13215 EVT VT) const { in isLegalT2ScaledAddressingMode()
13220 switch (VT.getSimpleVT().SimpleTy) { in isLegalT2ScaledAddressingMode()
13253 EVT VT) const { in isLegalT1ScaledAddressingMode()
13272 EVT VT = getValueType(DL, Ty, true); in isLegalAddressingMode() local
13273 if (!isLegalAddressImmediate(AM.BaseOffs, VT, Subtarget)) in isLegalAddressingMode()
13288 if (!VT.isSimple()) in isLegalAddressingMode()
13292 return isLegalT1ScaledAddressingMode(AM, VT); in isLegalAddressingMode()
13295 return isLegalT2ScaledAddressingMode(AM, VT); in isLegalAddressingMode()
13298 switch (VT.getSimpleVT().SimpleTy) { in isLegalAddressingMode()
13362 static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT, in getARMIndexedAddressParts() argument
13369 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) { in getARMIndexedAddressParts()
13384 } else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) { in getARMIndexedAddressParts()
13421 static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT, in getT2IndexedAddressParts() argument
13457 EVT VT; in getPreIndexedAddressParts() local
13462 VT = LD->getMemoryVT(); in getPreIndexedAddressParts()
13466 VT = ST->getMemoryVT(); in getPreIndexedAddressParts()
13473 isLegal = getT2IndexedAddressParts(Ptr.getNode(), VT, isSEXTLoad, Base, in getPreIndexedAddressParts()
13476 isLegal = getARMIndexedAddressParts(Ptr.getNode(), VT, isSEXTLoad, Base, in getPreIndexedAddressParts()
13493 EVT VT; in getPostIndexedAddressParts() local
13497 VT = LD->getMemoryVT(); in getPostIndexedAddressParts()
13502 VT = ST->getMemoryVT(); in getPostIndexedAddressParts()
13527 isLegal = getT2IndexedAddressParts(Op, VT, isSEXTLoad, Base, Offset, in getPostIndexedAddressParts()
13530 isLegal = getARMIndexedAddressParts(Op, VT, isSEXTLoad, Base, Offset, in getPostIndexedAddressParts()
13594 EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT(); in computeKnownBitsForTargetNode() local
13595 unsigned MemBits = VT.getScalarSizeInBits(); in computeKnownBitsForTargetNode()
13627 EVT VT = Op.getValueType(); in computeKnownBitsForTargetNode() local
13628 const unsigned DstSz = VT.getScalarSizeInBits(); in computeKnownBitsForTargetNode()
13657 EVT VT = Op.getValueType(); in targetShrinkDemandedConstant() local
13660 if (VT.isVector()) in targetShrinkDemandedConstant()
13663 assert(VT == MVT::i32 && "Unexpected integer type"); in targetShrinkDemandedConstant()
13690 auto UseMask = [Mask, Op, VT, &TLO](unsigned NewMask) -> bool { in targetShrinkDemandedConstant()
13694 SDValue NewC = TLO.DAG.getConstant(NewMask, DL, VT); in targetShrinkDemandedConstant()
13695 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC); in targetShrinkDemandedConstant()
13847 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const { in getRegForInlineAsmConstraint()
13864 if (VT == MVT::Other) in getRegForInlineAsmConstraint()
13866 if (VT == MVT::f32) in getRegForInlineAsmConstraint()
13868 if (VT.getSizeInBits() == 64) in getRegForInlineAsmConstraint()
13870 if (VT.getSizeInBits() == 128) in getRegForInlineAsmConstraint()
13874 if (VT == MVT::Other) in getRegForInlineAsmConstraint()
13876 if (VT == MVT::f32) in getRegForInlineAsmConstraint()
13878 if (VT.getSizeInBits() == 64) in getRegForInlineAsmConstraint()
13880 if (VT.getSizeInBits() == 128) in getRegForInlineAsmConstraint()
13884 if (VT == MVT::Other) in getRegForInlineAsmConstraint()
13886 if (VT == MVT::f32 || VT == MVT::i32) in getRegForInlineAsmConstraint()
13888 if (VT.getSizeInBits() == 64) in getRegForInlineAsmConstraint()
13890 if (VT.getSizeInBits() == 128) in getRegForInlineAsmConstraint()
13898 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); in getRegForInlineAsmConstraint()
14118 EVT VT = Op->getValueType(0); in LowerDivRem() local
14119 Type *Ty = VT.getTypeForEVT(*DAG.getContext()); in LowerDivRem()
14134 SDValue Div = DAG.getNode(DivOpcode, dl, VT, Dividend, Divisor); in LowerDivRem()
14135 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Div, Divisor); in LowerDivRem()
14136 SDValue Rem = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul); in LowerDivRem()
14139 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(VT, VT), Values); in LowerDivRem()
14143 VT.getSimpleVT().SimpleTy); in LowerDivRem()
14295 bool ARMTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { in isFPImmLegal()
14298 if (VT == MVT::f16 && Subtarget->hasFullFP16()) in isFPImmLegal()
14300 if (VT == MVT::f32) in isFPImmLegal()
14302 if (VT == MVT::f64 && !Subtarget->isFPOnlySP()) in isFPImmLegal()
15043 } else if (auto *VT = dyn_cast<VectorType>(Ty)) { in isHomogeneousAggregate() local
15050 return VT->getBitWidth() == 64; in isHomogeneousAggregate()
15052 return VT->getBitWidth() == 128; in isHomogeneousAggregate()
15054 switch (VT->getBitWidth()) { in isHomogeneousAggregate()