Lines Matching refs:SrcVT
5783 bool X86TargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, in isExtractSubvectorCheap() argument
5791 return Index == 0 || ((ResVT.getSizeInBits() == SrcVT.getSizeInBits()*2) && in isExtractSubvectorCheap()
7434 EVT SrcVT = Op.getOperand(0).getValueType(); in getTargetConstantBitsFromNode() local
7435 unsigned NumSrcElts = SrcVT.getVectorNumElements(); in getTargetConstantBitsFromNode()
8403 EVT SrcVT = SrcVec.getValueType(); in getFauxShuffleMask() local
8404 if (!SrcVT.getScalarType().isByteSized()) in getFauxShuffleMask()
8407 unsigned SrcByte = SrcIdx * (SrcVT.getScalarSizeInBits() / 8); in getFauxShuffleMask()
8410 std::min<unsigned>(MinBitsPerElt, SrcVT.getScalarSizeInBits()); in getFauxShuffleMask()
8501 EVT SrcVT = Src.getValueType(); in getFauxShuffleMask() local
8503 if (!SrcVT.isSimple() || (SrcVT.getSizeInBits() % 128) != 0 || in getFauxShuffleMask()
8504 (SrcVT.getScalarSizeInBits() % 8) != 0) in getFauxShuffleMask()
8506 unsigned NumSrcElts = SrcVT.getVectorNumElements(); in getFauxShuffleMask()
8507 unsigned NumBitsPerSrcElt = SrcVT.getScalarSizeInBits(); in getFauxShuffleMask()
8582 EVT SrcVT = Src.getValueType(); in getFauxShuffleMask() local
8585 if (!SrcVT.isSimple() || (SrcVT.getSizeInBits() % 128) != 0 || in getFauxShuffleMask()
8586 (SrcVT.getScalarSizeInBits() % 8) != 0) in getFauxShuffleMask()
8591 DecodeZeroExtendMask(SrcVT.getScalarSizeInBits(), NumBitsPerElt, NumElts, in getFauxShuffleMask()
8787 EVT SrcVT = Src.getValueType(); in getShuffleScalarElt() local
8788 if (SrcVT.isVector() && SrcVT.getVectorNumElements() == NumElems) in getShuffleScalarElt()
10778 EVT SrcVT = Idx.getValueType(); in createVariablePermute() local
10779 unsigned NumDstBits = SrcVT.getScalarSizeInBits() / Scale; in createVariablePermute()
10793 Idx = DAG.getNode(ISD::MUL, SDLoc(Idx), SrcVT, Idx, in createVariablePermute()
10794 DAG.getConstant(IndexScale, SDLoc(Idx), SrcVT)); in createVariablePermute()
10795 Idx = DAG.getNode(ISD::ADD, SDLoc(Idx), SrcVT, Idx, in createVariablePermute()
10796 DAG.getConstant(IndexOffset, SDLoc(Idx), SrcVT)); in createVariablePermute()
12275 static bool matchShuffleAsVTRUNC(MVT &SrcVT, MVT &DstVT, MVT VT, in matchShuffleAsVTRUNC() argument
12295 SrcVT = MVT::getIntegerVT(EltSizeInBits * Scale); in matchShuffleAsVTRUNC()
12296 SrcVT = MVT::getVectorVT(SrcVT, NumSrcElts); in matchShuffleAsVTRUNC()
12316 MVT SrcVT = Src.getSimpleValueType(); in getAVX512TruncNode() local
12319 unsigned NumSrcElts = SrcVT.getVectorNumElements(); in getAVX512TruncNode()
12322 if (!DAG.getTargetLoweringInfo().isTypeLegal(SrcVT)) in getAVX512TruncNode()
12344 if (!Subtarget.hasVLX() && !SrcVT.is512BitVector()) { in getAVX512TruncNode()
12404 MVT SrcVT = Src.getSimpleValueType(); in lowerShuffleWithVPMOV() local
12405 if (SrcVT.getVectorElementType() == MVT::i16 && VT == MVT::v16i8 && in lowerShuffleWithVPMOV()
12458 MVT SrcVT = MVT::getVectorVT(SrcSVT, NumSrcElts); in lowerShuffleAsVTRUNC() local
12459 Src = DAG.getBitcast(SrcVT, Src); in lowerShuffleAsVTRUNC()
12542 static bool matchShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2, in matchShuffleWithPACK() argument
12570 SrcVT = PackVT; in matchShuffleWithPACK()
12583 SrcVT = PackVT; in matchShuffleWithPACK()
12646 MVT SrcVT = MVT::getVectorVT(SrcSVT, NumSrcElts); in lowerShuffleWithPACK() local
12648 Res = DAG.getNode(PackOpcode, DL, DstVT, DAG.getBitcast(SrcVT, V1), in lowerShuffleWithPACK()
12649 DAG.getBitcast(SrcVT, V2)); in lowerShuffleWithPACK()
20678 MVT SrcVT = Src.getSimpleValueType(); in LowerI64IntToFP_AVX512DQ() local
20681 if (!Subtarget.hasDQI() || SrcVT != MVT::i64 || Subtarget.is64Bit() || in LowerI64IntToFP_AVX512DQ()
20719 MVT SrcVT = Src.getSimpleValueType(); in LowerI64IntToFP16() local
20722 if (SrcVT != MVT::i64 || Subtarget.is64Bit() || VT != MVT::f16) in LowerI64IntToFP16()
20823 MVT SrcVT = X.getSimpleValueType(); in lowerFPToIntToFP() local
20824 if (SrcVT != MVT::f32 && SrcVT != MVT::f64) in lowerFPToIntToFP()
20833 unsigned SrcSize = SrcVT.getSizeInBits(); in lowerFPToIntToFP()
20836 MVT VecSrcVT = MVT::getVectorVT(SrcVT, 128 / SrcSize); in lowerFPToIntToFP()
20994 MVT SrcVT = Src.getSimpleValueType(); in LowerSINT_TO_FP() local
21000 else if (isLegalConversion(SrcVT, true, Subtarget)) in LowerSINT_TO_FP()
21003 if (Subtarget.isTargetWin64() && SrcVT == MVT::i128) in LowerSINT_TO_FP()
21012 if (SrcVT.isVector()) { in LowerSINT_TO_FP()
21013 if (SrcVT == MVT::v2i32 && VT == MVT::v2f64) { in LowerSINT_TO_FP()
21020 DAG.getUNDEF(SrcVT))}); in LowerSINT_TO_FP()
21023 DAG.getUNDEF(SrcVT))); in LowerSINT_TO_FP()
21025 if (SrcVT == MVT::v2i64 || SrcVT == MVT::v4i64) in LowerSINT_TO_FP()
21031 assert(SrcVT <= MVT::i64 && SrcVT >= MVT::i16 && in LowerSINT_TO_FP()
21038 if (SrcVT == MVT::i32 && UseSSEReg) in LowerSINT_TO_FP()
21040 if (SrcVT == MVT::i64 && UseSSEReg && Subtarget.is64Bit()) in LowerSINT_TO_FP()
21049 if (SrcVT == MVT::i16 && (UseSSEReg || VT == MVT::f128)) { in LowerSINT_TO_FP()
21062 if (SrcVT == MVT::i64 && Subtarget.hasSSE2() && !Subtarget.is64Bit()) in LowerSINT_TO_FP()
21068 unsigned Size = SrcVT.getStoreSize(); in LowerSINT_TO_FP()
21078 BuildFILD(VT, SrcVT, dl, Chain, StackSlot, MPI, Alignment, DAG); in LowerSINT_TO_FP()
21087 EVT DstVT, EVT SrcVT, const SDLoc &DL, SDValue Chain, SDValue Pointer, in BuildFILD() argument
21099 DAG.getMemIntrinsicNode(X86ISD::FILD, DL, Tys, FILDOps, SrcVT, PtrInfo, in BuildFILD()
21471 MVT SrcVT = N0.getSimpleValueType(); in lowerUINT_TO_FP_vec() local
21474 switch (SrcVT.SimpleTy) { in lowerUINT_TO_FP_vec()
21495 MVT SrcVT = Src.getSimpleValueType(); in LowerUINT_TO_FP() local
21505 else if (isLegalConversion(SrcVT, false, Subtarget)) in LowerUINT_TO_FP()
21511 if (Subtarget.isTargetWin64() && SrcVT == MVT::i128) in LowerUINT_TO_FP()
21518 (SrcVT == MVT::i32 || (SrcVT == MVT::i64 && Subtarget.is64Bit()))) { in LowerUINT_TO_FP()
21525 if (SrcVT == MVT::i32 && Subtarget.is64Bit()) { in LowerUINT_TO_FP()
21540 if (SrcVT == MVT::i64 && DstVT == MVT::f64 && Subtarget.hasSSE2() && in LowerUINT_TO_FP()
21545 if (SrcVT == MVT::i32 && Subtarget.hasSSE2() && DstVT != MVT::f80 && in LowerUINT_TO_FP()
21548 if (Subtarget.is64Bit() && SrcVT == MVT::i64 && in LowerUINT_TO_FP()
21558 if (SrcVT == MVT::i32) { in LowerUINT_TO_FP()
21572 assert(SrcVT == MVT::i64 && "Unexpected type in UINT_TO_FP"); in LowerUINT_TO_FP()
21954 EVT SrcVT = In.getValueType(); in truncateVectorWithPACK() local
21957 if (SrcVT == DstVT) in truncateVectorWithPACK()
21963 unsigned SrcSizeInBits = SrcVT.getSizeInBits(); in truncateVectorWithPACK()
21967 unsigned NumElems = SrcVT.getVectorNumElements(); in truncateVectorWithPACK()
21975 EVT PackedSVT = EVT::getIntegerVT(Ctx, SrcVT.getScalarSizeInBits() / 2); in truncateVectorWithPACK()
21980 if (SrcVT.getScalarSizeInBits() > 16 && in truncateVectorWithPACK()
21987 if (SrcVT.is128BitVector()) { in truncateVectorWithPACK()
22005 if (SrcVT.is256BitVector() && DstVT.is128BitVector()) { in truncateVectorWithPACK()
22014 if (SrcVT.is512BitVector() && Subtarget.hasInt256()) { in truncateVectorWithPACK()
22284 MVT SrcVT = Src.getSimpleValueType(); in expandFP_TO_UINT_SSE() local
22293 DAG.getNode(ISD::FSUB, dl, SrcVT, Src, in expandFP_TO_UINT_SSE()
22294 DAG.getConstantFP(2147483648.0f, dl, SrcVT))); in expandFP_TO_UINT_SSE()
22324 MVT SrcVT = Src.getSimpleValueType(); in LowerFP_TO_INT() local
22328 if (isSoftFP16(SrcVT)) { in LowerFP_TO_INT()
22336 } else if (isTypeLegal(SrcVT) && isLegalConversion(VT, IsSigned, Subtarget)) { in LowerFP_TO_INT()
22341 if (VT == MVT::v2i1 && SrcVT == MVT::v2f64) { in LowerFP_TO_INT()
22379 if (Subtarget.hasFP16() && SrcVT.getVectorElementType() == MVT::f16) { in LowerFP_TO_INT()
22388 if (SrcVT != MVT::v8f16) { in LowerFP_TO_INT()
22390 IsStrict ? DAG.getConstantFP(0.0, dl, SrcVT) : DAG.getUNDEF(SrcVT); in LowerFP_TO_INT()
22391 SmallVector<SDValue, 4> Ops(SrcVT == MVT::v2f16 ? 4 : 2, Tmp); in LowerFP_TO_INT()
22421 if (VT == MVT::v8i16 && (SrcVT == MVT::v8f32 || SrcVT == MVT::v8f64)) { in LowerFP_TO_INT()
22441 if (VT == MVT::v8i32 && SrcVT == MVT::v8f64) { in LowerFP_TO_INT()
22449 (SrcVT == MVT::v4f64 || SrcVT == MVT::v4f32 || SrcVT == MVT::v8f32) && in LowerFP_TO_INT()
22453 MVT WideVT = SrcVT == MVT::v4f64 ? MVT::v8f64 : MVT::v16f32; in LowerFP_TO_INT()
22454 MVT ResVT = SrcVT == MVT::v4f64 ? MVT::v8i32 : MVT::v16i32; in LowerFP_TO_INT()
22481 (SrcVT == MVT::v2f64 || SrcVT == MVT::v4f64 || SrcVT == MVT::v4f32) && in LowerFP_TO_INT()
22484 MVT WideVT = SrcVT == MVT::v4f32 ? MVT::v8f32 : MVT::v8f64; in LowerFP_TO_INT()
22509 if (VT == MVT::v2i64 && SrcVT == MVT::v2f32) { in LowerFP_TO_INT()
22541 if ((VT == MVT::v4i32 && SrcVT == MVT::v4f32) || in LowerFP_TO_INT()
22542 (VT == MVT::v4i32 && SrcVT == MVT::v4f64) || in LowerFP_TO_INT()
22543 (VT == MVT::v8i32 && SrcVT == MVT::v8f32)) { in LowerFP_TO_INT()
22553 bool UseSSEReg = isScalarFPTypeInSSEReg(SrcVT); in LowerFP_TO_INT()
22566 SDValue FloatOffset = DAG.getNode(ISD::UINT_TO_FP, dl, SrcVT, in LowerFP_TO_INT()
22568 MVT SrcVecVT = MVT::getVectorVT(SrcVT, 128 / SrcVT.getScalarSizeInBits()); in LowerFP_TO_INT()
22579 DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FloatOffset))); in LowerFP_TO_INT()
22625 if (VT == MVT::i16 && (UseSSEReg || SrcVT == MVT::f128)) { in LowerFP_TO_INT()
22645 if (SrcVT == MVT::f128) { in LowerFP_TO_INT()
22648 LC = RTLIB::getFPTOSINT(SrcVT, VT); in LowerFP_TO_INT()
22650 LC = RTLIB::getFPTOUINT(SrcVT, VT); in LowerFP_TO_INT()
22675 MVT SrcVT = Src.getSimpleValueType(); in LowerLRINT_LLRINT() local
22677 if (SrcVT == MVT::f16) in LowerLRINT_LLRINT()
22681 if (isScalarFPTypeInSSEReg(SrcVT)) in LowerLRINT_LLRINT()
22691 EVT SrcVT = Src.getValueType(); in LRINT_LLRINTHelper() local
22693 if (SrcVT != MVT::f32 && SrcVT != MVT::f64 && SrcVT != MVT::f80) { in LRINT_LLRINTHelper()
22702 bool UseSSE = isScalarFPTypeInSSEReg(SrcVT); in LRINT_LLRINTHelper()
22706 EVT OtherVT = UseSSE ? SrcVT : DstVT; in LRINT_LLRINTHelper()
22718 Src = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, SrcVT, MPI, in LRINT_LLRINTHelper()
22745 EVT SrcVT = Src.getValueType(); in LowerFP_TO_INT_SAT() local
22751 if (!isScalarFPTypeInSSEReg(SrcVT) || isSoftFP16(SrcVT)) in LowerFP_TO_INT_SAT()
22790 APFloat MinFloat(DAG.EVTToAPFloatSemantics(SrcVT)); in LowerFP_TO_INT_SAT()
22791 APFloat MaxFloat(DAG.EVTToAPFloatSemantics(SrcVT)); in LowerFP_TO_INT_SAT()
22800 SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT); in LowerFP_TO_INT_SAT()
22801 SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT); in LowerFP_TO_INT_SAT()
22809 X86ISD::FMAX, dl, SrcVT, MinFloatNode, Src); in LowerFP_TO_INT_SAT()
22812 X86ISD::FMIN, dl, SrcVT, MaxFloatNode, MinClamped); in LowerFP_TO_INT_SAT()
22823 X86ISD::FMAX, dl, SrcVT, Src, MinFloatNode); in LowerFP_TO_INT_SAT()
22826 X86ISD::FMINC, dl, SrcVT, MinClamped, MaxFloatNode); in LowerFP_TO_INT_SAT()
23472 EVT SrcVT = Src.getValueType(); in LowerVectorAllZero() local
23473 SDValue MaskValue = DAG.getConstant(Mask, DL, SrcVT); in LowerVectorAllZero()
23474 return DAG.getNode(ISD::AND, DL, SrcVT, Src, MaskValue); in LowerVectorAllZero()
27105 MVT SrcVT = Src.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN() local
27106 MVT MaskVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorNumElements()); in LowerINTRINSIC_WO_CHAIN()
27120 MVT SrcVT = Src.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN() local
27121 MVT MaskVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorNumElements()); in LowerINTRINSIC_WO_CHAIN()
31228 MVT SrcVT = Src.getSimpleValueType(); in LowerBITCAST() local
31233 if (SrcVT == MVT::i64 && DstVT == MVT::v64i1) { in LowerBITCAST()
31247 if ((SrcVT == MVT::v16i1 || SrcVT == MVT::v32i1) && DstVT.isScalarInteger()) { in LowerBITCAST()
31249 MVT SExtVT = SrcVT == MVT::v16i1 ? MVT::v16i8 : MVT::v32i8; in LowerBITCAST()
31256 assert((SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8 || in LowerBITCAST()
31257 SrcVT == MVT::i64) && "Unexpected VT!"); in LowerBITCAST()
31260 if (!(DstVT == MVT::f64 && SrcVT == MVT::i64) && in LowerBITCAST()
31261 !(DstVT == MVT::x86mmx && SrcVT.isVector())) in LowerBITCAST()
31266 if (SrcVT.isVector()) { in LowerBITCAST()
31269 MVT NewVT = MVT::getVectorVT(SrcVT.getVectorElementType(), in LowerBITCAST()
31270 SrcVT.getVectorNumElements() * 2); in LowerBITCAST()
31272 DAG.getUNDEF(SrcVT)); in LowerBITCAST()
31274 assert(SrcVT == MVT::i64 && !Subtarget.is64Bit() && in LowerBITCAST()
32709 EVT SrcVT = Src.getValueType(); in ReplaceNodeResults() local
32712 if (isSoftFP16(SrcVT)) { in ReplaceNodeResults()
32732 SrcVT.getVectorElementType() == MVT::f16) { in ReplaceNodeResults()
32736 if (SrcVT != MVT::v8f16) { in ReplaceNodeResults()
32738 IsStrict ? DAG.getConstantFP(0.0, dl, SrcVT) : DAG.getUNDEF(SrcVT); in ReplaceNodeResults()
32739 SmallVector<SDValue, 4> Ops(SrcVT == MVT::v2f16 ? 4 : 2, Tmp); in ReplaceNodeResults()
32891 (SrcVT == MVT::f32 || SrcVT == MVT::f64)) || in ReplaceNodeResults()
32892 (Subtarget.hasFP16() && SrcVT == MVT::f16)) { in ReplaceNodeResults()
32897 std::max(NumElts, 128U / (unsigned)SrcVT.getSizeInBits()); in ReplaceNodeResults()
32899 MVT VecInVT = MVT::getVectorVT(SrcVT.getSimpleVT(), SrcElts); in ReplaceNodeResults()
32982 EVT SrcVT = Src.getValueType(); in ReplaceNodeResults() local
32983 if (Subtarget.hasDQI() && Subtarget.hasVLX() && SrcVT == MVT::v2i64) { in ReplaceNodeResults()
32997 if (SrcVT == MVT::v2i64 && !IsSigned && Subtarget.is64Bit() && in ReplaceNodeResults()
32999 SDValue Zero = DAG.getConstant(0, dl, SrcVT); in ReplaceNodeResults()
33000 SDValue One = DAG.getConstant(1, dl, SrcVT); in ReplaceNodeResults()
33001 SDValue Sign = DAG.getNode(ISD::OR, dl, SrcVT, in ReplaceNodeResults()
33002 DAG.getNode(ISD::SRL, dl, SrcVT, Src, One), in ReplaceNodeResults()
33003 DAG.getNode(ISD::AND, dl, SrcVT, Src, One)); in ReplaceNodeResults()
33005 SDValue SignSrc = DAG.getSelect(dl, SrcVT, IsNeg, Sign, Src); in ReplaceNodeResults()
33038 if (SrcVT != MVT::v2i32) in ReplaceNodeResults()
33084 EVT SrcVT = Src.getValueType(); in ReplaceNodeResults() local
33094 if (SrcVT.getVectorElementType() != MVT::f32) in ReplaceNodeResults()
33327 EVT SrcVT = N->getOperand(0).getValueType(); in ReplaceNodeResults() local
33331 if (SrcVT == MVT::v64i1 && DstVT == MVT::i64 && Subtarget.hasBWI()) { in ReplaceNodeResults()
33342 if (DstVT.isVector() && SrcVT == MVT::x86mmx) { in ReplaceNodeResults()
34096 EVT SrcVT = ExtVal.getOperand(0).getValueType(); in isVectorLoadExtDesirable() local
34099 if (SrcVT.getScalarType() == MVT::i1) in isVectorLoadExtDesirable()
36788 EVT SrcVT = Src.getValueType(); in computeKnownBitsForTargetNode() local
36789 APInt DemandedElt = APInt::getOneBitSet(SrcVT.getVectorNumElements(), in computeKnownBitsForTargetNode()
36793 Known.Zero.setBitsFrom(SrcVT.getScalarSizeInBits()); in computeKnownBitsForTargetNode()
36972 EVT SrcVT = Op.getOperand(0).getValueType(); in computeKnownBitsForTargetNode() local
36973 if (SrcVT.isVector()) { in computeKnownBitsForTargetNode()
36974 unsigned NumSrcElts = SrcVT.getVectorNumElements(); in computeKnownBitsForTargetNode()
36988 EVT SrcVT = Op.getOperand(1).getValueType(); in computeKnownBitsForTargetNode() local
36989 if (SrcVT.isVector()) { in computeKnownBitsForTargetNode()
36990 unsigned NumSrcElts = SrcVT.getVectorNumElements(); in computeKnownBitsForTargetNode()
37090 MVT SrcVT = Src.getSimpleValueType(); in ComputeNumSignBitsForTargetNode() local
37091 unsigned NumSrcBits = SrcVT.getScalarSizeInBits(); in ComputeNumSignBitsForTargetNode()
37093 APInt DemandedSrc = DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements()); in ComputeNumSignBitsForTargetNode()
37256 MVT &SrcVT, MVT &DstVT) { in matchUnaryShuffle() argument
37268 SrcVT = DstVT = MaskVT.changeVectorElementType(MVT::f16); in matchUnaryShuffle()
37270 SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT; in matchUnaryShuffle()
37297 SrcVT = MVT::getVectorVT(ScalarTy, SrcSize / MaskEltSize); in matchUnaryShuffle()
37300 if (SrcVT.getVectorNumElements() != NumDstElts) in matchUnaryShuffle()
37317 SrcVT = DstVT = MaskVT.changeVectorElementType(MVT::f16); in matchUnaryShuffle()
37319 SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT; in matchUnaryShuffle()
37329 SrcVT = DstVT = MVT::v2f64; in matchUnaryShuffle()
37334 SrcVT = DstVT = MVT::v4f32; in matchUnaryShuffle()
37339 SrcVT = DstVT = MVT::v4f32; in matchUnaryShuffle()
37348 SrcVT = DstVT = MVT::v4f64; in matchUnaryShuffle()
37354 SrcVT = DstVT = MVT::v8f32; in matchUnaryShuffle()
37360 SrcVT = DstVT = MVT::v8f32; in matchUnaryShuffle()
37371 SrcVT = DstVT = MVT::v8f64; in matchUnaryShuffle()
37378 SrcVT = DstVT = MVT::v16f32; in matchUnaryShuffle()
37385 SrcVT = DstVT = MVT::v16f32; in matchUnaryShuffle()
37537 unsigned &Shuffle, MVT &SrcVT, MVT &DstVT, in matchBinaryShuffle() argument
37549 SrcVT = DstVT = Subtarget.hasSSE2() ? MVT::v2f64 : MVT::v4f32; in matchBinaryShuffle()
37556 SrcVT = DstVT = Subtarget.hasSSE2() ? MVT::v2f64 : MVT::v4f32; in matchBinaryShuffle()
37563 SrcVT = DstVT = MVT::v2f64; in matchBinaryShuffle()
37569 SrcVT = DstVT = MVT::v4f32; in matchBinaryShuffle()
37576 SrcVT = DstVT = MVT::v8f16; in matchBinaryShuffle()
37585 if (matchShuffleWithPACK(MaskVT, SrcVT, V1, V2, Shuffle, Mask, DAG, in matchBinaryShuffle()
37600 SrcVT = DstVT = MaskVT; in matchBinaryShuffle()
37602 SrcVT = DstVT = (32 == EltSizeInBits ? MVT::v8f32 : MVT::v4f64); in matchBinaryShuffle()
37645 SrcVT = DstVT = MaskVT.changeTypeToInteger(); in matchBinaryShuffle()
37689 SrcVT = DstVT = MaskVT.changeTypeToInteger(); in matchBinaryShuffle()
38764 MVT SrcVT = BC0.getOperand(0).getSimpleValueType(); in canonicalizeShuffleMaskWithHorizOp() local
38794 SDValue LHS = DAG.getNode(Opcode0, DL, SrcVT, M0, M1); in canonicalizeShuffleMaskWithHorizOp()
38795 SDValue RHS = DAG.getNode(Opcode0, DL, SrcVT, M2, M3); in canonicalizeShuffleMaskWithHorizOp()
38824 LHS = DAG.getBitcast(SrcVT, LHS); in canonicalizeShuffleMaskWithHorizOp()
38825 RHS = DAG.getBitcast(SrcVT, RHS ? RHS : LHS); in canonicalizeShuffleMaskWithHorizOp()
38901 SDValue Undef = DAG.getUNDEF(SrcVT); in canonicalizeShuffleMaskWithHorizOp()
38902 SDValue Zero = getZeroVector(SrcVT, Subtarget, DAG, DL); in canonicalizeShuffleMaskWithHorizOp()
39880 EVT SrcVT = Src.getValueType(); in combineTargetShuffle() local
39898 DAG.getBitcast(SrcVT, Res)); in combineTargetShuffle()
39904 SrcVT.getScalarSizeInBits() == BCVT.getScalarSizeInBits() && in combineTargetShuffle()
39914 if (SrcVT.getSizeInBits() > 128) in combineTargetShuffle()
39942 if (!SrcVT.isVector() && (Src.hasOneUse() || VT.isFloatingPoint()) && in combineTargetShuffle()
39957 SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT, BcastLd, in combineTargetShuffle()
39966 if (SrcVT == MVT::i16 && Src.getOpcode() == ISD::TRUNCATE && in combineTargetShuffle()
40054 if ((SrcVT == MVT::v2f64 || SrcVT == MVT::v4f32 || SrcVT == MVT::v2i64 || in combineTargetShuffle()
40055 SrcVT == MVT::v4i32) && in combineTargetShuffle()
40063 X86ISD::VBROADCAST_LOAD, DL, Tys, Ops, SrcVT.getScalarType(), in combineTargetShuffle()
40181 MVT SrcVT = N0.getOperand(0).getSimpleValueType(); in combineTargetShuffle() local
40182 if ((VT.getScalarSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 && in combineTargetShuffle()
40183 SrcVT.getScalarSizeInBits() >= 32) { in combineTargetShuffle()
40186 unsigned Scale = VT.getScalarSizeInBits() / SrcVT.getScalarSizeInBits(); in combineTargetShuffle()
40189 VT, DAG.getNode(X86ISD::BLENDI, DL, SrcVT, N0.getOperand(0), in combineTargetShuffle()
40240 EVT SrcVT = Src.getValueType(); in combineTargetShuffle() local
40241 SDValue Res = DAG.getNode(X86ISD::VPERMI, DL, SrcVT, Src, N1); in combineTargetShuffle()
40252 EVT SrcVT = LHS.getOperand(0).getValueType(); in combineTargetShuffle() local
40253 if (RHS.isUndef() || SrcVT == RHS.getOperand(0).getValueType()) { in combineTargetShuffle()
40254 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT, in combineTargetShuffle()
40255 DAG.getBitcast(SrcVT, LHS), in combineTargetShuffle()
40256 DAG.getBitcast(SrcVT, RHS), in combineTargetShuffle()
41191 MVT SrcVT = Src.getSimpleValueType(); in SimplifyDemandedVectorEltsForTargetNode() local
41193 APInt SrcElts = DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements()); in SimplifyDemandedVectorEltsForTargetNode()
41275 MVT SrcVT = Src.getSimpleValueType(); in SimplifyDemandedVectorEltsForTargetNode() local
41276 APInt DemandedSrc = DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements()); in SimplifyDemandedVectorEltsForTargetNode()
41317 MVT SrcVT = Src.getSimpleValueType(); in SimplifyDemandedVectorEltsForTargetNode() local
41318 if (!SrcVT.isVector()) in SimplifyDemandedVectorEltsForTargetNode()
41328 APInt SrcElts = APInt::getOneBitSet(SrcVT.getVectorNumElements(), 0); in SimplifyDemandedVectorEltsForTargetNode()
41503 EVT SrcVT = SrcOp.getValueType(); in SimplifyDemandedVectorEltsForTargetNode() local
41504 assert((!SrcVT.isVector() || SrcVT.getSizeInBits() == SizeInBits) && in SimplifyDemandedVectorEltsForTargetNode()
41506 Ops.push_back(SrcVT.isVector() ? extractSubVector(SrcOp, 0, TLO.DAG, DL, in SimplifyDemandedVectorEltsForTargetNode()
41626 MVT SrcVT = Src.getSimpleValueType(); in SimplifyDemandedBitsForTargetNode() local
41629 APInt TruncMask = OriginalDemandedBits.zext(SrcVT.getScalarSizeInBits()); in SimplifyDemandedBitsForTargetNode()
41630 APInt DemandedElts = OriginalDemandedElts.trunc(SrcVT.getVectorNumElements()); in SimplifyDemandedBitsForTargetNode()
41913 MVT SrcVT = Src.getSimpleValueType(); in SimplifyDemandedBitsForTargetNode() local
41915 SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1, 0); in SimplifyDemandedBitsForTargetNode()
41922 if ((BitWidth == 64) && SrcVT.isScalarInteger() && !Subtarget.hasAVX512() && in SimplifyDemandedBitsForTargetNode()
41944 MVT SrcVT = Src.getSimpleValueType(); in SimplifyDemandedBitsForTargetNode() local
41945 unsigned SrcBits = SrcVT.getScalarSizeInBits(); in SimplifyDemandedBitsForTargetNode()
41946 unsigned NumElts = SrcVT.getVectorNumElements(); in SimplifyDemandedBitsForTargetNode()
41953 if (SrcVT.is256BitVector() && in SimplifyDemandedBitsForTargetNode()
42215 EVT SrcVT = Src.getValueType(); in adjustBitcastSrcVectorSSE1() local
42216 if (SrcVT != MVT::v4i1) in adjustBitcastSrcVectorSSE1()
42273 EVT SrcVT = Src.getValueType(); in combineBitcastvxi1() local
42274 if (!SrcVT.isSimple() || SrcVT.getScalarType() != MVT::i1) in combineBitcastvxi1()
42325 switch (SrcVT.getSimpleVT().SimpleTy) { in combineBitcastvxi1()
42394 EVT::getIntegerVT(*DAG.getContext(), SrcVT.getVectorNumElements()); in combineBitcastvxi1()
42401 EVT SrcVT = Op.getValueType(); in combinevXi1ConstantToInteger() local
42402 assert(SrcVT.getVectorElementType() == MVT::i1 && in combinevXi1ConstantToInteger()
42407 APInt Imm(SrcVT.getVectorNumElements(), 0); in combinevXi1ConstantToInteger()
42431 EVT SrcVT = Op.getValueType(); in combineCastedMaskArithmetic() local
42443 if (!(SrcVT.isVector() && SrcVT.getVectorElementType() == MVT::i1 && in combineCastedMaskArithmetic()
42446 SrcVT.isScalarInteger())) in combineCastedMaskArithmetic()
42562 EVT SrcVT = Src.getValueType(); in combineBitcastToBoolVector() local
42563 if (SrcVT.isVector() || SrcVT.isFloatingPoint()) in combineBitcastToBoolVector()
42626 EVT SrcVT = N0.getValueType(); in combineBitcast() local
42642 if ((VT == MVT::v4i1 || VT == MVT::v2i1) && SrcVT.isScalarInteger() && in combineBitcast()
42652 if ((SrcVT == MVT::v4i1 || SrcVT == MVT::v2i1) && VT.isScalarInteger() && in combineBitcast()
42664 SrcVT = LastOp.getValueType(); in combineBitcast()
42665 unsigned NumConcats = 8 / SrcVT.getVectorNumElements(); in combineBitcast()
42667 Ops.resize(NumConcats, DAG.getConstant(0, dl, SrcVT)); in combineBitcast()
42674 unsigned NumConcats = 8 / SrcVT.getVectorNumElements(); in combineBitcast()
42675 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(SrcVT)); in combineBitcast()
42685 SrcVT.isScalarInteger() && TLI.isTypeLegal(VT)) { in combineBitcast()
42697 if (VT == MVT::i8 && SrcVT == MVT::v8i1 && Subtarget.hasAVX512() && in combineBitcast()
42708 VT.isFloatingPoint() != SrcVT.isFloatingPoint() && VT.isVector()) { in combineBitcast()
42710 unsigned SrcVTSize = SrcVT.getScalarSizeInBits(); in combineBitcast()
42718 LoadVT = MVT::getVectorVT(LoadVT, SrcVT.getVectorNumElements()); in combineBitcast()
42751 (SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8) && in combineBitcast()
42752 N0.getOperand(0).getValueType() == SrcVT.getScalarType()) { in combineBitcast()
42754 for (unsigned i = 1, e = SrcVT.getVectorNumElements(); i != e; ++i) { in combineBitcast()
42772 (SrcVT == MVT::v2f32 || SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || in combineBitcast()
42773 SrcVT == MVT::v8i8)) in combineBitcast()
42787 if (SrcVT == MVT::v2i32 && N0.getOpcode() == ISD::FP_TO_SINT) { in combineBitcast()
42799 SrcVT.isVector() && SrcVT.getVectorElementType() == MVT::i1 && in combineBitcast()
42804 if (Subtarget.hasAVX512() && SrcVT.isScalarInteger() && in combineBitcast()
42817 if (Subtarget.hasAVX512() && SrcVT.isScalarInteger() && in combineBitcast()
43052 EVT SrcVT = Src.getValueType(); in combineMinMaxReduction() local
43053 EVT SrcSVT = SrcVT.getScalarType(); in combineMinMaxReduction()
43054 if (SrcSVT != ExtractVT || (SrcVT.getSizeInBits() % 128) != 0) in combineMinMaxReduction()
43061 while (SrcVT.getSizeInBits() > 128) { in combineMinMaxReduction()
43064 SrcVT = Lo.getValueType(); in combineMinMaxReduction()
43065 MinPos = DAG.getNode(BinOp, DL, SrcVT, Lo, Hi); in combineMinMaxReduction()
43067 assert(((SrcVT == MVT::v8i16 && ExtractVT == MVT::i16) || in combineMinMaxReduction()
43068 (SrcVT == MVT::v16i8 && ExtractVT == MVT::i8)) && in combineMinMaxReduction()
43076 Mask = DAG.getConstant(APInt::getSignedMaxValue(MaskEltsBits), DL, SrcVT); in combineMinMaxReduction()
43078 Mask = DAG.getConstant(APInt::getSignedMinValue(MaskEltsBits), DL, SrcVT); in combineMinMaxReduction()
43080 Mask = DAG.getAllOnesConstant(DL, SrcVT); in combineMinMaxReduction()
43083 MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos); in combineMinMaxReduction()
43091 SrcVT, DL, MinPos, DAG.getConstant(0, DL, MVT::v16i8), in combineMinMaxReduction()
43093 MinPos = DAG.getNode(ISD::UMIN, DL, SrcVT, MinPos, Upper); in combineMinMaxReduction()
43099 MinPos = DAG.getBitcast(SrcVT, MinPos); in combineMinMaxReduction()
43102 MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos); in combineMinMaxReduction()
43406 EVT SrcVT = Src.getValueType(); in combineExtractWithShuffle() local
43407 EVT SrcSVT = SrcVT.getVectorElementType(); in combineExtractWithShuffle()
43409 unsigned NumSrcElts = SrcVT.getVectorNumElements(); in combineExtractWithShuffle()
43431 SrcOp = DAG.getZExtOrTrunc(SrcOp, dl, SrcVT.getScalarType()); in combineExtractWithShuffle()
43464 unsigned Offset = IdxC.getZExtValue() * SrcVT.getScalarSizeInBits(); in combineExtractWithShuffle()
43471 Scl = DAG.getZExtOrTrunc(Scl, dl, SrcVT.getScalarType()); in combineExtractWithShuffle()
43481 (SrcVT.getSizeInBits() % 128) == 0) { in combineExtractWithShuffle()
43527 if (llvm::any_of(Ops, [SrcVT](SDValue Op) { in combineExtractWithShuffle()
43528 return SrcVT.getSizeInBits() != Op.getValueSizeInBits(); in combineExtractWithShuffle()
43561 ExtractVT = SrcVT; in combineExtractWithShuffle()
43564 if ((Mask.size() % NumSrcElts) != 0 || SrcVT.isFloatingPoint()) in combineExtractWithShuffle()
43572 assert(SrcVT.getSizeInBits() == ExtractVT.getSizeInBits() && in combineExtractWithShuffle()
43892 EVT SrcVT = InputVector.getValueType(); in combineExtractVectorElt() local
43896 unsigned NumSrcElts = SrcVT.getVectorNumElements(); in combineExtractVectorElt()
43905 unsigned VecEltBitWidth = SrcVT.getScalarSizeInBits(); in combineExtractVectorElt()
43926 assert(SrcVT == InputVector.getOperand(0).getValueType() && in combineExtractVectorElt()
43929 Scl = DAG.getNode(ISD::TRUNCATE, dl, SrcVT.getScalarType(), Scl); in combineExtractVectorElt()
43941 VT == MVT::i64 && SrcVT == MVT::v1i64 && isNullConstant(EltIdx)) { in combineExtractVectorElt()
43951 VT == MVT::i32 && SrcVT == MVT::v2i32 && isNullConstant(EltIdx)) { in combineExtractVectorElt()
43993 if (SrcVT.getScalarType() == MVT::i1) { in combineExtractVectorElt()
44042 SrcVT.getVectorElementType() == VT && DCI.isAfterLegalizeDAG() && in combineExtractVectorElt()
44046 TLI.getVectorElementPointer(DAG, LoadVec->getBasePtr(), SrcVT, EltIdx); in combineExtractVectorElt()
44805 EVT SrcVT = SelectableLHS ? LHS.getOperand(0).getValueType() in combineSelect() local
44807 EVT SrcCondVT = SrcVT.changeVectorElementType(MVT::i1); in combineSelect()
44808 LHS = insertSubVector(DAG.getUNDEF(SrcVT), LHS, 0, DAG, DL, in combineSelect()
44810 RHS = insertSubVector(DAG.getUNDEF(SrcVT), RHS, 0, DAG, DL, in combineSelect()
44815 SDValue Res = DAG.getSelect(DL, SrcVT, Cond, LHS, RHS); in combineSelect()
46739 EVT SrcVT = N0.getValueType(); in combineHorizOpWithShuffle() local
46751 if (VT.is128BitVector() && SrcVT.getScalarSizeInBits() <= 32) { in combineHorizOpWithShuffle()
46766 Lo = DAG.getBitcast(SrcVT, Lo); in combineHorizOpWithShuffle()
46767 Hi = DAG.getBitcast(SrcVT, Hi); in combineHorizOpWithShuffle()
46778 if (VT.is128BitVector() && SrcVT.getScalarSizeInBits() <= 32) { in combineHorizOpWithShuffle()
46823 LHS = DAG.getBitcast(SrcVT, LHS); in combineHorizOpWithShuffle()
46824 RHS = DAG.getBitcast(SrcVT, RHS ? RHS : LHS); in combineHorizOpWithShuffle()
46862 SDValue Res = DAG.getNode(Opcode, DL, VT, DAG.getBitcast(SrcVT, Op00), in combineHorizOpWithShuffle()
46863 DAG.getBitcast(SrcVT, Op01)); in combineHorizOpWithShuffle()
47343 EVT SrcVT = Src.getValueType(); in combineAndNotIntoANDNP() local
47344 if (!SrcVT.isVector()) in combineAndNotIntoANDNP()
47348 DAG.getBitcast(SrcVT, Not)); in combineAndNotIntoANDNP()
47822 EVT SrcVT = Src.getValueType(); in combineScalarAndWithMaskSetcc() local
47823 if (!SrcVT.isVector() || SrcVT.getVectorElementType() != MVT::i1 || in combineScalarAndWithMaskSetcc()
47824 !TLI.isTypeLegal(SrcVT)) in combineScalarAndWithMaskSetcc()
47855 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT, in combineScalarAndWithMaskSetcc()
49507 MVT SrcVT = Src.getSimpleValueType(); in combineStore() local
49508 unsigned NumSrcElts = SrcVT.getVectorNumElements(); in combineStore()
49512 TLI.isTruncStoreLegal(SrcVT, TruncVT)) { in combineStore()
50082 EVT SrcVT = Src.getValueType(); in combineTruncatedArithmetic() local
50125 if (SrcVT.getScalarType() == MVT::i64 && in combineTruncatedArithmetic()
50127 !TLI.isOperationLegal(SrcOpcode, SrcVT)) in combineTruncatedArithmetic()
52140 EVT SrcVT = LHS.getOperand(0).getValueType(); in combineSetCC() local
52141 APInt UpperBits = APInt::getBitsSetFrom(SrcVT.getScalarSizeInBits(), in combineSetCC()
52147 DAG.getConstant(0, DL, SrcVT), CC); in combineSetCC()
52222 MVT SrcVT = Src.getSimpleValueType(); in combineMOVMSK() local
52225 unsigned NumElts = SrcVT.getVectorNumElements(); in combineMOVMSK()
52226 unsigned NumBitsPerElt = SrcVT.getScalarSizeInBits(); in combineMOVMSK()
52242 unsigned EltWidth = SrcVT.getScalarSizeInBits(); in combineMOVMSK()
52252 NotSrc = DAG.getBitcast(SrcVT, NotSrc); in combineMOVMSK()
52279 MVT ShiftVT = SrcVT; in combineMOVMSK()
52288 LHS = DAG.getNOT(DL, DAG.getBitcast(SrcVT, LHS), SrcVT); in combineMOVMSK()
52594 EVT SrcVT = ExtElt.getValueType(); in combineToFPTruncExtElt() local
52596 unsigned SrcWidth = SrcVT.getSizeInBits(); in combineToFPTruncExtElt()
53089 EVT SrcVT = Src.getValueType(); in combineCMP() local
53090 if (SrcVT.getScalarSizeInBits() >= 8 && in combineCMP()
53091 DAG.getTargetLoweringInfo().isTypeLegal(SrcVT)) in combineCMP()
53093 DAG.getConstant(0, dl, SrcVT)); in combineCMP()
54076 MVT SrcVT = Op0.getOperand(0).getSimpleValueType(); in combineConcatVectorOps() local
54077 SrcVT = MVT::getVectorVT(SrcVT.getScalarType(), in combineConcatVectorOps()
54078 NumOps * SrcVT.getVectorNumElements()); in combineConcatVectorOps()
54080 ConcatSubOperand(SrcVT, Ops, 0), in combineConcatVectorOps()
54081 ConcatSubOperand(SrcVT, Ops, 1)); in combineConcatVectorOps()
54092 MVT SrcVT = Op0.getOperand(0).getSimpleValueType(); in combineConcatVectorOps() local
54093 SrcVT = MVT::getVectorVT(SrcVT.getScalarType(), in combineConcatVectorOps()
54094 NumOps * SrcVT.getVectorNumElements()); in combineConcatVectorOps()
54096 ConcatSubOperand(SrcVT, Ops, 0), in combineConcatVectorOps()
54097 ConcatSubOperand(SrcVT, Ops, 1)); in combineConcatVectorOps()
54166 EVT SrcVT = N->getOperand(0).getValueType(); in combineCONCAT_VECTORS() local
54173 if (Subtarget.hasAVX() && TLI.isTypeLegal(VT) && TLI.isTypeLegal(SrcVT)) { in combineCONCAT_VECTORS()
54837 EVT SrcVT = Src.getValueType(); in combineFP_EXTEND() local
54839 if (!SrcVT.isVector() || SrcVT.getVectorElementType() != MVT::f16) in combineFP_EXTEND()
54853 EVT IntVT = SrcVT.changeVectorElementTypeToInteger(); in combineFP_EXTEND()
54948 EVT SrcVT = Src.getValueType(); in combineFP_ROUND() local
54951 SrcVT.getVectorElementType() != MVT::f32) in combineFP_ROUND()
54963 DAG.getConstantFP(0.0, dl, SrcVT)); in combineFP_ROUND()