Lines Matching refs:MaskVT
7466 MVT MaskVT = BOperand.getSimpleValueType(); in lowerBuildVectorAsBroadcast() local
7467 if ((EltType == MVT::i64 && MaskVT == MVT::v8i1) || // for broadcastmb2q in lowerBuildVectorAsBroadcast()
7468 (EltType == MVT::i32 && MaskVT == MVT::v16i1)) { // for broadcastmw2d in lowerBuildVectorAsBroadcast()
9883 static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
16214 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts); in LowerVSELECT() local
16215 SDValue Mask = DAG.getSetCC(dl, MaskVT, Cond, in LowerVSELECT()
21373 static SDValue getMaskNode(SDValue Mask, MVT MaskVT, in getMaskNode() argument
21378 return DAG.getConstant(1, dl, MaskVT); in getMaskNode()
21380 return DAG.getConstant(0, dl, MaskVT); in getMaskNode()
21382 assert(MaskVT.bitsLE(Mask.getSimpleValueType()) && "Unexpected mask size!"); in getMaskNode()
21385 assert(MaskVT == MVT::v64i1 && "Expected v64i1 mask!"); in getMaskNode()
21403 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT, in getMaskNode()
21417 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements()); in getVectorMaskingNode() local
21424 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in getVectorMaskingNode()
21840 MVT MaskVT = Op.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN() local
21850 Cmp = DAG.getNode(IntrData->Opc1, dl, MaskVT, Op.getOperand(1), in LowerINTRINSIC_WO_CHAIN()
21855 Cmp = DAG.getNode(IntrData->Opc0, dl, MaskVT, Op.getOperand(1), in LowerINTRINSIC_WO_CHAIN()
22040 MVT MaskVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorNumElements()); in LowerINTRINSIC_WO_CHAIN() local
22041 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in LowerINTRINSIC_WO_CHAIN()
22055 MVT MaskVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorNumElements()); in LowerINTRINSIC_WO_CHAIN() local
22056 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in LowerINTRINSIC_WO_CHAIN()
22308 EVT MaskVT = Mask.getValueType(); in getAVX2GatherNode() local
22309 SDVTList VTs = DAG.getVTList(Op.getValueType(), MaskVT, MVT::Other); in getAVX2GatherNode()
22336 MVT MaskVT = MVT::getVectorVT(MVT::i1, MinElts); in getGatherNode() local
22340 if (Mask.getValueType() != MaskVT) in getGatherNode()
22341 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in getGatherNode()
22343 SDVTList VTs = DAG.getVTList(Op.getValueType(), MaskVT, MVT::Other); in getGatherNode()
22371 MVT MaskVT = MVT::getVectorVT(MVT::i1, MinElts); in getScatterNode() local
22375 if (Mask.getValueType() != MaskVT) in getScatterNode()
22376 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in getScatterNode()
22378 SDVTList VTs = DAG.getVTList(MaskVT, MVT::Other); in getScatterNode()
22396 MVT MaskVT = in getPrefetchNode() local
22398 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in getPrefetchNode()
22796 MVT MaskVT = MVT::getVectorVT(MVT::i1, MemVT.getVectorNumElements()); in LowerINTRINSIC_W_CHAIN() local
22797 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in LowerINTRINSIC_W_CHAIN()
22809 MVT MaskVT = MVT::getVectorVT(MVT::i1, MemVT.getVectorNumElements()); in LowerINTRINSIC_W_CHAIN() local
22810 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl); in LowerINTRINSIC_W_CHAIN()
23335 MVT MaskVT = MVT::getVectorVT(MVT::i1, CurrVT.getVectorNumElements()); in LowerVectorCTLZInRegLUT() local
23336 HiZ = DAG.getSetCC(DL, MaskVT, Hi, Zero, ISD::SETEQ); in LowerVectorCTLZInRegLUT()
23361 MVT MaskVT = MVT::getVectorVT(MVT::i1, CurrVT.getVectorNumElements()); in LowerVectorCTLZInRegLUT() local
23362 HiZ = DAG.getSetCC(DL, MaskVT, DAG.getBitcast(CurrVT, Op0), in LowerVectorCTLZInRegLUT()
24730 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements()); in LowerShift() local
24734 Sel = DAG.getSetCC(dl, MaskVT, DAG.getConstant(0, dl, VT), Sel, in LowerShift()
25963 MVT MaskVT = Mask.getSimpleValueType(); in LowerMSCATTER() local
25981 MaskVT = MVT::getVectorVT(MVT::i1, NumElts); in LowerMSCATTER()
25985 Mask = ExtendToType(Mask, MaskVT, DAG, true); in LowerMSCATTER()
25988 SDVTList VTs = DAG.getVTList(MaskVT, MVT::Other); in LowerMSCATTER()
26098 MVT MaskVT = Mask.getSimpleValueType(); in LowerMGATHER() local
26119 MaskVT = MVT::getVectorVT(MVT::i1, NumElts); in LowerMGATHER()
26123 Mask = ExtendToType(Mask, MaskVT, DAG, true); in LowerMGATHER()
26129 DAG.getVTList(VT, MaskVT, MVT::Other), Ops, dl, N->getMemoryVT(), in LowerMGATHER()
30319 static bool matchUnaryVectorShuffle(MVT MaskVT, ArrayRef<int> Mask, in matchUnaryVectorShuffle() argument
30326 unsigned MaskEltSize = MaskVT.getScalarSizeInBits(); in matchUnaryVectorShuffle()
30332 SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT; in matchUnaryVectorShuffle()
30338 if (AllowIntDomain && ((MaskVT.is128BitVector() && Subtarget.hasSSE41()) || in matchUnaryVectorShuffle()
30339 (MaskVT.is256BitVector() && Subtarget.hasInt256()))) { in matchUnaryVectorShuffle()
30350 MVT ScalarTy = MaskVT.isInteger() ? MaskVT.getScalarType() : in matchUnaryVectorShuffle()
30354 if (SrcVT.getSizeInBits() != MaskVT.getSizeInBits()) in matchUnaryVectorShuffle()
30374 SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT; in matchUnaryVectorShuffle()
30381 if (MaskVT.is128BitVector() && Subtarget.hasSSE3() && AllowFloatDomain) { in matchUnaryVectorShuffle()
30399 if (MaskVT.is256BitVector() && AllowFloatDomain) { in matchUnaryVectorShuffle()
30418 if (MaskVT.is512BitVector() && AllowFloatDomain) { in matchUnaryVectorShuffle()
30444 SrcVT = DstVT = MaskVT; in matchUnaryVectorShuffle()
30456 static bool matchUnaryPermuteVectorShuffle(MVT MaskVT, ArrayRef<int> Mask, in matchUnaryPermuteVectorShuffle() argument
30464 unsigned InputSizeInBits = MaskVT.getSizeInBits(); in matchUnaryPermuteVectorShuffle()
30476 if (Subtarget.hasAVX2() && MaskVT.is256BitVector()) { in matchUnaryPermuteVectorShuffle()
30482 if (Subtarget.hasAVX512() && MaskVT.is512BitVector()) { in matchUnaryPermuteVectorShuffle()
30561 if (AllowIntDomain && ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) || in matchUnaryPermuteVectorShuffle()
30562 (MaskVT.is256BitVector() && Subtarget.hasAVX2()))) { in matchUnaryPermuteVectorShuffle()
30578 static bool matchBinaryVectorShuffle(MVT MaskVT, ArrayRef<int> Mask, in matchBinaryVectorShuffle() argument
30585 unsigned EltSizeInBits = MaskVT.getScalarSizeInBits(); in matchBinaryVectorShuffle()
30587 if (MaskVT.is128BitVector()) { in matchBinaryVectorShuffle()
30617 if (((MaskVT == MVT::v8i16 || MaskVT == MVT::v16i8) && Subtarget.hasSSE2()) || in matchBinaryVectorShuffle()
30618 ((MaskVT == MVT::v16i16 || MaskVT == MVT::v32i8) && Subtarget.hasInt256()) || in matchBinaryVectorShuffle()
30619 ((MaskVT == MVT::v32i16 || MaskVT == MVT::v64i8) && Subtarget.hasBWI())) { in matchBinaryVectorShuffle()
30620 if (matchVectorShuffleWithPACK(MaskVT, SrcVT, V1, V2, Shuffle, Mask, DAG, in matchBinaryVectorShuffle()
30622 DstVT = MaskVT; in matchBinaryVectorShuffle()
30628 if ((MaskVT == MVT::v4f32 && Subtarget.hasSSE1()) || in matchBinaryVectorShuffle()
30629 (MaskVT.is128BitVector() && Subtarget.hasSSE2()) || in matchBinaryVectorShuffle()
30630 (MaskVT.is256BitVector() && 32 <= EltSizeInBits && Subtarget.hasAVX()) || in matchBinaryVectorShuffle()
30631 (MaskVT.is256BitVector() && Subtarget.hasAVX2()) || in matchBinaryVectorShuffle()
30632 (MaskVT.is512BitVector() && Subtarget.hasAVX512())) { in matchBinaryVectorShuffle()
30633 if (matchVectorShuffleWithUNPCK(MaskVT, V1, V2, Shuffle, IsUnary, Mask, DL, in matchBinaryVectorShuffle()
30635 SrcVT = DstVT = MaskVT; in matchBinaryVectorShuffle()
30636 if (MaskVT.is256BitVector() && !Subtarget.hasAVX2()) in matchBinaryVectorShuffle()
30646 MVT MaskVT, ArrayRef<int> Mask, const APInt &Zeroable, in matchBinaryPermuteVectorShuffle() argument
30651 unsigned EltSizeInBits = MaskVT.getScalarSizeInBits(); in matchBinaryPermuteVectorShuffle()
30654 if (AllowIntDomain && ((MaskVT.is128BitVector() && Subtarget.hasSSSE3()) || in matchBinaryPermuteVectorShuffle()
30655 (MaskVT.is256BitVector() && Subtarget.hasAVX2()))) { in matchBinaryPermuteVectorShuffle()
30656 int ByteRotation = matchVectorShuffleAsByteRotate(MaskVT, V1, V2, Mask); in matchBinaryPermuteVectorShuffle()
30659 ShuffleVT = MVT::getVectorVT(MVT::i8, MaskVT.getSizeInBits() / 8); in matchBinaryPermuteVectorShuffle()
30666 if ((NumMaskElts <= 8 && ((Subtarget.hasSSE41() && MaskVT.is128BitVector()) || in matchBinaryPermuteVectorShuffle()
30667 (Subtarget.hasAVX() && MaskVT.is256BitVector()))) || in matchBinaryPermuteVectorShuffle()
30668 (MaskVT == MVT::v16i16 && Subtarget.hasAVX2())) { in matchBinaryPermuteVectorShuffle()
30674 if (MaskVT == MVT::v16i16) { in matchBinaryPermuteVectorShuffle()
30677 if (isRepeatedTargetShuffleMask(128, MaskVT, TargetMask, in matchBinaryPermuteVectorShuffle()
30685 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1; in matchBinaryPermuteVectorShuffle()
30686 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2; in matchBinaryPermuteVectorShuffle()
30688 ShuffleVT = MaskVT; in matchBinaryPermuteVectorShuffle()
30693 ShuffleVT = MaskVT; in matchBinaryPermuteVectorShuffle()
30716 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1; in matchBinaryPermuteVectorShuffle()
30717 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2; in matchBinaryPermuteVectorShuffle()
30727 MaskVT.is128BitVector()) { in matchBinaryPermuteVectorShuffle()
30738 ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) || in matchBinaryPermuteVectorShuffle()
30739 (MaskVT.is256BitVector() && Subtarget.hasAVX()) || in matchBinaryPermuteVectorShuffle()
30740 (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) { in matchBinaryPermuteVectorShuffle()
30741 if (matchVectorShuffleWithSHUFPD(MaskVT, V1, V2, PermuteImm, Mask)) { in matchBinaryPermuteVectorShuffle()
30743 ShuffleVT = MVT::getVectorVT(MVT::f64, MaskVT.getSizeInBits() / 64); in matchBinaryPermuteVectorShuffle()
30750 ((MaskVT.is128BitVector() && Subtarget.hasSSE1()) || in matchBinaryPermuteVectorShuffle()
30751 (MaskVT.is256BitVector() && Subtarget.hasAVX()) || in matchBinaryPermuteVectorShuffle()
30752 (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) { in matchBinaryPermuteVectorShuffle()
30754 if (isRepeatedTargetShuffleMask(128, MaskVT, Mask, RepeatedMask)) { in matchBinaryPermuteVectorShuffle()
30762 return DAG.getUNDEF(MaskVT); in matchBinaryPermuteVectorShuffle()
30766 return getZeroVector(MaskVT, Subtarget, DAG, DL); in matchBinaryPermuteVectorShuffle()
30788 ShuffleVT = MVT::getVectorVT(MVT::f32, MaskVT.getSizeInBits() / 32); in matchBinaryPermuteVectorShuffle()
30894 MVT MaskVT = FloatDomain ? MVT::getFloatingPointVT(MaskEltSizeInBits) in combineX86ShuffleChain() local
30896 MaskVT = MVT::getVectorVT(MaskVT, NumMaskElts); in combineX86ShuffleChain()
30899 if (!DAG.getTargetLoweringInfo().isTypeLegal(MaskVT)) in combineX86ShuffleChain()
30910 (!MaskVT.is256BitVector() || Subtarget.hasAVX2()); in combineX86ShuffleChain()
30933 if (matchUnaryVectorShuffle(MaskVT, Mask, AllowFloatDomain, AllowIntDomain, in combineX86ShuffleChain()
30944 if (matchUnaryPermuteVectorShuffle(MaskVT, Mask, Zeroable, AllowFloatDomain, in combineX86ShuffleChain()
30959 if (matchBinaryVectorShuffle(MaskVT, Mask, AllowFloatDomain, AllowIntDomain, in combineX86ShuffleChain()
30974 MaskVT, Mask, Zeroable, AllowFloatDomain, AllowIntDomain, NewV1, in combineX86ShuffleChain()
31028 if (is128BitLaneCrossingShuffleMask(MaskVT, Mask)) { in combineX86ShuffleChain()
31032 (MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) || in combineX86ShuffleChain()
31034 (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 || in combineX86ShuffleChain()
31035 MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) || in combineX86ShuffleChain()
31036 (Subtarget.hasBWI() && MaskVT == MVT::v32i16) || in combineX86ShuffleChain()
31037 (Subtarget.hasBWI() && Subtarget.hasVLX() && MaskVT == MVT::v16i16) || in combineX86ShuffleChain()
31038 (Subtarget.hasVBMI() && MaskVT == MVT::v64i8) || in combineX86ShuffleChain()
31039 (Subtarget.hasVBMI() && Subtarget.hasVLX() && MaskVT == MVT::v32i8))) { in combineX86ShuffleChain()
31041 Res = DAG.getBitcast(MaskVT, V1); in combineX86ShuffleChain()
31042 Res = DAG.getNode(X86ISD::VPERMV, DL, MaskVT, VPermMask, Res); in combineX86ShuffleChain()
31050 (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 || in combineX86ShuffleChain()
31051 MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) || in combineX86ShuffleChain()
31053 (MaskVT == MVT::v4f64 || MaskVT == MVT::v4i64 || in combineX86ShuffleChain()
31054 MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) || in combineX86ShuffleChain()
31055 (Subtarget.hasBWI() && MaskVT == MVT::v32i16) || in combineX86ShuffleChain()
31056 (Subtarget.hasBWI() && Subtarget.hasVLX() && MaskVT == MVT::v16i16) || in combineX86ShuffleChain()
31057 (Subtarget.hasVBMI() && MaskVT == MVT::v64i8) || in combineX86ShuffleChain()
31058 (Subtarget.hasVBMI() && Subtarget.hasVLX() && MaskVT == MVT::v32i8))) { in combineX86ShuffleChain()
31065 Res = DAG.getBitcast(MaskVT, V1); in combineX86ShuffleChain()
31066 SDValue Zero = getZeroVector(MaskVT, Subtarget, DAG, DL); in combineX86ShuffleChain()
31067 Res = DAG.getNode(X86ISD::VPERMV3, DL, MaskVT, Res, VPermMask, Zero); in combineX86ShuffleChain()
31074 (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 || in combineX86ShuffleChain()
31075 MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) || in combineX86ShuffleChain()
31077 (MaskVT == MVT::v4f64 || MaskVT == MVT::v4i64 || in combineX86ShuffleChain()
31078 MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) || in combineX86ShuffleChain()
31079 (Subtarget.hasBWI() && MaskVT == MVT::v32i16) || in combineX86ShuffleChain()
31080 (Subtarget.hasBWI() && Subtarget.hasVLX() && MaskVT == MVT::v16i16) || in combineX86ShuffleChain()
31081 (Subtarget.hasVBMI() && MaskVT == MVT::v64i8) || in combineX86ShuffleChain()
31082 (Subtarget.hasVBMI() && Subtarget.hasVLX() && MaskVT == MVT::v32i8))) { in combineX86ShuffleChain()
31084 V1 = DAG.getBitcast(MaskVT, V1); in combineX86ShuffleChain()
31085 V2 = DAG.getBitcast(MaskVT, V2); in combineX86ShuffleChain()
31086 Res = DAG.getNode(X86ISD::VPERMV3, DL, MaskVT, V1, VPermMask, V2); in combineX86ShuffleChain()
31096 DAG.getTargetLoweringInfo().isTypeLegal(MaskVT)) { in combineX86ShuffleChain()
31111 SDValue BitMask = getConstVector(EltBits, UndefElts, MaskVT, DAG, DL); in combineX86ShuffleChain()
31112 Res = DAG.getBitcast(MaskVT, V1); in combineX86ShuffleChain()
31115 Res = DAG.getNode(AndOpcode, DL, MaskVT, Res, BitMask); in combineX86ShuffleChain()
31123 ((MaskVT == MVT::v8f32 && Subtarget.hasAVX()) || in combineX86ShuffleChain()
31124 (MaskVT == MVT::v16f32 && Subtarget.hasAVX512()))) { in combineX86ShuffleChain()
31132 Res = DAG.getBitcast(MaskVT, V1); in combineX86ShuffleChain()
31133 Res = DAG.getNode(X86ISD::VPERMILPV, DL, MaskVT, Res, VPermMask); in combineX86ShuffleChain()
31140 (MaskVT == MVT::v2f64 || MaskVT == MVT::v4f64 || MaskVT == MVT::v4f32 || in combineX86ShuffleChain()
31141 MaskVT == MVT::v8f32)) { in combineX86ShuffleChain()
31146 unsigned NumLanes = MaskVT.getSizeInBits() / 128; in combineX86ShuffleChain()
31161 Index = (MaskVT.getScalarSizeInBits() == 64 ? Index << 1 : Index); in combineX86ShuffleChain()
31164 V1 = DAG.getBitcast(MaskVT, V1); in combineX86ShuffleChain()
31165 V2 = DAG.getBitcast(MaskVT, V2); in combineX86ShuffleChain()
31167 Res = DAG.getNode(X86ISD::VPERMIL2, DL, MaskVT, V1, V2, VPerm2MaskOp, in combineX86ShuffleChain()
31318 MVT MaskVT = MVT::getVectorVT(MaskSVT, NumMaskElts); in combineX86ShufflesConstants() local
31321 SDValue CstOp = getConstVector(ConstantBitData, UndefElts, MaskVT, DAG, DL); in combineX86ShufflesConstants()
33531 MVT MaskVT; in combineHorizontalPredicateResult() local
33533 MaskVT = MVT::getVectorVT(MVT::getFloatingPointVT(BitWidth), in combineHorizontalPredicateResult()
33536 MaskVT = MVT::getVectorVT(MVT::i8, MatchSizeInBits / 8); in combineHorizontalPredicateResult()
33546 CompareBits = APInt::getLowBitsSet(32, MaskVT.getVectorNumElements()); in combineHorizontalPredicateResult()
33557 SDValue Res = DAG.getBitcast(MaskVT, Match); in combineHorizontalPredicateResult()
36512 EVT MaskVT = Mask.getValueType(); in combineLogicBlendIntoPBLENDV() local
36513 unsigned EltBits = MaskVT.getScalarSizeInBits(); in combineLogicBlendIntoPBLENDV()
36516 if (!MaskVT.isInteger() || DAG.ComputeNumSignBits(Mask) != EltBits) in combineLogicBlendIntoPBLENDV()
36537 if (X.getValueType() == MaskVT && Y.getValueType() == MaskVT && in combineLogicBlendIntoPBLENDV()
36538 DAG.getTargetLoweringInfo().isOperationLegal(ISD::SUB, MaskVT)) { in combineLogicBlendIntoPBLENDV()
36550 SDValue SubOp1 = DAG.getNode(ISD::XOR, DL, MaskVT, V, Mask); in combineLogicBlendIntoPBLENDV()
36566 SDValue Res = DAG.getNode(ISD::SUB, DL, MaskVT, SubOp1, SubOp2); in combineLogicBlendIntoPBLENDV()