Lines Matching refs:Zeroable
5197 const APInt &Zeroable, in canWidenShuffleElements() argument
5203 if (Zeroable[i]) in canWidenShuffleElements()
7005 std::bitset<4> Zeroable; in LowerBuildVectorv4x32() local
7008 Zeroable[i] = (Elt.isUndef() || X86::isZeroNode(Elt)); in LowerBuildVectorv4x32()
7010 assert(Zeroable.size() - Zeroable.count() > 1 && in LowerBuildVectorv4x32()
7018 if (Zeroable[i]) in LowerBuildVectorv4x32()
7043 if (Zeroable[EltIdx]) { in LowerBuildVectorv4x32()
7075 if (Zeroable[i]) in LowerBuildVectorv4x32()
7095 unsigned ZMask = Zeroable.to_ulong(); in LowerBuildVectorv4x32()
9636 const APInt &Zeroable) { in createTargetShuffleMask() argument
9638 assert(NumElts == (int)Zeroable.getBitWidth() && "Mismatch mask sizes"); in createTargetShuffleMask()
9646 TargetMask[i] = (Zeroable[i] ? SM_SentinelZero : M); in createTargetShuffleMask()
9728 APInt Zeroable(Mask.size(), 0); in computeZeroableShuffleElements() local
9743 Zeroable.setBit(i); in computeZeroableShuffleElements()
9761 Zeroable.setBit(i); in computeZeroableShuffleElements()
9767 Zeroable.setBit(i); in computeZeroableShuffleElements()
9773 Zeroable.setBit(i); in computeZeroableShuffleElements()
9788 Zeroable.setBit(i); in computeZeroableShuffleElements()
9793 return Zeroable; in computeZeroableShuffleElements()
9803 static bool isNonZeroElementsInOrder(const APInt &Zeroable, in isNonZeroElementsInOrder() argument
9813 if (Zeroable[i]) in isNonZeroElementsInOrder()
9832 const APInt &Zeroable, in lowerVectorShuffleWithPSHUFB() argument
9855 if (Zeroable[i / NumEltBytes]) { in lowerVectorShuffleWithPSHUFB()
9889 const APInt &Zeroable, in lowerVectorShuffleToEXPAND() argument
9894 if (!isNonZeroElementsInOrder(Zeroable, Mask, V1.getValueType(), in lowerVectorShuffleToEXPAND()
9897 unsigned VEXPANDMask = (~Zeroable).getZExtValue(); in lowerVectorShuffleToEXPAND()
10192 const APInt &Zeroable, in lowerVectorShuffleAsBitMask() argument
10201 if (Zeroable[i]) in lowerVectorShuffleAsBitMask()
10310 const APInt &Zeroable, in lowerVectorShuffleAsBlend() argument
10313 SmallVector<int, 64> Mask = createTargetShuffleMask(Original, Zeroable); in lowerVectorShuffleAsBlend()
10401 lowerVectorShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable, DAG)) in lowerVectorShuffleAsBlend()
10924 const APInt &Zeroable, in matchVectorShuffleAsShift() argument
10932 if (!Zeroable[i + j + (Left ? 0 : (Scale - Shift))]) in matchVectorShuffleAsShift()
10986 const APInt &Zeroable, in lowerVectorShuffleAsShift() argument
10998 ShiftVT, Opcode, VT.getScalarSizeInBits(), Mask, 0, Zeroable, Subtarget); in lowerVectorShuffleAsShift()
11004 Mask, Size, Zeroable, Subtarget); in lowerVectorShuffleAsShift()
11023 uint64_t &BitIdx, const APInt &Zeroable) { in matchVectorShuffleAsEXTRQ() argument
11027 assert(!Zeroable.isAllOnesValue() && "Fully zeroable shuffle mask"); in matchVectorShuffleAsEXTRQ()
11037 if (!Zeroable[Len - 1]) in matchVectorShuffleAsEXTRQ()
11145 const APInt &Zeroable, in lowerVectorShuffleWithSSE4A() argument
11148 if (matchVectorShuffleAsEXTRQ(VT, V1, V2, Mask, BitLen, BitIdx, Zeroable)) in lowerVectorShuffleWithSSE4A()
11331 const APInt &Zeroable, const X86Subtarget &Subtarget, in lowerVectorShuffleAsZeroOrAnyExtend() argument
11354 if (!Zeroable[i]) in lowerVectorShuffleAsZeroOrAnyExtend()
11426 if (!Zeroable[i]) in lowerVectorShuffleAsZeroOrAnyExtend()
11487 const APInt &Zeroable, const X86Subtarget &Subtarget, in lowerVectorShuffleAsElementInsertion() argument
11497 if (i != V2Index && !Zeroable[i]) { in lowerVectorShuffleAsElementInsertion()
11854 const APInt &Zeroable, in matchVectorShuffleAsInsertPS() argument
11873 if (Zeroable[i]) { in matchVectorShuffleAsInsertPS()
11943 const APInt &Zeroable, in lowerVectorShuffleAsInsertPS() argument
11950 if (!matchVectorShuffleAsInsertPS(V1, V2, InsertPSMask, Zeroable, Mask, DAG)) in lowerVectorShuffleAsInsertPS()
12083 const APInt &Zeroable, in lowerV2F64VectorShuffle() argument
12122 DL, MVT::v2f64, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV2F64VectorShuffle()
12129 DL, MVT::v2f64, V2, V1, InverseMask, Zeroable, Subtarget, DAG)) in lowerV2F64VectorShuffle()
12145 Zeroable, Subtarget, DAG)) in lowerV2F64VectorShuffle()
12165 const APInt &Zeroable, in lowerV2I64VectorShuffle() argument
12198 Zeroable, Subtarget, DAG)) in lowerV2I64VectorShuffle()
12204 DL, MVT::v2i64, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV2I64VectorShuffle()
12210 DL, MVT::v2i64, V2, V1, InverseMask, Zeroable, Subtarget, DAG)) in lowerV2I64VectorShuffle()
12218 Zeroable, Subtarget, DAG)) in lowerV2I64VectorShuffle()
12370 const APInt &Zeroable, in lowerV4F32VectorShuffle() argument
12423 DL, MVT::v4f32, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV4F32VectorShuffle()
12428 Zeroable, Subtarget, DAG)) in lowerV4F32VectorShuffle()
12433 lowerVectorShuffleAsInsertPS(DL, V1, V2, Mask, Zeroable, DAG)) in lowerV4F32VectorShuffle()
12465 const APInt &Zeroable, in lowerV4I32VectorShuffle() argument
12477 DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV4I32VectorShuffle()
12506 Zeroable, Subtarget, DAG)) in lowerV4I32VectorShuffle()
12512 DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV4I32VectorShuffle()
12520 Zeroable, Subtarget, DAG)) in lowerV4I32VectorShuffle()
12524 Zeroable, DAG)) in lowerV4I32VectorShuffle()
13089 const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse) { in lowerVectorShuffleAsBlendOfPSHUFBs() argument
13110 if (Zeroable[i / Scale]) in lowerVectorShuffleAsBlendOfPSHUFBs()
13151 const APInt &Zeroable, in lowerV8I16VectorShuffle() argument
13162 DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV8I16VectorShuffle()
13175 Zeroable, Subtarget, DAG)) in lowerV8I16VectorShuffle()
13206 Zeroable, Subtarget, DAG)) in lowerV8I16VectorShuffle()
13212 Zeroable, DAG)) in lowerV8I16VectorShuffle()
13218 DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV8I16VectorShuffle()
13226 Zeroable, Subtarget, DAG)) in lowerV8I16VectorShuffle()
13230 Zeroable, DAG)) in lowerV8I16VectorShuffle()
13262 Zeroable, DAG, V1InUse, V2InUse); in lowerV8I16VectorShuffle()
13358 const APInt &Zeroable, in lowerV16I8VectorShuffle() argument
13368 Zeroable, Subtarget, DAG)) in lowerV16I8VectorShuffle()
13383 DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV16I8VectorShuffle()
13389 Zeroable, DAG)) in lowerV16I8VectorShuffle()
13496 Zeroable, DAG)) in lowerV16I8VectorShuffle()
13522 DL, MVT::v16i8, V1, V2, Mask, Zeroable, DAG, V1InUse, V2InUse); in lowerV16I8VectorShuffle()
13530 DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV16I8VectorShuffle()
13562 DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV16I8VectorShuffle()
13666 const APInt &Zeroable, in lower128BitVectorShuffle() argument
13671 return lowerV2I64VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitVectorShuffle()
13673 return lowerV2F64VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitVectorShuffle()
13675 return lowerV4I32VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitVectorShuffle()
13677 return lowerV4F32VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitVectorShuffle()
13679 return lowerV8I16VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitVectorShuffle()
13681 return lowerV16I8VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower128BitVectorShuffle()
13995 const APInt &Zeroable, in lowerV2X128VectorShuffle() argument
14003 if (!canWidenShuffleElements(Mask, Zeroable, WidenedMask)) in lowerV2X128VectorShuffle()
14006 bool IsLowZero = (Zeroable & 0x3) == 0x3; in lowerV2X128VectorShuffle()
14007 bool IsHighZero = (Zeroable & 0xc) == 0xc; in lowerV2X128VectorShuffle()
14025 Zeroable, Subtarget, DAG)) in lowerV2X128VectorShuffle()
14628 const APInt &Zeroable, in lowerV4F64VectorShuffle() argument
14637 Zeroable, Subtarget, DAG)) in lowerV4F64VectorShuffle()
14686 Zeroable, Subtarget, DAG)) in lowerV4F64VectorShuffle()
14712 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v4f64, Zeroable, Mask, in lowerV4F64VectorShuffle()
14732 const APInt &Zeroable, in lowerV4I64VectorShuffle() argument
14742 Zeroable, Subtarget, DAG)) in lowerV4I64VectorShuffle()
14746 Zeroable, Subtarget, DAG)) in lowerV4I64VectorShuffle()
14776 Zeroable, Subtarget, DAG)) in lowerV4I64VectorShuffle()
14785 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v4i64, Zeroable, Mask, in lowerV4I64VectorShuffle()
14826 const APInt &Zeroable, in lowerV8F32VectorShuffle() argument
14835 Zeroable, Subtarget, DAG)) in lowerV8F32VectorShuffle()
14898 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v8f32, Zeroable, Mask, in lowerV8F32VectorShuffle()
14926 const APInt &Zeroable, in lowerV8I32VectorShuffle() argument
14939 DL, MVT::v8i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV8I32VectorShuffle()
14952 Zeroable, Subtarget, DAG)) in lowerV8I32VectorShuffle()
14980 Zeroable, Subtarget, DAG)) in lowerV8I32VectorShuffle()
14989 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v8i32, Zeroable, Mask, in lowerV8I32VectorShuffle()
15039 const APInt &Zeroable, in lowerV16I16VectorShuffle() argument
15052 DL, MVT::v16i16, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV16I16VectorShuffle()
15061 Zeroable, Subtarget, DAG)) in lowerV16I16VectorShuffle()
15076 Zeroable, Subtarget, DAG)) in lowerV16I16VectorShuffle()
15113 DL, MVT::v16i16, Mask, V1, V2, Zeroable, Subtarget, DAG)) in lowerV16I16VectorShuffle()
15141 const APInt &Zeroable, in lowerV32I8VectorShuffle() argument
15154 DL, MVT::v32i8, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV32I8VectorShuffle()
15163 Zeroable, Subtarget, DAG)) in lowerV32I8VectorShuffle()
15178 Zeroable, Subtarget, DAG)) in lowerV32I8VectorShuffle()
15204 DL, MVT::v32i8, Mask, V1, V2, Zeroable, Subtarget, DAG)) in lowerV32I8VectorShuffle()
15234 const APInt &Zeroable, in lower256BitVectorShuffle() argument
15244 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lower256BitVectorShuffle()
15264 lowerVectorShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable, DAG)) in lower256BitVectorShuffle()
15280 return lowerV4F64VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower256BitVectorShuffle()
15282 return lowerV4I64VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower256BitVectorShuffle()
15284 return lowerV8F32VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower256BitVectorShuffle()
15286 return lowerV8I32VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower256BitVectorShuffle()
15288 return lowerV16I16VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower256BitVectorShuffle()
15290 return lowerV32I8VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower256BitVectorShuffle()
15300 const APInt &Zeroable, in lowerV4X128VectorShuffle() argument
15317 if (WidenedMask[0] == 0 && (Zeroable & 0xf0) == 0xf0 && in lowerV4X128VectorShuffle()
15318 (WidenedMask[1] == 1 || (Zeroable & 0x0c) == 0x0c)) { in lowerV4X128VectorShuffle()
15319 unsigned NumElts = ((Zeroable & 0x0c) == 0x0c) ? 2 : 4; in lowerV4X128VectorShuffle()
15401 const APInt &Zeroable, in lowerV8F64VectorShuffle() argument
15432 lowerV4X128VectorShuffle(DL, MVT::v8f64, Mask, Zeroable, V1, V2, in lowerV8F64VectorShuffle()
15445 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v8f64, Zeroable, Mask, V1, in lowerV8F64VectorShuffle()
15450 Zeroable, Subtarget, DAG)) in lowerV8F64VectorShuffle()
15458 const APInt &Zeroable, in lowerV16F32VectorShuffle() argument
15488 Zeroable, Subtarget, DAG)) in lowerV16F32VectorShuffle()
15504 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v16f32, Zeroable, Mask, in lowerV16F32VectorShuffle()
15513 const APInt &Zeroable, in lowerV8I64VectorShuffle() argument
15543 lowerV4X128VectorShuffle(DL, MVT::v8i64, Mask, Zeroable, in lowerV8I64VectorShuffle()
15549 Zeroable, Subtarget, DAG)) in lowerV8I64VectorShuffle()
15566 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v8i64, Zeroable, Mask, V1, in lowerV8I64VectorShuffle()
15571 Zeroable, Subtarget, DAG)) in lowerV8I64VectorShuffle()
15579 const APInt &Zeroable, in lowerV16I32VectorShuffle() argument
15591 DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV16I32VectorShuffle()
15614 Zeroable, Subtarget, DAG)) in lowerV16I32VectorShuffle()
15638 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v16i32, Zeroable, Mask, in lowerV16I32VectorShuffle()
15643 Zeroable, Subtarget, DAG)) in lowerV16I32VectorShuffle()
15650 const APInt &Zeroable, in lowerV32I16VectorShuffle() argument
15663 DL, MVT::v32i16, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV32I16VectorShuffle()
15673 Zeroable, Subtarget, DAG)) in lowerV32I16VectorShuffle()
15693 Zeroable, Subtarget, DAG)) in lowerV32I16VectorShuffle()
15697 DL, MVT::v32i16, Mask, V1, V2, Zeroable, Subtarget, DAG)) in lowerV32I16VectorShuffle()
15705 const APInt &Zeroable, in lowerV64I8VectorShuffle() argument
15718 DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lowerV64I8VectorShuffle()
15733 Zeroable, Subtarget, DAG)) in lowerV64I8VectorShuffle()
15742 DL, MVT::v64i8, Mask, V1, V2, Zeroable, Subtarget, DAG)) in lowerV64I8VectorShuffle()
15756 Zeroable, Subtarget, DAG)) in lowerV64I8VectorShuffle()
15770 const APInt &Zeroable, in lower512BitVectorShuffle() argument
15783 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lower512BitVectorShuffle()
15802 return lowerV8F64VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower512BitVectorShuffle()
15804 return lowerV16F32VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower512BitVectorShuffle()
15806 return lowerV8I64VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower512BitVectorShuffle()
15808 return lowerV16I32VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower512BitVectorShuffle()
15810 return lowerV32I16VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower512BitVectorShuffle()
15812 return lowerV64I8VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG); in lower512BitVectorShuffle()
15823 int MaskOffset, const APInt &Zeroable) { in match1BitShuffleAsKSHIFT() argument
15828 if (!Zeroable[j + (Left ? 0 : (Size - Shift))]) in match1BitShuffleAsKSHIFT()
15858 const APInt &Zeroable, in lower1BitVectorShuffle() argument
15881 if (Zeroable.countLeadingOnes() >= (NumElts - SubvecElts)) { in lower1BitVectorShuffle()
15896 int ShiftAmt = match1BitShuffleAsKSHIFT(Opcode, Mask, Offset, Zeroable); in lower1BitVectorShuffle()
16068 APInt Zeroable = computeZeroableShuffleElements(Mask, V1, V2); in lowerVectorShuffle() local
16069 if (Zeroable.isAllOnesValue()) in lowerVectorShuffle()
16078 assert(!Zeroable.isNullValue() && "V2's non-undef elements are used?!"); in lowerVectorShuffle()
16080 if (Mask[i] != SM_SentinelUndef && Zeroable[i]) in lowerVectorShuffle()
16140 return lower128BitVectorShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, in lowerVectorShuffle()
16144 return lower256BitVectorShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, in lowerVectorShuffle()
16148 return lower512BitVectorShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, in lowerVectorShuffle()
16152 return lower1BitVectorShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, in lowerVectorShuffle()
30457 const APInt &Zeroable, in matchUnaryPermuteVectorShuffle() argument
30565 0, Zeroable, Subtarget); in matchUnaryPermuteVectorShuffle()
30646 MVT MaskVT, ArrayRef<int> Mask, const APInt &Zeroable, in matchBinaryPermuteVectorShuffle() argument
30728 if (Zeroable.getBoolValue() && in matchBinaryPermuteVectorShuffle()
30729 matchVectorShuffleAsInsertPS(V1, V2, PermuteImm, Zeroable, Mask, DAG)) { in matchBinaryPermuteVectorShuffle()
30913 APInt Zeroable(NumMaskElts, 0); in combineX86ShuffleChain() local
30916 Zeroable.setBit(i); in combineX86ShuffleChain()
30944 if (matchUnaryPermuteVectorShuffle(MaskVT, Mask, Zeroable, AllowFloatDomain, in combineX86ShuffleChain()
30974 MaskVT, Mask, Zeroable, AllowFloatDomain, AllowIntDomain, NewV1, in combineX86ShuffleChain()
30994 Zeroable)) { in combineX86ShuffleChain()