Lines Matching refs:NumElts
2442 handleMaskRegisterForCallingConv(unsigned NumElts, CallingConv::ID CC, in handleMaskRegisterForCallingConv() argument
2446 if (NumElts == 2) in handleMaskRegisterForCallingConv()
2448 if (NumElts == 4) in handleMaskRegisterForCallingConv()
2450 if (NumElts == 8 && CC != CallingConv::X86_RegCall && in handleMaskRegisterForCallingConv()
2453 if (NumElts == 16 && CC != CallingConv::X86_RegCall && in handleMaskRegisterForCallingConv()
2458 if (NumElts == 32 && (!Subtarget.hasBWI() || CC != CallingConv::X86_RegCall)) in handleMaskRegisterForCallingConv()
2461 if (NumElts == 64 && Subtarget.hasBWI() && CC != CallingConv::X86_RegCall) { in handleMaskRegisterForCallingConv()
2468 if (!isPowerOf2_32(NumElts) || (NumElts == 64 && !Subtarget.hasBWI()) || in handleMaskRegisterForCallingConv()
2469 NumElts > 64) in handleMaskRegisterForCallingConv()
2470 return {MVT::i8, NumElts}; in handleMaskRegisterForCallingConv()
2480 unsigned NumElts = VT.getVectorNumElements(); in getRegisterTypeForCallingConv() local
2485 handleMaskRegisterForCallingConv(NumElts, CC, Subtarget); in getRegisterTypeForCallingConv()
2511 unsigned NumElts = VT.getVectorNumElements(); in getNumRegistersForCallingConv() local
2516 handleMaskRegisterForCallingConv(NumElts, CC, Subtarget); in getNumRegistersForCallingConv()
5650 unsigned NumElts = std::min(DataVT.getVectorNumElements(), in getTgtMemIntrinsic() local
5652 Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts); in getTgtMemIntrinsic()
5662 unsigned NumElts = std::min(DataVT.getVectorNumElements(), in getTgtMemIntrinsic() local
5664 Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts); in getTgtMemIntrinsic()
6049 unsigned NumElts = Mask.size(); in isUndefLowerHalf() local
6050 return isUndefInRange(Mask, 0, NumElts / 2); in isUndefLowerHalf()
6055 unsigned NumElts = Mask.size(); in isUndefUpperHalf() local
6056 return isUndefInRange(Mask, NumElts / 2, NumElts / 2); in isUndefUpperHalf()
6264 unsigned NumElts = VT.getVectorNumElements(); in getConstVector() local
6267 ConstVecVT = MVT::getVectorVT(MVT::i32, NumElts * 2); in getConstVector()
6272 for (unsigned i = 0; i < NumElts; ++i) { in getConstVector()
6295 unsigned NumElts = VT.getVectorNumElements(); in getConstVector() local
6298 ConstVecVT = MVT::getVectorVT(MVT::i32, NumElts * 2); in getConstVector()
6367 unsigned NumElts = LHS.getValueType().getVectorNumElements(); in getSplitVectorSrc() local
6369 RHS.getConstantOperandAPInt(1) == NumElts) || in getSplitVectorSrc()
6371 LHS.getConstantOperandAPInt(1) == NumElts)) in getSplitVectorSrc()
6913 unsigned NumElts = VT.getSizeInBits() / 32; in getOnesVector() local
6914 SDValue Vec = DAG.getConstant(Ones, dl, MVT::getVectorVT(MVT::i32, NumElts)); in getOnesVector()
7008 int NumElts = VT.getVectorNumElements(); in createUnpackShuffleMask() local
7010 for (int i = 0; i < NumElts; ++i) { in createUnpackShuffleMask()
7013 Pos += (Unary ? 0 : NumElts * (i % 2)); in createUnpackShuffleMask()
7026 int NumElts = VT.getVectorNumElements(); in createSplat2ShuffleMask() local
7027 for (int i = 0; i < NumElts; ++i) { in createSplat2ShuffleMask()
7029 Pos += (Lo ? 0 : NumElts / 2); in createSplat2ShuffleMask()
7040 for (int I = 0, NumElts = Mask.size(); I != NumElts; ++I) { in getVectorShuffle() local
7044 SDValue V = (M < NumElts) ? V1 : V2; in getVectorShuffle()
7047 Ops[I] = V.getOperand(M % NumElts); in getVectorShuffle()
7091 int NumElts = VT.getVectorNumElements(); in getPack() local
7092 for (int I = 0; I != NumElts; I += 4) { in getPack()
7095 PackMask.push_back(I + Offset + NumElts); in getPack()
7096 PackMask.push_back(I + Offset + NumElts + 2); in getPack()
7198 unsigned NumElts = SizeInBits / EltSizeInBits; in getTargetConstantBitsFromNode() local
7213 if (NumSrcElts == NumElts) { in getTargetConstantBitsFromNode()
7231 UndefElts = APInt(NumElts, 0); in getTargetConstantBitsFromNode()
7232 EltBits.resize(NumElts, APInt(EltSizeInBits, 0)); in getTargetConstantBitsFromNode()
7234 for (unsigned i = 0; i != NumElts; ++i) { in getTargetConstantBitsFromNode()
7278 APInt UndefSrcElts = APInt::getAllOnes(NumElts); in getTargetConstantBitsFromNode()
7279 SmallVector<APInt, 64> SrcEltBits(NumElts, APInt(EltSizeInBits, 0)); in getTargetConstantBitsFromNode()
7460 if (isAnyInRange(Mask, 0, NumElts) && in getTargetConstantBitsFromNode()
7465 if (isAnyInRange(Mask, NumElts, 2 * NumElts) && in getTargetConstantBitsFromNode()
7471 UndefElts = APInt::getZero(NumElts); in getTargetConstantBitsFromNode()
7472 for (int i = 0; i != (int)NumElts; ++i) { in getTargetConstantBitsFromNode()
7477 } else if (M < (int)NumElts) { in getTargetConstantBitsFromNode()
7482 if (UndefElts1[M - NumElts]) in getTargetConstantBitsFromNode()
7484 EltBits.push_back(EltBits1[M - NumElts]); in getTargetConstantBitsFromNode()
7546 unsigned NumElts = VT.getVectorNumElements(); in createPackShuffleMask() local
7549 unsigned Offset = Unary ? 0 : NumElts; in createPackShuffleMask()
7568 int NumElts = DemandedElts.getBitWidth(); in getPackDemandedElts() local
7569 int NumInnerElts = NumElts / 2; in getPackDemandedElts()
7570 int NumEltsPerLane = NumElts / NumLanes; in getPackDemandedElts()
7593 int NumElts = DemandedElts.getBitWidth(); in getHorizDemandedElts() local
7594 int NumEltsPerLane = NumElts / NumLanes; in getHorizDemandedElts()
7597 DemandedLHS = APInt::getZero(NumElts); in getHorizDemandedElts()
7598 DemandedRHS = APInt::getZero(NumElts); in getHorizDemandedElts()
7601 for (int Idx = 0; Idx != NumElts; ++Idx) { in getHorizDemandedElts()
8128 unsigned NumElts = Mask.size(); in resolveTargetShuffleFromZeroables() local
8129 assert(KnownUndef.getBitWidth() == NumElts && in resolveTargetShuffleFromZeroables()
8130 KnownZero.getBitWidth() == NumElts && "Shuffle mask size mismatch"); in resolveTargetShuffleFromZeroables()
8132 for (unsigned i = 0; i != NumElts; ++i) { in resolveTargetShuffleFromZeroables()
8144 unsigned NumElts = Mask.size(); in resolveZeroablesFromTargetShuffle() local
8145 KnownUndef = KnownZero = APInt::getZero(NumElts); in resolveZeroablesFromTargetShuffle()
8147 for (unsigned i = 0; i != NumElts; ++i) { in resolveZeroablesFromTargetShuffle()
8161 unsigned NumElts = CondVT.getVectorNumElements(); in createShuffleMaskFromVSELECT() local
8169 Mask.resize(NumElts, SM_SentinelUndef); in createShuffleMaskFromVSELECT()
8171 for (int i = 0; i != (int)NumElts; ++i) { in createShuffleMaskFromVSELECT()
8178 Mask[i] += NumElts; in createShuffleMaskFromVSELECT()
8203 unsigned NumElts = VT.getVectorNumElements(); in getFauxShuffleMask() local
8208 assert(NumElts == DemandedElts.getBitWidth() && "Unexpected vector size"); in getFauxShuffleMask()
8217 if (isUndefOrInRange(ShuffleMask, 0, 2 * NumElts)) { in getFauxShuffleMask()
8298 for (int i = 0; i != (int)NumElts; ++i) in getFauxShuffleMask()
8301 Mask[InsertIdx + i] = NumElts + ExtractIdx + i; in getFauxShuffleMask()
8331 NumElts *= Scale; in getFauxShuffleMask()
8338 Mask.append(NumElts, SM_SentinelZero); in getFauxShuffleMask()
8340 for (int i = 0; i != (int)NumElts; ++i) in getFauxShuffleMask()
8346 M = (NumElts * (1 + InputIdx)) + (M % NumSubElts); in getFauxShuffleMask()
8364 N.getConstantOperandAPInt(2).uge(NumElts)) in getFauxShuffleMask()
8371 for (unsigned i = 0; i != NumElts; ++i) in getFauxShuffleMask()
8435 assert(N0.getValueType().getVectorNumElements() == (NumElts / 2) && in getFauxShuffleMask()
8436 N1.getValueType().getVectorNumElements() == (NumElts / 2) && in getFauxShuffleMask()
8483 if ((Offset0 && isInRange(M, 0, NumElts)) || in getFauxShuffleMask()
8484 (Offset1 && isInRange(M, NumElts, 2 * NumElts))) in getFauxShuffleMask()
8512 Mask.append(NumElts - NumSrcElts, SM_SentinelZero); in getFauxShuffleMask()
8521 Mask.append(NumElts, SM_SentinelZero); in getFauxShuffleMask()
8555 for (int i = 0; i != (int)NumElts; ++i) { in getFauxShuffleMask()
8574 Mask.append(NumElts, 0); in getFauxShuffleMask()
8591 DecodeZeroExtendMask(SrcVT.getScalarSizeInBits(), NumBitsPerElt, NumElts, in getFauxShuffleMask()
8681 unsigned NumElts = Op.getValueType().getVectorNumElements(); in getTargetShuffleInputs() local
8682 APInt DemandedElts = APInt::getAllOnes(NumElts); in getTargetShuffleInputs()
8820 unsigned NumElts = VT.getVectorNumElements(); in LowerBuildVectorAsInsert() local
8829 for (unsigned i = 0; i < NumElts; ++i) { in LowerBuildVectorAsInsert()
9561 unsigned NumElts = VT.getVectorNumElements(); in lowerBuildVectorAsBroadcast() local
9572 assert((NumElts % Sequence.size()) == 0 && "Sequence doesn't fit."); in lowerBuildVectorAsBroadcast()
9602 MVT BcstVT = MVT::getVectorVT(EltType, NumElts / SeqLen); in lowerBuildVectorAsBroadcast()
9605 BcstVT = MVT::getVectorVT(EltType, Scale * (NumElts / SeqLen)); in lowerBuildVectorAsBroadcast()
9616 if (!Ld || (NumElts - NumUndefElts) <= 1) { in lowerBuildVectorAsBroadcast()
9678 if (!Ld || NumElts - NumUndefElts != 1) in lowerBuildVectorAsBroadcast()
9757 if (!Ld->hasNUsesOfValue(NumElts - NumUndefElts, 0)) in lowerBuildVectorAsBroadcast()
10040 unsigned NumElts = LastIdx - BaseIdx; in isHorizontalBinOpPart() local
10045 for (unsigned i = 0, e = NumElts; i != e && CanFold; ++i) { in isHorizontalBinOpPart()
10051 if (i * 2 == NumElts) in isHorizontalBinOpPart()
10078 if (i * 2 < NumElts) { in isHorizontalBinOpPart()
10090 if (i * 2 == NumElts) in isHorizontalBinOpPart()
10094 SDValue Expected = (i * 2 < NumElts) ? V0 : V1; in isHorizontalBinOpPart()
10149 unsigned NumElts = VT.getVectorNumElements(); in ExpandHorizontalBinOp() local
10151 SDValue V0_HI = extract128BitVector(V0, NumElts/2, DAG, DL); in ExpandHorizontalBinOp()
10153 SDValue V1_HI = extract128BitVector(V1, NumElts/2, DAG, DL); in ExpandHorizontalBinOp()
10192 unsigned NumElts = VT.getVectorNumElements(); in isAddSubOrSubAdd() local
10203 for (unsigned i = 0, e = NumElts; i != e; ++i) { in isAddSubOrSubAdd()
10388 unsigned NumElts = VT.getVectorNumElements(); in isHopBuildVector() local
10391 unsigned NumEltsIn128Bits = NumElts / Num128BitChunks; in isHopBuildVector()
10482 unsigned NumElts = VT.getVectorNumElements(); in getHopForBuildVector() local
10483 APInt DemandedElts = APInt::getAllOnes(NumElts); in getHopForBuildVector()
10484 for (unsigned i = 0; i != NumElts; ++i) in getHopForBuildVector()
10489 unsigned HalfNumElts = NumElts / 2; in getHopForBuildVector()
10530 unsigned NumElts = VT.getVectorNumElements(); in LowerToHorizontalOp() local
10531 unsigned Half = NumElts / 2; in LowerToHorizontalOp()
10538 for (unsigned i = Half, e = NumElts; i != e; ++i) in LowerToHorizontalOp()
10550 isHorizontalBinOpPart(BV, ISD::ADD, DAG, Half, NumElts, InVec2, in LowerToHorizontalOp()
10557 isHorizontalBinOpPart(BV, ISD::SUB, DAG, Half, NumElts, InVec2, in LowerToHorizontalOp()
10587 if (isHorizontalBinOpPart(BV, ISD::ADD, DAG, 0, NumElts, InVec0, InVec1)) in LowerToHorizontalOp()
10589 else if (isHorizontalBinOpPart(BV, ISD::SUB, DAG, 0, NumElts, InVec0, in LowerToHorizontalOp()
10592 else if (isHorizontalBinOpPart(BV, ISD::FADD, DAG, 0, NumElts, InVec0, in LowerToHorizontalOp()
10595 else if (isHorizontalBinOpPart(BV, ISD::FSUB, DAG, 0, NumElts, InVec0, in LowerToHorizontalOp()
10734 unsigned NumElts = VT.getVectorNumElements(); in createVariablePermute() local
10738 assert(IndicesVec.getValueType().getVectorNumElements() >= NumElts && in createVariablePermute()
10740 if (IndicesVec.getValueType().getVectorNumElements() > NumElts) { in createVariablePermute()
10744 NumElts * VT.getScalarSizeInBits()); in createVariablePermute()
10749 if (IndicesVec.getValueType().getVectorNumElements() > NumElts) in createVariablePermute()
10760 VT = MVT::getVectorVT(VT.getScalarType(), Scale * NumElts); in createVariablePermute()
11174 unsigned NumElts = VT.getVectorNumElements(); in LowerBUILD_VECTOR() local
11175 for (unsigned i = 0; i != NumElts; ++i) in LowerBUILD_VECTOR()
11176 ShuffleMask.push_back(i == InsertC ? NumElts : i); in LowerBUILD_VECTOR()
11653 int NumElts = Mask.size(); in isMultiLaneShuffleMask() local
11655 int NumLanes = NumElts / NumEltsPerLane; in isMultiLaneShuffleMask()
11663 int Lane = (M % NumElts) / NumEltsPerLane; in isMultiLaneShuffleMask()
11812 int NumElts = VT.getVectorNumElements(); in IsElementEquivalent() local
11813 if (MaskSize == NumElts) { in IsElementEquivalent()
11815 int NumEltsPerLane = NumElts / NumLanes; in IsElementEquivalent()
12123 unsigned NumElts = VT.getVectorNumElements(); in lowerShuffleToEXPAND() local
12124 assert((NumElts == 4 || NumElts == 8 || NumElts == 16) && in lowerShuffleToEXPAND()
12126 SDValue VMask = getMaskNode(MaskNode, MVT::getVectorVT(MVT::i1, NumElts), in lowerShuffleToEXPAND()
12138 int NumElts = VT.getVectorNumElements(); in matchShuffleWithUNPCK() local
12141 for (int i = 0; i != NumElts; i += 2) { in matchShuffleWithUNPCK()
12176 isSequentialOrUndefOrZeroInRange(TargetMask, 0, NumElts, 0)) in matchShuffleWithUNPCK()
12180 for (int i = 0; (i != NumElts) && (MatchLo || MatchHi); ++i) { in matchShuffleWithUNPCK()
12281 unsigned NumElts = Mask.size(); in matchShuffleAsVTRUNC() local
12289 unsigned NumSrcElts = NumElts / Scale; in matchShuffleAsVTRUNC()
12292 unsigned UpperElts = NumElts - NumSrcElts; in matchShuffleAsVTRUNC()
12383 unsigned NumElts = VT.getVectorNumElements(); in lowerShuffleWithVPMOV() local
12387 unsigned NumSrcElts = NumElts / Scale; in lowerShuffleWithVPMOV()
12388 unsigned UpperElts = NumElts - NumSrcElts; in lowerShuffleWithVPMOV()
12427 unsigned NumElts = VT.getVectorNumElements(); in lowerShuffleAsVTRUNC() local
12438 unsigned NumHalfSrcElts = NumElts / Scale; in lowerShuffleAsVTRUNC()
12445 unsigned UpperElts = NumElts - NumSrcElts; in lowerShuffleAsVTRUNC()
12454 MVT ConcatVT = MVT::getVectorVT(VT.getScalarType(), NumElts * 2); in lowerShuffleAsVTRUNC()
12547 unsigned NumElts = VT.getVectorNumElements(); in matchShuffleWithPACK() local
12593 MVT PackVT = MVT::getVectorVT(PackSVT, NumElts >> NumStages); in matchShuffleWithPACK()
12824 unsigned NumElts = VT.getVectorNumElements(); in lowerShuffleAsBlend() local
12884 MVT IntegerType = MVT::getIntegerVT(std::max<unsigned>(NumElts, 8)); in lowerShuffleAsBlend()
12953 MVT IntegerType = MVT::getIntegerVT(std::max<unsigned>(NumElts, 8)); in lowerShuffleAsBlend()
13010 int NumElts = Mask.size(); in lowerShuffleAsUNPCKAndPermute() local
13012 int NumLaneElts = NumElts / NumLanes; in lowerShuffleAsUNPCKAndPermute()
13019 for (int Lane = 0; Lane != NumElts; Lane += NumLaneElts) { in lowerShuffleAsUNPCKAndPermute()
13026 if (M < NumElts && (Op.isUndef() || Op == V1)) in lowerShuffleAsUNPCKAndPermute()
13028 else if (NumElts <= M && (Op.isUndef() || Op == V2)) in lowerShuffleAsUNPCKAndPermute()
13035 isUndefOrInRange(M, NumElts + Lo, NumElts + Mid); in lowerShuffleAsUNPCKAndPermute()
13037 isUndefOrInRange(M, NumElts + Mid, NumElts + Hi); in lowerShuffleAsUNPCKAndPermute()
13047 SmallVector<int, 32> PermuteMask(NumElts, -1); in lowerShuffleAsUNPCKAndPermute()
13048 for (int Lane = 0; Lane != NumElts; Lane += NumLaneElts) { in lowerShuffleAsUNPCKAndPermute()
13083 int NumElts = VT.getVectorNumElements(); in lowerShuffleAsByteRotateAndPermute() local
13084 int NumEltsPerLane = NumElts / NumLanes; in lowerShuffleAsByteRotateAndPermute()
13091 for (int Lane = 0; Lane != NumElts; Lane += NumEltsPerLane) { in lowerShuffleAsByteRotateAndPermute()
13096 if (M < NumElts) { in lowerShuffleAsByteRotateAndPermute()
13103 M -= NumElts; in lowerShuffleAsByteRotateAndPermute()
13130 SmallVector<int, 64> PermMask(NumElts, SM_SentinelUndef); in lowerShuffleAsByteRotateAndPermute()
13131 for (int Lane = 0; Lane != NumElts; Lane += NumEltsPerLane) { in lowerShuffleAsByteRotateAndPermute()
13136 if (M < NumElts) in lowerShuffleAsByteRotateAndPermute()
13149 return RotateAndPermute(V2, V1, Range2.first, NumElts); in lowerShuffleAsByteRotateAndPermute()
13171 int NumElts = Mask.size(); in lowerShuffleAsDecomposedShuffleMerge() local
13173 int NumEltsPerLane = NumElts / NumLanes; in lowerShuffleAsDecomposedShuffleMerge()
13178 SmallVector<int, 32> V1Mask(NumElts, -1); in lowerShuffleAsDecomposedShuffleMerge()
13179 SmallVector<int, 32> V2Mask(NumElts, -1); in lowerShuffleAsDecomposedShuffleMerge()
13180 SmallVector<int, 32> FinalMask(NumElts, -1); in lowerShuffleAsDecomposedShuffleMerge()
13181 for (int i = 0; i < NumElts; ++i) { in lowerShuffleAsDecomposedShuffleMerge()
13183 if (M >= 0 && M < NumElts) { in lowerShuffleAsDecomposedShuffleMerge()
13187 } else if (M >= NumElts) { in lowerShuffleAsDecomposedShuffleMerge()
13188 V2Mask[i] = M - NumElts; in lowerShuffleAsDecomposedShuffleMerge()
13189 FinalMask[i] = i + NumElts; in lowerShuffleAsDecomposedShuffleMerge()
13253 V1Mask.assign(NumElts, -1); in lowerShuffleAsDecomposedShuffleMerge()
13254 V2Mask.assign(NumElts, -1); in lowerShuffleAsDecomposedShuffleMerge()
13255 FinalMask.assign(NumElts, -1); in lowerShuffleAsDecomposedShuffleMerge()
13256 for (int i = 0; i != NumElts; i += NumEltsPerLane) in lowerShuffleAsDecomposedShuffleMerge()
13259 if (M >= 0 && M < NumElts) { in lowerShuffleAsDecomposedShuffleMerge()
13262 } else if (M >= NumElts) { in lowerShuffleAsDecomposedShuffleMerge()
13263 V2Mask[i + (j / 2)] = M - NumElts; in lowerShuffleAsDecomposedShuffleMerge()
13264 FinalMask[i + j] = i + (j / 2) + NumElts; in lowerShuffleAsDecomposedShuffleMerge()
13279 int NumElts = Mask.size(); in matchShuffleAsBitRotate() local
13280 assert((NumElts % NumSubElts) == 0 && "Illegal shuffle mask"); in matchShuffleAsBitRotate()
13283 for (int i = 0; i != NumElts; i += NumSubElts) { in matchShuffleAsBitRotate()
13313 int NumElts = Mask.size(); in matchShuffleAsBitRotate() local
13315 RotateVT = MVT::getVectorVT(RotateSVT, NumElts / NumSubElts); in matchShuffleAsBitRotate()
13369 int NumElts = Mask.size(); in matchShuffleAsElementRotate() local
13380 for (int i = 0; i < NumElts; ++i) { in matchShuffleAsElementRotate()
13382 assert((M == SM_SentinelUndef || (0 <= M && M < (2*NumElts))) && in matchShuffleAsElementRotate()
13388 int StartIdx = i - (M % NumElts); in matchShuffleAsElementRotate()
13396 int CandidateRotation = StartIdx < 0 ? -StartIdx : NumElts - StartIdx; in matchShuffleAsElementRotate()
13405 SDValue MaskV = M < NumElts ? V1 : V2; in matchShuffleAsElementRotate()
13469 int NumElts = RepeatedMask.size(); in matchShuffleAsByteRotate() local
13470 int Scale = 16 / NumElts; in matchShuffleAsByteRotate()
13567 unsigned NumElts = Mask.size(); in lowerShuffleAsByteShiftMask() local
13568 unsigned Len = NumElts - (ZeroLo + ZeroHi); in lowerShuffleAsByteShiftMask()
13574 if (!isUndefOrInRange(StubMask, 0, NumElts) && in lowerShuffleAsByteShiftMask()
13575 !isUndefOrInRange(StubMask, NumElts, 2 * NumElts)) in lowerShuffleAsByteShiftMask()
13578 SDValue Res = Mask[ZeroLo] < (int)NumElts ? V1 : V2; in lowerShuffleAsByteShiftMask()
13587 unsigned Shift = (NumElts - 1) - (Mask[ZeroLo + Len - 1] % NumElts); in lowerShuffleAsByteShiftMask()
13593 unsigned Shift = Mask[ZeroLo] % NumElts; in lowerShuffleAsByteShiftMask()
13602 unsigned Shift = (NumElts - 1) - (Mask[ZeroLo + Len - 1] % NumElts); in lowerShuffleAsByteShiftMask()
13605 Shift += Mask[ZeroLo] % NumElts; in lowerShuffleAsByteShiftMask()
14417 unsigned NumElts = VT.getVectorNumElements(); in lowerShuffleOfExtractsAsVperm() local
14421 if (ExtIndex1 == 0 && ExtIndex0 == NumElts) in lowerShuffleOfExtractsAsVperm()
14423 else if (ExtIndex0 != 0 || ExtIndex1 != NumElts) in lowerShuffleOfExtractsAsVperm()
14429 if (NumElts == 4 && in lowerShuffleOfExtractsAsVperm()
14434 NewMask.append(NumElts, -1); in lowerShuffleOfExtractsAsVperm()
16160 int NumElts = VT.getVectorNumElements(); in lowerShuffleWithPERMV() local
16164 if (NumElts <= M) in lowerShuffleWithPERMV()
16165 M += (Scale - 1) * NumElts; in lowerShuffleWithPERMV()
16771 int NumElts = VT.getVectorNumElements(); in lowerShuffleAsLanePermuteAndPermute() local
16773 int NumEltsPerLane = NumElts / NumLanes; in lowerShuffleAsLanePermuteAndPermute()
16783 int NumEltsPerSublane = NumElts / NumSublanes; in lowerShuffleAsLanePermuteAndPermute()
16786 SmallVector<int, 16> InLaneMask(NumElts, SM_SentinelUndef); in lowerShuffleAsLanePermuteAndPermute()
16790 for (int i = 0; i != NumElts; ++i) { in lowerShuffleAsLanePermuteAndPermute()
17062 int NumElts = Mask.size(); in lowerShuffleAsLanePermuteAndRepeatedMask() local
17091 InLaneMask[i] = (M % NumLaneElts) + Src * NumElts; in lowerShuffleAsLanePermuteAndRepeatedMask()
17156 if (RepeatMask[i] < NumElts) { in lowerShuffleAsLanePermuteAndRepeatedMask()
17161 if (RepeatMask[i] != ((M % NumLaneElts) + NumElts)) in lowerShuffleAsLanePermuteAndRepeatedMask()
17171 SmallVector<int, 16> NewMask(NumElts, -1); in lowerShuffleAsLanePermuteAndRepeatedMask()
17206 for (int i = 0; i != NumElts; ++i) { in lowerShuffleAsLanePermuteAndRepeatedMask()
17443 int NumElts = VT.getVectorNumElements(); in lowerShuffleAsRepeatedMaskAndLanePermute() local
17445 int NumLaneElts = NumElts / NumLanes; in lowerShuffleAsRepeatedMaskAndLanePermute()
17459 for (int i = 0; i != NumElts; i += NumBroadcastElts) in lowerShuffleAsRepeatedMaskAndLanePermute()
17465 if (0 != ((M % NumElts) / NumLaneElts)) in lowerShuffleAsRepeatedMaskAndLanePermute()
17474 SmallVector<int, 8> RepeatMask((unsigned)NumElts, -1); in lowerShuffleAsRepeatedMaskAndLanePermute()
17482 SmallVector<int, 8> BroadcastMask((unsigned)NumElts, -1); in lowerShuffleAsRepeatedMaskAndLanePermute()
17483 for (int i = 0; i != NumElts; i += NumBroadcastElts) in lowerShuffleAsRepeatedMaskAndLanePermute()
17523 int Lane = (M % NumElts) / NumLaneElts; in lowerShuffleAsRepeatedMaskAndLanePermute()
17527 int LocalM = (M % NumLaneElts) + (M < NumElts ? 0 : NumElts); in lowerShuffleAsRepeatedMaskAndLanePermute()
17577 SmallVector<int, 8> RepeatedMask((unsigned)NumElts, -1); in lowerShuffleAsRepeatedMaskAndLanePermute()
17593 SmallVector<int, 8> SubLaneMask((unsigned)NumElts, -1); in lowerShuffleAsRepeatedMaskAndLanePermute()
17594 for (int i = 0; i != NumElts; i += NumSubLaneElts) { in lowerShuffleAsRepeatedMaskAndLanePermute()
17631 int NumElts = VT.getVectorNumElements(); in matchShuffleWithSHUFPD() local
17633 (NumElts == 2 || NumElts == 4 || NumElts == 8) && in matchShuffleWithSHUFPD()
17635 assert(isUndefOrZeroOrInRange(Mask, 0, 2 * NumElts) && in matchShuffleWithSHUFPD()
17639 for (int i = 0; i < NumElts; ++i) in matchShuffleWithSHUFPD()
17647 for (int i = 0; i < NumElts; ++i) { in matchShuffleWithSHUFPD()
17652 int Val = (i & 6) + NumElts * (i & 1); in matchShuffleWithSHUFPD()
17653 int CommutVal = (i & 0xe) + NumElts * ((i & 1) ^ 1); in matchShuffleWithSHUFPD()
18410 int NumElts = VT.getVectorNumElements(); in lower256BitShuffle() local
18411 int NumV2Elements = count_if(Mask, [NumElts](int M) { return M >= NumElts; }); in lower256BitShuffle()
18413 if (NumV2Elements == 1 && Mask[0] >= NumElts) in lower256BitShuffle()
18496 unsigned NumElts = ((Zeroable & 0x0c) == 0x0c) ? 2 : 4; in lowerV4X128Shuffle() local
18497 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), NumElts); in lowerV4X128Shuffle()
19009 int NumElts = Mask.size(); in lower512BitShuffle() local
19010 int NumV2Elements = count_if(Mask, [NumElts](int M) { return M >= NumElts; }); in lower512BitShuffle()
19012 if (NumV2Elements == 1 && Mask[0] >= NumElts) in lower512BitShuffle()
19078 int NumElts = Mask.size(); in lower1BitShuffleAsKSHIFTR() local
19079 for (int i = 0; i != NumElts; ++i) { in lower1BitShuffleAsKSHIFTR()
19081 assert((M == SM_SentinelUndef || (0 <= M && M < NumElts)) && in lower1BitShuffleAsKSHIFTR()
19101 if ((!Subtarget.hasDQI() && NumElts == 8) || NumElts < 8) in lower1BitShuffleAsKSHIFTR()
19157 int NumElts = Mask.size(); in lower1BitShuffle() local
19162 for (int i = 0; i != NumElts; ++i) { in lower1BitShuffle()
19167 Src = Mask[i] / NumElts; in lower1BitShuffle()
19168 if (Src != (Mask[i] / NumElts) || (Mask[i] % NumElts) != i) in lower1BitShuffle()
19174 assert(SubvecElts != NumElts && "Identity shuffle?"); in lower1BitShuffle()
19181 if ((int)Zeroable.countLeadingOnes() >= (NumElts - SubvecElts)) { in lower1BitShuffle()
19204 if ((!Subtarget.hasDQI() && NumElts == 8) || NumElts < 8) in lower1BitShuffle()
19214 DAG.getTargetConstant(WideElts - NumElts, DL, MVT::i8)); in lower1BitShuffle()
19216 ShiftAmt += WideElts - NumElts; in lower1BitShuffle()
19224 Offset += NumElts; // Increment for next iteration. in lower1BitShuffle()
19563 unsigned NumElts = VT.getVectorNumElements(); in LowerVSELECT() local
19574 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts); in LowerVSELECT()
19589 MVT NewCondVT = MVT::getVectorVT(NewCondSVT, NumElts); in LowerVSELECT()
19612 MVT CastVT = MVT::getVectorVT(MVT::i8, NumElts * 2); in LowerVSELECT()
19688 unsigned NumElts = VecVT.getVectorNumElements(); in ExtractBitFromMaskVector() local
19691 MVT ExtEltVT = (NumElts <= 8) ? MVT::getIntegerVT(128 / NumElts) : MVT::i8; in ExtractBitFromMaskVector()
19692 MVT ExtVecVT = MVT::getVectorVT(ExtEltVT, NumElts); in ExtractBitFromMaskVector()
19883 unsigned NumElts = VecVT.getVectorNumElements(); in InsertBitToMaskVector() local
19884 MVT ExtEltVT = (NumElts <= 8) ? MVT::getIntegerVT(128 / NumElts) : MVT::i8; in InsertBitToMaskVector()
19885 MVT ExtVecVT = MVT::getVectorVT(ExtEltVT, NumElts); in InsertBitToMaskVector()
19901 unsigned NumElts = VT.getVectorNumElements(); in LowerINSERT_VECTOR_ELT() local
19923 MVT IdxVT = MVT::getVectorVT(IdxSVT, NumElts); in LowerINSERT_VECTOR_ELT()
19932 for (unsigned I = 0; I != NumElts; ++I) in LowerINSERT_VECTOR_ELT()
19941 if (N2C->getAPIntValue().uge(NumElts)) in LowerINSERT_VECTOR_ELT()
19956 SmallVector<SDValue, 8> CstVectorElts(NumElts, ZeroCst); in LowerINSERT_VECTOR_ELT()
19966 for (unsigned i = 0; i != NumElts; ++i) in LowerINSERT_VECTOR_ELT()
19967 BlendMask.push_back(i == IdxVal ? i + NumElts : i); in LowerINSERT_VECTOR_ELT()
20004 for (unsigned i = 0; i != NumElts; ++i) in LowerINSERT_VECTOR_ELT()
20005 BlendMask.push_back(i == IdxVal ? i + NumElts : i); in LowerINSERT_VECTOR_ELT()
20688 unsigned NumElts = Subtarget.hasVLX() ? 4 : 8; in LowerI64IntToFP_AVX512DQ() local
20689 MVT VecInVT = MVT::getVectorVT(MVT::i64, NumElts); in LowerI64IntToFP_AVX512DQ()
20690 MVT VecVT = MVT::getVectorVT(VT, NumElts); in LowerI64IntToFP_AVX512DQ()
21876 unsigned NumElts = VT.getVectorNumElements(); in LowerZERO_EXTEND_Mask() local
21890 if (NumElts == 16 && !Subtarget.canExtendTo512DQ()) in LowerZERO_EXTEND_Mask()
21893 ExtVT = MVT::getVectorVT(MVT::i32, NumElts); in LowerZERO_EXTEND_Mask()
21899 NumElts *= 512 / ExtVT.getSizeInBits(); in LowerZERO_EXTEND_Mask()
21900 InVT = MVT::getVectorVT(MVT::i1, NumElts); in LowerZERO_EXTEND_Mask()
21904 NumElts); in LowerZERO_EXTEND_Mask()
21914 WideVT = MVT::getVectorVT(MVT::i8, NumElts); in LowerZERO_EXTEND_Mask()
22077 unsigned NumElts = InVT.getVectorNumElements(); in LowerTruncateVecI1() local
22078 assert((NumElts == 8 || NumElts == 16) && "Unexpected number of elements"); in LowerTruncateVecI1()
22088 if (NumElts == 16 && !Subtarget.canExtendTo512DQ()) { in LowerTruncateVecI1()
22110 MVT EltVT = Subtarget.hasVLX() ? MVT::i32 : MVT::getIntegerVT(512/NumElts); in LowerTruncateVecI1()
22111 MVT ExtVT = MVT::getVectorVT(EltVT, NumElts); in LowerTruncateVecI1()
23427 unsigned NumElts = VT.getVectorNumElements(); in matchScalarReduction() local
23428 APInt EltCount = APInt::getZero(NumElts); in matchScalarReduction()
24195 unsigned NumElts = VT.getVectorNumElements(); in incDecVectorConstant() local
24198 for (unsigned i = 0; i < NumElts; ++i) { in incDecVectorConstant()
25355 unsigned NumElts = VT.getVectorNumElements(); in LowerSIGN_EXTEND_Mask() local
25361 if (NumElts == 16 && !Subtarget.canExtendTo512DQ()) in LowerSIGN_EXTEND_Mask()
25364 ExtVT = MVT::getVectorVT(MVT::i32, NumElts); in LowerSIGN_EXTEND_Mask()
25370 NumElts *= 512 / ExtVT.getSizeInBits(); in LowerSIGN_EXTEND_Mask()
25371 InVT = MVT::getVectorVT(MVT::i1, NumElts); in LowerSIGN_EXTEND_Mask()
25374 WideVT = MVT::getVectorVT(ExtVT.getVectorElementType(), NumElts); in LowerSIGN_EXTEND_Mask()
25390 WideVT = MVT::getVectorVT(VTElt, NumElts); in LowerSIGN_EXTEND_Mask()
25440 unsigned NumElts = VT.getVectorNumElements(); in LowerEXTEND_VECTOR_INREG() local
25447 int InSize = InSVT.getSizeInBits() * NumElts; in LowerEXTEND_VECTOR_INREG()
25458 if (InVT.getVectorNumElements() != NumElts) in LowerEXTEND_VECTOR_INREG()
25660 unsigned NumElts = StoredVal.getValueType().getVectorNumElements(); in LowerStore() local
25661 assert(NumElts <= 8 && "Unexpected VT"); in LowerStore()
25673 if (NumElts < 8) in LowerStore()
25675 StoredVal, dl, EVT::getIntegerVT(*DAG.getContext(), NumElts)); in LowerStore()
28698 int NumElts = VT.getVectorNumElements(); in LowerVectorCTLZInRegLUT() local
28699 int NumBytes = NumElts * (VT.getScalarSizeInBits() / 8); in LowerVectorCTLZInRegLUT()
29028 unsigned NumElts = VT.getVectorNumElements(); in LowerMUL() local
29040 MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts / 2); in LowerMUL()
29054 for (unsigned i = 0; i != NumElts; i += 16) { in LowerMUL()
29158 unsigned NumElts = VT.getVectorNumElements(); in LowervXi8MulWithUNPCK() local
29172 MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts / 2); in LowervXi8MulWithUNPCK()
29188 for (unsigned i = 0; i != NumElts; i += 16) { in LowervXi8MulWithUNPCK()
29237 unsigned NumElts = VT.getVectorNumElements(); in LowerMULH() local
29269 makeArrayRef(&Mask[0], NumElts)); in LowerMULH()
29272 makeArrayRef(&Mask[0], NumElts)); in LowerMULH()
29276 MVT MulVT = MVT::getVectorVT(MVT::i64, NumElts / 2); in LowerMULH()
29291 SmallVector<int, 16> ShufMask(NumElts); in LowerMULH()
29292 for (int i = 0; i != (int)NumElts; ++i) in LowerMULH()
29293 ShufMask[i] = (i / 2) * 2 + ((i % 2) * NumElts) + 1; in LowerMULH()
29326 MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts); in LowerMULH()
29386 unsigned NumElts = VT.getVectorNumElements(); in LowerMULO() local
29387 MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts); in LowerMULO()
29720 unsigned NumElts = VT.getVectorNumElements(); in LowerShiftByScalarImmediate() local
29721 MVT ShiftVT = MVT::getVectorVT(MVT::i16, NumElts / 2); in LowerShiftByScalarImmediate()
29802 unsigned NumElts = VT.getVectorNumElements(); in LowerShiftByScalarVariable() local
29803 MVT ExtVT = MVT::getVectorVT(MVT::i16, NumElts / 2); in LowerShiftByScalarVariable()
29818 SmallVector<int, 64>(NumElts, 0)); in LowerShiftByScalarVariable()
29978 unsigned NumElts = VT.getVectorNumElements(); in LowerShift() local
29980 for (unsigned i = 0; i != NumElts; ++i) { in LowerShift()
29992 ShuffleMask.push_back(i + NumElts); in LowerShift()
30000 if (ShuffleMask.size() == NumElts && Amt1 && Amt2 && in LowerShift()
30144 int NumElts = VT.getVectorNumElements(); in LowerShift() local
30149 MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts); in LowerShift()
30165 for (int i = 0; i != NumElts; i += 16) { in LowerShift()
30172 MVT VT16 = MVT::getVectorVT(MVT::i16, NumElts / 2); in LowerShift()
30437 unsigned NumElts = VT.getVectorNumElements(); in LowerFunnelShift() local
30439 MVT ExtVT = MVT::getVectorVT(ExtSVT, NumElts / 2); in LowerFunnelShift()
30472 MVT WideVT = MVT::getVectorVT(WideSVT, NumElts); in LowerFunnelShift()
30564 int NumElts = VT.getVectorNumElements(); in LowerRotate() local
30647 MVT ExtVT = MVT::getVectorVT(ExtSVT, NumElts / 2); in LowerRotate()
30679 MVT::getVectorVT(Subtarget.hasBWI() ? MVT::i16 : MVT::i32, NumElts); in LowerRotate()
31364 int NumElts = VT.getVectorNumElements(); in LowerVectorCTPOPInRegLUT() local
31384 for (int i = 0; i < NumElts; ++i) in LowerVectorCTPOPInRegLUT()
31472 int NumElts = VT.getVectorNumElements(); in LowerBITREVERSE_XOP() local
31487 for (int i = 0; i != NumElts; ++i) { in LowerBITREVERSE_XOP()
31525 unsigned NumElts = VT.getVectorNumElements(); in LowerBITREVERSE() local
31529 MVT MatrixVT = MVT::getVectorVT(MVT::i64, NumElts / 8); in LowerBITREVERSE()
31555 for (unsigned i = 0; i < NumElts; ++i) { in LowerBITREVERSE()
31980 unsigned NumElts = VT.getVectorNumElements() * Factor; in LowerMSCATTER() local
31982 VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts); in LowerMSCATTER()
31983 IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(), NumElts); in LowerMSCATTER()
31984 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts); in LowerMSCATTER()
32133 unsigned NumElts = VT.getVectorNumElements() * Factor; in LowerMGATHER() local
32135 VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts); in LowerMGATHER()
32136 IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(), NumElts); in LowerMGATHER()
32137 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts); in LowerMGATHER()
32685 unsigned NumElts = InVT.getVectorNumElements(); in ReplaceNodeResults() local
32686 unsigned HalfNumElts = NumElts / 2; in ReplaceNodeResults()
32687 SmallVector<int, 16> ShufMask(NumElts, SM_SentinelUndef); in ReplaceNodeResults()
32894 unsigned NumElts = Subtarget.hasVLX() ? 2 : 8; in ReplaceNodeResults() local
32897 std::max(NumElts, 128U / (unsigned)SrcVT.getSizeInBits()); in ReplaceNodeResults()
32898 MVT VecVT = MVT::getVectorVT(MVT::i64, NumElts); in ReplaceNodeResults()
32901 if (NumElts != SrcElts) { in ReplaceNodeResults()
36764 unsigned NumElts = DemandedElts.getBitWidth(); in computeKnownBitsForTargetNode() local
36975 if (NumElts > NumSrcElts && in computeKnownBitsForTargetNode()
36991 if (NumElts > NumSrcElts && in computeKnownBitsForTargetNode()
36999 if (DemandedElts.countTrailingZeros() >= (NumElts / 2)) in computeKnownBitsForTargetNode()
37011 for (unsigned I = 0; I != NumElts; ++I) { in computeKnownBitsForTargetNode()
37034 unsigned NumElts = VT.getVectorNumElements(); in computeKnownBitsForTargetNode() local
37035 if (Mask.size() == NumElts) { in computeKnownBitsForTargetNode()
37036 SmallVector<APInt, 2> DemandedOps(NumOps, APInt(NumElts, 0)); in computeKnownBitsForTargetNode()
37038 for (unsigned i = 0; i != NumElts; ++i) { in computeKnownBitsForTargetNode()
37052 assert(0 <= M && (unsigned)M < (NumOps * NumElts) && in computeKnownBitsForTargetNode()
37055 unsigned OpIdx = (unsigned)M / NumElts; in computeKnownBitsForTargetNode()
37056 unsigned EltIdx = (unsigned)M % NumElts; in computeKnownBitsForTargetNode()
37185 unsigned NumElts = VT.getVectorNumElements(); in ComputeNumSignBitsForTargetNode() local
37186 if (Mask.size() == NumElts) { in ComputeNumSignBitsForTargetNode()
37187 SmallVector<APInt, 2> DemandedOps(NumOps, APInt(NumElts, 0)); in ComputeNumSignBitsForTargetNode()
37188 for (unsigned i = 0; i != NumElts; ++i) { in ComputeNumSignBitsForTargetNode()
37200 assert(0 <= M && (unsigned)M < (NumOps * NumElts) && in ComputeNumSignBitsForTargetNode()
37203 unsigned OpIdx = (unsigned)M / NumElts; in ComputeNumSignBitsForTargetNode()
37204 unsigned EltIdx = (unsigned)M % NumElts; in ComputeNumSignBitsForTargetNode()
37653 unsigned NumElts = V.getValueType().getVectorNumElements(); in matchBinaryShuffle() local
37654 KnownBits Known(NumElts); in matchBinaryShuffle()
37655 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) { in matchBinaryShuffle()
37656 APInt Mask = APInt::getOneBitSet(NumElts, EltIdx); in matchBinaryShuffle()
37954 unsigned NumElts = VT1.getVectorNumElements(); in combineX86ShuffleChain() local
37955 if (Mask.size() <= NumElts && in combineX86ShuffleChain()
37956 scaleShuffleElements(Mask, NumElts, ScaledMask)) { in combineX86ShuffleChain()
37957 for (unsigned i = 0; i != NumElts; ++i) in combineX86ShuffleChain()
38760 int NumElts = VT0.getVectorNumElements(); in canonicalizeShuffleMaskWithHorizOp() local
38762 int NumEltsPerLane = NumElts / NumLanes; in canonicalizeShuffleMaskWithHorizOp()
38862 if (M < NumElts) // BC0 element or UNDEF/Zero sentinel. in canonicalizeShuffleMaskWithHorizOp()
38865 M -= NumElts + (SubLane * NumHalfEltsPerLane); in canonicalizeShuffleMaskWithHorizOp()
38873 for (int i = 0; i != NumElts; ++i) { in canonicalizeShuffleMaskWithHorizOp()
38877 if (M < NumElts && BC0.getOperand(0) == BC0.getOperand(1) && in canonicalizeShuffleMaskWithHorizOp()
38880 if (NumElts <= M && BC1.getOperand(0) == BC1.getOperand(1) && in canonicalizeShuffleMaskWithHorizOp()
39097 unsigned NumElts = VT.getVectorNumElements(); in combineX86ShufflesRecursively() local
39099 OpMask.assign(NumElts, SM_SentinelUndef); in combineX86ShufflesRecursively()
39101 OpZero = OpUndef = APInt::getNullValue(NumElts); in combineX86ShufflesRecursively()
40134 unsigned NumElts = VT.getVectorNumElements(); in combineTargetShuffle() local
40136 SmallVector<Constant *, 32> ConstantVec(NumElts, Zero); in combineTargetShuffle()
40274 unsigned NumElts = Src.getValueType().getVectorNumElements(); in combineTargetShuffle() local
40277 Src.getConstantOperandAPInt(2) == (NumElts / 2)) { in combineTargetShuffle()
40729 int NumElts = VT.getVectorNumElements(); in combineShuffleOfConcatUndef() local
40733 Mask.push_back(Elt < NumElts ? Elt : (Elt - NumElts / 2)); in combineShuffleOfConcatUndef()
40839 unsigned NumElts = DemandedElts.getBitWidth(); in SimplifyDemandedVectorEltsForTargetShuffle() local
40872 if (NumCstElts != NumElts && NumCstElts != (NumElts * 2)) in SimplifyDemandedVectorEltsForTargetShuffle()
40874 unsigned Scale = NumCstElts / NumElts; in SimplifyDemandedVectorEltsForTargetShuffle()
40905 int NumElts = DemandedElts.getBitWidth(); in SimplifyDemandedVectorEltsForTargetNode() local
40932 APInt DemandedSrcElts = APIntOps::ScaleBitMask(DemandedElts, 2 * NumElts); in SimplifyDemandedVectorEltsForTargetNode()
41054 assert(Amt->getAPIntValue().ult(NumElts) && "Out of range shift amount"); in SimplifyDemandedVectorEltsForTargetNode()
41064 if (!DemandedElts.intersects(APInt::getLowBitsSet(NumElts, ShiftAmt))) { in SimplifyDemandedVectorEltsForTargetNode()
41093 assert(Amt->getAPIntValue().ult(NumElts) && "Out of range shift amount"); in SimplifyDemandedVectorEltsForTargetNode()
41103 if (!DemandedElts.intersects(APInt::getHighBitsSet(NumElts, ShiftAmt))) { in SimplifyDemandedVectorEltsForTargetNode()
41137 int NumElts = VT.getVectorNumElements(); in SimplifyDemandedVectorEltsForTargetNode() local
41145 for (int I = 0; I != NumElts; ++I) { in SimplifyDemandedVectorEltsForTargetNode()
41281 KnownZero = SrcZero.zextOrTrunc(NumElts); in SimplifyDemandedVectorEltsForTargetNode()
41282 KnownUndef = SrcUndef.zextOrTrunc(NumElts); in SimplifyDemandedVectorEltsForTargetNode()
41363 DemandedElts.lshr(NumElts / 2) == 0) { in SimplifyDemandedVectorEltsForTargetNode()
41368 if (VT.is512BitVector() && DemandedElts.lshr(NumElts / 4) == 0) in SimplifyDemandedVectorEltsForTargetNode()
41450 DecodeVPERMMask(NumElts, Op.getConstantOperandVal(1), Mask); in SimplifyDemandedVectorEltsForTargetNode()
41468 unsigned EltIdx = (LoMask & 0x1) * (NumElts / 2); in SimplifyDemandedVectorEltsForTargetNode()
41537 if (OpMask.size() != (unsigned)NumElts || in SimplifyDemandedVectorEltsForTargetNode()
41549 for (int i = 0; i != NumElts; ++i) in SimplifyDemandedVectorEltsForTargetNode()
41553 if (isUndefInRange(OpMask, 0, NumElts)) { in SimplifyDemandedVectorEltsForTargetNode()
41557 if (isUndefOrZeroInRange(OpMask, 0, NumElts)) { in SimplifyDemandedVectorEltsForTargetNode()
41563 if (isSequentialOrUndefInRange(OpMask, 0, NumElts, Src * NumElts)) in SimplifyDemandedVectorEltsForTargetNode()
41572 int Lo = Src * NumElts; in SimplifyDemandedVectorEltsForTargetNode()
41573 APInt SrcElts = APInt::getZero(NumElts); in SimplifyDemandedVectorEltsForTargetNode()
41574 for (int i = 0; i != NumElts; ++i) in SimplifyDemandedVectorEltsForTargetNode()
41577 if (0 <= M && M < NumElts) in SimplifyDemandedVectorEltsForTargetNode()
41598 SmallVector<int, 64> DemandedMask(NumElts, SM_SentinelUndef); in SimplifyDemandedVectorEltsForTargetNode()
41599 for (int i = 0; i != NumElts; ++i) in SimplifyDemandedVectorEltsForTargetNode()
41946 unsigned NumElts = SrcVT.getVectorNumElements(); in SimplifyDemandedBitsForTargetNode() local
41949 if (OriginalDemandedBits.countTrailingZeros() >= NumElts) in SimplifyDemandedBitsForTargetNode()
41954 OriginalDemandedBits.getActiveBits() <= (NumElts / 2)) { in SimplifyDemandedBitsForTargetNode()
41961 APInt DemandedElts = OriginalDemandedBits.zextOrTrunc(NumElts); in SimplifyDemandedBitsForTargetNode()
41967 Known.Zero.setHighBits(BitWidth - NumElts); in SimplifyDemandedBitsForTargetNode()
41977 Known.One.setLowBits(NumElts); in SimplifyDemandedBitsForTargetNode()
41979 Known.Zero.setLowBits(NumElts); in SimplifyDemandedBitsForTargetNode()
42074 int NumElts = DemandedElts.getBitWidth(); in SimplifyMultipleUseDemandedBitsForTargetNode() local
42125 if (ShuffleMask.size() == (unsigned)NumElts && in SimplifyMultipleUseDemandedBitsForTargetNode()
42137 for (int i = 0; i != NumElts; ++i) { in SimplifyMultipleUseDemandedBitsForTargetNode()
42141 int OpIdx = M / NumElts; in SimplifyMultipleUseDemandedBitsForTargetNode()
42142 int EltIdx = M % NumElts; in SimplifyMultipleUseDemandedBitsForTargetNode()
42167 unsigned NumElts = DemandedElts.getBitWidth(); in isSplatValueForTargetNode() local
42173 UndefElts = APInt::getNullValue(NumElts); in isSplatValueForTargetNode()
42476 unsigned NumElts = BV->getNumOperands(); in createMMXBuildVector() local
42510 if (NumElts == 8) in createMMXBuildVector()
42518 unsigned ShufMask = (NumElts > 2 ? 0 : 0x44); in createMMXBuildVector()
42525 Ops.append(NumElts, Splat); in createMMXBuildVector()
42527 for (unsigned i = 0; i != NumElts; ++i) in createMMXBuildVector()
42820 unsigned NumElts = VT.getVectorNumElements(); in combineBitcast() local
42834 if (MovMskElts <= NumElts && in combineBitcast()
42847 unsigned NumConcats = NumElts / MovMskElts; in combineBitcast()
43137 unsigned NumElts = MatchVT.getVectorNumElements(); in combinePredicateReduction() local
43143 if (NumElts > 64 || !isPowerOf2_32(NumElts)) in combinePredicateReduction()
43147 EVT MovmskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts); in combinePredicateReduction()
43156 NumElts *= VecSVT.getSizeInBits() / 8; in combinePredicateReduction()
43157 EVT CmpVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, NumElts); in combinePredicateReduction()
43158 MatchVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElts); in combinePredicateReduction()
43166 while (NumElts > MaxElts) { in combinePredicateReduction()
43170 NumElts /= 2; in combinePredicateReduction()
43172 EVT MovmskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts); in combinePredicateReduction()
43177 Movmsk = DAG.getZExtOrTrunc(Movmsk, DL, NumElts > 32 ? MVT::i64 : MVT::i32); in combinePredicateReduction()
43213 NumElts = MaskSrcVT.getVectorNumElements(); in combinePredicateReduction()
43215 assert((NumElts <= 32 || NumElts == 64) && in combinePredicateReduction()
43218 MVT CmpVT = NumElts == 64 ? MVT::i64 : MVT::i32; in combinePredicateReduction()
43233 CmpC = DAG.getConstant(APInt::getLowBitsSet(CmpVT.getSizeInBits(), NumElts), in combinePredicateReduction()
43718 unsigned NumElts = VecVT.getVectorNumElements(); in combineArithReduction() local
43741 if (VT != MVT::i8 || NumElts < 4 || !isPowerOf2_32(NumElts)) in combineArithReduction()
43744 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts / 2); in combineArithReduction()
43759 if (NumElts >= 8) in combineArithReduction()
43783 if ((VecVT.getSizeInBits() % 128) != 0 || !isPowerOf2_32(NumElts)) in combineArithReduction()
43810 if (Opc == ISD::ADD && NumElts >= 4 && EltSizeInBits >= 16 && in combineArithReduction()
43838 if (NumElts > 8) { in combineArithReduction()
43860 unsigned NumElts = VecVT.getVectorNumElements(); in combineArithReduction() local
43861 SDValue Hi = extract128BitVector(Rdx, NumElts / 2, DAG, DL); in combineArithReduction()
44093 unsigned NumElts = VT.getVectorNumElements(); in combineToExtendBoolVectorInReg() local
44094 assert(NumElts == SclVT.getSizeInBits() && "Unexpected bool vector size"); in combineToExtendBoolVectorInReg()
44097 if (NumElts > EltSizeInBits) { in combineToExtendBoolVectorInReg()
44102 assert((NumElts % EltSizeInBits) == 0 && "Unexpected integer scale"); in combineToExtendBoolVectorInReg()
44103 unsigned Scale = NumElts / EltSizeInBits; in combineToExtendBoolVectorInReg()
44111 } else if (Subtarget.hasAVX2() && NumElts < EltSizeInBits && in combineToExtendBoolVectorInReg()
44117 assert((EltSizeInBits % NumElts) == 0 && "Unexpected integer scale"); in combineToExtendBoolVectorInReg()
44118 unsigned Scale = EltSizeInBits / NumElts; in combineToExtendBoolVectorInReg()
44120 EVT::getVectorVT(*DAG.getContext(), SclVT, NumElts * Scale); in combineToExtendBoolVectorInReg()
44122 ShuffleMask.append(NumElts * Scale, 0); in combineToExtendBoolVectorInReg()
44131 ShuffleMask.append(NumElts, 0); in combineToExtendBoolVectorInReg()
44137 for (unsigned i = 0; i != NumElts; ++i) { in combineToExtendBoolVectorInReg()
44578 int NumElts = VT.getVectorNumElements(); in combineSelect() local
44579 for (int i = 0; i != NumElts; ++i) { in combineSelect()
44582 if (CondMask[i] < NumElts) { in combineSelect()
45564 unsigned NumElts = VecVT.getVectorNumElements(); in combineSetCCMOVMSK() local
45569 NumElts <= CmpBits && CmpVal.isMask(NumElts); in combineSetCCMOVMSK()
45587 if (Vec.getOpcode() == ISD::BITCAST && NumElts <= CmpBits) { in combineSetCCMOVMSK()
45607 if (VecVT.is256BitVector() && NumElts <= CmpBits && IsOneUse) { in combineSetCCMOVMSK()
45613 APInt CmpMask = APInt::getLowBitsSet(32, IsAnyOf ? 0 : NumElts / 2); in combineSetCCMOVMSK()
45632 if (BC.getValueType().getVectorNumElements() <= NumElts) { in combineSetCCMOVMSK()
45688 Result.getValueType().getVectorNumElements() <= NumElts) { in combineSetCCMOVMSK()
45712 if (NumElts <= CmpBits && in combineSetCCMOVMSK()
46071 unsigned NumElts = VT.getVectorNumElements(); in reduceVMULWidth() local
46072 if ((NumElts % 2) != 0) in reduceVMULWidth()
46075 EVT ReducedVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts); in reduceVMULWidth()
46089 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts / 2); in reduceVMULWidth()
46099 SmallVector<int, 16> ShuffleMask(NumElts); in reduceVMULWidth()
46100 for (unsigned i = 0, e = NumElts / 2; i < e; i++) { in reduceVMULWidth()
46102 ShuffleMask[2 * i + 1] = i + NumElts; in reduceVMULWidth()
46108 for (unsigned i = 0, e = NumElts / 2; i < e; i++) { in reduceVMULWidth()
46109 ShuffleMask[2 * i] = i + NumElts / 2; in reduceVMULWidth()
46110 ShuffleMask[2 * i + 1] = i + NumElts * 3 / 2; in reduceVMULWidth()
46223 unsigned NumElts = VT.getVectorNumElements(); in combineMulToPMADDWD() local
46224 if (NumElts == 1 || !isPowerOf2_32(NumElts)) in combineMulToPMADDWD()
46228 if (32 <= (2 * NumElts) && Subtarget.hasAVX512() && !Subtarget.hasBWI()) in combineMulToPMADDWD()
47521 unsigned NumElts = 128 / N00Type.getSizeInBits(); in convertIntLogicToFPLogic() local
47522 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), N00Type, NumElts); in convertIntLogicToFPLogic()
47523 EVT BoolVecVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElts); in convertIntLogicToFPLogic()
47896 unsigned NumElts = SrcOps[0].getValueType().getVectorNumElements(); in combineAnd() local
47897 EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts); in combineAnd()
47902 assert(SrcPartials[0].getBitWidth() == NumElts && in combineAnd()
47996 int NumElts = VT.getVectorNumElements(); in combineAnd() local
47999 APInt DemandedElts = APInt::getAllOnes(NumElts); in combineAnd()
48004 for (int I = 0; I != NumElts; ++I) { in combineAnd()
48055 unsigned NumElts = SrcVecVT.getVectorNumElements(); in combineAnd() local
48060 SmallVector<int, 16> ShuffleMask(NumElts * Scale, SM_SentinelUndef); in combineAnd()
48412 unsigned NumElts = SrcOps[0].getValueType().getVectorNumElements(); in combineOr() local
48413 EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts); in combineOr()
48418 assert(SrcPartials[0].getBitWidth() == NumElts && in combineOr()
48454 unsigned NumElts = VT.getVectorNumElements(); in combineOr() local
48455 unsigned HalfElts = NumElts / 2; in combineOr()
48456 APInt UpperElts = APInt::getHighBitsSet(NumElts, HalfElts); in combineOr()
48457 if (NumElts >= 16 && N1.getOpcode() == X86ISD::KSHIFTL && in combineOr()
48465 if (NumElts >= 16 && N0.getOpcode() == X86ISD::KSHIFTL && in combineOr()
48486 int NumElts = VT.getVectorNumElements(); in combineOr() local
48491 APInt DemandedElts = APInt::getZero(NumElts); in combineOr()
48492 for (int I = 0; I != NumElts; ++I) in combineOr()
49010 unsigned NumElts = RegVT.getVectorNumElements(); in combineLoad() local
49011 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), NumElts); in combineLoad()
49078 unsigned NumElts = BV->getValueType(0).getVectorNumElements(); in getOneTrueElt() local
49079 for (unsigned i = 0; i < NumElts; ++i) { in getOneTrueElt()
49183 unsigned NumElts = VT.getVectorNumElements(); in combineMaskedLoadConstantMask() local
49186 bool LoadLastElt = !isNullConstant(MaskBV->getOperand(NumElts - 1)); in combineMaskedLoadConstantMask()
49680 unsigned NumElts = VT.getVectorNumElements(); in isHorizontalBinOp() local
49700 scaleShuffleElements(SrcMask, NumElts, ScaledMask)) { in isHorizontalBinOp()
49706 scaleShuffleElements(SrcMask, 2 * NumElts, ScaledMask)) { in isHorizontalBinOp()
49708 ArrayRef<int> Mask = ArrayRef<int>(ScaledMask).slice(0, NumElts); in isHorizontalBinOp()
49736 for (unsigned i = 0; i != NumElts; ++i) in isHorizontalBinOp()
49742 for (unsigned i = 0; i != NumElts; ++i) in isHorizontalBinOp()
49747 if (isUndefOrInRange(LMask, 0, NumElts)) in isHorizontalBinOp()
49749 else if (isUndefOrInRange(LMask, NumElts, NumElts * 2)) in isHorizontalBinOp()
49752 if (isUndefOrInRange(RMask, 0, NumElts)) in isHorizontalBinOp()
49754 else if (isUndefOrInRange(RMask, NumElts, NumElts * 2)) in isHorizontalBinOp()
49768 PostShuffleMask.append(NumElts, SM_SentinelUndef); in isHorizontalBinOp()
49777 unsigned NumEltsPer128BitChunk = NumElts / Num128BitChunks; in isHorizontalBinOp()
49781 for (unsigned j = 0; j != NumElts; j += NumEltsPer128BitChunk) { in isHorizontalBinOp()
49786 (!A.getNode() && (LIdx < (int)NumElts || RIdx < (int)NumElts)) || in isHorizontalBinOp()
49787 (!B.getNode() && (LIdx >= (int)NumElts || RIdx >= (int)NumElts))) in isHorizontalBinOp()
49800 ((Base % NumElts) & ~(NumEltsPer128BitChunk - 1)); in isHorizontalBinOp()
49805 if ((B && Base >= (int)NumElts) || (!B && i >= NumEltsPer64BitChunk)) in isHorizontalBinOp()
49815 isSequentialOrUndefInRange(PostShuffleMask, 0, NumElts, 0); in isHorizontalBinOp()
51254 int NumElts = VT.getVectorNumElements(); in combineAndnp() local
51281 for (int I = 0; I != NumElts; ++I) in combineAndnp()
51283 APInt ResultUndefs = APInt::getZero(NumElts); in combineAndnp()
51302 APInt DemandedElts = APInt::getAllOnes(NumElts); in combineAndnp()
51307 for (int I = 0; I != NumElts; ++I) { in combineAndnp()
52225 unsigned NumElts = SrcVT.getVectorNumElements(); in combineMOVMSK() local
52227 assert(VT == MVT::i32 && NumElts <= NumBits && "Unexpected MOVMSK types"); in combineMOVMSK()
52234 for (unsigned Idx = 0; Idx != NumElts; ++Idx) in combineMOVMSK()
52251 APInt NotMask = APInt::getLowBitsSet(NumBits, NumElts); in combineMOVMSK()
52263 APInt NotMask = APInt::getLowBitsSet(NumBits, NumElts); in combineMOVMSK()
52282 ShiftVT = MVT::getVectorVT(MVT::i16, NumElts / 2); in combineMOVMSK()
52603 unsigned NumElts = VecWidth / DestWidth; in combineToFPTruncExtElt() local
52604 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), TruncVT, NumElts); in combineToFPTruncExtElt()
53537 unsigned NumElts = VT.getVectorNumElements(); in combineAddOfPMADDWD() local
53540 APInt DemandedHiElts = APInt::getSplat(2 * NumElts, APInt(2, 2)); in combineAddOfPMADDWD()
53556 for (int i = 0; i != (int)NumElts; ++i) { in combineAddOfPMADDWD()
53558 Mask.push_back(2 * (i + NumElts)); in combineAddOfPMADDWD()
54764 unsigned NumElts = VT.getVectorNumElements(); in combineEXTEND_VECTOR_INREG() local
54767 SmallVector<SDValue> Elts(Scale * NumElts, DAG.getConstant(0, DL, EltVT)); in combineEXTEND_VECTOR_INREG()
54768 for (unsigned I = 0; I != NumElts; ++I) in combineEXTEND_VECTOR_INREG()
54846 unsigned NumElts = VT.getVectorNumElements(); in combineFP_EXTEND() local
54847 if (NumElts == 1 || !isPowerOf2_32(NumElts)) in combineFP_EXTEND()
54857 if (NumElts < 8) { in combineFP_EXTEND()
54858 unsigned NumConcats = 8 / NumElts; in combineFP_EXTEND()
54859 SDValue Fill = NumElts == 4 ? DAG.getUNDEF(IntVT) in combineFP_EXTEND()
54868 std::max(4U, NumElts)); in combineFP_EXTEND()
54878 if (NumElts < 4) { in combineFP_EXTEND()
54879 assert(NumElts == 2 && "Unexpected size"); in combineFP_EXTEND()
54954 unsigned NumElts = VT.getVectorNumElements(); in combineFP_ROUND() local
54955 if (NumElts == 1 || !isPowerOf2_32(NumElts)) in combineFP_ROUND()
54961 if (NumElts < 4) in combineFP_ROUND()
54967 std::max(8U, NumElts)); in combineFP_ROUND()
54979 if (NumElts < 8) { in combineFP_ROUND()