Lines Matching refs:Part
416 Value *getOrCreateVectorValue(Value *V, unsigned Part);
552 unsigned Part,
1563 for (unsigned Part = 0; Part < UF; ++Part) { in createVectorIntOrFpInductionPHI() local
1564 VectorLoopValueMap.setVectorValue(EntryVal, Part, LastInduction); in createVectorIntOrFpInductionPHI()
1568 recordVectorLoopValueForInductionCast(II, EntryVal, LastInduction, Part); in createVectorIntOrFpInductionPHI()
1604 Value *VectorLoopVal, unsigned Part, unsigned Lane) { in recordVectorLoopValueForInductionCast() argument
1625 VectorLoopValueMap.setScalarValue(CastInst, {Part, Lane}, VectorLoopVal); in recordVectorLoopValueForInductionCast()
1627 VectorLoopValueMap.setVectorValue(CastInst, Part, VectorLoopVal); in recordVectorLoopValueForInductionCast()
1707 for (unsigned Part = 0; Part < UF; ++Part) { in widenIntOrFpInduction() local
1709 getStepVector(Broadcasted, VF * Part, Step, ID.getInductionOpcode()); in widenIntOrFpInduction()
1710 VectorLoopValueMap.setVectorValue(EntryVal, Part, EntryPart); in widenIntOrFpInduction()
1713 recordVectorLoopValueForInductionCast(ID, EntryVal, EntryPart, Part); in widenIntOrFpInduction()
1813 for (unsigned Part = 0; Part < UF; ++Part) { in buildScalarSteps() local
1815 auto *StartIdx = getSignedIntOrFpConstant(ScalarIVTy, VF * Part + Lane); in buildScalarSteps()
1818 VectorLoopValueMap.setScalarValue(EntryVal, {Part, Lane}, Add); in buildScalarSteps()
1819 recordVectorLoopValueForInductionCast(ID, EntryVal, Add, Part, Lane); in buildScalarSteps()
1824 Value *InnerLoopVectorizer::getOrCreateVectorValue(Value *V, unsigned Part) { in getOrCreateVectorValue() argument
1835 if (VectorLoopValueMap.hasVectorValue(V, Part)) in getOrCreateVectorValue()
1836 return VectorLoopValueMap.getVectorValue(V, Part); in getOrCreateVectorValue()
1842 Value *ScalarValue = VectorLoopValueMap.getScalarValue(V, {Part, 0}); in getOrCreateVectorValue()
1850 VectorLoopValueMap.setVectorValue(V, Part, ScalarValue); in getOrCreateVectorValue()
1860 VectorLoopValueMap.getScalarValue(V, {Part, LastLane})); in getOrCreateVectorValue()
1877 VectorLoopValueMap.setVectorValue(V, Part, VectorValue); in getOrCreateVectorValue()
1881 VectorLoopValueMap.setVectorValue(V, Part, Undef); in getOrCreateVectorValue()
1883 packScalarIntoVectorValue(V, {Part, Lane}); in getOrCreateVectorValue()
1884 VectorValue = VectorLoopValueMap.getVectorValue(V, Part); in getOrCreateVectorValue()
1893 VectorLoopValueMap.setVectorValue(V, Part, B); in getOrCreateVectorValue()
1919 auto *U = getOrCreateVectorValue(V, Instance.Part); in getOrCreateScalarValue()
1938 Value *VectorValue = VectorLoopValueMap.getVectorValue(V, Instance.Part); in packScalarIntoVectorValue()
1941 VectorLoopValueMap.resetVectorValue(V, Instance.Part, VectorValue); in packScalarIntoVectorValue()
2040 for (unsigned Part = 0; Part < UF; Part++) { in vectorizeInterleaveGroup() local
2041 Value *NewPtr = getOrCreateScalarValue(Ptr, {Part, 0}); in vectorizeInterleaveGroup()
2075 for (unsigned Part = 0; Part < UF; Part++) { in vectorizeInterleaveGroup() local
2082 auto *Undefs = UndefValue::get(Mask[Part]->getType()); in vectorizeInterleaveGroup()
2085 Mask[Part], Undefs, RepMask, "interleaved.mask"); in vectorizeInterleaveGroup()
2092 Builder.CreateMaskedLoad(NewPtrs[Part], Group->getAlignment(), in vectorizeInterleaveGroup()
2096 NewLoad = Builder.CreateAlignedLoad(NewPtrs[Part], in vectorizeInterleaveGroup()
2112 for (unsigned Part = 0; Part < UF; Part++) { in vectorizeInterleaveGroup() local
2114 NewLoads[Part], UndefVec, StrideMask, "strided.vec"); in vectorizeInterleaveGroup()
2125 VectorLoopValueMap.setVectorValue(Member, Part, StridedVec); in vectorizeInterleaveGroup()
2135 for (unsigned Part = 0; Part < UF; Part++) { in vectorizeInterleaveGroup() local
2144 cast<StoreInst>(Member)->getValueOperand(), Part); in vectorizeInterleaveGroup()
2166 auto *Undefs = UndefValue::get(Mask[Part]->getType()); in vectorizeInterleaveGroup()
2169 Mask[Part], Undefs, RepMask, "interleaved.mask"); in vectorizeInterleaveGroup()
2171 IVec, NewPtrs[Part], Group->getAlignment(), ShuffledMask); in vectorizeInterleaveGroup()
2174 NewStoreInstr = Builder.CreateAlignedStore(IVec, NewPtrs[Part], in vectorizeInterleaveGroup()
2234 const auto CreateVecPtr = [&](unsigned Part, Value *Ptr) -> Value * { in vectorizeMemoryInstruction() argument
2242 Builder.CreateGEP(Ptr, Builder.getInt32(-Part * VF))); in vectorizeMemoryInstruction()
2248 Mask[Part] = reverseVector(Mask[Part]); in vectorizeMemoryInstruction()
2251 Builder.CreateGEP(Ptr, Builder.getInt32(Part * VF))); in vectorizeMemoryInstruction()
2262 for (unsigned Part = 0; Part < UF; ++Part) { in vectorizeMemoryInstruction() local
2264 Value *StoredVal = getOrCreateVectorValue(SI->getValueOperand(), Part); in vectorizeMemoryInstruction()
2266 Value *MaskPart = isMaskRequired ? Mask[Part] : nullptr; in vectorizeMemoryInstruction()
2267 Value *VectorGep = getOrCreateVectorValue(Ptr, Part); in vectorizeMemoryInstruction()
2278 auto *VecPtr = CreateVecPtr(Part, Ptr); in vectorizeMemoryInstruction()
2281 Mask[Part]); in vectorizeMemoryInstruction()
2293 for (unsigned Part = 0; Part < UF; ++Part) { in vectorizeMemoryInstruction() local
2296 Value *MaskPart = isMaskRequired ? Mask[Part] : nullptr; in vectorizeMemoryInstruction()
2297 Value *VectorGep = getOrCreateVectorValue(Ptr, Part); in vectorizeMemoryInstruction()
2302 auto *VecPtr = CreateVecPtr(Part, Ptr); in vectorizeMemoryInstruction()
2304 NewLI = Builder.CreateMaskedLoad(VecPtr, Alignment, Mask[Part], in vectorizeMemoryInstruction()
2315 VectorLoopValueMap.setVectorValue(Instr, Part, NewLI); in vectorizeMemoryInstruction()
3144 for (unsigned Part = 0; Part < UF; ++Part) { in truncateToMinimalBitwidths() local
3145 Value *I = getOrCreateVectorValue(KV.first, Part); in truncateToMinimalBitwidths()
3236 VectorLoopValueMap.resetVectorValue(KV.first, Part, Res); in truncateToMinimalBitwidths()
3247 for (unsigned Part = 0; Part < UF; ++Part) { in truncateToMinimalBitwidths() local
3248 Value *I = getOrCreateVectorValue(KV.first, Part); in truncateToMinimalBitwidths()
3253 VectorLoopValueMap.resetVectorValue(KV.first, Part, NewI); in truncateToMinimalBitwidths()
3423 for (unsigned Part = 0; Part < UF; ++Part) { in fixFirstOrderRecurrence() local
3424 Value *PreviousPart = getOrCreateVectorValue(Previous, Part); in fixFirstOrderRecurrence()
3425 Value *PhiPart = VectorLoopValueMap.getVectorValue(Phi, Part); in fixFirstOrderRecurrence()
3432 VectorLoopValueMap.resetVectorValue(Phi, Part, Shuffle); in fixFirstOrderRecurrence()
3548 for (unsigned Part = 0; Part < UF; ++Part) { in fixReduction() local
3549 Value *VecRdxPhi = getOrCreateVectorValue(Phi, Part); in fixReduction()
3550 Value *Val = getOrCreateVectorValue(LoopVal, Part); in fixReduction()
3553 Value *StartVal = (Part == 0) ? VectorStart : Identity; in fixReduction()
3575 for (unsigned Part = 0; Part < UF; ++Part) { in fixReduction() local
3576 RdxParts[Part] = VectorLoopValueMap.getVectorValue(LoopExitInst, Part); in fixReduction()
3577 Value *Trunc = Builder.CreateTrunc(RdxParts[Part], RdxVecTy); in fixReduction()
3580 for (Value::user_iterator UI = RdxParts[Part]->user_begin(); in fixReduction()
3581 UI != RdxParts[Part]->user_end();) in fixReduction()
3583 (*UI++)->replaceUsesOfWith(RdxParts[Part], Extnd); in fixReduction()
3584 RdxParts[Part] = Extnd; in fixReduction()
3590 for (unsigned Part = 0; Part < UF; ++Part) { in fixReduction() local
3591 RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy); in fixReduction()
3592 VectorLoopValueMap.resetVectorValue(LoopExitInst, Part, RdxParts[Part]); in fixReduction()
3600 for (unsigned Part = 1; Part < UF; ++Part) { in fixReduction() local
3601 Value *RdxPart = VectorLoopValueMap.getVectorValue(LoopExitInst, Part); in fixReduction()
3805 for (unsigned Part = 0; Part < UF; ++Part) { in widenPHIInstruction() local
3811 VectorLoopValueMap.setVectorValue(P, Part, EntryPart); in widenPHIInstruction()
3845 for (unsigned Part = 0; Part < UF; ++Part) { in widenPHIInstruction() local
3847 Constant *Idx = ConstantInt::get(PtrInd->getType(), Lane + Part * VF); in widenPHIInstruction()
3852 VectorLoopValueMap.setScalarValue(P, {Part, Lane}, SclrGep); in widenPHIInstruction()
3905 for (unsigned Part = 0; Part < UF; ++Part) { in widenInstruction() local
3907 VectorLoopValueMap.setVectorValue(&I, Part, EntryPart); in widenInstruction()
3918 for (unsigned Part = 0; Part < UF; ++Part) { in widenInstruction() local
3924 : getOrCreateVectorValue(GEP->getPointerOperand(), Part); in widenInstruction()
3933 Indices.push_back(getOrCreateVectorValue(U.get(), Part)); in widenInstruction()
3943 VectorLoopValueMap.setVectorValue(&I, Part, NewGEP); in widenInstruction()
3972 for (unsigned Part = 0; Part < UF; ++Part) { in widenInstruction() local
3973 Value *A = getOrCreateVectorValue(BinOp->getOperand(0), Part); in widenInstruction()
3974 Value *B = getOrCreateVectorValue(BinOp->getOperand(1), Part); in widenInstruction()
3981 VectorLoopValueMap.setVectorValue(&I, Part, V); in widenInstruction()
4003 for (unsigned Part = 0; Part < UF; ++Part) { in widenInstruction() local
4004 Value *Cond = getOrCreateVectorValue(I.getOperand(0), Part); in widenInstruction()
4005 Value *Op0 = getOrCreateVectorValue(I.getOperand(1), Part); in widenInstruction()
4006 Value *Op1 = getOrCreateVectorValue(I.getOperand(2), Part); in widenInstruction()
4009 VectorLoopValueMap.setVectorValue(&I, Part, Sel); in widenInstruction()
4022 for (unsigned Part = 0; Part < UF; ++Part) { in widenInstruction() local
4023 Value *A = getOrCreateVectorValue(Cmp->getOperand(0), Part); in widenInstruction()
4024 Value *B = getOrCreateVectorValue(Cmp->getOperand(1), Part); in widenInstruction()
4034 VectorLoopValueMap.setVectorValue(&I, Part, C); in widenInstruction()
4060 for (unsigned Part = 0; Part < UF; ++Part) { in widenInstruction() local
4061 Value *A = getOrCreateVectorValue(CI->getOperand(0), Part); in widenInstruction()
4063 VectorLoopValueMap.setVectorValue(&I, Part, Cast); in widenInstruction()
4097 for (unsigned Part = 0; Part < UF; ++Part) { in widenInstruction() local
4104 Arg = getOrCreateVectorValue(CI->getArgOperand(i), Part); in widenInstruction()
4137 VectorLoopValueMap.setVectorValue(&I, Part, V); in widenInstruction()
6911 getOrCreateVectorValues(Value *V, unsigned Part) { in getOrCreateVectorValues() argument
6912 return ILV.getOrCreateVectorValue(V, Part); in getOrCreateVectorValues()
6963 for (unsigned Part = 0; Part < State.UF; ++Part) { in execute() local
6967 State.ILV->getOrCreateVectorValue(Phi->getIncomingValue(In), Part); in execute()
6969 Entry[Part] = In0; // Initialize with the first incoming value. in execute()
6973 Value *Cond = State.get(User->getOperand(In), Part); in execute()
6974 Entry[Part] = in execute()
6975 State.Builder.CreateSelect(Cond, In0, Entry[Part], "predphi"); in execute()
6979 for (unsigned Part = 0; Part < State.UF; ++Part) in execute() local
6980 State.ValueMap.setVectorValue(Phi, Part, Entry[Part]); in execute()
6991 for (unsigned Part = 0; Part < State.UF; ++Part) in execute() local
6992 MaskValues[Part] = State.get(Mask, Part); in execute()
7005 State.ValueMap.setVectorValue(Ingredient, State.Instance->Part, Undef); in execute()
7016 for (unsigned Part = 0; Part < State.UF; ++Part) in execute() local
7018 State.ILV->scalarizeInstruction(Ingredient, {Part, Lane}, IsPredicated); in execute()
7024 unsigned Part = State.Instance->Part; in execute() local
7032 ConditionBit = State.get(BlockInMask, Part); in execute()
7062 unsigned Part = State.Instance->Part; in execute() local
7063 if (State.ValueMap.hasVectorValue(PredInst, Part)) { in execute()
7064 Value *VectorValue = State.ValueMap.getVectorValue(PredInst, Part); in execute()
7069 State.ValueMap.resetVectorValue(PredInst, Part, VPhi); // Update cache. in execute()
7086 for (unsigned Part = 0; Part < State.UF; ++Part) in execute() local
7087 MaskValues[Part] = State.get(Mask, Part); in execute()