Lines Matching refs:VPValue
223 DenseMap<VPValue *, PerPartValuesTy> PerPartOutput;
226 DenseMap<VPValue *, ScalarsPerPartValuesTy> PerPartScalars;
234 Value *get(VPValue *Def, unsigned Part);
237 Value *get(VPValue *Def, const VPIteration &Instance);
239 bool hasVectorValue(VPValue *Def, unsigned Part) { in hasVectorValue()
245 bool hasAnyVectorValue(VPValue *Def) const { in hasAnyVectorValue()
249 bool hasScalarValue(VPValue *Def, VPIteration Instance) { in hasScalarValue()
260 void set(VPValue *Def, Value *V, unsigned Part) { in set()
268 void reset(VPValue *Def, Value *V, unsigned Part) { in reset()
276 void set(VPValue *Def, Value *V, const VPIteration &Instance) { in set()
290 void reset(VPValue *Def, Value *V, const VPIteration &Instance) { in reset()
597 virtual void dropAllReferences(VPValue *NewValue) = 0;
634 VPLiveOut(PHINode *Phi, VPValue *Op) in VPLiveOut()
645 bool usesScalars(const VPValue *Op) const override { in usesScalars()
669 VPRecipeBase(const unsigned char SC, ArrayRef<VPValue *> Operands) in VPRecipeBase()
771 class VPInstruction : public VPRecipeBase, public VPValue {
812 VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL,
815 VPValue(VPValue::VPVInstructionSC, nullptr, this), Opcode(Opcode), in VPRecipeBase()
818 VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
820 : VPInstruction(Opcode, ArrayRef<VPValue *>(Operands), DL, Name) {} in VPInstruction()
823 static inline bool classof(const VPValue *V) { in classof()
824 return V->getVPValueID() == VPValue::VPVInstructionSC; in classof()
828 SmallVector<VPValue *, 2> Operands(operands()); in clone()
897 bool onlyFirstLaneUsed(const VPValue *Op) const override { in onlyFirstLaneUsed()
920 class VPWidenRecipe : public VPRecipeBase, public VPValue {
925 VPValue(VPValue::VPVWidenSC, &I, this) {} in VPWidenRecipe()
933 static inline bool classof(const VPValue *V) { in classof()
934 return V->getVPValueID() == VPValue::VPVWidenSC; in classof()
948 class VPWidenCallRecipe : public VPRecipeBase, public VPValue {
954 VPValue(VPValue::VPVWidenCallSC, &I, this) {} in VPWidenCallRecipe()
974 class VPWidenSelectRecipe : public VPRecipeBase, public VPValue {
984 VPValue(VPValue::VPVWidenSelectSC, &I, this), in VPWidenSelectRecipe()
1005 class VPWidenGEPRecipe : public VPRecipeBase, public VPValue {
1013 VPValue(VPWidenGEPSC, GEP, this), in VPWidenGEPRecipe()
1020 VPValue(VPValue::VPVWidenGEPSC, GEP, this), in VPWidenGEPRecipe()
1046 class VPWidenIntOrFpInductionRecipe : public VPRecipeBase, public VPValue {
1052 VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, in VPWidenIntOrFpInductionRecipe()
1056 VPValue(IV, this), IV(IV), IndDesc(IndDesc),
1059 VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, in VPWidenIntOrFpInductionRecipe()
1063 VPValue(Trunc, this), IV(IV), IndDesc(IndDesc),
1084 VPValue *getStartValue() { return getOperand(0); } in getStartValue()
1085 const VPValue *getStartValue() const { return getOperand(0); } in getStartValue()
1088 VPValue *getStepValue() { return getOperand(1); } in getStepValue()
1089 const VPValue *getStepValue() const { return getOperand(1); } in getStepValue()
1123 class VPHeaderPHIRecipe : public VPRecipeBase, public VPValue {
1126 VPValue *Start = nullptr)
1127 : VPRecipeBase(VPDefID, {}), VPValue(VPVID, Phi, this) {
1144 static inline bool classof(const VPValue *V) { in classof()
1145 return V->getVPValueID() == VPValue::VPVCanonicalIVPHISC || in classof()
1146 V->getVPValueID() == VPValue::VPVActiveLaneMaskPHISC || in classof()
1147 V->getVPValueID() == VPValue::VPVFirstOrderRecurrencePHISC || in classof()
1148 V->getVPValueID() == VPValue::VPVReductionPHISC || in classof()
1149 V->getVPValueID() == VPValue::VPVWidenIntOrFpInductionSC || in classof()
1150 V->getVPValueID() == VPValue::VPVWidenPHISC; in classof()
1163 VPValue *getStartValue() { in getStartValue()
1166 VPValue *getStartValue() const { in getStartValue()
1171 VPValue *getBackedgeValue() { in getBackedgeValue()
1195 VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, in VPWidenPointerInductionRecipe()
1215 static inline bool classof(const VPValue *V) { in classof()
1216 return V->getVPValueID() == VPValue::VPVWidenPointerInductionSC; in classof()
1241 VPWidenPHIRecipe(PHINode *Phi, VPValue *Start = nullptr)
1256 static inline bool classof(const VPValue *V) { in classof()
1257 return V->getVPValueID() == VPValue::VPVWidenPHISC; in classof()
1270 void addIncoming(VPValue *IncomingV, VPBasicBlock *IncomingBlock) { in addIncoming()
1279 VPValue *getIncomingValue(unsigned I) { return getOperand(I); } in getIncomingValue()
1286 VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start) in VPFirstOrderRecurrencePHIRecipe()
1297 static inline bool classof(const VPValue *V) { in classof()
1298 return V->getVPValueID() == VPValue::VPVFirstOrderRecurrencePHISC; in classof()
1327 VPValue &Start, bool IsInLoop = false,
1343 static inline bool classof(const VPValue *V) { in classof()
1344 return V->getVPValueID() == VPValue::VPVReductionPHISC; in classof()
1369 class VPBlendRecipe : public VPRecipeBase, public VPValue {
1376 VPBlendRecipe(PHINode *Phi, ArrayRef<VPValue *> Operands) in VPBlendRecipe()
1378 VPValue(VPValue::VPVBlendSC, Phi, this), Phi(Phi) { in VPBlendRecipe()
1395 VPValue *getIncomingValue(unsigned Idx) const { return getOperand(Idx * 2); } in getIncomingValue()
1398 VPValue *getMask(unsigned Idx) const { return getOperand(Idx * 2 + 1); } in getMask()
1410 bool onlyFirstLaneUsed(const VPValue *Op) const override { in onlyFirstLaneUsed()
1430 VPInterleaveRecipe(const InterleaveGroup<Instruction> *IG, VPValue *Addr, in VPInterleaveRecipe()
1431 ArrayRef<VPValue *> StoredValues, VPValue *Mask) in VPInterleaveRecipe()
1437 new VPValue(I, this);
1455 VPValue *getAddr() const { in getAddr()
1461 VPValue *getMask() const { in getMask()
1468 ArrayRef<VPValue *> getStoredValues() const { in getStoredValues()
1471 return ArrayRef<VPValue *>(op_begin(), getNumOperands()) in getStoredValues()
1493 bool onlyFirstLaneUsed(const VPValue *Op) const override { in onlyFirstLaneUsed()
1496 return Op == getAddr() && all_of(getStoredValues(), [Op](VPValue *StoredV) { in onlyFirstLaneUsed()
1505 class VPReductionRecipe : public VPRecipeBase, public VPValue {
1513 VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp, in VPReductionRecipe()
1516 VPValue(VPValue::VPVReductionSC, I, this), RdxDesc(R), TTI(TTI) {
1524 static inline bool classof(const VPValue *V) { in classof()
1525 return V->getVPValueID() == VPValue::VPVReductionSC; in classof()
1538 VPValue *getChainOp() const { return getOperand(0); } in getChainOp()
1540 VPValue *getVecOp() const { return getOperand(1); } in getVecOp()
1542 VPValue *getCondOp() const { in getCondOp()
1551 class VPReplicateRecipe : public VPRecipeBase, public VPValue {
1565 : VPRecipeBase(VPReplicateSC, Operands), VPValue(VPVReplicateSC, I, this), in VPRecipeBase()
1582 static inline bool classof(const VPValue *V) { in classof()
1583 return V->getVPValueID() == VPValue::VPVReplicateSC; in classof()
1606 bool onlyFirstLaneUsed(const VPValue *Op) const override { in onlyFirstLaneUsed()
1613 bool usesScalars(const VPValue *Op) const override { in usesScalars()
1623 VPBranchOnMaskRecipe(VPValue *BlockInMask) in VPBranchOnMaskRecipe()
1643 if (VPValue *Mask = getMask()) in print()
1652 VPValue *getMask() const { in getMask()
1659 bool usesScalars(const VPValue *Op) const override { in usesScalars()
1671 class VPPredInstPHIRecipe : public VPRecipeBase, public VPValue {
1675 VPPredInstPHIRecipe(VPValue *PredV) in VPPredInstPHIRecipe()
1677 VPValue(VPValue::VPVPredInstPHI, nullptr, this) {} in VPPredInstPHIRecipe()
1695 bool usesScalars(const VPValue *Op) const override { in usesScalars()
1717 void setMask(VPValue *Mask) { in setMask()
1728 VPWidenMemoryInstructionRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask, in VPWidenMemoryInstructionRecipe()
1733 new VPValue(VPValue::VPVMemoryInstructionSC, &Load, this);
1737 VPWidenMemoryInstructionRecipe(StoreInst &Store, VPValue *Addr, in VPWidenMemoryInstructionRecipe()
1738 VPValue *StoredValue, VPValue *Mask, in VPWidenMemoryInstructionRecipe()
1752 VPValue *getAddr() const { in getAddr()
1758 VPValue *getMask() const { in getMask()
1767 VPValue *getStoredValue() const { in getStoredValue()
1789 bool onlyFirstLaneUsed(const VPValue *Op) const override { in onlyFirstLaneUsed()
1804 class VPExpandSCEVRecipe : public VPRecipeBase, public VPValue {
1810 : VPRecipeBase(VPExpandSCEVSC, {}), VPValue(nullptr, this), Expr(Expr), in VPExpandSCEVRecipe()
1840 VPCanonicalIVPHIRecipe(VPValue *StartV, DebugLoc DL) in VPCanonicalIVPHIRecipe()
1841 : VPHeaderPHIRecipe(VPValue::VPVCanonicalIVPHISC, VPCanonicalIVPHISC, in VPCanonicalIVPHIRecipe()
1854 static inline bool classof(const VPValue *V) { in classof()
1855 return V->getVPValueID() == VPValue::VPVCanonicalIVPHISC; in classof()
1873 bool onlyFirstLaneUsed(const VPValue *Op) const override { in onlyFirstLaneUsed()
1888 VPActiveLaneMaskPHIRecipe(VPValue *StartMask, DebugLoc DL) in VPActiveLaneMaskPHIRecipe()
1889 : VPHeaderPHIRecipe(VPValue::VPVActiveLaneMaskPHISC, in VPActiveLaneMaskPHIRecipe()
1902 static inline bool classof(const VPValue *V) { in classof()
1903 return V->getVPValueID() == VPValue::VPVActiveLaneMaskPHISC; in classof()
1917 class VPWidenCanonicalIVRecipe : public VPRecipeBase, public VPValue {
1921 VPValue(VPValue::VPVWidenCanonicalIVSC, nullptr, this) {}
1960 class VPScalarIVStepsRecipe : public VPRecipeBase, public VPValue {
1969 VPValue *CanonicalIV, VPValue *Start, VPValue *Step, in VPScalarIVStepsRecipe()
1972 VPValue(nullptr, this), Ty(Ty), TruncToTy(TruncToTy), IndDesc(IndDesc) {
2005 VPValue *getStartValue() const { return getOperand(1); } in getStartValue()
2006 VPValue *getStepValue() const { return getOperand(2); } in getStepValue()
2009 bool onlyFirstLaneUsed(const VPValue *Op) const override { in onlyFirstLaneUsed()
2101 void dropAllReferences(VPValue *NewValue) override;
2171 VPValue DummyValue; in ~VPRegionBlock()
2226 void dropAllReferences(VPValue *NewValue) override;
2513 DenseMap<Value *, VPValue *> VPExternalDefs;
2517 VPValue *TripCount = nullptr;
2521 VPValue *BackedgeTakenCount = nullptr;
2524 VPValue VectorTripCount;
2532 SmallVector<VPValue *, 16> VPValuesToFree;
2551 VPValue DummyValue;
2557 for (VPValue *VPV : VPValuesToFree)
2585 VPValue *getOrCreateTripCount() {
2587 TripCount = new VPValue();
2592 VPValue *getOrCreateBackedgeTakenCount() {
2594 BackedgeTakenCount = new VPValue();
2599 VPValue &getVectorTripCount() { return VectorTripCount; }
2614 VPValue *getOrAddExternalDef(Value *V) {
2617 I.first->second = new VPValue(V);
2626 VPValue *VPV = new VPValue(V);
2631 void addVPValue(Value *V, VPValue *VPV) {
2640 VPValue *getVPValue(Value *V, bool OverrideAllowed = false) {
2651 VPValue *getOrAddVPValue(Value *V, bool OverrideAllowed = false) {
2679 iterator_range<mapped_iterator<Use *, std::function<VPValue *(Value *)>>>
2681 std::function<VPValue *(Value *)> Fn = [this](Value *Op) {
2688 bool isUniformAfterVectorization(VPValue *VPV) const {
2715 void addLiveOut(PHINode *PN, VPValue *V);
2969 static SmallVector<VPValue *, 4> getEmptyKey() {
2970 return {reinterpret_cast<VPValue *>(-1)};
2973 static SmallVector<VPValue *, 4> getTombstoneKey() {
2974 return {reinterpret_cast<VPValue *>(-2)};
2977 static unsigned getHashValue(const SmallVector<VPValue *, 4> &V) {
2981 static bool isEqual(const SmallVector<VPValue *, 4> &LHS,
2982 const SmallVector<VPValue *, 4> &RHS) {
2988 DenseMap<SmallVector<VPValue *, 4>, VPInstruction *, BundleDenseMapInfo>
3004 typename std::pair<VPInstruction *, SmallVector<VPValue *, 4>>;
3016 bool areVectorizable(ArrayRef<VPValue *> Operands) const;
3019 void addCombined(ArrayRef<VPValue *> Operands, VPInstruction *New);
3031 std::pair<OpMode, VPValue *> getBest(OpMode Mode, VPValue *Last,
3032 SmallPtrSetImpl<VPValue *> &Candidates,
3037 void dumpBundle(ArrayRef<VPValue *> Values);
3047 VPInstruction *buildGraph(ArrayRef<VPValue *> Operands);
3059 bool onlyFirstLaneUsed(VPValue *Def);
3066 VPValue *getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr,