191   TargetTransformInfo::UnrollingPreferences UP;  in gatherUnrollingPreferences()  local
194   UP.Threshold =  in gatherUnrollingPreferences()
196   UP.MaxPercentThresholdBoost = 400;  in gatherUnrollingPreferences()
197   UP.OptSizeThreshold = UnrollOptSizeThreshold;  in gatherUnrollingPreferences()
198   UP.PartialThreshold = 150;  in gatherUnrollingPreferences()
199   UP.PartialOptSizeThreshold = UnrollOptSizeThreshold;  in gatherUnrollingPreferences()
200   UP.Count = 0;  in gatherUnrollingPreferences()
201   UP.DefaultUnrollRuntimeCount = 8;  in gatherUnrollingPreferences()
202   UP.MaxCount = std::numeric_limits<unsigned>::max();  in gatherUnrollingPreferences()
203   UP.MaxUpperBound = UnrollMaxUpperBound;  in gatherUnrollingPreferences()
204   UP.FullUnrollMaxCount = std::numeric_limits<unsigned>::max();  in gatherUnrollingPreferences()
205   UP.BEInsns = 2;  in gatherUnrollingPreferences()
206   UP.Partial = false;  in gatherUnrollingPreferences()
207   UP.Runtime = false;  in gatherUnrollingPreferences()
208   UP.AllowRemainder = true;  in gatherUnrollingPreferences()
209   UP.UnrollRemainder = false;  in gatherUnrollingPreferences()
210   UP.AllowExpensiveTripCount = false;  in gatherUnrollingPreferences()
211   UP.Force = false;  in gatherUnrollingPreferences()
212   UP.UpperBound = false;  in gatherUnrollingPreferences()
213   UP.UnrollAndJam = false;  in gatherUnrollingPreferences()
214   UP.UnrollAndJamInnerLoopThreshold = 60;  in gatherUnrollingPreferences()
215   UP.MaxIterationsCountToAnalyze = UnrollMaxIterationsCountToAnalyze;  in gatherUnrollingPreferences()
218   TTI.getUnrollingPreferences(L, SE, UP, &ORE);  in gatherUnrollingPreferences()
227     UP.Threshold = UP.OptSizeThreshold;  in gatherUnrollingPreferences()
228     UP.PartialThreshold = UP.PartialOptSizeThreshold;  in gatherUnrollingPreferences()
229     UP.MaxPercentThresholdBoost = 100;  in gatherUnrollingPreferences()
234     UP.Threshold = UnrollThreshold;  in gatherUnrollingPreferences()
236     UP.PartialThreshold = UnrollPartialThreshold;  in gatherUnrollingPreferences()
238     UP.MaxPercentThresholdBoost = UnrollMaxPercentThresholdBoost;  in gatherUnrollingPreferences()
240     UP.MaxCount = UnrollMaxCount;  in gatherUnrollingPreferences()
242     UP.MaxUpperBound = UnrollMaxUpperBound;  in gatherUnrollingPreferences()
244     UP.FullUnrollMaxCount = UnrollFullMaxCount;  in gatherUnrollingPreferences()
246     UP.Partial = UnrollAllowPartial;  in gatherUnrollingPreferences()
248     UP.AllowRemainder = UnrollAllowRemainder;  in gatherUnrollingPreferences()
250     UP.Runtime = UnrollRuntime;  in gatherUnrollingPreferences()
252     UP.UpperBound = false;  in gatherUnrollingPreferences()
254     UP.UnrollRemainder = UnrollUnrollRemainder;  in gatherUnrollingPreferences()
256     UP.MaxIterationsCountToAnalyze = UnrollMaxIterationsCountToAnalyze;  in gatherUnrollingPreferences()
260     UP.Threshold = *UserThreshold;  in gatherUnrollingPreferences()
261     UP.PartialThreshold = *UserThreshold;  in gatherUnrollingPreferences()
264     UP.Count = *UserCount;  in gatherUnrollingPreferences()
266     UP.Partial = *UserAllowPartial;  in gatherUnrollingPreferences()
268     UP.Runtime = *UserRuntime;  in gatherUnrollingPreferences()
270     UP.UpperBound = *UserUpperBound;  in gatherUnrollingPreferences()
272     UP.FullUnrollMaxCount = *UserFullUnrollMaxCount;  in gatherUnrollingPreferences()
274   return UP;  in gatherUnrollingPreferences()
691     const TargetTransformInfo::UnrollingPreferences &UP,  in getUnrolledLoopSize()  argument
694   assert(LS >= UP.BEInsns && "LoopSize should not be less than BEInsns!");  in getUnrolledLoopSize()
696     return static_cast<uint64_t>(LS - UP.BEInsns) * CountOverwrite + UP.BEInsns;  in getUnrolledLoopSize()
698     return static_cast<uint64_t>(LS - UP.BEInsns) * UP.Count + UP.BEInsns;  in getUnrolledLoopSize()
762                    const TargetTransformInfo::UnrollingPreferences &UP) {  in shouldPragmaUnroll()  argument
768     if (UP.AllowRemainder &&  in shouldPragmaUnroll()
769         UCE.getUnrolledLoopSize(UP, (unsigned)UnrollCount) < UP.Threshold)  in shouldPragmaUnroll()
775     if ((UP.AllowRemainder || (TripMultiple % PInfo.PragmaCount == 0)))  in shouldPragmaUnroll()
783       MaxTripCount <= UP.MaxUpperBound)  in shouldPragmaUnroll()
794     const TargetTransformInfo::UnrollingPreferences &UP) {  in shouldFullUnroll()  argument
797   if (FullUnrollTripCount > UP.FullUnrollMaxCount)  in shouldFullUnroll()
802   if (UCE.getUnrolledLoopSize(UP) < UP.Threshold)  in shouldFullUnroll()
810           UP.Threshold * UP.MaxPercentThresholdBoost / 100,  in shouldFullUnroll()
811           UP.MaxIterationsCountToAnalyze)) {  in shouldFullUnroll()
813       getFullUnrollBoostingFactor(*Cost, UP.MaxPercentThresholdBoost);  in shouldFullUnroll()
814     if (Cost->UnrolledCost < UP.Threshold * Boost / 100)  in shouldFullUnroll()
823                     const TargetTransformInfo::UnrollingPreferences &UP) {  in shouldPartialUnroll()  argument
828   if (!UP.Partial) {  in shouldPartialUnroll()
833   unsigned count = UP.Count;  in shouldPartialUnroll()
836   if (UP.PartialThreshold != NoThreshold) {  in shouldPartialUnroll()
838     if (UCE.getUnrolledLoopSize(UP, count) > UP.PartialThreshold)  in shouldPartialUnroll()
839       count = (std::max(UP.PartialThreshold, UP.BEInsns + 1) - UP.BEInsns) /  in shouldPartialUnroll()
840         (LoopSize - UP.BEInsns);  in shouldPartialUnroll()
841     if (count > UP.MaxCount)  in shouldPartialUnroll()
842       count = UP.MaxCount;  in shouldPartialUnroll()
845     if (UP.AllowRemainder && count <= 1) {  in shouldPartialUnroll()
850       count = UP.DefaultUnrollRuntimeCount;  in shouldPartialUnroll()
852              UCE.getUnrolledLoopSize(UP, count) > UP.PartialThreshold)  in shouldPartialUnroll()
861   if (count > UP.MaxCount)  in shouldPartialUnroll()
862     count = UP.MaxCount;  in shouldPartialUnroll()
882     TargetTransformInfo::UnrollingPreferences &UP,  in computeUnrollCount()  argument
904     UP.Count = 1;  in computeUnrollCount()
905     UP.Runtime = false;  in computeUnrollCount()
912                                              MaxTripCount, UCE, UP)) {  in computeUnrollCount()
913     UP.Count = *UnrollFactor;  in computeUnrollCount()
916       UP.AllowExpensiveTripCount = true;  in computeUnrollCount()
917       UP.Force = true;  in computeUnrollCount()
919     UP.Runtime |= (PragmaCount > 0);  in computeUnrollCount()
926       UP.Threshold = std::max<unsigned>(UP.Threshold, PragmaUnrollThreshold);  in computeUnrollCount()
927       UP.PartialThreshold =  in computeUnrollCount()
928           std::max<unsigned>(UP.PartialThreshold, PragmaUnrollThreshold);  in computeUnrollCount()
934   UP.Count = 0;  in computeUnrollCount()
936     UP.Count = TripCount;  in computeUnrollCount()
938                                              TripCount, UCE, UP)) {  in computeUnrollCount()
939       UP.Count = *UnrollFactor;  in computeUnrollCount()
957   if (!TripCount && MaxTripCount && (UP.UpperBound || MaxOrZero) &&  in computeUnrollCount()
958       MaxTripCount <= UP.MaxUpperBound) {  in computeUnrollCount()
959     UP.Count = MaxTripCount;  in computeUnrollCount()
961                                              MaxTripCount, UCE, UP)) {  in computeUnrollCount()
962       UP.Count = *UnrollFactor;  in computeUnrollCount()
969   computePeelCount(L, LoopSize, PP, TripCount, DT, SE, AC, UP.Threshold);  in computeUnrollCount()
971     UP.Runtime = false;  in computeUnrollCount()
972     UP.Count = 1;  in computeUnrollCount()
979     UP.Partial |= ExplicitUnroll;  in computeUnrollCount()
983   if (auto UnrollFactor = shouldPartialUnroll(LoopSize, TripCount, UCE, UP)) {  in computeUnrollCount()
984     UP.Count = *UnrollFactor;  in computeUnrollCount()
987         UP.Count != TripCount)  in computeUnrollCount()
997     if (UP.PartialThreshold != NoThreshold) {  in computeUnrollCount()
998       if (UP.Count == 0) {  in computeUnrollCount()
1027     UP.Count = 0;  in computeUnrollCount()
1032   if (MaxTripCount && !UP.Force && MaxTripCount < UP.MaxUpperBound) {  in computeUnrollCount()
1033     UP.Count = 0;  in computeUnrollCount()
1043         UP.AllowExpensiveTripCount = true;  in computeUnrollCount()
1046   UP.Runtime |= PragmaEnableUnroll || PragmaCount > 0 || UserUnrollCount;  in computeUnrollCount()
1047   if (!UP.Runtime) {  in computeUnrollCount()
1051     UP.Count = 0;  in computeUnrollCount()
1054   if (UP.Count == 0)  in computeUnrollCount()
1055     UP.Count = UP.DefaultUnrollRuntimeCount;  in computeUnrollCount()
1059   while (UP.Count != 0 &&  in computeUnrollCount()
1060          UCE.getUnrolledLoopSize(UP) > UP.PartialThreshold)  in computeUnrollCount()
1061     UP.Count >>= 1;  in computeUnrollCount()
1064   unsigned OrigCount = UP.Count;  in computeUnrollCount()
1067   if (!UP.AllowRemainder && UP.Count != 0 && (TripMultiple % UP.Count) != 0) {  in computeUnrollCount()
1068     while (UP.Count != 0 && TripMultiple % UP.Count != 0)  in computeUnrollCount()
1069       UP.Count >>= 1;  in computeUnrollCount()
1076                << " to " << UP.Count << ".\n");  in computeUnrollCount()
1080     if (unrollCountPragmaValue(L) > 0 && !UP.AllowRemainder)  in computeUnrollCount()
1092                << NV("UnrollCount", UP.Count) << " time(s).";  in computeUnrollCount()
1096   if (UP.Count > UP.MaxCount)  in computeUnrollCount()
1097     UP.Count = UP.MaxCount;  in computeUnrollCount()
1099   if (MaxTripCount && UP.Count > MaxTripCount)  in computeUnrollCount()
1100     UP.Count = MaxTripCount;  in computeUnrollCount()
1102   LLVM_DEBUG(dbgs() << "  runtime unrolling with count: " << UP.Count  in computeUnrollCount()
1104   if (UP.Count < 2)  in computeUnrollCount()
1105     UP.Count = 0;  in computeUnrollCount()
1167   TargetTransformInfo::UnrollingPreferences UP = gatherUnrollingPreferences(  in tryToUnrollLoop()  local
1176   if (UP.Threshold == 0 && (!UP.Partial || UP.PartialThreshold == 0) &&  in tryToUnrollLoop()
1183   UnrollCostEstimator UCE(L, TTI, EphValues, UP.BEInsns);  in tryToUnrollLoop()
1196     UP.Threshold = std::max(UP.Threshold, LoopSize + 1);  in tryToUnrollLoop()
1241     UP.AllowRemainder = false;  in tryToUnrollLoop()
1257       MaxOrZero, TripMultiple, UCE, UP, PP, UseUpperBound);  in tryToUnrollLoop()
1258   if (!UP.Count)  in tryToUnrollLoop()
1262     assert(UP.Count == 1 && "Cannot perform peel and unroll in the same step");  in tryToUnrollLoop()
1285   if (OnlyFullUnroll && !(UP.Count >= MaxTripCount)) {  in tryToUnrollLoop()
1296   UP.Runtime &= TripCount == 0 && TripMultiple % UP.Count != 0;  in tryToUnrollLoop()
1305       {UP.Count, UP.Force, UP.Runtime, UP.AllowExpensiveTripCount,  in tryToUnrollLoop()
1306        UP.UnrollRemainder, ForgetAllSCEV},  in tryToUnrollLoop()