Lines Matching refs:cfs_rq
312 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) in list_add_leaf_cfs_rq() argument
314 struct rq *rq = rq_of(cfs_rq); in list_add_leaf_cfs_rq()
317 if (cfs_rq->on_list) in list_add_leaf_cfs_rq()
320 cfs_rq->on_list = 1; in list_add_leaf_cfs_rq()
331 if (cfs_rq->tg->parent && in list_add_leaf_cfs_rq()
332 cfs_rq->tg->parent->cfs_rq[cpu]->on_list) { in list_add_leaf_cfs_rq()
339 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, in list_add_leaf_cfs_rq()
340 &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list)); in list_add_leaf_cfs_rq()
350 if (!cfs_rq->tg->parent) { in list_add_leaf_cfs_rq()
355 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, in list_add_leaf_cfs_rq()
371 list_add_rcu(&cfs_rq->leaf_cfs_rq_list, rq->tmp_alone_branch); in list_add_leaf_cfs_rq()
376 rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list; in list_add_leaf_cfs_rq()
380 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) in list_del_leaf_cfs_rq() argument
382 if (cfs_rq->on_list) { in list_del_leaf_cfs_rq()
383 struct rq *rq = rq_of(cfs_rq); in list_del_leaf_cfs_rq()
392 if (rq->tmp_alone_branch == &cfs_rq->leaf_cfs_rq_list) in list_del_leaf_cfs_rq()
393 rq->tmp_alone_branch = cfs_rq->leaf_cfs_rq_list.prev; in list_del_leaf_cfs_rq()
395 list_del_rcu(&cfs_rq->leaf_cfs_rq_list); in list_del_leaf_cfs_rq()
396 cfs_rq->on_list = 0; in list_del_leaf_cfs_rq()
406 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ argument
407 list_for_each_entry_safe(cfs_rq, pos, &rq->leaf_cfs_rq_list, \
411 static inline struct cfs_rq *
414 if (se->cfs_rq == pse->cfs_rq) in is_same_group()
415 return se->cfs_rq; in is_same_group()
462 static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) in cfs_rq_is_idle() argument
464 return cfs_rq->idle > 0; in cfs_rq_is_idle()
479 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) in list_add_leaf_cfs_rq() argument
484 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) in list_del_leaf_cfs_rq() argument
492 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ argument
493 for (cfs_rq = &rq->cfs, pos = NULL; cfs_rq; cfs_rq = pos)
510 static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) in cfs_rq_is_idle() argument
523 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec);
557 static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) in entity_key() argument
559 return (s64)(se->vruntime - cfs_rq->min_vruntime); in entity_key()
624 avg_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) in avg_vruntime_add() argument
627 s64 key = entity_key(cfs_rq, se); in avg_vruntime_add()
629 cfs_rq->avg_vruntime += key * weight; in avg_vruntime_add()
630 cfs_rq->avg_load += weight; in avg_vruntime_add()
634 avg_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) in avg_vruntime_sub() argument
637 s64 key = entity_key(cfs_rq, se); in avg_vruntime_sub()
639 cfs_rq->avg_vruntime -= key * weight; in avg_vruntime_sub()
640 cfs_rq->avg_load -= weight; in avg_vruntime_sub()
644 void avg_vruntime_update(struct cfs_rq *cfs_rq, s64 delta) in avg_vruntime_update() argument
649 cfs_rq->avg_vruntime -= cfs_rq->avg_load * delta; in avg_vruntime_update()
656 u64 avg_vruntime(struct cfs_rq *cfs_rq) in avg_vruntime() argument
658 struct sched_entity *curr = cfs_rq->curr; in avg_vruntime()
659 s64 avg = cfs_rq->avg_vruntime; in avg_vruntime()
660 long load = cfs_rq->avg_load; in avg_vruntime()
665 avg += entity_key(cfs_rq, curr) * weight; in avg_vruntime()
676 return cfs_rq->min_vruntime + avg; in avg_vruntime()
695 static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) in update_entity_lag() argument
701 vlag = avg_vruntime(cfs_rq) - se->vruntime; in update_entity_lag()
724 static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime) in vruntime_eligible() argument
726 struct sched_entity *curr = cfs_rq->curr; in vruntime_eligible()
727 s64 avg = cfs_rq->avg_vruntime; in vruntime_eligible()
728 long load = cfs_rq->avg_load; in vruntime_eligible()
733 avg += entity_key(cfs_rq, curr) * weight; in vruntime_eligible()
737 return avg >= (s64)(vruntime - cfs_rq->min_vruntime) * load; in vruntime_eligible()
740 int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se) in entity_eligible() argument
742 return vruntime_eligible(cfs_rq, se->vruntime); in entity_eligible()
745 static u64 __update_min_vruntime(struct cfs_rq *cfs_rq, u64 vruntime) in __update_min_vruntime() argument
747 u64 min_vruntime = cfs_rq->min_vruntime; in __update_min_vruntime()
753 avg_vruntime_update(cfs_rq, delta); in __update_min_vruntime()
759 static void update_min_vruntime(struct cfs_rq *cfs_rq) in update_min_vruntime() argument
761 struct sched_entity *se = __pick_root_entity(cfs_rq); in update_min_vruntime()
762 struct sched_entity *curr = cfs_rq->curr; in update_min_vruntime()
763 u64 vruntime = cfs_rq->min_vruntime; in update_min_vruntime()
780 cfs_rq->min_vruntime = __update_min_vruntime(cfs_rq, vruntime); in update_min_vruntime()
783 static inline u64 cfs_rq_min_slice(struct cfs_rq *cfs_rq) in cfs_rq_min_slice() argument
785 struct sched_entity *root = __pick_root_entity(cfs_rq); in cfs_rq_min_slice()
786 struct sched_entity *curr = cfs_rq->curr; in cfs_rq_min_slice()
850 static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) in __enqueue_entity() argument
852 avg_vruntime_add(cfs_rq, se); in __enqueue_entity()
855 rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, in __enqueue_entity()
859 static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) in __dequeue_entity() argument
861 rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, in __dequeue_entity()
863 avg_vruntime_sub(cfs_rq, se); in __dequeue_entity()
866 struct sched_entity *__pick_root_entity(struct cfs_rq *cfs_rq) in __pick_root_entity() argument
868 struct rb_node *root = cfs_rq->tasks_timeline.rb_root.rb_node; in __pick_root_entity()
876 struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) in __pick_first_entity() argument
878 struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline); in __pick_first_entity()
925 static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq) in pick_eevdf() argument
927 struct rb_node *node = cfs_rq->tasks_timeline.rb_root.rb_node; in pick_eevdf()
928 struct sched_entity *se = __pick_first_entity(cfs_rq); in pick_eevdf()
929 struct sched_entity *curr = cfs_rq->curr; in pick_eevdf()
936 if (cfs_rq->nr_queued == 1) in pick_eevdf()
939 if (curr && (!curr->on_rq || !entity_eligible(cfs_rq, curr))) in pick_eevdf()
946 if (se && entity_eligible(cfs_rq, se)) { in pick_eevdf()
959 if (left && vruntime_eligible(cfs_rq, in pick_eevdf()
972 if (entity_eligible(cfs_rq, se)) { in pick_eevdf()
986 struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) in __pick_last_entity() argument
988 struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root); in __pick_last_entity()
1013 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se);
1019 static bool update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se) in update_deadline() argument
1099 struct cfs_rq *cfs_rq = cfs_rq_of(se); in post_init_entity_util_avg() local
1101 long cpu_scale = arch_scale_cpu_capacity(cpu_of(rq_of(cfs_rq))); in post_init_entity_util_avg()
1102 long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2; in post_init_entity_util_avg()
1115 se->avg.last_update_time = cfs_rq_clock_pelt(cfs_rq); in post_init_entity_util_avg()
1120 if (cfs_rq->avg.util_avg != 0) { in post_init_entity_util_avg()
1121 sa->util_avg = cfs_rq->avg.util_avg * se_weight(se); in post_init_entity_util_avg()
1122 sa->util_avg /= (cfs_rq->avg.load_avg + 1); in post_init_entity_util_avg()
1141 static void update_tg_load_avg(struct cfs_rq *cfs_rq) in update_tg_load_avg() argument
1176 static inline bool did_preempt_short(struct cfs_rq *cfs_rq, struct sched_entity *curr) in did_preempt_short() argument
1184 return !entity_eligible(cfs_rq, curr); in did_preempt_short()
1187 static inline bool do_preempt_short(struct cfs_rq *cfs_rq, in do_preempt_short() argument
1196 if (!entity_eligible(cfs_rq, pse)) in do_preempt_short()
1202 if (!entity_eligible(cfs_rq, se)) in do_preempt_short()
1226 static void update_curr(struct cfs_rq *cfs_rq) in update_curr() argument
1228 struct sched_entity *curr = cfs_rq->curr; in update_curr()
1229 struct rq *rq = rq_of(cfs_rq); in update_curr()
1241 resched = update_deadline(cfs_rq, curr); in update_curr()
1242 update_min_vruntime(cfs_rq); in update_curr()
1263 account_cfs_rq_runtime(cfs_rq, delta_exec); in update_curr()
1265 if (cfs_rq->nr_queued == 1) in update_curr()
1268 if (resched || did_preempt_short(cfs_rq, curr)) { in update_curr()
1270 clear_buddies(cfs_rq, curr); in update_curr()
1280 update_stats_wait_start_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) in update_stats_wait_start_fair() argument
1293 __update_stats_wait_start(rq_of(cfs_rq), p, stats); in update_stats_wait_start_fair()
1297 update_stats_wait_end_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) in update_stats_wait_end_fair() argument
1319 __update_stats_wait_end(rq_of(cfs_rq), p, stats); in update_stats_wait_end_fair()
1323 update_stats_enqueue_sleeper_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) in update_stats_enqueue_sleeper_fair() argument
1336 __update_stats_enqueue_sleeper(rq_of(cfs_rq), tsk, stats); in update_stats_enqueue_sleeper_fair()
1343 update_stats_enqueue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) in update_stats_enqueue_fair() argument
1352 if (se != cfs_rq->curr) in update_stats_enqueue_fair()
1353 update_stats_wait_start_fair(cfs_rq, se); in update_stats_enqueue_fair()
1356 update_stats_enqueue_sleeper_fair(cfs_rq, se); in update_stats_enqueue_fair()
1360 update_stats_dequeue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) in update_stats_dequeue_fair() argument
1370 if (se != cfs_rq->curr) in update_stats_dequeue_fair()
1371 update_stats_wait_end_fair(cfs_rq, se); in update_stats_dequeue_fair()
1381 rq_clock(rq_of(cfs_rq))); in update_stats_dequeue_fair()
1384 rq_clock(rq_of(cfs_rq))); in update_stats_dequeue_fair()
1392 update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) in update_stats_curr_start() argument
1397 se->exec_start = rq_clock_task(rq_of(cfs_rq)); in update_stats_curr_start()
3686 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) in account_entity_enqueue() argument
3688 update_load_add(&cfs_rq->load, se->load.weight); in account_entity_enqueue()
3691 struct rq *rq = rq_of(cfs_rq); in account_entity_enqueue()
3697 cfs_rq->nr_queued++; in account_entity_enqueue()
3701 account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) in account_entity_dequeue() argument
3703 update_load_sub(&cfs_rq->load, se->load.weight); in account_entity_dequeue()
3706 account_numa_dequeue(rq_of(cfs_rq), task_of(se)); in account_entity_dequeue()
3710 cfs_rq->nr_queued--; in account_entity_dequeue()
3763 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) in enqueue_load_avg() argument
3765 cfs_rq->avg.load_avg += se->avg.load_avg; in enqueue_load_avg()
3766 cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum; in enqueue_load_avg()
3770 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) in dequeue_load_avg() argument
3772 sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg); in dequeue_load_avg()
3773 sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum); in dequeue_load_avg()
3775 cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum, in dequeue_load_avg()
3776 cfs_rq->avg.load_avg * PELT_MIN_DIVIDER); in dequeue_load_avg()
3780 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } in enqueue_load_avg() argument
3782 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } in dequeue_load_avg() argument
3785 static void place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags);
3787 static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, in reweight_entity() argument
3790 bool curr = cfs_rq->curr == se; in reweight_entity()
3794 update_curr(cfs_rq); in reweight_entity()
3795 update_entity_lag(cfs_rq, se); in reweight_entity()
3799 __dequeue_entity(cfs_rq, se); in reweight_entity()
3800 update_load_sub(&cfs_rq->load, se->load.weight); in reweight_entity()
3802 dequeue_load_avg(cfs_rq, se); in reweight_entity()
3822 enqueue_load_avg(cfs_rq, se); in reweight_entity()
3824 update_load_add(&cfs_rq->load, se->load.weight); in reweight_entity()
3825 place_entity(cfs_rq, se, 0); in reweight_entity()
3827 __enqueue_entity(cfs_rq, se); in reweight_entity()
3836 update_min_vruntime(cfs_rq); in reweight_entity()
3844 struct cfs_rq *cfs_rq = cfs_rq_of(se); in reweight_task_fair() local
3847 reweight_entity(cfs_rq, se, lw->weight); in reweight_task_fair()
3851 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);
3928 static long calc_group_shares(struct cfs_rq *cfs_rq) in calc_group_shares() argument
3931 struct task_group *tg = cfs_rq->tg; in calc_group_shares()
3935 load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); in calc_group_shares()
3940 tg_weight -= cfs_rq->tg_load_avg_contrib; in calc_group_shares()
3969 struct cfs_rq *gcfs_rq = group_cfs_rq(se); in update_cfs_group()
3997 static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq, int flags) in cfs_rq_util_change() argument
3999 struct rq *rq = rq_of(cfs_rq); in cfs_rq_util_change()
4001 if (&rq->cfs == cfs_rq) { in cfs_rq_util_change()
4044 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) in cfs_rq_last_update_time() argument
4046 return u64_u32_load_copy(cfs_rq->avg.last_update_time, in cfs_rq_last_update_time()
4047 cfs_rq->last_update_time_copy); in cfs_rq_last_update_time()
4058 static inline bool child_cfs_rq_on_list(struct cfs_rq *cfs_rq) in child_cfs_rq_on_list() argument
4060 struct cfs_rq *prev_cfs_rq; in child_cfs_rq_on_list()
4062 struct rq *rq = rq_of(cfs_rq); in child_cfs_rq_on_list()
4064 if (cfs_rq->on_list) { in child_cfs_rq_on_list()
4065 prev = cfs_rq->leaf_cfs_rq_list.prev; in child_cfs_rq_on_list()
4073 prev_cfs_rq = container_of(prev, struct cfs_rq, leaf_cfs_rq_list); in child_cfs_rq_on_list()
4075 return (prev_cfs_rq->tg->parent == cfs_rq->tg); in child_cfs_rq_on_list()
4078 static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) in cfs_rq_is_decayed() argument
4080 if (cfs_rq->load.weight) in cfs_rq_is_decayed()
4083 if (!load_avg_is_decayed(&cfs_rq->avg)) in cfs_rq_is_decayed()
4086 if (child_cfs_rq_on_list(cfs_rq)) in cfs_rq_is_decayed()
4106 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) in update_tg_load_avg() argument
4114 if (cfs_rq->tg == &root_task_group) in update_tg_load_avg()
4118 if (!cpu_active(cpu_of(rq_of(cfs_rq)))) in update_tg_load_avg()
4125 now = sched_clock_cpu(cpu_of(rq_of(cfs_rq))); in update_tg_load_avg()
4126 if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC) in update_tg_load_avg()
4129 delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; in update_tg_load_avg()
4130 if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { in update_tg_load_avg()
4131 atomic_long_add(delta, &cfs_rq->tg->load_avg); in update_tg_load_avg()
4132 cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; in update_tg_load_avg()
4133 cfs_rq->last_update_tg_load_avg = now; in update_tg_load_avg()
4137 static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) in clear_tg_load_avg() argument
4145 if (cfs_rq->tg == &root_task_group) in clear_tg_load_avg()
4148 now = sched_clock_cpu(cpu_of(rq_of(cfs_rq))); in clear_tg_load_avg()
4149 delta = 0 - cfs_rq->tg_load_avg_contrib; in clear_tg_load_avg()
4150 atomic_long_add(delta, &cfs_rq->tg->load_avg); in clear_tg_load_avg()
4151 cfs_rq->tg_load_avg_contrib = 0; in clear_tg_load_avg()
4152 cfs_rq->last_update_tg_load_avg = now; in clear_tg_load_avg()
4171 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; in clear_tg_offline_cfs_rqs() local
4173 clear_tg_load_avg(cfs_rq); in clear_tg_offline_cfs_rqs()
4186 struct cfs_rq *prev, struct cfs_rq *next) in set_task_rq_fair()
4279 update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) in update_tg_cfs_util() argument
4292 divider = get_pelt_divider(&cfs_rq->avg); in update_tg_cfs_util()
4302 add_positive(&cfs_rq->avg.util_avg, delta_avg); in update_tg_cfs_util()
4303 add_positive(&cfs_rq->avg.util_sum, delta_sum); in update_tg_cfs_util()
4306 cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum, in update_tg_cfs_util()
4307 cfs_rq->avg.util_avg * PELT_MIN_DIVIDER); in update_tg_cfs_util()
4311 update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) in update_tg_cfs_runnable() argument
4324 divider = get_pelt_divider(&cfs_rq->avg); in update_tg_cfs_runnable()
4333 add_positive(&cfs_rq->avg.runnable_avg, delta_avg); in update_tg_cfs_runnable()
4334 add_positive(&cfs_rq->avg.runnable_sum, delta_sum); in update_tg_cfs_runnable()
4336 cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum, in update_tg_cfs_runnable()
4337 cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER); in update_tg_cfs_runnable()
4341 update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) in update_tg_cfs_load() argument
4358 divider = get_pelt_divider(&cfs_rq->avg); in update_tg_cfs_load()
4401 add_positive(&cfs_rq->avg.load_avg, delta_avg); in update_tg_cfs_load()
4402 add_positive(&cfs_rq->avg.load_sum, delta_sum); in update_tg_cfs_load()
4404 cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum, in update_tg_cfs_load()
4405 cfs_rq->avg.load_avg * PELT_MIN_DIVIDER); in update_tg_cfs_load()
4408 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) in add_tg_cfs_propagate() argument
4410 cfs_rq->propagate = 1; in add_tg_cfs_propagate()
4411 cfs_rq->prop_runnable_sum += runnable_sum; in add_tg_cfs_propagate()
4417 struct cfs_rq *cfs_rq, *gcfs_rq; in propagate_entity_load_avg() local
4428 cfs_rq = cfs_rq_of(se); in propagate_entity_load_avg()
4430 add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum); in propagate_entity_load_avg()
4432 update_tg_cfs_util(cfs_rq, se, gcfs_rq); in propagate_entity_load_avg()
4433 update_tg_cfs_runnable(cfs_rq, se, gcfs_rq); in propagate_entity_load_avg()
4434 update_tg_cfs_load(cfs_rq, se, gcfs_rq); in propagate_entity_load_avg()
4436 trace_pelt_cfs_tp(cfs_rq); in propagate_entity_load_avg()
4448 struct cfs_rq *gcfs_rq = group_cfs_rq(se); in skip_blocked_update()
4474 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) {} in update_tg_load_avg() argument
4483 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) {} in add_tg_cfs_propagate() argument
4491 struct cfs_rq *cfs_rq; in migrate_se_pelt_lag() local
4498 cfs_rq = cfs_rq_of(se); in migrate_se_pelt_lag()
4499 rq = rq_of(cfs_rq); in migrate_se_pelt_lag()
4539 throttled = u64_u32_load(cfs_rq->throttled_pelt_idle); in migrate_se_pelt_lag()
4552 lut = cfs_rq_last_update_time(cfs_rq); in migrate_se_pelt_lag()
4586 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) in update_cfs_rq_load_avg() argument
4589 struct sched_avg *sa = &cfs_rq->avg; in update_cfs_rq_load_avg()
4592 if (cfs_rq->removed.nr) { in update_cfs_rq_load_avg()
4594 u32 divider = get_pelt_divider(&cfs_rq->avg); in update_cfs_rq_load_avg()
4596 raw_spin_lock(&cfs_rq->removed.lock); in update_cfs_rq_load_avg()
4597 swap(cfs_rq->removed.util_avg, removed_util); in update_cfs_rq_load_avg()
4598 swap(cfs_rq->removed.load_avg, removed_load); in update_cfs_rq_load_avg()
4599 swap(cfs_rq->removed.runnable_avg, removed_runnable); in update_cfs_rq_load_avg()
4600 cfs_rq->removed.nr = 0; in update_cfs_rq_load_avg()
4601 raw_spin_unlock(&cfs_rq->removed.lock); in update_cfs_rq_load_avg()
4636 add_tg_cfs_propagate(cfs_rq, in update_cfs_rq_load_avg()
4642 decayed |= __update_load_avg_cfs_rq(now, cfs_rq); in update_cfs_rq_load_avg()
4644 cfs_rq->last_update_time_copy, in update_cfs_rq_load_avg()
4657 static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) in attach_entity_load_avg() argument
4663 u32 divider = get_pelt_divider(&cfs_rq->avg); in attach_entity_load_avg()
4672 se->avg.last_update_time = cfs_rq->avg.last_update_time; in attach_entity_load_avg()
4673 se->avg.period_contrib = cfs_rq->avg.period_contrib; in attach_entity_load_avg()
4691 enqueue_load_avg(cfs_rq, se); in attach_entity_load_avg()
4692 cfs_rq->avg.util_avg += se->avg.util_avg; in attach_entity_load_avg()
4693 cfs_rq->avg.util_sum += se->avg.util_sum; in attach_entity_load_avg()
4694 cfs_rq->avg.runnable_avg += se->avg.runnable_avg; in attach_entity_load_avg()
4695 cfs_rq->avg.runnable_sum += se->avg.runnable_sum; in attach_entity_load_avg()
4697 add_tg_cfs_propagate(cfs_rq, se->avg.load_sum); in attach_entity_load_avg()
4699 cfs_rq_util_change(cfs_rq, 0); in attach_entity_load_avg()
4701 trace_pelt_cfs_tp(cfs_rq); in attach_entity_load_avg()
4712 static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) in detach_entity_load_avg() argument
4714 dequeue_load_avg(cfs_rq, se); in detach_entity_load_avg()
4715 sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg); in detach_entity_load_avg()
4716 sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum); in detach_entity_load_avg()
4718 cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum, in detach_entity_load_avg()
4719 cfs_rq->avg.util_avg * PELT_MIN_DIVIDER); in detach_entity_load_avg()
4721 sub_positive(&cfs_rq->avg.runnable_avg, se->avg.runnable_avg); in detach_entity_load_avg()
4722 sub_positive(&cfs_rq->avg.runnable_sum, se->avg.runnable_sum); in detach_entity_load_avg()
4724 cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum, in detach_entity_load_avg()
4725 cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER); in detach_entity_load_avg()
4727 add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum); in detach_entity_load_avg()
4729 cfs_rq_util_change(cfs_rq, 0); in detach_entity_load_avg()
4731 trace_pelt_cfs_tp(cfs_rq); in detach_entity_load_avg()
4743 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) in update_load_avg() argument
4745 u64 now = cfs_rq_clock_pelt(cfs_rq); in update_load_avg()
4753 __update_load_avg_se(now, cfs_rq, se); in update_load_avg()
4755 decayed = update_cfs_rq_load_avg(now, cfs_rq); in update_load_avg()
4767 attach_entity_load_avg(cfs_rq, se); in update_load_avg()
4768 update_tg_load_avg(cfs_rq); in update_load_avg()
4775 detach_entity_load_avg(cfs_rq, se); in update_load_avg()
4776 update_tg_load_avg(cfs_rq); in update_load_avg()
4778 cfs_rq_util_change(cfs_rq, 0); in update_load_avg()
4781 update_tg_load_avg(cfs_rq); in update_load_avg()
4791 struct cfs_rq *cfs_rq = cfs_rq_of(se); in sync_entity_load_avg() local
4794 last_update_time = cfs_rq_last_update_time(cfs_rq); in sync_entity_load_avg()
4804 struct cfs_rq *cfs_rq = cfs_rq_of(se); in remove_entity_load_avg() local
4815 raw_spin_lock_irqsave(&cfs_rq->removed.lock, flags); in remove_entity_load_avg()
4816 ++cfs_rq->removed.nr; in remove_entity_load_avg()
4817 cfs_rq->removed.util_avg += se->avg.util_avg; in remove_entity_load_avg()
4818 cfs_rq->removed.load_avg += se->avg.load_avg; in remove_entity_load_avg()
4819 cfs_rq->removed.runnable_avg += se->avg.runnable_avg; in remove_entity_load_avg()
4820 raw_spin_unlock_irqrestore(&cfs_rq->removed.lock, flags); in remove_entity_load_avg()
4823 static inline unsigned long cfs_rq_runnable_avg(struct cfs_rq *cfs_rq) in cfs_rq_runnable_avg() argument
4825 return cfs_rq->avg.runnable_avg; in cfs_rq_runnable_avg()
4828 static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) in cfs_rq_load_avg() argument
4830 return cfs_rq->avg.load_avg; in cfs_rq_load_avg()
4855 static inline void util_est_enqueue(struct cfs_rq *cfs_rq, in util_est_enqueue() argument
4864 enqueued = cfs_rq->avg.util_est; in util_est_enqueue()
4866 WRITE_ONCE(cfs_rq->avg.util_est, enqueued); in util_est_enqueue()
4868 trace_sched_util_est_cfs_tp(cfs_rq); in util_est_enqueue()
4871 static inline void util_est_dequeue(struct cfs_rq *cfs_rq, in util_est_dequeue() argument
4880 enqueued = cfs_rq->avg.util_est; in util_est_dequeue()
4882 WRITE_ONCE(cfs_rq->avg.util_est, enqueued); in util_est_dequeue()
4884 trace_sched_util_est_cfs_tp(cfs_rq); in util_est_dequeue()
4889 static inline void util_est_update(struct cfs_rq *cfs_rq, in util_est_update() argument
4939 if (dequeued > arch_scale_cpu_capacity(cpu_of(rq_of(cfs_rq)))) in util_est_update()
5144 static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) in cfs_rq_is_decayed() argument
5146 return !cfs_rq->nr_queued; in cfs_rq_is_decayed()
5154 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int not_used1) in update_load_avg() argument
5156 cfs_rq_util_change(cfs_rq, 0); in update_load_avg()
5162 attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} in attach_entity_load_avg() argument
5164 detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} in detach_entity_load_avg() argument
5172 util_est_enqueue(struct cfs_rq *cfs_rq, struct task_struct *p) {} in util_est_enqueue() argument
5175 util_est_dequeue(struct cfs_rq *cfs_rq, struct task_struct *p) {} in util_est_dequeue() argument
5178 util_est_update(struct cfs_rq *cfs_rq, struct task_struct *p, in util_est_update() argument
5201 place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) in place_entity() argument
5203 u64 vslice, vruntime = avg_vruntime(cfs_rq); in place_entity()
5218 if (sched_feat(PLACE_LAG) && cfs_rq->nr_queued && se->vlag) { in place_entity()
5219 struct sched_entity *curr = cfs_rq->curr; in place_entity()
5276 load = cfs_rq->avg_load; in place_entity()
5308 static void check_enqueue_throttle(struct cfs_rq *cfs_rq);
5309 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq);
5315 enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) in enqueue_entity() argument
5317 bool curr = cfs_rq->curr == se; in enqueue_entity()
5324 place_entity(cfs_rq, se, flags); in enqueue_entity()
5326 update_curr(cfs_rq); in enqueue_entity()
5337 update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH); in enqueue_entity()
5351 place_entity(cfs_rq, se, flags); in enqueue_entity()
5353 account_entity_enqueue(cfs_rq, se); in enqueue_entity()
5360 update_stats_enqueue_fair(cfs_rq, se, flags); in enqueue_entity()
5362 __enqueue_entity(cfs_rq, se); in enqueue_entity()
5365 if (cfs_rq->nr_queued == 1) { in enqueue_entity()
5366 check_enqueue_throttle(cfs_rq); in enqueue_entity()
5367 if (!throttled_hierarchy(cfs_rq)) { in enqueue_entity()
5368 list_add_leaf_cfs_rq(cfs_rq); in enqueue_entity()
5371 struct rq *rq = rq_of(cfs_rq); in enqueue_entity()
5373 if (cfs_rq_throttled(cfs_rq) && !cfs_rq->throttled_clock) in enqueue_entity()
5374 cfs_rq->throttled_clock = rq_clock(rq); in enqueue_entity()
5375 if (!cfs_rq->throttled_clock_self) in enqueue_entity()
5376 cfs_rq->throttled_clock_self = rq_clock(rq); in enqueue_entity()
5385 struct cfs_rq *cfs_rq = cfs_rq_of(se); in __clear_buddies_next() local
5386 if (cfs_rq->next != se) in __clear_buddies_next()
5389 cfs_rq->next = NULL; in __clear_buddies_next()
5393 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) in clear_buddies() argument
5395 if (cfs_rq->next == se) in clear_buddies()
5399 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
5414 struct cfs_rq *cfs_rq = cfs_rq_of(se); in set_delayed() local
5416 cfs_rq->h_nr_runnable--; in set_delayed()
5417 if (cfs_rq_throttled(cfs_rq)) in set_delayed()
5436 struct cfs_rq *cfs_rq = cfs_rq_of(se); in clear_delayed() local
5438 cfs_rq->h_nr_runnable++; in clear_delayed()
5439 if (cfs_rq_throttled(cfs_rq)) in clear_delayed()
5452 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) in dequeue_entity() argument
5457 update_curr(cfs_rq); in dequeue_entity()
5458 clear_buddies(cfs_rq, se); in dequeue_entity()
5474 !entity_eligible(cfs_rq, se)) { in dequeue_entity()
5475 update_load_avg(cfs_rq, se, 0); in dequeue_entity()
5493 update_load_avg(cfs_rq, se, action); in dequeue_entity()
5496 update_stats_dequeue_fair(cfs_rq, se, flags); in dequeue_entity()
5498 update_entity_lag(cfs_rq, se); in dequeue_entity()
5504 if (se != cfs_rq->curr) in dequeue_entity()
5505 __dequeue_entity(cfs_rq, se); in dequeue_entity()
5507 account_entity_dequeue(cfs_rq, se); in dequeue_entity()
5510 return_cfs_rq_runtime(cfs_rq); in dequeue_entity()
5521 update_min_vruntime(cfs_rq); in dequeue_entity()
5526 if (cfs_rq->nr_queued == 0) in dequeue_entity()
5527 update_idle_cfs_rq_clock_pelt(cfs_rq); in dequeue_entity()
5533 set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) in set_next_entity() argument
5535 clear_buddies(cfs_rq, se); in set_next_entity()
5544 update_stats_wait_end_fair(cfs_rq, se); in set_next_entity()
5545 __dequeue_entity(cfs_rq, se); in set_next_entity()
5546 update_load_avg(cfs_rq, se, UPDATE_TG); in set_next_entity()
5551 update_stats_curr_start(cfs_rq, se); in set_next_entity()
5552 WARN_ON_ONCE(cfs_rq->curr); in set_next_entity()
5553 cfs_rq->curr = se; in set_next_entity()
5561 rq_of(cfs_rq)->cfs.load.weight >= 2*se->load.weight) { in set_next_entity()
5583 pick_next_entity(struct rq *rq, struct cfs_rq *cfs_rq) in pick_next_entity() argument
5591 cfs_rq->next && entity_eligible(cfs_rq, cfs_rq->next)) { in pick_next_entity()
5593 WARN_ON_ONCE(cfs_rq->next->sched_delayed); in pick_next_entity()
5594 return cfs_rq->next; in pick_next_entity()
5597 se = pick_eevdf(cfs_rq); in pick_next_entity()
5608 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq);
5610 static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) in put_prev_entity() argument
5617 update_curr(cfs_rq); in put_prev_entity()
5620 check_cfs_rq_runtime(cfs_rq); in put_prev_entity()
5623 update_stats_wait_start_fair(cfs_rq, prev); in put_prev_entity()
5625 __enqueue_entity(cfs_rq, prev); in put_prev_entity()
5627 update_load_avg(cfs_rq, prev, 0); in put_prev_entity()
5629 WARN_ON_ONCE(cfs_rq->curr != prev); in put_prev_entity()
5630 cfs_rq->curr = NULL; in put_prev_entity()
5634 entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) in entity_tick() argument
5639 update_curr(cfs_rq); in entity_tick()
5644 update_load_avg(cfs_rq, curr, UPDATE_TG); in entity_tick()
5653 resched_curr_lazy(rq_of(cfs_rq)); in entity_tick()
5739 struct cfs_rq *cfs_rq, u64 target_runtime) in __assign_cfs_rq_runtime() argument
5746 min_amount = target_runtime - cfs_rq->runtime_remaining; in __assign_cfs_rq_runtime()
5760 cfs_rq->runtime_remaining += amount; in __assign_cfs_rq_runtime()
5762 return cfs_rq->runtime_remaining > 0; in __assign_cfs_rq_runtime()
5766 static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) in assign_cfs_rq_runtime() argument
5768 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); in assign_cfs_rq_runtime()
5772 ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice()); in assign_cfs_rq_runtime()
5778 static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) in __account_cfs_rq_runtime() argument
5781 cfs_rq->runtime_remaining -= delta_exec; in __account_cfs_rq_runtime()
5783 if (likely(cfs_rq->runtime_remaining > 0)) in __account_cfs_rq_runtime()
5786 if (cfs_rq->throttled) in __account_cfs_rq_runtime()
5792 if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) in __account_cfs_rq_runtime()
5793 resched_curr(rq_of(cfs_rq)); in __account_cfs_rq_runtime()
5797 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) in account_cfs_rq_runtime() argument
5799 if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled) in account_cfs_rq_runtime()
5802 __account_cfs_rq_runtime(cfs_rq, delta_exec); in account_cfs_rq_runtime()
5805 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) in cfs_rq_throttled() argument
5807 return cfs_bandwidth_used() && cfs_rq->throttled; in cfs_rq_throttled()
5811 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) in throttled_hierarchy() argument
5813 return cfs_bandwidth_used() && cfs_rq->throttle_count; in throttled_hierarchy()
5824 struct cfs_rq *src_cfs_rq, *dest_cfs_rq; in throttled_lb_pair()
5826 src_cfs_rq = tg->cfs_rq[src_cpu]; in throttled_lb_pair()
5827 dest_cfs_rq = tg->cfs_rq[dest_cpu]; in throttled_lb_pair()
5836 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; in tg_unthrottle_up() local
5838 cfs_rq->throttle_count--; in tg_unthrottle_up()
5839 if (!cfs_rq->throttle_count) { in tg_unthrottle_up()
5840 cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) - in tg_unthrottle_up()
5841 cfs_rq->throttled_clock_pelt; in tg_unthrottle_up()
5844 if (!cfs_rq_is_decayed(cfs_rq)) in tg_unthrottle_up()
5845 list_add_leaf_cfs_rq(cfs_rq); in tg_unthrottle_up()
5847 if (cfs_rq->throttled_clock_self) { in tg_unthrottle_up()
5848 u64 delta = rq_clock(rq) - cfs_rq->throttled_clock_self; in tg_unthrottle_up()
5850 cfs_rq->throttled_clock_self = 0; in tg_unthrottle_up()
5855 cfs_rq->throttled_clock_self_time += delta; in tg_unthrottle_up()
5865 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; in tg_throttle_down() local
5868 if (!cfs_rq->throttle_count) { in tg_throttle_down()
5869 cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq); in tg_throttle_down()
5870 list_del_leaf_cfs_rq(cfs_rq); in tg_throttle_down()
5872 WARN_ON_ONCE(cfs_rq->throttled_clock_self); in tg_throttle_down()
5873 if (cfs_rq->nr_queued) in tg_throttle_down()
5874 cfs_rq->throttled_clock_self = rq_clock(rq); in tg_throttle_down()
5876 cfs_rq->throttle_count++; in tg_throttle_down()
5881 static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) in throttle_cfs_rq() argument
5883 struct rq *rq = rq_of(cfs_rq); in throttle_cfs_rq()
5884 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); in throttle_cfs_rq()
5891 if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) { in throttle_cfs_rq()
5902 list_add_tail_rcu(&cfs_rq->throttled_list, in throttle_cfs_rq()
5910 se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))]; in throttle_cfs_rq()
5914 walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); in throttle_cfs_rq()
5917 queued_delta = cfs_rq->h_nr_queued; in throttle_cfs_rq()
5918 runnable_delta = cfs_rq->h_nr_runnable; in throttle_cfs_rq()
5919 idle_delta = cfs_rq->h_nr_idle; in throttle_cfs_rq()
5921 struct cfs_rq *qcfs_rq = cfs_rq_of(se); in throttle_cfs_rq()
5939 idle_delta = cfs_rq->h_nr_queued; in throttle_cfs_rq()
5953 struct cfs_rq *qcfs_rq = cfs_rq_of(se); in throttle_cfs_rq()
5962 idle_delta = cfs_rq->h_nr_queued; in throttle_cfs_rq()
5980 cfs_rq->throttled = 1; in throttle_cfs_rq()
5981 WARN_ON_ONCE(cfs_rq->throttled_clock); in throttle_cfs_rq()
5982 if (cfs_rq->nr_queued) in throttle_cfs_rq()
5983 cfs_rq->throttled_clock = rq_clock(rq); in throttle_cfs_rq()
5987 void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) in unthrottle_cfs_rq() argument
5989 struct rq *rq = rq_of(cfs_rq); in unthrottle_cfs_rq()
5990 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); in unthrottle_cfs_rq()
5995 se = cfs_rq->tg->se[cpu_of(rq)]; in unthrottle_cfs_rq()
5997 cfs_rq->throttled = 0; in unthrottle_cfs_rq()
6002 if (cfs_rq->throttled_clock) { in unthrottle_cfs_rq()
6003 cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; in unthrottle_cfs_rq()
6004 cfs_rq->throttled_clock = 0; in unthrottle_cfs_rq()
6006 list_del_rcu(&cfs_rq->throttled_list); in unthrottle_cfs_rq()
6010 walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); in unthrottle_cfs_rq()
6012 if (!cfs_rq->load.weight) { in unthrottle_cfs_rq()
6013 if (!cfs_rq->on_list) in unthrottle_cfs_rq()
6026 queued_delta = cfs_rq->h_nr_queued; in unthrottle_cfs_rq()
6027 runnable_delta = cfs_rq->h_nr_runnable; in unthrottle_cfs_rq()
6028 idle_delta = cfs_rq->h_nr_idle; in unthrottle_cfs_rq()
6030 struct cfs_rq *qcfs_rq = cfs_rq_of(se); in unthrottle_cfs_rq()
6042 idle_delta = cfs_rq->h_nr_queued; in unthrottle_cfs_rq()
6054 struct cfs_rq *qcfs_rq = cfs_rq_of(se); in unthrottle_cfs_rq()
6060 idle_delta = cfs_rq->h_nr_queued; in unthrottle_cfs_rq()
6089 struct cfs_rq *cursor, *tmp; in __cfsb_csd_unthrottle()
6126 static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) in __unthrottle_cfs_rq_async() argument
6128 struct rq *rq = rq_of(cfs_rq); in __unthrottle_cfs_rq_async()
6132 unthrottle_cfs_rq(cfs_rq); in __unthrottle_cfs_rq_async()
6137 if (WARN_ON_ONCE(!list_empty(&cfs_rq->throttled_csd_list))) in __unthrottle_cfs_rq_async()
6141 list_add_tail(&cfs_rq->throttled_csd_list, &rq->cfsb_csd_list); in __unthrottle_cfs_rq_async()
6146 static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) in __unthrottle_cfs_rq_async() argument
6148 unthrottle_cfs_rq(cfs_rq); in __unthrottle_cfs_rq_async()
6152 static void unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) in unthrottle_cfs_rq_async() argument
6154 lockdep_assert_rq_held(rq_of(cfs_rq)); in unthrottle_cfs_rq_async()
6156 if (WARN_ON_ONCE(!cfs_rq_throttled(cfs_rq) || in unthrottle_cfs_rq_async()
6157 cfs_rq->runtime_remaining <= 0)) in unthrottle_cfs_rq_async()
6160 __unthrottle_cfs_rq_async(cfs_rq); in unthrottle_cfs_rq_async()
6168 struct cfs_rq *cfs_rq, *tmp; in distribute_cfs_runtime() local
6174 list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, in distribute_cfs_runtime()
6176 rq = rq_of(cfs_rq); in distribute_cfs_runtime()
6184 if (!cfs_rq_throttled(cfs_rq)) in distribute_cfs_runtime()
6188 if (!list_empty(&cfs_rq->throttled_csd_list)) in distribute_cfs_runtime()
6192 WARN_ON_ONCE(cfs_rq->runtime_remaining > 0); in distribute_cfs_runtime()
6195 runtime = -cfs_rq->runtime_remaining + 1; in distribute_cfs_runtime()
6202 cfs_rq->runtime_remaining += runtime; in distribute_cfs_runtime()
6205 if (cfs_rq->runtime_remaining > 0) { in distribute_cfs_runtime()
6207 unthrottle_cfs_rq_async(cfs_rq); in distribute_cfs_runtime()
6214 list_add_tail(&cfs_rq->throttled_csd_list, in distribute_cfs_runtime()
6225 list_for_each_entry_safe(cfs_rq, tmp, &local_unthrottle, in distribute_cfs_runtime()
6227 struct rq *rq = rq_of(cfs_rq); in distribute_cfs_runtime()
6231 list_del_init(&cfs_rq->throttled_csd_list); in distribute_cfs_runtime()
6233 if (cfs_rq_throttled(cfs_rq)) in distribute_cfs_runtime()
6234 unthrottle_cfs_rq(cfs_rq); in distribute_cfs_runtime()
6355 static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq) in __return_cfs_rq_runtime() argument
6357 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); in __return_cfs_rq_runtime()
6358 s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime; in __return_cfs_rq_runtime()
6375 cfs_rq->runtime_remaining -= slack_runtime; in __return_cfs_rq_runtime()
6378 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) in return_cfs_rq_runtime() argument
6383 if (!cfs_rq->runtime_enabled || cfs_rq->nr_queued) in return_cfs_rq_runtime()
6386 __return_cfs_rq_runtime(cfs_rq); in return_cfs_rq_runtime()
6423 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) in check_enqueue_throttle() argument
6429 if (!cfs_rq->runtime_enabled || cfs_rq->curr) in check_enqueue_throttle()
6433 if (cfs_rq_throttled(cfs_rq)) in check_enqueue_throttle()
6437 account_cfs_rq_runtime(cfs_rq, 0); in check_enqueue_throttle()
6438 if (cfs_rq->runtime_remaining <= 0) in check_enqueue_throttle()
6439 throttle_cfs_rq(cfs_rq); in check_enqueue_throttle()
6444 struct cfs_rq *pcfs_rq, *cfs_rq; in sync_throttle() local
6452 cfs_rq = tg->cfs_rq[cpu]; in sync_throttle()
6453 pcfs_rq = tg->parent->cfs_rq[cpu]; in sync_throttle()
6455 cfs_rq->throttle_count = pcfs_rq->throttle_count; in sync_throttle()
6456 cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu)); in sync_throttle()
6460 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) in check_cfs_rq_runtime() argument
6465 if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0)) in check_cfs_rq_runtime()
6472 if (cfs_rq_throttled(cfs_rq)) in check_cfs_rq_runtime()
6475 return throttle_cfs_rq(cfs_rq); in check_cfs_rq_runtime()
6566 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) in init_cfs_rq_runtime() argument
6568 cfs_rq->runtime_enabled = 0; in init_cfs_rq_runtime()
6569 INIT_LIST_HEAD(&cfs_rq->throttled_list); in init_cfs_rq_runtime()
6570 INIT_LIST_HEAD(&cfs_rq->throttled_csd_list); in init_cfs_rq_runtime()
6638 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; in update_runtime_enabled() local
6641 cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; in update_runtime_enabled()
6667 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; in unthrottle_offline_cfs_rqs() local
6669 if (!cfs_rq->runtime_enabled) in unthrottle_offline_cfs_rqs()
6676 cfs_rq->runtime_enabled = 0; in unthrottle_offline_cfs_rqs()
6678 if (!cfs_rq_throttled(cfs_rq)) in unthrottle_offline_cfs_rqs()
6685 cfs_rq->runtime_remaining = 1; in unthrottle_offline_cfs_rqs()
6686 unthrottle_cfs_rq(cfs_rq); in unthrottle_offline_cfs_rqs()
6695 struct cfs_rq *cfs_rq = task_cfs_rq(p); in cfs_task_bw_constrained() local
6700 if (cfs_rq->runtime_enabled || in cfs_task_bw_constrained()
6701 tg_cfs_bandwidth(cfs_rq->tg)->hierarchical_quota != RUNTIME_INF) in cfs_task_bw_constrained()
6735 static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {} in account_cfs_rq_runtime() argument
6736 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; } in check_cfs_rq_runtime() argument
6737 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {} in check_enqueue_throttle() argument
6739 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} in return_cfs_rq_runtime() argument
6741 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) in cfs_rq_throttled() argument
6746 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) in throttled_hierarchy() argument
6759 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} in init_cfs_rq_runtime() argument
6894 struct cfs_rq *cfs_rq = cfs_rq_of(se); in requeue_delayed_entity() local
6905 update_entity_lag(cfs_rq, se); in requeue_delayed_entity()
6907 cfs_rq->nr_queued--; in requeue_delayed_entity()
6908 if (se != cfs_rq->curr) in requeue_delayed_entity()
6909 __dequeue_entity(cfs_rq, se); in requeue_delayed_entity()
6911 place_entity(cfs_rq, se, 0); in requeue_delayed_entity()
6912 if (se != cfs_rq->curr) in requeue_delayed_entity()
6913 __enqueue_entity(cfs_rq, se); in requeue_delayed_entity()
6914 cfs_rq->nr_queued++; in requeue_delayed_entity()
6918 update_load_avg(cfs_rq, se, 0); in requeue_delayed_entity()
6930 struct cfs_rq *cfs_rq; in enqueue_task_fair() local
6969 cfs_rq = cfs_rq_of(se); in enqueue_task_fair()
6980 enqueue_entity(cfs_rq, se, flags); in enqueue_task_fair()
6981 slice = cfs_rq_min_slice(cfs_rq); in enqueue_task_fair()
6983 cfs_rq->h_nr_runnable += h_nr_runnable; in enqueue_task_fair()
6984 cfs_rq->h_nr_queued++; in enqueue_task_fair()
6985 cfs_rq->h_nr_idle += h_nr_idle; in enqueue_task_fair()
6987 if (cfs_rq_is_idle(cfs_rq)) in enqueue_task_fair()
6991 if (cfs_rq_throttled(cfs_rq)) in enqueue_task_fair()
6998 cfs_rq = cfs_rq_of(se); in enqueue_task_fair()
7000 update_load_avg(cfs_rq, se, UPDATE_TG); in enqueue_task_fair()
7005 if (se != cfs_rq->curr) in enqueue_task_fair()
7007 slice = cfs_rq_min_slice(cfs_rq); in enqueue_task_fair()
7009 cfs_rq->h_nr_runnable += h_nr_runnable; in enqueue_task_fair()
7010 cfs_rq->h_nr_queued++; in enqueue_task_fair()
7011 cfs_rq->h_nr_idle += h_nr_idle; in enqueue_task_fair()
7013 if (cfs_rq_is_idle(cfs_rq)) in enqueue_task_fair()
7017 if (cfs_rq_throttled(cfs_rq)) in enqueue_task_fair()
7075 struct cfs_rq *cfs_rq; in dequeue_entities() local
7087 cfs_rq = cfs_rq_of(se); in dequeue_entities()
7089 if (!dequeue_entity(cfs_rq, se, flags)) { in dequeue_entities()
7093 slice = cfs_rq_min_slice(cfs_rq); in dequeue_entities()
7097 cfs_rq->h_nr_runnable -= h_nr_runnable; in dequeue_entities()
7098 cfs_rq->h_nr_queued -= h_nr_queued; in dequeue_entities()
7099 cfs_rq->h_nr_idle -= h_nr_idle; in dequeue_entities()
7101 if (cfs_rq_is_idle(cfs_rq)) in dequeue_entities()
7105 if (cfs_rq_throttled(cfs_rq)) in dequeue_entities()
7109 if (cfs_rq->load.weight) { in dequeue_entities()
7110 slice = cfs_rq_min_slice(cfs_rq); in dequeue_entities()
7118 if (task_sleep && se && !throttled_hierarchy(cfs_rq)) in dequeue_entities()
7127 cfs_rq = cfs_rq_of(se); in dequeue_entities()
7129 update_load_avg(cfs_rq, se, UPDATE_TG); in dequeue_entities()
7134 if (se != cfs_rq->curr) in dequeue_entities()
7136 slice = cfs_rq_min_slice(cfs_rq); in dequeue_entities()
7138 cfs_rq->h_nr_runnable -= h_nr_runnable; in dequeue_entities()
7139 cfs_rq->h_nr_queued -= h_nr_queued; in dequeue_entities()
7140 cfs_rq->h_nr_idle -= h_nr_idle; in dequeue_entities()
7142 if (cfs_rq_is_idle(cfs_rq)) in dequeue_entities()
7146 if (cfs_rq_throttled(cfs_rq)) in dequeue_entities()
7239 struct cfs_rq *cfs_rq; in cpu_load_without() local
7246 cfs_rq = &rq->cfs; in cpu_load_without()
7247 load = READ_ONCE(cfs_rq->avg.load_avg); in cpu_load_without()
7262 struct cfs_rq *cfs_rq; in cpu_runnable_without() local
7269 cfs_rq = &rq->cfs; in cpu_runnable_without()
7270 runnable = READ_ONCE(cfs_rq->avg.runnable_avg); in cpu_runnable_without()
8000 struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs; in cpu_util() local
8001 unsigned long util = READ_ONCE(cfs_rq->avg.util_avg); in cpu_util()
8005 runnable = READ_ONCE(cfs_rq->avg.runnable_avg); in cpu_util()
8023 util_est = READ_ONCE(cfs_rq->avg.util_est); in cpu_util()
8754 struct cfs_rq *cfs_rq = task_cfs_rq(donor); in check_preempt_wakeup_fair() local
8817 cfs_rq = cfs_rq_of(se); in check_preempt_wakeup_fair()
8818 update_curr(cfs_rq); in check_preempt_wakeup_fair()
8826 if (do_preempt_short(cfs_rq, pse, se)) in check_preempt_wakeup_fair()
8832 if (pick_eevdf(cfs_rq) == pse) in check_preempt_wakeup_fair()
8844 struct cfs_rq *cfs_rq; in pick_task_fair() local
8847 cfs_rq = &rq->cfs; in pick_task_fair()
8848 if (!cfs_rq->nr_queued) in pick_task_fair()
8853 if (cfs_rq->curr && cfs_rq->curr->on_rq) in pick_task_fair()
8854 update_curr(cfs_rq); in pick_task_fair()
8856 if (unlikely(check_cfs_rq_runtime(cfs_rq))) in pick_task_fair()
8859 se = pick_next_entity(rq, cfs_rq); in pick_task_fair()
8862 cfs_rq = group_cfs_rq(se); in pick_task_fair()
8863 } while (cfs_rq); in pick_task_fair()
8903 struct cfs_rq *cfs_rq; in pick_next_task_fair() local
8905 while (!(cfs_rq = is_same_group(se, pse))) { in pick_next_task_fair()
8919 put_prev_entity(cfs_rq, pse); in pick_next_task_fair()
8920 set_next_entity(cfs_rq, se); in pick_next_task_fair()
8988 struct cfs_rq *cfs_rq; in put_prev_task_fair() local
8991 cfs_rq = cfs_rq_of(se); in put_prev_task_fair()
8992 put_prev_entity(cfs_rq, se); in put_prev_task_fair()
9002 struct cfs_rq *cfs_rq = task_cfs_rq(curr); in yield_task_fair() local
9011 clear_buddies(cfs_rq, se); in yield_task_fair()
9017 update_curr(cfs_rq); in yield_task_fair()
9364 struct cfs_rq *dst_cfs_rq; in task_is_ineligible_on_dst_cpu()
9367 dst_cfs_rq = task_group(p)->cfs_rq[dest_cpu]; in task_is_ineligible_on_dst_cpu()
9723 static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) in cfs_rq_has_blocked() argument
9725 if (cfs_rq->avg.load_avg) in cfs_rq_has_blocked()
9728 if (cfs_rq->avg.util_avg) in cfs_rq_has_blocked()
9762 static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) { return false; } in cfs_rq_has_blocked() argument
9788 struct cfs_rq *cfs_rq, *pos; in __update_blocked_fair() local
9796 for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) { in __update_blocked_fair()
9799 if (update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq)) { in __update_blocked_fair()
9800 update_tg_load_avg(cfs_rq); in __update_blocked_fair()
9802 if (cfs_rq->nr_queued == 0) in __update_blocked_fair()
9803 update_idle_cfs_rq_clock_pelt(cfs_rq); in __update_blocked_fair()
9805 if (cfs_rq == &rq->cfs) in __update_blocked_fair()
9810 se = cfs_rq->tg->se[cpu]; in __update_blocked_fair()
9818 if (cfs_rq_is_decayed(cfs_rq)) in __update_blocked_fair()
9819 list_del_leaf_cfs_rq(cfs_rq); in __update_blocked_fair()
9822 if (cfs_rq_has_blocked(cfs_rq)) in __update_blocked_fair()
9834 static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) in update_cfs_rq_h_load() argument
9836 struct rq *rq = rq_of(cfs_rq); in update_cfs_rq_h_load()
9837 struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; in update_cfs_rq_h_load()
9841 if (cfs_rq->last_h_load_update == now) in update_cfs_rq_h_load()
9844 WRITE_ONCE(cfs_rq->h_load_next, NULL); in update_cfs_rq_h_load()
9846 cfs_rq = cfs_rq_of(se); in update_cfs_rq_h_load()
9847 WRITE_ONCE(cfs_rq->h_load_next, se); in update_cfs_rq_h_load()
9848 if (cfs_rq->last_h_load_update == now) in update_cfs_rq_h_load()
9853 cfs_rq->h_load = cfs_rq_load_avg(cfs_rq); in update_cfs_rq_h_load()
9854 cfs_rq->last_h_load_update = now; in update_cfs_rq_h_load()
9857 while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) { in update_cfs_rq_h_load()
9858 load = cfs_rq->h_load; in update_cfs_rq_h_load()
9860 cfs_rq_load_avg(cfs_rq) + 1); in update_cfs_rq_h_load()
9861 cfs_rq = group_cfs_rq(se); in update_cfs_rq_h_load()
9862 cfs_rq->h_load = load; in update_cfs_rq_h_load()
9863 cfs_rq->last_h_load_update = now; in update_cfs_rq_h_load()
9869 struct cfs_rq *cfs_rq = task_cfs_rq(p); in task_h_load() local
9871 update_cfs_rq_h_load(cfs_rq); in task_h_load()
9872 return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, in task_h_load()
9873 cfs_rq_load_avg(cfs_rq) + 1); in task_h_load()
9878 struct cfs_rq *cfs_rq = &rq->cfs; in __update_blocked_fair() local
9881 decayed = update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq); in __update_blocked_fair()
9882 if (cfs_rq_has_blocked(cfs_rq)) in __update_blocked_fair()
13010 struct cfs_rq *cfs_rq = cfs_rq_of(se); in se_fi_update() local
13013 if (cfs_rq->forceidle_seq == fi_seq) in se_fi_update()
13015 cfs_rq->forceidle_seq = fi_seq; in se_fi_update()
13018 cfs_rq->min_vruntime_fi = cfs_rq->min_vruntime; in se_fi_update()
13038 struct cfs_rq *cfs_rqa; in cfs_prio_less()
13039 struct cfs_rq *cfs_rqb; in cfs_prio_less()
13049 while (sea->cfs_rq->tg != seb->cfs_rq->tg) { in cfs_prio_less()
13062 cfs_rqa = sea->cfs_rq; in cfs_prio_less()
13063 cfs_rqb = seb->cfs_rq; in cfs_prio_less()
13082 struct cfs_rq *cfs_rq; in task_is_throttled_fair() local
13085 cfs_rq = task_group(p)->cfs_rq[cpu]; in task_is_throttled_fair()
13087 cfs_rq = &cpu_rq(cpu)->cfs; in task_is_throttled_fair()
13089 return throttled_hierarchy(cfs_rq); in task_is_throttled_fair()
13105 struct cfs_rq *cfs_rq; in task_tick_fair() local
13109 cfs_rq = cfs_rq_of(se); in task_tick_fair()
13110 entity_tick(cfs_rq, se, queued); in task_tick_fair()
13164 struct cfs_rq *cfs_rq = cfs_rq_of(se); in propagate_entity_cfs_rq() local
13166 if (cfs_rq_throttled(cfs_rq)) in propagate_entity_cfs_rq()
13169 if (!throttled_hierarchy(cfs_rq)) in propagate_entity_cfs_rq()
13170 list_add_leaf_cfs_rq(cfs_rq); in propagate_entity_cfs_rq()
13176 cfs_rq = cfs_rq_of(se); in propagate_entity_cfs_rq()
13178 update_load_avg(cfs_rq, se, UPDATE_TG); in propagate_entity_cfs_rq()
13180 if (cfs_rq_throttled(cfs_rq)) in propagate_entity_cfs_rq()
13183 if (!throttled_hierarchy(cfs_rq)) in propagate_entity_cfs_rq()
13184 list_add_leaf_cfs_rq(cfs_rq); in propagate_entity_cfs_rq()
13193 struct cfs_rq *cfs_rq = cfs_rq_of(se); in detach_entity_cfs_rq() local
13207 update_load_avg(cfs_rq, se, 0); in detach_entity_cfs_rq()
13208 detach_entity_load_avg(cfs_rq, se); in detach_entity_cfs_rq()
13209 update_tg_load_avg(cfs_rq); in detach_entity_cfs_rq()
13215 struct cfs_rq *cfs_rq = cfs_rq_of(se); in attach_entity_cfs_rq() local
13218 update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD); in attach_entity_cfs_rq()
13219 attach_entity_load_avg(cfs_rq, se); in attach_entity_cfs_rq()
13220 update_tg_load_avg(cfs_rq); in attach_entity_cfs_rq()
13300 struct cfs_rq *cfs_rq = cfs_rq_of(se); in set_next_task_fair() local
13302 set_next_entity(cfs_rq, se); in set_next_task_fair()
13304 account_cfs_rq_runtime(cfs_rq, 0); in set_next_task_fair()
13310 void init_cfs_rq(struct cfs_rq *cfs_rq) in init_cfs_rq() argument
13312 cfs_rq->tasks_timeline = RB_ROOT_CACHED; in init_cfs_rq()
13313 cfs_rq->min_vruntime = (u64)(-(1LL << 20)); in init_cfs_rq()
13315 raw_spin_lock_init(&cfs_rq->removed.lock); in init_cfs_rq()
13344 if (tg->cfs_rq) in free_fair_sched_group()
13345 kfree(tg->cfs_rq[i]); in free_fair_sched_group()
13350 kfree(tg->cfs_rq); in free_fair_sched_group()
13357 struct cfs_rq *cfs_rq; in alloc_fair_sched_group() local
13360 tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL); in alloc_fair_sched_group()
13361 if (!tg->cfs_rq) in alloc_fair_sched_group()
13372 cfs_rq = kzalloc_node(sizeof(struct cfs_rq), in alloc_fair_sched_group()
13374 if (!cfs_rq) in alloc_fair_sched_group()
13382 init_cfs_rq(cfs_rq); in alloc_fair_sched_group()
13383 init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]); in alloc_fair_sched_group()
13390 kfree(cfs_rq); in alloc_fair_sched_group()
13420 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu]; in unregister_fair_sched_group() local
13431 list_del_leaf_cfs_rq(cfs_rq); in unregister_fair_sched_group()
13440 if (cfs_rq->on_list) { in unregister_fair_sched_group()
13442 list_del_leaf_cfs_rq(cfs_rq); in unregister_fair_sched_group()
13447 void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, in init_tg_cfs_entry() argument
13453 cfs_rq->tg = tg; in init_tg_cfs_entry()
13454 cfs_rq->rq = rq; in init_tg_cfs_entry()
13455 init_cfs_rq_runtime(cfs_rq); in init_tg_cfs_entry()
13457 tg->cfs_rq[cpu] = cfs_rq; in init_tg_cfs_entry()
13465 se->cfs_rq = &rq->cfs; in init_tg_cfs_entry()
13468 se->cfs_rq = parent->my_q; in init_tg_cfs_entry()
13472 se->my_q = cfs_rq; in init_tg_cfs_entry()
13552 struct cfs_rq *grp_cfs_rq = tg->cfs_rq[i]; in sched_group_set_idle()
13569 struct cfs_rq *cfs_rq = cfs_rq_of(se); in sched_group_set_idle() local
13574 cfs_rq->h_nr_idle += idle_task_delta; in sched_group_set_idle()
13577 if (cfs_rq_is_idle(cfs_rq)) in sched_group_set_idle()
13669 struct cfs_rq *cfs_rq, *pos; in print_cfs_stats() local
13672 for_each_leaf_cfs_rq_safe(cpu_rq(cpu), cfs_rq, pos) in print_cfs_stats()
13673 print_cfs_rq(m, cpu, cfs_rq); in print_cfs_stats()