Lines Matching refs:se
292 static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) in calc_delta_fair() argument
294 if (unlikely(se->load.weight != NICE_0_LOAD)) in calc_delta_fair()
295 delta = __calc_delta(delta, NICE_0_LOAD, &se->load); in calc_delta_fair()
309 #define for_each_sched_entity(se) \ argument
310 for (; se; se = se->parent)
412 is_same_group(struct sched_entity *se, struct sched_entity *pse) in is_same_group() argument
414 if (se->cfs_rq == pse->cfs_rq) in is_same_group()
415 return se->cfs_rq; in is_same_group()
420 static inline struct sched_entity *parent_entity(const struct sched_entity *se) in parent_entity() argument
422 return se->parent; in parent_entity()
426 find_matching_se(struct sched_entity **se, struct sched_entity **pse) in find_matching_se() argument
438 se_depth = (*se)->depth; in find_matching_se()
443 *se = parent_entity(*se); in find_matching_se()
451 while (!is_same_group(*se, *pse)) { in find_matching_se()
452 *se = parent_entity(*se); in find_matching_se()
467 static int se_is_idle(struct sched_entity *se) in se_is_idle() argument
469 if (entity_is_task(se)) in se_is_idle()
470 return task_has_idle_policy(task_of(se)); in se_is_idle()
471 return cfs_rq_is_idle(group_cfs_rq(se)); in se_is_idle()
476 #define for_each_sched_entity(se) \ argument
477 for (; se; se = NULL)
495 static inline struct sched_entity *parent_entity(struct sched_entity *se) in parent_entity() argument
501 find_matching_se(struct sched_entity **se, struct sched_entity **pse) in find_matching_se() argument
515 static int se_is_idle(struct sched_entity *se) in se_is_idle() argument
517 return task_has_idle_policy(task_of(se)); in se_is_idle()
557 static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) in entity_key() argument
559 return (s64)(se->vruntime - cfs_rq->min_vruntime); in entity_key()
624 avg_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) in avg_vruntime_add() argument
626 unsigned long weight = scale_load_down(se->load.weight); in avg_vruntime_add()
627 s64 key = entity_key(cfs_rq, se); in avg_vruntime_add()
634 avg_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) in avg_vruntime_sub() argument
636 unsigned long weight = scale_load_down(se->load.weight); in avg_vruntime_sub()
637 s64 key = entity_key(cfs_rq, se); in avg_vruntime_sub()
695 static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) in update_entity_lag() argument
699 WARN_ON_ONCE(!se->on_rq); in update_entity_lag()
701 vlag = avg_vruntime(cfs_rq) - se->vruntime; in update_entity_lag()
702 limit = calc_delta_fair(max_t(u64, 2*se->slice, TICK_NSEC), se); in update_entity_lag()
704 se->vlag = clamp(vlag, -limit, limit); in update_entity_lag()
740 int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se) in entity_eligible() argument
742 return vruntime_eligible(cfs_rq, se->vruntime); in entity_eligible()
761 struct sched_entity *se = __pick_root_entity(cfs_rq); in update_min_vruntime() local
772 if (se) { in update_min_vruntime()
774 vruntime = se->min_vruntime; in update_min_vruntime()
776 vruntime = min_vruntime(vruntime, se->min_vruntime); in update_min_vruntime()
805 static inline void __min_vruntime_update(struct sched_entity *se, struct rb_node *node) in __min_vruntime_update() argument
809 if (vruntime_gt(min_vruntime, se, rse)) in __min_vruntime_update()
810 se->min_vruntime = rse->min_vruntime; in __min_vruntime_update()
814 static inline void __min_slice_update(struct sched_entity *se, struct rb_node *node) in __min_slice_update() argument
818 if (rse->min_slice < se->min_slice) in __min_slice_update()
819 se->min_slice = rse->min_slice; in __min_slice_update()
826 static inline bool min_vruntime_update(struct sched_entity *se, bool exit) in min_vruntime_update() argument
828 u64 old_min_vruntime = se->min_vruntime; in min_vruntime_update()
829 u64 old_min_slice = se->min_slice; in min_vruntime_update()
830 struct rb_node *node = &se->run_node; in min_vruntime_update()
832 se->min_vruntime = se->vruntime; in min_vruntime_update()
833 __min_vruntime_update(se, node->rb_right); in min_vruntime_update()
834 __min_vruntime_update(se, node->rb_left); in min_vruntime_update()
836 se->min_slice = se->slice; in min_vruntime_update()
837 __min_slice_update(se, node->rb_right); in min_vruntime_update()
838 __min_slice_update(se, node->rb_left); in min_vruntime_update()
840 return se->min_vruntime == old_min_vruntime && in min_vruntime_update()
841 se->min_slice == old_min_slice; in min_vruntime_update()
850 static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) in __enqueue_entity() argument
852 avg_vruntime_add(cfs_rq, se); in __enqueue_entity()
853 se->min_vruntime = se->vruntime; in __enqueue_entity()
854 se->min_slice = se->slice; in __enqueue_entity()
855 rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, in __enqueue_entity()
859 static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) in __dequeue_entity() argument
861 rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, in __dequeue_entity()
863 avg_vruntime_sub(cfs_rq, se); in __dequeue_entity()
890 static inline void set_protect_slice(struct sched_entity *se) in set_protect_slice() argument
892 se->vlag = se->deadline; in set_protect_slice()
895 static inline bool protect_slice(struct sched_entity *se) in protect_slice() argument
897 return se->vlag == se->deadline; in protect_slice()
900 static inline void cancel_protect_slice(struct sched_entity *se) in cancel_protect_slice() argument
902 if (protect_slice(se)) in cancel_protect_slice()
903 se->vlag = se->deadline + 1; in cancel_protect_slice()
928 struct sched_entity *se = __pick_first_entity(cfs_rq); in pick_eevdf() local
937 return curr && curr->on_rq ? curr : se; in pick_eevdf()
946 if (se && entity_eligible(cfs_rq, se)) { in pick_eevdf()
947 best = se; in pick_eevdf()
965 se = __node_2_se(node); in pick_eevdf()
972 if (entity_eligible(cfs_rq, se)) { in pick_eevdf()
973 best = se; in pick_eevdf()
1013 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se);
1019 static bool update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se) in update_deadline() argument
1021 if ((s64)(se->vruntime - se->deadline) < 0) in update_deadline()
1029 if (!se->custom_slice) in update_deadline()
1030 se->slice = sysctl_sched_base_slice; in update_deadline()
1035 se->deadline = se->vruntime + calc_delta_fair(se->slice, se); in update_deadline()
1051 void init_entity_runnable_average(struct sched_entity *se) in init_entity_runnable_average() argument
1053 struct sched_avg *sa = &se->avg; in init_entity_runnable_average()
1063 if (entity_is_task(se)) in init_entity_runnable_average()
1064 sa->load_avg = scale_load_down(se->load.weight); in init_entity_runnable_average()
1098 struct sched_entity *se = &p->se; in post_init_entity_util_avg() local
1099 struct cfs_rq *cfs_rq = cfs_rq_of(se); in post_init_entity_util_avg()
1100 struct sched_avg *sa = &se->avg; in post_init_entity_util_avg()
1115 se->avg.last_update_time = cfs_rq_clock_pelt(cfs_rq); in post_init_entity_util_avg()
1121 sa->util_avg = cfs_rq->avg.util_avg * se_weight(se); in post_init_entity_util_avg()
1135 void init_entity_runnable_average(struct sched_entity *se) in init_entity_runnable_average() argument
1188 struct sched_entity *pse, struct sched_entity *se) in do_preempt_short() argument
1193 if (pse->slice >= se->slice) in do_preempt_short()
1199 if (entity_before(pse, se)) in do_preempt_short()
1202 if (!entity_eligible(cfs_rq, se)) in do_preempt_short()
1216 delta_exec = update_curr_se(rq, &donor->se); in update_curr_common()
1276 update_curr(cfs_rq_of(&rq->donor->se)); in update_curr_fair()
1280 update_stats_wait_start_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) in update_stats_wait_start_fair() argument
1288 stats = __schedstats_from_se(se); in update_stats_wait_start_fair()
1290 if (entity_is_task(se)) in update_stats_wait_start_fair()
1291 p = task_of(se); in update_stats_wait_start_fair()
1297 update_stats_wait_end_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) in update_stats_wait_end_fair() argument
1305 stats = __schedstats_from_se(se); in update_stats_wait_end_fair()
1316 if (entity_is_task(se)) in update_stats_wait_end_fair()
1317 p = task_of(se); in update_stats_wait_end_fair()
1323 update_stats_enqueue_sleeper_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) in update_stats_enqueue_sleeper_fair() argument
1331 stats = __schedstats_from_se(se); in update_stats_enqueue_sleeper_fair()
1333 if (entity_is_task(se)) in update_stats_enqueue_sleeper_fair()
1334 tsk = task_of(se); in update_stats_enqueue_sleeper_fair()
1343 update_stats_enqueue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) in update_stats_enqueue_fair() argument
1352 if (se != cfs_rq->curr) in update_stats_enqueue_fair()
1353 update_stats_wait_start_fair(cfs_rq, se); in update_stats_enqueue_fair()
1356 update_stats_enqueue_sleeper_fair(cfs_rq, se); in update_stats_enqueue_fair()
1360 update_stats_dequeue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) in update_stats_dequeue_fair() argument
1370 if (se != cfs_rq->curr) in update_stats_dequeue_fair()
1371 update_stats_wait_end_fair(cfs_rq, se); in update_stats_dequeue_fair()
1373 if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) { in update_stats_dequeue_fair()
1374 struct task_struct *tsk = task_of(se); in update_stats_dequeue_fair()
1392 update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) in update_stats_curr_start() argument
1397 se->exec_start = rq_clock_task(rq_of(cfs_rq)); in update_stats_curr_start()
2785 now = p->se.exec_start; in numa_get_avg_runtime()
2786 runtime = p->se.sum_exec_runtime; in numa_get_avg_runtime()
2796 delta = p->se.avg.load_sum; in numa_get_avg_runtime()
3309 u64 runtime = p->se.sum_exec_runtime; in task_numa_work()
3546 if (unlikely(p->se.sum_exec_runtime != runtime)) { in task_numa_work()
3547 u64 diff = p->se.sum_exec_runtime - runtime; in task_numa_work()
3619 now = curr->se.sum_exec_runtime; in task_tick_numa()
3686 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) in account_entity_enqueue() argument
3688 update_load_add(&cfs_rq->load, se->load.weight); in account_entity_enqueue()
3690 if (entity_is_task(se)) { in account_entity_enqueue()
3693 account_numa_enqueue(rq, task_of(se)); in account_entity_enqueue()
3694 list_add(&se->group_node, &rq->cfs_tasks); in account_entity_enqueue()
3701 account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) in account_entity_dequeue() argument
3703 update_load_sub(&cfs_rq->load, se->load.weight); in account_entity_dequeue()
3705 if (entity_is_task(se)) { in account_entity_dequeue()
3706 account_numa_dequeue(rq_of(cfs_rq), task_of(se)); in account_entity_dequeue()
3707 list_del_init(&se->group_node); in account_entity_dequeue()
3763 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) in enqueue_load_avg() argument
3765 cfs_rq->avg.load_avg += se->avg.load_avg; in enqueue_load_avg()
3766 cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum; in enqueue_load_avg()
3770 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) in dequeue_load_avg() argument
3772 sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg); in dequeue_load_avg()
3773 sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum); in dequeue_load_avg()
3780 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } in enqueue_load_avg() argument
3782 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { } in dequeue_load_avg() argument
3785 static void place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags);
3787 static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, in reweight_entity() argument
3790 bool curr = cfs_rq->curr == se; in reweight_entity()
3792 if (se->on_rq) { in reweight_entity()
3795 update_entity_lag(cfs_rq, se); in reweight_entity()
3796 se->deadline -= se->vruntime; in reweight_entity()
3797 se->rel_deadline = 1; in reweight_entity()
3799 __dequeue_entity(cfs_rq, se); in reweight_entity()
3800 update_load_sub(&cfs_rq->load, se->load.weight); in reweight_entity()
3802 dequeue_load_avg(cfs_rq, se); in reweight_entity()
3808 se->vlag = div_s64(se->vlag * se->load.weight, weight); in reweight_entity()
3809 if (se->rel_deadline) in reweight_entity()
3810 se->deadline = div_s64(se->deadline * se->load.weight, weight); in reweight_entity()
3812 update_load_set(&se->load, weight); in reweight_entity()
3816 u32 divider = get_pelt_divider(&se->avg); in reweight_entity()
3818 se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); in reweight_entity()
3822 enqueue_load_avg(cfs_rq, se); in reweight_entity()
3823 if (se->on_rq) { in reweight_entity()
3824 update_load_add(&cfs_rq->load, se->load.weight); in reweight_entity()
3825 place_entity(cfs_rq, se, 0); in reweight_entity()
3827 __enqueue_entity(cfs_rq, se); in reweight_entity()
3843 struct sched_entity *se = &p->se; in reweight_task_fair() local
3844 struct cfs_rq *cfs_rq = cfs_rq_of(se); in reweight_task_fair()
3845 struct load_weight *load = &se->load; in reweight_task_fair()
3847 reweight_entity(cfs_rq, se, lw->weight); in reweight_task_fair()
3967 static void update_cfs_group(struct sched_entity *se) in update_cfs_group() argument
3969 struct cfs_rq *gcfs_rq = group_cfs_rq(se); in update_cfs_group()
3987 if (unlikely(se->load.weight != shares)) in update_cfs_group()
3988 reweight_entity(cfs_rq_of(se), se, shares); in update_cfs_group()
3992 static inline void update_cfs_group(struct sched_entity *se) in update_cfs_group() argument
4185 void set_task_rq_fair(struct sched_entity *se, in set_task_rq_fair() argument
4201 if (!(se->avg.last_update_time && prev)) in set_task_rq_fair()
4207 __update_load_avg_blocked_se(p_last_update_time, se); in set_task_rq_fair()
4208 se->avg.last_update_time = n_last_update_time; in set_task_rq_fair()
4279 update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) in update_tg_cfs_util() argument
4281 long delta_sum, delta_avg = gcfs_rq->avg.util_avg - se->avg.util_avg; in update_tg_cfs_util()
4296 se->avg.util_avg = gcfs_rq->avg.util_avg; in update_tg_cfs_util()
4297 new_sum = se->avg.util_avg * divider; in update_tg_cfs_util()
4298 delta_sum = (long)new_sum - (long)se->avg.util_sum; in update_tg_cfs_util()
4299 se->avg.util_sum = new_sum; in update_tg_cfs_util()
4311 update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) in update_tg_cfs_runnable() argument
4313 long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg; in update_tg_cfs_runnable()
4327 se->avg.runnable_avg = gcfs_rq->avg.runnable_avg; in update_tg_cfs_runnable()
4328 new_sum = se->avg.runnable_avg * divider; in update_tg_cfs_runnable()
4329 delta_sum = (long)new_sum - (long)se->avg.runnable_sum; in update_tg_cfs_runnable()
4330 se->avg.runnable_sum = new_sum; in update_tg_cfs_runnable()
4341 update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) in update_tg_cfs_load() argument
4365 runnable_sum += se->avg.load_sum; in update_tg_cfs_load()
4378 runnable_sum = min(se->avg.load_sum, load_sum); in update_tg_cfs_load()
4387 running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; in update_tg_cfs_load()
4390 load_sum = se_weight(se) * runnable_sum; in update_tg_cfs_load()
4393 delta_avg = load_avg - se->avg.load_avg; in update_tg_cfs_load()
4397 delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; in update_tg_cfs_load()
4399 se->avg.load_sum = runnable_sum; in update_tg_cfs_load()
4400 se->avg.load_avg = load_avg; in update_tg_cfs_load()
4415 static inline int propagate_entity_load_avg(struct sched_entity *se) in propagate_entity_load_avg() argument
4419 if (entity_is_task(se)) in propagate_entity_load_avg()
4422 gcfs_rq = group_cfs_rq(se); in propagate_entity_load_avg()
4428 cfs_rq = cfs_rq_of(se); in propagate_entity_load_avg()
4432 update_tg_cfs_util(cfs_rq, se, gcfs_rq); in propagate_entity_load_avg()
4433 update_tg_cfs_runnable(cfs_rq, se, gcfs_rq); in propagate_entity_load_avg()
4434 update_tg_cfs_load(cfs_rq, se, gcfs_rq); in propagate_entity_load_avg()
4437 trace_pelt_se_tp(se); in propagate_entity_load_avg()
4446 static inline bool skip_blocked_update(struct sched_entity *se) in skip_blocked_update() argument
4448 struct cfs_rq *gcfs_rq = group_cfs_rq(se); in skip_blocked_update()
4454 if (se->avg.load_avg || se->avg.util_avg) in skip_blocked_update()
4478 static inline int propagate_entity_load_avg(struct sched_entity *se) in propagate_entity_load_avg() argument
4488 static inline void migrate_se_pelt_lag(struct sched_entity *se) in migrate_se_pelt_lag() argument
4495 if (load_avg_is_decayed(&se->avg)) in migrate_se_pelt_lag()
4498 cfs_rq = cfs_rq_of(se); in migrate_se_pelt_lag()
4564 __update_load_avg_blocked_se(now, se); in migrate_se_pelt_lag()
4567 static void migrate_se_pelt_lag(struct sched_entity *se) {} in migrate_se_pelt_lag() argument
4657 static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) in attach_entity_load_avg() argument
4672 se->avg.last_update_time = cfs_rq->avg.last_update_time; in attach_entity_load_avg()
4673 se->avg.period_contrib = cfs_rq->avg.period_contrib; in attach_entity_load_avg()
4681 se->avg.util_sum = se->avg.util_avg * divider; in attach_entity_load_avg()
4683 se->avg.runnable_sum = se->avg.runnable_avg * divider; in attach_entity_load_avg()
4685 se->avg.load_sum = se->avg.load_avg * divider; in attach_entity_load_avg()
4686 if (se_weight(se) < se->avg.load_sum) in attach_entity_load_avg()
4687 se->avg.load_sum = div_u64(se->avg.load_sum, se_weight(se)); in attach_entity_load_avg()
4689 se->avg.load_sum = 1; in attach_entity_load_avg()
4691 enqueue_load_avg(cfs_rq, se); in attach_entity_load_avg()
4692 cfs_rq->avg.util_avg += se->avg.util_avg; in attach_entity_load_avg()
4693 cfs_rq->avg.util_sum += se->avg.util_sum; in attach_entity_load_avg()
4694 cfs_rq->avg.runnable_avg += se->avg.runnable_avg; in attach_entity_load_avg()
4695 cfs_rq->avg.runnable_sum += se->avg.runnable_sum; in attach_entity_load_avg()
4697 add_tg_cfs_propagate(cfs_rq, se->avg.load_sum); in attach_entity_load_avg()
4712 static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) in detach_entity_load_avg() argument
4714 dequeue_load_avg(cfs_rq, se); in detach_entity_load_avg()
4715 sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg); in detach_entity_load_avg()
4716 sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum); in detach_entity_load_avg()
4721 sub_positive(&cfs_rq->avg.runnable_avg, se->avg.runnable_avg); in detach_entity_load_avg()
4722 sub_positive(&cfs_rq->avg.runnable_sum, se->avg.runnable_sum); in detach_entity_load_avg()
4727 add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum); in detach_entity_load_avg()
4743 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) in update_load_avg() argument
4752 if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) in update_load_avg()
4753 __update_load_avg_se(now, cfs_rq, se); in update_load_avg()
4756 decayed |= propagate_entity_load_avg(se); in update_load_avg()
4758 if (!se->avg.last_update_time && (flags & DO_ATTACH)) { in update_load_avg()
4767 attach_entity_load_avg(cfs_rq, se); in update_load_avg()
4775 detach_entity_load_avg(cfs_rq, se); in update_load_avg()
4789 static void sync_entity_load_avg(struct sched_entity *se) in sync_entity_load_avg() argument
4791 struct cfs_rq *cfs_rq = cfs_rq_of(se); in sync_entity_load_avg()
4795 __update_load_avg_blocked_se(last_update_time, se); in sync_entity_load_avg()
4802 static void remove_entity_load_avg(struct sched_entity *se) in remove_entity_load_avg() argument
4804 struct cfs_rq *cfs_rq = cfs_rq_of(se); in remove_entity_load_avg()
4813 sync_entity_load_avg(se); in remove_entity_load_avg()
4817 cfs_rq->removed.util_avg += se->avg.util_avg; in remove_entity_load_avg()
4818 cfs_rq->removed.load_avg += se->avg.load_avg; in remove_entity_load_avg()
4819 cfs_rq->removed.runnable_avg += se->avg.runnable_avg; in remove_entity_load_avg()
4837 return READ_ONCE(p->se.avg.util_avg); in task_util()
4842 return READ_ONCE(p->se.avg.runnable_avg); in task_runnable()
4847 return READ_ONCE(p->se.avg.util_est) & ~UTIL_AVG_UNCHANGED; in _task_util_est()
4906 ewma = READ_ONCE(p->se.avg.util_est); in util_est_update()
4971 WRITE_ONCE(p->se.avg.util_est, ewma); in util_est_update()
4973 trace_sched_util_est_se_tp(&p->se); in util_est_update()
5154 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int not_used1) in update_load_avg() argument
5159 static inline void remove_entity_load_avg(struct sched_entity *se) {} in remove_entity_load_avg() argument
5162 attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} in attach_entity_load_avg() argument
5164 detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} in detach_entity_load_avg() argument
5186 struct sched_entity *se = &p->se; in __setparam_fair() local
5190 se->custom_slice = 1; in __setparam_fair()
5191 se->slice = clamp_t(u64, attr->sched_runtime, in __setparam_fair()
5195 se->custom_slice = 0; in __setparam_fair()
5196 se->slice = sysctl_sched_base_slice; in __setparam_fair()
5201 place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) in place_entity() argument
5206 if (!se->custom_slice) in place_entity()
5207 se->slice = sysctl_sched_base_slice; in place_entity()
5208 vslice = calc_delta_fair(se->slice, se); in place_entity()
5218 if (sched_feat(PLACE_LAG) && cfs_rq->nr_queued && se->vlag) { in place_entity()
5222 lag = se->vlag; in place_entity()
5280 lag *= load + scale_load_down(se->load.weight); in place_entity()
5286 se->vruntime = vruntime - lag; in place_entity()
5288 if (se->rel_deadline) { in place_entity()
5289 se->deadline += se->vruntime; in place_entity()
5290 se->rel_deadline = 0; in place_entity()
5305 se->deadline = se->vruntime + vslice; in place_entity()
5312 requeue_delayed_entity(struct sched_entity *se);
5315 enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) in enqueue_entity() argument
5317 bool curr = cfs_rq->curr == se; in enqueue_entity()
5324 place_entity(cfs_rq, se, flags); in enqueue_entity()
5337 update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH); in enqueue_entity()
5338 se_update_runnable(se); in enqueue_entity()
5344 update_cfs_group(se); in enqueue_entity()
5351 place_entity(cfs_rq, se, flags); in enqueue_entity()
5353 account_entity_enqueue(cfs_rq, se); in enqueue_entity()
5357 se->exec_start = 0; in enqueue_entity()
5360 update_stats_enqueue_fair(cfs_rq, se, flags); in enqueue_entity()
5362 __enqueue_entity(cfs_rq, se); in enqueue_entity()
5363 se->on_rq = 1; in enqueue_entity()
5382 static void __clear_buddies_next(struct sched_entity *se) in __clear_buddies_next() argument
5384 for_each_sched_entity(se) { in __clear_buddies_next()
5385 struct cfs_rq *cfs_rq = cfs_rq_of(se); in __clear_buddies_next()
5386 if (cfs_rq->next != se) in __clear_buddies_next()
5393 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) in clear_buddies() argument
5395 if (cfs_rq->next == se) in clear_buddies()
5396 __clear_buddies_next(se); in clear_buddies()
5401 static void set_delayed(struct sched_entity *se) in set_delayed() argument
5403 se->sched_delayed = 1; in set_delayed()
5410 if (!entity_is_task(se)) in set_delayed()
5413 for_each_sched_entity(se) { in set_delayed()
5414 struct cfs_rq *cfs_rq = cfs_rq_of(se); in set_delayed()
5422 static void clear_delayed(struct sched_entity *se) in clear_delayed() argument
5424 se->sched_delayed = 0; in clear_delayed()
5432 if (!entity_is_task(se)) in clear_delayed()
5435 for_each_sched_entity(se) { in clear_delayed()
5436 struct cfs_rq *cfs_rq = cfs_rq_of(se); in clear_delayed()
5444 static inline void finish_delayed_dequeue_entity(struct sched_entity *se) in finish_delayed_dequeue_entity() argument
5446 clear_delayed(se); in finish_delayed_dequeue_entity()
5447 if (sched_feat(DELAY_ZERO) && se->vlag > 0) in finish_delayed_dequeue_entity()
5448 se->vlag = 0; in finish_delayed_dequeue_entity()
5452 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) in dequeue_entity() argument
5458 clear_buddies(cfs_rq, se); in dequeue_entity()
5461 WARN_ON_ONCE(!se->sched_delayed); in dequeue_entity()
5471 WARN_ON_ONCE(delay && se->sched_delayed); in dequeue_entity()
5474 !entity_eligible(cfs_rq, se)) { in dequeue_entity()
5475 update_load_avg(cfs_rq, se, 0); in dequeue_entity()
5476 set_delayed(se); in dequeue_entity()
5481 if (entity_is_task(se) && task_on_rq_migrating(task_of(se))) in dequeue_entity()
5493 update_load_avg(cfs_rq, se, action); in dequeue_entity()
5494 se_update_runnable(se); in dequeue_entity()
5496 update_stats_dequeue_fair(cfs_rq, se, flags); in dequeue_entity()
5498 update_entity_lag(cfs_rq, se); in dequeue_entity()
5500 se->deadline -= se->vruntime; in dequeue_entity()
5501 se->rel_deadline = 1; in dequeue_entity()
5504 if (se != cfs_rq->curr) in dequeue_entity()
5505 __dequeue_entity(cfs_rq, se); in dequeue_entity()
5506 se->on_rq = 0; in dequeue_entity()
5507 account_entity_dequeue(cfs_rq, se); in dequeue_entity()
5512 update_cfs_group(se); in dequeue_entity()
5524 finish_delayed_dequeue_entity(se); in dequeue_entity()
5533 set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) in set_next_entity() argument
5535 clear_buddies(cfs_rq, se); in set_next_entity()
5538 if (se->on_rq) { in set_next_entity()
5544 update_stats_wait_end_fair(cfs_rq, se); in set_next_entity()
5545 __dequeue_entity(cfs_rq, se); in set_next_entity()
5546 update_load_avg(cfs_rq, se, UPDATE_TG); in set_next_entity()
5548 set_protect_slice(se); in set_next_entity()
5551 update_stats_curr_start(cfs_rq, se); in set_next_entity()
5553 cfs_rq->curr = se; in set_next_entity()
5561 rq_of(cfs_rq)->cfs.load.weight >= 2*se->load.weight) { in set_next_entity()
5564 stats = __schedstats_from_se(se); in set_next_entity()
5567 se->sum_exec_runtime - se->prev_sum_exec_runtime)); in set_next_entity()
5570 se->prev_sum_exec_runtime = se->sum_exec_runtime; in set_next_entity()
5573 static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags);
5585 struct sched_entity *se; in pick_next_entity() local
5597 se = pick_eevdf(cfs_rq); in pick_next_entity()
5598 if (se->sched_delayed) { in pick_next_entity()
5599 dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); in pick_next_entity()
5605 return se; in pick_next_entity()
5885 struct sched_entity *se; in throttle_cfs_rq() local
5910 se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))]; in throttle_cfs_rq()
5920 for_each_sched_entity(se) { in throttle_cfs_rq()
5921 struct cfs_rq *qcfs_rq = cfs_rq_of(se); in throttle_cfs_rq()
5925 if (!se->on_rq) in throttle_cfs_rq()
5934 if (se->sched_delayed) in throttle_cfs_rq()
5936 dequeue_entity(qcfs_rq, se, flags); in throttle_cfs_rq()
5938 if (cfs_rq_is_idle(group_cfs_rq(se))) in throttle_cfs_rq()
5947 se = parent_entity(se); in throttle_cfs_rq()
5952 for_each_sched_entity(se) { in throttle_cfs_rq()
5953 struct cfs_rq *qcfs_rq = cfs_rq_of(se); in throttle_cfs_rq()
5955 if (!se->on_rq) in throttle_cfs_rq()
5958 update_load_avg(qcfs_rq, se, 0); in throttle_cfs_rq()
5959 se_update_runnable(se); in throttle_cfs_rq()
5961 if (cfs_rq_is_idle(group_cfs_rq(se))) in throttle_cfs_rq()
5991 struct sched_entity *se; in unthrottle_cfs_rq() local
5995 se = cfs_rq->tg->se[cpu_of(rq)]; in unthrottle_cfs_rq()
6019 for_each_sched_entity(se) { in unthrottle_cfs_rq()
6020 if (list_add_leaf_cfs_rq(cfs_rq_of(se))) in unthrottle_cfs_rq()
6029 for_each_sched_entity(se) { in unthrottle_cfs_rq()
6030 struct cfs_rq *qcfs_rq = cfs_rq_of(se); in unthrottle_cfs_rq()
6033 if (se->sched_delayed) { in unthrottle_cfs_rq()
6036 dequeue_entity(qcfs_rq, se, flags); in unthrottle_cfs_rq()
6037 } else if (se->on_rq) in unthrottle_cfs_rq()
6039 enqueue_entity(qcfs_rq, se, ENQUEUE_WAKEUP); in unthrottle_cfs_rq()
6041 if (cfs_rq_is_idle(group_cfs_rq(se))) in unthrottle_cfs_rq()
6053 for_each_sched_entity(se) { in unthrottle_cfs_rq()
6054 struct cfs_rq *qcfs_rq = cfs_rq_of(se); in unthrottle_cfs_rq()
6056 update_load_avg(qcfs_rq, se, UPDATE_TG); in unthrottle_cfs_rq()
6057 se_update_runnable(se); in unthrottle_cfs_rq()
6059 if (cfs_rq_is_idle(group_cfs_rq(se))) in unthrottle_cfs_rq()
6788 struct sched_entity *se = &p->se; in hrtick_start_fair() local
6793 u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime; in hrtick_start_fair()
6794 u64 slice = se->slice; in hrtick_start_fair()
6892 requeue_delayed_entity(struct sched_entity *se) in requeue_delayed_entity() argument
6894 struct cfs_rq *cfs_rq = cfs_rq_of(se); in requeue_delayed_entity()
6901 WARN_ON_ONCE(!se->sched_delayed); in requeue_delayed_entity()
6902 WARN_ON_ONCE(!se->on_rq); in requeue_delayed_entity()
6905 update_entity_lag(cfs_rq, se); in requeue_delayed_entity()
6906 if (se->vlag > 0) { in requeue_delayed_entity()
6908 if (se != cfs_rq->curr) in requeue_delayed_entity()
6909 __dequeue_entity(cfs_rq, se); in requeue_delayed_entity()
6910 se->vlag = 0; in requeue_delayed_entity()
6911 place_entity(cfs_rq, se, 0); in requeue_delayed_entity()
6912 if (se != cfs_rq->curr) in requeue_delayed_entity()
6913 __enqueue_entity(cfs_rq, se); in requeue_delayed_entity()
6918 update_load_avg(cfs_rq, se, 0); in requeue_delayed_entity()
6919 clear_delayed(se); in requeue_delayed_entity()
6931 struct sched_entity *se = &p->se; in enqueue_task_fair() local
6944 if (!(p->se.sched_delayed && (task_on_rq_migrating(p) || (flags & ENQUEUE_RESTORE)))) in enqueue_task_fair()
6948 requeue_delayed_entity(se); in enqueue_task_fair()
6960 if (task_new && se->sched_delayed) in enqueue_task_fair()
6963 for_each_sched_entity(se) { in enqueue_task_fair()
6964 if (se->on_rq) { in enqueue_task_fair()
6965 if (se->sched_delayed) in enqueue_task_fair()
6966 requeue_delayed_entity(se); in enqueue_task_fair()
6969 cfs_rq = cfs_rq_of(se); in enqueue_task_fair()
6977 se->slice = slice; in enqueue_task_fair()
6978 se->custom_slice = 1; in enqueue_task_fair()
6980 enqueue_entity(cfs_rq, se, flags); in enqueue_task_fair()
6997 for_each_sched_entity(se) { in enqueue_task_fair()
6998 cfs_rq = cfs_rq_of(se); in enqueue_task_fair()
7000 update_load_avg(cfs_rq, se, UPDATE_TG); in enqueue_task_fair()
7001 se_update_runnable(se); in enqueue_task_fair()
7002 update_cfs_group(se); in enqueue_task_fair()
7004 se->slice = slice; in enqueue_task_fair()
7005 if (se != cfs_rq->curr) in enqueue_task_fair()
7006 min_vruntime_cb_propagate(&se->run_node, NULL); in enqueue_task_fair()
7054 static void set_next_buddy(struct sched_entity *se);
7065 static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) in dequeue_entities() argument
7078 if (entity_is_task(se)) { in dequeue_entities()
7079 p = task_of(se); in dequeue_entities()
7082 if (task_sleep || task_delayed || !se->sched_delayed) in dequeue_entities()
7086 for_each_sched_entity(se) { in dequeue_entities()
7087 cfs_rq = cfs_rq_of(se); in dequeue_entities()
7089 if (!dequeue_entity(cfs_rq, se, flags)) { in dequeue_entities()
7090 if (p && &p->se == se) in dequeue_entities()
7113 se = parent_entity(se); in dequeue_entities()
7118 if (task_sleep && se && !throttled_hierarchy(cfs_rq)) in dequeue_entities()
7119 set_next_buddy(se); in dequeue_entities()
7126 for_each_sched_entity(se) { in dequeue_entities()
7127 cfs_rq = cfs_rq_of(se); in dequeue_entities()
7129 update_load_avg(cfs_rq, se, UPDATE_TG); in dequeue_entities()
7130 se_update_runnable(se); in dequeue_entities()
7131 update_cfs_group(se); in dequeue_entities()
7133 se->slice = slice; in dequeue_entities()
7134 if (se != cfs_rq->curr) in dequeue_entities()
7135 min_vruntime_cb_propagate(&se->run_node, NULL); in dequeue_entities()
7184 if (!(p->se.sched_delayed && (task_on_rq_migrating(p) || (flags & DEQUEUE_SAVE)))) in dequeue_task_fair()
7188 if (dequeue_entities(rq, &p->se, flags) < 0) in dequeue_task_fair()
7243 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) in cpu_load_without()
7266 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) in cpu_runnable_without()
7273 lsub_positive(&runnable, p->se.avg.runnable_avg); in cpu_runnable_without()
7508 sync_entity_load_avg(&p->se); in sched_balance_find_dst_cpu()
7841 sync_entity_load_avg(&p->se); in select_idle_sibling()
8088 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) in cpu_util_without()
8409 sync_entity_load_avg(&p->se); in find_energy_efficient_cpu()
8651 struct sched_entity *se = &p->se; in migrate_task_rq_fair() local
8654 remove_entity_load_avg(se); in migrate_task_rq_fair()
8666 migrate_se_pelt_lag(se); in migrate_task_rq_fair()
8670 se->avg.last_update_time = 0; in migrate_task_rq_fair()
8677 struct sched_entity *se = &p->se; in task_dead_fair() local
8679 if (se->sched_delayed) { in task_dead_fair()
8684 if (se->sched_delayed) { in task_dead_fair()
8686 dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); in task_dead_fair()
8691 remove_entity_load_avg(se); in task_dead_fair()
8736 static void set_next_buddy(struct sched_entity *se) in set_next_buddy() argument
8738 for_each_sched_entity(se) { in set_next_buddy()
8739 if (WARN_ON_ONCE(!se->on_rq)) in set_next_buddy()
8741 if (se_is_idle(se)) in set_next_buddy()
8743 cfs_rq_of(se)->next = se; in set_next_buddy()
8753 struct sched_entity *se = &donor->se, *pse = &p->se; in check_preempt_wakeup_fair() local
8757 if (unlikely(se == pse)) in check_preempt_wakeup_fair()
8789 find_matching_se(&se, &pse); in check_preempt_wakeup_fair()
8792 cse_is_idle = se_is_idle(se); in check_preempt_wakeup_fair()
8804 cancel_protect_slice(se); in check_preempt_wakeup_fair()
8817 cfs_rq = cfs_rq_of(se); in check_preempt_wakeup_fair()
8826 if (do_preempt_short(cfs_rq, pse, se)) in check_preempt_wakeup_fair()
8827 cancel_protect_slice(se); in check_preempt_wakeup_fair()
8843 struct sched_entity *se; in pick_task_fair() local
8859 se = pick_next_entity(rq, cfs_rq); in pick_task_fair()
8860 if (!se) in pick_task_fair()
8862 cfs_rq = group_cfs_rq(se); in pick_task_fair()
8865 return task_of(se); in pick_task_fair()
8874 struct sched_entity *se; in pick_next_task_fair() local
8882 se = &p->se; in pick_next_task_fair()
8902 struct sched_entity *pse = &prev->se; in pick_next_task_fair()
8905 while (!(cfs_rq = is_same_group(se, pse))) { in pick_next_task_fair()
8906 int se_depth = se->depth; in pick_next_task_fair()
8914 set_next_entity(cfs_rq_of(se), se); in pick_next_task_fair()
8915 se = parent_entity(se); in pick_next_task_fair()
8920 set_next_entity(cfs_rq, se); in pick_next_task_fair()
8987 struct sched_entity *se = &prev->se; in put_prev_task_fair() local
8990 for_each_sched_entity(se) { in put_prev_task_fair()
8991 cfs_rq = cfs_rq_of(se); in put_prev_task_fair()
8992 put_prev_entity(cfs_rq, se); in put_prev_task_fair()
9003 struct sched_entity *se = &curr->se; in yield_task_fair() local
9011 clear_buddies(cfs_rq, se); in yield_task_fair()
9025 se->deadline += calc_delta_fair(se->slice, se); in yield_task_fair()
9030 struct sched_entity *se = &p->se; in yield_to_task_fair() local
9033 if (!se->on_rq || throttled_hierarchy(cfs_rq_of(se))) in yield_to_task_fair()
9037 set_next_buddy(se); in yield_to_task_fair()
9273 (&p->se == cfs_rq_of(&p->se)->next)) in task_hot()
9289 delta = rq_clock_task(env->src_rq) - p->se.exec_start; in task_hot()
9372 !entity_eligible(task_cfs_rq(p), &p->se)) in task_is_ineligible_on_dst_cpu()
9398 if ((p->se.sched_delayed) && (env->migration_type != migrate_load)) in can_migrate_task()
9514 &env->src_rq->cfs_tasks, se.group_node) { in detach_one_task()
9579 p = list_last_entry(tasks, struct task_struct, se.group_node); in detach_tasks()
9634 list_add(&p->se.group_node, &env->tasks); in detach_tasks()
9660 list_move(&p->se.group_node, tasks); in detach_tasks()
9713 p = list_first_entry(tasks, struct task_struct, se.group_node); in attach_tasks()
9714 list_del_init(&p->se.group_node); in attach_tasks()
9797 struct sched_entity *se; in __update_blocked_fair() local
9810 se = cfs_rq->tg->se[cpu]; in __update_blocked_fair()
9811 if (se && !skip_blocked_update(se)) in __update_blocked_fair()
9812 update_load_avg(cfs_rq_of(se), se, UPDATE_TG); in __update_blocked_fair()
9837 struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; in update_cfs_rq_h_load() local
9845 for_each_sched_entity(se) { in update_cfs_rq_h_load()
9846 cfs_rq = cfs_rq_of(se); in update_cfs_rq_h_load()
9847 WRITE_ONCE(cfs_rq->h_load_next, se); in update_cfs_rq_h_load()
9852 if (!se) { in update_cfs_rq_h_load()
9857 while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) { in update_cfs_rq_h_load()
9859 load = div64_ul(load * se->avg.load_avg, in update_cfs_rq_h_load()
9861 cfs_rq = group_cfs_rq(se); in update_cfs_rq_h_load()
9872 return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, in task_h_load()
9890 return p->se.avg.load_avg; in task_h_load()
10626 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) in task_running_on_cpu()
12970 __entity_slice_used(struct sched_entity *se, int min_nr_tasks) in __entity_slice_used() argument
12972 u64 rtime = se->sum_exec_runtime - se->prev_sum_exec_runtime; in __entity_slice_used()
12973 u64 slice = se->slice; in __entity_slice_used()
12999 __entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE)) in task_tick_core()
13006 static void se_fi_update(const struct sched_entity *se, unsigned int fi_seq, in se_fi_update() argument
13009 for_each_sched_entity(se) { in se_fi_update()
13010 struct cfs_rq *cfs_rq = cfs_rq_of(se); in se_fi_update()
13024 struct sched_entity *se = &p->se; in task_vruntime_update() local
13029 se_fi_update(se, rq->core->core_forceidle_seq, in_fi); in task_vruntime_update()
13036 const struct sched_entity *sea = &a->se; in cfs_prio_less()
13037 const struct sched_entity *seb = &b->se; in cfs_prio_less()
13106 struct sched_entity *se = &curr->se; in task_tick_fair() local
13108 for_each_sched_entity(se) { in task_tick_fair()
13109 cfs_rq = cfs_rq_of(se); in task_tick_fair()
13110 entity_tick(cfs_rq, se, queued); in task_tick_fair()
13162 static void propagate_entity_cfs_rq(struct sched_entity *se) in propagate_entity_cfs_rq() argument
13164 struct cfs_rq *cfs_rq = cfs_rq_of(se); in propagate_entity_cfs_rq()
13173 se = se->parent; in propagate_entity_cfs_rq()
13175 for_each_sched_entity(se) { in propagate_entity_cfs_rq()
13176 cfs_rq = cfs_rq_of(se); in propagate_entity_cfs_rq()
13178 update_load_avg(cfs_rq, se, UPDATE_TG); in propagate_entity_cfs_rq()
13188 static void propagate_entity_cfs_rq(struct sched_entity *se) { } in propagate_entity_cfs_rq() argument
13191 static void detach_entity_cfs_rq(struct sched_entity *se) in detach_entity_cfs_rq() argument
13193 struct cfs_rq *cfs_rq = cfs_rq_of(se); in detach_entity_cfs_rq()
13202 if (!se->avg.last_update_time) in detach_entity_cfs_rq()
13207 update_load_avg(cfs_rq, se, 0); in detach_entity_cfs_rq()
13208 detach_entity_load_avg(cfs_rq, se); in detach_entity_cfs_rq()
13210 propagate_entity_cfs_rq(se); in detach_entity_cfs_rq()
13213 static void attach_entity_cfs_rq(struct sched_entity *se) in attach_entity_cfs_rq() argument
13215 struct cfs_rq *cfs_rq = cfs_rq_of(se); in attach_entity_cfs_rq()
13218 update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD); in attach_entity_cfs_rq()
13219 attach_entity_load_avg(cfs_rq, se); in attach_entity_cfs_rq()
13221 propagate_entity_cfs_rq(se); in attach_entity_cfs_rq()
13226 struct sched_entity *se = &p->se; in detach_task_cfs_rq() local
13228 detach_entity_cfs_rq(se); in detach_task_cfs_rq()
13233 struct sched_entity *se = &p->se; in attach_task_cfs_rq() local
13235 attach_entity_cfs_rq(se); in attach_task_cfs_rq()
13245 WARN_ON_ONCE(p->se.sched_delayed); in switched_to_fair()
13266 struct sched_entity *se = &p->se; in __set_next_task_fair() local
13274 list_move(&se->group_node, &rq->cfs_tasks); in __set_next_task_fair()
13280 WARN_ON_ONCE(se->sched_delayed); in __set_next_task_fair()
13297 struct sched_entity *se = &p->se; in set_next_task_fair() local
13299 for_each_sched_entity(se) { in set_next_task_fair()
13300 struct cfs_rq *cfs_rq = cfs_rq_of(se); in set_next_task_fair()
13302 set_next_entity(cfs_rq, se); in set_next_task_fair()
13333 p->se.avg.last_update_time = 0; in task_change_group_fair()
13346 if (tg->se) in free_fair_sched_group()
13347 kfree(tg->se[i]); in free_fair_sched_group()
13351 kfree(tg->se); in free_fair_sched_group()
13356 struct sched_entity *se; in alloc_fair_sched_group() local
13363 tg->se = kcalloc(nr_cpu_ids, sizeof(se), GFP_KERNEL); in alloc_fair_sched_group()
13364 if (!tg->se) in alloc_fair_sched_group()
13377 se = kzalloc_node(sizeof(struct sched_entity_stats), in alloc_fair_sched_group()
13379 if (!se) in alloc_fair_sched_group()
13383 init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]); in alloc_fair_sched_group()
13384 init_entity_runnable_average(se); in alloc_fair_sched_group()
13397 struct sched_entity *se; in online_fair_sched_group() local
13404 se = tg->se[i]; in online_fair_sched_group()
13407 attach_entity_cfs_rq(se); in online_fair_sched_group()
13421 struct sched_entity *se = tg->se[cpu]; in unregister_fair_sched_group() local
13424 if (se) { in unregister_fair_sched_group()
13425 if (se->sched_delayed) { in unregister_fair_sched_group()
13427 if (se->sched_delayed) { in unregister_fair_sched_group()
13429 dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); in unregister_fair_sched_group()
13433 remove_entity_load_avg(se); in unregister_fair_sched_group()
13448 struct sched_entity *se, int cpu, in init_tg_cfs_entry() argument
13458 tg->se[cpu] = se; in init_tg_cfs_entry()
13461 if (!se) in init_tg_cfs_entry()
13465 se->cfs_rq = &rq->cfs; in init_tg_cfs_entry()
13466 se->depth = 0; in init_tg_cfs_entry()
13468 se->cfs_rq = parent->my_q; in init_tg_cfs_entry()
13469 se->depth = parent->depth + 1; in init_tg_cfs_entry()
13472 se->my_q = cfs_rq; in init_tg_cfs_entry()
13474 update_load_set(&se->load, NICE_0_LOAD); in init_tg_cfs_entry()
13475 se->parent = parent; in init_tg_cfs_entry()
13489 if (!tg->se[0]) in __sched_group_set_shares()
13500 struct sched_entity *se = tg->se[i]; in __sched_group_set_shares() local
13506 for_each_sched_entity(se) { in __sched_group_set_shares()
13507 update_load_avg(cfs_rq_of(se), se, UPDATE_TG); in __sched_group_set_shares()
13508 update_cfs_group(se); in __sched_group_set_shares()
13551 struct sched_entity *se = tg->se[i]; in sched_group_set_idle() local
13568 for_each_sched_entity(se) { in sched_group_set_idle()
13569 struct cfs_rq *cfs_rq = cfs_rq_of(se); in sched_group_set_idle()
13571 if (!se->on_rq) in sched_group_set_idle()
13600 struct sched_entity *se = &task->se; in get_rr_interval_fair() local
13608 rr_interval = NS_TO_JIFFIES(se->slice); in get_rr_interval_fair()