64 	return container_of(dl_se, struct task_struct, dl);  in dl_task_of()
69 	return container_of(dl_rq, struct rq, dl);  in rq_of_dl_rq()
84 	return &rq_of_dl_se(dl_se)->dl;  in dl_rq_of_se()
191 		rq->dl.extra_bw += bw;  in __dl_update()
197 	return &cpu_rq(i)->dl.dl_bw;  in dl_bw_of()
218 	struct dl_rq *dl = container_of(dl_b, struct dl_rq, dl_bw);  in __dl_update()  local
220 	dl->extra_bw += bw;  in __dl_update()
326 		sub_running_bw(dl_se, &rq->dl);  in dl_rq_change_utilization()
341 	__sub_rq_bw(dl_se->dl_bw, &rq->dl);  in dl_rq_change_utilization()
342 	__add_rq_bw(new_bw, &rq->dl);  in dl_rq_change_utilization()
370 	WARN_ON_ONCE(p->dl.flags & SCHED_FLAG_SUGOV);  in dl_change_utilization()
375 	dl_rq_change_utilization(task_rq(p), &p->dl, new_bw);  in dl_change_utilization()
438 	struct dl_rq *dl_rq = &rq->dl;  in task_non_contending()
480 					sub_rq_bw(dl_se, &rq->dl);  in task_non_contending()
607 	return dl_entity_preempt(&__node_2_pdl(a)->dl, &__node_2_pdl(b)->dl);  in __pushable_less()
612 	return !RB_EMPTY_ROOT(&rq->dl.pushable_dl_tasks_root.rb_root);  in has_pushable_dl_tasks()
626 				 &rq->dl.pushable_dl_tasks_root,  in enqueue_pushable_dl_task()
629 		rq->dl.earliest_dl.next = p->dl.deadline;  in enqueue_pushable_dl_task()
631 	if (!rq->dl.overloaded) {  in enqueue_pushable_dl_task()
633 		rq->dl.overloaded = 1;  in enqueue_pushable_dl_task()
639 	struct dl_rq *dl_rq = &rq->dl;  in dequeue_pushable_dl_task()
648 		dl_rq->earliest_dl.next = __node_2_pdl(leftmost)->dl.deadline;  in dequeue_pushable_dl_task()
652 	if (!has_pushable_dl_tasks(rq) && rq->dl.overloaded) {  in dequeue_pushable_dl_task()
654 		rq->dl.overloaded = 0;  in dequeue_pushable_dl_task()
718 	if (p->dl.dl_non_contending || p->dl.dl_throttled) {  in dl_task_offline_migration()
725 		sub_running_bw(&p->dl, &rq->dl);  in dl_task_offline_migration()
726 		sub_rq_bw(&p->dl, &rq->dl);  in dl_task_offline_migration()
728 		add_rq_bw(&p->dl, &later_rq->dl);  in dl_task_offline_migration()
729 		add_running_bw(&p->dl, &later_rq->dl);  in dl_task_offline_migration()
731 		sub_rq_bw(&p->dl, &rq->dl);  in dl_task_offline_migration()
732 		add_rq_bw(&p->dl, &later_rq->dl);  in dl_task_offline_migration()
742 	__dl_sub(dl_b, p->dl.dl_bw, cpumask_weight(rq->rd->span));  in dl_task_offline_migration()
747 	__dl_add(dl_b, p->dl.dl_bw, cpumask_weight(later_rq->rd->span));  in dl_task_offline_migration()
1262 		if (!dl_task(dl_se->rq->curr) || dl_entity_preempt(dl_se, &dl_se->rq->curr->dl))  in dl_server_timer()
1446 	u64 u_inact = rq->dl.this_bw - rq->dl.running_bw; /* Utot - Uact */  in grub_reclaim()
1454 	if (u_inact + rq->dl.extra_bw > rq->dl.max_bw - dl_se->dl_bw)  in grub_reclaim()
1457 		u_act = rq->dl.max_bw - u_inact - rq->dl.extra_bw;  in grub_reclaim()
1459 	u_act = (u_act * rq->dl.bw_ratio) >> RATIO_SHIFT;  in grub_reclaim()
1554 			update_stats_dequeue_dl(&rq->dl, dl_se, 0);  in update_curr_dl_se()
1565 		if (!is_leftmost(dl_se, &rq->dl))  in update_curr_dl_se()
1671 	if (!dl_task(dl_se->rq->curr) || dl_entity_preempt(dl_se, &rq->curr->dl))  in dl_server_start()
1732 		__add_rq_bw(new_bw, &rq->dl);  in dl_server_apply_params()
1761 	struct sched_dl_entity *dl_se = &donor->dl;  in update_curr_dl()
1806 			sub_running_bw(&p->dl, dl_rq_of_se(&p->dl));  in inactive_task_timer()
1807 			sub_rq_bw(&p->dl, dl_rq_of_se(&p->dl));  in inactive_task_timer()
1812 		__dl_sub(dl_b, p->dl.dl_bw, dl_bw_cpus(task_cpu(p)));  in inactive_task_timer()
1823 	sub_running_bw(dl_se, &rq->dl);  in inactive_task_timer()
2118 	if (is_dl_boosted(&p->dl)) {  in enqueue_task_dl()
2131 		if (p->dl.dl_throttled) {  in enqueue_task_dl()
2137 			cancel_replenish_timer(&p->dl);  in enqueue_task_dl()
2138 			p->dl.dl_throttled = 0;  in enqueue_task_dl()
2150 		p->dl.dl_throttled = 0;  in enqueue_task_dl()
2159 	update_stats_wait_start_dl(dl_rq_of_se(&p->dl), &p->dl);  in enqueue_task_dl()
2164 	enqueue_dl_entity(&p->dl, flags);  in enqueue_task_dl()
2166 	if (dl_server(&p->dl))  in enqueue_task_dl()
2169 	if (!task_current(rq, p) && !p->dl.dl_throttled && p->nr_cpus_allowed > 1)  in enqueue_task_dl()
2180 	dequeue_dl_entity(&p->dl, flags);  in dequeue_task_dl()
2181 	if (!p->dl.dl_throttled && !dl_server(&p->dl))  in dequeue_task_dl()
2205 	rq->curr->dl.dl_yielded = 1;  in yield_task_dl()
2222 	return (!rq->dl.dl_nr_running ||  in dl_task_is_earliest_deadline()
2223 		dl_time_before(p->dl.deadline,  in dl_task_is_earliest_deadline()
2224 			       rq->dl.earliest_dl.curr));  in dl_task_is_earliest_deadline()
2256 		     !dl_entity_preempt(&p->dl, &donor->dl)) &&  in select_task_rq_dl()
2294 	if (p->dl.dl_non_contending) {  in migrate_task_rq_dl()
2296 		sub_running_bw(&p->dl, &rq->dl);  in migrate_task_rq_dl()
2297 		p->dl.dl_non_contending = 0;  in migrate_task_rq_dl()
2305 		cancel_inactive_timer(&p->dl);  in migrate_task_rq_dl()
2307 	sub_rq_bw(&p->dl, &rq->dl);  in migrate_task_rq_dl()
2334 	if (!on_dl_rq(&p->dl) && need_pull_dl_task(rq, p)) {  in balance_dl()
2357 	if (dl_entity_preempt(&p->dl, &rq->donor->dl)) {  in wakeup_preempt_dl()
2367 	if ((p->dl.deadline == rq->donor->dl.deadline) &&  in wakeup_preempt_dl()
2386 	struct sched_dl_entity *dl_se = &p->dl;  in set_next_task_dl()
2387 	struct dl_rq *dl_rq = &rq->dl;  in set_next_task_dl()
2390 	if (on_dl_rq(&p->dl))  in set_next_task_dl()
2405 		start_hrtick_dl(rq, &p->dl);  in set_next_task_dl()
2425 	struct dl_rq *dl_rq = &rq->dl;  in __pick_task_dl()
2459 	struct sched_dl_entity *dl_se = &p->dl;  in put_prev_task_dl()
2460 	struct dl_rq *dl_rq = &rq->dl;  in put_prev_task_dl()
2462 	if (on_dl_rq(&p->dl))  in put_prev_task_dl()
2468 	if (on_dl_rq(&p->dl) && p->nr_cpus_allowed > 1)  in put_prev_task_dl()
2490 	if (hrtick_enabled_dl(rq) && queued && p->dl.runtime > 0 &&  in task_tick_dl()
2491 	    is_leftmost(&p->dl, &rq->dl))  in task_tick_dl()
2492 		start_hrtick_dl(rq, &p->dl);  in task_tick_dl()
2520 	next_node = rb_first_cached(&rq->dl.pushable_dl_tasks_root);  in pick_earliest_pushable_dl_task()
2684 	p = __node_2_pdl(rb_first_cached(&rq->dl.pushable_dl_tasks_root));  in pick_next_pushable_dl_task()
2718 	    dl_time_before(next_task->dl.deadline, rq->donor->dl.deadline) &&  in push_dl_task()
2808 		if (this_rq->dl.dl_nr_running &&  in pull_dl_task()
2809 		    dl_time_before(this_rq->dl.earliest_dl.curr,  in pull_dl_task()
2810 				   src_rq->dl.earliest_dl.next))  in pull_dl_task()
2821 		if (src_rq->dl.dl_nr_running <= 1)  in pull_dl_task()
2831 		if (p && dl_time_before(p->dl.deadline, dmin) &&  in pull_dl_task()
2840 			if (dl_time_before(p->dl.deadline,  in pull_dl_task()
2841 					   src_rq->donor->dl.deadline))  in pull_dl_task()
2848 				dmin = p->dl.deadline;  in pull_dl_task()
2882 	     !dl_entity_preempt(&p->dl, &rq->donor->dl))) {  in task_woken_dl()
2913 		__dl_sub(src_dl_b, p->dl.dl_bw, dl_bw_cpus(task_cpu(p)));  in set_cpus_allowed_dl()
2923 	if (rq->dl.overloaded)  in rq_online_dl()
2927 	if (rq->dl.dl_nr_running > 0)  in rq_online_dl()
2928 		cpudl_set(&rq->rd->cpudl, rq->cpu, rq->dl.earliest_dl.curr);  in rq_online_dl()
2934 	if (rq->dl.overloaded)  in rq_offline_dl()
2957 	if (!dl_task(p) || dl_entity_is_special(&p->dl)) {  in dl_add_task_root_domain()
2967 	__dl_add(dl_b, p->dl.dl_bw, cpumask_weight(rq->rd->span));  in dl_add_task_root_domain()
3010 	if (task_on_rq_queued(p) && p->dl.dl_runtime)  in switched_from_dl()
3011 		task_non_contending(&p->dl);  in switched_from_dl()
3026 		if (p->dl.dl_non_contending)  in switched_from_dl()
3027 			sub_running_bw(&p->dl, &rq->dl);  in switched_from_dl()
3028 		sub_rq_bw(&p->dl, &rq->dl);  in switched_from_dl()
3036 	if (p->dl.dl_non_contending)  in switched_from_dl()
3037 		p->dl.dl_non_contending = 0;  in switched_from_dl()
3044 	if (!task_on_rq_queued(p) || rq->dl.dl_nr_running)  in switched_from_dl()
3056 	cancel_inactive_timer(&p->dl);  in switched_to_dl()
3066 		add_rq_bw(&p->dl, &rq->dl);  in switched_to_dl()
3073 		if (p->nr_cpus_allowed > 1 && rq->dl.overloaded)  in switched_to_dl()
3102 	if (!rq->dl.overloaded)  in prio_changed_dl()
3111 		if (dl_time_before(rq->dl.earliest_dl.curr, p->dl.deadline))  in prio_changed_dl()
3121 		    dl_time_before(p->dl.deadline, rq->curr->dl.deadline))  in prio_changed_dl()
3136 	return p->dl.dl_throttled;  in task_is_throttled_dl()
3140 DEFINE_SCHED_CLASS(dl) = {
3260 		init_dl_rq_bw_ratio(&cpu_rq(cpu)->dl);  in sched_dl_do_global()
3286 	if (new_bw == p->dl.dl_bw && task_has_dl_policy(p))  in sched_dl_overflow()
3300 		if (hrtimer_active(&p->dl.inactive_timer))  in sched_dl_overflow()
3301 			__dl_sub(dl_b, p->dl.dl_bw, cpus);  in sched_dl_overflow()
3305 		   !__dl_overflow(dl_b, cap, p->dl.dl_bw, new_bw)) {  in sched_dl_overflow()
3313 		__dl_sub(dl_b, p->dl.dl_bw, cpus);  in sched_dl_overflow()
3340 	struct sched_dl_entity *dl_se = &p->dl;  in __setparam_dl()
3352 	struct sched_dl_entity *dl_se = &p->dl;  in __getparam_dl()
3450 	struct sched_dl_entity *dl_se = &p->dl;  in dl_param_changed()
3577 	print_dl_rq(m, cpu, &cpu_rq(cpu)->dl);  in print_dl_stats()