176 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)  in rq_of_rt_rq()
178 	return rt_rq->rq;  in rq_of_rt_rq()
186 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)  in rq_of_rt_se()
190 	return rt_rq->rq;  in rq_of_rt_se()
218 	struct rq *rq = cpu_rq(cpu);  in init_tg_rt_entry()  local
222 	rt_rq->rq = rq;  in init_tg_rt_entry()
232 		rt_se->rt_rq = &rq->rt;  in init_tg_rt_entry()
289 static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)  in rq_of_rt_rq()
291 	return container_of(rt_rq, struct rq, rt);  in rq_of_rt_rq()
294 static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)  in rq_of_rt_se()
303 	struct rq *rq = rq_of_rt_se(rt_se);  in rt_rq_of_se()  local
305 	return &rq->rt;  in rt_rq_of_se()
320 static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev)  in need_pull_rt_task()  argument
323 	return rq->online && rq->rt.highest_prio.curr > prev->prio;  in need_pull_rt_task()
326 static inline int rt_overloaded(struct rq *rq)  in rt_overloaded()  argument
328 	return atomic_read(&rq->rd->rto_count);  in rt_overloaded()
331 static inline void rt_set_overload(struct rq *rq)  in rt_set_overload()  argument
333 	if (!rq->online)  in rt_set_overload()
336 	cpumask_set_cpu(rq->cpu, rq->rd->rto_mask);  in rt_set_overload()
347 	atomic_inc(&rq->rd->rto_count);  in rt_set_overload()
350 static inline void rt_clear_overload(struct rq *rq)  in rt_clear_overload()  argument
352 	if (!rq->online)  in rt_clear_overload()
356 	atomic_dec(&rq->rd->rto_count);  in rt_clear_overload()
357 	cpumask_clear_cpu(rq->cpu, rq->rd->rto_mask);  in rt_clear_overload()
360 static inline int has_pushable_tasks(struct rq *rq)  in has_pushable_tasks()  argument
362 	return !plist_head_empty(&rq->rt.pushable_tasks);  in has_pushable_tasks()
368 static void push_rt_tasks(struct rq *);
369 static void pull_rt_task(struct rq *);
371 static inline void rt_queue_push_tasks(struct rq *rq)  in rt_queue_push_tasks()  argument
373 	if (!has_pushable_tasks(rq))  in rt_queue_push_tasks()
376 	queue_balance_callback(rq, &per_cpu(rt_push_head, rq->cpu), push_rt_tasks);  in rt_queue_push_tasks()
379 static inline void rt_queue_pull_task(struct rq *rq)  in rt_queue_pull_task()  argument
381 	queue_balance_callback(rq, &per_cpu(rt_pull_head, rq->cpu), pull_rt_task);  in rt_queue_pull_task()
384 static void enqueue_pushable_task(struct rq *rq, struct task_struct *p)  in enqueue_pushable_task()  argument
386 	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);  in enqueue_pushable_task()
388 	plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks);  in enqueue_pushable_task()
391 	if (p->prio < rq->rt.highest_prio.next)  in enqueue_pushable_task()
392 		rq->rt.highest_prio.next = p->prio;  in enqueue_pushable_task()
394 	if (!rq->rt.overloaded) {  in enqueue_pushable_task()
395 		rt_set_overload(rq);  in enqueue_pushable_task()
396 		rq->rt.overloaded = 1;  in enqueue_pushable_task()
400 static void dequeue_pushable_task(struct rq *rq, struct task_struct *p)  in dequeue_pushable_task()  argument
402 	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);  in dequeue_pushable_task()
405 	if (has_pushable_tasks(rq)) {  in dequeue_pushable_task()
406 		p = plist_first_entry(&rq->rt.pushable_tasks,  in dequeue_pushable_task()
408 		rq->rt.highest_prio.next = p->prio;  in dequeue_pushable_task()
410 		rq->rt.highest_prio.next = MAX_RT_PRIO-1;  in dequeue_pushable_task()
412 		if (rq->rt.overloaded) {  in dequeue_pushable_task()
413 			rt_clear_overload(rq);  in dequeue_pushable_task()
414 			rq->rt.overloaded = 0;  in dequeue_pushable_task()
421 static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p)  in enqueue_pushable_task()  argument
425 static inline void dequeue_pushable_task(struct rq *rq, struct task_struct *p)  in dequeue_pushable_task()  argument
429 static inline void rt_queue_push_tasks(struct rq *rq)  in rt_queue_push_tasks()  argument
511 #define for_each_rt_rq(rt_rq, iter, rq)					\  argument
514 		(rt_rq = iter->rt_rq[cpu_of(rq)]);)
530 	struct rq *rq = rq_of_rt_rq(rt_rq);  in sched_rt_rq_enqueue()  local
533 	int cpu = cpu_of(rq);  in sched_rt_rq_enqueue()
544 			resched_curr(rq);  in sched_rt_rq_enqueue()
668 static void __disable_runtime(struct rq *rq)  in __disable_runtime()  argument
670 	struct root_domain *rd = rq->rd;  in __disable_runtime()
677 	for_each_rt_rq(rt_rq, iter, rq) {  in __disable_runtime()
750 static void __enable_runtime(struct rq *rq)  in __enable_runtime()  argument
761 	for_each_rt_rq(rt_rq, iter, rq) {  in __enable_runtime()
811 		struct rq *rq = rq_of_rt_rq(rt_rq);  in do_sched_rt_period_timer()  local
827 		rq_lock(rq, &rf);  in do_sched_rt_period_timer()
828 		update_rq_clock(rq);  in do_sched_rt_period_timer()
849 				if (rt_rq->rt_nr_running && rq->curr == rq->idle)  in do_sched_rt_period_timer()
850 					rq_clock_cancel_skipupdate(rq);  in do_sched_rt_period_timer()
865 		rq_unlock(rq, &rf);  in do_sched_rt_period_timer()
921 #define for_each_rt_rq(rt_rq, iter, rq) \  argument
922 	for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
934 	struct rq *rq = rq_of_rt_rq(rt_rq);  in sched_rt_rq_enqueue()  local
940 	resched_curr(rq);  in sched_rt_rq_enqueue()
965 static void __enable_runtime(struct rq *rq) { }  in __enable_runtime()  argument
966 static void __disable_runtime(struct rq *rq) { }  in __disable_runtime()  argument
987 static void update_curr_rt(struct rq *rq)  in update_curr_rt()  argument
989 	struct task_struct *donor = rq->donor;  in update_curr_rt()
995 	delta_exec = update_curr_common(rq);  in update_curr_rt()
1014 				resched_curr(rq);  in update_curr_rt()
1026 	struct rq *rq = rq_of_rt_rq(rt_rq);  in dequeue_top_rt_rq()  local
1028 	BUG_ON(&rq->rt != rt_rq);  in dequeue_top_rt_rq()
1033 	BUG_ON(!rq->nr_running);  in dequeue_top_rt_rq()
1035 	sub_nr_running(rq, count);  in dequeue_top_rt_rq()
1043 	struct rq *rq = rq_of_rt_rq(rt_rq);  in enqueue_top_rt_rq()  local
1045 	BUG_ON(&rq->rt != rt_rq);  in enqueue_top_rt_rq()
1054 		add_nr_running(rq, rt_rq->rt_nr_running);  in enqueue_top_rt_rq()
1059 	cpufreq_update_util(rq, 0);  in enqueue_top_rt_rq()
1067 	struct rq *rq = rq_of_rt_rq(rt_rq);  in inc_rt_prio_smp()  local
1073 	if (&rq->rt != rt_rq)  in inc_rt_prio_smp()
1076 	if (rq->online && prio < prev_prio)  in inc_rt_prio_smp()
1077 		cpupri_set(&rq->rd->cpupri, rq->cpu, prio);  in inc_rt_prio_smp()
1083 	struct rq *rq = rq_of_rt_rq(rt_rq);  in dec_rt_prio_smp()  local
1089 	if (&rq->rt != rt_rq)  in dec_rt_prio_smp()
1092 	if (rq->online && rt_rq->highest_prio.curr != prev_prio)  in dec_rt_prio_smp()
1093 		cpupri_set(&rq->rd->cpupri, rq->cpu, rt_rq->highest_prio.curr);  in dec_rt_prio_smp()
1437 	struct rq *rq = rq_of_rt_se(rt_se);  in enqueue_rt_entity()  local
1444 	enqueue_top_rt_rq(&rq->rt);  in enqueue_rt_entity()
1449 	struct rq *rq = rq_of_rt_se(rt_se);  in dequeue_rt_entity()  local
1461 	enqueue_top_rt_rq(&rq->rt);  in dequeue_rt_entity()
1468 enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)  in enqueue_task_rt()  argument
1480 	if (!task_current(rq, p) && p->nr_cpus_allowed > 1)  in enqueue_task_rt()
1481 		enqueue_pushable_task(rq, p);  in enqueue_task_rt()
1484 static bool dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)  in dequeue_task_rt()  argument
1488 	update_curr_rt(rq);  in dequeue_task_rt()
1491 	dequeue_pushable_task(rq, p);  in dequeue_task_rt()
1514 static void requeue_task_rt(struct rq *rq, struct task_struct *p, int head)  in requeue_task_rt()  argument
1525 static void yield_task_rt(struct rq *rq)  in yield_task_rt()  argument
1527 	requeue_task_rt(rq, rq->curr, 0);  in yield_task_rt()
1537 	struct rq *rq;  in select_task_rq_rt()  local
1544 	rq = cpu_rq(cpu);  in select_task_rq_rt()
1547 	curr = READ_ONCE(rq->curr); /* unlocked access */  in select_task_rq_rt()
1548 	donor = READ_ONCE(rq->donor);  in select_task_rq_rt()
1606 static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)  in check_preempt_equal_prio()  argument
1608 	if (rq->curr->nr_cpus_allowed == 1 ||  in check_preempt_equal_prio()
1609 	    !cpupri_find(&rq->rd->cpupri, rq->donor, NULL))  in check_preempt_equal_prio()
1617 	    cpupri_find(&rq->rd->cpupri, p, NULL))  in check_preempt_equal_prio()
1625 	requeue_task_rt(rq, p, 1);  in check_preempt_equal_prio()
1626 	resched_curr(rq);  in check_preempt_equal_prio()
1629 static int balance_rt(struct rq *rq, struct task_struct *p, struct rq_flags *rf)  in balance_rt()  argument
1631 	if (!on_rt_rq(&p->rt) && need_pull_rt_task(rq, p)) {  in balance_rt()
1638 		rq_unpin_lock(rq, rf);  in balance_rt()
1639 		pull_rt_task(rq);  in balance_rt()
1640 		rq_repin_lock(rq, rf);  in balance_rt()
1643 	return sched_stop_runnable(rq) || sched_dl_runnable(rq) || sched_rt_runnable(rq);  in balance_rt()
1650 static void wakeup_preempt_rt(struct rq *rq, struct task_struct *p, int flags)  in wakeup_preempt_rt()  argument
1652 	struct task_struct *donor = rq->donor;  in wakeup_preempt_rt()
1655 		resched_curr(rq);  in wakeup_preempt_rt()
1672 	if (p->prio == donor->prio && !test_tsk_need_resched(rq->curr))  in wakeup_preempt_rt()
1673 		check_preempt_equal_prio(rq, p);  in wakeup_preempt_rt()
1677 static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool first)  in set_next_task_rt()  argument
1680 	struct rt_rq *rt_rq = &rq->rt;  in set_next_task_rt()
1682 	p->se.exec_start = rq_clock_task(rq);  in set_next_task_rt()
1687 	dequeue_pushable_task(rq, p);  in set_next_task_rt()
1697 	if (rq->donor->sched_class != &rt_sched_class)  in set_next_task_rt()
1698 		update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0);  in set_next_task_rt()
1700 	rt_queue_push_tasks(rq);  in set_next_task_rt()
1721 static struct task_struct *_pick_next_task_rt(struct rq *rq)  in _pick_next_task_rt()  argument
1724 	struct rt_rq *rt_rq  = &rq->rt;  in _pick_next_task_rt()
1736 static struct task_struct *pick_task_rt(struct rq *rq)  in pick_task_rt()  argument
1740 	if (!sched_rt_runnable(rq))  in pick_task_rt()
1743 	p = _pick_next_task_rt(rq);  in pick_task_rt()
1748 static void put_prev_task_rt(struct rq *rq, struct task_struct *p, struct task_struct *next)  in put_prev_task_rt()  argument
1751 	struct rt_rq *rt_rq = &rq->rt;  in put_prev_task_rt()
1756 	update_curr_rt(rq);  in put_prev_task_rt()
1758 	update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1);  in put_prev_task_rt()
1765 		enqueue_pushable_task(rq, p);  in put_prev_task_rt()
1777 static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu)  in pick_highest_pushable_task()  argument
1779 	struct plist_head *head = &rq->rt.pushable_tasks;  in pick_highest_pushable_task()
1782 	if (!has_pushable_tasks(rq))  in pick_highest_pushable_task()
1786 		if (task_is_pushable(rq, p, cpu))  in pick_highest_pushable_task()
1887 static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)  in find_lock_lowest_rq()  argument
1889 	struct rq *lowest_rq = NULL;  in find_lock_lowest_rq()
1896 		if ((cpu == -1) || (cpu == rq->cpu))  in find_lock_lowest_rq()
1912 		if (double_lock_balance(rq, lowest_rq)) {  in find_lock_lowest_rq()
1922 			if (unlikely(task_rq(task) != rq ||  in find_lock_lowest_rq()
1924 				     task_on_cpu(rq, task) ||  in find_lock_lowest_rq()
1929 				double_unlock_balance(rq, lowest_rq);  in find_lock_lowest_rq()
1940 		double_unlock_balance(rq, lowest_rq);  in find_lock_lowest_rq()
1947 static struct task_struct *pick_next_pushable_task(struct rq *rq)  in pick_next_pushable_task()  argument
1951 	if (!has_pushable_tasks(rq))  in pick_next_pushable_task()
1954 	p = plist_first_entry(&rq->rt.pushable_tasks,  in pick_next_pushable_task()
1957 	BUG_ON(rq->cpu != task_cpu(p));  in pick_next_pushable_task()
1958 	BUG_ON(task_current(rq, p));  in pick_next_pushable_task()
1959 	BUG_ON(task_current_donor(rq, p));  in pick_next_pushable_task()
1973 static int push_rt_task(struct rq *rq, bool pull)  in push_rt_task()  argument
1976 	struct rq *lowest_rq;  in push_rt_task()
1979 	if (!rq->rt.overloaded)  in push_rt_task()
1982 	next_task = pick_next_pushable_task(rq);  in push_rt_task()
1992 	if (unlikely(next_task->prio < rq->donor->prio)) {  in push_rt_task()
1993 		resched_curr(rq);  in push_rt_task()
2001 		if (!pull || rq->push_busy)  in push_rt_task()
2013 		if (rq->donor->sched_class != &rt_sched_class)  in push_rt_task()
2016 		cpu = find_lowest_rq(rq->curr);  in push_rt_task()
2017 		if (cpu == -1 || cpu == rq->cpu)  in push_rt_task()
2026 		push_task = get_push_task(rq);  in push_rt_task()
2029 			raw_spin_rq_unlock(rq);  in push_rt_task()
2030 			stop_one_cpu_nowait(rq->cpu, push_cpu_stop,  in push_rt_task()
2031 					    push_task, &rq->push_work);  in push_rt_task()
2033 			raw_spin_rq_lock(rq);  in push_rt_task()
2039 	if (WARN_ON(next_task == rq->curr))  in push_rt_task()
2046 	lowest_rq = find_lock_lowest_rq(next_task, rq);  in push_rt_task()
2057 		task = pick_next_pushable_task(rq);  in push_rt_task()
2080 	move_queued_task_locked(rq, lowest_rq, next_task);  in push_rt_task()
2084 	double_unlock_balance(rq, lowest_rq);  in push_rt_task()
2091 static void push_rt_tasks(struct rq *rq)  in push_rt_tasks()  argument
2094 	while (push_rt_task(rq, false))  in push_rt_tasks()
2198 static void tell_cpu_to_push(struct rq *rq)  in tell_cpu_to_push()  argument
2203 	atomic_inc(&rq->rd->rto_loop_next);  in tell_cpu_to_push()
2206 	if (!rto_start_trylock(&rq->rd->rto_loop_start))  in tell_cpu_to_push()
2209 	raw_spin_lock(&rq->rd->rto_lock);  in tell_cpu_to_push()
2217 	if (rq->rd->rto_cpu < 0)  in tell_cpu_to_push()
2218 		cpu = rto_next_cpu(rq->rd);  in tell_cpu_to_push()
2220 	raw_spin_unlock(&rq->rd->rto_lock);  in tell_cpu_to_push()
2222 	rto_start_unlock(&rq->rd->rto_loop_start);  in tell_cpu_to_push()
2226 		sched_get_rd(rq->rd);  in tell_cpu_to_push()
2227 		irq_work_queue_on(&rq->rd->rto_push_work, cpu);  in tell_cpu_to_push()
2236 	struct rq *rq;  in rto_push_irq_work_func()  local
2239 	rq = this_rq();  in rto_push_irq_work_func()
2245 	if (has_pushable_tasks(rq)) {  in rto_push_irq_work_func()
2246 		raw_spin_rq_lock(rq);  in rto_push_irq_work_func()
2247 		while (push_rt_task(rq, true))  in rto_push_irq_work_func()
2249 		raw_spin_rq_unlock(rq);  in rto_push_irq_work_func()
2269 static void pull_rt_task(struct rq *this_rq)  in pull_rt_task()
2274 	struct rq *src_rq;  in pull_rt_task()
2382 static void task_woken_rt(struct rq *rq, struct task_struct *p)  in task_woken_rt()  argument
2384 	bool need_to_push = !task_on_cpu(rq, p) &&  in task_woken_rt()
2385 			    !test_tsk_need_resched(rq->curr) &&  in task_woken_rt()
2387 			    (dl_task(rq->donor) || rt_task(rq->donor)) &&  in task_woken_rt()
2388 			    (rq->curr->nr_cpus_allowed < 2 ||  in task_woken_rt()
2389 			     rq->donor->prio <= p->prio);  in task_woken_rt()
2392 		push_rt_tasks(rq);  in task_woken_rt()
2396 static void rq_online_rt(struct rq *rq)  in rq_online_rt()  argument
2398 	if (rq->rt.overloaded)  in rq_online_rt()
2399 		rt_set_overload(rq);  in rq_online_rt()
2401 	__enable_runtime(rq);  in rq_online_rt()
2403 	cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio.curr);  in rq_online_rt()
2407 static void rq_offline_rt(struct rq *rq)  in rq_offline_rt()  argument
2409 	if (rq->rt.overloaded)  in rq_offline_rt()
2410 		rt_clear_overload(rq);  in rq_offline_rt()
2412 	__disable_runtime(rq);  in rq_offline_rt()
2414 	cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_INVALID);  in rq_offline_rt()
2421 static void switched_from_rt(struct rq *rq, struct task_struct *p)  in switched_from_rt()  argument
2430 	if (!task_on_rq_queued(p) || rq->rt.rt_nr_running)  in switched_from_rt()
2433 	rt_queue_pull_task(rq);  in switched_from_rt()
2452 static void switched_to_rt(struct rq *rq, struct task_struct *p)  in switched_to_rt()  argument
2458 	if (task_current(rq, p)) {  in switched_to_rt()
2459 		update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0);  in switched_to_rt()
2470 		if (p->nr_cpus_allowed > 1 && rq->rt.overloaded)  in switched_to_rt()
2471 			rt_queue_push_tasks(rq);  in switched_to_rt()
2473 		if (p->prio < rq->donor->prio && cpu_online(cpu_of(rq)))  in switched_to_rt()
2474 			resched_curr(rq);  in switched_to_rt()
2483 prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio)  in prio_changed_rt()  argument
2488 	if (task_current_donor(rq, p)) {  in prio_changed_rt()
2495 			rt_queue_pull_task(rq);  in prio_changed_rt()
2501 		if (p->prio > rq->rt.highest_prio.curr)  in prio_changed_rt()
2502 			resched_curr(rq);  in prio_changed_rt()
2506 			resched_curr(rq);  in prio_changed_rt()
2514 		if (p->prio < rq->donor->prio)  in prio_changed_rt()
2515 			resched_curr(rq);  in prio_changed_rt()
2520 static void watchdog(struct rq *rq, struct task_struct *p)  in watchdog()  argument
2544 static inline void watchdog(struct rq *rq, struct task_struct *p) { }  in watchdog()  argument
2555 static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued)  in task_tick_rt()  argument
2559 	update_curr_rt(rq);  in task_tick_rt()
2560 	update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1);  in task_tick_rt()
2562 	watchdog(rq, p);  in task_tick_rt()
2582 			requeue_task_rt(rq, p, 0);  in task_tick_rt()
2583 			resched_curr(rq);  in task_tick_rt()
2589 static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task)  in get_rr_interval_rt()  argument