97 static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
101 	struct rt_bandwidth *rt_b =  in sched_rt_period_timer()  local
106 	raw_spin_lock(&rt_b->rt_runtime_lock);  in sched_rt_period_timer()
108 		overrun = hrtimer_forward_now(timer, rt_b->rt_period);  in sched_rt_period_timer()
112 		raw_spin_unlock(&rt_b->rt_runtime_lock);  in sched_rt_period_timer()
113 		idle = do_sched_rt_period_timer(rt_b, overrun);  in sched_rt_period_timer()
114 		raw_spin_lock(&rt_b->rt_runtime_lock);  in sched_rt_period_timer()
117 		rt_b->rt_period_active = 0;  in sched_rt_period_timer()
118 	raw_spin_unlock(&rt_b->rt_runtime_lock);  in sched_rt_period_timer()
123 void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)  in init_rt_bandwidth()  argument
125 	rt_b->rt_period = ns_to_ktime(period);  in init_rt_bandwidth()
126 	rt_b->rt_runtime = runtime;  in init_rt_bandwidth()
128 	raw_spin_lock_init(&rt_b->rt_runtime_lock);  in init_rt_bandwidth()
130 	hrtimer_setup(&rt_b->rt_period_timer, sched_rt_period_timer, CLOCK_MONOTONIC,  in init_rt_bandwidth()
134 static inline void do_start_rt_bandwidth(struct rt_bandwidth *rt_b)  in do_start_rt_bandwidth()  argument
136 	raw_spin_lock(&rt_b->rt_runtime_lock);  in do_start_rt_bandwidth()
137 	if (!rt_b->rt_period_active) {  in do_start_rt_bandwidth()
138 		rt_b->rt_period_active = 1;  in do_start_rt_bandwidth()
147 		hrtimer_forward_now(&rt_b->rt_period_timer, ns_to_ktime(0));  in do_start_rt_bandwidth()
148 		hrtimer_start_expires(&rt_b->rt_period_timer,  in do_start_rt_bandwidth()
151 	raw_spin_unlock(&rt_b->rt_runtime_lock);  in do_start_rt_bandwidth()
154 static void start_rt_bandwidth(struct rt_bandwidth *rt_b)  in start_rt_bandwidth()  argument
156 	if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)  in start_rt_bandwidth()
159 	do_start_rt_bandwidth(rt_b);  in start_rt_bandwidth()
162 static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)  in destroy_rt_bandwidth()  argument
164 	hrtimer_cancel(&rt_b->rt_period_timer);  in destroy_rt_bandwidth()
594 struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)  in sched_rt_period_rt_rq()  argument
596 	return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu];  in sched_rt_period_rt_rq()
606 	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in sched_rt_bandwidth_account()  local
608 	return (hrtimer_active(&rt_b->rt_period_timer) ||  in sched_rt_bandwidth_account()
609 		rt_rq->rt_time < rt_b->rt_runtime);  in sched_rt_bandwidth_account()
618 	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in do_balance_runtime()  local
625 	raw_spin_lock(&rt_b->rt_runtime_lock);  in do_balance_runtime()
626 	rt_period = ktime_to_ns(rt_b->rt_period);  in do_balance_runtime()
628 		struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);  in do_balance_runtime()
662 	raw_spin_unlock(&rt_b->rt_runtime_lock);  in do_balance_runtime()
678 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in __disable_runtime()  local
682 		raw_spin_lock(&rt_b->rt_runtime_lock);  in __disable_runtime()
690 				rt_rq->rt_runtime == rt_b->rt_runtime)  in __disable_runtime()
699 		want = rt_b->rt_runtime - rt_rq->rt_runtime;  in __disable_runtime()
705 			struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);  in __disable_runtime()
743 		raw_spin_unlock(&rt_b->rt_runtime_lock);  in __disable_runtime()
762 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in __enable_runtime()  local
764 		raw_spin_lock(&rt_b->rt_runtime_lock);  in __enable_runtime()
766 		rt_rq->rt_runtime = rt_b->rt_runtime;  in __enable_runtime()
770 		raw_spin_unlock(&rt_b->rt_runtime_lock);  in __enable_runtime()
789 static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)  in do_sched_rt_period_timer()  argument
805 	if (rt_b == &root_task_group.rt_bandwidth)  in do_sched_rt_period_timer()
810 		struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);  in do_sched_rt_period_timer()
821 			rt_rq->rt_runtime = rt_b->rt_runtime;  in do_sched_rt_period_timer()
868 	if (!throttled && (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF))  in do_sched_rt_period_timer()
890 		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);  in sched_rt_runtime_exceeded()  local
896 		if (likely(rt_b->rt_runtime)) {  in sched_rt_runtime_exceeded()
959 struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)  in sched_rt_period_rt_rq()  argument