193 void unregister_rt_sched_group(struct task_group *tg)  in unregister_rt_sched_group()  argument
195 	if (tg->rt_se)  in unregister_rt_sched_group()
196 		destroy_rt_bandwidth(&tg->rt_bandwidth);  in unregister_rt_sched_group()
199 void free_rt_sched_group(struct task_group *tg)  in free_rt_sched_group()  argument
204 		if (tg->rt_rq)  in free_rt_sched_group()
205 			kfree(tg->rt_rq[i]);  in free_rt_sched_group()
206 		if (tg->rt_se)  in free_rt_sched_group()
207 			kfree(tg->rt_se[i]);  in free_rt_sched_group()
210 	kfree(tg->rt_rq);  in free_rt_sched_group()
211 	kfree(tg->rt_se);  in free_rt_sched_group()
214 void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,  in init_tg_rt_entry()  argument
223 	rt_rq->tg = tg;  in init_tg_rt_entry()
225 	tg->rt_rq[cpu] = rt_rq;  in init_tg_rt_entry()
226 	tg->rt_se[cpu] = rt_se;  in init_tg_rt_entry()
241 int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)  in alloc_rt_sched_group()  argument
247 	tg->rt_rq = kcalloc(nr_cpu_ids, sizeof(rt_rq), GFP_KERNEL);  in alloc_rt_sched_group()
248 	if (!tg->rt_rq)  in alloc_rt_sched_group()
250 	tg->rt_se = kcalloc(nr_cpu_ids, sizeof(rt_se), GFP_KERNEL);  in alloc_rt_sched_group()
251 	if (!tg->rt_se)  in alloc_rt_sched_group()
254 	init_rt_bandwidth(&tg->rt_bandwidth, ktime_to_ns(global_rt_period()), 0);  in alloc_rt_sched_group()
268 		rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;  in alloc_rt_sched_group()
269 		init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);  in alloc_rt_sched_group()
308 void unregister_rt_sched_group(struct task_group *tg) { }  in unregister_rt_sched_group()  argument
310 void free_rt_sched_group(struct task_group *tg) { }  in free_rt_sched_group()  argument
312 int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)  in alloc_rt_sched_group()  argument
485 	if (!rt_rq->tg)  in sched_rt_runtime()
493 	return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period);  in sched_rt_period()
498 static inline struct task_group *next_task_group(struct task_group *tg)  in next_task_group()  argument
501 		tg = list_entry_rcu(tg->list.next,  in next_task_group()
503 	} while (&tg->list != &task_groups && task_group_is_autogroup(tg));  in next_task_group()
505 	if (&tg->list == &task_groups)  in next_task_group()
506 		tg = NULL;  in next_task_group()
508 	return tg;  in next_task_group()
535 	rt_se = rt_rq->tg->rt_se[cpu];  in sched_rt_rq_enqueue()
553 	rt_se = rt_rq->tg->rt_se[cpu];  in sched_rt_rq_dequeue()
601 	return &rt_rq->tg->rt_bandwidth;  in sched_rt_bandwidth()
1159 	if (rt_rq->tg)  in inc_rt_group()
1160 		start_rt_bandwidth(&rt_rq->tg->rt_bandwidth);  in inc_rt_group()
2662 static inline int tg_has_rt_tasks(struct task_group *tg)  in tg_has_rt_tasks()  argument
2671 	if (task_group_is_autogroup(tg))  in tg_has_rt_tasks()
2674 	css_task_iter_start(&tg->css, 0, &it);  in tg_has_rt_tasks()
2683 	struct task_group *tg;  member
2688 static int tg_rt_schedulable(struct task_group *tg, void *data)  in tg_rt_schedulable()  argument
2695 	period = ktime_to_ns(tg->rt_bandwidth.rt_period);  in tg_rt_schedulable()
2696 	runtime = tg->rt_bandwidth.rt_runtime;  in tg_rt_schedulable()
2698 	if (tg == d->tg) {  in tg_rt_schedulable()
2713 	    tg->rt_bandwidth.rt_runtime && tg_has_rt_tasks(tg))  in tg_rt_schedulable()
2727 	list_for_each_entry_rcu(child, &tg->children, siblings) {  in tg_rt_schedulable()
2731 		if (child == d->tg) {  in tg_rt_schedulable()
2745 static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)  in __rt_schedulable()  argument
2750 		.tg = tg,  in __rt_schedulable()
2762 static int tg_set_rt_bandwidth(struct task_group *tg,  in tg_set_rt_bandwidth()  argument
2771 	if (tg == &root_task_group && rt_runtime == 0)  in tg_set_rt_bandwidth()
2785 	err = __rt_schedulable(tg, rt_period, rt_runtime);  in tg_set_rt_bandwidth()
2789 	raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);  in tg_set_rt_bandwidth()
2790 	tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);  in tg_set_rt_bandwidth()
2791 	tg->rt_bandwidth.rt_runtime = rt_runtime;  in tg_set_rt_bandwidth()
2794 		struct rt_rq *rt_rq = tg->rt_rq[i];  in tg_set_rt_bandwidth()
2800 	raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);  in tg_set_rt_bandwidth()
2807 int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)  in sched_group_set_rt_runtime()  argument
2811 	rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);  in sched_group_set_rt_runtime()
2818 	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);  in sched_group_set_rt_runtime()
2821 long sched_group_rt_runtime(struct task_group *tg)  in sched_group_rt_runtime()  argument
2825 	if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)  in sched_group_rt_runtime()
2828 	rt_runtime_us = tg->rt_bandwidth.rt_runtime;  in sched_group_rt_runtime()
2833 int sched_group_set_rt_period(struct task_group *tg, u64 rt_period_us)  in sched_group_set_rt_period()  argument
2841 	rt_runtime = tg->rt_bandwidth.rt_runtime;  in sched_group_set_rt_period()
2843 	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);  in sched_group_set_rt_period()
2846 long sched_group_rt_period(struct task_group *tg)  in sched_group_rt_period()  argument
2850 	rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);  in sched_group_rt_period()
2868 int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)  in sched_rt_can_attach()  argument
2871 	if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)  in sched_rt_can_attach()