5714 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)  in __refill_cfs_bandwidth_runtime()  argument
5718 	if (unlikely(cfs_b->quota == RUNTIME_INF))  in __refill_cfs_bandwidth_runtime()
5721 	cfs_b->runtime += cfs_b->quota;  in __refill_cfs_bandwidth_runtime()
5722 	runtime = cfs_b->runtime_snap - cfs_b->runtime;  in __refill_cfs_bandwidth_runtime()
5724 		cfs_b->burst_time += runtime;  in __refill_cfs_bandwidth_runtime()
5725 		cfs_b->nr_burst++;  in __refill_cfs_bandwidth_runtime()
5728 	cfs_b->runtime = min(cfs_b->runtime, cfs_b->quota + cfs_b->burst);  in __refill_cfs_bandwidth_runtime()
5729 	cfs_b->runtime_snap = cfs_b->runtime;  in __refill_cfs_bandwidth_runtime()
5738 static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b,  in __assign_cfs_rq_runtime()  argument
5743 	lockdep_assert_held(&cfs_b->lock);  in __assign_cfs_rq_runtime()
5748 	if (cfs_b->quota == RUNTIME_INF)  in __assign_cfs_rq_runtime()
5751 		start_cfs_bandwidth(cfs_b);  in __assign_cfs_rq_runtime()
5753 		if (cfs_b->runtime > 0) {  in __assign_cfs_rq_runtime()
5754 			amount = min(cfs_b->runtime, min_amount);  in __assign_cfs_rq_runtime()
5755 			cfs_b->runtime -= amount;  in __assign_cfs_rq_runtime()
5756 			cfs_b->idle = 0;  in __assign_cfs_rq_runtime()
5768 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in assign_cfs_rq_runtime()  local
5771 	raw_spin_lock(&cfs_b->lock);  in assign_cfs_rq_runtime()
5772 	ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice());  in assign_cfs_rq_runtime()
5773 	raw_spin_unlock(&cfs_b->lock);  in assign_cfs_rq_runtime()
5884 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in throttle_cfs_rq()  local
5889 	raw_spin_lock(&cfs_b->lock);  in throttle_cfs_rq()
5891 	if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) {  in throttle_cfs_rq()
5903 				  &cfs_b->throttled_cfs_rq);  in throttle_cfs_rq()
5905 	raw_spin_unlock(&cfs_b->lock);  in throttle_cfs_rq()
5990 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in unthrottle_cfs_rq()  local
6001 	raw_spin_lock(&cfs_b->lock);  in unthrottle_cfs_rq()
6003 		cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock;  in unthrottle_cfs_rq()
6007 	raw_spin_unlock(&cfs_b->lock);  in unthrottle_cfs_rq()
6163 static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b)  in distribute_cfs_runtime()  argument
6174 	list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,  in distribute_cfs_runtime()
6194 		raw_spin_lock(&cfs_b->lock);  in distribute_cfs_runtime()
6196 		if (runtime > cfs_b->runtime)  in distribute_cfs_runtime()
6197 			runtime = cfs_b->runtime;  in distribute_cfs_runtime()
6198 		cfs_b->runtime -= runtime;  in distribute_cfs_runtime()
6199 		remaining = cfs_b->runtime;  in distribute_cfs_runtime()
6200 		raw_spin_unlock(&cfs_b->lock);  in distribute_cfs_runtime()
6251 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags)  in do_sched_cfs_period_timer()  argument
6256 	if (cfs_b->quota == RUNTIME_INF)  in do_sched_cfs_period_timer()
6259 	throttled = !list_empty(&cfs_b->throttled_cfs_rq);  in do_sched_cfs_period_timer()
6260 	cfs_b->nr_periods += overrun;  in do_sched_cfs_period_timer()
6263 	__refill_cfs_bandwidth_runtime(cfs_b);  in do_sched_cfs_period_timer()
6269 	if (cfs_b->idle && !throttled)  in do_sched_cfs_period_timer()
6274 		cfs_b->idle = 1;  in do_sched_cfs_period_timer()
6279 	cfs_b->nr_throttled += overrun;  in do_sched_cfs_period_timer()
6284 	while (throttled && cfs_b->runtime > 0) {  in do_sched_cfs_period_timer()
6285 		raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in do_sched_cfs_period_timer()
6287 		throttled = distribute_cfs_runtime(cfs_b);  in do_sched_cfs_period_timer()
6288 		raw_spin_lock_irqsave(&cfs_b->lock, flags);  in do_sched_cfs_period_timer()
6297 	cfs_b->idle = 0;  in do_sched_cfs_period_timer()
6319 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire)  in runtime_refresh_within()  argument
6321 	struct hrtimer *refresh_timer = &cfs_b->period_timer;  in runtime_refresh_within()
6336 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)  in start_cfs_slack_bandwidth()  argument
6341 	if (runtime_refresh_within(cfs_b, min_left))  in start_cfs_slack_bandwidth()
6345 	if (cfs_b->slack_started)  in start_cfs_slack_bandwidth()
6347 	cfs_b->slack_started = true;  in start_cfs_slack_bandwidth()
6349 	hrtimer_start(&cfs_b->slack_timer,  in start_cfs_slack_bandwidth()
6357 	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);  in __return_cfs_rq_runtime()  local
6363 	raw_spin_lock(&cfs_b->lock);  in __return_cfs_rq_runtime()
6364 	if (cfs_b->quota != RUNTIME_INF) {  in __return_cfs_rq_runtime()
6365 		cfs_b->runtime += slack_runtime;  in __return_cfs_rq_runtime()
6368 		if (cfs_b->runtime > sched_cfs_bandwidth_slice() &&  in __return_cfs_rq_runtime()
6369 		    !list_empty(&cfs_b->throttled_cfs_rq))  in __return_cfs_rq_runtime()
6370 			start_cfs_slack_bandwidth(cfs_b);  in __return_cfs_rq_runtime()
6372 	raw_spin_unlock(&cfs_b->lock);  in __return_cfs_rq_runtime()
6393 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)  in do_sched_cfs_slack_timer()  argument
6399 	raw_spin_lock_irqsave(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
6400 	cfs_b->slack_started = false;  in do_sched_cfs_slack_timer()
6402 	if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) {  in do_sched_cfs_slack_timer()
6403 		raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
6407 	if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice)  in do_sched_cfs_slack_timer()
6408 		runtime = cfs_b->runtime;  in do_sched_cfs_slack_timer()
6410 	raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in do_sched_cfs_slack_timer()
6415 	distribute_cfs_runtime(cfs_b);  in do_sched_cfs_slack_timer()
6480 	struct cfs_bandwidth *cfs_b =  in sched_cfs_slack_timer()  local
6483 	do_sched_cfs_slack_timer(cfs_b);  in sched_cfs_slack_timer()
6492 	struct cfs_bandwidth *cfs_b =  in sched_cfs_period_timer()  local
6499 	raw_spin_lock_irqsave(&cfs_b->lock, flags);  in sched_cfs_period_timer()
6501 		overrun = hrtimer_forward_now(timer, cfs_b->period);  in sched_cfs_period_timer()
6505 		idle = do_sched_cfs_period_timer(cfs_b, overrun, flags);  in sched_cfs_period_timer()
6508 			u64 new, old = ktime_to_ns(cfs_b->period);  in sched_cfs_period_timer()
6517 				cfs_b->period = ns_to_ktime(new);  in sched_cfs_period_timer()
6518 				cfs_b->quota *= 2;  in sched_cfs_period_timer()
6519 				cfs_b->burst *= 2;  in sched_cfs_period_timer()
6525 					div_u64(cfs_b->quota, NSEC_PER_USEC));  in sched_cfs_period_timer()
6531 					div_u64(cfs_b->quota, NSEC_PER_USEC));  in sched_cfs_period_timer()
6539 		cfs_b->period_active = 0;  in sched_cfs_period_timer()
6540 	raw_spin_unlock_irqrestore(&cfs_b->lock, flags);  in sched_cfs_period_timer()
6545 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent)  in init_cfs_bandwidth()  argument
6547 	raw_spin_lock_init(&cfs_b->lock);  in init_cfs_bandwidth()
6548 	cfs_b->runtime = 0;  in init_cfs_bandwidth()
6549 	cfs_b->quota = RUNTIME_INF;  in init_cfs_bandwidth()
6550 	cfs_b->period = ns_to_ktime(default_cfs_period());  in init_cfs_bandwidth()
6551 	cfs_b->burst = 0;  in init_cfs_bandwidth()
6552 	cfs_b->hierarchical_quota = parent ? parent->hierarchical_quota : RUNTIME_INF;  in init_cfs_bandwidth()
6554 	INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq);  in init_cfs_bandwidth()
6555 	hrtimer_setup(&cfs_b->period_timer, sched_cfs_period_timer, CLOCK_MONOTONIC,  in init_cfs_bandwidth()
6559 	hrtimer_set_expires(&cfs_b->period_timer,  in init_cfs_bandwidth()
6560 			    get_random_u32_below(cfs_b->period));  in init_cfs_bandwidth()
6561 	hrtimer_setup(&cfs_b->slack_timer, sched_cfs_slack_timer, CLOCK_MONOTONIC,  in init_cfs_bandwidth()
6563 	cfs_b->slack_started = false;  in init_cfs_bandwidth()
6573 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)  in start_cfs_bandwidth()  argument
6575 	lockdep_assert_held(&cfs_b->lock);  in start_cfs_bandwidth()
6577 	if (cfs_b->period_active)  in start_cfs_bandwidth()
6580 	cfs_b->period_active = 1;  in start_cfs_bandwidth()
6581 	hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period);  in start_cfs_bandwidth()
6582 	hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED);  in start_cfs_bandwidth()
6585 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)  in destroy_cfs_bandwidth()  argument
6590 	if (!cfs_b->throttled_cfs_rq.next)  in destroy_cfs_bandwidth()
6593 	hrtimer_cancel(&cfs_b->period_timer);  in destroy_cfs_bandwidth()
6594 	hrtimer_cancel(&cfs_b->slack_timer);  in destroy_cfs_bandwidth()
6637 		struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;  in update_runtime_enabled()  local
6640 		raw_spin_lock(&cfs_b->lock);  in update_runtime_enabled()
6641 		cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;  in update_runtime_enabled()
6642 		raw_spin_unlock(&cfs_b->lock);  in update_runtime_enabled()
6758 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) {}  in init_cfs_bandwidth()  argument
6766 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}  in destroy_cfs_bandwidth()  argument