15 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu);
17 static struct cgroup_rstat_cpu *cgroup_rstat_cpu(struct cgroup *cgrp, int cpu)  in cgroup_rstat_cpu()  argument
19 	return per_cpu_ptr(cgrp->rstat_cpu, cpu);  in cgroup_rstat_cpu()
32 				     struct cgroup *cgrp, const bool fast_path)  in _cgroup_rstat_cpu_lock()  argument
48 			trace_cgroup_rstat_cpu_lock_contended_fastpath(cgrp, cpu, contended);  in _cgroup_rstat_cpu_lock()
50 			trace_cgroup_rstat_cpu_lock_contended(cgrp, cpu, contended);  in _cgroup_rstat_cpu_lock()
56 		trace_cgroup_rstat_cpu_locked_fastpath(cgrp, cpu, contended);  in _cgroup_rstat_cpu_lock()
58 		trace_cgroup_rstat_cpu_locked(cgrp, cpu, contended);  in _cgroup_rstat_cpu_lock()
65 			      struct cgroup *cgrp, unsigned long flags,  in _cgroup_rstat_cpu_unlock()  argument
69 		trace_cgroup_rstat_cpu_unlock_fastpath(cgrp, cpu, false);  in _cgroup_rstat_cpu_unlock()
71 		trace_cgroup_rstat_cpu_unlock(cgrp, cpu, false);  in _cgroup_rstat_cpu_unlock()
85 __bpf_kfunc void cgroup_rstat_updated(struct cgroup *cgrp, int cpu)  in cgroup_rstat_updated()  argument
98 	if (data_race(cgroup_rstat_cpu(cgrp, cpu)->updated_next))  in cgroup_rstat_updated()
101 	flags = _cgroup_rstat_cpu_lock(cpu_lock, cpu, cgrp, true);  in cgroup_rstat_updated()
105 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_updated()
106 		struct cgroup *parent = cgroup_parent(cgrp);  in cgroup_rstat_updated()
118 			rstatc->updated_next = cgrp;  in cgroup_rstat_updated()
124 		prstatc->updated_children = cgrp;  in cgroup_rstat_updated()
126 		cgrp = parent;  in cgroup_rstat_updated()
129 	_cgroup_rstat_cpu_unlock(cpu_lock, cpu, cgrp, flags, true);  in cgroup_rstat_updated()
266 __weak noinline void bpf_rstat_flush(struct cgroup *cgrp,  in bpf_rstat_flush()  argument
282 static inline void __cgroup_rstat_lock(struct cgroup *cgrp, int cpu_in_loop)  in __cgroup_rstat_lock()  argument
289 		trace_cgroup_rstat_lock_contended(cgrp, cpu_in_loop, contended);  in __cgroup_rstat_lock()
292 	trace_cgroup_rstat_locked(cgrp, cpu_in_loop, contended);  in __cgroup_rstat_lock()
295 static inline void __cgroup_rstat_unlock(struct cgroup *cgrp, int cpu_in_loop)  in __cgroup_rstat_unlock()  argument
298 	trace_cgroup_rstat_unlock(cgrp, cpu_in_loop, false);  in __cgroup_rstat_unlock()
315 __bpf_kfunc void cgroup_rstat_flush(struct cgroup *cgrp)  in cgroup_rstat_flush()  argument
324 		__cgroup_rstat_lock(cgrp, cpu);  in cgroup_rstat_flush()
325 		pos = cgroup_rstat_updated_list(cgrp, cpu);  in cgroup_rstat_flush()
338 		__cgroup_rstat_unlock(cgrp, cpu);  in cgroup_rstat_flush()
344 int cgroup_rstat_init(struct cgroup *cgrp)  in cgroup_rstat_init()  argument
349 	if (!cgrp->rstat_cpu) {  in cgroup_rstat_init()
350 		cgrp->rstat_cpu = alloc_percpu(struct cgroup_rstat_cpu);  in cgroup_rstat_init()
351 		if (!cgrp->rstat_cpu)  in cgroup_rstat_init()
357 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_init()
359 		rstatc->updated_children = cgrp;  in cgroup_rstat_init()
366 void cgroup_rstat_exit(struct cgroup *cgrp)  in cgroup_rstat_exit()  argument
370 	cgroup_rstat_flush(cgrp);  in cgroup_rstat_exit()
374 		struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_rstat_exit()
376 		if (WARN_ON_ONCE(rstatc->updated_children != cgrp) ||  in cgroup_rstat_exit()
381 	free_percpu(cgrp->rstat_cpu);  in cgroup_rstat_exit()
382 	cgrp->rstat_cpu = NULL;  in cgroup_rstat_exit()
421 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)  in cgroup_base_stat_flush()  argument
423 	struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);  in cgroup_base_stat_flush()
424 	struct cgroup *parent = cgroup_parent(cgrp);  in cgroup_base_stat_flush()
441 	cgroup_base_stat_add(&cgrp->bstat, &delta);  in cgroup_base_stat_flush()
447 		delta = cgrp->bstat;  in cgroup_base_stat_flush()
448 		cgroup_base_stat_sub(&delta, &cgrp->last_bstat);  in cgroup_base_stat_flush()
450 		cgroup_base_stat_add(&cgrp->last_bstat, &delta);  in cgroup_base_stat_flush()
461 cgroup_base_stat_cputime_account_begin(struct cgroup *cgrp, unsigned long *flags)  in cgroup_base_stat_cputime_account_begin()  argument
465 	rstatc = get_cpu_ptr(cgrp->rstat_cpu);  in cgroup_base_stat_cputime_account_begin()
470 static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp,  in cgroup_base_stat_cputime_account_end()  argument
475 	cgroup_rstat_updated(cgrp, smp_processor_id());  in cgroup_base_stat_cputime_account_end()
479 void __cgroup_account_cputime(struct cgroup *cgrp, u64 delta_exec)  in __cgroup_account_cputime()  argument
484 	rstatc = cgroup_base_stat_cputime_account_begin(cgrp, &flags);  in __cgroup_account_cputime()
486 	cgroup_base_stat_cputime_account_end(cgrp, rstatc, flags);  in __cgroup_account_cputime()
489 void __cgroup_account_cputime_field(struct cgroup *cgrp,  in __cgroup_account_cputime_field()  argument
495 	rstatc = cgroup_base_stat_cputime_account_begin(cgrp, &flags);  in __cgroup_account_cputime_field()
518 	cgroup_base_stat_cputime_account_end(cgrp, rstatc, flags);  in __cgroup_account_cputime_field()
573 	struct cgroup *cgrp = seq_css(seq)->cgroup;  in cgroup_base_stat_cputime_show()  local
576 	if (cgroup_parent(cgrp)) {  in cgroup_base_stat_cputime_show()
577 		cgroup_rstat_flush(cgrp);  in cgroup_base_stat_cputime_show()
578 		__cgroup_rstat_lock(cgrp, -1);  in cgroup_base_stat_cputime_show()
579 		bstat = cgrp->bstat;  in cgroup_base_stat_cputime_show()
580 		cputime_adjust(&cgrp->bstat.cputime, &cgrp->prev_cputime,  in cgroup_base_stat_cputime_show()
582 		__cgroup_rstat_unlock(cgrp, -1);  in cgroup_base_stat_cputime_show()