49 		      enum cgroup_bpf_attach_type atype,  in bpf_prog_run_array_cg()  argument
63 	array = rcu_dereference(cgrp->effective[atype]);  in bpf_prog_run_array_cg()
300 	unsigned int atype;  in cgroup_bpf_release()  local
304 	for (atype = 0; atype < ARRAY_SIZE(cgrp->bpf.progs); atype++) {  in cgroup_bpf_release()
305 		struct hlist_head *progs = &cgrp->bpf.progs[atype];  in cgroup_bpf_release()
322 			static_branch_dec(&cgroup_bpf_enabled_key[atype]);  in cgroup_bpf_release()
325 				cgrp->bpf.effective[atype],  in cgroup_bpf_release()
392 				    enum cgroup_bpf_attach_type atype)  in hierarchy_allows_attach()  argument
400 		u32 flags = p->bpf.flags[atype];  in hierarchy_allows_attach()
405 		cnt = prog_list_length(&p->bpf.progs[atype], NULL);  in hierarchy_allows_attach()
421 				   enum cgroup_bpf_attach_type atype,  in compute_effective_progs()  argument
432 		if (cnt == 0 || (p->bpf.flags[atype] & BPF_F_ALLOW_MULTI))  in compute_effective_progs()
433 			cnt += prog_list_length(&p->bpf.progs[atype], &preorder_cnt);  in compute_effective_progs()
447 		if (cnt > 0 && !(p->bpf.flags[atype] & BPF_F_ALLOW_MULTI))  in compute_effective_progs()
451 		hlist_for_each_entry(pl, &p->bpf.progs[atype], node) {  in compute_effective_progs()
479 				     enum cgroup_bpf_attach_type atype,  in activate_effective_progs()  argument
482 	old_array = rcu_replace_pointer(cgrp->bpf.effective[atype], old_array,  in activate_effective_progs()
538 				  enum cgroup_bpf_attach_type atype)  in update_effective_progs()  argument
550 		err = compute_effective_progs(desc, atype, &desc->bpf.inactive);  in update_effective_progs()
567 		activate_effective_progs(desc, atype, desc->bpf.inactive);  in update_effective_progs()
650 	enum cgroup_bpf_attach_type atype;  in __cgroup_bpf_attach()  local
666 	atype = bpf_cgroup_atype_find(type, new_prog->aux->attach_btf_id);  in __cgroup_bpf_attach()
667 	if (atype < 0)  in __cgroup_bpf_attach()
670 	progs = &cgrp->bpf.progs[atype];  in __cgroup_bpf_attach()
672 	if (!hierarchy_allows_attach(cgrp, atype))  in __cgroup_bpf_attach()
675 	if (!hlist_empty(progs) && cgrp->bpf.flags[atype] != saved_flags)  in __cgroup_bpf_attach()
719 	cgrp->bpf.flags[atype] = saved_flags;  in __cgroup_bpf_attach()
722 		err = bpf_trampoline_link_cgroup_shim(new_prog, atype);  in __cgroup_bpf_attach()
727 	err = update_effective_progs(cgrp, atype);  in __cgroup_bpf_attach()
736 		static_branch_inc(&cgroup_bpf_enabled_key[atype]);  in __cgroup_bpf_attach()
776 				   enum cgroup_bpf_attach_type atype,  in replace_effective_prog()  argument
795 			if (pos && !(cg->bpf.flags[atype] & BPF_F_ALLOW_MULTI))  in replace_effective_prog()
798 			head = &cg->bpf.progs[atype];  in replace_effective_prog()
810 				desc->bpf.effective[atype],  in replace_effective_prog()
831 	enum cgroup_bpf_attach_type atype;  in __cgroup_bpf_replace()  local
837 	atype = bpf_cgroup_atype_find(link->type, new_prog->aux->attach_btf_id);  in __cgroup_bpf_replace()
838 	if (atype < 0)  in __cgroup_bpf_replace()
841 	progs = &cgrp->bpf.progs[atype];  in __cgroup_bpf_replace()
856 	replace_effective_prog(cgrp, atype, link);  in __cgroup_bpf_replace()
929 				  enum cgroup_bpf_attach_type atype)  in purge_effective_progs()  argument
947 			if (pos && !(cg->bpf.flags[atype] & BPF_F_ALLOW_MULTI))  in purge_effective_progs()
950 			head = &cg->bpf.progs[atype];  in purge_effective_progs()
964 				desc->bpf.effective[atype],  in purge_effective_progs()
987 	enum cgroup_bpf_attach_type atype;  in __cgroup_bpf_detach()  local
999 	atype = bpf_cgroup_atype_find(type, attach_btf_id);  in __cgroup_bpf_detach()
1000 	if (atype < 0)  in __cgroup_bpf_detach()
1003 	progs = &cgrp->bpf.progs[atype];  in __cgroup_bpf_detach()
1004 	flags = cgrp->bpf.flags[atype];  in __cgroup_bpf_detach()
1019 	if (update_effective_progs(cgrp, atype)) {  in __cgroup_bpf_detach()
1023 		purge_effective_progs(cgrp, old_prog, link, atype);  in __cgroup_bpf_detach()
1032 		cgrp->bpf.flags[atype] = 0;  in __cgroup_bpf_detach()
1038 	static_branch_dec(&cgroup_bpf_enabled_key[atype]);  in __cgroup_bpf_detach()
1062 	enum cgroup_bpf_attach_type atype;  in __cgroup_bpf_query()  local
1087 	for (atype = from_atype; atype <= to_atype; atype++) {  in __cgroup_bpf_query()
1089 			effective = rcu_dereference_protected(cgrp->bpf.effective[atype],  in __cgroup_bpf_query()
1093 			total_cnt += prog_list_length(&cgrp->bpf.progs[atype], NULL);  in __cgroup_bpf_query()
1112 	for (atype = from_atype; atype <= to_atype && total_cnt; atype++) {  in __cgroup_bpf_query()
1114 			effective = rcu_dereference_protected(cgrp->bpf.effective[atype],  in __cgroup_bpf_query()
1124 			progs = &cgrp->bpf.progs[atype];  in __cgroup_bpf_query()
1137 				flags = cgrp->bpf.flags[atype];  in __cgroup_bpf_query()
1393 				enum cgroup_bpf_attach_type atype)  in __cgroup_bpf_run_filter_skb()  argument
1412 	if (atype == CGROUP_INET_EGRESS) {  in __cgroup_bpf_run_filter_skb()
1416 		ret = bpf_prog_run_array_cg(&cgrp->bpf, atype, skb,  in __cgroup_bpf_run_filter_skb()
1442 		ret = bpf_prog_run_array_cg(&cgrp->bpf, atype,  in __cgroup_bpf_run_filter_skb()
1470 			       enum cgroup_bpf_attach_type atype)  in __cgroup_bpf_run_filter_sk()  argument
1474 	return bpf_prog_run_array_cg(&cgrp->bpf, atype, sk, bpf_prog_run, 0,  in __cgroup_bpf_run_filter_sk()
1500 				      enum cgroup_bpf_attach_type atype,  in __cgroup_bpf_run_filter_sock_addr()  argument
1529 	ret = bpf_prog_run_array_cg(&cgrp->bpf, atype, &ctx, bpf_prog_run,  in __cgroup_bpf_run_filter_sock_addr()
1557 				     enum cgroup_bpf_attach_type atype)  in __cgroup_bpf_run_filter_sock_ops()  argument
1561 	return bpf_prog_run_array_cg(&cgrp->bpf, atype, sock_ops, bpf_prog_run,  in __cgroup_bpf_run_filter_sock_ops()
1567 				      short access, enum cgroup_bpf_attach_type atype)  in __cgroup_bpf_check_dev_permission()  argument
1579 	ret = bpf_prog_run_array_cg(&cgrp->bpf, atype, &ctx, bpf_prog_run, 0,  in __cgroup_bpf_check_dev_permission()
1730 				   enum cgroup_bpf_attach_type atype)  in __cgroup_bpf_run_filter_sysctl()  argument
1770 	ret = bpf_prog_run_array_cg(&cgrp->bpf, atype, &ctx, bpf_prog_run, 0,  in __cgroup_bpf_run_filter_sysctl()