xref: /linux-6.15/include/linux/bpf-cgroup.h (revision b910eaaa)
1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */
230070984SDaniel Mack #ifndef _BPF_CGROUP_H
330070984SDaniel Mack #define _BPF_CGROUP_H
430070984SDaniel Mack 
58bad74f9SRoman Gushchin #include <linux/bpf.h>
6f292b87dSRoman Gushchin #include <linux/errno.h>
730070984SDaniel Mack #include <linux/jump_label.h>
8aa0ad5b0SRoman Gushchin #include <linux/percpu.h>
94bfc0bb2SRoman Gushchin #include <linux/percpu-refcount.h>
10de9cbbaaSRoman Gushchin #include <linux/rbtree.h>
1130070984SDaniel Mack #include <uapi/linux/bpf.h>
1230070984SDaniel Mack 
1330070984SDaniel Mack struct sock;
144fbac77dSAndrey Ignatov struct sockaddr;
1530070984SDaniel Mack struct cgroup;
1630070984SDaniel Mack struct sk_buff;
17de9cbbaaSRoman Gushchin struct bpf_map;
18de9cbbaaSRoman Gushchin struct bpf_prog;
1940304b2aSLawrence Brakmo struct bpf_sock_ops_kern;
20de9cbbaaSRoman Gushchin struct bpf_cgroup_storage;
217b146cebSAndrey Ignatov struct ctl_table;
227b146cebSAndrey Ignatov struct ctl_table_header;
23*b910eaaaSYonghong Song struct task_struct;
2430070984SDaniel Mack 
2530070984SDaniel Mack #ifdef CONFIG_CGROUP_BPF
2630070984SDaniel Mack 
27a9ed15daSStanislav Fomichev extern struct static_key_false cgroup_bpf_enabled_key[MAX_BPF_ATTACH_TYPE];
28a9ed15daSStanislav Fomichev #define cgroup_bpf_enabled(type) static_branch_unlikely(&cgroup_bpf_enabled_key[type])
2930070984SDaniel Mack 
30*b910eaaaSYonghong Song #define BPF_CGROUP_STORAGE_NEST_MAX	8
31*b910eaaaSYonghong Song 
32*b910eaaaSYonghong Song struct bpf_cgroup_storage_info {
33*b910eaaaSYonghong Song 	struct task_struct *task;
34*b910eaaaSYonghong Song 	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];
35*b910eaaaSYonghong Song };
36*b910eaaaSYonghong Song 
37*b910eaaaSYonghong Song /* For each cpu, permit maximum BPF_CGROUP_STORAGE_NEST_MAX number of tasks
38*b910eaaaSYonghong Song  * to use bpf cgroup storage simultaneously.
39*b910eaaaSYonghong Song  */
40*b910eaaaSYonghong Song DECLARE_PER_CPU(struct bpf_cgroup_storage_info,
41*b910eaaaSYonghong Song 		bpf_cgroup_storage_info[BPF_CGROUP_STORAGE_NEST_MAX]);
428bad74f9SRoman Gushchin 
438bad74f9SRoman Gushchin #define for_each_cgroup_storage_type(stype) \
448bad74f9SRoman Gushchin 	for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++)
45aa0ad5b0SRoman Gushchin 
46de9cbbaaSRoman Gushchin struct bpf_cgroup_storage_map;
47de9cbbaaSRoman Gushchin 
48de9cbbaaSRoman Gushchin struct bpf_storage_buffer {
49de9cbbaaSRoman Gushchin 	struct rcu_head rcu;
50d7f10df8SGustavo A. R. Silva 	char data[];
51de9cbbaaSRoman Gushchin };
52de9cbbaaSRoman Gushchin 
53de9cbbaaSRoman Gushchin struct bpf_cgroup_storage {
54b741f163SRoman Gushchin 	union {
55de9cbbaaSRoman Gushchin 		struct bpf_storage_buffer *buf;
56b741f163SRoman Gushchin 		void __percpu *percpu_buf;
57b741f163SRoman Gushchin 	};
58de9cbbaaSRoman Gushchin 	struct bpf_cgroup_storage_map *map;
59de9cbbaaSRoman Gushchin 	struct bpf_cgroup_storage_key key;
607d9c3427SYiFei Zhu 	struct list_head list_map;
617d9c3427SYiFei Zhu 	struct list_head list_cg;
62de9cbbaaSRoman Gushchin 	struct rb_node node;
63de9cbbaaSRoman Gushchin 	struct rcu_head rcu;
64de9cbbaaSRoman Gushchin };
65de9cbbaaSRoman Gushchin 
66af6eea57SAndrii Nakryiko struct bpf_cgroup_link {
67af6eea57SAndrii Nakryiko 	struct bpf_link link;
68af6eea57SAndrii Nakryiko 	struct cgroup *cgroup;
69af6eea57SAndrii Nakryiko 	enum bpf_attach_type type;
70af6eea57SAndrii Nakryiko };
71af6eea57SAndrii Nakryiko 
72324bda9eSAlexei Starovoitov struct bpf_prog_list {
73324bda9eSAlexei Starovoitov 	struct list_head node;
74324bda9eSAlexei Starovoitov 	struct bpf_prog *prog;
75af6eea57SAndrii Nakryiko 	struct bpf_cgroup_link *link;
768bad74f9SRoman Gushchin 	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];
77324bda9eSAlexei Starovoitov };
78324bda9eSAlexei Starovoitov 
79324bda9eSAlexei Starovoitov struct bpf_prog_array;
80324bda9eSAlexei Starovoitov 
8130070984SDaniel Mack struct cgroup_bpf {
82324bda9eSAlexei Starovoitov 	/* array of effective progs in this cgroup */
83324bda9eSAlexei Starovoitov 	struct bpf_prog_array __rcu *effective[MAX_BPF_ATTACH_TYPE];
84324bda9eSAlexei Starovoitov 
85324bda9eSAlexei Starovoitov 	/* attached progs to this cgroup and attach flags
86324bda9eSAlexei Starovoitov 	 * when flags == 0 or BPF_F_ALLOW_OVERRIDE the progs list will
87324bda9eSAlexei Starovoitov 	 * have either zero or one element
88324bda9eSAlexei Starovoitov 	 * when BPF_F_ALLOW_MULTI the list can have up to BPF_CGROUP_MAX_PROGS
8930070984SDaniel Mack 	 */
90324bda9eSAlexei Starovoitov 	struct list_head progs[MAX_BPF_ATTACH_TYPE];
91324bda9eSAlexei Starovoitov 	u32 flags[MAX_BPF_ATTACH_TYPE];
92324bda9eSAlexei Starovoitov 
937d9c3427SYiFei Zhu 	/* list of cgroup shared storages */
947d9c3427SYiFei Zhu 	struct list_head storages;
957d9c3427SYiFei Zhu 
96324bda9eSAlexei Starovoitov 	/* temp storage for effective prog array used by prog_attach/detach */
97dbcc1ba2SStanislav Fomichev 	struct bpf_prog_array *inactive;
984bfc0bb2SRoman Gushchin 
994bfc0bb2SRoman Gushchin 	/* reference counter used to detach bpf programs after cgroup removal */
1004bfc0bb2SRoman Gushchin 	struct percpu_ref refcnt;
1014bfc0bb2SRoman Gushchin 
1024bfc0bb2SRoman Gushchin 	/* cgroup_bpf is released using a work queue */
1034bfc0bb2SRoman Gushchin 	struct work_struct release_work;
10430070984SDaniel Mack };
10530070984SDaniel Mack 
106324bda9eSAlexei Starovoitov int cgroup_bpf_inherit(struct cgroup *cgrp);
1074bfc0bb2SRoman Gushchin void cgroup_bpf_offline(struct cgroup *cgrp);
10830070984SDaniel Mack 
109af6eea57SAndrii Nakryiko int __cgroup_bpf_attach(struct cgroup *cgrp,
110af6eea57SAndrii Nakryiko 			struct bpf_prog *prog, struct bpf_prog *replace_prog,
111af6eea57SAndrii Nakryiko 			struct bpf_cgroup_link *link,
112324bda9eSAlexei Starovoitov 			enum bpf_attach_type type, u32 flags);
113324bda9eSAlexei Starovoitov int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
114af6eea57SAndrii Nakryiko 			struct bpf_cgroup_link *link,
1151832f4efSValdis Kletnieks 			enum bpf_attach_type type);
116468e2f64SAlexei Starovoitov int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
117468e2f64SAlexei Starovoitov 		       union bpf_attr __user *uattr);
11830070984SDaniel Mack 
119324bda9eSAlexei Starovoitov /* Wrapper for __cgroup_bpf_*() protected by cgroup_mutex */
120af6eea57SAndrii Nakryiko int cgroup_bpf_attach(struct cgroup *cgrp,
121af6eea57SAndrii Nakryiko 		      struct bpf_prog *prog, struct bpf_prog *replace_prog,
122af6eea57SAndrii Nakryiko 		      struct bpf_cgroup_link *link, enum bpf_attach_type type,
1237dd68b32SAndrey Ignatov 		      u32 flags);
124324bda9eSAlexei Starovoitov int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
125af6eea57SAndrii Nakryiko 		      enum bpf_attach_type type);
126468e2f64SAlexei Starovoitov int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
127468e2f64SAlexei Starovoitov 		     union bpf_attr __user *uattr);
12830070984SDaniel Mack 
129b2cd1257SDavid Ahern int __cgroup_bpf_run_filter_skb(struct sock *sk,
13030070984SDaniel Mack 				struct sk_buff *skb,
13130070984SDaniel Mack 				enum bpf_attach_type type);
13230070984SDaniel Mack 
13361023658SDavid Ahern int __cgroup_bpf_run_filter_sk(struct sock *sk,
13461023658SDavid Ahern 			       enum bpf_attach_type type);
13561023658SDavid Ahern 
1364fbac77dSAndrey Ignatov int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,
1374fbac77dSAndrey Ignatov 				      struct sockaddr *uaddr,
1381cedee13SAndrey Ignatov 				      enum bpf_attach_type type,
13977241217SStanislav Fomichev 				      void *t_ctx,
14077241217SStanislav Fomichev 				      u32 *flags);
1414fbac77dSAndrey Ignatov 
14240304b2aSLawrence Brakmo int __cgroup_bpf_run_filter_sock_ops(struct sock *sk,
14340304b2aSLawrence Brakmo 				     struct bpf_sock_ops_kern *sock_ops,
14440304b2aSLawrence Brakmo 				     enum bpf_attach_type type);
14540304b2aSLawrence Brakmo 
146ebc614f6SRoman Gushchin int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor,
147ebc614f6SRoman Gushchin 				      short access, enum bpf_attach_type type);
148ebc614f6SRoman Gushchin 
1497b146cebSAndrey Ignatov int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head,
1507b146cebSAndrey Ignatov 				   struct ctl_table *table, int write,
1514bd6a735SMatthew Wilcox (Oracle) 				   char **buf, size_t *pcount, loff_t *ppos,
152e1550bfeSAndrey Ignatov 				   enum bpf_attach_type type);
1537b146cebSAndrey Ignatov 
1540d01da6aSStanislav Fomichev int __cgroup_bpf_run_filter_setsockopt(struct sock *sock, int *level,
1550d01da6aSStanislav Fomichev 				       int *optname, char __user *optval,
1560d01da6aSStanislav Fomichev 				       int *optlen, char **kernel_optval);
1570d01da6aSStanislav Fomichev int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level,
1580d01da6aSStanislav Fomichev 				       int optname, char __user *optval,
1590d01da6aSStanislav Fomichev 				       int __user *optlen, int max_optlen,
1600d01da6aSStanislav Fomichev 				       int retval);
1610d01da6aSStanislav Fomichev 
1629cacf81fSStanislav Fomichev int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level,
1639cacf81fSStanislav Fomichev 					    int optname, void *optval,
1649cacf81fSStanislav Fomichev 					    int *optlen, int retval);
1659cacf81fSStanislav Fomichev 
1668bad74f9SRoman Gushchin static inline enum bpf_cgroup_storage_type cgroup_storage_type(
1678bad74f9SRoman Gushchin 	struct bpf_map *map)
168aa0ad5b0SRoman Gushchin {
169b741f163SRoman Gushchin 	if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
170b741f163SRoman Gushchin 		return BPF_CGROUP_STORAGE_PERCPU;
171b741f163SRoman Gushchin 
1728bad74f9SRoman Gushchin 	return BPF_CGROUP_STORAGE_SHARED;
173aa0ad5b0SRoman Gushchin }
174aa0ad5b0SRoman Gushchin 
175*b910eaaaSYonghong Song static inline int bpf_cgroup_storage_set(struct bpf_cgroup_storage
1768bad74f9SRoman Gushchin 					 *storage[MAX_BPF_CGROUP_STORAGE_TYPE])
1778bad74f9SRoman Gushchin {
1788bad74f9SRoman Gushchin 	enum bpf_cgroup_storage_type stype;
179*b910eaaaSYonghong Song 	int i, err = 0;
1808bad74f9SRoman Gushchin 
181*b910eaaaSYonghong Song 	preempt_disable();
182*b910eaaaSYonghong Song 	for (i = 0; i < BPF_CGROUP_STORAGE_NEST_MAX; i++) {
183*b910eaaaSYonghong Song 		if (unlikely(this_cpu_read(bpf_cgroup_storage_info[i].task) != NULL))
184*b910eaaaSYonghong Song 			continue;
185*b910eaaaSYonghong Song 
186*b910eaaaSYonghong Song 		this_cpu_write(bpf_cgroup_storage_info[i].task, current);
187f294b37eSRoman Gushchin 		for_each_cgroup_storage_type(stype)
188*b910eaaaSYonghong Song 			this_cpu_write(bpf_cgroup_storage_info[i].storage[stype],
189*b910eaaaSYonghong Song 				       storage[stype]);
190*b910eaaaSYonghong Song 		goto out;
191*b910eaaaSYonghong Song 	}
192*b910eaaaSYonghong Song 	err = -EBUSY;
193*b910eaaaSYonghong Song 	WARN_ON_ONCE(1);
194*b910eaaaSYonghong Song 
195*b910eaaaSYonghong Song out:
196*b910eaaaSYonghong Song 	preempt_enable();
197*b910eaaaSYonghong Song 	return err;
198*b910eaaaSYonghong Song }
199*b910eaaaSYonghong Song 
200*b910eaaaSYonghong Song static inline void bpf_cgroup_storage_unset(void)
201*b910eaaaSYonghong Song {
202*b910eaaaSYonghong Song 	int i;
203*b910eaaaSYonghong Song 
204*b910eaaaSYonghong Song 	for (i = 0; i < BPF_CGROUP_STORAGE_NEST_MAX; i++) {
205*b910eaaaSYonghong Song 		if (unlikely(this_cpu_read(bpf_cgroup_storage_info[i].task) != current))
206*b910eaaaSYonghong Song 			continue;
207*b910eaaaSYonghong Song 
208*b910eaaaSYonghong Song 		this_cpu_write(bpf_cgroup_storage_info[i].task, NULL);
209*b910eaaaSYonghong Song 		return;
210*b910eaaaSYonghong Song 	}
2118bad74f9SRoman Gushchin }
2128bad74f9SRoman Gushchin 
2137d9c3427SYiFei Zhu struct bpf_cgroup_storage *
2147d9c3427SYiFei Zhu cgroup_storage_lookup(struct bpf_cgroup_storage_map *map,
2157d9c3427SYiFei Zhu 		      void *key, bool locked);
2168bad74f9SRoman Gushchin struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(struct bpf_prog *prog,
2178bad74f9SRoman Gushchin 					enum bpf_cgroup_storage_type stype);
218de9cbbaaSRoman Gushchin void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage);
219de9cbbaaSRoman Gushchin void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage,
220de9cbbaaSRoman Gushchin 			     struct cgroup *cgroup,
221de9cbbaaSRoman Gushchin 			     enum bpf_attach_type type);
222de9cbbaaSRoman Gushchin void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage);
223e4730423SDaniel Borkmann int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux, struct bpf_map *map);
224de9cbbaaSRoman Gushchin 
225b741f163SRoman Gushchin int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key, void *value);
226b741f163SRoman Gushchin int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,
227b741f163SRoman Gushchin 				     void *value, u64 flags);
228b741f163SRoman Gushchin 
229b2cd1257SDavid Ahern /* Wrappers for __cgroup_bpf_run_filter_skb() guarded by cgroup_bpf_enabled. */
23030070984SDaniel Mack #define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb)			      \
23130070984SDaniel Mack ({									      \
23230070984SDaniel Mack 	int __ret = 0;							      \
233a9ed15daSStanislav Fomichev 	if (cgroup_bpf_enabled(BPF_CGROUP_INET_INGRESS))		      \
234b2cd1257SDavid Ahern 		__ret = __cgroup_bpf_run_filter_skb(sk, skb,		      \
23530070984SDaniel Mack 						    BPF_CGROUP_INET_INGRESS); \
23630070984SDaniel Mack 									      \
23730070984SDaniel Mack 	__ret;								      \
23830070984SDaniel Mack })
23930070984SDaniel Mack 
24030070984SDaniel Mack #define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb)			       \
24130070984SDaniel Mack ({									       \
24230070984SDaniel Mack 	int __ret = 0;							       \
243a9ed15daSStanislav Fomichev 	if (cgroup_bpf_enabled(BPF_CGROUP_INET_EGRESS) && sk && sk == skb->sk) { \
24430070984SDaniel Mack 		typeof(sk) __sk = sk_to_full_sk(sk);			       \
24530070984SDaniel Mack 		if (sk_fullsock(__sk))					       \
246b2cd1257SDavid Ahern 			__ret = __cgroup_bpf_run_filter_skb(__sk, skb,	       \
24730070984SDaniel Mack 						      BPF_CGROUP_INET_EGRESS); \
24830070984SDaniel Mack 	}								       \
24930070984SDaniel Mack 	__ret;								       \
25030070984SDaniel Mack })
25130070984SDaniel Mack 
252aac3fc32SAndrey Ignatov #define BPF_CGROUP_RUN_SK_PROG(sk, type)				       \
25361023658SDavid Ahern ({									       \
25461023658SDavid Ahern 	int __ret = 0;							       \
255a9ed15daSStanislav Fomichev 	if (cgroup_bpf_enabled(type)) {					       \
256aac3fc32SAndrey Ignatov 		__ret = __cgroup_bpf_run_filter_sk(sk, type);		       \
25761023658SDavid Ahern 	}								       \
25861023658SDavid Ahern 	__ret;								       \
25961023658SDavid Ahern })
26061023658SDavid Ahern 
261aac3fc32SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET_SOCK(sk)				       \
262aac3fc32SAndrey Ignatov 	BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET_SOCK_CREATE)
263aac3fc32SAndrey Ignatov 
264f5836749SStanislav Fomichev #define BPF_CGROUP_RUN_PROG_INET_SOCK_RELEASE(sk)			       \
265f5836749SStanislav Fomichev 	BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET_SOCK_RELEASE)
266f5836749SStanislav Fomichev 
267aac3fc32SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk)				       \
268aac3fc32SAndrey Ignatov 	BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET4_POST_BIND)
269aac3fc32SAndrey Ignatov 
270aac3fc32SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk)				       \
271aac3fc32SAndrey Ignatov 	BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET6_POST_BIND)
272aac3fc32SAndrey Ignatov 
2734fbac77dSAndrey Ignatov #define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, type)				       \
2744fbac77dSAndrey Ignatov ({									       \
27577241217SStanislav Fomichev 	u32 __unused_flags;						       \
2764fbac77dSAndrey Ignatov 	int __ret = 0;							       \
277a9ed15daSStanislav Fomichev 	if (cgroup_bpf_enabled(type))					       \
2781cedee13SAndrey Ignatov 		__ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type,     \
27977241217SStanislav Fomichev 							  NULL,		       \
28077241217SStanislav Fomichev 							  &__unused_flags);    \
2814fbac77dSAndrey Ignatov 	__ret;								       \
2824fbac77dSAndrey Ignatov })
2834fbac77dSAndrey Ignatov 
2841cedee13SAndrey Ignatov #define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx)		       \
285d74bad4eSAndrey Ignatov ({									       \
28677241217SStanislav Fomichev 	u32 __unused_flags;						       \
287d74bad4eSAndrey Ignatov 	int __ret = 0;							       \
288a9ed15daSStanislav Fomichev 	if (cgroup_bpf_enabled(type))	{				       \
289d74bad4eSAndrey Ignatov 		lock_sock(sk);						       \
2901cedee13SAndrey Ignatov 		__ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type,     \
29177241217SStanislav Fomichev 							  t_ctx,	       \
29277241217SStanislav Fomichev 							  &__unused_flags);    \
293d74bad4eSAndrey Ignatov 		release_sock(sk);					       \
294d74bad4eSAndrey Ignatov 	}								       \
295d74bad4eSAndrey Ignatov 	__ret;								       \
296d74bad4eSAndrey Ignatov })
297d74bad4eSAndrey Ignatov 
29877241217SStanislav Fomichev /* BPF_CGROUP_INET4_BIND and BPF_CGROUP_INET6_BIND can return extra flags
29977241217SStanislav Fomichev  * via upper bits of return code. The only flag that is supported
30077241217SStanislav Fomichev  * (at bit position 0) is to indicate CAP_NET_BIND_SERVICE capability check
30177241217SStanislav Fomichev  * should be bypassed (BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE).
30277241217SStanislav Fomichev  */
30377241217SStanislav Fomichev #define BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr, type, bind_flags)	       \
30477241217SStanislav Fomichev ({									       \
30577241217SStanislav Fomichev 	u32 __flags = 0;						       \
30677241217SStanislav Fomichev 	int __ret = 0;							       \
30777241217SStanislav Fomichev 	if (cgroup_bpf_enabled(type))	{				       \
30877241217SStanislav Fomichev 		lock_sock(sk);						       \
30977241217SStanislav Fomichev 		__ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type,     \
31077241217SStanislav Fomichev 							  NULL, &__flags);     \
31177241217SStanislav Fomichev 		release_sock(sk);					       \
31277241217SStanislav Fomichev 		if (__flags & BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE)	       \
31377241217SStanislav Fomichev 			*bind_flags |= BIND_NO_CAP_NET_BIND_SERVICE;	       \
31477241217SStanislav Fomichev 	}								       \
31577241217SStanislav Fomichev 	__ret;								       \
31677241217SStanislav Fomichev })
3174fbac77dSAndrey Ignatov 
318a9ed15daSStanislav Fomichev #define BPF_CGROUP_PRE_CONNECT_ENABLED(sk)				       \
319a9ed15daSStanislav Fomichev 	((cgroup_bpf_enabled(BPF_CGROUP_INET4_CONNECT) ||		       \
320a9ed15daSStanislav Fomichev 	  cgroup_bpf_enabled(BPF_CGROUP_INET6_CONNECT)) &&		       \
321a9ed15daSStanislav Fomichev 	 (sk)->sk_prot->pre_connect)
322d74bad4eSAndrey Ignatov 
323d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr)			       \
324d74bad4eSAndrey Ignatov 	BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET4_CONNECT)
325d74bad4eSAndrey Ignatov 
326d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr)			       \
327d74bad4eSAndrey Ignatov 	BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET6_CONNECT)
328d74bad4eSAndrey Ignatov 
329d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr)		       \
3301cedee13SAndrey Ignatov 	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET4_CONNECT, NULL)
331d74bad4eSAndrey Ignatov 
332d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr)		       \
3331cedee13SAndrey Ignatov 	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET6_CONNECT, NULL)
3341cedee13SAndrey Ignatov 
3351cedee13SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, t_ctx)		       \
3361cedee13SAndrey Ignatov 	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP4_SENDMSG, t_ctx)
3371cedee13SAndrey Ignatov 
3381cedee13SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, t_ctx)		       \
3391cedee13SAndrey Ignatov 	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP6_SENDMSG, t_ctx)
340d74bad4eSAndrey Ignatov 
341983695faSDaniel Borkmann #define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr)			\
342983695faSDaniel Borkmann 	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP4_RECVMSG, NULL)
343983695faSDaniel Borkmann 
344983695faSDaniel Borkmann #define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr)			\
345983695faSDaniel Borkmann 	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP6_RECVMSG, NULL)
346983695faSDaniel Borkmann 
3470813a841SMartin KaFai Lau /* The SOCK_OPS"_SK" macro should be used when sock_ops->sk is not a
3480813a841SMartin KaFai Lau  * fullsock and its parent fullsock cannot be traced by
3490813a841SMartin KaFai Lau  * sk_to_full_sk().
3500813a841SMartin KaFai Lau  *
3510813a841SMartin KaFai Lau  * e.g. sock_ops->sk is a request_sock and it is under syncookie mode.
3520813a841SMartin KaFai Lau  * Its listener-sk is not attached to the rsk_listener.
3530813a841SMartin KaFai Lau  * In this case, the caller holds the listener-sk (unlocked),
3540813a841SMartin KaFai Lau  * set its sock_ops->sk to req_sk, and call this SOCK_OPS"_SK" with
3550813a841SMartin KaFai Lau  * the listener-sk such that the cgroup-bpf-progs of the
3560813a841SMartin KaFai Lau  * listener-sk will be run.
3570813a841SMartin KaFai Lau  *
3580813a841SMartin KaFai Lau  * Regardless of syncookie mode or not,
3590813a841SMartin KaFai Lau  * calling bpf_setsockopt on listener-sk will not make sense anyway,
3600813a841SMartin KaFai Lau  * so passing 'sock_ops->sk == req_sk' to the bpf prog is appropriate here.
3610813a841SMartin KaFai Lau  */
3620813a841SMartin KaFai Lau #define BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk)			\
3630813a841SMartin KaFai Lau ({									\
3640813a841SMartin KaFai Lau 	int __ret = 0;							\
365a9ed15daSStanislav Fomichev 	if (cgroup_bpf_enabled(BPF_CGROUP_SOCK_OPS))			\
3660813a841SMartin KaFai Lau 		__ret = __cgroup_bpf_run_filter_sock_ops(sk,		\
3670813a841SMartin KaFai Lau 							 sock_ops,	\
3680813a841SMartin KaFai Lau 							 BPF_CGROUP_SOCK_OPS); \
3690813a841SMartin KaFai Lau 	__ret;								\
3700813a841SMartin KaFai Lau })
3710813a841SMartin KaFai Lau 
37240304b2aSLawrence Brakmo #define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops)				       \
37340304b2aSLawrence Brakmo ({									       \
37440304b2aSLawrence Brakmo 	int __ret = 0;							       \
375a9ed15daSStanislav Fomichev 	if (cgroup_bpf_enabled(BPF_CGROUP_SOCK_OPS) && (sock_ops)->sk) {       \
37640304b2aSLawrence Brakmo 		typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk);	       \
377df39a9f1SWANG Cong 		if (__sk && sk_fullsock(__sk))				       \
37840304b2aSLawrence Brakmo 			__ret = __cgroup_bpf_run_filter_sock_ops(__sk,	       \
37940304b2aSLawrence Brakmo 								 sock_ops,     \
38040304b2aSLawrence Brakmo 							 BPF_CGROUP_SOCK_OPS); \
38140304b2aSLawrence Brakmo 	}								       \
38240304b2aSLawrence Brakmo 	__ret;								       \
38340304b2aSLawrence Brakmo })
384ebc614f6SRoman Gushchin 
385ebc614f6SRoman Gushchin #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type, major, minor, access)	      \
386ebc614f6SRoman Gushchin ({									      \
387ebc614f6SRoman Gushchin 	int __ret = 0;							      \
388a9ed15daSStanislav Fomichev 	if (cgroup_bpf_enabled(BPF_CGROUP_DEVICE))			      \
389ebc614f6SRoman Gushchin 		__ret = __cgroup_bpf_check_dev_permission(type, major, minor, \
390ebc614f6SRoman Gushchin 							  access,	      \
391ebc614f6SRoman Gushchin 							  BPF_CGROUP_DEVICE); \
392ebc614f6SRoman Gushchin 									      \
393ebc614f6SRoman Gushchin 	__ret;								      \
394ebc614f6SRoman Gushchin })
3957b146cebSAndrey Ignatov 
3967b146cebSAndrey Ignatov 
39732927393SChristoph Hellwig #define BPF_CGROUP_RUN_PROG_SYSCTL(head, table, write, buf, count, pos)  \
3987b146cebSAndrey Ignatov ({									       \
3997b146cebSAndrey Ignatov 	int __ret = 0;							       \
400a9ed15daSStanislav Fomichev 	if (cgroup_bpf_enabled(BPF_CGROUP_SYSCTL))			       \
4017b146cebSAndrey Ignatov 		__ret = __cgroup_bpf_run_filter_sysctl(head, table, write,     \
40232927393SChristoph Hellwig 						       buf, count, pos,        \
4037b146cebSAndrey Ignatov 						       BPF_CGROUP_SYSCTL);     \
4047b146cebSAndrey Ignatov 	__ret;								       \
4057b146cebSAndrey Ignatov })
4067b146cebSAndrey Ignatov 
4070d01da6aSStanislav Fomichev #define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen,   \
4080d01da6aSStanislav Fomichev 				       kernel_optval)			       \
4090d01da6aSStanislav Fomichev ({									       \
4100d01da6aSStanislav Fomichev 	int __ret = 0;							       \
411a9ed15daSStanislav Fomichev 	if (cgroup_bpf_enabled(BPF_CGROUP_SETSOCKOPT))			       \
4120d01da6aSStanislav Fomichev 		__ret = __cgroup_bpf_run_filter_setsockopt(sock, level,	       \
4130d01da6aSStanislav Fomichev 							   optname, optval,    \
4140d01da6aSStanislav Fomichev 							   optlen,	       \
4150d01da6aSStanislav Fomichev 							   kernel_optval);     \
4160d01da6aSStanislav Fomichev 	__ret;								       \
4170d01da6aSStanislav Fomichev })
4180d01da6aSStanislav Fomichev 
4190d01da6aSStanislav Fomichev #define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen)			       \
4200d01da6aSStanislav Fomichev ({									       \
4210d01da6aSStanislav Fomichev 	int __ret = 0;							       \
422a9ed15daSStanislav Fomichev 	if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT))			       \
4230d01da6aSStanislav Fomichev 		get_user(__ret, optlen);				       \
4240d01da6aSStanislav Fomichev 	__ret;								       \
4250d01da6aSStanislav Fomichev })
4260d01da6aSStanislav Fomichev 
4270d01da6aSStanislav Fomichev #define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, optlen,   \
4280d01da6aSStanislav Fomichev 				       max_optlen, retval)		       \
4290d01da6aSStanislav Fomichev ({									       \
4300d01da6aSStanislav Fomichev 	int __ret = retval;						       \
431a9ed15daSStanislav Fomichev 	if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT))			       \
4329cacf81fSStanislav Fomichev 		if (!(sock)->sk_prot->bpf_bypass_getsockopt ||		       \
4339cacf81fSStanislav Fomichev 		    !INDIRECT_CALL_INET_1((sock)->sk_prot->bpf_bypass_getsockopt, \
4349cacf81fSStanislav Fomichev 					tcp_bpf_bypass_getsockopt,	       \
4359cacf81fSStanislav Fomichev 					level, optname))		       \
4369cacf81fSStanislav Fomichev 			__ret = __cgroup_bpf_run_filter_getsockopt(	       \
4379cacf81fSStanislav Fomichev 				sock, level, optname, optval, optlen,	       \
4389cacf81fSStanislav Fomichev 				max_optlen, retval);			       \
4399cacf81fSStanislav Fomichev 	__ret;								       \
4409cacf81fSStanislav Fomichev })
4419cacf81fSStanislav Fomichev 
4429cacf81fSStanislav Fomichev #define BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sock, level, optname, optval,      \
4439cacf81fSStanislav Fomichev 					    optlen, retval)		       \
4449cacf81fSStanislav Fomichev ({									       \
4459cacf81fSStanislav Fomichev 	int __ret = retval;						       \
446a9ed15daSStanislav Fomichev 	if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT))			       \
4479cacf81fSStanislav Fomichev 		__ret = __cgroup_bpf_run_filter_getsockopt_kern(	       \
4489cacf81fSStanislav Fomichev 			sock, level, optname, optval, optlen, retval);	       \
4490d01da6aSStanislav Fomichev 	__ret;								       \
4500d01da6aSStanislav Fomichev })
4510d01da6aSStanislav Fomichev 
452fdb5c453SSean Young int cgroup_bpf_prog_attach(const union bpf_attr *attr,
453fdb5c453SSean Young 			   enum bpf_prog_type ptype, struct bpf_prog *prog);
454fdb5c453SSean Young int cgroup_bpf_prog_detach(const union bpf_attr *attr,
455fdb5c453SSean Young 			   enum bpf_prog_type ptype);
456af6eea57SAndrii Nakryiko int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
457fdb5c453SSean Young int cgroup_bpf_prog_query(const union bpf_attr *attr,
458fdb5c453SSean Young 			  union bpf_attr __user *uattr);
45930070984SDaniel Mack #else
46030070984SDaniel Mack 
461fdb5c453SSean Young struct bpf_prog;
46230070984SDaniel Mack struct cgroup_bpf {};
463324bda9eSAlexei Starovoitov static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; }
4644bfc0bb2SRoman Gushchin static inline void cgroup_bpf_offline(struct cgroup *cgrp) {}
46530070984SDaniel Mack 
466fdb5c453SSean Young static inline int cgroup_bpf_prog_attach(const union bpf_attr *attr,
467fdb5c453SSean Young 					 enum bpf_prog_type ptype,
468fdb5c453SSean Young 					 struct bpf_prog *prog)
469fdb5c453SSean Young {
470fdb5c453SSean Young 	return -EINVAL;
471fdb5c453SSean Young }
472fdb5c453SSean Young 
473fdb5c453SSean Young static inline int cgroup_bpf_prog_detach(const union bpf_attr *attr,
474fdb5c453SSean Young 					 enum bpf_prog_type ptype)
475fdb5c453SSean Young {
476fdb5c453SSean Young 	return -EINVAL;
477fdb5c453SSean Young }
478fdb5c453SSean Young 
479af6eea57SAndrii Nakryiko static inline int cgroup_bpf_link_attach(const union bpf_attr *attr,
480af6eea57SAndrii Nakryiko 					 struct bpf_prog *prog)
481af6eea57SAndrii Nakryiko {
482af6eea57SAndrii Nakryiko 	return -EINVAL;
483af6eea57SAndrii Nakryiko }
484af6eea57SAndrii Nakryiko 
485fdb5c453SSean Young static inline int cgroup_bpf_prog_query(const union bpf_attr *attr,
486fdb5c453SSean Young 					union bpf_attr __user *uattr)
487fdb5c453SSean Young {
488fdb5c453SSean Young 	return -EINVAL;
489fdb5c453SSean Young }
490fdb5c453SSean Young 
491*b910eaaaSYonghong Song static inline int bpf_cgroup_storage_set(
492*b910eaaaSYonghong Song 	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) { return 0; }
493*b910eaaaSYonghong Song static inline void bpf_cgroup_storage_unset(void) {}
494e4730423SDaniel Borkmann static inline int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux,
495de9cbbaaSRoman Gushchin 					    struct bpf_map *map) { return 0; }
496de9cbbaaSRoman Gushchin static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(
49771b91a50SBo YU 	struct bpf_prog *prog, enum bpf_cgroup_storage_type stype) { return NULL; }
498de9cbbaaSRoman Gushchin static inline void bpf_cgroup_storage_free(
499de9cbbaaSRoman Gushchin 	struct bpf_cgroup_storage *storage) {}
500b741f163SRoman Gushchin static inline int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key,
501b741f163SRoman Gushchin 						 void *value) {
502b741f163SRoman Gushchin 	return 0;
503b741f163SRoman Gushchin }
504b741f163SRoman Gushchin static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map,
505b741f163SRoman Gushchin 					void *key, void *value, u64 flags) {
506b741f163SRoman Gushchin 	return 0;
507b741f163SRoman Gushchin }
508de9cbbaaSRoman Gushchin 
509a9ed15daSStanislav Fomichev #define cgroup_bpf_enabled(type) (0)
5101b66d253SDaniel Borkmann #define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx) ({ 0; })
511d74bad4eSAndrey Ignatov #define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (0)
51230070984SDaniel Mack #define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk,skb) ({ 0; })
51330070984SDaniel Mack #define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk,skb) ({ 0; })
51461023658SDavid Ahern #define BPF_CGROUP_RUN_PROG_INET_SOCK(sk) ({ 0; })
515f5836749SStanislav Fomichev #define BPF_CGROUP_RUN_PROG_INET_SOCK_RELEASE(sk) ({ 0; })
51677241217SStanislav Fomichev #define BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr, type, flags) ({ 0; })
517aac3fc32SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk) ({ 0; })
518aac3fc32SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk) ({ 0; })
519d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) ({ 0; })
520d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr) ({ 0; })
521d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr) ({ 0; })
522d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr) ({ 0; })
5231cedee13SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, t_ctx) ({ 0; })
5241cedee13SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, t_ctx) ({ 0; })
525983695faSDaniel Borkmann #define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr) ({ 0; })
526983695faSDaniel Borkmann #define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr) ({ 0; })
52740304b2aSLawrence Brakmo #define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; })
528ebc614f6SRoman Gushchin #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type,major,minor,access) ({ 0; })
52932927393SChristoph Hellwig #define BPF_CGROUP_RUN_PROG_SYSCTL(head,table,write,buf,count,pos) ({ 0; })
5300d01da6aSStanislav Fomichev #define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) ({ 0; })
5310d01da6aSStanislav Fomichev #define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \
5320d01da6aSStanislav Fomichev 				       optlen, max_optlen, retval) ({ retval; })
5339cacf81fSStanislav Fomichev #define BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sock, level, optname, optval, \
5349cacf81fSStanislav Fomichev 					    optlen, retval) ({ retval; })
5350d01da6aSStanislav Fomichev #define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \
5360d01da6aSStanislav Fomichev 				       kernel_optval) ({ 0; })
53730070984SDaniel Mack 
5388bad74f9SRoman Gushchin #define for_each_cgroup_storage_type(stype) for (; false; )
5398bad74f9SRoman Gushchin 
54030070984SDaniel Mack #endif /* CONFIG_CGROUP_BPF */
54130070984SDaniel Mack 
54230070984SDaniel Mack #endif /* _BPF_CGROUP_H */
543