xref: /linux-6.15/include/linux/bpf-cgroup.h (revision 9dfc685e)
1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */
230070984SDaniel Mack #ifndef _BPF_CGROUP_H
330070984SDaniel Mack #define _BPF_CGROUP_H
430070984SDaniel Mack 
58bad74f9SRoman Gushchin #include <linux/bpf.h>
6f292b87dSRoman Gushchin #include <linux/errno.h>
730070984SDaniel Mack #include <linux/jump_label.h>
8aa0ad5b0SRoman Gushchin #include <linux/percpu.h>
94bfc0bb2SRoman Gushchin #include <linux/percpu-refcount.h>
10de9cbbaaSRoman Gushchin #include <linux/rbtree.h>
1130070984SDaniel Mack #include <uapi/linux/bpf.h>
1230070984SDaniel Mack 
1330070984SDaniel Mack struct sock;
144fbac77dSAndrey Ignatov struct sockaddr;
1530070984SDaniel Mack struct cgroup;
1630070984SDaniel Mack struct sk_buff;
17de9cbbaaSRoman Gushchin struct bpf_map;
18de9cbbaaSRoman Gushchin struct bpf_prog;
1940304b2aSLawrence Brakmo struct bpf_sock_ops_kern;
20de9cbbaaSRoman Gushchin struct bpf_cgroup_storage;
217b146cebSAndrey Ignatov struct ctl_table;
227b146cebSAndrey Ignatov struct ctl_table_header;
23b910eaaaSYonghong Song struct task_struct;
2430070984SDaniel Mack 
2530070984SDaniel Mack #ifdef CONFIG_CGROUP_BPF
266fc88c35SDave Marchevsky enum cgroup_bpf_attach_type {
276fc88c35SDave Marchevsky 	CGROUP_BPF_ATTACH_TYPE_INVALID = -1,
286fc88c35SDave Marchevsky 	CGROUP_INET_INGRESS = 0,
296fc88c35SDave Marchevsky 	CGROUP_INET_EGRESS,
306fc88c35SDave Marchevsky 	CGROUP_INET_SOCK_CREATE,
316fc88c35SDave Marchevsky 	CGROUP_SOCK_OPS,
326fc88c35SDave Marchevsky 	CGROUP_DEVICE,
336fc88c35SDave Marchevsky 	CGROUP_INET4_BIND,
346fc88c35SDave Marchevsky 	CGROUP_INET6_BIND,
356fc88c35SDave Marchevsky 	CGROUP_INET4_CONNECT,
366fc88c35SDave Marchevsky 	CGROUP_INET6_CONNECT,
376fc88c35SDave Marchevsky 	CGROUP_INET4_POST_BIND,
386fc88c35SDave Marchevsky 	CGROUP_INET6_POST_BIND,
396fc88c35SDave Marchevsky 	CGROUP_UDP4_SENDMSG,
406fc88c35SDave Marchevsky 	CGROUP_UDP6_SENDMSG,
416fc88c35SDave Marchevsky 	CGROUP_SYSCTL,
426fc88c35SDave Marchevsky 	CGROUP_UDP4_RECVMSG,
436fc88c35SDave Marchevsky 	CGROUP_UDP6_RECVMSG,
446fc88c35SDave Marchevsky 	CGROUP_GETSOCKOPT,
456fc88c35SDave Marchevsky 	CGROUP_SETSOCKOPT,
466fc88c35SDave Marchevsky 	CGROUP_INET4_GETPEERNAME,
476fc88c35SDave Marchevsky 	CGROUP_INET6_GETPEERNAME,
486fc88c35SDave Marchevsky 	CGROUP_INET4_GETSOCKNAME,
496fc88c35SDave Marchevsky 	CGROUP_INET6_GETSOCKNAME,
506fc88c35SDave Marchevsky 	CGROUP_INET_SOCK_RELEASE,
516fc88c35SDave Marchevsky 	MAX_CGROUP_BPF_ATTACH_TYPE
526fc88c35SDave Marchevsky };
5330070984SDaniel Mack 
546fc88c35SDave Marchevsky #define CGROUP_ATYPE(type) \
556fc88c35SDave Marchevsky 	case BPF_##type: return type
566fc88c35SDave Marchevsky 
576fc88c35SDave Marchevsky static inline enum cgroup_bpf_attach_type
586fc88c35SDave Marchevsky to_cgroup_bpf_attach_type(enum bpf_attach_type attach_type)
596fc88c35SDave Marchevsky {
606fc88c35SDave Marchevsky 	switch (attach_type) {
616fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_INET_INGRESS);
626fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_INET_EGRESS);
636fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_INET_SOCK_CREATE);
646fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_SOCK_OPS);
656fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_DEVICE);
666fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_INET4_BIND);
676fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_INET6_BIND);
686fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_INET4_CONNECT);
696fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_INET6_CONNECT);
706fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_INET4_POST_BIND);
716fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_INET6_POST_BIND);
726fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_UDP4_SENDMSG);
736fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_UDP6_SENDMSG);
746fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_SYSCTL);
756fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_UDP4_RECVMSG);
766fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_UDP6_RECVMSG);
776fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_GETSOCKOPT);
786fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_SETSOCKOPT);
796fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_INET4_GETPEERNAME);
806fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_INET6_GETPEERNAME);
816fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_INET4_GETSOCKNAME);
826fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_INET6_GETSOCKNAME);
836fc88c35SDave Marchevsky 	CGROUP_ATYPE(CGROUP_INET_SOCK_RELEASE);
846fc88c35SDave Marchevsky 	default:
856fc88c35SDave Marchevsky 		return CGROUP_BPF_ATTACH_TYPE_INVALID;
866fc88c35SDave Marchevsky 	}
876fc88c35SDave Marchevsky }
886fc88c35SDave Marchevsky 
896fc88c35SDave Marchevsky #undef CGROUP_ATYPE
906fc88c35SDave Marchevsky 
916fc88c35SDave Marchevsky extern struct static_key_false cgroup_bpf_enabled_key[MAX_CGROUP_BPF_ATTACH_TYPE];
926fc88c35SDave Marchevsky #define cgroup_bpf_enabled(atype) static_branch_unlikely(&cgroup_bpf_enabled_key[atype])
9330070984SDaniel Mack 
948bad74f9SRoman Gushchin #define for_each_cgroup_storage_type(stype) \
958bad74f9SRoman Gushchin 	for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++)
96aa0ad5b0SRoman Gushchin 
97de9cbbaaSRoman Gushchin struct bpf_cgroup_storage_map;
98de9cbbaaSRoman Gushchin 
99de9cbbaaSRoman Gushchin struct bpf_storage_buffer {
100de9cbbaaSRoman Gushchin 	struct rcu_head rcu;
101d7f10df8SGustavo A. R. Silva 	char data[];
102de9cbbaaSRoman Gushchin };
103de9cbbaaSRoman Gushchin 
104de9cbbaaSRoman Gushchin struct bpf_cgroup_storage {
105b741f163SRoman Gushchin 	union {
106de9cbbaaSRoman Gushchin 		struct bpf_storage_buffer *buf;
107b741f163SRoman Gushchin 		void __percpu *percpu_buf;
108b741f163SRoman Gushchin 	};
109de9cbbaaSRoman Gushchin 	struct bpf_cgroup_storage_map *map;
110de9cbbaaSRoman Gushchin 	struct bpf_cgroup_storage_key key;
1117d9c3427SYiFei Zhu 	struct list_head list_map;
1127d9c3427SYiFei Zhu 	struct list_head list_cg;
113de9cbbaaSRoman Gushchin 	struct rb_node node;
114de9cbbaaSRoman Gushchin 	struct rcu_head rcu;
115de9cbbaaSRoman Gushchin };
116de9cbbaaSRoman Gushchin 
117af6eea57SAndrii Nakryiko struct bpf_cgroup_link {
118af6eea57SAndrii Nakryiko 	struct bpf_link link;
119af6eea57SAndrii Nakryiko 	struct cgroup *cgroup;
120af6eea57SAndrii Nakryiko 	enum bpf_attach_type type;
121af6eea57SAndrii Nakryiko };
122af6eea57SAndrii Nakryiko 
123324bda9eSAlexei Starovoitov struct bpf_prog_list {
124324bda9eSAlexei Starovoitov 	struct list_head node;
125324bda9eSAlexei Starovoitov 	struct bpf_prog *prog;
126af6eea57SAndrii Nakryiko 	struct bpf_cgroup_link *link;
1278bad74f9SRoman Gushchin 	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];
128324bda9eSAlexei Starovoitov };
129324bda9eSAlexei Starovoitov 
130324bda9eSAlexei Starovoitov struct bpf_prog_array;
131324bda9eSAlexei Starovoitov 
13230070984SDaniel Mack struct cgroup_bpf {
133324bda9eSAlexei Starovoitov 	/* array of effective progs in this cgroup */
1346fc88c35SDave Marchevsky 	struct bpf_prog_array __rcu *effective[MAX_CGROUP_BPF_ATTACH_TYPE];
135324bda9eSAlexei Starovoitov 
136324bda9eSAlexei Starovoitov 	/* attached progs to this cgroup and attach flags
137324bda9eSAlexei Starovoitov 	 * when flags == 0 or BPF_F_ALLOW_OVERRIDE the progs list will
138324bda9eSAlexei Starovoitov 	 * have either zero or one element
139324bda9eSAlexei Starovoitov 	 * when BPF_F_ALLOW_MULTI the list can have up to BPF_CGROUP_MAX_PROGS
14030070984SDaniel Mack 	 */
1416fc88c35SDave Marchevsky 	struct list_head progs[MAX_CGROUP_BPF_ATTACH_TYPE];
1426fc88c35SDave Marchevsky 	u32 flags[MAX_CGROUP_BPF_ATTACH_TYPE];
143324bda9eSAlexei Starovoitov 
1447d9c3427SYiFei Zhu 	/* list of cgroup shared storages */
1457d9c3427SYiFei Zhu 	struct list_head storages;
1467d9c3427SYiFei Zhu 
147324bda9eSAlexei Starovoitov 	/* temp storage for effective prog array used by prog_attach/detach */
148dbcc1ba2SStanislav Fomichev 	struct bpf_prog_array *inactive;
1494bfc0bb2SRoman Gushchin 
1504bfc0bb2SRoman Gushchin 	/* reference counter used to detach bpf programs after cgroup removal */
1514bfc0bb2SRoman Gushchin 	struct percpu_ref refcnt;
1524bfc0bb2SRoman Gushchin 
1534bfc0bb2SRoman Gushchin 	/* cgroup_bpf is released using a work queue */
1544bfc0bb2SRoman Gushchin 	struct work_struct release_work;
15530070984SDaniel Mack };
15630070984SDaniel Mack 
157324bda9eSAlexei Starovoitov int cgroup_bpf_inherit(struct cgroup *cgrp);
1584bfc0bb2SRoman Gushchin void cgroup_bpf_offline(struct cgroup *cgrp);
15930070984SDaniel Mack 
160af6eea57SAndrii Nakryiko int __cgroup_bpf_attach(struct cgroup *cgrp,
161af6eea57SAndrii Nakryiko 			struct bpf_prog *prog, struct bpf_prog *replace_prog,
162af6eea57SAndrii Nakryiko 			struct bpf_cgroup_link *link,
163324bda9eSAlexei Starovoitov 			enum bpf_attach_type type, u32 flags);
164324bda9eSAlexei Starovoitov int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
165af6eea57SAndrii Nakryiko 			struct bpf_cgroup_link *link,
1661832f4efSValdis Kletnieks 			enum bpf_attach_type type);
167468e2f64SAlexei Starovoitov int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
168468e2f64SAlexei Starovoitov 		       union bpf_attr __user *uattr);
16930070984SDaniel Mack 
170324bda9eSAlexei Starovoitov /* Wrapper for __cgroup_bpf_*() protected by cgroup_mutex */
171af6eea57SAndrii Nakryiko int cgroup_bpf_attach(struct cgroup *cgrp,
172af6eea57SAndrii Nakryiko 		      struct bpf_prog *prog, struct bpf_prog *replace_prog,
173af6eea57SAndrii Nakryiko 		      struct bpf_cgroup_link *link, enum bpf_attach_type type,
1747dd68b32SAndrey Ignatov 		      u32 flags);
175324bda9eSAlexei Starovoitov int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
176af6eea57SAndrii Nakryiko 		      enum bpf_attach_type type);
177468e2f64SAlexei Starovoitov int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
178468e2f64SAlexei Starovoitov 		     union bpf_attr __user *uattr);
17930070984SDaniel Mack 
180b2cd1257SDavid Ahern int __cgroup_bpf_run_filter_skb(struct sock *sk,
18130070984SDaniel Mack 				struct sk_buff *skb,
1826fc88c35SDave Marchevsky 				enum cgroup_bpf_attach_type atype);
18330070984SDaniel Mack 
18461023658SDavid Ahern int __cgroup_bpf_run_filter_sk(struct sock *sk,
1856fc88c35SDave Marchevsky 			       enum cgroup_bpf_attach_type atype);
18661023658SDavid Ahern 
1874fbac77dSAndrey Ignatov int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,
1884fbac77dSAndrey Ignatov 				      struct sockaddr *uaddr,
1896fc88c35SDave Marchevsky 				      enum cgroup_bpf_attach_type atype,
19077241217SStanislav Fomichev 				      void *t_ctx,
19177241217SStanislav Fomichev 				      u32 *flags);
1924fbac77dSAndrey Ignatov 
19340304b2aSLawrence Brakmo int __cgroup_bpf_run_filter_sock_ops(struct sock *sk,
19440304b2aSLawrence Brakmo 				     struct bpf_sock_ops_kern *sock_ops,
1956fc88c35SDave Marchevsky 				     enum cgroup_bpf_attach_type atype);
19640304b2aSLawrence Brakmo 
197ebc614f6SRoman Gushchin int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor,
1986fc88c35SDave Marchevsky 				      short access, enum cgroup_bpf_attach_type atype);
199ebc614f6SRoman Gushchin 
2007b146cebSAndrey Ignatov int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head,
2017b146cebSAndrey Ignatov 				   struct ctl_table *table, int write,
2024bd6a735SMatthew Wilcox (Oracle) 				   char **buf, size_t *pcount, loff_t *ppos,
2036fc88c35SDave Marchevsky 				   enum cgroup_bpf_attach_type atype);
2047b146cebSAndrey Ignatov 
2050d01da6aSStanislav Fomichev int __cgroup_bpf_run_filter_setsockopt(struct sock *sock, int *level,
2060d01da6aSStanislav Fomichev 				       int *optname, char __user *optval,
2070d01da6aSStanislav Fomichev 				       int *optlen, char **kernel_optval);
2080d01da6aSStanislav Fomichev int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level,
2090d01da6aSStanislav Fomichev 				       int optname, char __user *optval,
2100d01da6aSStanislav Fomichev 				       int __user *optlen, int max_optlen,
2110d01da6aSStanislav Fomichev 				       int retval);
2120d01da6aSStanislav Fomichev 
2139cacf81fSStanislav Fomichev int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level,
2149cacf81fSStanislav Fomichev 					    int optname, void *optval,
2159cacf81fSStanislav Fomichev 					    int *optlen, int retval);
2169cacf81fSStanislav Fomichev 
2178bad74f9SRoman Gushchin static inline enum bpf_cgroup_storage_type cgroup_storage_type(
2188bad74f9SRoman Gushchin 	struct bpf_map *map)
219aa0ad5b0SRoman Gushchin {
220b741f163SRoman Gushchin 	if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
221b741f163SRoman Gushchin 		return BPF_CGROUP_STORAGE_PERCPU;
222b741f163SRoman Gushchin 
2238bad74f9SRoman Gushchin 	return BPF_CGROUP_STORAGE_SHARED;
224aa0ad5b0SRoman Gushchin }
225aa0ad5b0SRoman Gushchin 
2267d9c3427SYiFei Zhu struct bpf_cgroup_storage *
2277d9c3427SYiFei Zhu cgroup_storage_lookup(struct bpf_cgroup_storage_map *map,
2287d9c3427SYiFei Zhu 		      void *key, bool locked);
2298bad74f9SRoman Gushchin struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(struct bpf_prog *prog,
2308bad74f9SRoman Gushchin 					enum bpf_cgroup_storage_type stype);
231de9cbbaaSRoman Gushchin void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage);
232de9cbbaaSRoman Gushchin void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage,
233de9cbbaaSRoman Gushchin 			     struct cgroup *cgroup,
234de9cbbaaSRoman Gushchin 			     enum bpf_attach_type type);
235de9cbbaaSRoman Gushchin void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage);
236e4730423SDaniel Borkmann int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux, struct bpf_map *map);
237de9cbbaaSRoman Gushchin 
238b741f163SRoman Gushchin int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key, void *value);
239b741f163SRoman Gushchin int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,
240b741f163SRoman Gushchin 				     void *value, u64 flags);
241b741f163SRoman Gushchin 
242b2cd1257SDavid Ahern /* Wrappers for __cgroup_bpf_run_filter_skb() guarded by cgroup_bpf_enabled. */
24330070984SDaniel Mack #define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb)			      \
24430070984SDaniel Mack ({									      \
24530070984SDaniel Mack 	int __ret = 0;							      \
2466fc88c35SDave Marchevsky 	if (cgroup_bpf_enabled(CGROUP_INET_INGRESS))		      \
247b2cd1257SDavid Ahern 		__ret = __cgroup_bpf_run_filter_skb(sk, skb,		      \
2486fc88c35SDave Marchevsky 						    CGROUP_INET_INGRESS); \
24930070984SDaniel Mack 									      \
25030070984SDaniel Mack 	__ret;								      \
25130070984SDaniel Mack })
25230070984SDaniel Mack 
25330070984SDaniel Mack #define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb)			       \
25430070984SDaniel Mack ({									       \
25530070984SDaniel Mack 	int __ret = 0;							       \
2566fc88c35SDave Marchevsky 	if (cgroup_bpf_enabled(CGROUP_INET_EGRESS) && sk && sk == skb->sk) { \
25730070984SDaniel Mack 		typeof(sk) __sk = sk_to_full_sk(sk);			       \
25830070984SDaniel Mack 		if (sk_fullsock(__sk))					       \
259b2cd1257SDavid Ahern 			__ret = __cgroup_bpf_run_filter_skb(__sk, skb,	       \
2606fc88c35SDave Marchevsky 						      CGROUP_INET_EGRESS); \
26130070984SDaniel Mack 	}								       \
26230070984SDaniel Mack 	__ret;								       \
26330070984SDaniel Mack })
26430070984SDaniel Mack 
2656fc88c35SDave Marchevsky #define BPF_CGROUP_RUN_SK_PROG(sk, atype)				       \
26661023658SDavid Ahern ({									       \
26761023658SDavid Ahern 	int __ret = 0;							       \
2686fc88c35SDave Marchevsky 	if (cgroup_bpf_enabled(atype)) {					       \
2696fc88c35SDave Marchevsky 		__ret = __cgroup_bpf_run_filter_sk(sk, atype);		       \
27061023658SDavid Ahern 	}								       \
27161023658SDavid Ahern 	__ret;								       \
27261023658SDavid Ahern })
27361023658SDavid Ahern 
274aac3fc32SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET_SOCK(sk)				       \
2756fc88c35SDave Marchevsky 	BPF_CGROUP_RUN_SK_PROG(sk, CGROUP_INET_SOCK_CREATE)
276aac3fc32SAndrey Ignatov 
277f5836749SStanislav Fomichev #define BPF_CGROUP_RUN_PROG_INET_SOCK_RELEASE(sk)			       \
2786fc88c35SDave Marchevsky 	BPF_CGROUP_RUN_SK_PROG(sk, CGROUP_INET_SOCK_RELEASE)
279f5836749SStanislav Fomichev 
280aac3fc32SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk)				       \
2816fc88c35SDave Marchevsky 	BPF_CGROUP_RUN_SK_PROG(sk, CGROUP_INET4_POST_BIND)
282aac3fc32SAndrey Ignatov 
283aac3fc32SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk)				       \
2846fc88c35SDave Marchevsky 	BPF_CGROUP_RUN_SK_PROG(sk, CGROUP_INET6_POST_BIND)
285aac3fc32SAndrey Ignatov 
2866fc88c35SDave Marchevsky #define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, atype)				       \
2874fbac77dSAndrey Ignatov ({									       \
28877241217SStanislav Fomichev 	u32 __unused_flags;						       \
2894fbac77dSAndrey Ignatov 	int __ret = 0;							       \
2906fc88c35SDave Marchevsky 	if (cgroup_bpf_enabled(atype))					       \
2916fc88c35SDave Marchevsky 		__ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, atype,     \
29277241217SStanislav Fomichev 							  NULL,		       \
29377241217SStanislav Fomichev 							  &__unused_flags);    \
2944fbac77dSAndrey Ignatov 	__ret;								       \
2954fbac77dSAndrey Ignatov })
2964fbac77dSAndrey Ignatov 
2976fc88c35SDave Marchevsky #define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, atype, t_ctx)		       \
298d74bad4eSAndrey Ignatov ({									       \
29977241217SStanislav Fomichev 	u32 __unused_flags;						       \
300d74bad4eSAndrey Ignatov 	int __ret = 0;							       \
3016fc88c35SDave Marchevsky 	if (cgroup_bpf_enabled(atype))	{				       \
302d74bad4eSAndrey Ignatov 		lock_sock(sk);						       \
3036fc88c35SDave Marchevsky 		__ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, atype,     \
30477241217SStanislav Fomichev 							  t_ctx,	       \
30577241217SStanislav Fomichev 							  &__unused_flags);    \
306d74bad4eSAndrey Ignatov 		release_sock(sk);					       \
307d74bad4eSAndrey Ignatov 	}								       \
308d74bad4eSAndrey Ignatov 	__ret;								       \
309d74bad4eSAndrey Ignatov })
310d74bad4eSAndrey Ignatov 
31177241217SStanislav Fomichev /* BPF_CGROUP_INET4_BIND and BPF_CGROUP_INET6_BIND can return extra flags
31277241217SStanislav Fomichev  * via upper bits of return code. The only flag that is supported
31377241217SStanislav Fomichev  * (at bit position 0) is to indicate CAP_NET_BIND_SERVICE capability check
31477241217SStanislav Fomichev  * should be bypassed (BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE).
31577241217SStanislav Fomichev  */
3166fc88c35SDave Marchevsky #define BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr, atype, bind_flags)	       \
31777241217SStanislav Fomichev ({									       \
31877241217SStanislav Fomichev 	u32 __flags = 0;						       \
31977241217SStanislav Fomichev 	int __ret = 0;							       \
3206fc88c35SDave Marchevsky 	if (cgroup_bpf_enabled(atype))	{				       \
32177241217SStanislav Fomichev 		lock_sock(sk);						       \
3226fc88c35SDave Marchevsky 		__ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, atype,     \
32377241217SStanislav Fomichev 							  NULL, &__flags);     \
32477241217SStanislav Fomichev 		release_sock(sk);					       \
32577241217SStanislav Fomichev 		if (__flags & BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE)	       \
32677241217SStanislav Fomichev 			*bind_flags |= BIND_NO_CAP_NET_BIND_SERVICE;	       \
32777241217SStanislav Fomichev 	}								       \
32877241217SStanislav Fomichev 	__ret;								       \
32977241217SStanislav Fomichev })
3304fbac77dSAndrey Ignatov 
331a9ed15daSStanislav Fomichev #define BPF_CGROUP_PRE_CONNECT_ENABLED(sk)				       \
3326fc88c35SDave Marchevsky 	((cgroup_bpf_enabled(CGROUP_INET4_CONNECT) ||		       \
3336fc88c35SDave Marchevsky 	  cgroup_bpf_enabled(CGROUP_INET6_CONNECT)) &&		       \
334a9ed15daSStanislav Fomichev 	 (sk)->sk_prot->pre_connect)
335d74bad4eSAndrey Ignatov 
336d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr)			       \
3376fc88c35SDave Marchevsky 	BPF_CGROUP_RUN_SA_PROG(sk, uaddr, CGROUP_INET4_CONNECT)
338d74bad4eSAndrey Ignatov 
339d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr)			       \
3406fc88c35SDave Marchevsky 	BPF_CGROUP_RUN_SA_PROG(sk, uaddr, CGROUP_INET6_CONNECT)
341d74bad4eSAndrey Ignatov 
342d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr)		       \
3436fc88c35SDave Marchevsky 	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, CGROUP_INET4_CONNECT, NULL)
344d74bad4eSAndrey Ignatov 
345d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr)		       \
3466fc88c35SDave Marchevsky 	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, CGROUP_INET6_CONNECT, NULL)
3471cedee13SAndrey Ignatov 
3481cedee13SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, t_ctx)		       \
3496fc88c35SDave Marchevsky 	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, CGROUP_UDP4_SENDMSG, t_ctx)
3501cedee13SAndrey Ignatov 
3511cedee13SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, t_ctx)		       \
3526fc88c35SDave Marchevsky 	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, CGROUP_UDP6_SENDMSG, t_ctx)
353d74bad4eSAndrey Ignatov 
354983695faSDaniel Borkmann #define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr)			\
3556fc88c35SDave Marchevsky 	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, CGROUP_UDP4_RECVMSG, NULL)
356983695faSDaniel Borkmann 
357983695faSDaniel Borkmann #define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr)			\
3586fc88c35SDave Marchevsky 	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, CGROUP_UDP6_RECVMSG, NULL)
359983695faSDaniel Borkmann 
3600813a841SMartin KaFai Lau /* The SOCK_OPS"_SK" macro should be used when sock_ops->sk is not a
3610813a841SMartin KaFai Lau  * fullsock and its parent fullsock cannot be traced by
3620813a841SMartin KaFai Lau  * sk_to_full_sk().
3630813a841SMartin KaFai Lau  *
3640813a841SMartin KaFai Lau  * e.g. sock_ops->sk is a request_sock and it is under syncookie mode.
3650813a841SMartin KaFai Lau  * Its listener-sk is not attached to the rsk_listener.
3660813a841SMartin KaFai Lau  * In this case, the caller holds the listener-sk (unlocked),
3670813a841SMartin KaFai Lau  * set its sock_ops->sk to req_sk, and call this SOCK_OPS"_SK" with
3680813a841SMartin KaFai Lau  * the listener-sk such that the cgroup-bpf-progs of the
3690813a841SMartin KaFai Lau  * listener-sk will be run.
3700813a841SMartin KaFai Lau  *
3710813a841SMartin KaFai Lau  * Regardless of syncookie mode or not,
3720813a841SMartin KaFai Lau  * calling bpf_setsockopt on listener-sk will not make sense anyway,
3730813a841SMartin KaFai Lau  * so passing 'sock_ops->sk == req_sk' to the bpf prog is appropriate here.
3740813a841SMartin KaFai Lau  */
3750813a841SMartin KaFai Lau #define BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk)			\
3760813a841SMartin KaFai Lau ({									\
3770813a841SMartin KaFai Lau 	int __ret = 0;							\
3786fc88c35SDave Marchevsky 	if (cgroup_bpf_enabled(CGROUP_SOCK_OPS))			\
3790813a841SMartin KaFai Lau 		__ret = __cgroup_bpf_run_filter_sock_ops(sk,		\
3800813a841SMartin KaFai Lau 							 sock_ops,	\
3816fc88c35SDave Marchevsky 							 CGROUP_SOCK_OPS); \
3820813a841SMartin KaFai Lau 	__ret;								\
3830813a841SMartin KaFai Lau })
3840813a841SMartin KaFai Lau 
38540304b2aSLawrence Brakmo #define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops)				       \
38640304b2aSLawrence Brakmo ({									       \
38740304b2aSLawrence Brakmo 	int __ret = 0;							       \
3886fc88c35SDave Marchevsky 	if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) && (sock_ops)->sk) {       \
38940304b2aSLawrence Brakmo 		typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk);	       \
390df39a9f1SWANG Cong 		if (__sk && sk_fullsock(__sk))				       \
39140304b2aSLawrence Brakmo 			__ret = __cgroup_bpf_run_filter_sock_ops(__sk,	       \
39240304b2aSLawrence Brakmo 								 sock_ops,     \
3936fc88c35SDave Marchevsky 							 CGROUP_SOCK_OPS); \
39440304b2aSLawrence Brakmo 	}								       \
39540304b2aSLawrence Brakmo 	__ret;								       \
39640304b2aSLawrence Brakmo })
397ebc614f6SRoman Gushchin 
3986fc88c35SDave Marchevsky #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(atype, major, minor, access)	      \
399ebc614f6SRoman Gushchin ({									      \
400ebc614f6SRoman Gushchin 	int __ret = 0;							      \
4016fc88c35SDave Marchevsky 	if (cgroup_bpf_enabled(CGROUP_DEVICE))			      \
4026fc88c35SDave Marchevsky 		__ret = __cgroup_bpf_check_dev_permission(atype, major, minor, \
403ebc614f6SRoman Gushchin 							  access,	      \
4046fc88c35SDave Marchevsky 							  CGROUP_DEVICE); \
405ebc614f6SRoman Gushchin 									      \
406ebc614f6SRoman Gushchin 	__ret;								      \
407ebc614f6SRoman Gushchin })
4087b146cebSAndrey Ignatov 
4097b146cebSAndrey Ignatov 
41032927393SChristoph Hellwig #define BPF_CGROUP_RUN_PROG_SYSCTL(head, table, write, buf, count, pos)  \
4117b146cebSAndrey Ignatov ({									       \
4127b146cebSAndrey Ignatov 	int __ret = 0;							       \
4136fc88c35SDave Marchevsky 	if (cgroup_bpf_enabled(CGROUP_SYSCTL))			       \
4147b146cebSAndrey Ignatov 		__ret = __cgroup_bpf_run_filter_sysctl(head, table, write,     \
41532927393SChristoph Hellwig 						       buf, count, pos,        \
4166fc88c35SDave Marchevsky 						       CGROUP_SYSCTL);     \
4177b146cebSAndrey Ignatov 	__ret;								       \
4187b146cebSAndrey Ignatov })
4197b146cebSAndrey Ignatov 
4200d01da6aSStanislav Fomichev #define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen,   \
4210d01da6aSStanislav Fomichev 				       kernel_optval)			       \
4220d01da6aSStanislav Fomichev ({									       \
4230d01da6aSStanislav Fomichev 	int __ret = 0;							       \
4246fc88c35SDave Marchevsky 	if (cgroup_bpf_enabled(CGROUP_SETSOCKOPT))			       \
4250d01da6aSStanislav Fomichev 		__ret = __cgroup_bpf_run_filter_setsockopt(sock, level,	       \
4260d01da6aSStanislav Fomichev 							   optname, optval,    \
4270d01da6aSStanislav Fomichev 							   optlen,	       \
4280d01da6aSStanislav Fomichev 							   kernel_optval);     \
4290d01da6aSStanislav Fomichev 	__ret;								       \
4300d01da6aSStanislav Fomichev })
4310d01da6aSStanislav Fomichev 
4320d01da6aSStanislav Fomichev #define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen)			       \
4330d01da6aSStanislav Fomichev ({									       \
4340d01da6aSStanislav Fomichev 	int __ret = 0;							       \
4356fc88c35SDave Marchevsky 	if (cgroup_bpf_enabled(CGROUP_GETSOCKOPT))			       \
4360d01da6aSStanislav Fomichev 		get_user(__ret, optlen);				       \
4370d01da6aSStanislav Fomichev 	__ret;								       \
4380d01da6aSStanislav Fomichev })
4390d01da6aSStanislav Fomichev 
4400d01da6aSStanislav Fomichev #define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, optlen,   \
4410d01da6aSStanislav Fomichev 				       max_optlen, retval)		       \
4420d01da6aSStanislav Fomichev ({									       \
4430d01da6aSStanislav Fomichev 	int __ret = retval;						       \
4446fc88c35SDave Marchevsky 	if (cgroup_bpf_enabled(CGROUP_GETSOCKOPT))			       \
4459cacf81fSStanislav Fomichev 		if (!(sock)->sk_prot->bpf_bypass_getsockopt ||		       \
4469cacf81fSStanislav Fomichev 		    !INDIRECT_CALL_INET_1((sock)->sk_prot->bpf_bypass_getsockopt, \
4479cacf81fSStanislav Fomichev 					tcp_bpf_bypass_getsockopt,	       \
4489cacf81fSStanislav Fomichev 					level, optname))		       \
4499cacf81fSStanislav Fomichev 			__ret = __cgroup_bpf_run_filter_getsockopt(	       \
4509cacf81fSStanislav Fomichev 				sock, level, optname, optval, optlen,	       \
4519cacf81fSStanislav Fomichev 				max_optlen, retval);			       \
4529cacf81fSStanislav Fomichev 	__ret;								       \
4539cacf81fSStanislav Fomichev })
4549cacf81fSStanislav Fomichev 
4559cacf81fSStanislav Fomichev #define BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sock, level, optname, optval,      \
4569cacf81fSStanislav Fomichev 					    optlen, retval)		       \
4579cacf81fSStanislav Fomichev ({									       \
4589cacf81fSStanislav Fomichev 	int __ret = retval;						       \
4596fc88c35SDave Marchevsky 	if (cgroup_bpf_enabled(CGROUP_GETSOCKOPT))			       \
4609cacf81fSStanislav Fomichev 		__ret = __cgroup_bpf_run_filter_getsockopt_kern(	       \
4619cacf81fSStanislav Fomichev 			sock, level, optname, optval, optlen, retval);	       \
4620d01da6aSStanislav Fomichev 	__ret;								       \
4630d01da6aSStanislav Fomichev })
4640d01da6aSStanislav Fomichev 
465fdb5c453SSean Young int cgroup_bpf_prog_attach(const union bpf_attr *attr,
466fdb5c453SSean Young 			   enum bpf_prog_type ptype, struct bpf_prog *prog);
467fdb5c453SSean Young int cgroup_bpf_prog_detach(const union bpf_attr *attr,
468fdb5c453SSean Young 			   enum bpf_prog_type ptype);
469af6eea57SAndrii Nakryiko int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
470fdb5c453SSean Young int cgroup_bpf_prog_query(const union bpf_attr *attr,
471fdb5c453SSean Young 			  union bpf_attr __user *uattr);
47230070984SDaniel Mack #else
47330070984SDaniel Mack 
47430070984SDaniel Mack struct cgroup_bpf {};
475324bda9eSAlexei Starovoitov static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; }
4764bfc0bb2SRoman Gushchin static inline void cgroup_bpf_offline(struct cgroup *cgrp) {}
47730070984SDaniel Mack 
478fdb5c453SSean Young static inline int cgroup_bpf_prog_attach(const union bpf_attr *attr,
479fdb5c453SSean Young 					 enum bpf_prog_type ptype,
480fdb5c453SSean Young 					 struct bpf_prog *prog)
481fdb5c453SSean Young {
482fdb5c453SSean Young 	return -EINVAL;
483fdb5c453SSean Young }
484fdb5c453SSean Young 
485fdb5c453SSean Young static inline int cgroup_bpf_prog_detach(const union bpf_attr *attr,
486fdb5c453SSean Young 					 enum bpf_prog_type ptype)
487fdb5c453SSean Young {
488fdb5c453SSean Young 	return -EINVAL;
489fdb5c453SSean Young }
490fdb5c453SSean Young 
491af6eea57SAndrii Nakryiko static inline int cgroup_bpf_link_attach(const union bpf_attr *attr,
492af6eea57SAndrii Nakryiko 					 struct bpf_prog *prog)
493af6eea57SAndrii Nakryiko {
494af6eea57SAndrii Nakryiko 	return -EINVAL;
495af6eea57SAndrii Nakryiko }
496af6eea57SAndrii Nakryiko 
497fdb5c453SSean Young static inline int cgroup_bpf_prog_query(const union bpf_attr *attr,
498fdb5c453SSean Young 					union bpf_attr __user *uattr)
499fdb5c453SSean Young {
500fdb5c453SSean Young 	return -EINVAL;
501fdb5c453SSean Young }
502fdb5c453SSean Young 
503e4730423SDaniel Borkmann static inline int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux,
504de9cbbaaSRoman Gushchin 					    struct bpf_map *map) { return 0; }
505de9cbbaaSRoman Gushchin static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(
50671b91a50SBo YU 	struct bpf_prog *prog, enum bpf_cgroup_storage_type stype) { return NULL; }
507de9cbbaaSRoman Gushchin static inline void bpf_cgroup_storage_free(
508de9cbbaaSRoman Gushchin 	struct bpf_cgroup_storage *storage) {}
509b741f163SRoman Gushchin static inline int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key,
510b741f163SRoman Gushchin 						 void *value) {
511b741f163SRoman Gushchin 	return 0;
512b741f163SRoman Gushchin }
513b741f163SRoman Gushchin static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map,
514b741f163SRoman Gushchin 					void *key, void *value, u64 flags) {
515b741f163SRoman Gushchin 	return 0;
516b741f163SRoman Gushchin }
517de9cbbaaSRoman Gushchin 
5186fc88c35SDave Marchevsky #define cgroup_bpf_enabled(atype) (0)
5196fc88c35SDave Marchevsky #define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, atype, t_ctx) ({ 0; })
520*9dfc685eSEric Dumazet #define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, atype) ({ 0; })
521d74bad4eSAndrey Ignatov #define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (0)
52230070984SDaniel Mack #define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk,skb) ({ 0; })
52330070984SDaniel Mack #define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk,skb) ({ 0; })
52461023658SDavid Ahern #define BPF_CGROUP_RUN_PROG_INET_SOCK(sk) ({ 0; })
525f5836749SStanislav Fomichev #define BPF_CGROUP_RUN_PROG_INET_SOCK_RELEASE(sk) ({ 0; })
5266fc88c35SDave Marchevsky #define BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr, atype, flags) ({ 0; })
527aac3fc32SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk) ({ 0; })
528aac3fc32SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk) ({ 0; })
529d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) ({ 0; })
530d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr) ({ 0; })
531d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr) ({ 0; })
532d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr) ({ 0; })
5331cedee13SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, t_ctx) ({ 0; })
5341cedee13SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, t_ctx) ({ 0; })
535983695faSDaniel Borkmann #define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr) ({ 0; })
536983695faSDaniel Borkmann #define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr) ({ 0; })
53740304b2aSLawrence Brakmo #define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; })
5386fc88c35SDave Marchevsky #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(atype, major, minor, access) ({ 0; })
53932927393SChristoph Hellwig #define BPF_CGROUP_RUN_PROG_SYSCTL(head,table,write,buf,count,pos) ({ 0; })
5400d01da6aSStanislav Fomichev #define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) ({ 0; })
5410d01da6aSStanislav Fomichev #define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \
5420d01da6aSStanislav Fomichev 				       optlen, max_optlen, retval) ({ retval; })
5439cacf81fSStanislav Fomichev #define BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sock, level, optname, optval, \
5449cacf81fSStanislav Fomichev 					    optlen, retval) ({ retval; })
5450d01da6aSStanislav Fomichev #define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \
5460d01da6aSStanislav Fomichev 				       kernel_optval) ({ 0; })
54730070984SDaniel Mack 
5488bad74f9SRoman Gushchin #define for_each_cgroup_storage_type(stype) for (; false; )
5498bad74f9SRoman Gushchin 
55030070984SDaniel Mack #endif /* CONFIG_CGROUP_BPF */
55130070984SDaniel Mack 
55230070984SDaniel Mack #endif /* _BPF_CGROUP_H */
553