1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */ 230070984SDaniel Mack #ifndef _BPF_CGROUP_H 330070984SDaniel Mack #define _BPF_CGROUP_H 430070984SDaniel Mack 58bad74f9SRoman Gushchin #include <linux/bpf.h> 6f292b87dSRoman Gushchin #include <linux/errno.h> 730070984SDaniel Mack #include <linux/jump_label.h> 8aa0ad5b0SRoman Gushchin #include <linux/percpu.h> 94bfc0bb2SRoman Gushchin #include <linux/percpu-refcount.h> 10de9cbbaaSRoman Gushchin #include <linux/rbtree.h> 1130070984SDaniel Mack #include <uapi/linux/bpf.h> 1230070984SDaniel Mack 1330070984SDaniel Mack struct sock; 144fbac77dSAndrey Ignatov struct sockaddr; 1530070984SDaniel Mack struct cgroup; 1630070984SDaniel Mack struct sk_buff; 17de9cbbaaSRoman Gushchin struct bpf_map; 18de9cbbaaSRoman Gushchin struct bpf_prog; 1940304b2aSLawrence Brakmo struct bpf_sock_ops_kern; 20de9cbbaaSRoman Gushchin struct bpf_cgroup_storage; 217b146cebSAndrey Ignatov struct ctl_table; 227b146cebSAndrey Ignatov struct ctl_table_header; 23*b910eaaaSYonghong Song struct task_struct; 2430070984SDaniel Mack 2530070984SDaniel Mack #ifdef CONFIG_CGROUP_BPF 2630070984SDaniel Mack 27a9ed15daSStanislav Fomichev extern struct static_key_false cgroup_bpf_enabled_key[MAX_BPF_ATTACH_TYPE]; 28a9ed15daSStanislav Fomichev #define cgroup_bpf_enabled(type) static_branch_unlikely(&cgroup_bpf_enabled_key[type]) 2930070984SDaniel Mack 30*b910eaaaSYonghong Song #define BPF_CGROUP_STORAGE_NEST_MAX 8 31*b910eaaaSYonghong Song 32*b910eaaaSYonghong Song struct bpf_cgroup_storage_info { 33*b910eaaaSYonghong Song struct task_struct *task; 34*b910eaaaSYonghong Song struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 35*b910eaaaSYonghong Song }; 36*b910eaaaSYonghong Song 37*b910eaaaSYonghong Song /* For each cpu, permit maximum BPF_CGROUP_STORAGE_NEST_MAX number of tasks 38*b910eaaaSYonghong Song * to use bpf cgroup storage simultaneously. 39*b910eaaaSYonghong Song */ 40*b910eaaaSYonghong Song DECLARE_PER_CPU(struct bpf_cgroup_storage_info, 41*b910eaaaSYonghong Song bpf_cgroup_storage_info[BPF_CGROUP_STORAGE_NEST_MAX]); 428bad74f9SRoman Gushchin 438bad74f9SRoman Gushchin #define for_each_cgroup_storage_type(stype) \ 448bad74f9SRoman Gushchin for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++) 45aa0ad5b0SRoman Gushchin 46de9cbbaaSRoman Gushchin struct bpf_cgroup_storage_map; 47de9cbbaaSRoman Gushchin 48de9cbbaaSRoman Gushchin struct bpf_storage_buffer { 49de9cbbaaSRoman Gushchin struct rcu_head rcu; 50d7f10df8SGustavo A. R. Silva char data[]; 51de9cbbaaSRoman Gushchin }; 52de9cbbaaSRoman Gushchin 53de9cbbaaSRoman Gushchin struct bpf_cgroup_storage { 54b741f163SRoman Gushchin union { 55de9cbbaaSRoman Gushchin struct bpf_storage_buffer *buf; 56b741f163SRoman Gushchin void __percpu *percpu_buf; 57b741f163SRoman Gushchin }; 58de9cbbaaSRoman Gushchin struct bpf_cgroup_storage_map *map; 59de9cbbaaSRoman Gushchin struct bpf_cgroup_storage_key key; 607d9c3427SYiFei Zhu struct list_head list_map; 617d9c3427SYiFei Zhu struct list_head list_cg; 62de9cbbaaSRoman Gushchin struct rb_node node; 63de9cbbaaSRoman Gushchin struct rcu_head rcu; 64de9cbbaaSRoman Gushchin }; 65de9cbbaaSRoman Gushchin 66af6eea57SAndrii Nakryiko struct bpf_cgroup_link { 67af6eea57SAndrii Nakryiko struct bpf_link link; 68af6eea57SAndrii Nakryiko struct cgroup *cgroup; 69af6eea57SAndrii Nakryiko enum bpf_attach_type type; 70af6eea57SAndrii Nakryiko }; 71af6eea57SAndrii Nakryiko 72324bda9eSAlexei Starovoitov struct bpf_prog_list { 73324bda9eSAlexei Starovoitov struct list_head node; 74324bda9eSAlexei Starovoitov struct bpf_prog *prog; 75af6eea57SAndrii Nakryiko struct bpf_cgroup_link *link; 768bad74f9SRoman Gushchin struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 77324bda9eSAlexei Starovoitov }; 78324bda9eSAlexei Starovoitov 79324bda9eSAlexei Starovoitov struct bpf_prog_array; 80324bda9eSAlexei Starovoitov 8130070984SDaniel Mack struct cgroup_bpf { 82324bda9eSAlexei Starovoitov /* array of effective progs in this cgroup */ 83324bda9eSAlexei Starovoitov struct bpf_prog_array __rcu *effective[MAX_BPF_ATTACH_TYPE]; 84324bda9eSAlexei Starovoitov 85324bda9eSAlexei Starovoitov /* attached progs to this cgroup and attach flags 86324bda9eSAlexei Starovoitov * when flags == 0 or BPF_F_ALLOW_OVERRIDE the progs list will 87324bda9eSAlexei Starovoitov * have either zero or one element 88324bda9eSAlexei Starovoitov * when BPF_F_ALLOW_MULTI the list can have up to BPF_CGROUP_MAX_PROGS 8930070984SDaniel Mack */ 90324bda9eSAlexei Starovoitov struct list_head progs[MAX_BPF_ATTACH_TYPE]; 91324bda9eSAlexei Starovoitov u32 flags[MAX_BPF_ATTACH_TYPE]; 92324bda9eSAlexei Starovoitov 937d9c3427SYiFei Zhu /* list of cgroup shared storages */ 947d9c3427SYiFei Zhu struct list_head storages; 957d9c3427SYiFei Zhu 96324bda9eSAlexei Starovoitov /* temp storage for effective prog array used by prog_attach/detach */ 97dbcc1ba2SStanislav Fomichev struct bpf_prog_array *inactive; 984bfc0bb2SRoman Gushchin 994bfc0bb2SRoman Gushchin /* reference counter used to detach bpf programs after cgroup removal */ 1004bfc0bb2SRoman Gushchin struct percpu_ref refcnt; 1014bfc0bb2SRoman Gushchin 1024bfc0bb2SRoman Gushchin /* cgroup_bpf is released using a work queue */ 1034bfc0bb2SRoman Gushchin struct work_struct release_work; 10430070984SDaniel Mack }; 10530070984SDaniel Mack 106324bda9eSAlexei Starovoitov int cgroup_bpf_inherit(struct cgroup *cgrp); 1074bfc0bb2SRoman Gushchin void cgroup_bpf_offline(struct cgroup *cgrp); 10830070984SDaniel Mack 109af6eea57SAndrii Nakryiko int __cgroup_bpf_attach(struct cgroup *cgrp, 110af6eea57SAndrii Nakryiko struct bpf_prog *prog, struct bpf_prog *replace_prog, 111af6eea57SAndrii Nakryiko struct bpf_cgroup_link *link, 112324bda9eSAlexei Starovoitov enum bpf_attach_type type, u32 flags); 113324bda9eSAlexei Starovoitov int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, 114af6eea57SAndrii Nakryiko struct bpf_cgroup_link *link, 1151832f4efSValdis Kletnieks enum bpf_attach_type type); 116468e2f64SAlexei Starovoitov int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, 117468e2f64SAlexei Starovoitov union bpf_attr __user *uattr); 11830070984SDaniel Mack 119324bda9eSAlexei Starovoitov /* Wrapper for __cgroup_bpf_*() protected by cgroup_mutex */ 120af6eea57SAndrii Nakryiko int cgroup_bpf_attach(struct cgroup *cgrp, 121af6eea57SAndrii Nakryiko struct bpf_prog *prog, struct bpf_prog *replace_prog, 122af6eea57SAndrii Nakryiko struct bpf_cgroup_link *link, enum bpf_attach_type type, 1237dd68b32SAndrey Ignatov u32 flags); 124324bda9eSAlexei Starovoitov int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, 125af6eea57SAndrii Nakryiko enum bpf_attach_type type); 126468e2f64SAlexei Starovoitov int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, 127468e2f64SAlexei Starovoitov union bpf_attr __user *uattr); 12830070984SDaniel Mack 129b2cd1257SDavid Ahern int __cgroup_bpf_run_filter_skb(struct sock *sk, 13030070984SDaniel Mack struct sk_buff *skb, 13130070984SDaniel Mack enum bpf_attach_type type); 13230070984SDaniel Mack 13361023658SDavid Ahern int __cgroup_bpf_run_filter_sk(struct sock *sk, 13461023658SDavid Ahern enum bpf_attach_type type); 13561023658SDavid Ahern 1364fbac77dSAndrey Ignatov int __cgroup_bpf_run_filter_sock_addr(struct sock *sk, 1374fbac77dSAndrey Ignatov struct sockaddr *uaddr, 1381cedee13SAndrey Ignatov enum bpf_attach_type type, 13977241217SStanislav Fomichev void *t_ctx, 14077241217SStanislav Fomichev u32 *flags); 1414fbac77dSAndrey Ignatov 14240304b2aSLawrence Brakmo int __cgroup_bpf_run_filter_sock_ops(struct sock *sk, 14340304b2aSLawrence Brakmo struct bpf_sock_ops_kern *sock_ops, 14440304b2aSLawrence Brakmo enum bpf_attach_type type); 14540304b2aSLawrence Brakmo 146ebc614f6SRoman Gushchin int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor, 147ebc614f6SRoman Gushchin short access, enum bpf_attach_type type); 148ebc614f6SRoman Gushchin 1497b146cebSAndrey Ignatov int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head, 1507b146cebSAndrey Ignatov struct ctl_table *table, int write, 1514bd6a735SMatthew Wilcox (Oracle) char **buf, size_t *pcount, loff_t *ppos, 152e1550bfeSAndrey Ignatov enum bpf_attach_type type); 1537b146cebSAndrey Ignatov 1540d01da6aSStanislav Fomichev int __cgroup_bpf_run_filter_setsockopt(struct sock *sock, int *level, 1550d01da6aSStanislav Fomichev int *optname, char __user *optval, 1560d01da6aSStanislav Fomichev int *optlen, char **kernel_optval); 1570d01da6aSStanislav Fomichev int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, 1580d01da6aSStanislav Fomichev int optname, char __user *optval, 1590d01da6aSStanislav Fomichev int __user *optlen, int max_optlen, 1600d01da6aSStanislav Fomichev int retval); 1610d01da6aSStanislav Fomichev 1629cacf81fSStanislav Fomichev int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level, 1639cacf81fSStanislav Fomichev int optname, void *optval, 1649cacf81fSStanislav Fomichev int *optlen, int retval); 1659cacf81fSStanislav Fomichev 1668bad74f9SRoman Gushchin static inline enum bpf_cgroup_storage_type cgroup_storage_type( 1678bad74f9SRoman Gushchin struct bpf_map *map) 168aa0ad5b0SRoman Gushchin { 169b741f163SRoman Gushchin if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 170b741f163SRoman Gushchin return BPF_CGROUP_STORAGE_PERCPU; 171b741f163SRoman Gushchin 1728bad74f9SRoman Gushchin return BPF_CGROUP_STORAGE_SHARED; 173aa0ad5b0SRoman Gushchin } 174aa0ad5b0SRoman Gushchin 175*b910eaaaSYonghong Song static inline int bpf_cgroup_storage_set(struct bpf_cgroup_storage 1768bad74f9SRoman Gushchin *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) 1778bad74f9SRoman Gushchin { 1788bad74f9SRoman Gushchin enum bpf_cgroup_storage_type stype; 179*b910eaaaSYonghong Song int i, err = 0; 1808bad74f9SRoman Gushchin 181*b910eaaaSYonghong Song preempt_disable(); 182*b910eaaaSYonghong Song for (i = 0; i < BPF_CGROUP_STORAGE_NEST_MAX; i++) { 183*b910eaaaSYonghong Song if (unlikely(this_cpu_read(bpf_cgroup_storage_info[i].task) != NULL)) 184*b910eaaaSYonghong Song continue; 185*b910eaaaSYonghong Song 186*b910eaaaSYonghong Song this_cpu_write(bpf_cgroup_storage_info[i].task, current); 187f294b37eSRoman Gushchin for_each_cgroup_storage_type(stype) 188*b910eaaaSYonghong Song this_cpu_write(bpf_cgroup_storage_info[i].storage[stype], 189*b910eaaaSYonghong Song storage[stype]); 190*b910eaaaSYonghong Song goto out; 191*b910eaaaSYonghong Song } 192*b910eaaaSYonghong Song err = -EBUSY; 193*b910eaaaSYonghong Song WARN_ON_ONCE(1); 194*b910eaaaSYonghong Song 195*b910eaaaSYonghong Song out: 196*b910eaaaSYonghong Song preempt_enable(); 197*b910eaaaSYonghong Song return err; 198*b910eaaaSYonghong Song } 199*b910eaaaSYonghong Song 200*b910eaaaSYonghong Song static inline void bpf_cgroup_storage_unset(void) 201*b910eaaaSYonghong Song { 202*b910eaaaSYonghong Song int i; 203*b910eaaaSYonghong Song 204*b910eaaaSYonghong Song for (i = 0; i < BPF_CGROUP_STORAGE_NEST_MAX; i++) { 205*b910eaaaSYonghong Song if (unlikely(this_cpu_read(bpf_cgroup_storage_info[i].task) != current)) 206*b910eaaaSYonghong Song continue; 207*b910eaaaSYonghong Song 208*b910eaaaSYonghong Song this_cpu_write(bpf_cgroup_storage_info[i].task, NULL); 209*b910eaaaSYonghong Song return; 210*b910eaaaSYonghong Song } 2118bad74f9SRoman Gushchin } 2128bad74f9SRoman Gushchin 2137d9c3427SYiFei Zhu struct bpf_cgroup_storage * 2147d9c3427SYiFei Zhu cgroup_storage_lookup(struct bpf_cgroup_storage_map *map, 2157d9c3427SYiFei Zhu void *key, bool locked); 2168bad74f9SRoman Gushchin struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(struct bpf_prog *prog, 2178bad74f9SRoman Gushchin enum bpf_cgroup_storage_type stype); 218de9cbbaaSRoman Gushchin void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage); 219de9cbbaaSRoman Gushchin void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage, 220de9cbbaaSRoman Gushchin struct cgroup *cgroup, 221de9cbbaaSRoman Gushchin enum bpf_attach_type type); 222de9cbbaaSRoman Gushchin void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage); 223e4730423SDaniel Borkmann int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux, struct bpf_map *map); 224de9cbbaaSRoman Gushchin 225b741f163SRoman Gushchin int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key, void *value); 226b741f163SRoman Gushchin int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key, 227b741f163SRoman Gushchin void *value, u64 flags); 228b741f163SRoman Gushchin 229b2cd1257SDavid Ahern /* Wrappers for __cgroup_bpf_run_filter_skb() guarded by cgroup_bpf_enabled. */ 23030070984SDaniel Mack #define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb) \ 23130070984SDaniel Mack ({ \ 23230070984SDaniel Mack int __ret = 0; \ 233a9ed15daSStanislav Fomichev if (cgroup_bpf_enabled(BPF_CGROUP_INET_INGRESS)) \ 234b2cd1257SDavid Ahern __ret = __cgroup_bpf_run_filter_skb(sk, skb, \ 23530070984SDaniel Mack BPF_CGROUP_INET_INGRESS); \ 23630070984SDaniel Mack \ 23730070984SDaniel Mack __ret; \ 23830070984SDaniel Mack }) 23930070984SDaniel Mack 24030070984SDaniel Mack #define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb) \ 24130070984SDaniel Mack ({ \ 24230070984SDaniel Mack int __ret = 0; \ 243a9ed15daSStanislav Fomichev if (cgroup_bpf_enabled(BPF_CGROUP_INET_EGRESS) && sk && sk == skb->sk) { \ 24430070984SDaniel Mack typeof(sk) __sk = sk_to_full_sk(sk); \ 24530070984SDaniel Mack if (sk_fullsock(__sk)) \ 246b2cd1257SDavid Ahern __ret = __cgroup_bpf_run_filter_skb(__sk, skb, \ 24730070984SDaniel Mack BPF_CGROUP_INET_EGRESS); \ 24830070984SDaniel Mack } \ 24930070984SDaniel Mack __ret; \ 25030070984SDaniel Mack }) 25130070984SDaniel Mack 252aac3fc32SAndrey Ignatov #define BPF_CGROUP_RUN_SK_PROG(sk, type) \ 25361023658SDavid Ahern ({ \ 25461023658SDavid Ahern int __ret = 0; \ 255a9ed15daSStanislav Fomichev if (cgroup_bpf_enabled(type)) { \ 256aac3fc32SAndrey Ignatov __ret = __cgroup_bpf_run_filter_sk(sk, type); \ 25761023658SDavid Ahern } \ 25861023658SDavid Ahern __ret; \ 25961023658SDavid Ahern }) 26061023658SDavid Ahern 261aac3fc32SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET_SOCK(sk) \ 262aac3fc32SAndrey Ignatov BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET_SOCK_CREATE) 263aac3fc32SAndrey Ignatov 264f5836749SStanislav Fomichev #define BPF_CGROUP_RUN_PROG_INET_SOCK_RELEASE(sk) \ 265f5836749SStanislav Fomichev BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET_SOCK_RELEASE) 266f5836749SStanislav Fomichev 267aac3fc32SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk) \ 268aac3fc32SAndrey Ignatov BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET4_POST_BIND) 269aac3fc32SAndrey Ignatov 270aac3fc32SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk) \ 271aac3fc32SAndrey Ignatov BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET6_POST_BIND) 272aac3fc32SAndrey Ignatov 2734fbac77dSAndrey Ignatov #define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, type) \ 2744fbac77dSAndrey Ignatov ({ \ 27577241217SStanislav Fomichev u32 __unused_flags; \ 2764fbac77dSAndrey Ignatov int __ret = 0; \ 277a9ed15daSStanislav Fomichev if (cgroup_bpf_enabled(type)) \ 2781cedee13SAndrey Ignatov __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \ 27977241217SStanislav Fomichev NULL, \ 28077241217SStanislav Fomichev &__unused_flags); \ 2814fbac77dSAndrey Ignatov __ret; \ 2824fbac77dSAndrey Ignatov }) 2834fbac77dSAndrey Ignatov 2841cedee13SAndrey Ignatov #define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx) \ 285d74bad4eSAndrey Ignatov ({ \ 28677241217SStanislav Fomichev u32 __unused_flags; \ 287d74bad4eSAndrey Ignatov int __ret = 0; \ 288a9ed15daSStanislav Fomichev if (cgroup_bpf_enabled(type)) { \ 289d74bad4eSAndrey Ignatov lock_sock(sk); \ 2901cedee13SAndrey Ignatov __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \ 29177241217SStanislav Fomichev t_ctx, \ 29277241217SStanislav Fomichev &__unused_flags); \ 293d74bad4eSAndrey Ignatov release_sock(sk); \ 294d74bad4eSAndrey Ignatov } \ 295d74bad4eSAndrey Ignatov __ret; \ 296d74bad4eSAndrey Ignatov }) 297d74bad4eSAndrey Ignatov 29877241217SStanislav Fomichev /* BPF_CGROUP_INET4_BIND and BPF_CGROUP_INET6_BIND can return extra flags 29977241217SStanislav Fomichev * via upper bits of return code. The only flag that is supported 30077241217SStanislav Fomichev * (at bit position 0) is to indicate CAP_NET_BIND_SERVICE capability check 30177241217SStanislav Fomichev * should be bypassed (BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE). 30277241217SStanislav Fomichev */ 30377241217SStanislav Fomichev #define BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr, type, bind_flags) \ 30477241217SStanislav Fomichev ({ \ 30577241217SStanislav Fomichev u32 __flags = 0; \ 30677241217SStanislav Fomichev int __ret = 0; \ 30777241217SStanislav Fomichev if (cgroup_bpf_enabled(type)) { \ 30877241217SStanislav Fomichev lock_sock(sk); \ 30977241217SStanislav Fomichev __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \ 31077241217SStanislav Fomichev NULL, &__flags); \ 31177241217SStanislav Fomichev release_sock(sk); \ 31277241217SStanislav Fomichev if (__flags & BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE) \ 31377241217SStanislav Fomichev *bind_flags |= BIND_NO_CAP_NET_BIND_SERVICE; \ 31477241217SStanislav Fomichev } \ 31577241217SStanislav Fomichev __ret; \ 31677241217SStanislav Fomichev }) 3174fbac77dSAndrey Ignatov 318a9ed15daSStanislav Fomichev #define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) \ 319a9ed15daSStanislav Fomichev ((cgroup_bpf_enabled(BPF_CGROUP_INET4_CONNECT) || \ 320a9ed15daSStanislav Fomichev cgroup_bpf_enabled(BPF_CGROUP_INET6_CONNECT)) && \ 321a9ed15daSStanislav Fomichev (sk)->sk_prot->pre_connect) 322d74bad4eSAndrey Ignatov 323d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) \ 324d74bad4eSAndrey Ignatov BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET4_CONNECT) 325d74bad4eSAndrey Ignatov 326d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr) \ 327d74bad4eSAndrey Ignatov BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET6_CONNECT) 328d74bad4eSAndrey Ignatov 329d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr) \ 3301cedee13SAndrey Ignatov BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET4_CONNECT, NULL) 331d74bad4eSAndrey Ignatov 332d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr) \ 3331cedee13SAndrey Ignatov BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET6_CONNECT, NULL) 3341cedee13SAndrey Ignatov 3351cedee13SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, t_ctx) \ 3361cedee13SAndrey Ignatov BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP4_SENDMSG, t_ctx) 3371cedee13SAndrey Ignatov 3381cedee13SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, t_ctx) \ 3391cedee13SAndrey Ignatov BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP6_SENDMSG, t_ctx) 340d74bad4eSAndrey Ignatov 341983695faSDaniel Borkmann #define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr) \ 342983695faSDaniel Borkmann BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP4_RECVMSG, NULL) 343983695faSDaniel Borkmann 344983695faSDaniel Borkmann #define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr) \ 345983695faSDaniel Borkmann BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP6_RECVMSG, NULL) 346983695faSDaniel Borkmann 3470813a841SMartin KaFai Lau /* The SOCK_OPS"_SK" macro should be used when sock_ops->sk is not a 3480813a841SMartin KaFai Lau * fullsock and its parent fullsock cannot be traced by 3490813a841SMartin KaFai Lau * sk_to_full_sk(). 3500813a841SMartin KaFai Lau * 3510813a841SMartin KaFai Lau * e.g. sock_ops->sk is a request_sock and it is under syncookie mode. 3520813a841SMartin KaFai Lau * Its listener-sk is not attached to the rsk_listener. 3530813a841SMartin KaFai Lau * In this case, the caller holds the listener-sk (unlocked), 3540813a841SMartin KaFai Lau * set its sock_ops->sk to req_sk, and call this SOCK_OPS"_SK" with 3550813a841SMartin KaFai Lau * the listener-sk such that the cgroup-bpf-progs of the 3560813a841SMartin KaFai Lau * listener-sk will be run. 3570813a841SMartin KaFai Lau * 3580813a841SMartin KaFai Lau * Regardless of syncookie mode or not, 3590813a841SMartin KaFai Lau * calling bpf_setsockopt on listener-sk will not make sense anyway, 3600813a841SMartin KaFai Lau * so passing 'sock_ops->sk == req_sk' to the bpf prog is appropriate here. 3610813a841SMartin KaFai Lau */ 3620813a841SMartin KaFai Lau #define BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk) \ 3630813a841SMartin KaFai Lau ({ \ 3640813a841SMartin KaFai Lau int __ret = 0; \ 365a9ed15daSStanislav Fomichev if (cgroup_bpf_enabled(BPF_CGROUP_SOCK_OPS)) \ 3660813a841SMartin KaFai Lau __ret = __cgroup_bpf_run_filter_sock_ops(sk, \ 3670813a841SMartin KaFai Lau sock_ops, \ 3680813a841SMartin KaFai Lau BPF_CGROUP_SOCK_OPS); \ 3690813a841SMartin KaFai Lau __ret; \ 3700813a841SMartin KaFai Lau }) 3710813a841SMartin KaFai Lau 37240304b2aSLawrence Brakmo #define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \ 37340304b2aSLawrence Brakmo ({ \ 37440304b2aSLawrence Brakmo int __ret = 0; \ 375a9ed15daSStanislav Fomichev if (cgroup_bpf_enabled(BPF_CGROUP_SOCK_OPS) && (sock_ops)->sk) { \ 37640304b2aSLawrence Brakmo typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \ 377df39a9f1SWANG Cong if (__sk && sk_fullsock(__sk)) \ 37840304b2aSLawrence Brakmo __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \ 37940304b2aSLawrence Brakmo sock_ops, \ 38040304b2aSLawrence Brakmo BPF_CGROUP_SOCK_OPS); \ 38140304b2aSLawrence Brakmo } \ 38240304b2aSLawrence Brakmo __ret; \ 38340304b2aSLawrence Brakmo }) 384ebc614f6SRoman Gushchin 385ebc614f6SRoman Gushchin #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type, major, minor, access) \ 386ebc614f6SRoman Gushchin ({ \ 387ebc614f6SRoman Gushchin int __ret = 0; \ 388a9ed15daSStanislav Fomichev if (cgroup_bpf_enabled(BPF_CGROUP_DEVICE)) \ 389ebc614f6SRoman Gushchin __ret = __cgroup_bpf_check_dev_permission(type, major, minor, \ 390ebc614f6SRoman Gushchin access, \ 391ebc614f6SRoman Gushchin BPF_CGROUP_DEVICE); \ 392ebc614f6SRoman Gushchin \ 393ebc614f6SRoman Gushchin __ret; \ 394ebc614f6SRoman Gushchin }) 3957b146cebSAndrey Ignatov 3967b146cebSAndrey Ignatov 39732927393SChristoph Hellwig #define BPF_CGROUP_RUN_PROG_SYSCTL(head, table, write, buf, count, pos) \ 3987b146cebSAndrey Ignatov ({ \ 3997b146cebSAndrey Ignatov int __ret = 0; \ 400a9ed15daSStanislav Fomichev if (cgroup_bpf_enabled(BPF_CGROUP_SYSCTL)) \ 4017b146cebSAndrey Ignatov __ret = __cgroup_bpf_run_filter_sysctl(head, table, write, \ 40232927393SChristoph Hellwig buf, count, pos, \ 4037b146cebSAndrey Ignatov BPF_CGROUP_SYSCTL); \ 4047b146cebSAndrey Ignatov __ret; \ 4057b146cebSAndrey Ignatov }) 4067b146cebSAndrey Ignatov 4070d01da6aSStanislav Fomichev #define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \ 4080d01da6aSStanislav Fomichev kernel_optval) \ 4090d01da6aSStanislav Fomichev ({ \ 4100d01da6aSStanislav Fomichev int __ret = 0; \ 411a9ed15daSStanislav Fomichev if (cgroup_bpf_enabled(BPF_CGROUP_SETSOCKOPT)) \ 4120d01da6aSStanislav Fomichev __ret = __cgroup_bpf_run_filter_setsockopt(sock, level, \ 4130d01da6aSStanislav Fomichev optname, optval, \ 4140d01da6aSStanislav Fomichev optlen, \ 4150d01da6aSStanislav Fomichev kernel_optval); \ 4160d01da6aSStanislav Fomichev __ret; \ 4170d01da6aSStanislav Fomichev }) 4180d01da6aSStanislav Fomichev 4190d01da6aSStanislav Fomichev #define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) \ 4200d01da6aSStanislav Fomichev ({ \ 4210d01da6aSStanislav Fomichev int __ret = 0; \ 422a9ed15daSStanislav Fomichev if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT)) \ 4230d01da6aSStanislav Fomichev get_user(__ret, optlen); \ 4240d01da6aSStanislav Fomichev __ret; \ 4250d01da6aSStanislav Fomichev }) 4260d01da6aSStanislav Fomichev 4270d01da6aSStanislav Fomichev #define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, optlen, \ 4280d01da6aSStanislav Fomichev max_optlen, retval) \ 4290d01da6aSStanislav Fomichev ({ \ 4300d01da6aSStanislav Fomichev int __ret = retval; \ 431a9ed15daSStanislav Fomichev if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT)) \ 4329cacf81fSStanislav Fomichev if (!(sock)->sk_prot->bpf_bypass_getsockopt || \ 4339cacf81fSStanislav Fomichev !INDIRECT_CALL_INET_1((sock)->sk_prot->bpf_bypass_getsockopt, \ 4349cacf81fSStanislav Fomichev tcp_bpf_bypass_getsockopt, \ 4359cacf81fSStanislav Fomichev level, optname)) \ 4369cacf81fSStanislav Fomichev __ret = __cgroup_bpf_run_filter_getsockopt( \ 4379cacf81fSStanislav Fomichev sock, level, optname, optval, optlen, \ 4389cacf81fSStanislav Fomichev max_optlen, retval); \ 4399cacf81fSStanislav Fomichev __ret; \ 4409cacf81fSStanislav Fomichev }) 4419cacf81fSStanislav Fomichev 4429cacf81fSStanislav Fomichev #define BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sock, level, optname, optval, \ 4439cacf81fSStanislav Fomichev optlen, retval) \ 4449cacf81fSStanislav Fomichev ({ \ 4459cacf81fSStanislav Fomichev int __ret = retval; \ 446a9ed15daSStanislav Fomichev if (cgroup_bpf_enabled(BPF_CGROUP_GETSOCKOPT)) \ 4479cacf81fSStanislav Fomichev __ret = __cgroup_bpf_run_filter_getsockopt_kern( \ 4489cacf81fSStanislav Fomichev sock, level, optname, optval, optlen, retval); \ 4490d01da6aSStanislav Fomichev __ret; \ 4500d01da6aSStanislav Fomichev }) 4510d01da6aSStanislav Fomichev 452fdb5c453SSean Young int cgroup_bpf_prog_attach(const union bpf_attr *attr, 453fdb5c453SSean Young enum bpf_prog_type ptype, struct bpf_prog *prog); 454fdb5c453SSean Young int cgroup_bpf_prog_detach(const union bpf_attr *attr, 455fdb5c453SSean Young enum bpf_prog_type ptype); 456af6eea57SAndrii Nakryiko int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog); 457fdb5c453SSean Young int cgroup_bpf_prog_query(const union bpf_attr *attr, 458fdb5c453SSean Young union bpf_attr __user *uattr); 45930070984SDaniel Mack #else 46030070984SDaniel Mack 461fdb5c453SSean Young struct bpf_prog; 46230070984SDaniel Mack struct cgroup_bpf {}; 463324bda9eSAlexei Starovoitov static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; } 4644bfc0bb2SRoman Gushchin static inline void cgroup_bpf_offline(struct cgroup *cgrp) {} 46530070984SDaniel Mack 466fdb5c453SSean Young static inline int cgroup_bpf_prog_attach(const union bpf_attr *attr, 467fdb5c453SSean Young enum bpf_prog_type ptype, 468fdb5c453SSean Young struct bpf_prog *prog) 469fdb5c453SSean Young { 470fdb5c453SSean Young return -EINVAL; 471fdb5c453SSean Young } 472fdb5c453SSean Young 473fdb5c453SSean Young static inline int cgroup_bpf_prog_detach(const union bpf_attr *attr, 474fdb5c453SSean Young enum bpf_prog_type ptype) 475fdb5c453SSean Young { 476fdb5c453SSean Young return -EINVAL; 477fdb5c453SSean Young } 478fdb5c453SSean Young 479af6eea57SAndrii Nakryiko static inline int cgroup_bpf_link_attach(const union bpf_attr *attr, 480af6eea57SAndrii Nakryiko struct bpf_prog *prog) 481af6eea57SAndrii Nakryiko { 482af6eea57SAndrii Nakryiko return -EINVAL; 483af6eea57SAndrii Nakryiko } 484af6eea57SAndrii Nakryiko 485fdb5c453SSean Young static inline int cgroup_bpf_prog_query(const union bpf_attr *attr, 486fdb5c453SSean Young union bpf_attr __user *uattr) 487fdb5c453SSean Young { 488fdb5c453SSean Young return -EINVAL; 489fdb5c453SSean Young } 490fdb5c453SSean Young 491*b910eaaaSYonghong Song static inline int bpf_cgroup_storage_set( 492*b910eaaaSYonghong Song struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) { return 0; } 493*b910eaaaSYonghong Song static inline void bpf_cgroup_storage_unset(void) {} 494e4730423SDaniel Borkmann static inline int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux, 495de9cbbaaSRoman Gushchin struct bpf_map *map) { return 0; } 496de9cbbaaSRoman Gushchin static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc( 49771b91a50SBo YU struct bpf_prog *prog, enum bpf_cgroup_storage_type stype) { return NULL; } 498de9cbbaaSRoman Gushchin static inline void bpf_cgroup_storage_free( 499de9cbbaaSRoman Gushchin struct bpf_cgroup_storage *storage) {} 500b741f163SRoman Gushchin static inline int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key, 501b741f163SRoman Gushchin void *value) { 502b741f163SRoman Gushchin return 0; 503b741f163SRoman Gushchin } 504b741f163SRoman Gushchin static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, 505b741f163SRoman Gushchin void *key, void *value, u64 flags) { 506b741f163SRoman Gushchin return 0; 507b741f163SRoman Gushchin } 508de9cbbaaSRoman Gushchin 509a9ed15daSStanislav Fomichev #define cgroup_bpf_enabled(type) (0) 5101b66d253SDaniel Borkmann #define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx) ({ 0; }) 511d74bad4eSAndrey Ignatov #define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (0) 51230070984SDaniel Mack #define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk,skb) ({ 0; }) 51330070984SDaniel Mack #define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk,skb) ({ 0; }) 51461023658SDavid Ahern #define BPF_CGROUP_RUN_PROG_INET_SOCK(sk) ({ 0; }) 515f5836749SStanislav Fomichev #define BPF_CGROUP_RUN_PROG_INET_SOCK_RELEASE(sk) ({ 0; }) 51677241217SStanislav Fomichev #define BPF_CGROUP_RUN_PROG_INET_BIND_LOCK(sk, uaddr, type, flags) ({ 0; }) 517aac3fc32SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk) ({ 0; }) 518aac3fc32SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk) ({ 0; }) 519d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) ({ 0; }) 520d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr) ({ 0; }) 521d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr) ({ 0; }) 522d74bad4eSAndrey Ignatov #define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr) ({ 0; }) 5231cedee13SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, t_ctx) ({ 0; }) 5241cedee13SAndrey Ignatov #define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, t_ctx) ({ 0; }) 525983695faSDaniel Borkmann #define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr) ({ 0; }) 526983695faSDaniel Borkmann #define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr) ({ 0; }) 52740304b2aSLawrence Brakmo #define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; }) 528ebc614f6SRoman Gushchin #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type,major,minor,access) ({ 0; }) 52932927393SChristoph Hellwig #define BPF_CGROUP_RUN_PROG_SYSCTL(head,table,write,buf,count,pos) ({ 0; }) 5300d01da6aSStanislav Fomichev #define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) ({ 0; }) 5310d01da6aSStanislav Fomichev #define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \ 5320d01da6aSStanislav Fomichev optlen, max_optlen, retval) ({ retval; }) 5339cacf81fSStanislav Fomichev #define BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sock, level, optname, optval, \ 5349cacf81fSStanislav Fomichev optlen, retval) ({ retval; }) 5350d01da6aSStanislav Fomichev #define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \ 5360d01da6aSStanislav Fomichev kernel_optval) ({ 0; }) 53730070984SDaniel Mack 5388bad74f9SRoman Gushchin #define for_each_cgroup_storage_type(stype) for (; false; ) 5398bad74f9SRoman Gushchin 54030070984SDaniel Mack #endif /* CONFIG_CGROUP_BPF */ 54130070984SDaniel Mack 54230070984SDaniel Mack #endif /* _BPF_CGROUP_H */ 543