1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 2 * 3 * This program is free software; you can redistribute it and/or 4 * modify it under the terms of version 2 of the GNU General Public 5 * License as published by the Free Software Foundation. 6 */ 7 #ifndef _LINUX_BPF_H 8 #define _LINUX_BPF_H 1 9 10 #include <uapi/linux/bpf.h> 11 12 #include <linux/workqueue.h> 13 #include <linux/file.h> 14 #include <linux/percpu.h> 15 #include <linux/err.h> 16 #include <linux/rbtree_latch.h> 17 #include <linux/numa.h> 18 #include <linux/wait.h> 19 #include <linux/u64_stats_sync.h> 20 21 struct bpf_verifier_env; 22 struct perf_event; 23 struct bpf_prog; 24 struct bpf_map; 25 struct sock; 26 struct seq_file; 27 struct btf; 28 struct btf_type; 29 30 /* map is generic key/value storage optionally accesible by eBPF programs */ 31 struct bpf_map_ops { 32 /* funcs callable from userspace (via syscall) */ 33 int (*map_alloc_check)(union bpf_attr *attr); 34 struct bpf_map *(*map_alloc)(union bpf_attr *attr); 35 void (*map_release)(struct bpf_map *map, struct file *map_file); 36 void (*map_free)(struct bpf_map *map); 37 int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key); 38 void (*map_release_uref)(struct bpf_map *map); 39 40 /* funcs callable from userspace and from eBPF programs */ 41 void *(*map_lookup_elem)(struct bpf_map *map, void *key); 42 int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags); 43 int (*map_delete_elem)(struct bpf_map *map, void *key); 44 int (*map_push_elem)(struct bpf_map *map, void *value, u64 flags); 45 int (*map_pop_elem)(struct bpf_map *map, void *value); 46 int (*map_peek_elem)(struct bpf_map *map, void *value); 47 48 /* funcs called by prog_array and perf_event_array map */ 49 void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file, 50 int fd); 51 void (*map_fd_put_ptr)(void *ptr); 52 u32 (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf); 53 u32 (*map_fd_sys_lookup_elem)(void *ptr); 54 void (*map_seq_show_elem)(struct bpf_map *map, void *key, 55 struct seq_file *m); 56 int (*map_check_btf)(const struct bpf_map *map, 57 const struct btf *btf, 58 const struct btf_type *key_type, 59 const struct btf_type *value_type); 60 }; 61 62 struct bpf_map { 63 /* The first two cachelines with read-mostly members of which some 64 * are also accessed in fast-path (e.g. ops, max_entries). 65 */ 66 const struct bpf_map_ops *ops ____cacheline_aligned; 67 struct bpf_map *inner_map_meta; 68 #ifdef CONFIG_SECURITY 69 void *security; 70 #endif 71 enum bpf_map_type map_type; 72 u32 key_size; 73 u32 value_size; 74 u32 max_entries; 75 u32 map_flags; 76 int spin_lock_off; /* >=0 valid offset, <0 error */ 77 u32 id; 78 int numa_node; 79 u32 btf_key_type_id; 80 u32 btf_value_type_id; 81 struct btf *btf; 82 u32 pages; 83 bool unpriv_array; 84 /* 51 bytes hole */ 85 86 /* The 3rd and 4th cacheline with misc members to avoid false sharing 87 * particularly with refcounting. 88 */ 89 struct user_struct *user ____cacheline_aligned; 90 atomic_t refcnt; 91 atomic_t usercnt; 92 struct work_struct work; 93 char name[BPF_OBJ_NAME_LEN]; 94 }; 95 96 static inline bool map_value_has_spin_lock(const struct bpf_map *map) 97 { 98 return map->spin_lock_off >= 0; 99 } 100 101 static inline void check_and_init_map_lock(struct bpf_map *map, void *dst) 102 { 103 if (likely(!map_value_has_spin_lock(map))) 104 return; 105 *(struct bpf_spin_lock *)(dst + map->spin_lock_off) = 106 (struct bpf_spin_lock){}; 107 } 108 109 /* copy everything but bpf_spin_lock */ 110 static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) 111 { 112 if (unlikely(map_value_has_spin_lock(map))) { 113 u32 off = map->spin_lock_off; 114 115 memcpy(dst, src, off); 116 memcpy(dst + off + sizeof(struct bpf_spin_lock), 117 src + off + sizeof(struct bpf_spin_lock), 118 map->value_size - off - sizeof(struct bpf_spin_lock)); 119 } else { 120 memcpy(dst, src, map->value_size); 121 } 122 } 123 void copy_map_value_locked(struct bpf_map *map, void *dst, void *src, 124 bool lock_src); 125 126 struct bpf_offload_dev; 127 struct bpf_offloaded_map; 128 129 struct bpf_map_dev_ops { 130 int (*map_get_next_key)(struct bpf_offloaded_map *map, 131 void *key, void *next_key); 132 int (*map_lookup_elem)(struct bpf_offloaded_map *map, 133 void *key, void *value); 134 int (*map_update_elem)(struct bpf_offloaded_map *map, 135 void *key, void *value, u64 flags); 136 int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key); 137 }; 138 139 struct bpf_offloaded_map { 140 struct bpf_map map; 141 struct net_device *netdev; 142 const struct bpf_map_dev_ops *dev_ops; 143 void *dev_priv; 144 struct list_head offloads; 145 }; 146 147 static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map) 148 { 149 return container_of(map, struct bpf_offloaded_map, map); 150 } 151 152 static inline bool bpf_map_offload_neutral(const struct bpf_map *map) 153 { 154 return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 155 } 156 157 static inline bool bpf_map_support_seq_show(const struct bpf_map *map) 158 { 159 return map->btf && map->ops->map_seq_show_elem; 160 } 161 162 int map_check_no_btf(const struct bpf_map *map, 163 const struct btf *btf, 164 const struct btf_type *key_type, 165 const struct btf_type *value_type); 166 167 extern const struct bpf_map_ops bpf_map_offload_ops; 168 169 /* function argument constraints */ 170 enum bpf_arg_type { 171 ARG_DONTCARE = 0, /* unused argument in helper function */ 172 173 /* the following constraints used to prototype 174 * bpf_map_lookup/update/delete_elem() functions 175 */ 176 ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */ 177 ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */ 178 ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */ 179 ARG_PTR_TO_UNINIT_MAP_VALUE, /* pointer to valid memory used to store a map value */ 180 181 /* the following constraints used to prototype bpf_memcmp() and other 182 * functions that access data on eBPF program stack 183 */ 184 ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ 185 ARG_PTR_TO_MEM_OR_NULL, /* pointer to valid memory or NULL */ 186 ARG_PTR_TO_UNINIT_MEM, /* pointer to memory does not need to be initialized, 187 * helper function must fill all bytes or clear 188 * them in error case. 189 */ 190 191 ARG_CONST_SIZE, /* number of bytes accessed from memory */ 192 ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ 193 194 ARG_PTR_TO_CTX, /* pointer to context */ 195 ARG_ANYTHING, /* any (initialized) argument is ok */ 196 ARG_PTR_TO_SOCKET, /* pointer to bpf_sock */ 197 ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */ 198 ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */ 199 }; 200 201 /* type of values returned from helper functions */ 202 enum bpf_return_type { 203 RET_INTEGER, /* function returns integer */ 204 RET_VOID, /* function doesn't return anything */ 205 RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */ 206 RET_PTR_TO_MAP_VALUE_OR_NULL, /* returns a pointer to map elem value or NULL */ 207 RET_PTR_TO_SOCKET_OR_NULL, /* returns a pointer to a socket or NULL */ 208 RET_PTR_TO_TCP_SOCK_OR_NULL, /* returns a pointer to a tcp_sock or NULL */ 209 }; 210 211 /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs 212 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL 213 * instructions after verifying 214 */ 215 struct bpf_func_proto { 216 u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 217 bool gpl_only; 218 bool pkt_access; 219 enum bpf_return_type ret_type; 220 enum bpf_arg_type arg1_type; 221 enum bpf_arg_type arg2_type; 222 enum bpf_arg_type arg3_type; 223 enum bpf_arg_type arg4_type; 224 enum bpf_arg_type arg5_type; 225 }; 226 227 /* bpf_context is intentionally undefined structure. Pointer to bpf_context is 228 * the first argument to eBPF programs. 229 * For socket filters: 'struct bpf_context *' == 'struct sk_buff *' 230 */ 231 struct bpf_context; 232 233 enum bpf_access_type { 234 BPF_READ = 1, 235 BPF_WRITE = 2 236 }; 237 238 /* types of values stored in eBPF registers */ 239 /* Pointer types represent: 240 * pointer 241 * pointer + imm 242 * pointer + (u16) var 243 * pointer + (u16) var + imm 244 * if (range > 0) then [ptr, ptr + range - off) is safe to access 245 * if (id > 0) means that some 'var' was added 246 * if (off > 0) means that 'imm' was added 247 */ 248 enum bpf_reg_type { 249 NOT_INIT = 0, /* nothing was written into register */ 250 SCALAR_VALUE, /* reg doesn't contain a valid pointer */ 251 PTR_TO_CTX, /* reg points to bpf_context */ 252 CONST_PTR_TO_MAP, /* reg points to struct bpf_map */ 253 PTR_TO_MAP_VALUE, /* reg points to map element value */ 254 PTR_TO_MAP_VALUE_OR_NULL,/* points to map elem value or NULL */ 255 PTR_TO_STACK, /* reg == frame_pointer + offset */ 256 PTR_TO_PACKET_META, /* skb->data - meta_len */ 257 PTR_TO_PACKET, /* reg points to skb->data */ 258 PTR_TO_PACKET_END, /* skb->data + headlen */ 259 PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */ 260 PTR_TO_SOCKET, /* reg points to struct bpf_sock */ 261 PTR_TO_SOCKET_OR_NULL, /* reg points to struct bpf_sock or NULL */ 262 PTR_TO_SOCK_COMMON, /* reg points to sock_common */ 263 PTR_TO_SOCK_COMMON_OR_NULL, /* reg points to sock_common or NULL */ 264 PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */ 265 PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */ 266 }; 267 268 /* The information passed from prog-specific *_is_valid_access 269 * back to the verifier. 270 */ 271 struct bpf_insn_access_aux { 272 enum bpf_reg_type reg_type; 273 int ctx_field_size; 274 }; 275 276 static inline void 277 bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size) 278 { 279 aux->ctx_field_size = size; 280 } 281 282 struct bpf_prog_ops { 283 int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, 284 union bpf_attr __user *uattr); 285 }; 286 287 struct bpf_verifier_ops { 288 /* return eBPF function prototype for verification */ 289 const struct bpf_func_proto * 290 (*get_func_proto)(enum bpf_func_id func_id, 291 const struct bpf_prog *prog); 292 293 /* return true if 'size' wide access at offset 'off' within bpf_context 294 * with 'type' (read or write) is allowed 295 */ 296 bool (*is_valid_access)(int off, int size, enum bpf_access_type type, 297 const struct bpf_prog *prog, 298 struct bpf_insn_access_aux *info); 299 int (*gen_prologue)(struct bpf_insn *insn, bool direct_write, 300 const struct bpf_prog *prog); 301 int (*gen_ld_abs)(const struct bpf_insn *orig, 302 struct bpf_insn *insn_buf); 303 u32 (*convert_ctx_access)(enum bpf_access_type type, 304 const struct bpf_insn *src, 305 struct bpf_insn *dst, 306 struct bpf_prog *prog, u32 *target_size); 307 }; 308 309 struct bpf_prog_offload_ops { 310 /* verifier basic callbacks */ 311 int (*insn_hook)(struct bpf_verifier_env *env, 312 int insn_idx, int prev_insn_idx); 313 int (*finalize)(struct bpf_verifier_env *env); 314 /* verifier optimization callbacks (called after .finalize) */ 315 int (*replace_insn)(struct bpf_verifier_env *env, u32 off, 316 struct bpf_insn *insn); 317 int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt); 318 /* program management callbacks */ 319 int (*prepare)(struct bpf_prog *prog); 320 int (*translate)(struct bpf_prog *prog); 321 void (*destroy)(struct bpf_prog *prog); 322 }; 323 324 struct bpf_prog_offload { 325 struct bpf_prog *prog; 326 struct net_device *netdev; 327 struct bpf_offload_dev *offdev; 328 void *dev_priv; 329 struct list_head offloads; 330 bool dev_state; 331 bool opt_failed; 332 void *jited_image; 333 u32 jited_len; 334 }; 335 336 enum bpf_cgroup_storage_type { 337 BPF_CGROUP_STORAGE_SHARED, 338 BPF_CGROUP_STORAGE_PERCPU, 339 __BPF_CGROUP_STORAGE_MAX 340 }; 341 342 #define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX 343 344 struct bpf_prog_stats { 345 u64 cnt; 346 u64 nsecs; 347 struct u64_stats_sync syncp; 348 }; 349 350 struct bpf_prog_aux { 351 atomic_t refcnt; 352 u32 used_map_cnt; 353 u32 max_ctx_offset; 354 u32 max_pkt_offset; 355 u32 stack_depth; 356 u32 id; 357 u32 func_cnt; /* used by non-func prog as the number of func progs */ 358 u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */ 359 bool offload_requested; 360 struct bpf_prog **func; 361 void *jit_data; /* JIT specific data. arch dependent */ 362 struct latch_tree_node ksym_tnode; 363 struct list_head ksym_lnode; 364 const struct bpf_prog_ops *ops; 365 struct bpf_map **used_maps; 366 struct bpf_prog *prog; 367 struct user_struct *user; 368 u64 load_time; /* ns since boottime */ 369 struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 370 char name[BPF_OBJ_NAME_LEN]; 371 #ifdef CONFIG_SECURITY 372 void *security; 373 #endif 374 struct bpf_prog_offload *offload; 375 struct btf *btf; 376 struct bpf_func_info *func_info; 377 /* bpf_line_info loaded from userspace. linfo->insn_off 378 * has the xlated insn offset. 379 * Both the main and sub prog share the same linfo. 380 * The subprog can access its first linfo by 381 * using the linfo_idx. 382 */ 383 struct bpf_line_info *linfo; 384 /* jited_linfo is the jited addr of the linfo. It has a 385 * one to one mapping to linfo: 386 * jited_linfo[i] is the jited addr for the linfo[i]->insn_off. 387 * Both the main and sub prog share the same jited_linfo. 388 * The subprog can access its first jited_linfo by 389 * using the linfo_idx. 390 */ 391 void **jited_linfo; 392 u32 func_info_cnt; 393 u32 nr_linfo; 394 /* subprog can use linfo_idx to access its first linfo and 395 * jited_linfo. 396 * main prog always has linfo_idx == 0 397 */ 398 u32 linfo_idx; 399 struct bpf_prog_stats __percpu *stats; 400 union { 401 struct work_struct work; 402 struct rcu_head rcu; 403 }; 404 }; 405 406 struct bpf_array { 407 struct bpf_map map; 408 u32 elem_size; 409 u32 index_mask; 410 /* 'ownership' of prog_array is claimed by the first program that 411 * is going to use this map or by the first program which FD is stored 412 * in the map to make sure that all callers and callees have the same 413 * prog_type and JITed flag 414 */ 415 enum bpf_prog_type owner_prog_type; 416 bool owner_jited; 417 union { 418 char value[0] __aligned(8); 419 void *ptrs[0] __aligned(8); 420 void __percpu *pptrs[0] __aligned(8); 421 }; 422 }; 423 424 #define MAX_TAIL_CALL_CNT 32 425 426 struct bpf_event_entry { 427 struct perf_event *event; 428 struct file *perf_file; 429 struct file *map_file; 430 struct rcu_head rcu; 431 }; 432 433 bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp); 434 int bpf_prog_calc_tag(struct bpf_prog *fp); 435 436 const struct bpf_func_proto *bpf_get_trace_printk_proto(void); 437 438 typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src, 439 unsigned long off, unsigned long len); 440 typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type, 441 const struct bpf_insn *src, 442 struct bpf_insn *dst, 443 struct bpf_prog *prog, 444 u32 *target_size); 445 446 u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, 447 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy); 448 449 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 450 union bpf_attr __user *uattr); 451 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 452 union bpf_attr __user *uattr); 453 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 454 const union bpf_attr *kattr, 455 union bpf_attr __user *uattr); 456 457 /* an array of programs to be executed under rcu_lock. 458 * 459 * Typical usage: 460 * ret = BPF_PROG_RUN_ARRAY(&bpf_prog_array, ctx, BPF_PROG_RUN); 461 * 462 * the structure returned by bpf_prog_array_alloc() should be populated 463 * with program pointers and the last pointer must be NULL. 464 * The user has to keep refcnt on the program and make sure the program 465 * is removed from the array before bpf_prog_put(). 466 * The 'struct bpf_prog_array *' should only be replaced with xchg() 467 * since other cpus are walking the array of pointers in parallel. 468 */ 469 struct bpf_prog_array_item { 470 struct bpf_prog *prog; 471 struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 472 }; 473 474 struct bpf_prog_array { 475 struct rcu_head rcu; 476 struct bpf_prog_array_item items[0]; 477 }; 478 479 struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags); 480 void bpf_prog_array_free(struct bpf_prog_array __rcu *progs); 481 int bpf_prog_array_length(struct bpf_prog_array __rcu *progs); 482 int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *progs, 483 __u32 __user *prog_ids, u32 cnt); 484 485 void bpf_prog_array_delete_safe(struct bpf_prog_array __rcu *progs, 486 struct bpf_prog *old_prog); 487 int bpf_prog_array_copy_info(struct bpf_prog_array __rcu *array, 488 u32 *prog_ids, u32 request_cnt, 489 u32 *prog_cnt); 490 int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array, 491 struct bpf_prog *exclude_prog, 492 struct bpf_prog *include_prog, 493 struct bpf_prog_array **new_array); 494 495 #define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null) \ 496 ({ \ 497 struct bpf_prog_array_item *_item; \ 498 struct bpf_prog *_prog; \ 499 struct bpf_prog_array *_array; \ 500 u32 _ret = 1; \ 501 preempt_disable(); \ 502 rcu_read_lock(); \ 503 _array = rcu_dereference(array); \ 504 if (unlikely(check_non_null && !_array))\ 505 goto _out; \ 506 _item = &_array->items[0]; \ 507 while ((_prog = READ_ONCE(_item->prog))) { \ 508 bpf_cgroup_storage_set(_item->cgroup_storage); \ 509 _ret &= func(_prog, ctx); \ 510 _item++; \ 511 } \ 512 _out: \ 513 rcu_read_unlock(); \ 514 preempt_enable_no_resched(); \ 515 _ret; \ 516 }) 517 518 #define BPF_PROG_RUN_ARRAY(array, ctx, func) \ 519 __BPF_PROG_RUN_ARRAY(array, ctx, func, false) 520 521 #define BPF_PROG_RUN_ARRAY_CHECK(array, ctx, func) \ 522 __BPF_PROG_RUN_ARRAY(array, ctx, func, true) 523 524 #ifdef CONFIG_BPF_SYSCALL 525 DECLARE_PER_CPU(int, bpf_prog_active); 526 527 extern const struct file_operations bpf_map_fops; 528 extern const struct file_operations bpf_prog_fops; 529 530 #define BPF_PROG_TYPE(_id, _name) \ 531 extern const struct bpf_prog_ops _name ## _prog_ops; \ 532 extern const struct bpf_verifier_ops _name ## _verifier_ops; 533 #define BPF_MAP_TYPE(_id, _ops) \ 534 extern const struct bpf_map_ops _ops; 535 #include <linux/bpf_types.h> 536 #undef BPF_PROG_TYPE 537 #undef BPF_MAP_TYPE 538 539 extern const struct bpf_prog_ops bpf_offload_prog_ops; 540 extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops; 541 extern const struct bpf_verifier_ops xdp_analyzer_ops; 542 543 struct bpf_prog *bpf_prog_get(u32 ufd); 544 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 545 bool attach_drv); 546 struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog, int i); 547 void bpf_prog_sub(struct bpf_prog *prog, int i); 548 struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog); 549 struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog); 550 void bpf_prog_put(struct bpf_prog *prog); 551 int __bpf_prog_charge(struct user_struct *user, u32 pages); 552 void __bpf_prog_uncharge(struct user_struct *user, u32 pages); 553 554 void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock); 555 void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock); 556 557 struct bpf_map *bpf_map_get_with_uref(u32 ufd); 558 struct bpf_map *__bpf_map_get(struct fd f); 559 struct bpf_map * __must_check bpf_map_inc(struct bpf_map *map, bool uref); 560 void bpf_map_put_with_uref(struct bpf_map *map); 561 void bpf_map_put(struct bpf_map *map); 562 int bpf_map_precharge_memlock(u32 pages); 563 int bpf_map_charge_memlock(struct bpf_map *map, u32 pages); 564 void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages); 565 void *bpf_map_area_alloc(size_t size, int numa_node); 566 void bpf_map_area_free(void *base); 567 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr); 568 569 extern int sysctl_unprivileged_bpf_disabled; 570 extern int sysctl_bpf_stats_enabled; 571 572 int bpf_map_new_fd(struct bpf_map *map, int flags); 573 int bpf_prog_new_fd(struct bpf_prog *prog); 574 575 int bpf_obj_pin_user(u32 ufd, const char __user *pathname); 576 int bpf_obj_get_user(const char __user *pathname, int flags); 577 578 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value); 579 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value); 580 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, 581 u64 flags); 582 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, 583 u64 flags); 584 585 int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value); 586 587 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, 588 void *key, void *value, u64 map_flags); 589 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); 590 int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, 591 void *key, void *value, u64 map_flags); 592 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); 593 594 int bpf_get_file_flag(int flags); 595 int bpf_check_uarg_tail_zero(void __user *uaddr, size_t expected_size, 596 size_t actual_size); 597 598 /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and 599 * forced to use 'long' read/writes to try to atomically copy long counters. 600 * Best-effort only. No barriers here, since it _will_ race with concurrent 601 * updates from BPF programs. Called from bpf syscall and mostly used with 602 * size 8 or 16 bytes, so ask compiler to inline it. 603 */ 604 static inline void bpf_long_memcpy(void *dst, const void *src, u32 size) 605 { 606 const long *lsrc = src; 607 long *ldst = dst; 608 609 size /= sizeof(long); 610 while (size--) 611 *ldst++ = *lsrc++; 612 } 613 614 /* verify correctness of eBPF program */ 615 int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, 616 union bpf_attr __user *uattr); 617 void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); 618 619 /* Map specifics */ 620 struct xdp_buff; 621 struct sk_buff; 622 623 struct bpf_dtab_netdev *__dev_map_lookup_elem(struct bpf_map *map, u32 key); 624 void __dev_map_insert_ctx(struct bpf_map *map, u32 index); 625 void __dev_map_flush(struct bpf_map *map); 626 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, 627 struct net_device *dev_rx); 628 int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, 629 struct bpf_prog *xdp_prog); 630 631 struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key); 632 void __cpu_map_insert_ctx(struct bpf_map *map, u32 index); 633 void __cpu_map_flush(struct bpf_map *map); 634 int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp, 635 struct net_device *dev_rx); 636 637 /* Return map's numa specified by userspace */ 638 static inline int bpf_map_attr_numa_node(const union bpf_attr *attr) 639 { 640 return (attr->map_flags & BPF_F_NUMA_NODE) ? 641 attr->numa_node : NUMA_NO_NODE; 642 } 643 644 struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type); 645 int array_map_alloc_check(union bpf_attr *attr); 646 647 #else /* !CONFIG_BPF_SYSCALL */ 648 static inline struct bpf_prog *bpf_prog_get(u32 ufd) 649 { 650 return ERR_PTR(-EOPNOTSUPP); 651 } 652 653 static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, 654 enum bpf_prog_type type, 655 bool attach_drv) 656 { 657 return ERR_PTR(-EOPNOTSUPP); 658 } 659 660 static inline struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog, 661 int i) 662 { 663 return ERR_PTR(-EOPNOTSUPP); 664 } 665 666 static inline void bpf_prog_sub(struct bpf_prog *prog, int i) 667 { 668 } 669 670 static inline void bpf_prog_put(struct bpf_prog *prog) 671 { 672 } 673 674 static inline struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog) 675 { 676 return ERR_PTR(-EOPNOTSUPP); 677 } 678 679 static inline struct bpf_prog *__must_check 680 bpf_prog_inc_not_zero(struct bpf_prog *prog) 681 { 682 return ERR_PTR(-EOPNOTSUPP); 683 } 684 685 static inline int __bpf_prog_charge(struct user_struct *user, u32 pages) 686 { 687 return 0; 688 } 689 690 static inline void __bpf_prog_uncharge(struct user_struct *user, u32 pages) 691 { 692 } 693 694 static inline int bpf_obj_get_user(const char __user *pathname, int flags) 695 { 696 return -EOPNOTSUPP; 697 } 698 699 static inline struct net_device *__dev_map_lookup_elem(struct bpf_map *map, 700 u32 key) 701 { 702 return NULL; 703 } 704 705 static inline void __dev_map_insert_ctx(struct bpf_map *map, u32 index) 706 { 707 } 708 709 static inline void __dev_map_flush(struct bpf_map *map) 710 { 711 } 712 713 struct xdp_buff; 714 struct bpf_dtab_netdev; 715 716 static inline 717 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, 718 struct net_device *dev_rx) 719 { 720 return 0; 721 } 722 723 struct sk_buff; 724 725 static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, 726 struct sk_buff *skb, 727 struct bpf_prog *xdp_prog) 728 { 729 return 0; 730 } 731 732 static inline 733 struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key) 734 { 735 return NULL; 736 } 737 738 static inline void __cpu_map_insert_ctx(struct bpf_map *map, u32 index) 739 { 740 } 741 742 static inline void __cpu_map_flush(struct bpf_map *map) 743 { 744 } 745 746 static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, 747 struct xdp_buff *xdp, 748 struct net_device *dev_rx) 749 { 750 return 0; 751 } 752 753 static inline struct bpf_prog *bpf_prog_get_type_path(const char *name, 754 enum bpf_prog_type type) 755 { 756 return ERR_PTR(-EOPNOTSUPP); 757 } 758 #endif /* CONFIG_BPF_SYSCALL */ 759 760 static inline struct bpf_prog *bpf_prog_get_type(u32 ufd, 761 enum bpf_prog_type type) 762 { 763 return bpf_prog_get_type_dev(ufd, type, false); 764 } 765 766 bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool); 767 768 int bpf_prog_offload_compile(struct bpf_prog *prog); 769 void bpf_prog_offload_destroy(struct bpf_prog *prog); 770 int bpf_prog_offload_info_fill(struct bpf_prog_info *info, 771 struct bpf_prog *prog); 772 773 int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map); 774 775 int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value); 776 int bpf_map_offload_update_elem(struct bpf_map *map, 777 void *key, void *value, u64 flags); 778 int bpf_map_offload_delete_elem(struct bpf_map *map, void *key); 779 int bpf_map_offload_get_next_key(struct bpf_map *map, 780 void *key, void *next_key); 781 782 bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map); 783 784 struct bpf_offload_dev * 785 bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv); 786 void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev); 787 void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev); 788 int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev, 789 struct net_device *netdev); 790 void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev, 791 struct net_device *netdev); 792 bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev); 793 794 #if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL) 795 int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr); 796 797 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) 798 { 799 return aux->offload_requested; 800 } 801 802 static inline bool bpf_map_is_dev_bound(struct bpf_map *map) 803 { 804 return unlikely(map->ops == &bpf_map_offload_ops); 805 } 806 807 struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr); 808 void bpf_map_offload_map_free(struct bpf_map *map); 809 #else 810 static inline int bpf_prog_offload_init(struct bpf_prog *prog, 811 union bpf_attr *attr) 812 { 813 return -EOPNOTSUPP; 814 } 815 816 static inline bool bpf_prog_is_dev_bound(struct bpf_prog_aux *aux) 817 { 818 return false; 819 } 820 821 static inline bool bpf_map_is_dev_bound(struct bpf_map *map) 822 { 823 return false; 824 } 825 826 static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) 827 { 828 return ERR_PTR(-EOPNOTSUPP); 829 } 830 831 static inline void bpf_map_offload_map_free(struct bpf_map *map) 832 { 833 } 834 #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */ 835 836 #if defined(CONFIG_BPF_STREAM_PARSER) 837 int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, u32 which); 838 int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog); 839 #else 840 static inline int sock_map_prog_update(struct bpf_map *map, 841 struct bpf_prog *prog, u32 which) 842 { 843 return -EOPNOTSUPP; 844 } 845 846 static inline int sock_map_get_from_fd(const union bpf_attr *attr, 847 struct bpf_prog *prog) 848 { 849 return -EINVAL; 850 } 851 #endif 852 853 #if defined(CONFIG_XDP_SOCKETS) 854 struct xdp_sock; 855 struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, u32 key); 856 int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp, 857 struct xdp_sock *xs); 858 void __xsk_map_flush(struct bpf_map *map); 859 #else 860 struct xdp_sock; 861 static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, 862 u32 key) 863 { 864 return NULL; 865 } 866 867 static inline int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp, 868 struct xdp_sock *xs) 869 { 870 return -EOPNOTSUPP; 871 } 872 873 static inline void __xsk_map_flush(struct bpf_map *map) 874 { 875 } 876 #endif 877 878 #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) 879 void bpf_sk_reuseport_detach(struct sock *sk); 880 int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, 881 void *value); 882 int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key, 883 void *value, u64 map_flags); 884 #else 885 static inline void bpf_sk_reuseport_detach(struct sock *sk) 886 { 887 } 888 889 #ifdef CONFIG_BPF_SYSCALL 890 static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, 891 void *key, void *value) 892 { 893 return -EOPNOTSUPP; 894 } 895 896 static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, 897 void *key, void *value, 898 u64 map_flags) 899 { 900 return -EOPNOTSUPP; 901 } 902 #endif /* CONFIG_BPF_SYSCALL */ 903 #endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */ 904 905 /* verifier prototypes for helper functions called from eBPF programs */ 906 extern const struct bpf_func_proto bpf_map_lookup_elem_proto; 907 extern const struct bpf_func_proto bpf_map_update_elem_proto; 908 extern const struct bpf_func_proto bpf_map_delete_elem_proto; 909 extern const struct bpf_func_proto bpf_map_push_elem_proto; 910 extern const struct bpf_func_proto bpf_map_pop_elem_proto; 911 extern const struct bpf_func_proto bpf_map_peek_elem_proto; 912 913 extern const struct bpf_func_proto bpf_get_prandom_u32_proto; 914 extern const struct bpf_func_proto bpf_get_smp_processor_id_proto; 915 extern const struct bpf_func_proto bpf_get_numa_node_id_proto; 916 extern const struct bpf_func_proto bpf_tail_call_proto; 917 extern const struct bpf_func_proto bpf_ktime_get_ns_proto; 918 extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto; 919 extern const struct bpf_func_proto bpf_get_current_uid_gid_proto; 920 extern const struct bpf_func_proto bpf_get_current_comm_proto; 921 extern const struct bpf_func_proto bpf_get_stackid_proto; 922 extern const struct bpf_func_proto bpf_get_stack_proto; 923 extern const struct bpf_func_proto bpf_sock_map_update_proto; 924 extern const struct bpf_func_proto bpf_sock_hash_update_proto; 925 extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto; 926 extern const struct bpf_func_proto bpf_msg_redirect_hash_proto; 927 extern const struct bpf_func_proto bpf_msg_redirect_map_proto; 928 extern const struct bpf_func_proto bpf_sk_redirect_hash_proto; 929 extern const struct bpf_func_proto bpf_sk_redirect_map_proto; 930 extern const struct bpf_func_proto bpf_spin_lock_proto; 931 extern const struct bpf_func_proto bpf_spin_unlock_proto; 932 extern const struct bpf_func_proto bpf_get_local_storage_proto; 933 934 /* Shared helpers among cBPF and eBPF. */ 935 void bpf_user_rnd_init_once(void); 936 u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 937 938 #if defined(CONFIG_NET) 939 bool bpf_sock_common_is_valid_access(int off, int size, 940 enum bpf_access_type type, 941 struct bpf_insn_access_aux *info); 942 bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type, 943 struct bpf_insn_access_aux *info); 944 u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, 945 const struct bpf_insn *si, 946 struct bpf_insn *insn_buf, 947 struct bpf_prog *prog, 948 u32 *target_size); 949 #else 950 static inline bool bpf_sock_common_is_valid_access(int off, int size, 951 enum bpf_access_type type, 952 struct bpf_insn_access_aux *info) 953 { 954 return false; 955 } 956 static inline bool bpf_sock_is_valid_access(int off, int size, 957 enum bpf_access_type type, 958 struct bpf_insn_access_aux *info) 959 { 960 return false; 961 } 962 static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, 963 const struct bpf_insn *si, 964 struct bpf_insn *insn_buf, 965 struct bpf_prog *prog, 966 u32 *target_size) 967 { 968 return 0; 969 } 970 #endif 971 972 #ifdef CONFIG_INET 973 bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type, 974 struct bpf_insn_access_aux *info); 975 976 u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, 977 const struct bpf_insn *si, 978 struct bpf_insn *insn_buf, 979 struct bpf_prog *prog, 980 u32 *target_size); 981 #else 982 static inline bool bpf_tcp_sock_is_valid_access(int off, int size, 983 enum bpf_access_type type, 984 struct bpf_insn_access_aux *info) 985 { 986 return false; 987 } 988 989 static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, 990 const struct bpf_insn *si, 991 struct bpf_insn *insn_buf, 992 struct bpf_prog *prog, 993 u32 *target_size) 994 { 995 return 0; 996 } 997 #endif /* CONFIG_INET */ 998 999 #endif /* _LINUX_BPF_H */ 1000