1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 2 * 3 * This program is free software; you can redistribute it and/or 4 * modify it under the terms of version 2 of the GNU General Public 5 * License as published by the Free Software Foundation. 6 */ 7 #ifndef _LINUX_BPF_H 8 #define _LINUX_BPF_H 1 9 10 #include <uapi/linux/bpf.h> 11 12 #include <linux/workqueue.h> 13 #include <linux/file.h> 14 #include <linux/percpu.h> 15 #include <linux/err.h> 16 #include <linux/rbtree_latch.h> 17 #include <linux/numa.h> 18 #include <linux/wait.h> 19 #include <linux/u64_stats_sync.h> 20 21 struct bpf_verifier_env; 22 struct perf_event; 23 struct bpf_prog; 24 struct bpf_map; 25 struct sock; 26 struct seq_file; 27 struct btf; 28 struct btf_type; 29 30 /* map is generic key/value storage optionally accesible by eBPF programs */ 31 struct bpf_map_ops { 32 /* funcs callable from userspace (via syscall) */ 33 int (*map_alloc_check)(union bpf_attr *attr); 34 struct bpf_map *(*map_alloc)(union bpf_attr *attr); 35 void (*map_release)(struct bpf_map *map, struct file *map_file); 36 void (*map_free)(struct bpf_map *map); 37 int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key); 38 void (*map_release_uref)(struct bpf_map *map); 39 40 /* funcs callable from userspace and from eBPF programs */ 41 void *(*map_lookup_elem)(struct bpf_map *map, void *key); 42 int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags); 43 int (*map_delete_elem)(struct bpf_map *map, void *key); 44 int (*map_push_elem)(struct bpf_map *map, void *value, u64 flags); 45 int (*map_pop_elem)(struct bpf_map *map, void *value); 46 int (*map_peek_elem)(struct bpf_map *map, void *value); 47 48 /* funcs called by prog_array and perf_event_array map */ 49 void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file, 50 int fd); 51 void (*map_fd_put_ptr)(void *ptr); 52 u32 (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf); 53 u32 (*map_fd_sys_lookup_elem)(void *ptr); 54 void (*map_seq_show_elem)(struct bpf_map *map, void *key, 55 struct seq_file *m); 56 int (*map_check_btf)(const struct bpf_map *map, 57 const struct btf *btf, 58 const struct btf_type *key_type, 59 const struct btf_type *value_type); 60 }; 61 62 struct bpf_map { 63 /* The first two cachelines with read-mostly members of which some 64 * are also accessed in fast-path (e.g. ops, max_entries). 65 */ 66 const struct bpf_map_ops *ops ____cacheline_aligned; 67 struct bpf_map *inner_map_meta; 68 #ifdef CONFIG_SECURITY 69 void *security; 70 #endif 71 enum bpf_map_type map_type; 72 u32 key_size; 73 u32 value_size; 74 u32 max_entries; 75 u32 map_flags; 76 int spin_lock_off; /* >=0 valid offset, <0 error */ 77 u32 id; 78 int numa_node; 79 u32 btf_key_type_id; 80 u32 btf_value_type_id; 81 struct btf *btf; 82 u32 pages; 83 bool unpriv_array; 84 /* 51 bytes hole */ 85 86 /* The 3rd and 4th cacheline with misc members to avoid false sharing 87 * particularly with refcounting. 88 */ 89 struct user_struct *user ____cacheline_aligned; 90 atomic_t refcnt; 91 atomic_t usercnt; 92 struct work_struct work; 93 char name[BPF_OBJ_NAME_LEN]; 94 }; 95 96 static inline bool map_value_has_spin_lock(const struct bpf_map *map) 97 { 98 return map->spin_lock_off >= 0; 99 } 100 101 static inline void check_and_init_map_lock(struct bpf_map *map, void *dst) 102 { 103 if (likely(!map_value_has_spin_lock(map))) 104 return; 105 *(struct bpf_spin_lock *)(dst + map->spin_lock_off) = 106 (struct bpf_spin_lock){}; 107 } 108 109 /* copy everything but bpf_spin_lock */ 110 static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) 111 { 112 if (unlikely(map_value_has_spin_lock(map))) { 113 u32 off = map->spin_lock_off; 114 115 memcpy(dst, src, off); 116 memcpy(dst + off + sizeof(struct bpf_spin_lock), 117 src + off + sizeof(struct bpf_spin_lock), 118 map->value_size - off - sizeof(struct bpf_spin_lock)); 119 } else { 120 memcpy(dst, src, map->value_size); 121 } 122 } 123 void copy_map_value_locked(struct bpf_map *map, void *dst, void *src, 124 bool lock_src); 125 126 struct bpf_offload_dev; 127 struct bpf_offloaded_map; 128 129 struct bpf_map_dev_ops { 130 int (*map_get_next_key)(struct bpf_offloaded_map *map, 131 void *key, void *next_key); 132 int (*map_lookup_elem)(struct bpf_offloaded_map *map, 133 void *key, void *value); 134 int (*map_update_elem)(struct bpf_offloaded_map *map, 135 void *key, void *value, u64 flags); 136 int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key); 137 }; 138 139 struct bpf_offloaded_map { 140 struct bpf_map map; 141 struct net_device *netdev; 142 const struct bpf_map_dev_ops *dev_ops; 143 void *dev_priv; 144 struct list_head offloads; 145 }; 146 147 static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map) 148 { 149 return container_of(map, struct bpf_offloaded_map, map); 150 } 151 152 static inline bool bpf_map_offload_neutral(const struct bpf_map *map) 153 { 154 return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 155 } 156 157 static inline bool bpf_map_support_seq_show(const struct bpf_map *map) 158 { 159 return map->btf && map->ops->map_seq_show_elem; 160 } 161 162 int map_check_no_btf(const struct bpf_map *map, 163 const struct btf *btf, 164 const struct btf_type *key_type, 165 const struct btf_type *value_type); 166 167 extern const struct bpf_map_ops bpf_map_offload_ops; 168 169 /* function argument constraints */ 170 enum bpf_arg_type { 171 ARG_DONTCARE = 0, /* unused argument in helper function */ 172 173 /* the following constraints used to prototype 174 * bpf_map_lookup/update/delete_elem() functions 175 */ 176 ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */ 177 ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */ 178 ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */ 179 ARG_PTR_TO_UNINIT_MAP_VALUE, /* pointer to valid memory used to store a map value */ 180 181 /* the following constraints used to prototype bpf_memcmp() and other 182 * functions that access data on eBPF program stack 183 */ 184 ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ 185 ARG_PTR_TO_MEM_OR_NULL, /* pointer to valid memory or NULL */ 186 ARG_PTR_TO_UNINIT_MEM, /* pointer to memory does not need to be initialized, 187 * helper function must fill all bytes or clear 188 * them in error case. 189 */ 190 191 ARG_CONST_SIZE, /* number of bytes accessed from memory */ 192 ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ 193 194 ARG_PTR_TO_CTX, /* pointer to context */ 195 ARG_ANYTHING, /* any (initialized) argument is ok */ 196 ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */ 197 ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */ 198 }; 199 200 /* type of values returned from helper functions */ 201 enum bpf_return_type { 202 RET_INTEGER, /* function returns integer */ 203 RET_VOID, /* function doesn't return anything */ 204 RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */ 205 RET_PTR_TO_MAP_VALUE_OR_NULL, /* returns a pointer to map elem value or NULL */ 206 RET_PTR_TO_SOCKET_OR_NULL, /* returns a pointer to a socket or NULL */ 207 RET_PTR_TO_TCP_SOCK_OR_NULL, /* returns a pointer to a tcp_sock or NULL */ 208 }; 209 210 /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs 211 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL 212 * instructions after verifying 213 */ 214 struct bpf_func_proto { 215 u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 216 bool gpl_only; 217 bool pkt_access; 218 enum bpf_return_type ret_type; 219 enum bpf_arg_type arg1_type; 220 enum bpf_arg_type arg2_type; 221 enum bpf_arg_type arg3_type; 222 enum bpf_arg_type arg4_type; 223 enum bpf_arg_type arg5_type; 224 }; 225 226 /* bpf_context is intentionally undefined structure. Pointer to bpf_context is 227 * the first argument to eBPF programs. 228 * For socket filters: 'struct bpf_context *' == 'struct sk_buff *' 229 */ 230 struct bpf_context; 231 232 enum bpf_access_type { 233 BPF_READ = 1, 234 BPF_WRITE = 2 235 }; 236 237 /* types of values stored in eBPF registers */ 238 /* Pointer types represent: 239 * pointer 240 * pointer + imm 241 * pointer + (u16) var 242 * pointer + (u16) var + imm 243 * if (range > 0) then [ptr, ptr + range - off) is safe to access 244 * if (id > 0) means that some 'var' was added 245 * if (off > 0) means that 'imm' was added 246 */ 247 enum bpf_reg_type { 248 NOT_INIT = 0, /* nothing was written into register */ 249 SCALAR_VALUE, /* reg doesn't contain a valid pointer */ 250 PTR_TO_CTX, /* reg points to bpf_context */ 251 CONST_PTR_TO_MAP, /* reg points to struct bpf_map */ 252 PTR_TO_MAP_VALUE, /* reg points to map element value */ 253 PTR_TO_MAP_VALUE_OR_NULL,/* points to map elem value or NULL */ 254 PTR_TO_STACK, /* reg == frame_pointer + offset */ 255 PTR_TO_PACKET_META, /* skb->data - meta_len */ 256 PTR_TO_PACKET, /* reg points to skb->data */ 257 PTR_TO_PACKET_END, /* skb->data + headlen */ 258 PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */ 259 PTR_TO_SOCKET, /* reg points to struct bpf_sock */ 260 PTR_TO_SOCKET_OR_NULL, /* reg points to struct bpf_sock or NULL */ 261 PTR_TO_SOCK_COMMON, /* reg points to sock_common */ 262 PTR_TO_SOCK_COMMON_OR_NULL, /* reg points to sock_common or NULL */ 263 PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */ 264 PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */ 265 }; 266 267 /* The information passed from prog-specific *_is_valid_access 268 * back to the verifier. 269 */ 270 struct bpf_insn_access_aux { 271 enum bpf_reg_type reg_type; 272 int ctx_field_size; 273 }; 274 275 static inline void 276 bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size) 277 { 278 aux->ctx_field_size = size; 279 } 280 281 struct bpf_prog_ops { 282 int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, 283 union bpf_attr __user *uattr); 284 }; 285 286 struct bpf_verifier_ops { 287 /* return eBPF function prototype for verification */ 288 const struct bpf_func_proto * 289 (*get_func_proto)(enum bpf_func_id func_id, 290 const struct bpf_prog *prog); 291 292 /* return true if 'size' wide access at offset 'off' within bpf_context 293 * with 'type' (read or write) is allowed 294 */ 295 bool (*is_valid_access)(int off, int size, enum bpf_access_type type, 296 const struct bpf_prog *prog, 297 struct bpf_insn_access_aux *info); 298 int (*gen_prologue)(struct bpf_insn *insn, bool direct_write, 299 const struct bpf_prog *prog); 300 int (*gen_ld_abs)(const struct bpf_insn *orig, 301 struct bpf_insn *insn_buf); 302 u32 (*convert_ctx_access)(enum bpf_access_type type, 303 const struct bpf_insn *src, 304 struct bpf_insn *dst, 305 struct bpf_prog *prog, u32 *target_size); 306 }; 307 308 struct bpf_prog_offload_ops { 309 /* verifier basic callbacks */ 310 int (*insn_hook)(struct bpf_verifier_env *env, 311 int insn_idx, int prev_insn_idx); 312 int (*finalize)(struct bpf_verifier_env *env); 313 /* verifier optimization callbacks (called after .finalize) */ 314 int (*replace_insn)(struct bpf_verifier_env *env, u32 off, 315 struct bpf_insn *insn); 316 int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt); 317 /* program management callbacks */ 318 int (*prepare)(struct bpf_prog *prog); 319 int (*translate)(struct bpf_prog *prog); 320 void (*destroy)(struct bpf_prog *prog); 321 }; 322 323 struct bpf_prog_offload { 324 struct bpf_prog *prog; 325 struct net_device *netdev; 326 struct bpf_offload_dev *offdev; 327 void *dev_priv; 328 struct list_head offloads; 329 bool dev_state; 330 bool opt_failed; 331 void *jited_image; 332 u32 jited_len; 333 }; 334 335 enum bpf_cgroup_storage_type { 336 BPF_CGROUP_STORAGE_SHARED, 337 BPF_CGROUP_STORAGE_PERCPU, 338 __BPF_CGROUP_STORAGE_MAX 339 }; 340 341 #define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX 342 343 struct bpf_prog_stats { 344 u64 cnt; 345 u64 nsecs; 346 struct u64_stats_sync syncp; 347 }; 348 349 struct bpf_prog_aux { 350 atomic_t refcnt; 351 u32 used_map_cnt; 352 u32 max_ctx_offset; 353 u32 max_pkt_offset; 354 u32 stack_depth; 355 u32 id; 356 u32 func_cnt; /* used by non-func prog as the number of func progs */ 357 u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */ 358 bool offload_requested; 359 struct bpf_prog **func; 360 void *jit_data; /* JIT specific data. arch dependent */ 361 struct latch_tree_node ksym_tnode; 362 struct list_head ksym_lnode; 363 const struct bpf_prog_ops *ops; 364 struct bpf_map **used_maps; 365 struct bpf_prog *prog; 366 struct user_struct *user; 367 u64 load_time; /* ns since boottime */ 368 struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 369 char name[BPF_OBJ_NAME_LEN]; 370 #ifdef CONFIG_SECURITY 371 void *security; 372 #endif 373 struct bpf_prog_offload *offload; 374 struct btf *btf; 375 struct bpf_func_info *func_info; 376 /* bpf_line_info loaded from userspace. linfo->insn_off 377 * has the xlated insn offset. 378 * Both the main and sub prog share the same linfo. 379 * The subprog can access its first linfo by 380 * using the linfo_idx. 381 */ 382 struct bpf_line_info *linfo; 383 /* jited_linfo is the jited addr of the linfo. It has a 384 * one to one mapping to linfo: 385 * jited_linfo[i] is the jited addr for the linfo[i]->insn_off. 386 * Both the main and sub prog share the same jited_linfo. 387 * The subprog can access its first jited_linfo by 388 * using the linfo_idx. 389 */ 390 void **jited_linfo; 391 u32 func_info_cnt; 392 u32 nr_linfo; 393 /* subprog can use linfo_idx to access its first linfo and 394 * jited_linfo. 395 * main prog always has linfo_idx == 0 396 */ 397 u32 linfo_idx; 398 struct bpf_prog_stats __percpu *stats; 399 union { 400 struct work_struct work; 401 struct rcu_head rcu; 402 }; 403 }; 404 405 struct bpf_array { 406 struct bpf_map map; 407 u32 elem_size; 408 u32 index_mask; 409 /* 'ownership' of prog_array is claimed by the first program that 410 * is going to use this map or by the first program which FD is stored 411 * in the map to make sure that all callers and callees have the same 412 * prog_type and JITed flag 413 */ 414 enum bpf_prog_type owner_prog_type; 415 bool owner_jited; 416 union { 417 char value[0] __aligned(8); 418 void *ptrs[0] __aligned(8); 419 void __percpu *pptrs[0] __aligned(8); 420 }; 421 }; 422 423 #define MAX_TAIL_CALL_CNT 32 424 425 struct bpf_event_entry { 426 struct perf_event *event; 427 struct file *perf_file; 428 struct file *map_file; 429 struct rcu_head rcu; 430 }; 431 432 bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp); 433 int bpf_prog_calc_tag(struct bpf_prog *fp); 434 435 const struct bpf_func_proto *bpf_get_trace_printk_proto(void); 436 437 typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src, 438 unsigned long off, unsigned long len); 439 typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type, 440 const struct bpf_insn *src, 441 struct bpf_insn *dst, 442 struct bpf_prog *prog, 443 u32 *target_size); 444 445 u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, 446 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy); 447 448 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 449 union bpf_attr __user *uattr); 450 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 451 union bpf_attr __user *uattr); 452 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 453 const union bpf_attr *kattr, 454 union bpf_attr __user *uattr); 455 456 /* an array of programs to be executed under rcu_lock. 457 * 458 * Typical usage: 459 * ret = BPF_PROG_RUN_ARRAY(&bpf_prog_array, ctx, BPF_PROG_RUN); 460 * 461 * the structure returned by bpf_prog_array_alloc() should be populated 462 * with program pointers and the last pointer must be NULL. 463 * The user has to keep refcnt on the program and make sure the program 464 * is removed from the array before bpf_prog_put(). 465 * The 'struct bpf_prog_array *' should only be replaced with xchg() 466 * since other cpus are walking the array of pointers in parallel. 467 */ 468 struct bpf_prog_array_item { 469 struct bpf_prog *prog; 470 struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 471 }; 472 473 struct bpf_prog_array { 474 struct rcu_head rcu; 475 struct bpf_prog_array_item items[0]; 476 }; 477 478 struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags); 479 void bpf_prog_array_free(struct bpf_prog_array __rcu *progs); 480 int bpf_prog_array_length(struct bpf_prog_array __rcu *progs); 481 int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *progs, 482 __u32 __user *prog_ids, u32 cnt); 483 484 void bpf_prog_array_delete_safe(struct bpf_prog_array __rcu *progs, 485 struct bpf_prog *old_prog); 486 int bpf_prog_array_copy_info(struct bpf_prog_array __rcu *array, 487 u32 *prog_ids, u32 request_cnt, 488 u32 *prog_cnt); 489 int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array, 490 struct bpf_prog *exclude_prog, 491 struct bpf_prog *include_prog, 492 struct bpf_prog_array **new_array); 493 494 #define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null) \ 495 ({ \ 496 struct bpf_prog_array_item *_item; \ 497 struct bpf_prog *_prog; \ 498 struct bpf_prog_array *_array; \ 499 u32 _ret = 1; \ 500 preempt_disable(); \ 501 rcu_read_lock(); \ 502 _array = rcu_dereference(array); \ 503 if (unlikely(check_non_null && !_array))\ 504 goto _out; \ 505 _item = &_array->items[0]; \ 506 while ((_prog = READ_ONCE(_item->prog))) { \ 507 bpf_cgroup_storage_set(_item->cgroup_storage); \ 508 _ret &= func(_prog, ctx); \ 509 _item++; \ 510 } \ 511 _out: \ 512 rcu_read_unlock(); \ 513 preempt_enable_no_resched(); \ 514 _ret; \ 515 }) 516 517 #define BPF_PROG_RUN_ARRAY(array, ctx, func) \ 518 __BPF_PROG_RUN_ARRAY(array, ctx, func, false) 519 520 #define BPF_PROG_RUN_ARRAY_CHECK(array, ctx, func) \ 521 __BPF_PROG_RUN_ARRAY(array, ctx, func, true) 522 523 #ifdef CONFIG_BPF_SYSCALL 524 DECLARE_PER_CPU(int, bpf_prog_active); 525 526 extern const struct file_operations bpf_map_fops; 527 extern const struct file_operations bpf_prog_fops; 528 529 #define BPF_PROG_TYPE(_id, _name) \ 530 extern const struct bpf_prog_ops _name ## _prog_ops; \ 531 extern const struct bpf_verifier_ops _name ## _verifier_ops; 532 #define BPF_MAP_TYPE(_id, _ops) \ 533 extern const struct bpf_map_ops _ops; 534 #include <linux/bpf_types.h> 535 #undef BPF_PROG_TYPE 536 #undef BPF_MAP_TYPE 537 538 extern const struct bpf_prog_ops bpf_offload_prog_ops; 539 extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops; 540 extern const struct bpf_verifier_ops xdp_analyzer_ops; 541 542 struct bpf_prog *bpf_prog_get(u32 ufd); 543 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 544 bool attach_drv); 545 struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog, int i); 546 void bpf_prog_sub(struct bpf_prog *prog, int i); 547 struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog); 548 struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog); 549 void bpf_prog_put(struct bpf_prog *prog); 550 int __bpf_prog_charge(struct user_struct *user, u32 pages); 551 void __bpf_prog_uncharge(struct user_struct *user, u32 pages); 552 553 void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock); 554 void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock); 555 556 struct bpf_map *bpf_map_get_with_uref(u32 ufd); 557 struct bpf_map *__bpf_map_get(struct fd f); 558 struct bpf_map * __must_check bpf_map_inc(struct bpf_map *map, bool uref); 559 void bpf_map_put_with_uref(struct bpf_map *map); 560 void bpf_map_put(struct bpf_map *map); 561 int bpf_map_precharge_memlock(u32 pages); 562 int bpf_map_charge_memlock(struct bpf_map *map, u32 pages); 563 void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages); 564 void *bpf_map_area_alloc(size_t size, int numa_node); 565 void bpf_map_area_free(void *base); 566 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr); 567 568 extern int sysctl_unprivileged_bpf_disabled; 569 extern int sysctl_bpf_stats_enabled; 570 571 int bpf_map_new_fd(struct bpf_map *map, int flags); 572 int bpf_prog_new_fd(struct bpf_prog *prog); 573 574 int bpf_obj_pin_user(u32 ufd, const char __user *pathname); 575 int bpf_obj_get_user(const char __user *pathname, int flags); 576 577 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value); 578 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value); 579 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, 580 u64 flags); 581 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, 582 u64 flags); 583 584 int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value); 585 586 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, 587 void *key, void *value, u64 map_flags); 588 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); 589 int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, 590 void *key, void *value, u64 map_flags); 591 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); 592 593 int bpf_get_file_flag(int flags); 594 int bpf_check_uarg_tail_zero(void __user *uaddr, size_t expected_size, 595 size_t actual_size); 596 597 /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and 598 * forced to use 'long' read/writes to try to atomically copy long counters. 599 * Best-effort only. No barriers here, since it _will_ race with concurrent 600 * updates from BPF programs. Called from bpf syscall and mostly used with 601 * size 8 or 16 bytes, so ask compiler to inline it. 602 */ 603 static inline void bpf_long_memcpy(void *dst, const void *src, u32 size) 604 { 605 const long *lsrc = src; 606 long *ldst = dst; 607 608 size /= sizeof(long); 609 while (size--) 610 *ldst++ = *lsrc++; 611 } 612 613 /* verify correctness of eBPF program */ 614 int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, 615 union bpf_attr __user *uattr); 616 void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); 617 618 /* Map specifics */ 619 struct xdp_buff; 620 struct sk_buff; 621 622 struct bpf_dtab_netdev *__dev_map_lookup_elem(struct bpf_map *map, u32 key); 623 void __dev_map_insert_ctx(struct bpf_map *map, u32 index); 624 void __dev_map_flush(struct bpf_map *map); 625 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, 626 struct net_device *dev_rx); 627 int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, 628 struct bpf_prog *xdp_prog); 629 630 struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key); 631 void __cpu_map_insert_ctx(struct bpf_map *map, u32 index); 632 void __cpu_map_flush(struct bpf_map *map); 633 int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp, 634 struct net_device *dev_rx); 635 636 /* Return map's numa specified by userspace */ 637 static inline int bpf_map_attr_numa_node(const union bpf_attr *attr) 638 { 639 return (attr->map_flags & BPF_F_NUMA_NODE) ? 640 attr->numa_node : NUMA_NO_NODE; 641 } 642 643 struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type); 644 int array_map_alloc_check(union bpf_attr *attr); 645 646 #else /* !CONFIG_BPF_SYSCALL */ 647 static inline struct bpf_prog *bpf_prog_get(u32 ufd) 648 { 649 return ERR_PTR(-EOPNOTSUPP); 650 } 651 652 static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, 653 enum bpf_prog_type type, 654 bool attach_drv) 655 { 656 return ERR_PTR(-EOPNOTSUPP); 657 } 658 659 static inline struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog, 660 int i) 661 { 662 return ERR_PTR(-EOPNOTSUPP); 663 } 664 665 static inline void bpf_prog_sub(struct bpf_prog *prog, int i) 666 { 667 } 668 669 static inline void bpf_prog_put(struct bpf_prog *prog) 670 { 671 } 672 673 static inline struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog) 674 { 675 return ERR_PTR(-EOPNOTSUPP); 676 } 677 678 static inline struct bpf_prog *__must_check 679 bpf_prog_inc_not_zero(struct bpf_prog *prog) 680 { 681 return ERR_PTR(-EOPNOTSUPP); 682 } 683 684 static inline int __bpf_prog_charge(struct user_struct *user, u32 pages) 685 { 686 return 0; 687 } 688 689 static inline void __bpf_prog_uncharge(struct user_struct *user, u32 pages) 690 { 691 } 692 693 static inline int bpf_obj_get_user(const char __user *pathname, int flags) 694 { 695 return -EOPNOTSUPP; 696 } 697 698 static inline struct net_device *__dev_map_lookup_elem(struct bpf_map *map, 699 u32 key) 700 { 701 return NULL; 702 } 703 704 static inline void __dev_map_insert_ctx(struct bpf_map *map, u32 index) 705 { 706 } 707 708 static inline void __dev_map_flush(struct bpf_map *map) 709 { 710 } 711 712 struct xdp_buff; 713 struct bpf_dtab_netdev; 714 715 static inline 716 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, 717 struct net_device *dev_rx) 718 { 719 return 0; 720 } 721 722 struct sk_buff; 723 724 static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, 725 struct sk_buff *skb, 726 struct bpf_prog *xdp_prog) 727 { 728 return 0; 729 } 730 731 static inline 732 struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key) 733 { 734 return NULL; 735 } 736 737 static inline void __cpu_map_insert_ctx(struct bpf_map *map, u32 index) 738 { 739 } 740 741 static inline void __cpu_map_flush(struct bpf_map *map) 742 { 743 } 744 745 static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, 746 struct xdp_buff *xdp, 747 struct net_device *dev_rx) 748 { 749 return 0; 750 } 751 752 static inline struct bpf_prog *bpf_prog_get_type_path(const char *name, 753 enum bpf_prog_type type) 754 { 755 return ERR_PTR(-EOPNOTSUPP); 756 } 757 #endif /* CONFIG_BPF_SYSCALL */ 758 759 static inline struct bpf_prog *bpf_prog_get_type(u32 ufd, 760 enum bpf_prog_type type) 761 { 762 return bpf_prog_get_type_dev(ufd, type, false); 763 } 764 765 bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool); 766 767 int bpf_prog_offload_compile(struct bpf_prog *prog); 768 void bpf_prog_offload_destroy(struct bpf_prog *prog); 769 int bpf_prog_offload_info_fill(struct bpf_prog_info *info, 770 struct bpf_prog *prog); 771 772 int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map); 773 774 int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value); 775 int bpf_map_offload_update_elem(struct bpf_map *map, 776 void *key, void *value, u64 flags); 777 int bpf_map_offload_delete_elem(struct bpf_map *map, void *key); 778 int bpf_map_offload_get_next_key(struct bpf_map *map, 779 void *key, void *next_key); 780 781 bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map); 782 783 struct bpf_offload_dev * 784 bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv); 785 void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev); 786 void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev); 787 int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev, 788 struct net_device *netdev); 789 void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev, 790 struct net_device *netdev); 791 bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev); 792 793 #if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL) 794 int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr); 795 796 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) 797 { 798 return aux->offload_requested; 799 } 800 801 static inline bool bpf_map_is_dev_bound(struct bpf_map *map) 802 { 803 return unlikely(map->ops == &bpf_map_offload_ops); 804 } 805 806 struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr); 807 void bpf_map_offload_map_free(struct bpf_map *map); 808 #else 809 static inline int bpf_prog_offload_init(struct bpf_prog *prog, 810 union bpf_attr *attr) 811 { 812 return -EOPNOTSUPP; 813 } 814 815 static inline bool bpf_prog_is_dev_bound(struct bpf_prog_aux *aux) 816 { 817 return false; 818 } 819 820 static inline bool bpf_map_is_dev_bound(struct bpf_map *map) 821 { 822 return false; 823 } 824 825 static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) 826 { 827 return ERR_PTR(-EOPNOTSUPP); 828 } 829 830 static inline void bpf_map_offload_map_free(struct bpf_map *map) 831 { 832 } 833 #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */ 834 835 #if defined(CONFIG_BPF_STREAM_PARSER) 836 int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, u32 which); 837 int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog); 838 #else 839 static inline int sock_map_prog_update(struct bpf_map *map, 840 struct bpf_prog *prog, u32 which) 841 { 842 return -EOPNOTSUPP; 843 } 844 845 static inline int sock_map_get_from_fd(const union bpf_attr *attr, 846 struct bpf_prog *prog) 847 { 848 return -EINVAL; 849 } 850 #endif 851 852 #if defined(CONFIG_XDP_SOCKETS) 853 struct xdp_sock; 854 struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, u32 key); 855 int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp, 856 struct xdp_sock *xs); 857 void __xsk_map_flush(struct bpf_map *map); 858 #else 859 struct xdp_sock; 860 static inline struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, 861 u32 key) 862 { 863 return NULL; 864 } 865 866 static inline int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp, 867 struct xdp_sock *xs) 868 { 869 return -EOPNOTSUPP; 870 } 871 872 static inline void __xsk_map_flush(struct bpf_map *map) 873 { 874 } 875 #endif 876 877 #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) 878 void bpf_sk_reuseport_detach(struct sock *sk); 879 int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, 880 void *value); 881 int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key, 882 void *value, u64 map_flags); 883 #else 884 static inline void bpf_sk_reuseport_detach(struct sock *sk) 885 { 886 } 887 888 #ifdef CONFIG_BPF_SYSCALL 889 static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, 890 void *key, void *value) 891 { 892 return -EOPNOTSUPP; 893 } 894 895 static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, 896 void *key, void *value, 897 u64 map_flags) 898 { 899 return -EOPNOTSUPP; 900 } 901 #endif /* CONFIG_BPF_SYSCALL */ 902 #endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */ 903 904 /* verifier prototypes for helper functions called from eBPF programs */ 905 extern const struct bpf_func_proto bpf_map_lookup_elem_proto; 906 extern const struct bpf_func_proto bpf_map_update_elem_proto; 907 extern const struct bpf_func_proto bpf_map_delete_elem_proto; 908 extern const struct bpf_func_proto bpf_map_push_elem_proto; 909 extern const struct bpf_func_proto bpf_map_pop_elem_proto; 910 extern const struct bpf_func_proto bpf_map_peek_elem_proto; 911 912 extern const struct bpf_func_proto bpf_get_prandom_u32_proto; 913 extern const struct bpf_func_proto bpf_get_smp_processor_id_proto; 914 extern const struct bpf_func_proto bpf_get_numa_node_id_proto; 915 extern const struct bpf_func_proto bpf_tail_call_proto; 916 extern const struct bpf_func_proto bpf_ktime_get_ns_proto; 917 extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto; 918 extern const struct bpf_func_proto bpf_get_current_uid_gid_proto; 919 extern const struct bpf_func_proto bpf_get_current_comm_proto; 920 extern const struct bpf_func_proto bpf_get_stackid_proto; 921 extern const struct bpf_func_proto bpf_get_stack_proto; 922 extern const struct bpf_func_proto bpf_sock_map_update_proto; 923 extern const struct bpf_func_proto bpf_sock_hash_update_proto; 924 extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto; 925 extern const struct bpf_func_proto bpf_msg_redirect_hash_proto; 926 extern const struct bpf_func_proto bpf_msg_redirect_map_proto; 927 extern const struct bpf_func_proto bpf_sk_redirect_hash_proto; 928 extern const struct bpf_func_proto bpf_sk_redirect_map_proto; 929 extern const struct bpf_func_proto bpf_spin_lock_proto; 930 extern const struct bpf_func_proto bpf_spin_unlock_proto; 931 extern const struct bpf_func_proto bpf_get_local_storage_proto; 932 933 /* Shared helpers among cBPF and eBPF. */ 934 void bpf_user_rnd_init_once(void); 935 u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 936 937 #if defined(CONFIG_NET) 938 bool bpf_sock_common_is_valid_access(int off, int size, 939 enum bpf_access_type type, 940 struct bpf_insn_access_aux *info); 941 bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type, 942 struct bpf_insn_access_aux *info); 943 u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, 944 const struct bpf_insn *si, 945 struct bpf_insn *insn_buf, 946 struct bpf_prog *prog, 947 u32 *target_size); 948 #else 949 static inline bool bpf_sock_common_is_valid_access(int off, int size, 950 enum bpf_access_type type, 951 struct bpf_insn_access_aux *info) 952 { 953 return false; 954 } 955 static inline bool bpf_sock_is_valid_access(int off, int size, 956 enum bpf_access_type type, 957 struct bpf_insn_access_aux *info) 958 { 959 return false; 960 } 961 static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, 962 const struct bpf_insn *si, 963 struct bpf_insn *insn_buf, 964 struct bpf_prog *prog, 965 u32 *target_size) 966 { 967 return 0; 968 } 969 #endif 970 971 #ifdef CONFIG_INET 972 bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type, 973 struct bpf_insn_access_aux *info); 974 975 u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, 976 const struct bpf_insn *si, 977 struct bpf_insn *insn_buf, 978 struct bpf_prog *prog, 979 u32 *target_size); 980 #else 981 static inline bool bpf_tcp_sock_is_valid_access(int off, int size, 982 enum bpf_access_type type, 983 struct bpf_insn_access_aux *info) 984 { 985 return false; 986 } 987 988 static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, 989 const struct bpf_insn *si, 990 struct bpf_insn *insn_buf, 991 struct bpf_prog *prog, 992 u32 *target_size) 993 { 994 return 0; 995 } 996 #endif /* CONFIG_INET */ 997 998 #endif /* _LINUX_BPF_H */ 999