1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 3 */ 4 #ifndef _LINUX_BPF_H 5 #define _LINUX_BPF_H 1 6 7 #include <uapi/linux/bpf.h> 8 9 #include <linux/workqueue.h> 10 #include <linux/file.h> 11 #include <linux/percpu.h> 12 #include <linux/err.h> 13 #include <linux/rbtree_latch.h> 14 #include <linux/numa.h> 15 #include <linux/mm_types.h> 16 #include <linux/wait.h> 17 #include <linux/u64_stats_sync.h> 18 #include <linux/refcount.h> 19 #include <linux/mutex.h> 20 21 struct bpf_verifier_env; 22 struct bpf_verifier_log; 23 struct perf_event; 24 struct bpf_prog; 25 struct bpf_prog_aux; 26 struct bpf_map; 27 struct sock; 28 struct seq_file; 29 struct btf; 30 struct btf_type; 31 struct exception_table_entry; 32 33 extern struct idr btf_idr; 34 extern spinlock_t btf_idr_lock; 35 36 /* map is generic key/value storage optionally accesible by eBPF programs */ 37 struct bpf_map_ops { 38 /* funcs callable from userspace (via syscall) */ 39 int (*map_alloc_check)(union bpf_attr *attr); 40 struct bpf_map *(*map_alloc)(union bpf_attr *attr); 41 void (*map_release)(struct bpf_map *map, struct file *map_file); 42 void (*map_free)(struct bpf_map *map); 43 int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key); 44 void (*map_release_uref)(struct bpf_map *map); 45 void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key); 46 47 /* funcs callable from userspace and from eBPF programs */ 48 void *(*map_lookup_elem)(struct bpf_map *map, void *key); 49 int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags); 50 int (*map_delete_elem)(struct bpf_map *map, void *key); 51 int (*map_push_elem)(struct bpf_map *map, void *value, u64 flags); 52 int (*map_pop_elem)(struct bpf_map *map, void *value); 53 int (*map_peek_elem)(struct bpf_map *map, void *value); 54 55 /* funcs called by prog_array and perf_event_array map */ 56 void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file, 57 int fd); 58 void (*map_fd_put_ptr)(void *ptr); 59 u32 (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf); 60 u32 (*map_fd_sys_lookup_elem)(void *ptr); 61 void (*map_seq_show_elem)(struct bpf_map *map, void *key, 62 struct seq_file *m); 63 int (*map_check_btf)(const struct bpf_map *map, 64 const struct btf *btf, 65 const struct btf_type *key_type, 66 const struct btf_type *value_type); 67 68 /* Prog poke tracking helpers. */ 69 int (*map_poke_track)(struct bpf_map *map, struct bpf_prog_aux *aux); 70 void (*map_poke_untrack)(struct bpf_map *map, struct bpf_prog_aux *aux); 71 void (*map_poke_run)(struct bpf_map *map, u32 key, struct bpf_prog *old, 72 struct bpf_prog *new); 73 74 /* Direct value access helpers. */ 75 int (*map_direct_value_addr)(const struct bpf_map *map, 76 u64 *imm, u32 off); 77 int (*map_direct_value_meta)(const struct bpf_map *map, 78 u64 imm, u32 *off); 79 int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma); 80 }; 81 82 struct bpf_map_memory { 83 u32 pages; 84 struct user_struct *user; 85 }; 86 87 struct bpf_map { 88 /* The first two cachelines with read-mostly members of which some 89 * are also accessed in fast-path (e.g. ops, max_entries). 90 */ 91 const struct bpf_map_ops *ops ____cacheline_aligned; 92 struct bpf_map *inner_map_meta; 93 #ifdef CONFIG_SECURITY 94 void *security; 95 #endif 96 enum bpf_map_type map_type; 97 u32 key_size; 98 u32 value_size; 99 u32 max_entries; 100 u32 map_flags; 101 int spin_lock_off; /* >=0 valid offset, <0 error */ 102 u32 id; 103 int numa_node; 104 u32 btf_key_type_id; 105 u32 btf_value_type_id; 106 struct btf *btf; 107 struct bpf_map_memory memory; 108 char name[BPF_OBJ_NAME_LEN]; 109 bool unpriv_array; 110 bool frozen; /* write-once; write-protected by freeze_mutex */ 111 /* 22 bytes hole */ 112 113 /* The 3rd and 4th cacheline with misc members to avoid false sharing 114 * particularly with refcounting. 115 */ 116 atomic64_t refcnt ____cacheline_aligned; 117 atomic64_t usercnt; 118 struct work_struct work; 119 struct mutex freeze_mutex; 120 u64 writecnt; /* writable mmap cnt; protected by freeze_mutex */ 121 }; 122 123 static inline bool map_value_has_spin_lock(const struct bpf_map *map) 124 { 125 return map->spin_lock_off >= 0; 126 } 127 128 static inline void check_and_init_map_lock(struct bpf_map *map, void *dst) 129 { 130 if (likely(!map_value_has_spin_lock(map))) 131 return; 132 *(struct bpf_spin_lock *)(dst + map->spin_lock_off) = 133 (struct bpf_spin_lock){}; 134 } 135 136 /* copy everything but bpf_spin_lock */ 137 static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) 138 { 139 if (unlikely(map_value_has_spin_lock(map))) { 140 u32 off = map->spin_lock_off; 141 142 memcpy(dst, src, off); 143 memcpy(dst + off + sizeof(struct bpf_spin_lock), 144 src + off + sizeof(struct bpf_spin_lock), 145 map->value_size - off - sizeof(struct bpf_spin_lock)); 146 } else { 147 memcpy(dst, src, map->value_size); 148 } 149 } 150 void copy_map_value_locked(struct bpf_map *map, void *dst, void *src, 151 bool lock_src); 152 153 struct bpf_offload_dev; 154 struct bpf_offloaded_map; 155 156 struct bpf_map_dev_ops { 157 int (*map_get_next_key)(struct bpf_offloaded_map *map, 158 void *key, void *next_key); 159 int (*map_lookup_elem)(struct bpf_offloaded_map *map, 160 void *key, void *value); 161 int (*map_update_elem)(struct bpf_offloaded_map *map, 162 void *key, void *value, u64 flags); 163 int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key); 164 }; 165 166 struct bpf_offloaded_map { 167 struct bpf_map map; 168 struct net_device *netdev; 169 const struct bpf_map_dev_ops *dev_ops; 170 void *dev_priv; 171 struct list_head offloads; 172 }; 173 174 static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map) 175 { 176 return container_of(map, struct bpf_offloaded_map, map); 177 } 178 179 static inline bool bpf_map_offload_neutral(const struct bpf_map *map) 180 { 181 return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 182 } 183 184 static inline bool bpf_map_support_seq_show(const struct bpf_map *map) 185 { 186 return map->btf && map->ops->map_seq_show_elem; 187 } 188 189 int map_check_no_btf(const struct bpf_map *map, 190 const struct btf *btf, 191 const struct btf_type *key_type, 192 const struct btf_type *value_type); 193 194 extern const struct bpf_map_ops bpf_map_offload_ops; 195 196 /* function argument constraints */ 197 enum bpf_arg_type { 198 ARG_DONTCARE = 0, /* unused argument in helper function */ 199 200 /* the following constraints used to prototype 201 * bpf_map_lookup/update/delete_elem() functions 202 */ 203 ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */ 204 ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */ 205 ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */ 206 ARG_PTR_TO_UNINIT_MAP_VALUE, /* pointer to valid memory used to store a map value */ 207 ARG_PTR_TO_MAP_VALUE_OR_NULL, /* pointer to stack used as map value or NULL */ 208 209 /* the following constraints used to prototype bpf_memcmp() and other 210 * functions that access data on eBPF program stack 211 */ 212 ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ 213 ARG_PTR_TO_MEM_OR_NULL, /* pointer to valid memory or NULL */ 214 ARG_PTR_TO_UNINIT_MEM, /* pointer to memory does not need to be initialized, 215 * helper function must fill all bytes or clear 216 * them in error case. 217 */ 218 219 ARG_CONST_SIZE, /* number of bytes accessed from memory */ 220 ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ 221 222 ARG_PTR_TO_CTX, /* pointer to context */ 223 ARG_ANYTHING, /* any (initialized) argument is ok */ 224 ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */ 225 ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */ 226 ARG_PTR_TO_INT, /* pointer to int */ 227 ARG_PTR_TO_LONG, /* pointer to long */ 228 ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */ 229 ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */ 230 }; 231 232 /* type of values returned from helper functions */ 233 enum bpf_return_type { 234 RET_INTEGER, /* function returns integer */ 235 RET_VOID, /* function doesn't return anything */ 236 RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */ 237 RET_PTR_TO_MAP_VALUE_OR_NULL, /* returns a pointer to map elem value or NULL */ 238 RET_PTR_TO_SOCKET_OR_NULL, /* returns a pointer to a socket or NULL */ 239 RET_PTR_TO_TCP_SOCK_OR_NULL, /* returns a pointer to a tcp_sock or NULL */ 240 RET_PTR_TO_SOCK_COMMON_OR_NULL, /* returns a pointer to a sock_common or NULL */ 241 }; 242 243 /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs 244 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL 245 * instructions after verifying 246 */ 247 struct bpf_func_proto { 248 u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 249 bool gpl_only; 250 bool pkt_access; 251 enum bpf_return_type ret_type; 252 union { 253 struct { 254 enum bpf_arg_type arg1_type; 255 enum bpf_arg_type arg2_type; 256 enum bpf_arg_type arg3_type; 257 enum bpf_arg_type arg4_type; 258 enum bpf_arg_type arg5_type; 259 }; 260 enum bpf_arg_type arg_type[5]; 261 }; 262 int *btf_id; /* BTF ids of arguments */ 263 }; 264 265 /* bpf_context is intentionally undefined structure. Pointer to bpf_context is 266 * the first argument to eBPF programs. 267 * For socket filters: 'struct bpf_context *' == 'struct sk_buff *' 268 */ 269 struct bpf_context; 270 271 enum bpf_access_type { 272 BPF_READ = 1, 273 BPF_WRITE = 2 274 }; 275 276 /* types of values stored in eBPF registers */ 277 /* Pointer types represent: 278 * pointer 279 * pointer + imm 280 * pointer + (u16) var 281 * pointer + (u16) var + imm 282 * if (range > 0) then [ptr, ptr + range - off) is safe to access 283 * if (id > 0) means that some 'var' was added 284 * if (off > 0) means that 'imm' was added 285 */ 286 enum bpf_reg_type { 287 NOT_INIT = 0, /* nothing was written into register */ 288 SCALAR_VALUE, /* reg doesn't contain a valid pointer */ 289 PTR_TO_CTX, /* reg points to bpf_context */ 290 CONST_PTR_TO_MAP, /* reg points to struct bpf_map */ 291 PTR_TO_MAP_VALUE, /* reg points to map element value */ 292 PTR_TO_MAP_VALUE_OR_NULL,/* points to map elem value or NULL */ 293 PTR_TO_STACK, /* reg == frame_pointer + offset */ 294 PTR_TO_PACKET_META, /* skb->data - meta_len */ 295 PTR_TO_PACKET, /* reg points to skb->data */ 296 PTR_TO_PACKET_END, /* skb->data + headlen */ 297 PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */ 298 PTR_TO_SOCKET, /* reg points to struct bpf_sock */ 299 PTR_TO_SOCKET_OR_NULL, /* reg points to struct bpf_sock or NULL */ 300 PTR_TO_SOCK_COMMON, /* reg points to sock_common */ 301 PTR_TO_SOCK_COMMON_OR_NULL, /* reg points to sock_common or NULL */ 302 PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */ 303 PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */ 304 PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */ 305 PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */ 306 PTR_TO_BTF_ID, /* reg points to kernel struct */ 307 }; 308 309 /* The information passed from prog-specific *_is_valid_access 310 * back to the verifier. 311 */ 312 struct bpf_insn_access_aux { 313 enum bpf_reg_type reg_type; 314 union { 315 int ctx_field_size; 316 u32 btf_id; 317 }; 318 struct bpf_verifier_log *log; /* for verbose logs */ 319 }; 320 321 static inline void 322 bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size) 323 { 324 aux->ctx_field_size = size; 325 } 326 327 struct bpf_prog_ops { 328 int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, 329 union bpf_attr __user *uattr); 330 }; 331 332 struct bpf_verifier_ops { 333 /* return eBPF function prototype for verification */ 334 const struct bpf_func_proto * 335 (*get_func_proto)(enum bpf_func_id func_id, 336 const struct bpf_prog *prog); 337 338 /* return true if 'size' wide access at offset 'off' within bpf_context 339 * with 'type' (read or write) is allowed 340 */ 341 bool (*is_valid_access)(int off, int size, enum bpf_access_type type, 342 const struct bpf_prog *prog, 343 struct bpf_insn_access_aux *info); 344 int (*gen_prologue)(struct bpf_insn *insn, bool direct_write, 345 const struct bpf_prog *prog); 346 int (*gen_ld_abs)(const struct bpf_insn *orig, 347 struct bpf_insn *insn_buf); 348 u32 (*convert_ctx_access)(enum bpf_access_type type, 349 const struct bpf_insn *src, 350 struct bpf_insn *dst, 351 struct bpf_prog *prog, u32 *target_size); 352 }; 353 354 struct bpf_prog_offload_ops { 355 /* verifier basic callbacks */ 356 int (*insn_hook)(struct bpf_verifier_env *env, 357 int insn_idx, int prev_insn_idx); 358 int (*finalize)(struct bpf_verifier_env *env); 359 /* verifier optimization callbacks (called after .finalize) */ 360 int (*replace_insn)(struct bpf_verifier_env *env, u32 off, 361 struct bpf_insn *insn); 362 int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt); 363 /* program management callbacks */ 364 int (*prepare)(struct bpf_prog *prog); 365 int (*translate)(struct bpf_prog *prog); 366 void (*destroy)(struct bpf_prog *prog); 367 }; 368 369 struct bpf_prog_offload { 370 struct bpf_prog *prog; 371 struct net_device *netdev; 372 struct bpf_offload_dev *offdev; 373 void *dev_priv; 374 struct list_head offloads; 375 bool dev_state; 376 bool opt_failed; 377 void *jited_image; 378 u32 jited_len; 379 }; 380 381 enum bpf_cgroup_storage_type { 382 BPF_CGROUP_STORAGE_SHARED, 383 BPF_CGROUP_STORAGE_PERCPU, 384 __BPF_CGROUP_STORAGE_MAX 385 }; 386 387 #define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX 388 389 /* The longest tracepoint has 12 args. 390 * See include/trace/bpf_probe.h 391 */ 392 #define MAX_BPF_FUNC_ARGS 12 393 394 struct bpf_prog_stats { 395 u64 cnt; 396 u64 nsecs; 397 struct u64_stats_sync syncp; 398 } __aligned(2 * sizeof(u64)); 399 400 struct btf_func_model { 401 u8 ret_size; 402 u8 nr_args; 403 u8 arg_size[MAX_BPF_FUNC_ARGS]; 404 }; 405 406 /* Restore arguments before returning from trampoline to let original function 407 * continue executing. This flag is used for fentry progs when there are no 408 * fexit progs. 409 */ 410 #define BPF_TRAMP_F_RESTORE_REGS BIT(0) 411 /* Call original function after fentry progs, but before fexit progs. 412 * Makes sense for fentry/fexit, normal calls and indirect calls. 413 */ 414 #define BPF_TRAMP_F_CALL_ORIG BIT(1) 415 /* Skip current frame and return to parent. Makes sense for fentry/fexit 416 * programs only. Should not be used with normal calls and indirect calls. 417 */ 418 #define BPF_TRAMP_F_SKIP_FRAME BIT(2) 419 420 /* Different use cases for BPF trampoline: 421 * 1. replace nop at the function entry (kprobe equivalent) 422 * flags = BPF_TRAMP_F_RESTORE_REGS 423 * fentry = a set of programs to run before returning from trampoline 424 * 425 * 2. replace nop at the function entry (kprobe + kretprobe equivalent) 426 * flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME 427 * orig_call = fentry_ip + MCOUNT_INSN_SIZE 428 * fentry = a set of program to run before calling original function 429 * fexit = a set of program to run after original function 430 * 431 * 3. replace direct call instruction anywhere in the function body 432 * or assign a function pointer for indirect call (like tcp_congestion_ops->cong_avoid) 433 * With flags = 0 434 * fentry = a set of programs to run before returning from trampoline 435 * With flags = BPF_TRAMP_F_CALL_ORIG 436 * orig_call = original callback addr or direct function addr 437 * fentry = a set of program to run before calling original function 438 * fexit = a set of program to run after original function 439 */ 440 int arch_prepare_bpf_trampoline(void *image, struct btf_func_model *m, u32 flags, 441 struct bpf_prog **fentry_progs, int fentry_cnt, 442 struct bpf_prog **fexit_progs, int fexit_cnt, 443 void *orig_call); 444 /* these two functions are called from generated trampoline */ 445 u64 notrace __bpf_prog_enter(void); 446 void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start); 447 448 enum bpf_tramp_prog_type { 449 BPF_TRAMP_FENTRY, 450 BPF_TRAMP_FEXIT, 451 BPF_TRAMP_MAX 452 }; 453 454 struct bpf_trampoline { 455 /* hlist for trampoline_table */ 456 struct hlist_node hlist; 457 /* serializes access to fields of this trampoline */ 458 struct mutex mutex; 459 refcount_t refcnt; 460 u64 key; 461 struct { 462 struct btf_func_model model; 463 void *addr; 464 bool ftrace_managed; 465 } func; 466 /* list of BPF programs using this trampoline */ 467 struct hlist_head progs_hlist[BPF_TRAMP_MAX]; 468 /* Number of attached programs. A counter per kind. */ 469 int progs_cnt[BPF_TRAMP_MAX]; 470 /* Executable image of trampoline */ 471 void *image; 472 u64 selector; 473 }; 474 475 #define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */ 476 477 struct bpf_dispatcher_prog { 478 struct bpf_prog *prog; 479 refcount_t users; 480 }; 481 482 struct bpf_dispatcher { 483 /* dispatcher mutex */ 484 struct mutex mutex; 485 void *func; 486 struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX]; 487 int num_progs; 488 void *image; 489 u32 image_off; 490 }; 491 492 static __always_inline unsigned int bpf_dispatcher_nopfunc( 493 const void *ctx, 494 const struct bpf_insn *insnsi, 495 unsigned int (*bpf_func)(const void *, 496 const struct bpf_insn *)) 497 { 498 return bpf_func(ctx, insnsi); 499 } 500 #ifdef CONFIG_BPF_JIT 501 struct bpf_trampoline *bpf_trampoline_lookup(u64 key); 502 int bpf_trampoline_link_prog(struct bpf_prog *prog); 503 int bpf_trampoline_unlink_prog(struct bpf_prog *prog); 504 void bpf_trampoline_put(struct bpf_trampoline *tr); 505 void *bpf_jit_alloc_exec_page(void); 506 #define BPF_DISPATCHER_INIT(name) { \ 507 .mutex = __MUTEX_INITIALIZER(name.mutex), \ 508 .func = &name##func, \ 509 .progs = {}, \ 510 .num_progs = 0, \ 511 .image = NULL, \ 512 .image_off = 0 \ 513 } 514 515 #define DEFINE_BPF_DISPATCHER(name) \ 516 noinline unsigned int name##func( \ 517 const void *ctx, \ 518 const struct bpf_insn *insnsi, \ 519 unsigned int (*bpf_func)(const void *, \ 520 const struct bpf_insn *)) \ 521 { \ 522 return bpf_func(ctx, insnsi); \ 523 } \ 524 EXPORT_SYMBOL(name##func); \ 525 struct bpf_dispatcher name = BPF_DISPATCHER_INIT(name); 526 #define DECLARE_BPF_DISPATCHER(name) \ 527 unsigned int name##func( \ 528 const void *ctx, \ 529 const struct bpf_insn *insnsi, \ 530 unsigned int (*bpf_func)(const void *, \ 531 const struct bpf_insn *)); \ 532 extern struct bpf_dispatcher name; 533 #define BPF_DISPATCHER_FUNC(name) name##func 534 #define BPF_DISPATCHER_PTR(name) (&name) 535 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, 536 struct bpf_prog *to); 537 #else 538 static inline struct bpf_trampoline *bpf_trampoline_lookup(u64 key) 539 { 540 return NULL; 541 } 542 static inline int bpf_trampoline_link_prog(struct bpf_prog *prog) 543 { 544 return -ENOTSUPP; 545 } 546 static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog) 547 { 548 return -ENOTSUPP; 549 } 550 static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {} 551 #define DEFINE_BPF_DISPATCHER(name) 552 #define DECLARE_BPF_DISPATCHER(name) 553 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nopfunc 554 #define BPF_DISPATCHER_PTR(name) NULL 555 static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, 556 struct bpf_prog *from, 557 struct bpf_prog *to) {} 558 #endif 559 560 struct bpf_func_info_aux { 561 bool unreliable; 562 }; 563 564 enum bpf_jit_poke_reason { 565 BPF_POKE_REASON_TAIL_CALL, 566 }; 567 568 /* Descriptor of pokes pointing /into/ the JITed image. */ 569 struct bpf_jit_poke_descriptor { 570 void *ip; 571 union { 572 struct { 573 struct bpf_map *map; 574 u32 key; 575 } tail_call; 576 }; 577 bool ip_stable; 578 u8 adj_off; 579 u16 reason; 580 }; 581 582 struct bpf_prog_aux { 583 atomic64_t refcnt; 584 u32 used_map_cnt; 585 u32 max_ctx_offset; 586 u32 max_pkt_offset; 587 u32 max_tp_access; 588 u32 stack_depth; 589 u32 id; 590 u32 func_cnt; /* used by non-func prog as the number of func progs */ 591 u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */ 592 u32 attach_btf_id; /* in-kernel BTF type id to attach to */ 593 struct bpf_prog *linked_prog; 594 bool verifier_zext; /* Zero extensions has been inserted by verifier. */ 595 bool offload_requested; 596 bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ 597 bool func_proto_unreliable; 598 enum bpf_tramp_prog_type trampoline_prog_type; 599 struct bpf_trampoline *trampoline; 600 struct hlist_node tramp_hlist; 601 /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ 602 const struct btf_type *attach_func_proto; 603 /* function name for valid attach_btf_id */ 604 const char *attach_func_name; 605 struct bpf_prog **func; 606 void *jit_data; /* JIT specific data. arch dependent */ 607 struct bpf_jit_poke_descriptor *poke_tab; 608 u32 size_poke_tab; 609 struct latch_tree_node ksym_tnode; 610 struct list_head ksym_lnode; 611 const struct bpf_prog_ops *ops; 612 struct bpf_map **used_maps; 613 struct bpf_prog *prog; 614 struct user_struct *user; 615 u64 load_time; /* ns since boottime */ 616 struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 617 char name[BPF_OBJ_NAME_LEN]; 618 #ifdef CONFIG_SECURITY 619 void *security; 620 #endif 621 struct bpf_prog_offload *offload; 622 struct btf *btf; 623 struct bpf_func_info *func_info; 624 struct bpf_func_info_aux *func_info_aux; 625 /* bpf_line_info loaded from userspace. linfo->insn_off 626 * has the xlated insn offset. 627 * Both the main and sub prog share the same linfo. 628 * The subprog can access its first linfo by 629 * using the linfo_idx. 630 */ 631 struct bpf_line_info *linfo; 632 /* jited_linfo is the jited addr of the linfo. It has a 633 * one to one mapping to linfo: 634 * jited_linfo[i] is the jited addr for the linfo[i]->insn_off. 635 * Both the main and sub prog share the same jited_linfo. 636 * The subprog can access its first jited_linfo by 637 * using the linfo_idx. 638 */ 639 void **jited_linfo; 640 u32 func_info_cnt; 641 u32 nr_linfo; 642 /* subprog can use linfo_idx to access its first linfo and 643 * jited_linfo. 644 * main prog always has linfo_idx == 0 645 */ 646 u32 linfo_idx; 647 u32 num_exentries; 648 struct exception_table_entry *extable; 649 struct bpf_prog_stats __percpu *stats; 650 union { 651 struct work_struct work; 652 struct rcu_head rcu; 653 }; 654 }; 655 656 struct bpf_array_aux { 657 /* 'Ownership' of prog array is claimed by the first program that 658 * is going to use this map or by the first program which FD is 659 * stored in the map to make sure that all callers and callees have 660 * the same prog type and JITed flag. 661 */ 662 enum bpf_prog_type type; 663 bool jited; 664 /* Programs with direct jumps into programs part of this array. */ 665 struct list_head poke_progs; 666 struct bpf_map *map; 667 struct mutex poke_mutex; 668 struct work_struct work; 669 }; 670 671 struct bpf_array { 672 struct bpf_map map; 673 u32 elem_size; 674 u32 index_mask; 675 struct bpf_array_aux *aux; 676 union { 677 char value[0] __aligned(8); 678 void *ptrs[0] __aligned(8); 679 void __percpu *pptrs[0] __aligned(8); 680 }; 681 }; 682 683 #define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */ 684 #define MAX_TAIL_CALL_CNT 32 685 686 #define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \ 687 BPF_F_RDONLY_PROG | \ 688 BPF_F_WRONLY | \ 689 BPF_F_WRONLY_PROG) 690 691 #define BPF_MAP_CAN_READ BIT(0) 692 #define BPF_MAP_CAN_WRITE BIT(1) 693 694 static inline u32 bpf_map_flags_to_cap(struct bpf_map *map) 695 { 696 u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); 697 698 /* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is 699 * not possible. 700 */ 701 if (access_flags & BPF_F_RDONLY_PROG) 702 return BPF_MAP_CAN_READ; 703 else if (access_flags & BPF_F_WRONLY_PROG) 704 return BPF_MAP_CAN_WRITE; 705 else 706 return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE; 707 } 708 709 static inline bool bpf_map_flags_access_ok(u32 access_flags) 710 { 711 return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) != 712 (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); 713 } 714 715 struct bpf_event_entry { 716 struct perf_event *event; 717 struct file *perf_file; 718 struct file *map_file; 719 struct rcu_head rcu; 720 }; 721 722 bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp); 723 int bpf_prog_calc_tag(struct bpf_prog *fp); 724 const char *kernel_type_name(u32 btf_type_id); 725 726 const struct bpf_func_proto *bpf_get_trace_printk_proto(void); 727 728 typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src, 729 unsigned long off, unsigned long len); 730 typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type, 731 const struct bpf_insn *src, 732 struct bpf_insn *dst, 733 struct bpf_prog *prog, 734 u32 *target_size); 735 736 u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, 737 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy); 738 739 /* an array of programs to be executed under rcu_lock. 740 * 741 * Typical usage: 742 * ret = BPF_PROG_RUN_ARRAY(&bpf_prog_array, ctx, BPF_PROG_RUN); 743 * 744 * the structure returned by bpf_prog_array_alloc() should be populated 745 * with program pointers and the last pointer must be NULL. 746 * The user has to keep refcnt on the program and make sure the program 747 * is removed from the array before bpf_prog_put(). 748 * The 'struct bpf_prog_array *' should only be replaced with xchg() 749 * since other cpus are walking the array of pointers in parallel. 750 */ 751 struct bpf_prog_array_item { 752 struct bpf_prog *prog; 753 struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 754 }; 755 756 struct bpf_prog_array { 757 struct rcu_head rcu; 758 struct bpf_prog_array_item items[0]; 759 }; 760 761 struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags); 762 void bpf_prog_array_free(struct bpf_prog_array *progs); 763 int bpf_prog_array_length(struct bpf_prog_array *progs); 764 bool bpf_prog_array_is_empty(struct bpf_prog_array *array); 765 int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs, 766 __u32 __user *prog_ids, u32 cnt); 767 768 void bpf_prog_array_delete_safe(struct bpf_prog_array *progs, 769 struct bpf_prog *old_prog); 770 int bpf_prog_array_copy_info(struct bpf_prog_array *array, 771 u32 *prog_ids, u32 request_cnt, 772 u32 *prog_cnt); 773 int bpf_prog_array_copy(struct bpf_prog_array *old_array, 774 struct bpf_prog *exclude_prog, 775 struct bpf_prog *include_prog, 776 struct bpf_prog_array **new_array); 777 778 #define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null) \ 779 ({ \ 780 struct bpf_prog_array_item *_item; \ 781 struct bpf_prog *_prog; \ 782 struct bpf_prog_array *_array; \ 783 u32 _ret = 1; \ 784 preempt_disable(); \ 785 rcu_read_lock(); \ 786 _array = rcu_dereference(array); \ 787 if (unlikely(check_non_null && !_array))\ 788 goto _out; \ 789 _item = &_array->items[0]; \ 790 while ((_prog = READ_ONCE(_item->prog))) { \ 791 bpf_cgroup_storage_set(_item->cgroup_storage); \ 792 _ret &= func(_prog, ctx); \ 793 _item++; \ 794 } \ 795 _out: \ 796 rcu_read_unlock(); \ 797 preempt_enable(); \ 798 _ret; \ 799 }) 800 801 /* To be used by __cgroup_bpf_run_filter_skb for EGRESS BPF progs 802 * so BPF programs can request cwr for TCP packets. 803 * 804 * Current cgroup skb programs can only return 0 or 1 (0 to drop the 805 * packet. This macro changes the behavior so the low order bit 806 * indicates whether the packet should be dropped (0) or not (1) 807 * and the next bit is a congestion notification bit. This could be 808 * used by TCP to call tcp_enter_cwr() 809 * 810 * Hence, new allowed return values of CGROUP EGRESS BPF programs are: 811 * 0: drop packet 812 * 1: keep packet 813 * 2: drop packet and cn 814 * 3: keep packet and cn 815 * 816 * This macro then converts it to one of the NET_XMIT or an error 817 * code that is then interpreted as drop packet (and no cn): 818 * 0: NET_XMIT_SUCCESS skb should be transmitted 819 * 1: NET_XMIT_DROP skb should be dropped and cn 820 * 2: NET_XMIT_CN skb should be transmitted and cn 821 * 3: -EPERM skb should be dropped 822 */ 823 #define BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY(array, ctx, func) \ 824 ({ \ 825 struct bpf_prog_array_item *_item; \ 826 struct bpf_prog *_prog; \ 827 struct bpf_prog_array *_array; \ 828 u32 ret; \ 829 u32 _ret = 1; \ 830 u32 _cn = 0; \ 831 preempt_disable(); \ 832 rcu_read_lock(); \ 833 _array = rcu_dereference(array); \ 834 _item = &_array->items[0]; \ 835 while ((_prog = READ_ONCE(_item->prog))) { \ 836 bpf_cgroup_storage_set(_item->cgroup_storage); \ 837 ret = func(_prog, ctx); \ 838 _ret &= (ret & 1); \ 839 _cn |= (ret & 2); \ 840 _item++; \ 841 } \ 842 rcu_read_unlock(); \ 843 preempt_enable(); \ 844 if (_ret) \ 845 _ret = (_cn ? NET_XMIT_CN : NET_XMIT_SUCCESS); \ 846 else \ 847 _ret = (_cn ? NET_XMIT_DROP : -EPERM); \ 848 _ret; \ 849 }) 850 851 #define BPF_PROG_RUN_ARRAY(array, ctx, func) \ 852 __BPF_PROG_RUN_ARRAY(array, ctx, func, false) 853 854 #define BPF_PROG_RUN_ARRAY_CHECK(array, ctx, func) \ 855 __BPF_PROG_RUN_ARRAY(array, ctx, func, true) 856 857 #ifdef CONFIG_BPF_SYSCALL 858 DECLARE_PER_CPU(int, bpf_prog_active); 859 860 extern const struct file_operations bpf_map_fops; 861 extern const struct file_operations bpf_prog_fops; 862 863 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 864 extern const struct bpf_prog_ops _name ## _prog_ops; \ 865 extern const struct bpf_verifier_ops _name ## _verifier_ops; 866 #define BPF_MAP_TYPE(_id, _ops) \ 867 extern const struct bpf_map_ops _ops; 868 #include <linux/bpf_types.h> 869 #undef BPF_PROG_TYPE 870 #undef BPF_MAP_TYPE 871 872 extern const struct bpf_prog_ops bpf_offload_prog_ops; 873 extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops; 874 extern const struct bpf_verifier_ops xdp_analyzer_ops; 875 876 struct bpf_prog *bpf_prog_get(u32 ufd); 877 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 878 bool attach_drv); 879 void bpf_prog_add(struct bpf_prog *prog, int i); 880 void bpf_prog_sub(struct bpf_prog *prog, int i); 881 void bpf_prog_inc(struct bpf_prog *prog); 882 struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog); 883 void bpf_prog_put(struct bpf_prog *prog); 884 int __bpf_prog_charge(struct user_struct *user, u32 pages); 885 void __bpf_prog_uncharge(struct user_struct *user, u32 pages); 886 void __bpf_free_used_maps(struct bpf_prog_aux *aux, 887 struct bpf_map **used_maps, u32 len); 888 889 void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock); 890 void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock); 891 892 struct bpf_map *bpf_map_get_with_uref(u32 ufd); 893 struct bpf_map *__bpf_map_get(struct fd f); 894 void bpf_map_inc(struct bpf_map *map); 895 void bpf_map_inc_with_uref(struct bpf_map *map); 896 struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map); 897 void bpf_map_put_with_uref(struct bpf_map *map); 898 void bpf_map_put(struct bpf_map *map); 899 int bpf_map_charge_memlock(struct bpf_map *map, u32 pages); 900 void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages); 901 int bpf_map_charge_init(struct bpf_map_memory *mem, u64 size); 902 void bpf_map_charge_finish(struct bpf_map_memory *mem); 903 void bpf_map_charge_move(struct bpf_map_memory *dst, 904 struct bpf_map_memory *src); 905 void *bpf_map_area_alloc(u64 size, int numa_node); 906 void *bpf_map_area_mmapable_alloc(u64 size, int numa_node); 907 void bpf_map_area_free(void *base); 908 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr); 909 910 extern int sysctl_unprivileged_bpf_disabled; 911 912 int bpf_map_new_fd(struct bpf_map *map, int flags); 913 int bpf_prog_new_fd(struct bpf_prog *prog); 914 915 int bpf_obj_pin_user(u32 ufd, const char __user *pathname); 916 int bpf_obj_get_user(const char __user *pathname, int flags); 917 918 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value); 919 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value); 920 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, 921 u64 flags); 922 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, 923 u64 flags); 924 925 int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value); 926 927 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, 928 void *key, void *value, u64 map_flags); 929 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); 930 int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, 931 void *key, void *value, u64 map_flags); 932 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); 933 934 int bpf_get_file_flag(int flags); 935 int bpf_check_uarg_tail_zero(void __user *uaddr, size_t expected_size, 936 size_t actual_size); 937 938 /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and 939 * forced to use 'long' read/writes to try to atomically copy long counters. 940 * Best-effort only. No barriers here, since it _will_ race with concurrent 941 * updates from BPF programs. Called from bpf syscall and mostly used with 942 * size 8 or 16 bytes, so ask compiler to inline it. 943 */ 944 static inline void bpf_long_memcpy(void *dst, const void *src, u32 size) 945 { 946 const long *lsrc = src; 947 long *ldst = dst; 948 949 size /= sizeof(long); 950 while (size--) 951 *ldst++ = *lsrc++; 952 } 953 954 /* verify correctness of eBPF program */ 955 int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, 956 union bpf_attr __user *uattr); 957 void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); 958 959 /* Map specifics */ 960 struct xdp_buff; 961 struct sk_buff; 962 963 struct bpf_dtab_netdev *__dev_map_lookup_elem(struct bpf_map *map, u32 key); 964 struct bpf_dtab_netdev *__dev_map_hash_lookup_elem(struct bpf_map *map, u32 key); 965 void __dev_map_flush(void); 966 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, 967 struct net_device *dev_rx); 968 int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, 969 struct bpf_prog *xdp_prog); 970 971 struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key); 972 void __cpu_map_flush(void); 973 int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp, 974 struct net_device *dev_rx); 975 976 /* Return map's numa specified by userspace */ 977 static inline int bpf_map_attr_numa_node(const union bpf_attr *attr) 978 { 979 return (attr->map_flags & BPF_F_NUMA_NODE) ? 980 attr->numa_node : NUMA_NO_NODE; 981 } 982 983 struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type); 984 int array_map_alloc_check(union bpf_attr *attr); 985 986 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 987 union bpf_attr __user *uattr); 988 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 989 union bpf_attr __user *uattr); 990 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 991 const union bpf_attr *kattr, 992 union bpf_attr __user *uattr); 993 bool btf_ctx_access(int off, int size, enum bpf_access_type type, 994 const struct bpf_prog *prog, 995 struct bpf_insn_access_aux *info); 996 int btf_struct_access(struct bpf_verifier_log *log, 997 const struct btf_type *t, int off, int size, 998 enum bpf_access_type atype, 999 u32 *next_btf_id); 1000 int btf_resolve_helper_id(struct bpf_verifier_log *log, 1001 const struct bpf_func_proto *fn, int); 1002 1003 int btf_distill_func_proto(struct bpf_verifier_log *log, 1004 struct btf *btf, 1005 const struct btf_type *func_proto, 1006 const char *func_name, 1007 struct btf_func_model *m); 1008 1009 int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog); 1010 1011 struct bpf_prog *bpf_prog_by_id(u32 id); 1012 1013 #else /* !CONFIG_BPF_SYSCALL */ 1014 static inline struct bpf_prog *bpf_prog_get(u32 ufd) 1015 { 1016 return ERR_PTR(-EOPNOTSUPP); 1017 } 1018 1019 static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, 1020 enum bpf_prog_type type, 1021 bool attach_drv) 1022 { 1023 return ERR_PTR(-EOPNOTSUPP); 1024 } 1025 1026 static inline void bpf_prog_add(struct bpf_prog *prog, int i) 1027 { 1028 } 1029 1030 static inline void bpf_prog_sub(struct bpf_prog *prog, int i) 1031 { 1032 } 1033 1034 static inline void bpf_prog_put(struct bpf_prog *prog) 1035 { 1036 } 1037 1038 static inline void bpf_prog_inc(struct bpf_prog *prog) 1039 { 1040 } 1041 1042 static inline struct bpf_prog *__must_check 1043 bpf_prog_inc_not_zero(struct bpf_prog *prog) 1044 { 1045 return ERR_PTR(-EOPNOTSUPP); 1046 } 1047 1048 static inline int __bpf_prog_charge(struct user_struct *user, u32 pages) 1049 { 1050 return 0; 1051 } 1052 1053 static inline void __bpf_prog_uncharge(struct user_struct *user, u32 pages) 1054 { 1055 } 1056 1057 static inline int bpf_obj_get_user(const char __user *pathname, int flags) 1058 { 1059 return -EOPNOTSUPP; 1060 } 1061 1062 static inline struct net_device *__dev_map_lookup_elem(struct bpf_map *map, 1063 u32 key) 1064 { 1065 return NULL; 1066 } 1067 1068 static inline struct net_device *__dev_map_hash_lookup_elem(struct bpf_map *map, 1069 u32 key) 1070 { 1071 return NULL; 1072 } 1073 1074 static inline void __dev_map_flush(void) 1075 { 1076 } 1077 1078 struct xdp_buff; 1079 struct bpf_dtab_netdev; 1080 1081 static inline 1082 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, 1083 struct net_device *dev_rx) 1084 { 1085 return 0; 1086 } 1087 1088 struct sk_buff; 1089 1090 static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, 1091 struct sk_buff *skb, 1092 struct bpf_prog *xdp_prog) 1093 { 1094 return 0; 1095 } 1096 1097 static inline 1098 struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key) 1099 { 1100 return NULL; 1101 } 1102 1103 static inline void __cpu_map_flush(void) 1104 { 1105 } 1106 1107 static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, 1108 struct xdp_buff *xdp, 1109 struct net_device *dev_rx) 1110 { 1111 return 0; 1112 } 1113 1114 static inline struct bpf_prog *bpf_prog_get_type_path(const char *name, 1115 enum bpf_prog_type type) 1116 { 1117 return ERR_PTR(-EOPNOTSUPP); 1118 } 1119 1120 static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog, 1121 const union bpf_attr *kattr, 1122 union bpf_attr __user *uattr) 1123 { 1124 return -ENOTSUPP; 1125 } 1126 1127 static inline int bpf_prog_test_run_skb(struct bpf_prog *prog, 1128 const union bpf_attr *kattr, 1129 union bpf_attr __user *uattr) 1130 { 1131 return -ENOTSUPP; 1132 } 1133 1134 static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 1135 const union bpf_attr *kattr, 1136 union bpf_attr __user *uattr) 1137 { 1138 return -ENOTSUPP; 1139 } 1140 1141 static inline void bpf_map_put(struct bpf_map *map) 1142 { 1143 } 1144 1145 static inline struct bpf_prog *bpf_prog_by_id(u32 id) 1146 { 1147 return ERR_PTR(-ENOTSUPP); 1148 } 1149 #endif /* CONFIG_BPF_SYSCALL */ 1150 1151 static inline struct bpf_prog *bpf_prog_get_type(u32 ufd, 1152 enum bpf_prog_type type) 1153 { 1154 return bpf_prog_get_type_dev(ufd, type, false); 1155 } 1156 1157 bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool); 1158 1159 int bpf_prog_offload_compile(struct bpf_prog *prog); 1160 void bpf_prog_offload_destroy(struct bpf_prog *prog); 1161 int bpf_prog_offload_info_fill(struct bpf_prog_info *info, 1162 struct bpf_prog *prog); 1163 1164 int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map); 1165 1166 int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value); 1167 int bpf_map_offload_update_elem(struct bpf_map *map, 1168 void *key, void *value, u64 flags); 1169 int bpf_map_offload_delete_elem(struct bpf_map *map, void *key); 1170 int bpf_map_offload_get_next_key(struct bpf_map *map, 1171 void *key, void *next_key); 1172 1173 bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map); 1174 1175 struct bpf_offload_dev * 1176 bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv); 1177 void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev); 1178 void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev); 1179 int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev, 1180 struct net_device *netdev); 1181 void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev, 1182 struct net_device *netdev); 1183 bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev); 1184 1185 #if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL) 1186 int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr); 1187 1188 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) 1189 { 1190 return aux->offload_requested; 1191 } 1192 1193 static inline bool bpf_map_is_dev_bound(struct bpf_map *map) 1194 { 1195 return unlikely(map->ops == &bpf_map_offload_ops); 1196 } 1197 1198 struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr); 1199 void bpf_map_offload_map_free(struct bpf_map *map); 1200 #else 1201 static inline int bpf_prog_offload_init(struct bpf_prog *prog, 1202 union bpf_attr *attr) 1203 { 1204 return -EOPNOTSUPP; 1205 } 1206 1207 static inline bool bpf_prog_is_dev_bound(struct bpf_prog_aux *aux) 1208 { 1209 return false; 1210 } 1211 1212 static inline bool bpf_map_is_dev_bound(struct bpf_map *map) 1213 { 1214 return false; 1215 } 1216 1217 static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) 1218 { 1219 return ERR_PTR(-EOPNOTSUPP); 1220 } 1221 1222 static inline void bpf_map_offload_map_free(struct bpf_map *map) 1223 { 1224 } 1225 #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */ 1226 1227 #if defined(CONFIG_BPF_STREAM_PARSER) 1228 int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, u32 which); 1229 int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog); 1230 #else 1231 static inline int sock_map_prog_update(struct bpf_map *map, 1232 struct bpf_prog *prog, u32 which) 1233 { 1234 return -EOPNOTSUPP; 1235 } 1236 1237 static inline int sock_map_get_from_fd(const union bpf_attr *attr, 1238 struct bpf_prog *prog) 1239 { 1240 return -EINVAL; 1241 } 1242 #endif 1243 1244 #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) 1245 void bpf_sk_reuseport_detach(struct sock *sk); 1246 int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, 1247 void *value); 1248 int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key, 1249 void *value, u64 map_flags); 1250 #else 1251 static inline void bpf_sk_reuseport_detach(struct sock *sk) 1252 { 1253 } 1254 1255 #ifdef CONFIG_BPF_SYSCALL 1256 static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, 1257 void *key, void *value) 1258 { 1259 return -EOPNOTSUPP; 1260 } 1261 1262 static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, 1263 void *key, void *value, 1264 u64 map_flags) 1265 { 1266 return -EOPNOTSUPP; 1267 } 1268 #endif /* CONFIG_BPF_SYSCALL */ 1269 #endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */ 1270 1271 /* verifier prototypes for helper functions called from eBPF programs */ 1272 extern const struct bpf_func_proto bpf_map_lookup_elem_proto; 1273 extern const struct bpf_func_proto bpf_map_update_elem_proto; 1274 extern const struct bpf_func_proto bpf_map_delete_elem_proto; 1275 extern const struct bpf_func_proto bpf_map_push_elem_proto; 1276 extern const struct bpf_func_proto bpf_map_pop_elem_proto; 1277 extern const struct bpf_func_proto bpf_map_peek_elem_proto; 1278 1279 extern const struct bpf_func_proto bpf_get_prandom_u32_proto; 1280 extern const struct bpf_func_proto bpf_get_smp_processor_id_proto; 1281 extern const struct bpf_func_proto bpf_get_numa_node_id_proto; 1282 extern const struct bpf_func_proto bpf_tail_call_proto; 1283 extern const struct bpf_func_proto bpf_ktime_get_ns_proto; 1284 extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto; 1285 extern const struct bpf_func_proto bpf_get_current_uid_gid_proto; 1286 extern const struct bpf_func_proto bpf_get_current_comm_proto; 1287 extern const struct bpf_func_proto bpf_get_stackid_proto; 1288 extern const struct bpf_func_proto bpf_get_stack_proto; 1289 extern const struct bpf_func_proto bpf_sock_map_update_proto; 1290 extern const struct bpf_func_proto bpf_sock_hash_update_proto; 1291 extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto; 1292 extern const struct bpf_func_proto bpf_msg_redirect_hash_proto; 1293 extern const struct bpf_func_proto bpf_msg_redirect_map_proto; 1294 extern const struct bpf_func_proto bpf_sk_redirect_hash_proto; 1295 extern const struct bpf_func_proto bpf_sk_redirect_map_proto; 1296 extern const struct bpf_func_proto bpf_spin_lock_proto; 1297 extern const struct bpf_func_proto bpf_spin_unlock_proto; 1298 extern const struct bpf_func_proto bpf_get_local_storage_proto; 1299 extern const struct bpf_func_proto bpf_strtol_proto; 1300 extern const struct bpf_func_proto bpf_strtoul_proto; 1301 extern const struct bpf_func_proto bpf_tcp_sock_proto; 1302 1303 /* Shared helpers among cBPF and eBPF. */ 1304 void bpf_user_rnd_init_once(void); 1305 u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 1306 1307 #if defined(CONFIG_NET) 1308 bool bpf_sock_common_is_valid_access(int off, int size, 1309 enum bpf_access_type type, 1310 struct bpf_insn_access_aux *info); 1311 bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type, 1312 struct bpf_insn_access_aux *info); 1313 u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, 1314 const struct bpf_insn *si, 1315 struct bpf_insn *insn_buf, 1316 struct bpf_prog *prog, 1317 u32 *target_size); 1318 #else 1319 static inline bool bpf_sock_common_is_valid_access(int off, int size, 1320 enum bpf_access_type type, 1321 struct bpf_insn_access_aux *info) 1322 { 1323 return false; 1324 } 1325 static inline bool bpf_sock_is_valid_access(int off, int size, 1326 enum bpf_access_type type, 1327 struct bpf_insn_access_aux *info) 1328 { 1329 return false; 1330 } 1331 static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, 1332 const struct bpf_insn *si, 1333 struct bpf_insn *insn_buf, 1334 struct bpf_prog *prog, 1335 u32 *target_size) 1336 { 1337 return 0; 1338 } 1339 #endif 1340 1341 #ifdef CONFIG_INET 1342 struct sk_reuseport_kern { 1343 struct sk_buff *skb; 1344 struct sock *sk; 1345 struct sock *selected_sk; 1346 void *data_end; 1347 u32 hash; 1348 u32 reuseport_id; 1349 bool bind_inany; 1350 }; 1351 bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type, 1352 struct bpf_insn_access_aux *info); 1353 1354 u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, 1355 const struct bpf_insn *si, 1356 struct bpf_insn *insn_buf, 1357 struct bpf_prog *prog, 1358 u32 *target_size); 1359 1360 bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type, 1361 struct bpf_insn_access_aux *info); 1362 1363 u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, 1364 const struct bpf_insn *si, 1365 struct bpf_insn *insn_buf, 1366 struct bpf_prog *prog, 1367 u32 *target_size); 1368 #else 1369 static inline bool bpf_tcp_sock_is_valid_access(int off, int size, 1370 enum bpf_access_type type, 1371 struct bpf_insn_access_aux *info) 1372 { 1373 return false; 1374 } 1375 1376 static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, 1377 const struct bpf_insn *si, 1378 struct bpf_insn *insn_buf, 1379 struct bpf_prog *prog, 1380 u32 *target_size) 1381 { 1382 return 0; 1383 } 1384 static inline bool bpf_xdp_sock_is_valid_access(int off, int size, 1385 enum bpf_access_type type, 1386 struct bpf_insn_access_aux *info) 1387 { 1388 return false; 1389 } 1390 1391 static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, 1392 const struct bpf_insn *si, 1393 struct bpf_insn *insn_buf, 1394 struct bpf_prog *prog, 1395 u32 *target_size) 1396 { 1397 return 0; 1398 } 1399 #endif /* CONFIG_INET */ 1400 1401 enum bpf_text_poke_type { 1402 BPF_MOD_CALL, 1403 BPF_MOD_JUMP, 1404 }; 1405 1406 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, 1407 void *addr1, void *addr2); 1408 1409 #endif /* _LINUX_BPF_H */ 1410