1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 3 */ 4 #ifndef _LINUX_BPF_H 5 #define _LINUX_BPF_H 1 6 7 #include <uapi/linux/bpf.h> 8 9 #include <linux/workqueue.h> 10 #include <linux/file.h> 11 #include <linux/percpu.h> 12 #include <linux/err.h> 13 #include <linux/rbtree_latch.h> 14 #include <linux/numa.h> 15 #include <linux/mm_types.h> 16 #include <linux/wait.h> 17 #include <linux/u64_stats_sync.h> 18 #include <linux/refcount.h> 19 #include <linux/mutex.h> 20 #include <linux/module.h> 21 #include <linux/kallsyms.h> 22 #include <linux/capability.h> 23 24 struct bpf_verifier_env; 25 struct bpf_verifier_log; 26 struct perf_event; 27 struct bpf_prog; 28 struct bpf_prog_aux; 29 struct bpf_map; 30 struct sock; 31 struct seq_file; 32 struct btf; 33 struct btf_type; 34 struct exception_table_entry; 35 struct seq_operations; 36 struct bpf_iter_aux_info; 37 38 extern struct idr btf_idr; 39 extern spinlock_t btf_idr_lock; 40 41 typedef int (*bpf_iter_init_seq_priv_t)(void *private_data, 42 struct bpf_iter_aux_info *aux); 43 typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data); 44 struct bpf_iter_seq_info { 45 const struct seq_operations *seq_ops; 46 bpf_iter_init_seq_priv_t init_seq_private; 47 bpf_iter_fini_seq_priv_t fini_seq_private; 48 u32 seq_priv_size; 49 }; 50 51 /* map is generic key/value storage optionally accesible by eBPF programs */ 52 struct bpf_map_ops { 53 /* funcs callable from userspace (via syscall) */ 54 int (*map_alloc_check)(union bpf_attr *attr); 55 struct bpf_map *(*map_alloc)(union bpf_attr *attr); 56 void (*map_release)(struct bpf_map *map, struct file *map_file); 57 void (*map_free)(struct bpf_map *map); 58 int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key); 59 void (*map_release_uref)(struct bpf_map *map); 60 void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key); 61 int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr, 62 union bpf_attr __user *uattr); 63 int (*map_lookup_and_delete_batch)(struct bpf_map *map, 64 const union bpf_attr *attr, 65 union bpf_attr __user *uattr); 66 int (*map_update_batch)(struct bpf_map *map, const union bpf_attr *attr, 67 union bpf_attr __user *uattr); 68 int (*map_delete_batch)(struct bpf_map *map, const union bpf_attr *attr, 69 union bpf_attr __user *uattr); 70 71 /* funcs callable from userspace and from eBPF programs */ 72 void *(*map_lookup_elem)(struct bpf_map *map, void *key); 73 int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags); 74 int (*map_delete_elem)(struct bpf_map *map, void *key); 75 int (*map_push_elem)(struct bpf_map *map, void *value, u64 flags); 76 int (*map_pop_elem)(struct bpf_map *map, void *value); 77 int (*map_peek_elem)(struct bpf_map *map, void *value); 78 79 /* funcs called by prog_array and perf_event_array map */ 80 void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file, 81 int fd); 82 void (*map_fd_put_ptr)(void *ptr); 83 u32 (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf); 84 u32 (*map_fd_sys_lookup_elem)(void *ptr); 85 void (*map_seq_show_elem)(struct bpf_map *map, void *key, 86 struct seq_file *m); 87 int (*map_check_btf)(const struct bpf_map *map, 88 const struct btf *btf, 89 const struct btf_type *key_type, 90 const struct btf_type *value_type); 91 92 /* Prog poke tracking helpers. */ 93 int (*map_poke_track)(struct bpf_map *map, struct bpf_prog_aux *aux); 94 void (*map_poke_untrack)(struct bpf_map *map, struct bpf_prog_aux *aux); 95 void (*map_poke_run)(struct bpf_map *map, u32 key, struct bpf_prog *old, 96 struct bpf_prog *new); 97 98 /* Direct value access helpers. */ 99 int (*map_direct_value_addr)(const struct bpf_map *map, 100 u64 *imm, u32 off); 101 int (*map_direct_value_meta)(const struct bpf_map *map, 102 u64 imm, u32 *off); 103 int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma); 104 __poll_t (*map_poll)(struct bpf_map *map, struct file *filp, 105 struct poll_table_struct *pts); 106 107 /* BTF name and id of struct allocated by map_alloc */ 108 const char * const map_btf_name; 109 int *map_btf_id; 110 111 /* bpf_iter info used to open a seq_file */ 112 const struct bpf_iter_seq_info *iter_seq_info; 113 }; 114 115 struct bpf_map_memory { 116 u32 pages; 117 struct user_struct *user; 118 }; 119 120 struct bpf_map { 121 /* The first two cachelines with read-mostly members of which some 122 * are also accessed in fast-path (e.g. ops, max_entries). 123 */ 124 const struct bpf_map_ops *ops ____cacheline_aligned; 125 struct bpf_map *inner_map_meta; 126 #ifdef CONFIG_SECURITY 127 void *security; 128 #endif 129 enum bpf_map_type map_type; 130 u32 key_size; 131 u32 value_size; 132 u32 max_entries; 133 u32 map_flags; 134 int spin_lock_off; /* >=0 valid offset, <0 error */ 135 u32 id; 136 int numa_node; 137 u32 btf_key_type_id; 138 u32 btf_value_type_id; 139 struct btf *btf; 140 struct bpf_map_memory memory; 141 char name[BPF_OBJ_NAME_LEN]; 142 u32 btf_vmlinux_value_type_id; 143 bool bypass_spec_v1; 144 bool frozen; /* write-once; write-protected by freeze_mutex */ 145 /* 22 bytes hole */ 146 147 /* The 3rd and 4th cacheline with misc members to avoid false sharing 148 * particularly with refcounting. 149 */ 150 atomic64_t refcnt ____cacheline_aligned; 151 atomic64_t usercnt; 152 struct work_struct work; 153 struct mutex freeze_mutex; 154 u64 writecnt; /* writable mmap cnt; protected by freeze_mutex */ 155 }; 156 157 static inline bool map_value_has_spin_lock(const struct bpf_map *map) 158 { 159 return map->spin_lock_off >= 0; 160 } 161 162 static inline void check_and_init_map_lock(struct bpf_map *map, void *dst) 163 { 164 if (likely(!map_value_has_spin_lock(map))) 165 return; 166 *(struct bpf_spin_lock *)(dst + map->spin_lock_off) = 167 (struct bpf_spin_lock){}; 168 } 169 170 /* copy everything but bpf_spin_lock */ 171 static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) 172 { 173 if (unlikely(map_value_has_spin_lock(map))) { 174 u32 off = map->spin_lock_off; 175 176 memcpy(dst, src, off); 177 memcpy(dst + off + sizeof(struct bpf_spin_lock), 178 src + off + sizeof(struct bpf_spin_lock), 179 map->value_size - off - sizeof(struct bpf_spin_lock)); 180 } else { 181 memcpy(dst, src, map->value_size); 182 } 183 } 184 void copy_map_value_locked(struct bpf_map *map, void *dst, void *src, 185 bool lock_src); 186 int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size); 187 188 struct bpf_offload_dev; 189 struct bpf_offloaded_map; 190 191 struct bpf_map_dev_ops { 192 int (*map_get_next_key)(struct bpf_offloaded_map *map, 193 void *key, void *next_key); 194 int (*map_lookup_elem)(struct bpf_offloaded_map *map, 195 void *key, void *value); 196 int (*map_update_elem)(struct bpf_offloaded_map *map, 197 void *key, void *value, u64 flags); 198 int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key); 199 }; 200 201 struct bpf_offloaded_map { 202 struct bpf_map map; 203 struct net_device *netdev; 204 const struct bpf_map_dev_ops *dev_ops; 205 void *dev_priv; 206 struct list_head offloads; 207 }; 208 209 static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map) 210 { 211 return container_of(map, struct bpf_offloaded_map, map); 212 } 213 214 static inline bool bpf_map_offload_neutral(const struct bpf_map *map) 215 { 216 return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 217 } 218 219 static inline bool bpf_map_support_seq_show(const struct bpf_map *map) 220 { 221 return (map->btf_value_type_id || map->btf_vmlinux_value_type_id) && 222 map->ops->map_seq_show_elem; 223 } 224 225 int map_check_no_btf(const struct bpf_map *map, 226 const struct btf *btf, 227 const struct btf_type *key_type, 228 const struct btf_type *value_type); 229 230 extern const struct bpf_map_ops bpf_map_offload_ops; 231 232 /* function argument constraints */ 233 enum bpf_arg_type { 234 ARG_DONTCARE = 0, /* unused argument in helper function */ 235 236 /* the following constraints used to prototype 237 * bpf_map_lookup/update/delete_elem() functions 238 */ 239 ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */ 240 ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */ 241 ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */ 242 ARG_PTR_TO_UNINIT_MAP_VALUE, /* pointer to valid memory used to store a map value */ 243 ARG_PTR_TO_MAP_VALUE_OR_NULL, /* pointer to stack used as map value or NULL */ 244 245 /* the following constraints used to prototype bpf_memcmp() and other 246 * functions that access data on eBPF program stack 247 */ 248 ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ 249 ARG_PTR_TO_MEM_OR_NULL, /* pointer to valid memory or NULL */ 250 ARG_PTR_TO_UNINIT_MEM, /* pointer to memory does not need to be initialized, 251 * helper function must fill all bytes or clear 252 * them in error case. 253 */ 254 255 ARG_CONST_SIZE, /* number of bytes accessed from memory */ 256 ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ 257 258 ARG_PTR_TO_CTX, /* pointer to context */ 259 ARG_PTR_TO_CTX_OR_NULL, /* pointer to context or NULL */ 260 ARG_ANYTHING, /* any (initialized) argument is ok */ 261 ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */ 262 ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */ 263 ARG_PTR_TO_INT, /* pointer to int */ 264 ARG_PTR_TO_LONG, /* pointer to long */ 265 ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */ 266 ARG_PTR_TO_SOCKET_OR_NULL, /* pointer to bpf_sock (fullsock) or NULL */ 267 ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */ 268 ARG_PTR_TO_ALLOC_MEM, /* pointer to dynamically allocated memory */ 269 ARG_PTR_TO_ALLOC_MEM_OR_NULL, /* pointer to dynamically allocated memory or NULL */ 270 ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */ 271 }; 272 273 /* type of values returned from helper functions */ 274 enum bpf_return_type { 275 RET_INTEGER, /* function returns integer */ 276 RET_VOID, /* function doesn't return anything */ 277 RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */ 278 RET_PTR_TO_MAP_VALUE_OR_NULL, /* returns a pointer to map elem value or NULL */ 279 RET_PTR_TO_SOCKET_OR_NULL, /* returns a pointer to a socket or NULL */ 280 RET_PTR_TO_TCP_SOCK_OR_NULL, /* returns a pointer to a tcp_sock or NULL */ 281 RET_PTR_TO_SOCK_COMMON_OR_NULL, /* returns a pointer to a sock_common or NULL */ 282 RET_PTR_TO_ALLOC_MEM_OR_NULL, /* returns a pointer to dynamically allocated memory or NULL */ 283 RET_PTR_TO_BTF_ID_OR_NULL, /* returns a pointer to a btf_id or NULL */ 284 }; 285 286 /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs 287 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL 288 * instructions after verifying 289 */ 290 struct bpf_func_proto { 291 u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 292 bool gpl_only; 293 bool pkt_access; 294 enum bpf_return_type ret_type; 295 union { 296 struct { 297 enum bpf_arg_type arg1_type; 298 enum bpf_arg_type arg2_type; 299 enum bpf_arg_type arg3_type; 300 enum bpf_arg_type arg4_type; 301 enum bpf_arg_type arg5_type; 302 }; 303 enum bpf_arg_type arg_type[5]; 304 }; 305 int *btf_id; /* BTF ids of arguments */ 306 bool (*check_btf_id)(u32 btf_id, u32 arg); /* if the argument btf_id is 307 * valid. Often used if more 308 * than one btf id is permitted 309 * for this argument. 310 */ 311 int *ret_btf_id; /* return value btf_id */ 312 }; 313 314 /* bpf_context is intentionally undefined structure. Pointer to bpf_context is 315 * the first argument to eBPF programs. 316 * For socket filters: 'struct bpf_context *' == 'struct sk_buff *' 317 */ 318 struct bpf_context; 319 320 enum bpf_access_type { 321 BPF_READ = 1, 322 BPF_WRITE = 2 323 }; 324 325 /* types of values stored in eBPF registers */ 326 /* Pointer types represent: 327 * pointer 328 * pointer + imm 329 * pointer + (u16) var 330 * pointer + (u16) var + imm 331 * if (range > 0) then [ptr, ptr + range - off) is safe to access 332 * if (id > 0) means that some 'var' was added 333 * if (off > 0) means that 'imm' was added 334 */ 335 enum bpf_reg_type { 336 NOT_INIT = 0, /* nothing was written into register */ 337 SCALAR_VALUE, /* reg doesn't contain a valid pointer */ 338 PTR_TO_CTX, /* reg points to bpf_context */ 339 CONST_PTR_TO_MAP, /* reg points to struct bpf_map */ 340 PTR_TO_MAP_VALUE, /* reg points to map element value */ 341 PTR_TO_MAP_VALUE_OR_NULL,/* points to map elem value or NULL */ 342 PTR_TO_STACK, /* reg == frame_pointer + offset */ 343 PTR_TO_PACKET_META, /* skb->data - meta_len */ 344 PTR_TO_PACKET, /* reg points to skb->data */ 345 PTR_TO_PACKET_END, /* skb->data + headlen */ 346 PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */ 347 PTR_TO_SOCKET, /* reg points to struct bpf_sock */ 348 PTR_TO_SOCKET_OR_NULL, /* reg points to struct bpf_sock or NULL */ 349 PTR_TO_SOCK_COMMON, /* reg points to sock_common */ 350 PTR_TO_SOCK_COMMON_OR_NULL, /* reg points to sock_common or NULL */ 351 PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */ 352 PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */ 353 PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */ 354 PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */ 355 PTR_TO_BTF_ID, /* reg points to kernel struct */ 356 PTR_TO_BTF_ID_OR_NULL, /* reg points to kernel struct or NULL */ 357 PTR_TO_MEM, /* reg points to valid memory region */ 358 PTR_TO_MEM_OR_NULL, /* reg points to valid memory region or NULL */ 359 PTR_TO_RDONLY_BUF, /* reg points to a readonly buffer */ 360 PTR_TO_RDONLY_BUF_OR_NULL, /* reg points to a readonly buffer or NULL */ 361 PTR_TO_RDWR_BUF, /* reg points to a read/write buffer */ 362 PTR_TO_RDWR_BUF_OR_NULL, /* reg points to a read/write buffer or NULL */ 363 }; 364 365 /* The information passed from prog-specific *_is_valid_access 366 * back to the verifier. 367 */ 368 struct bpf_insn_access_aux { 369 enum bpf_reg_type reg_type; 370 union { 371 int ctx_field_size; 372 u32 btf_id; 373 }; 374 struct bpf_verifier_log *log; /* for verbose logs */ 375 }; 376 377 static inline void 378 bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size) 379 { 380 aux->ctx_field_size = size; 381 } 382 383 struct bpf_prog_ops { 384 int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, 385 union bpf_attr __user *uattr); 386 }; 387 388 struct bpf_verifier_ops { 389 /* return eBPF function prototype for verification */ 390 const struct bpf_func_proto * 391 (*get_func_proto)(enum bpf_func_id func_id, 392 const struct bpf_prog *prog); 393 394 /* return true if 'size' wide access at offset 'off' within bpf_context 395 * with 'type' (read or write) is allowed 396 */ 397 bool (*is_valid_access)(int off, int size, enum bpf_access_type type, 398 const struct bpf_prog *prog, 399 struct bpf_insn_access_aux *info); 400 int (*gen_prologue)(struct bpf_insn *insn, bool direct_write, 401 const struct bpf_prog *prog); 402 int (*gen_ld_abs)(const struct bpf_insn *orig, 403 struct bpf_insn *insn_buf); 404 u32 (*convert_ctx_access)(enum bpf_access_type type, 405 const struct bpf_insn *src, 406 struct bpf_insn *dst, 407 struct bpf_prog *prog, u32 *target_size); 408 int (*btf_struct_access)(struct bpf_verifier_log *log, 409 const struct btf_type *t, int off, int size, 410 enum bpf_access_type atype, 411 u32 *next_btf_id); 412 }; 413 414 struct bpf_prog_offload_ops { 415 /* verifier basic callbacks */ 416 int (*insn_hook)(struct bpf_verifier_env *env, 417 int insn_idx, int prev_insn_idx); 418 int (*finalize)(struct bpf_verifier_env *env); 419 /* verifier optimization callbacks (called after .finalize) */ 420 int (*replace_insn)(struct bpf_verifier_env *env, u32 off, 421 struct bpf_insn *insn); 422 int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt); 423 /* program management callbacks */ 424 int (*prepare)(struct bpf_prog *prog); 425 int (*translate)(struct bpf_prog *prog); 426 void (*destroy)(struct bpf_prog *prog); 427 }; 428 429 struct bpf_prog_offload { 430 struct bpf_prog *prog; 431 struct net_device *netdev; 432 struct bpf_offload_dev *offdev; 433 void *dev_priv; 434 struct list_head offloads; 435 bool dev_state; 436 bool opt_failed; 437 void *jited_image; 438 u32 jited_len; 439 }; 440 441 enum bpf_cgroup_storage_type { 442 BPF_CGROUP_STORAGE_SHARED, 443 BPF_CGROUP_STORAGE_PERCPU, 444 __BPF_CGROUP_STORAGE_MAX 445 }; 446 447 #define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX 448 449 /* The longest tracepoint has 12 args. 450 * See include/trace/bpf_probe.h 451 */ 452 #define MAX_BPF_FUNC_ARGS 12 453 454 struct bpf_prog_stats { 455 u64 cnt; 456 u64 nsecs; 457 struct u64_stats_sync syncp; 458 } __aligned(2 * sizeof(u64)); 459 460 struct btf_func_model { 461 u8 ret_size; 462 u8 nr_args; 463 u8 arg_size[MAX_BPF_FUNC_ARGS]; 464 }; 465 466 /* Restore arguments before returning from trampoline to let original function 467 * continue executing. This flag is used for fentry progs when there are no 468 * fexit progs. 469 */ 470 #define BPF_TRAMP_F_RESTORE_REGS BIT(0) 471 /* Call original function after fentry progs, but before fexit progs. 472 * Makes sense for fentry/fexit, normal calls and indirect calls. 473 */ 474 #define BPF_TRAMP_F_CALL_ORIG BIT(1) 475 /* Skip current frame and return to parent. Makes sense for fentry/fexit 476 * programs only. Should not be used with normal calls and indirect calls. 477 */ 478 #define BPF_TRAMP_F_SKIP_FRAME BIT(2) 479 480 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 481 * bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2 482 */ 483 #define BPF_MAX_TRAMP_PROGS 40 484 485 struct bpf_tramp_progs { 486 struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS]; 487 int nr_progs; 488 }; 489 490 /* Different use cases for BPF trampoline: 491 * 1. replace nop at the function entry (kprobe equivalent) 492 * flags = BPF_TRAMP_F_RESTORE_REGS 493 * fentry = a set of programs to run before returning from trampoline 494 * 495 * 2. replace nop at the function entry (kprobe + kretprobe equivalent) 496 * flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME 497 * orig_call = fentry_ip + MCOUNT_INSN_SIZE 498 * fentry = a set of program to run before calling original function 499 * fexit = a set of program to run after original function 500 * 501 * 3. replace direct call instruction anywhere in the function body 502 * or assign a function pointer for indirect call (like tcp_congestion_ops->cong_avoid) 503 * With flags = 0 504 * fentry = a set of programs to run before returning from trampoline 505 * With flags = BPF_TRAMP_F_CALL_ORIG 506 * orig_call = original callback addr or direct function addr 507 * fentry = a set of program to run before calling original function 508 * fexit = a set of program to run after original function 509 */ 510 int arch_prepare_bpf_trampoline(void *image, void *image_end, 511 const struct btf_func_model *m, u32 flags, 512 struct bpf_tramp_progs *tprogs, 513 void *orig_call); 514 /* these two functions are called from generated trampoline */ 515 u64 notrace __bpf_prog_enter(void); 516 void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start); 517 518 struct bpf_ksym { 519 unsigned long start; 520 unsigned long end; 521 char name[KSYM_NAME_LEN]; 522 struct list_head lnode; 523 struct latch_tree_node tnode; 524 bool prog; 525 }; 526 527 enum bpf_tramp_prog_type { 528 BPF_TRAMP_FENTRY, 529 BPF_TRAMP_FEXIT, 530 BPF_TRAMP_MODIFY_RETURN, 531 BPF_TRAMP_MAX, 532 BPF_TRAMP_REPLACE, /* more than MAX */ 533 }; 534 535 struct bpf_trampoline { 536 /* hlist for trampoline_table */ 537 struct hlist_node hlist; 538 /* serializes access to fields of this trampoline */ 539 struct mutex mutex; 540 refcount_t refcnt; 541 u64 key; 542 struct { 543 struct btf_func_model model; 544 void *addr; 545 bool ftrace_managed; 546 } func; 547 /* if !NULL this is BPF_PROG_TYPE_EXT program that extends another BPF 548 * program by replacing one of its functions. func.addr is the address 549 * of the function it replaced. 550 */ 551 struct bpf_prog *extension_prog; 552 /* list of BPF programs using this trampoline */ 553 struct hlist_head progs_hlist[BPF_TRAMP_MAX]; 554 /* Number of attached programs. A counter per kind. */ 555 int progs_cnt[BPF_TRAMP_MAX]; 556 /* Executable image of trampoline */ 557 void *image; 558 u64 selector; 559 struct bpf_ksym ksym; 560 }; 561 562 #define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */ 563 564 struct bpf_dispatcher_prog { 565 struct bpf_prog *prog; 566 refcount_t users; 567 }; 568 569 struct bpf_dispatcher { 570 /* dispatcher mutex */ 571 struct mutex mutex; 572 void *func; 573 struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX]; 574 int num_progs; 575 void *image; 576 u32 image_off; 577 struct bpf_ksym ksym; 578 }; 579 580 static __always_inline unsigned int bpf_dispatcher_nop_func( 581 const void *ctx, 582 const struct bpf_insn *insnsi, 583 unsigned int (*bpf_func)(const void *, 584 const struct bpf_insn *)) 585 { 586 return bpf_func(ctx, insnsi); 587 } 588 #ifdef CONFIG_BPF_JIT 589 struct bpf_trampoline *bpf_trampoline_lookup(u64 key); 590 int bpf_trampoline_link_prog(struct bpf_prog *prog); 591 int bpf_trampoline_unlink_prog(struct bpf_prog *prog); 592 void bpf_trampoline_put(struct bpf_trampoline *tr); 593 #define BPF_DISPATCHER_INIT(_name) { \ 594 .mutex = __MUTEX_INITIALIZER(_name.mutex), \ 595 .func = &_name##_func, \ 596 .progs = {}, \ 597 .num_progs = 0, \ 598 .image = NULL, \ 599 .image_off = 0, \ 600 .ksym = { \ 601 .name = #_name, \ 602 .lnode = LIST_HEAD_INIT(_name.ksym.lnode), \ 603 }, \ 604 } 605 606 #define DEFINE_BPF_DISPATCHER(name) \ 607 noinline unsigned int bpf_dispatcher_##name##_func( \ 608 const void *ctx, \ 609 const struct bpf_insn *insnsi, \ 610 unsigned int (*bpf_func)(const void *, \ 611 const struct bpf_insn *)) \ 612 { \ 613 return bpf_func(ctx, insnsi); \ 614 } \ 615 EXPORT_SYMBOL(bpf_dispatcher_##name##_func); \ 616 struct bpf_dispatcher bpf_dispatcher_##name = \ 617 BPF_DISPATCHER_INIT(bpf_dispatcher_##name); 618 #define DECLARE_BPF_DISPATCHER(name) \ 619 unsigned int bpf_dispatcher_##name##_func( \ 620 const void *ctx, \ 621 const struct bpf_insn *insnsi, \ 622 unsigned int (*bpf_func)(const void *, \ 623 const struct bpf_insn *)); \ 624 extern struct bpf_dispatcher bpf_dispatcher_##name; 625 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func 626 #define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name) 627 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, 628 struct bpf_prog *to); 629 /* Called only from JIT-enabled code, so there's no need for stubs. */ 630 void *bpf_jit_alloc_exec_page(void); 631 void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym); 632 void bpf_image_ksym_del(struct bpf_ksym *ksym); 633 void bpf_ksym_add(struct bpf_ksym *ksym); 634 void bpf_ksym_del(struct bpf_ksym *ksym); 635 #else 636 static inline struct bpf_trampoline *bpf_trampoline_lookup(u64 key) 637 { 638 return NULL; 639 } 640 static inline int bpf_trampoline_link_prog(struct bpf_prog *prog) 641 { 642 return -ENOTSUPP; 643 } 644 static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog) 645 { 646 return -ENOTSUPP; 647 } 648 static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {} 649 #define DEFINE_BPF_DISPATCHER(name) 650 #define DECLARE_BPF_DISPATCHER(name) 651 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nop_func 652 #define BPF_DISPATCHER_PTR(name) NULL 653 static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, 654 struct bpf_prog *from, 655 struct bpf_prog *to) {} 656 static inline bool is_bpf_image_address(unsigned long address) 657 { 658 return false; 659 } 660 #endif 661 662 struct bpf_func_info_aux { 663 u16 linkage; 664 bool unreliable; 665 }; 666 667 enum bpf_jit_poke_reason { 668 BPF_POKE_REASON_TAIL_CALL, 669 }; 670 671 /* Descriptor of pokes pointing /into/ the JITed image. */ 672 struct bpf_jit_poke_descriptor { 673 void *ip; 674 union { 675 struct { 676 struct bpf_map *map; 677 u32 key; 678 } tail_call; 679 }; 680 bool ip_stable; 681 u8 adj_off; 682 u16 reason; 683 }; 684 685 /* reg_type info for ctx arguments */ 686 struct bpf_ctx_arg_aux { 687 u32 offset; 688 enum bpf_reg_type reg_type; 689 u32 btf_id; 690 }; 691 692 struct bpf_prog_aux { 693 atomic64_t refcnt; 694 u32 used_map_cnt; 695 u32 max_ctx_offset; 696 u32 max_pkt_offset; 697 u32 max_tp_access; 698 u32 stack_depth; 699 u32 id; 700 u32 func_cnt; /* used by non-func prog as the number of func progs */ 701 u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */ 702 u32 attach_btf_id; /* in-kernel BTF type id to attach to */ 703 u32 ctx_arg_info_size; 704 u32 max_rdonly_access; 705 u32 max_rdwr_access; 706 const struct bpf_ctx_arg_aux *ctx_arg_info; 707 struct bpf_prog *linked_prog; 708 bool verifier_zext; /* Zero extensions has been inserted by verifier. */ 709 bool offload_requested; 710 bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ 711 bool func_proto_unreliable; 712 enum bpf_tramp_prog_type trampoline_prog_type; 713 struct bpf_trampoline *trampoline; 714 struct hlist_node tramp_hlist; 715 /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ 716 const struct btf_type *attach_func_proto; 717 /* function name for valid attach_btf_id */ 718 const char *attach_func_name; 719 struct bpf_prog **func; 720 void *jit_data; /* JIT specific data. arch dependent */ 721 struct bpf_jit_poke_descriptor *poke_tab; 722 u32 size_poke_tab; 723 struct bpf_ksym ksym; 724 const struct bpf_prog_ops *ops; 725 struct bpf_map **used_maps; 726 struct bpf_prog *prog; 727 struct user_struct *user; 728 u64 load_time; /* ns since boottime */ 729 struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 730 char name[BPF_OBJ_NAME_LEN]; 731 #ifdef CONFIG_SECURITY 732 void *security; 733 #endif 734 struct bpf_prog_offload *offload; 735 struct btf *btf; 736 struct bpf_func_info *func_info; 737 struct bpf_func_info_aux *func_info_aux; 738 /* bpf_line_info loaded from userspace. linfo->insn_off 739 * has the xlated insn offset. 740 * Both the main and sub prog share the same linfo. 741 * The subprog can access its first linfo by 742 * using the linfo_idx. 743 */ 744 struct bpf_line_info *linfo; 745 /* jited_linfo is the jited addr of the linfo. It has a 746 * one to one mapping to linfo: 747 * jited_linfo[i] is the jited addr for the linfo[i]->insn_off. 748 * Both the main and sub prog share the same jited_linfo. 749 * The subprog can access its first jited_linfo by 750 * using the linfo_idx. 751 */ 752 void **jited_linfo; 753 u32 func_info_cnt; 754 u32 nr_linfo; 755 /* subprog can use linfo_idx to access its first linfo and 756 * jited_linfo. 757 * main prog always has linfo_idx == 0 758 */ 759 u32 linfo_idx; 760 u32 num_exentries; 761 struct exception_table_entry *extable; 762 struct bpf_prog_stats __percpu *stats; 763 union { 764 struct work_struct work; 765 struct rcu_head rcu; 766 }; 767 }; 768 769 struct bpf_array_aux { 770 /* 'Ownership' of prog array is claimed by the first program that 771 * is going to use this map or by the first program which FD is 772 * stored in the map to make sure that all callers and callees have 773 * the same prog type and JITed flag. 774 */ 775 enum bpf_prog_type type; 776 bool jited; 777 /* Programs with direct jumps into programs part of this array. */ 778 struct list_head poke_progs; 779 struct bpf_map *map; 780 struct mutex poke_mutex; 781 struct work_struct work; 782 }; 783 784 struct bpf_link { 785 atomic64_t refcnt; 786 u32 id; 787 enum bpf_link_type type; 788 const struct bpf_link_ops *ops; 789 struct bpf_prog *prog; 790 struct work_struct work; 791 }; 792 793 struct bpf_link_ops { 794 void (*release)(struct bpf_link *link); 795 void (*dealloc)(struct bpf_link *link); 796 int (*detach)(struct bpf_link *link); 797 int (*update_prog)(struct bpf_link *link, struct bpf_prog *new_prog, 798 struct bpf_prog *old_prog); 799 void (*show_fdinfo)(const struct bpf_link *link, struct seq_file *seq); 800 int (*fill_link_info)(const struct bpf_link *link, 801 struct bpf_link_info *info); 802 }; 803 804 struct bpf_link_primer { 805 struct bpf_link *link; 806 struct file *file; 807 int fd; 808 u32 id; 809 }; 810 811 struct bpf_struct_ops_value; 812 struct btf_type; 813 struct btf_member; 814 815 #define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64 816 struct bpf_struct_ops { 817 const struct bpf_verifier_ops *verifier_ops; 818 int (*init)(struct btf *btf); 819 int (*check_member)(const struct btf_type *t, 820 const struct btf_member *member); 821 int (*init_member)(const struct btf_type *t, 822 const struct btf_member *member, 823 void *kdata, const void *udata); 824 int (*reg)(void *kdata); 825 void (*unreg)(void *kdata); 826 const struct btf_type *type; 827 const struct btf_type *value_type; 828 const char *name; 829 struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; 830 u32 type_id; 831 u32 value_id; 832 }; 833 834 #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) 835 #define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA)) 836 const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id); 837 void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log); 838 bool bpf_struct_ops_get(const void *kdata); 839 void bpf_struct_ops_put(const void *kdata); 840 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, 841 void *value); 842 static inline bool bpf_try_module_get(const void *data, struct module *owner) 843 { 844 if (owner == BPF_MODULE_OWNER) 845 return bpf_struct_ops_get(data); 846 else 847 return try_module_get(owner); 848 } 849 static inline void bpf_module_put(const void *data, struct module *owner) 850 { 851 if (owner == BPF_MODULE_OWNER) 852 bpf_struct_ops_put(data); 853 else 854 module_put(owner); 855 } 856 #else 857 static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id) 858 { 859 return NULL; 860 } 861 static inline void bpf_struct_ops_init(struct btf *btf, 862 struct bpf_verifier_log *log) 863 { 864 } 865 static inline bool bpf_try_module_get(const void *data, struct module *owner) 866 { 867 return try_module_get(owner); 868 } 869 static inline void bpf_module_put(const void *data, struct module *owner) 870 { 871 module_put(owner); 872 } 873 static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, 874 void *key, 875 void *value) 876 { 877 return -EINVAL; 878 } 879 #endif 880 881 struct bpf_array { 882 struct bpf_map map; 883 u32 elem_size; 884 u32 index_mask; 885 struct bpf_array_aux *aux; 886 union { 887 char value[0] __aligned(8); 888 void *ptrs[0] __aligned(8); 889 void __percpu *pptrs[0] __aligned(8); 890 }; 891 }; 892 893 #define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */ 894 #define MAX_TAIL_CALL_CNT 32 895 896 #define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \ 897 BPF_F_RDONLY_PROG | \ 898 BPF_F_WRONLY | \ 899 BPF_F_WRONLY_PROG) 900 901 #define BPF_MAP_CAN_READ BIT(0) 902 #define BPF_MAP_CAN_WRITE BIT(1) 903 904 static inline u32 bpf_map_flags_to_cap(struct bpf_map *map) 905 { 906 u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); 907 908 /* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is 909 * not possible. 910 */ 911 if (access_flags & BPF_F_RDONLY_PROG) 912 return BPF_MAP_CAN_READ; 913 else if (access_flags & BPF_F_WRONLY_PROG) 914 return BPF_MAP_CAN_WRITE; 915 else 916 return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE; 917 } 918 919 static inline bool bpf_map_flags_access_ok(u32 access_flags) 920 { 921 return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) != 922 (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); 923 } 924 925 struct bpf_event_entry { 926 struct perf_event *event; 927 struct file *perf_file; 928 struct file *map_file; 929 struct rcu_head rcu; 930 }; 931 932 bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp); 933 int bpf_prog_calc_tag(struct bpf_prog *fp); 934 const char *kernel_type_name(u32 btf_type_id); 935 936 const struct bpf_func_proto *bpf_get_trace_printk_proto(void); 937 938 typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src, 939 unsigned long off, unsigned long len); 940 typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type, 941 const struct bpf_insn *src, 942 struct bpf_insn *dst, 943 struct bpf_prog *prog, 944 u32 *target_size); 945 946 u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, 947 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy); 948 949 /* an array of programs to be executed under rcu_lock. 950 * 951 * Typical usage: 952 * ret = BPF_PROG_RUN_ARRAY(&bpf_prog_array, ctx, BPF_PROG_RUN); 953 * 954 * the structure returned by bpf_prog_array_alloc() should be populated 955 * with program pointers and the last pointer must be NULL. 956 * The user has to keep refcnt on the program and make sure the program 957 * is removed from the array before bpf_prog_put(). 958 * The 'struct bpf_prog_array *' should only be replaced with xchg() 959 * since other cpus are walking the array of pointers in parallel. 960 */ 961 struct bpf_prog_array_item { 962 struct bpf_prog *prog; 963 struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 964 }; 965 966 struct bpf_prog_array { 967 struct rcu_head rcu; 968 struct bpf_prog_array_item items[]; 969 }; 970 971 struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags); 972 void bpf_prog_array_free(struct bpf_prog_array *progs); 973 int bpf_prog_array_length(struct bpf_prog_array *progs); 974 bool bpf_prog_array_is_empty(struct bpf_prog_array *array); 975 int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs, 976 __u32 __user *prog_ids, u32 cnt); 977 978 void bpf_prog_array_delete_safe(struct bpf_prog_array *progs, 979 struct bpf_prog *old_prog); 980 int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index); 981 int bpf_prog_array_update_at(struct bpf_prog_array *array, int index, 982 struct bpf_prog *prog); 983 int bpf_prog_array_copy_info(struct bpf_prog_array *array, 984 u32 *prog_ids, u32 request_cnt, 985 u32 *prog_cnt); 986 int bpf_prog_array_copy(struct bpf_prog_array *old_array, 987 struct bpf_prog *exclude_prog, 988 struct bpf_prog *include_prog, 989 struct bpf_prog_array **new_array); 990 991 #define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null) \ 992 ({ \ 993 struct bpf_prog_array_item *_item; \ 994 struct bpf_prog *_prog; \ 995 struct bpf_prog_array *_array; \ 996 u32 _ret = 1; \ 997 migrate_disable(); \ 998 rcu_read_lock(); \ 999 _array = rcu_dereference(array); \ 1000 if (unlikely(check_non_null && !_array))\ 1001 goto _out; \ 1002 _item = &_array->items[0]; \ 1003 while ((_prog = READ_ONCE(_item->prog))) { \ 1004 bpf_cgroup_storage_set(_item->cgroup_storage); \ 1005 _ret &= func(_prog, ctx); \ 1006 _item++; \ 1007 } \ 1008 _out: \ 1009 rcu_read_unlock(); \ 1010 migrate_enable(); \ 1011 _ret; \ 1012 }) 1013 1014 /* To be used by __cgroup_bpf_run_filter_skb for EGRESS BPF progs 1015 * so BPF programs can request cwr for TCP packets. 1016 * 1017 * Current cgroup skb programs can only return 0 or 1 (0 to drop the 1018 * packet. This macro changes the behavior so the low order bit 1019 * indicates whether the packet should be dropped (0) or not (1) 1020 * and the next bit is a congestion notification bit. This could be 1021 * used by TCP to call tcp_enter_cwr() 1022 * 1023 * Hence, new allowed return values of CGROUP EGRESS BPF programs are: 1024 * 0: drop packet 1025 * 1: keep packet 1026 * 2: drop packet and cn 1027 * 3: keep packet and cn 1028 * 1029 * This macro then converts it to one of the NET_XMIT or an error 1030 * code that is then interpreted as drop packet (and no cn): 1031 * 0: NET_XMIT_SUCCESS skb should be transmitted 1032 * 1: NET_XMIT_DROP skb should be dropped and cn 1033 * 2: NET_XMIT_CN skb should be transmitted and cn 1034 * 3: -EPERM skb should be dropped 1035 */ 1036 #define BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY(array, ctx, func) \ 1037 ({ \ 1038 struct bpf_prog_array_item *_item; \ 1039 struct bpf_prog *_prog; \ 1040 struct bpf_prog_array *_array; \ 1041 u32 ret; \ 1042 u32 _ret = 1; \ 1043 u32 _cn = 0; \ 1044 migrate_disable(); \ 1045 rcu_read_lock(); \ 1046 _array = rcu_dereference(array); \ 1047 _item = &_array->items[0]; \ 1048 while ((_prog = READ_ONCE(_item->prog))) { \ 1049 bpf_cgroup_storage_set(_item->cgroup_storage); \ 1050 ret = func(_prog, ctx); \ 1051 _ret &= (ret & 1); \ 1052 _cn |= (ret & 2); \ 1053 _item++; \ 1054 } \ 1055 rcu_read_unlock(); \ 1056 migrate_enable(); \ 1057 if (_ret) \ 1058 _ret = (_cn ? NET_XMIT_CN : NET_XMIT_SUCCESS); \ 1059 else \ 1060 _ret = (_cn ? NET_XMIT_DROP : -EPERM); \ 1061 _ret; \ 1062 }) 1063 1064 #define BPF_PROG_RUN_ARRAY(array, ctx, func) \ 1065 __BPF_PROG_RUN_ARRAY(array, ctx, func, false) 1066 1067 #define BPF_PROG_RUN_ARRAY_CHECK(array, ctx, func) \ 1068 __BPF_PROG_RUN_ARRAY(array, ctx, func, true) 1069 1070 #ifdef CONFIG_BPF_SYSCALL 1071 DECLARE_PER_CPU(int, bpf_prog_active); 1072 extern struct mutex bpf_stats_enabled_mutex; 1073 1074 /* 1075 * Block execution of BPF programs attached to instrumentation (perf, 1076 * kprobes, tracepoints) to prevent deadlocks on map operations as any of 1077 * these events can happen inside a region which holds a map bucket lock 1078 * and can deadlock on it. 1079 * 1080 * Use the preemption safe inc/dec variants on RT because migrate disable 1081 * is preemptible on RT and preemption in the middle of the RMW operation 1082 * might lead to inconsistent state. Use the raw variants for non RT 1083 * kernels as migrate_disable() maps to preempt_disable() so the slightly 1084 * more expensive save operation can be avoided. 1085 */ 1086 static inline void bpf_disable_instrumentation(void) 1087 { 1088 migrate_disable(); 1089 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 1090 this_cpu_inc(bpf_prog_active); 1091 else 1092 __this_cpu_inc(bpf_prog_active); 1093 } 1094 1095 static inline void bpf_enable_instrumentation(void) 1096 { 1097 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 1098 this_cpu_dec(bpf_prog_active); 1099 else 1100 __this_cpu_dec(bpf_prog_active); 1101 migrate_enable(); 1102 } 1103 1104 extern const struct file_operations bpf_map_fops; 1105 extern const struct file_operations bpf_prog_fops; 1106 extern const struct file_operations bpf_iter_fops; 1107 1108 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 1109 extern const struct bpf_prog_ops _name ## _prog_ops; \ 1110 extern const struct bpf_verifier_ops _name ## _verifier_ops; 1111 #define BPF_MAP_TYPE(_id, _ops) \ 1112 extern const struct bpf_map_ops _ops; 1113 #define BPF_LINK_TYPE(_id, _name) 1114 #include <linux/bpf_types.h> 1115 #undef BPF_PROG_TYPE 1116 #undef BPF_MAP_TYPE 1117 #undef BPF_LINK_TYPE 1118 1119 extern const struct bpf_prog_ops bpf_offload_prog_ops; 1120 extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops; 1121 extern const struct bpf_verifier_ops xdp_analyzer_ops; 1122 1123 struct bpf_prog *bpf_prog_get(u32 ufd); 1124 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 1125 bool attach_drv); 1126 void bpf_prog_add(struct bpf_prog *prog, int i); 1127 void bpf_prog_sub(struct bpf_prog *prog, int i); 1128 void bpf_prog_inc(struct bpf_prog *prog); 1129 struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog); 1130 void bpf_prog_put(struct bpf_prog *prog); 1131 int __bpf_prog_charge(struct user_struct *user, u32 pages); 1132 void __bpf_prog_uncharge(struct user_struct *user, u32 pages); 1133 void __bpf_free_used_maps(struct bpf_prog_aux *aux, 1134 struct bpf_map **used_maps, u32 len); 1135 1136 void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock); 1137 void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock); 1138 1139 struct bpf_map *bpf_map_get(u32 ufd); 1140 struct bpf_map *bpf_map_get_with_uref(u32 ufd); 1141 struct bpf_map *__bpf_map_get(struct fd f); 1142 void bpf_map_inc(struct bpf_map *map); 1143 void bpf_map_inc_with_uref(struct bpf_map *map); 1144 struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map); 1145 void bpf_map_put_with_uref(struct bpf_map *map); 1146 void bpf_map_put(struct bpf_map *map); 1147 int bpf_map_charge_memlock(struct bpf_map *map, u32 pages); 1148 void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages); 1149 int bpf_map_charge_init(struct bpf_map_memory *mem, u64 size); 1150 void bpf_map_charge_finish(struct bpf_map_memory *mem); 1151 void bpf_map_charge_move(struct bpf_map_memory *dst, 1152 struct bpf_map_memory *src); 1153 void *bpf_map_area_alloc(u64 size, int numa_node); 1154 void *bpf_map_area_mmapable_alloc(u64 size, int numa_node); 1155 void bpf_map_area_free(void *base); 1156 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr); 1157 int generic_map_lookup_batch(struct bpf_map *map, 1158 const union bpf_attr *attr, 1159 union bpf_attr __user *uattr); 1160 int generic_map_update_batch(struct bpf_map *map, 1161 const union bpf_attr *attr, 1162 union bpf_attr __user *uattr); 1163 int generic_map_delete_batch(struct bpf_map *map, 1164 const union bpf_attr *attr, 1165 union bpf_attr __user *uattr); 1166 struct bpf_map *bpf_map_get_curr_or_next(u32 *id); 1167 struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id); 1168 1169 extern int sysctl_unprivileged_bpf_disabled; 1170 1171 static inline bool bpf_allow_ptr_leaks(void) 1172 { 1173 return perfmon_capable(); 1174 } 1175 1176 static inline bool bpf_allow_ptr_to_map_access(void) 1177 { 1178 return perfmon_capable(); 1179 } 1180 1181 static inline bool bpf_bypass_spec_v1(void) 1182 { 1183 return perfmon_capable(); 1184 } 1185 1186 static inline bool bpf_bypass_spec_v4(void) 1187 { 1188 return perfmon_capable(); 1189 } 1190 1191 int bpf_map_new_fd(struct bpf_map *map, int flags); 1192 int bpf_prog_new_fd(struct bpf_prog *prog); 1193 1194 void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, 1195 const struct bpf_link_ops *ops, struct bpf_prog *prog); 1196 int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer); 1197 int bpf_link_settle(struct bpf_link_primer *primer); 1198 void bpf_link_cleanup(struct bpf_link_primer *primer); 1199 void bpf_link_inc(struct bpf_link *link); 1200 void bpf_link_put(struct bpf_link *link); 1201 int bpf_link_new_fd(struct bpf_link *link); 1202 struct file *bpf_link_new_file(struct bpf_link *link, int *reserved_fd); 1203 struct bpf_link *bpf_link_get_from_fd(u32 ufd); 1204 1205 int bpf_obj_pin_user(u32 ufd, const char __user *pathname); 1206 int bpf_obj_get_user(const char __user *pathname, int flags); 1207 1208 #define BPF_ITER_FUNC_PREFIX "bpf_iter_" 1209 #define DEFINE_BPF_ITER_FUNC(target, args...) \ 1210 extern int bpf_iter_ ## target(args); \ 1211 int __init bpf_iter_ ## target(args) { return 0; } 1212 1213 struct bpf_iter_aux_info { 1214 struct bpf_map *map; 1215 }; 1216 1217 typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog, 1218 union bpf_iter_link_info *linfo, 1219 struct bpf_iter_aux_info *aux); 1220 typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux); 1221 1222 #define BPF_ITER_CTX_ARG_MAX 2 1223 struct bpf_iter_reg { 1224 const char *target; 1225 bpf_iter_attach_target_t attach_target; 1226 bpf_iter_detach_target_t detach_target; 1227 u32 ctx_arg_info_size; 1228 struct bpf_ctx_arg_aux ctx_arg_info[BPF_ITER_CTX_ARG_MAX]; 1229 const struct bpf_iter_seq_info *seq_info; 1230 }; 1231 1232 struct bpf_iter_meta { 1233 __bpf_md_ptr(struct seq_file *, seq); 1234 u64 session_id; 1235 u64 seq_num; 1236 }; 1237 1238 struct bpf_iter__bpf_map_elem { 1239 __bpf_md_ptr(struct bpf_iter_meta *, meta); 1240 __bpf_md_ptr(struct bpf_map *, map); 1241 __bpf_md_ptr(void *, key); 1242 __bpf_md_ptr(void *, value); 1243 }; 1244 1245 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info); 1246 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info); 1247 bool bpf_iter_prog_supported(struct bpf_prog *prog); 1248 int bpf_iter_link_attach(const union bpf_attr *attr, struct bpf_prog *prog); 1249 int bpf_iter_new_fd(struct bpf_link *link); 1250 bool bpf_link_is_iter(struct bpf_link *link); 1251 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop); 1252 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx); 1253 1254 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value); 1255 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value); 1256 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, 1257 u64 flags); 1258 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, 1259 u64 flags); 1260 1261 int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value); 1262 1263 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, 1264 void *key, void *value, u64 map_flags); 1265 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); 1266 int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, 1267 void *key, void *value, u64 map_flags); 1268 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); 1269 1270 int bpf_get_file_flag(int flags); 1271 int bpf_check_uarg_tail_zero(void __user *uaddr, size_t expected_size, 1272 size_t actual_size); 1273 1274 /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and 1275 * forced to use 'long' read/writes to try to atomically copy long counters. 1276 * Best-effort only. No barriers here, since it _will_ race with concurrent 1277 * updates from BPF programs. Called from bpf syscall and mostly used with 1278 * size 8 or 16 bytes, so ask compiler to inline it. 1279 */ 1280 static inline void bpf_long_memcpy(void *dst, const void *src, u32 size) 1281 { 1282 const long *lsrc = src; 1283 long *ldst = dst; 1284 1285 size /= sizeof(long); 1286 while (size--) 1287 *ldst++ = *lsrc++; 1288 } 1289 1290 /* verify correctness of eBPF program */ 1291 int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, 1292 union bpf_attr __user *uattr); 1293 void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); 1294 1295 /* Map specifics */ 1296 struct xdp_buff; 1297 struct sk_buff; 1298 1299 struct bpf_dtab_netdev *__dev_map_lookup_elem(struct bpf_map *map, u32 key); 1300 struct bpf_dtab_netdev *__dev_map_hash_lookup_elem(struct bpf_map *map, u32 key); 1301 void __dev_flush(void); 1302 int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp, 1303 struct net_device *dev_rx); 1304 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, 1305 struct net_device *dev_rx); 1306 int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, 1307 struct bpf_prog *xdp_prog); 1308 bool dev_map_can_have_prog(struct bpf_map *map); 1309 1310 struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key); 1311 void __cpu_map_flush(void); 1312 int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp, 1313 struct net_device *dev_rx); 1314 bool cpu_map_prog_allowed(struct bpf_map *map); 1315 1316 /* Return map's numa specified by userspace */ 1317 static inline int bpf_map_attr_numa_node(const union bpf_attr *attr) 1318 { 1319 return (attr->map_flags & BPF_F_NUMA_NODE) ? 1320 attr->numa_node : NUMA_NO_NODE; 1321 } 1322 1323 struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type); 1324 int array_map_alloc_check(union bpf_attr *attr); 1325 1326 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 1327 union bpf_attr __user *uattr); 1328 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 1329 union bpf_attr __user *uattr); 1330 int bpf_prog_test_run_tracing(struct bpf_prog *prog, 1331 const union bpf_attr *kattr, 1332 union bpf_attr __user *uattr); 1333 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 1334 const union bpf_attr *kattr, 1335 union bpf_attr __user *uattr); 1336 bool btf_ctx_access(int off, int size, enum bpf_access_type type, 1337 const struct bpf_prog *prog, 1338 struct bpf_insn_access_aux *info); 1339 int btf_struct_access(struct bpf_verifier_log *log, 1340 const struct btf_type *t, int off, int size, 1341 enum bpf_access_type atype, 1342 u32 *next_btf_id); 1343 int btf_resolve_helper_id(struct bpf_verifier_log *log, 1344 const struct bpf_func_proto *fn, int); 1345 1346 int btf_distill_func_proto(struct bpf_verifier_log *log, 1347 struct btf *btf, 1348 const struct btf_type *func_proto, 1349 const char *func_name, 1350 struct btf_func_model *m); 1351 1352 struct bpf_reg_state; 1353 int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, 1354 struct bpf_reg_state *regs); 1355 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, 1356 struct bpf_reg_state *reg); 1357 int btf_check_type_match(struct bpf_verifier_env *env, struct bpf_prog *prog, 1358 struct btf *btf, const struct btf_type *t); 1359 1360 struct bpf_prog *bpf_prog_by_id(u32 id); 1361 1362 const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id); 1363 #else /* !CONFIG_BPF_SYSCALL */ 1364 static inline struct bpf_prog *bpf_prog_get(u32 ufd) 1365 { 1366 return ERR_PTR(-EOPNOTSUPP); 1367 } 1368 1369 static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, 1370 enum bpf_prog_type type, 1371 bool attach_drv) 1372 { 1373 return ERR_PTR(-EOPNOTSUPP); 1374 } 1375 1376 static inline void bpf_prog_add(struct bpf_prog *prog, int i) 1377 { 1378 } 1379 1380 static inline void bpf_prog_sub(struct bpf_prog *prog, int i) 1381 { 1382 } 1383 1384 static inline void bpf_prog_put(struct bpf_prog *prog) 1385 { 1386 } 1387 1388 static inline void bpf_prog_inc(struct bpf_prog *prog) 1389 { 1390 } 1391 1392 static inline struct bpf_prog *__must_check 1393 bpf_prog_inc_not_zero(struct bpf_prog *prog) 1394 { 1395 return ERR_PTR(-EOPNOTSUPP); 1396 } 1397 1398 static inline int __bpf_prog_charge(struct user_struct *user, u32 pages) 1399 { 1400 return 0; 1401 } 1402 1403 static inline void __bpf_prog_uncharge(struct user_struct *user, u32 pages) 1404 { 1405 } 1406 1407 static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, 1408 const struct bpf_link_ops *ops, 1409 struct bpf_prog *prog) 1410 { 1411 } 1412 1413 static inline int bpf_link_prime(struct bpf_link *link, 1414 struct bpf_link_primer *primer) 1415 { 1416 return -EOPNOTSUPP; 1417 } 1418 1419 static inline int bpf_link_settle(struct bpf_link_primer *primer) 1420 { 1421 return -EOPNOTSUPP; 1422 } 1423 1424 static inline void bpf_link_cleanup(struct bpf_link_primer *primer) 1425 { 1426 } 1427 1428 static inline void bpf_link_inc(struct bpf_link *link) 1429 { 1430 } 1431 1432 static inline void bpf_link_put(struct bpf_link *link) 1433 { 1434 } 1435 1436 static inline int bpf_obj_get_user(const char __user *pathname, int flags) 1437 { 1438 return -EOPNOTSUPP; 1439 } 1440 1441 static inline struct net_device *__dev_map_lookup_elem(struct bpf_map *map, 1442 u32 key) 1443 { 1444 return NULL; 1445 } 1446 1447 static inline struct net_device *__dev_map_hash_lookup_elem(struct bpf_map *map, 1448 u32 key) 1449 { 1450 return NULL; 1451 } 1452 static inline bool dev_map_can_have_prog(struct bpf_map *map) 1453 { 1454 return false; 1455 } 1456 1457 static inline void __dev_flush(void) 1458 { 1459 } 1460 1461 struct xdp_buff; 1462 struct bpf_dtab_netdev; 1463 1464 static inline 1465 int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp, 1466 struct net_device *dev_rx) 1467 { 1468 return 0; 1469 } 1470 1471 static inline 1472 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, 1473 struct net_device *dev_rx) 1474 { 1475 return 0; 1476 } 1477 1478 struct sk_buff; 1479 1480 static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, 1481 struct sk_buff *skb, 1482 struct bpf_prog *xdp_prog) 1483 { 1484 return 0; 1485 } 1486 1487 static inline 1488 struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key) 1489 { 1490 return NULL; 1491 } 1492 1493 static inline void __cpu_map_flush(void) 1494 { 1495 } 1496 1497 static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, 1498 struct xdp_buff *xdp, 1499 struct net_device *dev_rx) 1500 { 1501 return 0; 1502 } 1503 1504 static inline bool cpu_map_prog_allowed(struct bpf_map *map) 1505 { 1506 return false; 1507 } 1508 1509 static inline struct bpf_prog *bpf_prog_get_type_path(const char *name, 1510 enum bpf_prog_type type) 1511 { 1512 return ERR_PTR(-EOPNOTSUPP); 1513 } 1514 1515 static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog, 1516 const union bpf_attr *kattr, 1517 union bpf_attr __user *uattr) 1518 { 1519 return -ENOTSUPP; 1520 } 1521 1522 static inline int bpf_prog_test_run_skb(struct bpf_prog *prog, 1523 const union bpf_attr *kattr, 1524 union bpf_attr __user *uattr) 1525 { 1526 return -ENOTSUPP; 1527 } 1528 1529 static inline int bpf_prog_test_run_tracing(struct bpf_prog *prog, 1530 const union bpf_attr *kattr, 1531 union bpf_attr __user *uattr) 1532 { 1533 return -ENOTSUPP; 1534 } 1535 1536 static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 1537 const union bpf_attr *kattr, 1538 union bpf_attr __user *uattr) 1539 { 1540 return -ENOTSUPP; 1541 } 1542 1543 static inline void bpf_map_put(struct bpf_map *map) 1544 { 1545 } 1546 1547 static inline struct bpf_prog *bpf_prog_by_id(u32 id) 1548 { 1549 return ERR_PTR(-ENOTSUPP); 1550 } 1551 1552 static inline const struct bpf_func_proto * 1553 bpf_base_func_proto(enum bpf_func_id func_id) 1554 { 1555 return NULL; 1556 } 1557 #endif /* CONFIG_BPF_SYSCALL */ 1558 1559 static inline struct bpf_prog *bpf_prog_get_type(u32 ufd, 1560 enum bpf_prog_type type) 1561 { 1562 return bpf_prog_get_type_dev(ufd, type, false); 1563 } 1564 1565 bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool); 1566 1567 int bpf_prog_offload_compile(struct bpf_prog *prog); 1568 void bpf_prog_offload_destroy(struct bpf_prog *prog); 1569 int bpf_prog_offload_info_fill(struct bpf_prog_info *info, 1570 struct bpf_prog *prog); 1571 1572 int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map); 1573 1574 int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value); 1575 int bpf_map_offload_update_elem(struct bpf_map *map, 1576 void *key, void *value, u64 flags); 1577 int bpf_map_offload_delete_elem(struct bpf_map *map, void *key); 1578 int bpf_map_offload_get_next_key(struct bpf_map *map, 1579 void *key, void *next_key); 1580 1581 bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map); 1582 1583 struct bpf_offload_dev * 1584 bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv); 1585 void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev); 1586 void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev); 1587 int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev, 1588 struct net_device *netdev); 1589 void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev, 1590 struct net_device *netdev); 1591 bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev); 1592 1593 #if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL) 1594 int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr); 1595 1596 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) 1597 { 1598 return aux->offload_requested; 1599 } 1600 1601 static inline bool bpf_map_is_dev_bound(struct bpf_map *map) 1602 { 1603 return unlikely(map->ops == &bpf_map_offload_ops); 1604 } 1605 1606 struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr); 1607 void bpf_map_offload_map_free(struct bpf_map *map); 1608 #else 1609 static inline int bpf_prog_offload_init(struct bpf_prog *prog, 1610 union bpf_attr *attr) 1611 { 1612 return -EOPNOTSUPP; 1613 } 1614 1615 static inline bool bpf_prog_is_dev_bound(struct bpf_prog_aux *aux) 1616 { 1617 return false; 1618 } 1619 1620 static inline bool bpf_map_is_dev_bound(struct bpf_map *map) 1621 { 1622 return false; 1623 } 1624 1625 static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) 1626 { 1627 return ERR_PTR(-EOPNOTSUPP); 1628 } 1629 1630 static inline void bpf_map_offload_map_free(struct bpf_map *map) 1631 { 1632 } 1633 #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */ 1634 1635 #if defined(CONFIG_BPF_STREAM_PARSER) 1636 int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, 1637 struct bpf_prog *old, u32 which); 1638 int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog); 1639 int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype); 1640 void sock_map_unhash(struct sock *sk); 1641 void sock_map_close(struct sock *sk, long timeout); 1642 #else 1643 static inline int sock_map_prog_update(struct bpf_map *map, 1644 struct bpf_prog *prog, 1645 struct bpf_prog *old, u32 which) 1646 { 1647 return -EOPNOTSUPP; 1648 } 1649 1650 static inline int sock_map_get_from_fd(const union bpf_attr *attr, 1651 struct bpf_prog *prog) 1652 { 1653 return -EINVAL; 1654 } 1655 1656 static inline int sock_map_prog_detach(const union bpf_attr *attr, 1657 enum bpf_prog_type ptype) 1658 { 1659 return -EOPNOTSUPP; 1660 } 1661 #endif /* CONFIG_BPF_STREAM_PARSER */ 1662 1663 #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) 1664 void bpf_sk_reuseport_detach(struct sock *sk); 1665 int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, 1666 void *value); 1667 int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key, 1668 void *value, u64 map_flags); 1669 #else 1670 static inline void bpf_sk_reuseport_detach(struct sock *sk) 1671 { 1672 } 1673 1674 #ifdef CONFIG_BPF_SYSCALL 1675 static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, 1676 void *key, void *value) 1677 { 1678 return -EOPNOTSUPP; 1679 } 1680 1681 static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, 1682 void *key, void *value, 1683 u64 map_flags) 1684 { 1685 return -EOPNOTSUPP; 1686 } 1687 #endif /* CONFIG_BPF_SYSCALL */ 1688 #endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */ 1689 1690 /* verifier prototypes for helper functions called from eBPF programs */ 1691 extern const struct bpf_func_proto bpf_map_lookup_elem_proto; 1692 extern const struct bpf_func_proto bpf_map_update_elem_proto; 1693 extern const struct bpf_func_proto bpf_map_delete_elem_proto; 1694 extern const struct bpf_func_proto bpf_map_push_elem_proto; 1695 extern const struct bpf_func_proto bpf_map_pop_elem_proto; 1696 extern const struct bpf_func_proto bpf_map_peek_elem_proto; 1697 1698 extern const struct bpf_func_proto bpf_get_prandom_u32_proto; 1699 extern const struct bpf_func_proto bpf_get_smp_processor_id_proto; 1700 extern const struct bpf_func_proto bpf_get_numa_node_id_proto; 1701 extern const struct bpf_func_proto bpf_tail_call_proto; 1702 extern const struct bpf_func_proto bpf_ktime_get_ns_proto; 1703 extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto; 1704 extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto; 1705 extern const struct bpf_func_proto bpf_get_current_uid_gid_proto; 1706 extern const struct bpf_func_proto bpf_get_current_comm_proto; 1707 extern const struct bpf_func_proto bpf_get_stackid_proto; 1708 extern const struct bpf_func_proto bpf_get_stack_proto; 1709 extern const struct bpf_func_proto bpf_get_task_stack_proto; 1710 extern const struct bpf_func_proto bpf_get_stackid_proto_pe; 1711 extern const struct bpf_func_proto bpf_get_stack_proto_pe; 1712 extern const struct bpf_func_proto bpf_sock_map_update_proto; 1713 extern const struct bpf_func_proto bpf_sock_hash_update_proto; 1714 extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto; 1715 extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto; 1716 extern const struct bpf_func_proto bpf_msg_redirect_hash_proto; 1717 extern const struct bpf_func_proto bpf_msg_redirect_map_proto; 1718 extern const struct bpf_func_proto bpf_sk_redirect_hash_proto; 1719 extern const struct bpf_func_proto bpf_sk_redirect_map_proto; 1720 extern const struct bpf_func_proto bpf_spin_lock_proto; 1721 extern const struct bpf_func_proto bpf_spin_unlock_proto; 1722 extern const struct bpf_func_proto bpf_get_local_storage_proto; 1723 extern const struct bpf_func_proto bpf_strtol_proto; 1724 extern const struct bpf_func_proto bpf_strtoul_proto; 1725 extern const struct bpf_func_proto bpf_tcp_sock_proto; 1726 extern const struct bpf_func_proto bpf_jiffies64_proto; 1727 extern const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto; 1728 extern const struct bpf_func_proto bpf_event_output_data_proto; 1729 extern const struct bpf_func_proto bpf_ringbuf_output_proto; 1730 extern const struct bpf_func_proto bpf_ringbuf_reserve_proto; 1731 extern const struct bpf_func_proto bpf_ringbuf_submit_proto; 1732 extern const struct bpf_func_proto bpf_ringbuf_discard_proto; 1733 extern const struct bpf_func_proto bpf_ringbuf_query_proto; 1734 extern const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto; 1735 extern const struct bpf_func_proto bpf_skc_to_tcp_sock_proto; 1736 extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto; 1737 extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto; 1738 extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto; 1739 1740 const struct bpf_func_proto *bpf_tracing_func_proto( 1741 enum bpf_func_id func_id, const struct bpf_prog *prog); 1742 1743 const struct bpf_func_proto *tracing_prog_func_proto( 1744 enum bpf_func_id func_id, const struct bpf_prog *prog); 1745 1746 /* Shared helpers among cBPF and eBPF. */ 1747 void bpf_user_rnd_init_once(void); 1748 u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 1749 u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 1750 1751 #if defined(CONFIG_NET) 1752 bool bpf_sock_common_is_valid_access(int off, int size, 1753 enum bpf_access_type type, 1754 struct bpf_insn_access_aux *info); 1755 bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type, 1756 struct bpf_insn_access_aux *info); 1757 u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, 1758 const struct bpf_insn *si, 1759 struct bpf_insn *insn_buf, 1760 struct bpf_prog *prog, 1761 u32 *target_size); 1762 #else 1763 static inline bool bpf_sock_common_is_valid_access(int off, int size, 1764 enum bpf_access_type type, 1765 struct bpf_insn_access_aux *info) 1766 { 1767 return false; 1768 } 1769 static inline bool bpf_sock_is_valid_access(int off, int size, 1770 enum bpf_access_type type, 1771 struct bpf_insn_access_aux *info) 1772 { 1773 return false; 1774 } 1775 static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, 1776 const struct bpf_insn *si, 1777 struct bpf_insn *insn_buf, 1778 struct bpf_prog *prog, 1779 u32 *target_size) 1780 { 1781 return 0; 1782 } 1783 #endif 1784 1785 #ifdef CONFIG_INET 1786 struct sk_reuseport_kern { 1787 struct sk_buff *skb; 1788 struct sock *sk; 1789 struct sock *selected_sk; 1790 void *data_end; 1791 u32 hash; 1792 u32 reuseport_id; 1793 bool bind_inany; 1794 }; 1795 bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type, 1796 struct bpf_insn_access_aux *info); 1797 1798 u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, 1799 const struct bpf_insn *si, 1800 struct bpf_insn *insn_buf, 1801 struct bpf_prog *prog, 1802 u32 *target_size); 1803 1804 bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type, 1805 struct bpf_insn_access_aux *info); 1806 1807 u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, 1808 const struct bpf_insn *si, 1809 struct bpf_insn *insn_buf, 1810 struct bpf_prog *prog, 1811 u32 *target_size); 1812 #else 1813 static inline bool bpf_tcp_sock_is_valid_access(int off, int size, 1814 enum bpf_access_type type, 1815 struct bpf_insn_access_aux *info) 1816 { 1817 return false; 1818 } 1819 1820 static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, 1821 const struct bpf_insn *si, 1822 struct bpf_insn *insn_buf, 1823 struct bpf_prog *prog, 1824 u32 *target_size) 1825 { 1826 return 0; 1827 } 1828 static inline bool bpf_xdp_sock_is_valid_access(int off, int size, 1829 enum bpf_access_type type, 1830 struct bpf_insn_access_aux *info) 1831 { 1832 return false; 1833 } 1834 1835 static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, 1836 const struct bpf_insn *si, 1837 struct bpf_insn *insn_buf, 1838 struct bpf_prog *prog, 1839 u32 *target_size) 1840 { 1841 return 0; 1842 } 1843 #endif /* CONFIG_INET */ 1844 1845 enum bpf_text_poke_type { 1846 BPF_MOD_CALL, 1847 BPF_MOD_JUMP, 1848 }; 1849 1850 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, 1851 void *addr1, void *addr2); 1852 1853 #endif /* _LINUX_BPF_H */ 1854