1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 3 */ 4 #ifndef _LINUX_BPF_H 5 #define _LINUX_BPF_H 1 6 7 #include <uapi/linux/bpf.h> 8 9 #include <linux/workqueue.h> 10 #include <linux/file.h> 11 #include <linux/percpu.h> 12 #include <linux/err.h> 13 #include <linux/rbtree_latch.h> 14 #include <linux/numa.h> 15 #include <linux/mm_types.h> 16 #include <linux/wait.h> 17 #include <linux/u64_stats_sync.h> 18 #include <linux/refcount.h> 19 #include <linux/mutex.h> 20 #include <linux/module.h> 21 #include <linux/kallsyms.h> 22 #include <linux/capability.h> 23 24 struct bpf_verifier_env; 25 struct bpf_verifier_log; 26 struct perf_event; 27 struct bpf_prog; 28 struct bpf_prog_aux; 29 struct bpf_map; 30 struct sock; 31 struct seq_file; 32 struct btf; 33 struct btf_type; 34 struct exception_table_entry; 35 struct seq_operations; 36 struct bpf_iter_aux_info; 37 38 extern struct idr btf_idr; 39 extern spinlock_t btf_idr_lock; 40 41 typedef int (*bpf_iter_init_seq_priv_t)(void *private_data, 42 struct bpf_iter_aux_info *aux); 43 typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data); 44 struct bpf_iter_seq_info { 45 const struct seq_operations *seq_ops; 46 bpf_iter_init_seq_priv_t init_seq_private; 47 bpf_iter_fini_seq_priv_t fini_seq_private; 48 u32 seq_priv_size; 49 }; 50 51 /* map is generic key/value storage optionally accesible by eBPF programs */ 52 struct bpf_map_ops { 53 /* funcs callable from userspace (via syscall) */ 54 int (*map_alloc_check)(union bpf_attr *attr); 55 struct bpf_map *(*map_alloc)(union bpf_attr *attr); 56 void (*map_release)(struct bpf_map *map, struct file *map_file); 57 void (*map_free)(struct bpf_map *map); 58 int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key); 59 void (*map_release_uref)(struct bpf_map *map); 60 void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key); 61 int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr, 62 union bpf_attr __user *uattr); 63 int (*map_lookup_and_delete_batch)(struct bpf_map *map, 64 const union bpf_attr *attr, 65 union bpf_attr __user *uattr); 66 int (*map_update_batch)(struct bpf_map *map, const union bpf_attr *attr, 67 union bpf_attr __user *uattr); 68 int (*map_delete_batch)(struct bpf_map *map, const union bpf_attr *attr, 69 union bpf_attr __user *uattr); 70 71 /* funcs callable from userspace and from eBPF programs */ 72 void *(*map_lookup_elem)(struct bpf_map *map, void *key); 73 int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags); 74 int (*map_delete_elem)(struct bpf_map *map, void *key); 75 int (*map_push_elem)(struct bpf_map *map, void *value, u64 flags); 76 int (*map_pop_elem)(struct bpf_map *map, void *value); 77 int (*map_peek_elem)(struct bpf_map *map, void *value); 78 79 /* funcs called by prog_array and perf_event_array map */ 80 void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file, 81 int fd); 82 void (*map_fd_put_ptr)(void *ptr); 83 u32 (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf); 84 u32 (*map_fd_sys_lookup_elem)(void *ptr); 85 void (*map_seq_show_elem)(struct bpf_map *map, void *key, 86 struct seq_file *m); 87 int (*map_check_btf)(const struct bpf_map *map, 88 const struct btf *btf, 89 const struct btf_type *key_type, 90 const struct btf_type *value_type); 91 92 /* Prog poke tracking helpers. */ 93 int (*map_poke_track)(struct bpf_map *map, struct bpf_prog_aux *aux); 94 void (*map_poke_untrack)(struct bpf_map *map, struct bpf_prog_aux *aux); 95 void (*map_poke_run)(struct bpf_map *map, u32 key, struct bpf_prog *old, 96 struct bpf_prog *new); 97 98 /* Direct value access helpers. */ 99 int (*map_direct_value_addr)(const struct bpf_map *map, 100 u64 *imm, u32 off); 101 int (*map_direct_value_meta)(const struct bpf_map *map, 102 u64 imm, u32 *off); 103 int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma); 104 __poll_t (*map_poll)(struct bpf_map *map, struct file *filp, 105 struct poll_table_struct *pts); 106 107 /* BTF name and id of struct allocated by map_alloc */ 108 const char * const map_btf_name; 109 int *map_btf_id; 110 111 /* bpf_iter info used to open a seq_file */ 112 const struct bpf_iter_seq_info *iter_seq_info; 113 }; 114 115 struct bpf_map_memory { 116 u32 pages; 117 struct user_struct *user; 118 }; 119 120 struct bpf_map { 121 /* The first two cachelines with read-mostly members of which some 122 * are also accessed in fast-path (e.g. ops, max_entries). 123 */ 124 const struct bpf_map_ops *ops ____cacheline_aligned; 125 struct bpf_map *inner_map_meta; 126 #ifdef CONFIG_SECURITY 127 void *security; 128 #endif 129 enum bpf_map_type map_type; 130 u32 key_size; 131 u32 value_size; 132 u32 max_entries; 133 u32 map_flags; 134 int spin_lock_off; /* >=0 valid offset, <0 error */ 135 u32 id; 136 int numa_node; 137 u32 btf_key_type_id; 138 u32 btf_value_type_id; 139 struct btf *btf; 140 struct bpf_map_memory memory; 141 char name[BPF_OBJ_NAME_LEN]; 142 u32 btf_vmlinux_value_type_id; 143 bool bypass_spec_v1; 144 bool frozen; /* write-once; write-protected by freeze_mutex */ 145 /* 22 bytes hole */ 146 147 /* The 3rd and 4th cacheline with misc members to avoid false sharing 148 * particularly with refcounting. 149 */ 150 atomic64_t refcnt ____cacheline_aligned; 151 atomic64_t usercnt; 152 struct work_struct work; 153 struct mutex freeze_mutex; 154 u64 writecnt; /* writable mmap cnt; protected by freeze_mutex */ 155 }; 156 157 static inline bool map_value_has_spin_lock(const struct bpf_map *map) 158 { 159 return map->spin_lock_off >= 0; 160 } 161 162 static inline void check_and_init_map_lock(struct bpf_map *map, void *dst) 163 { 164 if (likely(!map_value_has_spin_lock(map))) 165 return; 166 *(struct bpf_spin_lock *)(dst + map->spin_lock_off) = 167 (struct bpf_spin_lock){}; 168 } 169 170 /* copy everything but bpf_spin_lock */ 171 static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) 172 { 173 if (unlikely(map_value_has_spin_lock(map))) { 174 u32 off = map->spin_lock_off; 175 176 memcpy(dst, src, off); 177 memcpy(dst + off + sizeof(struct bpf_spin_lock), 178 src + off + sizeof(struct bpf_spin_lock), 179 map->value_size - off - sizeof(struct bpf_spin_lock)); 180 } else { 181 memcpy(dst, src, map->value_size); 182 } 183 } 184 void copy_map_value_locked(struct bpf_map *map, void *dst, void *src, 185 bool lock_src); 186 int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size); 187 188 struct bpf_offload_dev; 189 struct bpf_offloaded_map; 190 191 struct bpf_map_dev_ops { 192 int (*map_get_next_key)(struct bpf_offloaded_map *map, 193 void *key, void *next_key); 194 int (*map_lookup_elem)(struct bpf_offloaded_map *map, 195 void *key, void *value); 196 int (*map_update_elem)(struct bpf_offloaded_map *map, 197 void *key, void *value, u64 flags); 198 int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key); 199 }; 200 201 struct bpf_offloaded_map { 202 struct bpf_map map; 203 struct net_device *netdev; 204 const struct bpf_map_dev_ops *dev_ops; 205 void *dev_priv; 206 struct list_head offloads; 207 }; 208 209 static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map) 210 { 211 return container_of(map, struct bpf_offloaded_map, map); 212 } 213 214 static inline bool bpf_map_offload_neutral(const struct bpf_map *map) 215 { 216 return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 217 } 218 219 static inline bool bpf_map_support_seq_show(const struct bpf_map *map) 220 { 221 return (map->btf_value_type_id || map->btf_vmlinux_value_type_id) && 222 map->ops->map_seq_show_elem; 223 } 224 225 int map_check_no_btf(const struct bpf_map *map, 226 const struct btf *btf, 227 const struct btf_type *key_type, 228 const struct btf_type *value_type); 229 230 extern const struct bpf_map_ops bpf_map_offload_ops; 231 232 /* function argument constraints */ 233 enum bpf_arg_type { 234 ARG_DONTCARE = 0, /* unused argument in helper function */ 235 236 /* the following constraints used to prototype 237 * bpf_map_lookup/update/delete_elem() functions 238 */ 239 ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */ 240 ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */ 241 ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */ 242 ARG_PTR_TO_UNINIT_MAP_VALUE, /* pointer to valid memory used to store a map value */ 243 ARG_PTR_TO_MAP_VALUE_OR_NULL, /* pointer to stack used as map value or NULL */ 244 245 /* the following constraints used to prototype bpf_memcmp() and other 246 * functions that access data on eBPF program stack 247 */ 248 ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ 249 ARG_PTR_TO_MEM_OR_NULL, /* pointer to valid memory or NULL */ 250 ARG_PTR_TO_UNINIT_MEM, /* pointer to memory does not need to be initialized, 251 * helper function must fill all bytes or clear 252 * them in error case. 253 */ 254 255 ARG_CONST_SIZE, /* number of bytes accessed from memory */ 256 ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ 257 258 ARG_PTR_TO_CTX, /* pointer to context */ 259 ARG_PTR_TO_CTX_OR_NULL, /* pointer to context or NULL */ 260 ARG_ANYTHING, /* any (initialized) argument is ok */ 261 ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */ 262 ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */ 263 ARG_PTR_TO_INT, /* pointer to int */ 264 ARG_PTR_TO_LONG, /* pointer to long */ 265 ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */ 266 ARG_PTR_TO_SOCKET_OR_NULL, /* pointer to bpf_sock (fullsock) or NULL */ 267 ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */ 268 ARG_PTR_TO_ALLOC_MEM, /* pointer to dynamically allocated memory */ 269 ARG_PTR_TO_ALLOC_MEM_OR_NULL, /* pointer to dynamically allocated memory or NULL */ 270 ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */ 271 }; 272 273 /* type of values returned from helper functions */ 274 enum bpf_return_type { 275 RET_INTEGER, /* function returns integer */ 276 RET_VOID, /* function doesn't return anything */ 277 RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */ 278 RET_PTR_TO_MAP_VALUE_OR_NULL, /* returns a pointer to map elem value or NULL */ 279 RET_PTR_TO_SOCKET_OR_NULL, /* returns a pointer to a socket or NULL */ 280 RET_PTR_TO_TCP_SOCK_OR_NULL, /* returns a pointer to a tcp_sock or NULL */ 281 RET_PTR_TO_SOCK_COMMON_OR_NULL, /* returns a pointer to a sock_common or NULL */ 282 RET_PTR_TO_ALLOC_MEM_OR_NULL, /* returns a pointer to dynamically allocated memory or NULL */ 283 RET_PTR_TO_BTF_ID_OR_NULL, /* returns a pointer to a btf_id or NULL */ 284 }; 285 286 /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs 287 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL 288 * instructions after verifying 289 */ 290 struct bpf_func_proto { 291 u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 292 bool gpl_only; 293 bool pkt_access; 294 enum bpf_return_type ret_type; 295 union { 296 struct { 297 enum bpf_arg_type arg1_type; 298 enum bpf_arg_type arg2_type; 299 enum bpf_arg_type arg3_type; 300 enum bpf_arg_type arg4_type; 301 enum bpf_arg_type arg5_type; 302 }; 303 enum bpf_arg_type arg_type[5]; 304 }; 305 int *btf_id; /* BTF ids of arguments */ 306 bool (*check_btf_id)(u32 btf_id, u32 arg); /* if the argument btf_id is 307 * valid. Often used if more 308 * than one btf id is permitted 309 * for this argument. 310 */ 311 int *ret_btf_id; /* return value btf_id */ 312 }; 313 314 /* bpf_context is intentionally undefined structure. Pointer to bpf_context is 315 * the first argument to eBPF programs. 316 * For socket filters: 'struct bpf_context *' == 'struct sk_buff *' 317 */ 318 struct bpf_context; 319 320 enum bpf_access_type { 321 BPF_READ = 1, 322 BPF_WRITE = 2 323 }; 324 325 /* types of values stored in eBPF registers */ 326 /* Pointer types represent: 327 * pointer 328 * pointer + imm 329 * pointer + (u16) var 330 * pointer + (u16) var + imm 331 * if (range > 0) then [ptr, ptr + range - off) is safe to access 332 * if (id > 0) means that some 'var' was added 333 * if (off > 0) means that 'imm' was added 334 */ 335 enum bpf_reg_type { 336 NOT_INIT = 0, /* nothing was written into register */ 337 SCALAR_VALUE, /* reg doesn't contain a valid pointer */ 338 PTR_TO_CTX, /* reg points to bpf_context */ 339 CONST_PTR_TO_MAP, /* reg points to struct bpf_map */ 340 PTR_TO_MAP_VALUE, /* reg points to map element value */ 341 PTR_TO_MAP_VALUE_OR_NULL,/* points to map elem value or NULL */ 342 PTR_TO_STACK, /* reg == frame_pointer + offset */ 343 PTR_TO_PACKET_META, /* skb->data - meta_len */ 344 PTR_TO_PACKET, /* reg points to skb->data */ 345 PTR_TO_PACKET_END, /* skb->data + headlen */ 346 PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */ 347 PTR_TO_SOCKET, /* reg points to struct bpf_sock */ 348 PTR_TO_SOCKET_OR_NULL, /* reg points to struct bpf_sock or NULL */ 349 PTR_TO_SOCK_COMMON, /* reg points to sock_common */ 350 PTR_TO_SOCK_COMMON_OR_NULL, /* reg points to sock_common or NULL */ 351 PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */ 352 PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */ 353 PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */ 354 PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */ 355 PTR_TO_BTF_ID, /* reg points to kernel struct */ 356 PTR_TO_BTF_ID_OR_NULL, /* reg points to kernel struct or NULL */ 357 PTR_TO_MEM, /* reg points to valid memory region */ 358 PTR_TO_MEM_OR_NULL, /* reg points to valid memory region or NULL */ 359 PTR_TO_RDONLY_BUF, /* reg points to a readonly buffer */ 360 PTR_TO_RDONLY_BUF_OR_NULL, /* reg points to a readonly buffer or NULL */ 361 PTR_TO_RDWR_BUF, /* reg points to a read/write buffer */ 362 PTR_TO_RDWR_BUF_OR_NULL, /* reg points to a read/write buffer or NULL */ 363 }; 364 365 /* The information passed from prog-specific *_is_valid_access 366 * back to the verifier. 367 */ 368 struct bpf_insn_access_aux { 369 enum bpf_reg_type reg_type; 370 union { 371 int ctx_field_size; 372 u32 btf_id; 373 }; 374 struct bpf_verifier_log *log; /* for verbose logs */ 375 }; 376 377 static inline void 378 bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size) 379 { 380 aux->ctx_field_size = size; 381 } 382 383 struct bpf_prog_ops { 384 int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, 385 union bpf_attr __user *uattr); 386 }; 387 388 struct bpf_verifier_ops { 389 /* return eBPF function prototype for verification */ 390 const struct bpf_func_proto * 391 (*get_func_proto)(enum bpf_func_id func_id, 392 const struct bpf_prog *prog); 393 394 /* return true if 'size' wide access at offset 'off' within bpf_context 395 * with 'type' (read or write) is allowed 396 */ 397 bool (*is_valid_access)(int off, int size, enum bpf_access_type type, 398 const struct bpf_prog *prog, 399 struct bpf_insn_access_aux *info); 400 int (*gen_prologue)(struct bpf_insn *insn, bool direct_write, 401 const struct bpf_prog *prog); 402 int (*gen_ld_abs)(const struct bpf_insn *orig, 403 struct bpf_insn *insn_buf); 404 u32 (*convert_ctx_access)(enum bpf_access_type type, 405 const struct bpf_insn *src, 406 struct bpf_insn *dst, 407 struct bpf_prog *prog, u32 *target_size); 408 int (*btf_struct_access)(struct bpf_verifier_log *log, 409 const struct btf_type *t, int off, int size, 410 enum bpf_access_type atype, 411 u32 *next_btf_id); 412 }; 413 414 struct bpf_prog_offload_ops { 415 /* verifier basic callbacks */ 416 int (*insn_hook)(struct bpf_verifier_env *env, 417 int insn_idx, int prev_insn_idx); 418 int (*finalize)(struct bpf_verifier_env *env); 419 /* verifier optimization callbacks (called after .finalize) */ 420 int (*replace_insn)(struct bpf_verifier_env *env, u32 off, 421 struct bpf_insn *insn); 422 int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt); 423 /* program management callbacks */ 424 int (*prepare)(struct bpf_prog *prog); 425 int (*translate)(struct bpf_prog *prog); 426 void (*destroy)(struct bpf_prog *prog); 427 }; 428 429 struct bpf_prog_offload { 430 struct bpf_prog *prog; 431 struct net_device *netdev; 432 struct bpf_offload_dev *offdev; 433 void *dev_priv; 434 struct list_head offloads; 435 bool dev_state; 436 bool opt_failed; 437 void *jited_image; 438 u32 jited_len; 439 }; 440 441 enum bpf_cgroup_storage_type { 442 BPF_CGROUP_STORAGE_SHARED, 443 BPF_CGROUP_STORAGE_PERCPU, 444 __BPF_CGROUP_STORAGE_MAX 445 }; 446 447 #define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX 448 449 /* The longest tracepoint has 12 args. 450 * See include/trace/bpf_probe.h 451 */ 452 #define MAX_BPF_FUNC_ARGS 12 453 454 struct bpf_prog_stats { 455 u64 cnt; 456 u64 nsecs; 457 struct u64_stats_sync syncp; 458 } __aligned(2 * sizeof(u64)); 459 460 struct btf_func_model { 461 u8 ret_size; 462 u8 nr_args; 463 u8 arg_size[MAX_BPF_FUNC_ARGS]; 464 }; 465 466 /* Restore arguments before returning from trampoline to let original function 467 * continue executing. This flag is used for fentry progs when there are no 468 * fexit progs. 469 */ 470 #define BPF_TRAMP_F_RESTORE_REGS BIT(0) 471 /* Call original function after fentry progs, but before fexit progs. 472 * Makes sense for fentry/fexit, normal calls and indirect calls. 473 */ 474 #define BPF_TRAMP_F_CALL_ORIG BIT(1) 475 /* Skip current frame and return to parent. Makes sense for fentry/fexit 476 * programs only. Should not be used with normal calls and indirect calls. 477 */ 478 #define BPF_TRAMP_F_SKIP_FRAME BIT(2) 479 480 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 481 * bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2 482 */ 483 #define BPF_MAX_TRAMP_PROGS 40 484 485 struct bpf_tramp_progs { 486 struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS]; 487 int nr_progs; 488 }; 489 490 /* Different use cases for BPF trampoline: 491 * 1. replace nop at the function entry (kprobe equivalent) 492 * flags = BPF_TRAMP_F_RESTORE_REGS 493 * fentry = a set of programs to run before returning from trampoline 494 * 495 * 2. replace nop at the function entry (kprobe + kretprobe equivalent) 496 * flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME 497 * orig_call = fentry_ip + MCOUNT_INSN_SIZE 498 * fentry = a set of program to run before calling original function 499 * fexit = a set of program to run after original function 500 * 501 * 3. replace direct call instruction anywhere in the function body 502 * or assign a function pointer for indirect call (like tcp_congestion_ops->cong_avoid) 503 * With flags = 0 504 * fentry = a set of programs to run before returning from trampoline 505 * With flags = BPF_TRAMP_F_CALL_ORIG 506 * orig_call = original callback addr or direct function addr 507 * fentry = a set of program to run before calling original function 508 * fexit = a set of program to run after original function 509 */ 510 int arch_prepare_bpf_trampoline(void *image, void *image_end, 511 const struct btf_func_model *m, u32 flags, 512 struct bpf_tramp_progs *tprogs, 513 void *orig_call); 514 /* these two functions are called from generated trampoline */ 515 u64 notrace __bpf_prog_enter(void); 516 void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start); 517 518 struct bpf_ksym { 519 unsigned long start; 520 unsigned long end; 521 char name[KSYM_NAME_LEN]; 522 struct list_head lnode; 523 struct latch_tree_node tnode; 524 bool prog; 525 }; 526 527 enum bpf_tramp_prog_type { 528 BPF_TRAMP_FENTRY, 529 BPF_TRAMP_FEXIT, 530 BPF_TRAMP_MODIFY_RETURN, 531 BPF_TRAMP_MAX, 532 BPF_TRAMP_REPLACE, /* more than MAX */ 533 }; 534 535 struct bpf_trampoline { 536 /* hlist for trampoline_table */ 537 struct hlist_node hlist; 538 /* serializes access to fields of this trampoline */ 539 struct mutex mutex; 540 refcount_t refcnt; 541 u64 key; 542 struct { 543 struct btf_func_model model; 544 void *addr; 545 bool ftrace_managed; 546 } func; 547 /* if !NULL this is BPF_PROG_TYPE_EXT program that extends another BPF 548 * program by replacing one of its functions. func.addr is the address 549 * of the function it replaced. 550 */ 551 struct bpf_prog *extension_prog; 552 /* list of BPF programs using this trampoline */ 553 struct hlist_head progs_hlist[BPF_TRAMP_MAX]; 554 /* Number of attached programs. A counter per kind. */ 555 int progs_cnt[BPF_TRAMP_MAX]; 556 /* Executable image of trampoline */ 557 void *image; 558 u64 selector; 559 struct bpf_ksym ksym; 560 }; 561 562 #define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */ 563 564 struct bpf_dispatcher_prog { 565 struct bpf_prog *prog; 566 refcount_t users; 567 }; 568 569 struct bpf_dispatcher { 570 /* dispatcher mutex */ 571 struct mutex mutex; 572 void *func; 573 struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX]; 574 int num_progs; 575 void *image; 576 u32 image_off; 577 struct bpf_ksym ksym; 578 }; 579 580 static __always_inline unsigned int bpf_dispatcher_nop_func( 581 const void *ctx, 582 const struct bpf_insn *insnsi, 583 unsigned int (*bpf_func)(const void *, 584 const struct bpf_insn *)) 585 { 586 return bpf_func(ctx, insnsi); 587 } 588 #ifdef CONFIG_BPF_JIT 589 struct bpf_trampoline *bpf_trampoline_lookup(u64 key); 590 int bpf_trampoline_link_prog(struct bpf_prog *prog); 591 int bpf_trampoline_unlink_prog(struct bpf_prog *prog); 592 void bpf_trampoline_put(struct bpf_trampoline *tr); 593 #define BPF_DISPATCHER_INIT(_name) { \ 594 .mutex = __MUTEX_INITIALIZER(_name.mutex), \ 595 .func = &_name##_func, \ 596 .progs = {}, \ 597 .num_progs = 0, \ 598 .image = NULL, \ 599 .image_off = 0, \ 600 .ksym = { \ 601 .name = #_name, \ 602 .lnode = LIST_HEAD_INIT(_name.ksym.lnode), \ 603 }, \ 604 } 605 606 #define DEFINE_BPF_DISPATCHER(name) \ 607 noinline unsigned int bpf_dispatcher_##name##_func( \ 608 const void *ctx, \ 609 const struct bpf_insn *insnsi, \ 610 unsigned int (*bpf_func)(const void *, \ 611 const struct bpf_insn *)) \ 612 { \ 613 return bpf_func(ctx, insnsi); \ 614 } \ 615 EXPORT_SYMBOL(bpf_dispatcher_##name##_func); \ 616 struct bpf_dispatcher bpf_dispatcher_##name = \ 617 BPF_DISPATCHER_INIT(bpf_dispatcher_##name); 618 #define DECLARE_BPF_DISPATCHER(name) \ 619 unsigned int bpf_dispatcher_##name##_func( \ 620 const void *ctx, \ 621 const struct bpf_insn *insnsi, \ 622 unsigned int (*bpf_func)(const void *, \ 623 const struct bpf_insn *)); \ 624 extern struct bpf_dispatcher bpf_dispatcher_##name; 625 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func 626 #define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name) 627 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, 628 struct bpf_prog *to); 629 /* Called only from JIT-enabled code, so there's no need for stubs. */ 630 void *bpf_jit_alloc_exec_page(void); 631 void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym); 632 void bpf_image_ksym_del(struct bpf_ksym *ksym); 633 void bpf_ksym_add(struct bpf_ksym *ksym); 634 void bpf_ksym_del(struct bpf_ksym *ksym); 635 #else 636 static inline struct bpf_trampoline *bpf_trampoline_lookup(u64 key) 637 { 638 return NULL; 639 } 640 static inline int bpf_trampoline_link_prog(struct bpf_prog *prog) 641 { 642 return -ENOTSUPP; 643 } 644 static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog) 645 { 646 return -ENOTSUPP; 647 } 648 static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {} 649 #define DEFINE_BPF_DISPATCHER(name) 650 #define DECLARE_BPF_DISPATCHER(name) 651 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nop_func 652 #define BPF_DISPATCHER_PTR(name) NULL 653 static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, 654 struct bpf_prog *from, 655 struct bpf_prog *to) {} 656 static inline bool is_bpf_image_address(unsigned long address) 657 { 658 return false; 659 } 660 #endif 661 662 struct bpf_func_info_aux { 663 u16 linkage; 664 bool unreliable; 665 }; 666 667 enum bpf_jit_poke_reason { 668 BPF_POKE_REASON_TAIL_CALL, 669 }; 670 671 /* Descriptor of pokes pointing /into/ the JITed image. */ 672 struct bpf_jit_poke_descriptor { 673 void *ip; 674 union { 675 struct { 676 struct bpf_map *map; 677 u32 key; 678 } tail_call; 679 }; 680 bool ip_stable; 681 u8 adj_off; 682 u16 reason; 683 }; 684 685 /* reg_type info for ctx arguments */ 686 struct bpf_ctx_arg_aux { 687 u32 offset; 688 enum bpf_reg_type reg_type; 689 u32 btf_id; 690 }; 691 692 struct bpf_prog_aux { 693 atomic64_t refcnt; 694 u32 used_map_cnt; 695 u32 max_ctx_offset; 696 u32 max_pkt_offset; 697 u32 max_tp_access; 698 u32 stack_depth; 699 u32 id; 700 u32 func_cnt; /* used by non-func prog as the number of func progs */ 701 u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */ 702 u32 attach_btf_id; /* in-kernel BTF type id to attach to */ 703 u32 ctx_arg_info_size; 704 u32 max_rdonly_access; 705 u32 max_rdwr_access; 706 const struct bpf_ctx_arg_aux *ctx_arg_info; 707 struct bpf_prog *linked_prog; 708 bool verifier_zext; /* Zero extensions has been inserted by verifier. */ 709 bool offload_requested; 710 bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ 711 bool func_proto_unreliable; 712 enum bpf_tramp_prog_type trampoline_prog_type; 713 struct bpf_trampoline *trampoline; 714 struct hlist_node tramp_hlist; 715 /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ 716 const struct btf_type *attach_func_proto; 717 /* function name for valid attach_btf_id */ 718 const char *attach_func_name; 719 struct bpf_prog **func; 720 void *jit_data; /* JIT specific data. arch dependent */ 721 struct bpf_jit_poke_descriptor *poke_tab; 722 u32 size_poke_tab; 723 struct bpf_ksym ksym; 724 const struct bpf_prog_ops *ops; 725 struct bpf_map **used_maps; 726 struct bpf_prog *prog; 727 struct user_struct *user; 728 u64 load_time; /* ns since boottime */ 729 struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 730 char name[BPF_OBJ_NAME_LEN]; 731 #ifdef CONFIG_SECURITY 732 void *security; 733 #endif 734 struct bpf_prog_offload *offload; 735 struct btf *btf; 736 struct bpf_func_info *func_info; 737 struct bpf_func_info_aux *func_info_aux; 738 /* bpf_line_info loaded from userspace. linfo->insn_off 739 * has the xlated insn offset. 740 * Both the main and sub prog share the same linfo. 741 * The subprog can access its first linfo by 742 * using the linfo_idx. 743 */ 744 struct bpf_line_info *linfo; 745 /* jited_linfo is the jited addr of the linfo. It has a 746 * one to one mapping to linfo: 747 * jited_linfo[i] is the jited addr for the linfo[i]->insn_off. 748 * Both the main and sub prog share the same jited_linfo. 749 * The subprog can access its first jited_linfo by 750 * using the linfo_idx. 751 */ 752 void **jited_linfo; 753 u32 func_info_cnt; 754 u32 nr_linfo; 755 /* subprog can use linfo_idx to access its first linfo and 756 * jited_linfo. 757 * main prog always has linfo_idx == 0 758 */ 759 u32 linfo_idx; 760 u32 num_exentries; 761 struct exception_table_entry *extable; 762 struct bpf_prog_stats __percpu *stats; 763 union { 764 struct work_struct work; 765 struct rcu_head rcu; 766 }; 767 }; 768 769 struct bpf_array_aux { 770 /* 'Ownership' of prog array is claimed by the first program that 771 * is going to use this map or by the first program which FD is 772 * stored in the map to make sure that all callers and callees have 773 * the same prog type and JITed flag. 774 */ 775 enum bpf_prog_type type; 776 bool jited; 777 /* Programs with direct jumps into programs part of this array. */ 778 struct list_head poke_progs; 779 struct bpf_map *map; 780 struct mutex poke_mutex; 781 struct work_struct work; 782 }; 783 784 struct bpf_link { 785 atomic64_t refcnt; 786 u32 id; 787 enum bpf_link_type type; 788 const struct bpf_link_ops *ops; 789 struct bpf_prog *prog; 790 struct work_struct work; 791 }; 792 793 struct bpf_link_ops { 794 void (*release)(struct bpf_link *link); 795 void (*dealloc)(struct bpf_link *link); 796 int (*detach)(struct bpf_link *link); 797 int (*update_prog)(struct bpf_link *link, struct bpf_prog *new_prog, 798 struct bpf_prog *old_prog); 799 void (*show_fdinfo)(const struct bpf_link *link, struct seq_file *seq); 800 int (*fill_link_info)(const struct bpf_link *link, 801 struct bpf_link_info *info); 802 }; 803 804 struct bpf_link_primer { 805 struct bpf_link *link; 806 struct file *file; 807 int fd; 808 u32 id; 809 }; 810 811 struct bpf_struct_ops_value; 812 struct btf_type; 813 struct btf_member; 814 815 #define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64 816 struct bpf_struct_ops { 817 const struct bpf_verifier_ops *verifier_ops; 818 int (*init)(struct btf *btf); 819 int (*check_member)(const struct btf_type *t, 820 const struct btf_member *member); 821 int (*init_member)(const struct btf_type *t, 822 const struct btf_member *member, 823 void *kdata, const void *udata); 824 int (*reg)(void *kdata); 825 void (*unreg)(void *kdata); 826 const struct btf_type *type; 827 const struct btf_type *value_type; 828 const char *name; 829 struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; 830 u32 type_id; 831 u32 value_id; 832 }; 833 834 #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) 835 #define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA)) 836 const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id); 837 void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log); 838 bool bpf_struct_ops_get(const void *kdata); 839 void bpf_struct_ops_put(const void *kdata); 840 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, 841 void *value); 842 static inline bool bpf_try_module_get(const void *data, struct module *owner) 843 { 844 if (owner == BPF_MODULE_OWNER) 845 return bpf_struct_ops_get(data); 846 else 847 return try_module_get(owner); 848 } 849 static inline void bpf_module_put(const void *data, struct module *owner) 850 { 851 if (owner == BPF_MODULE_OWNER) 852 bpf_struct_ops_put(data); 853 else 854 module_put(owner); 855 } 856 #else 857 static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id) 858 { 859 return NULL; 860 } 861 static inline void bpf_struct_ops_init(struct btf *btf, 862 struct bpf_verifier_log *log) 863 { 864 } 865 static inline bool bpf_try_module_get(const void *data, struct module *owner) 866 { 867 return try_module_get(owner); 868 } 869 static inline void bpf_module_put(const void *data, struct module *owner) 870 { 871 module_put(owner); 872 } 873 static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, 874 void *key, 875 void *value) 876 { 877 return -EINVAL; 878 } 879 #endif 880 881 struct bpf_array { 882 struct bpf_map map; 883 u32 elem_size; 884 u32 index_mask; 885 struct bpf_array_aux *aux; 886 union { 887 char value[0] __aligned(8); 888 void *ptrs[0] __aligned(8); 889 void __percpu *pptrs[0] __aligned(8); 890 }; 891 }; 892 893 #define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */ 894 #define MAX_TAIL_CALL_CNT 32 895 896 #define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \ 897 BPF_F_RDONLY_PROG | \ 898 BPF_F_WRONLY | \ 899 BPF_F_WRONLY_PROG) 900 901 #define BPF_MAP_CAN_READ BIT(0) 902 #define BPF_MAP_CAN_WRITE BIT(1) 903 904 static inline u32 bpf_map_flags_to_cap(struct bpf_map *map) 905 { 906 u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); 907 908 /* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is 909 * not possible. 910 */ 911 if (access_flags & BPF_F_RDONLY_PROG) 912 return BPF_MAP_CAN_READ; 913 else if (access_flags & BPF_F_WRONLY_PROG) 914 return BPF_MAP_CAN_WRITE; 915 else 916 return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE; 917 } 918 919 static inline bool bpf_map_flags_access_ok(u32 access_flags) 920 { 921 return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) != 922 (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); 923 } 924 925 struct bpf_event_entry { 926 struct perf_event *event; 927 struct file *perf_file; 928 struct file *map_file; 929 struct rcu_head rcu; 930 }; 931 932 bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp); 933 int bpf_prog_calc_tag(struct bpf_prog *fp); 934 const char *kernel_type_name(u32 btf_type_id); 935 936 const struct bpf_func_proto *bpf_get_trace_printk_proto(void); 937 938 typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src, 939 unsigned long off, unsigned long len); 940 typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type, 941 const struct bpf_insn *src, 942 struct bpf_insn *dst, 943 struct bpf_prog *prog, 944 u32 *target_size); 945 946 u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, 947 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy); 948 949 /* an array of programs to be executed under rcu_lock. 950 * 951 * Typical usage: 952 * ret = BPF_PROG_RUN_ARRAY(&bpf_prog_array, ctx, BPF_PROG_RUN); 953 * 954 * the structure returned by bpf_prog_array_alloc() should be populated 955 * with program pointers and the last pointer must be NULL. 956 * The user has to keep refcnt on the program and make sure the program 957 * is removed from the array before bpf_prog_put(). 958 * The 'struct bpf_prog_array *' should only be replaced with xchg() 959 * since other cpus are walking the array of pointers in parallel. 960 */ 961 struct bpf_prog_array_item { 962 struct bpf_prog *prog; 963 struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 964 }; 965 966 struct bpf_prog_array { 967 struct rcu_head rcu; 968 struct bpf_prog_array_item items[]; 969 }; 970 971 struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags); 972 void bpf_prog_array_free(struct bpf_prog_array *progs); 973 int bpf_prog_array_length(struct bpf_prog_array *progs); 974 bool bpf_prog_array_is_empty(struct bpf_prog_array *array); 975 int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs, 976 __u32 __user *prog_ids, u32 cnt); 977 978 void bpf_prog_array_delete_safe(struct bpf_prog_array *progs, 979 struct bpf_prog *old_prog); 980 int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index); 981 int bpf_prog_array_update_at(struct bpf_prog_array *array, int index, 982 struct bpf_prog *prog); 983 int bpf_prog_array_copy_info(struct bpf_prog_array *array, 984 u32 *prog_ids, u32 request_cnt, 985 u32 *prog_cnt); 986 int bpf_prog_array_copy(struct bpf_prog_array *old_array, 987 struct bpf_prog *exclude_prog, 988 struct bpf_prog *include_prog, 989 struct bpf_prog_array **new_array); 990 991 #define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null) \ 992 ({ \ 993 struct bpf_prog_array_item *_item; \ 994 struct bpf_prog *_prog; \ 995 struct bpf_prog_array *_array; \ 996 u32 _ret = 1; \ 997 migrate_disable(); \ 998 rcu_read_lock(); \ 999 _array = rcu_dereference(array); \ 1000 if (unlikely(check_non_null && !_array))\ 1001 goto _out; \ 1002 _item = &_array->items[0]; \ 1003 while ((_prog = READ_ONCE(_item->prog))) { \ 1004 bpf_cgroup_storage_set(_item->cgroup_storage); \ 1005 _ret &= func(_prog, ctx); \ 1006 _item++; \ 1007 } \ 1008 _out: \ 1009 rcu_read_unlock(); \ 1010 migrate_enable(); \ 1011 _ret; \ 1012 }) 1013 1014 /* To be used by __cgroup_bpf_run_filter_skb for EGRESS BPF progs 1015 * so BPF programs can request cwr for TCP packets. 1016 * 1017 * Current cgroup skb programs can only return 0 or 1 (0 to drop the 1018 * packet. This macro changes the behavior so the low order bit 1019 * indicates whether the packet should be dropped (0) or not (1) 1020 * and the next bit is a congestion notification bit. This could be 1021 * used by TCP to call tcp_enter_cwr() 1022 * 1023 * Hence, new allowed return values of CGROUP EGRESS BPF programs are: 1024 * 0: drop packet 1025 * 1: keep packet 1026 * 2: drop packet and cn 1027 * 3: keep packet and cn 1028 * 1029 * This macro then converts it to one of the NET_XMIT or an error 1030 * code that is then interpreted as drop packet (and no cn): 1031 * 0: NET_XMIT_SUCCESS skb should be transmitted 1032 * 1: NET_XMIT_DROP skb should be dropped and cn 1033 * 2: NET_XMIT_CN skb should be transmitted and cn 1034 * 3: -EPERM skb should be dropped 1035 */ 1036 #define BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY(array, ctx, func) \ 1037 ({ \ 1038 struct bpf_prog_array_item *_item; \ 1039 struct bpf_prog *_prog; \ 1040 struct bpf_prog_array *_array; \ 1041 u32 ret; \ 1042 u32 _ret = 1; \ 1043 u32 _cn = 0; \ 1044 migrate_disable(); \ 1045 rcu_read_lock(); \ 1046 _array = rcu_dereference(array); \ 1047 _item = &_array->items[0]; \ 1048 while ((_prog = READ_ONCE(_item->prog))) { \ 1049 bpf_cgroup_storage_set(_item->cgroup_storage); \ 1050 ret = func(_prog, ctx); \ 1051 _ret &= (ret & 1); \ 1052 _cn |= (ret & 2); \ 1053 _item++; \ 1054 } \ 1055 rcu_read_unlock(); \ 1056 migrate_enable(); \ 1057 if (_ret) \ 1058 _ret = (_cn ? NET_XMIT_CN : NET_XMIT_SUCCESS); \ 1059 else \ 1060 _ret = (_cn ? NET_XMIT_DROP : -EPERM); \ 1061 _ret; \ 1062 }) 1063 1064 #define BPF_PROG_RUN_ARRAY(array, ctx, func) \ 1065 __BPF_PROG_RUN_ARRAY(array, ctx, func, false) 1066 1067 #define BPF_PROG_RUN_ARRAY_CHECK(array, ctx, func) \ 1068 __BPF_PROG_RUN_ARRAY(array, ctx, func, true) 1069 1070 #ifdef CONFIG_BPF_SYSCALL 1071 DECLARE_PER_CPU(int, bpf_prog_active); 1072 extern struct mutex bpf_stats_enabled_mutex; 1073 1074 /* 1075 * Block execution of BPF programs attached to instrumentation (perf, 1076 * kprobes, tracepoints) to prevent deadlocks on map operations as any of 1077 * these events can happen inside a region which holds a map bucket lock 1078 * and can deadlock on it. 1079 * 1080 * Use the preemption safe inc/dec variants on RT because migrate disable 1081 * is preemptible on RT and preemption in the middle of the RMW operation 1082 * might lead to inconsistent state. Use the raw variants for non RT 1083 * kernels as migrate_disable() maps to preempt_disable() so the slightly 1084 * more expensive save operation can be avoided. 1085 */ 1086 static inline void bpf_disable_instrumentation(void) 1087 { 1088 migrate_disable(); 1089 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 1090 this_cpu_inc(bpf_prog_active); 1091 else 1092 __this_cpu_inc(bpf_prog_active); 1093 } 1094 1095 static inline void bpf_enable_instrumentation(void) 1096 { 1097 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 1098 this_cpu_dec(bpf_prog_active); 1099 else 1100 __this_cpu_dec(bpf_prog_active); 1101 migrate_enable(); 1102 } 1103 1104 extern const struct file_operations bpf_map_fops; 1105 extern const struct file_operations bpf_prog_fops; 1106 extern const struct file_operations bpf_iter_fops; 1107 1108 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 1109 extern const struct bpf_prog_ops _name ## _prog_ops; \ 1110 extern const struct bpf_verifier_ops _name ## _verifier_ops; 1111 #define BPF_MAP_TYPE(_id, _ops) \ 1112 extern const struct bpf_map_ops _ops; 1113 #define BPF_LINK_TYPE(_id, _name) 1114 #include <linux/bpf_types.h> 1115 #undef BPF_PROG_TYPE 1116 #undef BPF_MAP_TYPE 1117 #undef BPF_LINK_TYPE 1118 1119 extern const struct bpf_prog_ops bpf_offload_prog_ops; 1120 extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops; 1121 extern const struct bpf_verifier_ops xdp_analyzer_ops; 1122 1123 struct bpf_prog *bpf_prog_get(u32 ufd); 1124 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 1125 bool attach_drv); 1126 void bpf_prog_add(struct bpf_prog *prog, int i); 1127 void bpf_prog_sub(struct bpf_prog *prog, int i); 1128 void bpf_prog_inc(struct bpf_prog *prog); 1129 struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog); 1130 void bpf_prog_put(struct bpf_prog *prog); 1131 int __bpf_prog_charge(struct user_struct *user, u32 pages); 1132 void __bpf_prog_uncharge(struct user_struct *user, u32 pages); 1133 void __bpf_free_used_maps(struct bpf_prog_aux *aux, 1134 struct bpf_map **used_maps, u32 len); 1135 1136 void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock); 1137 void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock); 1138 1139 struct bpf_map *bpf_map_get(u32 ufd); 1140 struct bpf_map *bpf_map_get_with_uref(u32 ufd); 1141 struct bpf_map *__bpf_map_get(struct fd f); 1142 void bpf_map_inc(struct bpf_map *map); 1143 void bpf_map_inc_with_uref(struct bpf_map *map); 1144 struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map); 1145 void bpf_map_put_with_uref(struct bpf_map *map); 1146 void bpf_map_put(struct bpf_map *map); 1147 int bpf_map_charge_memlock(struct bpf_map *map, u32 pages); 1148 void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages); 1149 int bpf_map_charge_init(struct bpf_map_memory *mem, u64 size); 1150 void bpf_map_charge_finish(struct bpf_map_memory *mem); 1151 void bpf_map_charge_move(struct bpf_map_memory *dst, 1152 struct bpf_map_memory *src); 1153 void *bpf_map_area_alloc(u64 size, int numa_node); 1154 void *bpf_map_area_mmapable_alloc(u64 size, int numa_node); 1155 void bpf_map_area_free(void *base); 1156 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr); 1157 int generic_map_lookup_batch(struct bpf_map *map, 1158 const union bpf_attr *attr, 1159 union bpf_attr __user *uattr); 1160 int generic_map_update_batch(struct bpf_map *map, 1161 const union bpf_attr *attr, 1162 union bpf_attr __user *uattr); 1163 int generic_map_delete_batch(struct bpf_map *map, 1164 const union bpf_attr *attr, 1165 union bpf_attr __user *uattr); 1166 struct bpf_map *bpf_map_get_curr_or_next(u32 *id); 1167 struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id); 1168 1169 extern int sysctl_unprivileged_bpf_disabled; 1170 1171 static inline bool bpf_allow_ptr_leaks(void) 1172 { 1173 return perfmon_capable(); 1174 } 1175 1176 static inline bool bpf_allow_ptr_to_map_access(void) 1177 { 1178 return perfmon_capable(); 1179 } 1180 1181 static inline bool bpf_bypass_spec_v1(void) 1182 { 1183 return perfmon_capable(); 1184 } 1185 1186 static inline bool bpf_bypass_spec_v4(void) 1187 { 1188 return perfmon_capable(); 1189 } 1190 1191 int bpf_map_new_fd(struct bpf_map *map, int flags); 1192 int bpf_prog_new_fd(struct bpf_prog *prog); 1193 1194 void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, 1195 const struct bpf_link_ops *ops, struct bpf_prog *prog); 1196 int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer); 1197 int bpf_link_settle(struct bpf_link_primer *primer); 1198 void bpf_link_cleanup(struct bpf_link_primer *primer); 1199 void bpf_link_inc(struct bpf_link *link); 1200 void bpf_link_put(struct bpf_link *link); 1201 int bpf_link_new_fd(struct bpf_link *link); 1202 struct file *bpf_link_new_file(struct bpf_link *link, int *reserved_fd); 1203 struct bpf_link *bpf_link_get_from_fd(u32 ufd); 1204 1205 int bpf_obj_pin_user(u32 ufd, const char __user *pathname); 1206 int bpf_obj_get_user(const char __user *pathname, int flags); 1207 1208 #define BPF_ITER_FUNC_PREFIX "bpf_iter_" 1209 #define DEFINE_BPF_ITER_FUNC(target, args...) \ 1210 extern int bpf_iter_ ## target(args); \ 1211 int __init bpf_iter_ ## target(args) { return 0; } 1212 1213 struct bpf_iter_aux_info { 1214 struct bpf_map *map; 1215 }; 1216 1217 typedef int (*bpf_iter_check_target_t)(struct bpf_prog *prog, 1218 struct bpf_iter_aux_info *aux); 1219 1220 #define BPF_ITER_CTX_ARG_MAX 2 1221 struct bpf_iter_reg { 1222 const char *target; 1223 bpf_iter_check_target_t check_target; 1224 u32 ctx_arg_info_size; 1225 enum bpf_iter_link_info req_linfo; 1226 struct bpf_ctx_arg_aux ctx_arg_info[BPF_ITER_CTX_ARG_MAX]; 1227 const struct bpf_iter_seq_info *seq_info; 1228 }; 1229 1230 struct bpf_iter_meta { 1231 __bpf_md_ptr(struct seq_file *, seq); 1232 u64 session_id; 1233 u64 seq_num; 1234 }; 1235 1236 struct bpf_iter__bpf_map_elem { 1237 __bpf_md_ptr(struct bpf_iter_meta *, meta); 1238 __bpf_md_ptr(struct bpf_map *, map); 1239 __bpf_md_ptr(void *, key); 1240 __bpf_md_ptr(void *, value); 1241 }; 1242 1243 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info); 1244 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info); 1245 bool bpf_iter_prog_supported(struct bpf_prog *prog); 1246 int bpf_iter_link_attach(const union bpf_attr *attr, struct bpf_prog *prog); 1247 int bpf_iter_new_fd(struct bpf_link *link); 1248 bool bpf_link_is_iter(struct bpf_link *link); 1249 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop); 1250 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx); 1251 1252 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value); 1253 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value); 1254 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, 1255 u64 flags); 1256 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, 1257 u64 flags); 1258 1259 int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value); 1260 1261 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, 1262 void *key, void *value, u64 map_flags); 1263 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); 1264 int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, 1265 void *key, void *value, u64 map_flags); 1266 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); 1267 1268 int bpf_get_file_flag(int flags); 1269 int bpf_check_uarg_tail_zero(void __user *uaddr, size_t expected_size, 1270 size_t actual_size); 1271 1272 /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and 1273 * forced to use 'long' read/writes to try to atomically copy long counters. 1274 * Best-effort only. No barriers here, since it _will_ race with concurrent 1275 * updates from BPF programs. Called from bpf syscall and mostly used with 1276 * size 8 or 16 bytes, so ask compiler to inline it. 1277 */ 1278 static inline void bpf_long_memcpy(void *dst, const void *src, u32 size) 1279 { 1280 const long *lsrc = src; 1281 long *ldst = dst; 1282 1283 size /= sizeof(long); 1284 while (size--) 1285 *ldst++ = *lsrc++; 1286 } 1287 1288 /* verify correctness of eBPF program */ 1289 int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, 1290 union bpf_attr __user *uattr); 1291 void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); 1292 1293 /* Map specifics */ 1294 struct xdp_buff; 1295 struct sk_buff; 1296 1297 struct bpf_dtab_netdev *__dev_map_lookup_elem(struct bpf_map *map, u32 key); 1298 struct bpf_dtab_netdev *__dev_map_hash_lookup_elem(struct bpf_map *map, u32 key); 1299 void __dev_flush(void); 1300 int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp, 1301 struct net_device *dev_rx); 1302 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, 1303 struct net_device *dev_rx); 1304 int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, 1305 struct bpf_prog *xdp_prog); 1306 bool dev_map_can_have_prog(struct bpf_map *map); 1307 1308 struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key); 1309 void __cpu_map_flush(void); 1310 int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp, 1311 struct net_device *dev_rx); 1312 bool cpu_map_prog_allowed(struct bpf_map *map); 1313 1314 /* Return map's numa specified by userspace */ 1315 static inline int bpf_map_attr_numa_node(const union bpf_attr *attr) 1316 { 1317 return (attr->map_flags & BPF_F_NUMA_NODE) ? 1318 attr->numa_node : NUMA_NO_NODE; 1319 } 1320 1321 struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type); 1322 int array_map_alloc_check(union bpf_attr *attr); 1323 1324 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 1325 union bpf_attr __user *uattr); 1326 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 1327 union bpf_attr __user *uattr); 1328 int bpf_prog_test_run_tracing(struct bpf_prog *prog, 1329 const union bpf_attr *kattr, 1330 union bpf_attr __user *uattr); 1331 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 1332 const union bpf_attr *kattr, 1333 union bpf_attr __user *uattr); 1334 bool btf_ctx_access(int off, int size, enum bpf_access_type type, 1335 const struct bpf_prog *prog, 1336 struct bpf_insn_access_aux *info); 1337 int btf_struct_access(struct bpf_verifier_log *log, 1338 const struct btf_type *t, int off, int size, 1339 enum bpf_access_type atype, 1340 u32 *next_btf_id); 1341 int btf_resolve_helper_id(struct bpf_verifier_log *log, 1342 const struct bpf_func_proto *fn, int); 1343 1344 int btf_distill_func_proto(struct bpf_verifier_log *log, 1345 struct btf *btf, 1346 const struct btf_type *func_proto, 1347 const char *func_name, 1348 struct btf_func_model *m); 1349 1350 struct bpf_reg_state; 1351 int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, 1352 struct bpf_reg_state *regs); 1353 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, 1354 struct bpf_reg_state *reg); 1355 int btf_check_type_match(struct bpf_verifier_env *env, struct bpf_prog *prog, 1356 struct btf *btf, const struct btf_type *t); 1357 1358 struct bpf_prog *bpf_prog_by_id(u32 id); 1359 1360 const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id); 1361 #else /* !CONFIG_BPF_SYSCALL */ 1362 static inline struct bpf_prog *bpf_prog_get(u32 ufd) 1363 { 1364 return ERR_PTR(-EOPNOTSUPP); 1365 } 1366 1367 static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, 1368 enum bpf_prog_type type, 1369 bool attach_drv) 1370 { 1371 return ERR_PTR(-EOPNOTSUPP); 1372 } 1373 1374 static inline void bpf_prog_add(struct bpf_prog *prog, int i) 1375 { 1376 } 1377 1378 static inline void bpf_prog_sub(struct bpf_prog *prog, int i) 1379 { 1380 } 1381 1382 static inline void bpf_prog_put(struct bpf_prog *prog) 1383 { 1384 } 1385 1386 static inline void bpf_prog_inc(struct bpf_prog *prog) 1387 { 1388 } 1389 1390 static inline struct bpf_prog *__must_check 1391 bpf_prog_inc_not_zero(struct bpf_prog *prog) 1392 { 1393 return ERR_PTR(-EOPNOTSUPP); 1394 } 1395 1396 static inline int __bpf_prog_charge(struct user_struct *user, u32 pages) 1397 { 1398 return 0; 1399 } 1400 1401 static inline void __bpf_prog_uncharge(struct user_struct *user, u32 pages) 1402 { 1403 } 1404 1405 static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, 1406 const struct bpf_link_ops *ops, 1407 struct bpf_prog *prog) 1408 { 1409 } 1410 1411 static inline int bpf_link_prime(struct bpf_link *link, 1412 struct bpf_link_primer *primer) 1413 { 1414 return -EOPNOTSUPP; 1415 } 1416 1417 static inline int bpf_link_settle(struct bpf_link_primer *primer) 1418 { 1419 return -EOPNOTSUPP; 1420 } 1421 1422 static inline void bpf_link_cleanup(struct bpf_link_primer *primer) 1423 { 1424 } 1425 1426 static inline void bpf_link_inc(struct bpf_link *link) 1427 { 1428 } 1429 1430 static inline void bpf_link_put(struct bpf_link *link) 1431 { 1432 } 1433 1434 static inline int bpf_obj_get_user(const char __user *pathname, int flags) 1435 { 1436 return -EOPNOTSUPP; 1437 } 1438 1439 static inline struct net_device *__dev_map_lookup_elem(struct bpf_map *map, 1440 u32 key) 1441 { 1442 return NULL; 1443 } 1444 1445 static inline struct net_device *__dev_map_hash_lookup_elem(struct bpf_map *map, 1446 u32 key) 1447 { 1448 return NULL; 1449 } 1450 static inline bool dev_map_can_have_prog(struct bpf_map *map) 1451 { 1452 return false; 1453 } 1454 1455 static inline void __dev_flush(void) 1456 { 1457 } 1458 1459 struct xdp_buff; 1460 struct bpf_dtab_netdev; 1461 1462 static inline 1463 int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp, 1464 struct net_device *dev_rx) 1465 { 1466 return 0; 1467 } 1468 1469 static inline 1470 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, 1471 struct net_device *dev_rx) 1472 { 1473 return 0; 1474 } 1475 1476 struct sk_buff; 1477 1478 static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, 1479 struct sk_buff *skb, 1480 struct bpf_prog *xdp_prog) 1481 { 1482 return 0; 1483 } 1484 1485 static inline 1486 struct bpf_cpu_map_entry *__cpu_map_lookup_elem(struct bpf_map *map, u32 key) 1487 { 1488 return NULL; 1489 } 1490 1491 static inline void __cpu_map_flush(void) 1492 { 1493 } 1494 1495 static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, 1496 struct xdp_buff *xdp, 1497 struct net_device *dev_rx) 1498 { 1499 return 0; 1500 } 1501 1502 static inline bool cpu_map_prog_allowed(struct bpf_map *map) 1503 { 1504 return false; 1505 } 1506 1507 static inline struct bpf_prog *bpf_prog_get_type_path(const char *name, 1508 enum bpf_prog_type type) 1509 { 1510 return ERR_PTR(-EOPNOTSUPP); 1511 } 1512 1513 static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog, 1514 const union bpf_attr *kattr, 1515 union bpf_attr __user *uattr) 1516 { 1517 return -ENOTSUPP; 1518 } 1519 1520 static inline int bpf_prog_test_run_skb(struct bpf_prog *prog, 1521 const union bpf_attr *kattr, 1522 union bpf_attr __user *uattr) 1523 { 1524 return -ENOTSUPP; 1525 } 1526 1527 static inline int bpf_prog_test_run_tracing(struct bpf_prog *prog, 1528 const union bpf_attr *kattr, 1529 union bpf_attr __user *uattr) 1530 { 1531 return -ENOTSUPP; 1532 } 1533 1534 static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 1535 const union bpf_attr *kattr, 1536 union bpf_attr __user *uattr) 1537 { 1538 return -ENOTSUPP; 1539 } 1540 1541 static inline void bpf_map_put(struct bpf_map *map) 1542 { 1543 } 1544 1545 static inline struct bpf_prog *bpf_prog_by_id(u32 id) 1546 { 1547 return ERR_PTR(-ENOTSUPP); 1548 } 1549 1550 static inline const struct bpf_func_proto * 1551 bpf_base_func_proto(enum bpf_func_id func_id) 1552 { 1553 return NULL; 1554 } 1555 #endif /* CONFIG_BPF_SYSCALL */ 1556 1557 static inline struct bpf_prog *bpf_prog_get_type(u32 ufd, 1558 enum bpf_prog_type type) 1559 { 1560 return bpf_prog_get_type_dev(ufd, type, false); 1561 } 1562 1563 bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool); 1564 1565 int bpf_prog_offload_compile(struct bpf_prog *prog); 1566 void bpf_prog_offload_destroy(struct bpf_prog *prog); 1567 int bpf_prog_offload_info_fill(struct bpf_prog_info *info, 1568 struct bpf_prog *prog); 1569 1570 int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map); 1571 1572 int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value); 1573 int bpf_map_offload_update_elem(struct bpf_map *map, 1574 void *key, void *value, u64 flags); 1575 int bpf_map_offload_delete_elem(struct bpf_map *map, void *key); 1576 int bpf_map_offload_get_next_key(struct bpf_map *map, 1577 void *key, void *next_key); 1578 1579 bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map); 1580 1581 struct bpf_offload_dev * 1582 bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv); 1583 void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev); 1584 void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev); 1585 int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev, 1586 struct net_device *netdev); 1587 void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev, 1588 struct net_device *netdev); 1589 bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev); 1590 1591 #if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL) 1592 int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr); 1593 1594 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) 1595 { 1596 return aux->offload_requested; 1597 } 1598 1599 static inline bool bpf_map_is_dev_bound(struct bpf_map *map) 1600 { 1601 return unlikely(map->ops == &bpf_map_offload_ops); 1602 } 1603 1604 struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr); 1605 void bpf_map_offload_map_free(struct bpf_map *map); 1606 #else 1607 static inline int bpf_prog_offload_init(struct bpf_prog *prog, 1608 union bpf_attr *attr) 1609 { 1610 return -EOPNOTSUPP; 1611 } 1612 1613 static inline bool bpf_prog_is_dev_bound(struct bpf_prog_aux *aux) 1614 { 1615 return false; 1616 } 1617 1618 static inline bool bpf_map_is_dev_bound(struct bpf_map *map) 1619 { 1620 return false; 1621 } 1622 1623 static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) 1624 { 1625 return ERR_PTR(-EOPNOTSUPP); 1626 } 1627 1628 static inline void bpf_map_offload_map_free(struct bpf_map *map) 1629 { 1630 } 1631 #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */ 1632 1633 #if defined(CONFIG_BPF_STREAM_PARSER) 1634 int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, 1635 struct bpf_prog *old, u32 which); 1636 int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog); 1637 int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype); 1638 void sock_map_unhash(struct sock *sk); 1639 void sock_map_close(struct sock *sk, long timeout); 1640 #else 1641 static inline int sock_map_prog_update(struct bpf_map *map, 1642 struct bpf_prog *prog, 1643 struct bpf_prog *old, u32 which) 1644 { 1645 return -EOPNOTSUPP; 1646 } 1647 1648 static inline int sock_map_get_from_fd(const union bpf_attr *attr, 1649 struct bpf_prog *prog) 1650 { 1651 return -EINVAL; 1652 } 1653 1654 static inline int sock_map_prog_detach(const union bpf_attr *attr, 1655 enum bpf_prog_type ptype) 1656 { 1657 return -EOPNOTSUPP; 1658 } 1659 #endif /* CONFIG_BPF_STREAM_PARSER */ 1660 1661 #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) 1662 void bpf_sk_reuseport_detach(struct sock *sk); 1663 int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, 1664 void *value); 1665 int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key, 1666 void *value, u64 map_flags); 1667 #else 1668 static inline void bpf_sk_reuseport_detach(struct sock *sk) 1669 { 1670 } 1671 1672 #ifdef CONFIG_BPF_SYSCALL 1673 static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, 1674 void *key, void *value) 1675 { 1676 return -EOPNOTSUPP; 1677 } 1678 1679 static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, 1680 void *key, void *value, 1681 u64 map_flags) 1682 { 1683 return -EOPNOTSUPP; 1684 } 1685 #endif /* CONFIG_BPF_SYSCALL */ 1686 #endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */ 1687 1688 /* verifier prototypes for helper functions called from eBPF programs */ 1689 extern const struct bpf_func_proto bpf_map_lookup_elem_proto; 1690 extern const struct bpf_func_proto bpf_map_update_elem_proto; 1691 extern const struct bpf_func_proto bpf_map_delete_elem_proto; 1692 extern const struct bpf_func_proto bpf_map_push_elem_proto; 1693 extern const struct bpf_func_proto bpf_map_pop_elem_proto; 1694 extern const struct bpf_func_proto bpf_map_peek_elem_proto; 1695 1696 extern const struct bpf_func_proto bpf_get_prandom_u32_proto; 1697 extern const struct bpf_func_proto bpf_get_smp_processor_id_proto; 1698 extern const struct bpf_func_proto bpf_get_numa_node_id_proto; 1699 extern const struct bpf_func_proto bpf_tail_call_proto; 1700 extern const struct bpf_func_proto bpf_ktime_get_ns_proto; 1701 extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto; 1702 extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto; 1703 extern const struct bpf_func_proto bpf_get_current_uid_gid_proto; 1704 extern const struct bpf_func_proto bpf_get_current_comm_proto; 1705 extern const struct bpf_func_proto bpf_get_stackid_proto; 1706 extern const struct bpf_func_proto bpf_get_stack_proto; 1707 extern const struct bpf_func_proto bpf_get_task_stack_proto; 1708 extern const struct bpf_func_proto bpf_get_stackid_proto_pe; 1709 extern const struct bpf_func_proto bpf_get_stack_proto_pe; 1710 extern const struct bpf_func_proto bpf_sock_map_update_proto; 1711 extern const struct bpf_func_proto bpf_sock_hash_update_proto; 1712 extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto; 1713 extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto; 1714 extern const struct bpf_func_proto bpf_msg_redirect_hash_proto; 1715 extern const struct bpf_func_proto bpf_msg_redirect_map_proto; 1716 extern const struct bpf_func_proto bpf_sk_redirect_hash_proto; 1717 extern const struct bpf_func_proto bpf_sk_redirect_map_proto; 1718 extern const struct bpf_func_proto bpf_spin_lock_proto; 1719 extern const struct bpf_func_proto bpf_spin_unlock_proto; 1720 extern const struct bpf_func_proto bpf_get_local_storage_proto; 1721 extern const struct bpf_func_proto bpf_strtol_proto; 1722 extern const struct bpf_func_proto bpf_strtoul_proto; 1723 extern const struct bpf_func_proto bpf_tcp_sock_proto; 1724 extern const struct bpf_func_proto bpf_jiffies64_proto; 1725 extern const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto; 1726 extern const struct bpf_func_proto bpf_event_output_data_proto; 1727 extern const struct bpf_func_proto bpf_ringbuf_output_proto; 1728 extern const struct bpf_func_proto bpf_ringbuf_reserve_proto; 1729 extern const struct bpf_func_proto bpf_ringbuf_submit_proto; 1730 extern const struct bpf_func_proto bpf_ringbuf_discard_proto; 1731 extern const struct bpf_func_proto bpf_ringbuf_query_proto; 1732 extern const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto; 1733 extern const struct bpf_func_proto bpf_skc_to_tcp_sock_proto; 1734 extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto; 1735 extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto; 1736 extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto; 1737 1738 const struct bpf_func_proto *bpf_tracing_func_proto( 1739 enum bpf_func_id func_id, const struct bpf_prog *prog); 1740 1741 const struct bpf_func_proto *tracing_prog_func_proto( 1742 enum bpf_func_id func_id, const struct bpf_prog *prog); 1743 1744 /* Shared helpers among cBPF and eBPF. */ 1745 void bpf_user_rnd_init_once(void); 1746 u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 1747 u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 1748 1749 #if defined(CONFIG_NET) 1750 bool bpf_sock_common_is_valid_access(int off, int size, 1751 enum bpf_access_type type, 1752 struct bpf_insn_access_aux *info); 1753 bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type, 1754 struct bpf_insn_access_aux *info); 1755 u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, 1756 const struct bpf_insn *si, 1757 struct bpf_insn *insn_buf, 1758 struct bpf_prog *prog, 1759 u32 *target_size); 1760 #else 1761 static inline bool bpf_sock_common_is_valid_access(int off, int size, 1762 enum bpf_access_type type, 1763 struct bpf_insn_access_aux *info) 1764 { 1765 return false; 1766 } 1767 static inline bool bpf_sock_is_valid_access(int off, int size, 1768 enum bpf_access_type type, 1769 struct bpf_insn_access_aux *info) 1770 { 1771 return false; 1772 } 1773 static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, 1774 const struct bpf_insn *si, 1775 struct bpf_insn *insn_buf, 1776 struct bpf_prog *prog, 1777 u32 *target_size) 1778 { 1779 return 0; 1780 } 1781 #endif 1782 1783 #ifdef CONFIG_INET 1784 struct sk_reuseport_kern { 1785 struct sk_buff *skb; 1786 struct sock *sk; 1787 struct sock *selected_sk; 1788 void *data_end; 1789 u32 hash; 1790 u32 reuseport_id; 1791 bool bind_inany; 1792 }; 1793 bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type, 1794 struct bpf_insn_access_aux *info); 1795 1796 u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, 1797 const struct bpf_insn *si, 1798 struct bpf_insn *insn_buf, 1799 struct bpf_prog *prog, 1800 u32 *target_size); 1801 1802 bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type, 1803 struct bpf_insn_access_aux *info); 1804 1805 u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, 1806 const struct bpf_insn *si, 1807 struct bpf_insn *insn_buf, 1808 struct bpf_prog *prog, 1809 u32 *target_size); 1810 #else 1811 static inline bool bpf_tcp_sock_is_valid_access(int off, int size, 1812 enum bpf_access_type type, 1813 struct bpf_insn_access_aux *info) 1814 { 1815 return false; 1816 } 1817 1818 static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, 1819 const struct bpf_insn *si, 1820 struct bpf_insn *insn_buf, 1821 struct bpf_prog *prog, 1822 u32 *target_size) 1823 { 1824 return 0; 1825 } 1826 static inline bool bpf_xdp_sock_is_valid_access(int off, int size, 1827 enum bpf_access_type type, 1828 struct bpf_insn_access_aux *info) 1829 { 1830 return false; 1831 } 1832 1833 static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, 1834 const struct bpf_insn *si, 1835 struct bpf_insn *insn_buf, 1836 struct bpf_prog *prog, 1837 u32 *target_size) 1838 { 1839 return 0; 1840 } 1841 #endif /* CONFIG_INET */ 1842 1843 enum bpf_text_poke_type { 1844 BPF_MOD_CALL, 1845 BPF_MOD_JUMP, 1846 }; 1847 1848 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, 1849 void *addr1, void *addr2); 1850 1851 #endif /* _LINUX_BPF_H */ 1852