1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 3 */ 4 #ifndef _LINUX_BPF_H 5 #define _LINUX_BPF_H 1 6 7 #include <uapi/linux/bpf.h> 8 #include <uapi/linux/filter.h> 9 10 #include <linux/workqueue.h> 11 #include <linux/file.h> 12 #include <linux/percpu.h> 13 #include <linux/err.h> 14 #include <linux/rbtree_latch.h> 15 #include <linux/numa.h> 16 #include <linux/mm_types.h> 17 #include <linux/wait.h> 18 #include <linux/refcount.h> 19 #include <linux/mutex.h> 20 #include <linux/module.h> 21 #include <linux/kallsyms.h> 22 #include <linux/capability.h> 23 #include <linux/sched/mm.h> 24 #include <linux/slab.h> 25 #include <linux/percpu-refcount.h> 26 #include <linux/stddef.h> 27 #include <linux/bpfptr.h> 28 #include <linux/btf.h> 29 #include <linux/rcupdate_trace.h> 30 #include <linux/static_call.h> 31 #include <linux/memcontrol.h> 32 33 struct bpf_verifier_env; 34 struct bpf_verifier_log; 35 struct perf_event; 36 struct bpf_prog; 37 struct bpf_prog_aux; 38 struct bpf_map; 39 struct sock; 40 struct seq_file; 41 struct btf; 42 struct btf_type; 43 struct exception_table_entry; 44 struct seq_operations; 45 struct bpf_iter_aux_info; 46 struct bpf_local_storage; 47 struct bpf_local_storage_map; 48 struct kobject; 49 struct mem_cgroup; 50 struct module; 51 struct bpf_func_state; 52 struct ftrace_ops; 53 struct cgroup; 54 55 extern struct idr btf_idr; 56 extern spinlock_t btf_idr_lock; 57 extern struct kobject *btf_kobj; 58 extern struct bpf_mem_alloc bpf_global_ma; 59 extern bool bpf_global_ma_set; 60 61 typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64); 62 typedef int (*bpf_iter_init_seq_priv_t)(void *private_data, 63 struct bpf_iter_aux_info *aux); 64 typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data); 65 typedef unsigned int (*bpf_func_t)(const void *, 66 const struct bpf_insn *); 67 struct bpf_iter_seq_info { 68 const struct seq_operations *seq_ops; 69 bpf_iter_init_seq_priv_t init_seq_private; 70 bpf_iter_fini_seq_priv_t fini_seq_private; 71 u32 seq_priv_size; 72 }; 73 74 /* map is generic key/value storage optionally accessible by eBPF programs */ 75 struct bpf_map_ops { 76 /* funcs callable from userspace (via syscall) */ 77 int (*map_alloc_check)(union bpf_attr *attr); 78 struct bpf_map *(*map_alloc)(union bpf_attr *attr); 79 void (*map_release)(struct bpf_map *map, struct file *map_file); 80 void (*map_free)(struct bpf_map *map); 81 int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key); 82 void (*map_release_uref)(struct bpf_map *map); 83 void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key); 84 int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr, 85 union bpf_attr __user *uattr); 86 int (*map_lookup_and_delete_elem)(struct bpf_map *map, void *key, 87 void *value, u64 flags); 88 int (*map_lookup_and_delete_batch)(struct bpf_map *map, 89 const union bpf_attr *attr, 90 union bpf_attr __user *uattr); 91 int (*map_update_batch)(struct bpf_map *map, struct file *map_file, 92 const union bpf_attr *attr, 93 union bpf_attr __user *uattr); 94 int (*map_delete_batch)(struct bpf_map *map, const union bpf_attr *attr, 95 union bpf_attr __user *uattr); 96 97 /* funcs callable from userspace and from eBPF programs */ 98 void *(*map_lookup_elem)(struct bpf_map *map, void *key); 99 int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags); 100 int (*map_delete_elem)(struct bpf_map *map, void *key); 101 int (*map_push_elem)(struct bpf_map *map, void *value, u64 flags); 102 int (*map_pop_elem)(struct bpf_map *map, void *value); 103 int (*map_peek_elem)(struct bpf_map *map, void *value); 104 void *(*map_lookup_percpu_elem)(struct bpf_map *map, void *key, u32 cpu); 105 106 /* funcs called by prog_array and perf_event_array map */ 107 void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file, 108 int fd); 109 void (*map_fd_put_ptr)(void *ptr); 110 int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf); 111 u32 (*map_fd_sys_lookup_elem)(void *ptr); 112 void (*map_seq_show_elem)(struct bpf_map *map, void *key, 113 struct seq_file *m); 114 int (*map_check_btf)(const struct bpf_map *map, 115 const struct btf *btf, 116 const struct btf_type *key_type, 117 const struct btf_type *value_type); 118 119 /* Prog poke tracking helpers. */ 120 int (*map_poke_track)(struct bpf_map *map, struct bpf_prog_aux *aux); 121 void (*map_poke_untrack)(struct bpf_map *map, struct bpf_prog_aux *aux); 122 void (*map_poke_run)(struct bpf_map *map, u32 key, struct bpf_prog *old, 123 struct bpf_prog *new); 124 125 /* Direct value access helpers. */ 126 int (*map_direct_value_addr)(const struct bpf_map *map, 127 u64 *imm, u32 off); 128 int (*map_direct_value_meta)(const struct bpf_map *map, 129 u64 imm, u32 *off); 130 int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma); 131 __poll_t (*map_poll)(struct bpf_map *map, struct file *filp, 132 struct poll_table_struct *pts); 133 134 /* Functions called by bpf_local_storage maps */ 135 int (*map_local_storage_charge)(struct bpf_local_storage_map *smap, 136 void *owner, u32 size); 137 void (*map_local_storage_uncharge)(struct bpf_local_storage_map *smap, 138 void *owner, u32 size); 139 struct bpf_local_storage __rcu ** (*map_owner_storage_ptr)(void *owner); 140 141 /* Misc helpers.*/ 142 int (*map_redirect)(struct bpf_map *map, u64 key, u64 flags); 143 144 /* map_meta_equal must be implemented for maps that can be 145 * used as an inner map. It is a runtime check to ensure 146 * an inner map can be inserted to an outer map. 147 * 148 * Some properties of the inner map has been used during the 149 * verification time. When inserting an inner map at the runtime, 150 * map_meta_equal has to ensure the inserting map has the same 151 * properties that the verifier has used earlier. 152 */ 153 bool (*map_meta_equal)(const struct bpf_map *meta0, 154 const struct bpf_map *meta1); 155 156 157 int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env, 158 struct bpf_func_state *caller, 159 struct bpf_func_state *callee); 160 int (*map_for_each_callback)(struct bpf_map *map, 161 bpf_callback_t callback_fn, 162 void *callback_ctx, u64 flags); 163 164 u64 (*map_mem_usage)(const struct bpf_map *map); 165 166 /* BTF id of struct allocated by map_alloc */ 167 int *map_btf_id; 168 169 /* bpf_iter info used to open a seq_file */ 170 const struct bpf_iter_seq_info *iter_seq_info; 171 }; 172 173 enum { 174 /* Support at most 10 fields in a BTF type */ 175 BTF_FIELDS_MAX = 10, 176 }; 177 178 enum btf_field_type { 179 BPF_SPIN_LOCK = (1 << 0), 180 BPF_TIMER = (1 << 1), 181 BPF_KPTR_UNREF = (1 << 2), 182 BPF_KPTR_REF = (1 << 3), 183 BPF_KPTR = BPF_KPTR_UNREF | BPF_KPTR_REF, 184 BPF_LIST_HEAD = (1 << 4), 185 BPF_LIST_NODE = (1 << 5), 186 BPF_RB_ROOT = (1 << 6), 187 BPF_RB_NODE = (1 << 7), 188 BPF_GRAPH_NODE_OR_ROOT = BPF_LIST_NODE | BPF_LIST_HEAD | 189 BPF_RB_NODE | BPF_RB_ROOT, 190 }; 191 192 typedef void (*btf_dtor_kfunc_t)(void *); 193 194 struct btf_field_kptr { 195 struct btf *btf; 196 struct module *module; 197 /* dtor used if btf_is_kernel(btf), otherwise the type is 198 * program-allocated, dtor is NULL, and __bpf_obj_drop_impl is used 199 */ 200 btf_dtor_kfunc_t dtor; 201 u32 btf_id; 202 }; 203 204 struct btf_field_graph_root { 205 struct btf *btf; 206 u32 value_btf_id; 207 u32 node_offset; 208 struct btf_record *value_rec; 209 }; 210 211 struct btf_field { 212 u32 offset; 213 enum btf_field_type type; 214 union { 215 struct btf_field_kptr kptr; 216 struct btf_field_graph_root graph_root; 217 }; 218 }; 219 220 struct btf_record { 221 u32 cnt; 222 u32 field_mask; 223 int spin_lock_off; 224 int timer_off; 225 struct btf_field fields[]; 226 }; 227 228 struct btf_field_offs { 229 u32 cnt; 230 u32 field_off[BTF_FIELDS_MAX]; 231 u8 field_sz[BTF_FIELDS_MAX]; 232 }; 233 234 struct bpf_map { 235 /* The first two cachelines with read-mostly members of which some 236 * are also accessed in fast-path (e.g. ops, max_entries). 237 */ 238 const struct bpf_map_ops *ops ____cacheline_aligned; 239 struct bpf_map *inner_map_meta; 240 #ifdef CONFIG_SECURITY 241 void *security; 242 #endif 243 enum bpf_map_type map_type; 244 u32 key_size; 245 u32 value_size; 246 u32 max_entries; 247 u64 map_extra; /* any per-map-type extra fields */ 248 u32 map_flags; 249 u32 id; 250 struct btf_record *record; 251 int numa_node; 252 u32 btf_key_type_id; 253 u32 btf_value_type_id; 254 u32 btf_vmlinux_value_type_id; 255 struct btf *btf; 256 #ifdef CONFIG_MEMCG_KMEM 257 struct obj_cgroup *objcg; 258 #endif 259 char name[BPF_OBJ_NAME_LEN]; 260 struct btf_field_offs *field_offs; 261 /* The 3rd and 4th cacheline with misc members to avoid false sharing 262 * particularly with refcounting. 263 */ 264 atomic64_t refcnt ____cacheline_aligned; 265 atomic64_t usercnt; 266 struct work_struct work; 267 struct mutex freeze_mutex; 268 atomic64_t writecnt; 269 /* 'Ownership' of program-containing map is claimed by the first program 270 * that is going to use this map or by the first program which FD is 271 * stored in the map to make sure that all callers and callees have the 272 * same prog type, JITed flag and xdp_has_frags flag. 273 */ 274 struct { 275 spinlock_t lock; 276 enum bpf_prog_type type; 277 bool jited; 278 bool xdp_has_frags; 279 } owner; 280 bool bypass_spec_v1; 281 bool frozen; /* write-once; write-protected by freeze_mutex */ 282 }; 283 284 static inline const char *btf_field_type_name(enum btf_field_type type) 285 { 286 switch (type) { 287 case BPF_SPIN_LOCK: 288 return "bpf_spin_lock"; 289 case BPF_TIMER: 290 return "bpf_timer"; 291 case BPF_KPTR_UNREF: 292 case BPF_KPTR_REF: 293 return "kptr"; 294 case BPF_LIST_HEAD: 295 return "bpf_list_head"; 296 case BPF_LIST_NODE: 297 return "bpf_list_node"; 298 case BPF_RB_ROOT: 299 return "bpf_rb_root"; 300 case BPF_RB_NODE: 301 return "bpf_rb_node"; 302 default: 303 WARN_ON_ONCE(1); 304 return "unknown"; 305 } 306 } 307 308 static inline u32 btf_field_type_size(enum btf_field_type type) 309 { 310 switch (type) { 311 case BPF_SPIN_LOCK: 312 return sizeof(struct bpf_spin_lock); 313 case BPF_TIMER: 314 return sizeof(struct bpf_timer); 315 case BPF_KPTR_UNREF: 316 case BPF_KPTR_REF: 317 return sizeof(u64); 318 case BPF_LIST_HEAD: 319 return sizeof(struct bpf_list_head); 320 case BPF_LIST_NODE: 321 return sizeof(struct bpf_list_node); 322 case BPF_RB_ROOT: 323 return sizeof(struct bpf_rb_root); 324 case BPF_RB_NODE: 325 return sizeof(struct bpf_rb_node); 326 default: 327 WARN_ON_ONCE(1); 328 return 0; 329 } 330 } 331 332 static inline u32 btf_field_type_align(enum btf_field_type type) 333 { 334 switch (type) { 335 case BPF_SPIN_LOCK: 336 return __alignof__(struct bpf_spin_lock); 337 case BPF_TIMER: 338 return __alignof__(struct bpf_timer); 339 case BPF_KPTR_UNREF: 340 case BPF_KPTR_REF: 341 return __alignof__(u64); 342 case BPF_LIST_HEAD: 343 return __alignof__(struct bpf_list_head); 344 case BPF_LIST_NODE: 345 return __alignof__(struct bpf_list_node); 346 case BPF_RB_ROOT: 347 return __alignof__(struct bpf_rb_root); 348 case BPF_RB_NODE: 349 return __alignof__(struct bpf_rb_node); 350 default: 351 WARN_ON_ONCE(1); 352 return 0; 353 } 354 } 355 356 static inline bool btf_record_has_field(const struct btf_record *rec, enum btf_field_type type) 357 { 358 if (IS_ERR_OR_NULL(rec)) 359 return false; 360 return rec->field_mask & type; 361 } 362 363 static inline void bpf_obj_init(const struct btf_field_offs *foffs, void *obj) 364 { 365 int i; 366 367 if (!foffs) 368 return; 369 for (i = 0; i < foffs->cnt; i++) 370 memset(obj + foffs->field_off[i], 0, foffs->field_sz[i]); 371 } 372 373 /* 'dst' must be a temporary buffer and should not point to memory that is being 374 * used in parallel by a bpf program or bpf syscall, otherwise the access from 375 * the bpf program or bpf syscall may be corrupted by the reinitialization, 376 * leading to weird problems. Even 'dst' is newly-allocated from bpf memory 377 * allocator, it is still possible for 'dst' to be used in parallel by a bpf 378 * program or bpf syscall. 379 */ 380 static inline void check_and_init_map_value(struct bpf_map *map, void *dst) 381 { 382 bpf_obj_init(map->field_offs, dst); 383 } 384 385 /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and 386 * forced to use 'long' read/writes to try to atomically copy long counters. 387 * Best-effort only. No barriers here, since it _will_ race with concurrent 388 * updates from BPF programs. Called from bpf syscall and mostly used with 389 * size 8 or 16 bytes, so ask compiler to inline it. 390 */ 391 static inline void bpf_long_memcpy(void *dst, const void *src, u32 size) 392 { 393 const long *lsrc = src; 394 long *ldst = dst; 395 396 size /= sizeof(long); 397 while (size--) 398 *ldst++ = *lsrc++; 399 } 400 401 /* copy everything but bpf_spin_lock, bpf_timer, and kptrs. There could be one of each. */ 402 static inline void bpf_obj_memcpy(struct btf_field_offs *foffs, 403 void *dst, void *src, u32 size, 404 bool long_memcpy) 405 { 406 u32 curr_off = 0; 407 int i; 408 409 if (likely(!foffs)) { 410 if (long_memcpy) 411 bpf_long_memcpy(dst, src, round_up(size, 8)); 412 else 413 memcpy(dst, src, size); 414 return; 415 } 416 417 for (i = 0; i < foffs->cnt; i++) { 418 u32 next_off = foffs->field_off[i]; 419 u32 sz = next_off - curr_off; 420 421 memcpy(dst + curr_off, src + curr_off, sz); 422 curr_off += foffs->field_sz[i] + sz; 423 } 424 memcpy(dst + curr_off, src + curr_off, size - curr_off); 425 } 426 427 static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) 428 { 429 bpf_obj_memcpy(map->field_offs, dst, src, map->value_size, false); 430 } 431 432 static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src) 433 { 434 bpf_obj_memcpy(map->field_offs, dst, src, map->value_size, true); 435 } 436 437 static inline void bpf_obj_memzero(struct btf_field_offs *foffs, void *dst, u32 size) 438 { 439 u32 curr_off = 0; 440 int i; 441 442 if (likely(!foffs)) { 443 memset(dst, 0, size); 444 return; 445 } 446 447 for (i = 0; i < foffs->cnt; i++) { 448 u32 next_off = foffs->field_off[i]; 449 u32 sz = next_off - curr_off; 450 451 memset(dst + curr_off, 0, sz); 452 curr_off += foffs->field_sz[i] + sz; 453 } 454 memset(dst + curr_off, 0, size - curr_off); 455 } 456 457 static inline void zero_map_value(struct bpf_map *map, void *dst) 458 { 459 bpf_obj_memzero(map->field_offs, dst, map->value_size); 460 } 461 462 void copy_map_value_locked(struct bpf_map *map, void *dst, void *src, 463 bool lock_src); 464 void bpf_timer_cancel_and_free(void *timer); 465 void bpf_list_head_free(const struct btf_field *field, void *list_head, 466 struct bpf_spin_lock *spin_lock); 467 void bpf_rb_root_free(const struct btf_field *field, void *rb_root, 468 struct bpf_spin_lock *spin_lock); 469 470 471 int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size); 472 473 struct bpf_offload_dev; 474 struct bpf_offloaded_map; 475 476 struct bpf_map_dev_ops { 477 int (*map_get_next_key)(struct bpf_offloaded_map *map, 478 void *key, void *next_key); 479 int (*map_lookup_elem)(struct bpf_offloaded_map *map, 480 void *key, void *value); 481 int (*map_update_elem)(struct bpf_offloaded_map *map, 482 void *key, void *value, u64 flags); 483 int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key); 484 }; 485 486 struct bpf_offloaded_map { 487 struct bpf_map map; 488 struct net_device *netdev; 489 const struct bpf_map_dev_ops *dev_ops; 490 void *dev_priv; 491 struct list_head offloads; 492 }; 493 494 static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map) 495 { 496 return container_of(map, struct bpf_offloaded_map, map); 497 } 498 499 static inline bool bpf_map_offload_neutral(const struct bpf_map *map) 500 { 501 return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 502 } 503 504 static inline bool bpf_map_support_seq_show(const struct bpf_map *map) 505 { 506 return (map->btf_value_type_id || map->btf_vmlinux_value_type_id) && 507 map->ops->map_seq_show_elem; 508 } 509 510 int map_check_no_btf(const struct bpf_map *map, 511 const struct btf *btf, 512 const struct btf_type *key_type, 513 const struct btf_type *value_type); 514 515 bool bpf_map_meta_equal(const struct bpf_map *meta0, 516 const struct bpf_map *meta1); 517 518 extern const struct bpf_map_ops bpf_map_offload_ops; 519 520 /* bpf_type_flag contains a set of flags that are applicable to the values of 521 * arg_type, ret_type and reg_type. For example, a pointer value may be null, 522 * or a memory is read-only. We classify types into two categories: base types 523 * and extended types. Extended types are base types combined with a type flag. 524 * 525 * Currently there are no more than 32 base types in arg_type, ret_type and 526 * reg_types. 527 */ 528 #define BPF_BASE_TYPE_BITS 8 529 530 enum bpf_type_flag { 531 /* PTR may be NULL. */ 532 PTR_MAYBE_NULL = BIT(0 + BPF_BASE_TYPE_BITS), 533 534 /* MEM is read-only. When applied on bpf_arg, it indicates the arg is 535 * compatible with both mutable and immutable memory. 536 */ 537 MEM_RDONLY = BIT(1 + BPF_BASE_TYPE_BITS), 538 539 /* MEM points to BPF ring buffer reservation. */ 540 MEM_RINGBUF = BIT(2 + BPF_BASE_TYPE_BITS), 541 542 /* MEM is in user address space. */ 543 MEM_USER = BIT(3 + BPF_BASE_TYPE_BITS), 544 545 /* MEM is a percpu memory. MEM_PERCPU tags PTR_TO_BTF_ID. When tagged 546 * with MEM_PERCPU, PTR_TO_BTF_ID _cannot_ be directly accessed. In 547 * order to drop this tag, it must be passed into bpf_per_cpu_ptr() 548 * or bpf_this_cpu_ptr(), which will return the pointer corresponding 549 * to the specified cpu. 550 */ 551 MEM_PERCPU = BIT(4 + BPF_BASE_TYPE_BITS), 552 553 /* Indicates that the argument will be released. */ 554 OBJ_RELEASE = BIT(5 + BPF_BASE_TYPE_BITS), 555 556 /* PTR is not trusted. This is only used with PTR_TO_BTF_ID, to mark 557 * unreferenced and referenced kptr loaded from map value using a load 558 * instruction, so that they can only be dereferenced but not escape the 559 * BPF program into the kernel (i.e. cannot be passed as arguments to 560 * kfunc or bpf helpers). 561 */ 562 PTR_UNTRUSTED = BIT(6 + BPF_BASE_TYPE_BITS), 563 564 MEM_UNINIT = BIT(7 + BPF_BASE_TYPE_BITS), 565 566 /* DYNPTR points to memory local to the bpf program. */ 567 DYNPTR_TYPE_LOCAL = BIT(8 + BPF_BASE_TYPE_BITS), 568 569 /* DYNPTR points to a kernel-produced ringbuf record. */ 570 DYNPTR_TYPE_RINGBUF = BIT(9 + BPF_BASE_TYPE_BITS), 571 572 /* Size is known at compile time. */ 573 MEM_FIXED_SIZE = BIT(10 + BPF_BASE_TYPE_BITS), 574 575 /* MEM is of an allocated object of type in program BTF. This is used to 576 * tag PTR_TO_BTF_ID allocated using bpf_obj_new. 577 */ 578 MEM_ALLOC = BIT(11 + BPF_BASE_TYPE_BITS), 579 580 /* PTR was passed from the kernel in a trusted context, and may be 581 * passed to KF_TRUSTED_ARGS kfuncs or BPF helper functions. 582 * Confusingly, this is _not_ the opposite of PTR_UNTRUSTED above. 583 * PTR_UNTRUSTED refers to a kptr that was read directly from a map 584 * without invoking bpf_kptr_xchg(). What we really need to know is 585 * whether a pointer is safe to pass to a kfunc or BPF helper function. 586 * While PTR_UNTRUSTED pointers are unsafe to pass to kfuncs and BPF 587 * helpers, they do not cover all possible instances of unsafe 588 * pointers. For example, a pointer that was obtained from walking a 589 * struct will _not_ get the PTR_UNTRUSTED type modifier, despite the 590 * fact that it may be NULL, invalid, etc. This is due to backwards 591 * compatibility requirements, as this was the behavior that was first 592 * introduced when kptrs were added. The behavior is now considered 593 * deprecated, and PTR_UNTRUSTED will eventually be removed. 594 * 595 * PTR_TRUSTED, on the other hand, is a pointer that the kernel 596 * guarantees to be valid and safe to pass to kfuncs and BPF helpers. 597 * For example, pointers passed to tracepoint arguments are considered 598 * PTR_TRUSTED, as are pointers that are passed to struct_ops 599 * callbacks. As alluded to above, pointers that are obtained from 600 * walking PTR_TRUSTED pointers are _not_ trusted. For example, if a 601 * struct task_struct *task is PTR_TRUSTED, then accessing 602 * task->last_wakee will lose the PTR_TRUSTED modifier when it's stored 603 * in a BPF register. Similarly, pointers passed to certain programs 604 * types such as kretprobes are not guaranteed to be valid, as they may 605 * for example contain an object that was recently freed. 606 */ 607 PTR_TRUSTED = BIT(12 + BPF_BASE_TYPE_BITS), 608 609 /* MEM is tagged with rcu and memory access needs rcu_read_lock protection. */ 610 MEM_RCU = BIT(13 + BPF_BASE_TYPE_BITS), 611 612 /* Used to tag PTR_TO_BTF_ID | MEM_ALLOC references which are non-owning. 613 * Currently only valid for linked-list and rbtree nodes. 614 */ 615 NON_OWN_REF = BIT(14 + BPF_BASE_TYPE_BITS), 616 617 /* DYNPTR points to sk_buff */ 618 DYNPTR_TYPE_SKB = BIT(15 + BPF_BASE_TYPE_BITS), 619 620 /* DYNPTR points to xdp_buff */ 621 DYNPTR_TYPE_XDP = BIT(16 + BPF_BASE_TYPE_BITS), 622 623 __BPF_TYPE_FLAG_MAX, 624 __BPF_TYPE_LAST_FLAG = __BPF_TYPE_FLAG_MAX - 1, 625 }; 626 627 #define DYNPTR_TYPE_FLAG_MASK (DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF | DYNPTR_TYPE_SKB \ 628 | DYNPTR_TYPE_XDP) 629 630 /* Max number of base types. */ 631 #define BPF_BASE_TYPE_LIMIT (1UL << BPF_BASE_TYPE_BITS) 632 633 /* Max number of all types. */ 634 #define BPF_TYPE_LIMIT (__BPF_TYPE_LAST_FLAG | (__BPF_TYPE_LAST_FLAG - 1)) 635 636 /* function argument constraints */ 637 enum bpf_arg_type { 638 ARG_DONTCARE = 0, /* unused argument in helper function */ 639 640 /* the following constraints used to prototype 641 * bpf_map_lookup/update/delete_elem() functions 642 */ 643 ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */ 644 ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */ 645 ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */ 646 647 /* Used to prototype bpf_memcmp() and other functions that access data 648 * on eBPF program stack 649 */ 650 ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ 651 652 ARG_CONST_SIZE, /* number of bytes accessed from memory */ 653 ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ 654 655 ARG_PTR_TO_CTX, /* pointer to context */ 656 ARG_ANYTHING, /* any (initialized) argument is ok */ 657 ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */ 658 ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */ 659 ARG_PTR_TO_INT, /* pointer to int */ 660 ARG_PTR_TO_LONG, /* pointer to long */ 661 ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */ 662 ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */ 663 ARG_PTR_TO_RINGBUF_MEM, /* pointer to dynamically reserved ringbuf memory */ 664 ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */ 665 ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */ 666 ARG_PTR_TO_PERCPU_BTF_ID, /* pointer to in-kernel percpu type */ 667 ARG_PTR_TO_FUNC, /* pointer to a bpf program function */ 668 ARG_PTR_TO_STACK, /* pointer to stack */ 669 ARG_PTR_TO_CONST_STR, /* pointer to a null terminated read-only string */ 670 ARG_PTR_TO_TIMER, /* pointer to bpf_timer */ 671 ARG_PTR_TO_KPTR, /* pointer to referenced kptr */ 672 ARG_PTR_TO_DYNPTR, /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */ 673 __BPF_ARG_TYPE_MAX, 674 675 /* Extended arg_types. */ 676 ARG_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MAP_VALUE, 677 ARG_PTR_TO_MEM_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MEM, 678 ARG_PTR_TO_CTX_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_CTX, 679 ARG_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_SOCKET, 680 ARG_PTR_TO_STACK_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_STACK, 681 ARG_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_BTF_ID, 682 /* pointer to memory does not need to be initialized, helper function must fill 683 * all bytes or clear them in error case. 684 */ 685 ARG_PTR_TO_UNINIT_MEM = MEM_UNINIT | ARG_PTR_TO_MEM, 686 /* Pointer to valid memory of size known at compile time. */ 687 ARG_PTR_TO_FIXED_SIZE_MEM = MEM_FIXED_SIZE | ARG_PTR_TO_MEM, 688 689 /* This must be the last entry. Its purpose is to ensure the enum is 690 * wide enough to hold the higher bits reserved for bpf_type_flag. 691 */ 692 __BPF_ARG_TYPE_LIMIT = BPF_TYPE_LIMIT, 693 }; 694 static_assert(__BPF_ARG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); 695 696 /* type of values returned from helper functions */ 697 enum bpf_return_type { 698 RET_INTEGER, /* function returns integer */ 699 RET_VOID, /* function doesn't return anything */ 700 RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */ 701 RET_PTR_TO_SOCKET, /* returns a pointer to a socket */ 702 RET_PTR_TO_TCP_SOCK, /* returns a pointer to a tcp_sock */ 703 RET_PTR_TO_SOCK_COMMON, /* returns a pointer to a sock_common */ 704 RET_PTR_TO_MEM, /* returns a pointer to memory */ 705 RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */ 706 RET_PTR_TO_BTF_ID, /* returns a pointer to a btf_id */ 707 __BPF_RET_TYPE_MAX, 708 709 /* Extended ret_types. */ 710 RET_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_MAP_VALUE, 711 RET_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCKET, 712 RET_PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_TCP_SOCK, 713 RET_PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCK_COMMON, 714 RET_PTR_TO_RINGBUF_MEM_OR_NULL = PTR_MAYBE_NULL | MEM_RINGBUF | RET_PTR_TO_MEM, 715 RET_PTR_TO_DYNPTR_MEM_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_MEM, 716 RET_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_BTF_ID, 717 RET_PTR_TO_BTF_ID_TRUSTED = PTR_TRUSTED | RET_PTR_TO_BTF_ID, 718 719 /* This must be the last entry. Its purpose is to ensure the enum is 720 * wide enough to hold the higher bits reserved for bpf_type_flag. 721 */ 722 __BPF_RET_TYPE_LIMIT = BPF_TYPE_LIMIT, 723 }; 724 static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); 725 726 /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs 727 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL 728 * instructions after verifying 729 */ 730 struct bpf_func_proto { 731 u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 732 bool gpl_only; 733 bool pkt_access; 734 bool might_sleep; 735 enum bpf_return_type ret_type; 736 union { 737 struct { 738 enum bpf_arg_type arg1_type; 739 enum bpf_arg_type arg2_type; 740 enum bpf_arg_type arg3_type; 741 enum bpf_arg_type arg4_type; 742 enum bpf_arg_type arg5_type; 743 }; 744 enum bpf_arg_type arg_type[5]; 745 }; 746 union { 747 struct { 748 u32 *arg1_btf_id; 749 u32 *arg2_btf_id; 750 u32 *arg3_btf_id; 751 u32 *arg4_btf_id; 752 u32 *arg5_btf_id; 753 }; 754 u32 *arg_btf_id[5]; 755 struct { 756 size_t arg1_size; 757 size_t arg2_size; 758 size_t arg3_size; 759 size_t arg4_size; 760 size_t arg5_size; 761 }; 762 size_t arg_size[5]; 763 }; 764 int *ret_btf_id; /* return value btf_id */ 765 bool (*allowed)(const struct bpf_prog *prog); 766 }; 767 768 /* bpf_context is intentionally undefined structure. Pointer to bpf_context is 769 * the first argument to eBPF programs. 770 * For socket filters: 'struct bpf_context *' == 'struct sk_buff *' 771 */ 772 struct bpf_context; 773 774 enum bpf_access_type { 775 BPF_READ = 1, 776 BPF_WRITE = 2 777 }; 778 779 /* types of values stored in eBPF registers */ 780 /* Pointer types represent: 781 * pointer 782 * pointer + imm 783 * pointer + (u16) var 784 * pointer + (u16) var + imm 785 * if (range > 0) then [ptr, ptr + range - off) is safe to access 786 * if (id > 0) means that some 'var' was added 787 * if (off > 0) means that 'imm' was added 788 */ 789 enum bpf_reg_type { 790 NOT_INIT = 0, /* nothing was written into register */ 791 SCALAR_VALUE, /* reg doesn't contain a valid pointer */ 792 PTR_TO_CTX, /* reg points to bpf_context */ 793 CONST_PTR_TO_MAP, /* reg points to struct bpf_map */ 794 PTR_TO_MAP_VALUE, /* reg points to map element value */ 795 PTR_TO_MAP_KEY, /* reg points to a map element key */ 796 PTR_TO_STACK, /* reg == frame_pointer + offset */ 797 PTR_TO_PACKET_META, /* skb->data - meta_len */ 798 PTR_TO_PACKET, /* reg points to skb->data */ 799 PTR_TO_PACKET_END, /* skb->data + headlen */ 800 PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */ 801 PTR_TO_SOCKET, /* reg points to struct bpf_sock */ 802 PTR_TO_SOCK_COMMON, /* reg points to sock_common */ 803 PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */ 804 PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */ 805 PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */ 806 /* PTR_TO_BTF_ID points to a kernel struct that does not need 807 * to be null checked by the BPF program. This does not imply the 808 * pointer is _not_ null and in practice this can easily be a null 809 * pointer when reading pointer chains. The assumption is program 810 * context will handle null pointer dereference typically via fault 811 * handling. The verifier must keep this in mind and can make no 812 * assumptions about null or non-null when doing branch analysis. 813 * Further, when passed into helpers the helpers can not, without 814 * additional context, assume the value is non-null. 815 */ 816 PTR_TO_BTF_ID, 817 /* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not 818 * been checked for null. Used primarily to inform the verifier 819 * an explicit null check is required for this struct. 820 */ 821 PTR_TO_MEM, /* reg points to valid memory region */ 822 PTR_TO_BUF, /* reg points to a read/write buffer */ 823 PTR_TO_FUNC, /* reg points to a bpf program function */ 824 CONST_PTR_TO_DYNPTR, /* reg points to a const struct bpf_dynptr */ 825 __BPF_REG_TYPE_MAX, 826 827 /* Extended reg_types. */ 828 PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | PTR_TO_MAP_VALUE, 829 PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCKET, 830 PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON, 831 PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | PTR_TO_TCP_SOCK, 832 PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | PTR_TO_BTF_ID, 833 834 /* This must be the last entry. Its purpose is to ensure the enum is 835 * wide enough to hold the higher bits reserved for bpf_type_flag. 836 */ 837 __BPF_REG_TYPE_LIMIT = BPF_TYPE_LIMIT, 838 }; 839 static_assert(__BPF_REG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); 840 841 /* The information passed from prog-specific *_is_valid_access 842 * back to the verifier. 843 */ 844 struct bpf_insn_access_aux { 845 enum bpf_reg_type reg_type; 846 union { 847 int ctx_field_size; 848 struct { 849 struct btf *btf; 850 u32 btf_id; 851 }; 852 }; 853 struct bpf_verifier_log *log; /* for verbose logs */ 854 }; 855 856 static inline void 857 bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size) 858 { 859 aux->ctx_field_size = size; 860 } 861 862 static inline bool bpf_pseudo_func(const struct bpf_insn *insn) 863 { 864 return insn->code == (BPF_LD | BPF_IMM | BPF_DW) && 865 insn->src_reg == BPF_PSEUDO_FUNC; 866 } 867 868 struct bpf_prog_ops { 869 int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, 870 union bpf_attr __user *uattr); 871 }; 872 873 struct bpf_reg_state; 874 struct bpf_verifier_ops { 875 /* return eBPF function prototype for verification */ 876 const struct bpf_func_proto * 877 (*get_func_proto)(enum bpf_func_id func_id, 878 const struct bpf_prog *prog); 879 880 /* return true if 'size' wide access at offset 'off' within bpf_context 881 * with 'type' (read or write) is allowed 882 */ 883 bool (*is_valid_access)(int off, int size, enum bpf_access_type type, 884 const struct bpf_prog *prog, 885 struct bpf_insn_access_aux *info); 886 int (*gen_prologue)(struct bpf_insn *insn, bool direct_write, 887 const struct bpf_prog *prog); 888 int (*gen_ld_abs)(const struct bpf_insn *orig, 889 struct bpf_insn *insn_buf); 890 u32 (*convert_ctx_access)(enum bpf_access_type type, 891 const struct bpf_insn *src, 892 struct bpf_insn *dst, 893 struct bpf_prog *prog, u32 *target_size); 894 int (*btf_struct_access)(struct bpf_verifier_log *log, 895 const struct bpf_reg_state *reg, 896 int off, int size, enum bpf_access_type atype, 897 u32 *next_btf_id, enum bpf_type_flag *flag); 898 }; 899 900 struct bpf_prog_offload_ops { 901 /* verifier basic callbacks */ 902 int (*insn_hook)(struct bpf_verifier_env *env, 903 int insn_idx, int prev_insn_idx); 904 int (*finalize)(struct bpf_verifier_env *env); 905 /* verifier optimization callbacks (called after .finalize) */ 906 int (*replace_insn)(struct bpf_verifier_env *env, u32 off, 907 struct bpf_insn *insn); 908 int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt); 909 /* program management callbacks */ 910 int (*prepare)(struct bpf_prog *prog); 911 int (*translate)(struct bpf_prog *prog); 912 void (*destroy)(struct bpf_prog *prog); 913 }; 914 915 struct bpf_prog_offload { 916 struct bpf_prog *prog; 917 struct net_device *netdev; 918 struct bpf_offload_dev *offdev; 919 void *dev_priv; 920 struct list_head offloads; 921 bool dev_state; 922 bool opt_failed; 923 void *jited_image; 924 u32 jited_len; 925 }; 926 927 enum bpf_cgroup_storage_type { 928 BPF_CGROUP_STORAGE_SHARED, 929 BPF_CGROUP_STORAGE_PERCPU, 930 __BPF_CGROUP_STORAGE_MAX 931 }; 932 933 #define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX 934 935 /* The longest tracepoint has 12 args. 936 * See include/trace/bpf_probe.h 937 */ 938 #define MAX_BPF_FUNC_ARGS 12 939 940 /* The maximum number of arguments passed through registers 941 * a single function may have. 942 */ 943 #define MAX_BPF_FUNC_REG_ARGS 5 944 945 /* The argument is a structure. */ 946 #define BTF_FMODEL_STRUCT_ARG BIT(0) 947 948 /* The argument is signed. */ 949 #define BTF_FMODEL_SIGNED_ARG BIT(1) 950 951 struct btf_func_model { 952 u8 ret_size; 953 u8 ret_flags; 954 u8 nr_args; 955 u8 arg_size[MAX_BPF_FUNC_ARGS]; 956 u8 arg_flags[MAX_BPF_FUNC_ARGS]; 957 }; 958 959 /* Restore arguments before returning from trampoline to let original function 960 * continue executing. This flag is used for fentry progs when there are no 961 * fexit progs. 962 */ 963 #define BPF_TRAMP_F_RESTORE_REGS BIT(0) 964 /* Call original function after fentry progs, but before fexit progs. 965 * Makes sense for fentry/fexit, normal calls and indirect calls. 966 */ 967 #define BPF_TRAMP_F_CALL_ORIG BIT(1) 968 /* Skip current frame and return to parent. Makes sense for fentry/fexit 969 * programs only. Should not be used with normal calls and indirect calls. 970 */ 971 #define BPF_TRAMP_F_SKIP_FRAME BIT(2) 972 /* Store IP address of the caller on the trampoline stack, 973 * so it's available for trampoline's programs. 974 */ 975 #define BPF_TRAMP_F_IP_ARG BIT(3) 976 /* Return the return value of fentry prog. Only used by bpf_struct_ops. */ 977 #define BPF_TRAMP_F_RET_FENTRY_RET BIT(4) 978 979 /* Get original function from stack instead of from provided direct address. 980 * Makes sense for trampolines with fexit or fmod_ret programs. 981 */ 982 #define BPF_TRAMP_F_ORIG_STACK BIT(5) 983 984 /* This trampoline is on a function with another ftrace_ops with IPMODIFY, 985 * e.g., a live patch. This flag is set and cleared by ftrace call backs, 986 */ 987 #define BPF_TRAMP_F_SHARE_IPMODIFY BIT(6) 988 989 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 990 * bytes on x86. 991 */ 992 enum { 993 #if defined(__s390x__) 994 BPF_MAX_TRAMP_LINKS = 27, 995 #else 996 BPF_MAX_TRAMP_LINKS = 38, 997 #endif 998 }; 999 1000 struct bpf_tramp_links { 1001 struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS]; 1002 int nr_links; 1003 }; 1004 1005 struct bpf_tramp_run_ctx; 1006 1007 /* Different use cases for BPF trampoline: 1008 * 1. replace nop at the function entry (kprobe equivalent) 1009 * flags = BPF_TRAMP_F_RESTORE_REGS 1010 * fentry = a set of programs to run before returning from trampoline 1011 * 1012 * 2. replace nop at the function entry (kprobe + kretprobe equivalent) 1013 * flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME 1014 * orig_call = fentry_ip + MCOUNT_INSN_SIZE 1015 * fentry = a set of program to run before calling original function 1016 * fexit = a set of program to run after original function 1017 * 1018 * 3. replace direct call instruction anywhere in the function body 1019 * or assign a function pointer for indirect call (like tcp_congestion_ops->cong_avoid) 1020 * With flags = 0 1021 * fentry = a set of programs to run before returning from trampoline 1022 * With flags = BPF_TRAMP_F_CALL_ORIG 1023 * orig_call = original callback addr or direct function addr 1024 * fentry = a set of program to run before calling original function 1025 * fexit = a set of program to run after original function 1026 */ 1027 struct bpf_tramp_image; 1028 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end, 1029 const struct btf_func_model *m, u32 flags, 1030 struct bpf_tramp_links *tlinks, 1031 void *orig_call); 1032 u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog, 1033 struct bpf_tramp_run_ctx *run_ctx); 1034 void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start, 1035 struct bpf_tramp_run_ctx *run_ctx); 1036 void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr); 1037 void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr); 1038 typedef u64 (*bpf_trampoline_enter_t)(struct bpf_prog *prog, 1039 struct bpf_tramp_run_ctx *run_ctx); 1040 typedef void (*bpf_trampoline_exit_t)(struct bpf_prog *prog, u64 start, 1041 struct bpf_tramp_run_ctx *run_ctx); 1042 bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog); 1043 bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog); 1044 1045 struct bpf_ksym { 1046 unsigned long start; 1047 unsigned long end; 1048 char name[KSYM_NAME_LEN]; 1049 struct list_head lnode; 1050 struct latch_tree_node tnode; 1051 bool prog; 1052 }; 1053 1054 enum bpf_tramp_prog_type { 1055 BPF_TRAMP_FENTRY, 1056 BPF_TRAMP_FEXIT, 1057 BPF_TRAMP_MODIFY_RETURN, 1058 BPF_TRAMP_MAX, 1059 BPF_TRAMP_REPLACE, /* more than MAX */ 1060 }; 1061 1062 struct bpf_tramp_image { 1063 void *image; 1064 struct bpf_ksym ksym; 1065 struct percpu_ref pcref; 1066 void *ip_after_call; 1067 void *ip_epilogue; 1068 union { 1069 struct rcu_head rcu; 1070 struct work_struct work; 1071 }; 1072 }; 1073 1074 struct bpf_trampoline { 1075 /* hlist for trampoline_table */ 1076 struct hlist_node hlist; 1077 struct ftrace_ops *fops; 1078 /* serializes access to fields of this trampoline */ 1079 struct mutex mutex; 1080 refcount_t refcnt; 1081 u32 flags; 1082 u64 key; 1083 struct { 1084 struct btf_func_model model; 1085 void *addr; 1086 bool ftrace_managed; 1087 } func; 1088 /* if !NULL this is BPF_PROG_TYPE_EXT program that extends another BPF 1089 * program by replacing one of its functions. func.addr is the address 1090 * of the function it replaced. 1091 */ 1092 struct bpf_prog *extension_prog; 1093 /* list of BPF programs using this trampoline */ 1094 struct hlist_head progs_hlist[BPF_TRAMP_MAX]; 1095 /* Number of attached programs. A counter per kind. */ 1096 int progs_cnt[BPF_TRAMP_MAX]; 1097 /* Executable image of trampoline */ 1098 struct bpf_tramp_image *cur_image; 1099 u64 selector; 1100 struct module *mod; 1101 }; 1102 1103 struct bpf_attach_target_info { 1104 struct btf_func_model fmodel; 1105 long tgt_addr; 1106 struct module *tgt_mod; 1107 const char *tgt_name; 1108 const struct btf_type *tgt_type; 1109 }; 1110 1111 #define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */ 1112 1113 struct bpf_dispatcher_prog { 1114 struct bpf_prog *prog; 1115 refcount_t users; 1116 }; 1117 1118 struct bpf_dispatcher { 1119 /* dispatcher mutex */ 1120 struct mutex mutex; 1121 void *func; 1122 struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX]; 1123 int num_progs; 1124 void *image; 1125 void *rw_image; 1126 u32 image_off; 1127 struct bpf_ksym ksym; 1128 #ifdef CONFIG_HAVE_STATIC_CALL 1129 struct static_call_key *sc_key; 1130 void *sc_tramp; 1131 #endif 1132 }; 1133 1134 static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( 1135 const void *ctx, 1136 const struct bpf_insn *insnsi, 1137 bpf_func_t bpf_func) 1138 { 1139 return bpf_func(ctx, insnsi); 1140 } 1141 1142 /* the implementation of the opaque uapi struct bpf_dynptr */ 1143 struct bpf_dynptr_kern { 1144 void *data; 1145 /* Size represents the number of usable bytes of dynptr data. 1146 * If for example the offset is at 4 for a local dynptr whose data is 1147 * of type u64, the number of usable bytes is 4. 1148 * 1149 * The upper 8 bits are reserved. It is as follows: 1150 * Bits 0 - 23 = size 1151 * Bits 24 - 30 = dynptr type 1152 * Bit 31 = whether dynptr is read-only 1153 */ 1154 u32 size; 1155 u32 offset; 1156 } __aligned(8); 1157 1158 enum bpf_dynptr_type { 1159 BPF_DYNPTR_TYPE_INVALID, 1160 /* Points to memory that is local to the bpf program */ 1161 BPF_DYNPTR_TYPE_LOCAL, 1162 /* Underlying data is a ringbuf record */ 1163 BPF_DYNPTR_TYPE_RINGBUF, 1164 /* Underlying data is a sk_buff */ 1165 BPF_DYNPTR_TYPE_SKB, 1166 /* Underlying data is a xdp_buff */ 1167 BPF_DYNPTR_TYPE_XDP, 1168 }; 1169 1170 int bpf_dynptr_check_size(u32 size); 1171 u32 bpf_dynptr_get_size(const struct bpf_dynptr_kern *ptr); 1172 1173 #ifdef CONFIG_BPF_JIT 1174 int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr); 1175 int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr); 1176 struct bpf_trampoline *bpf_trampoline_get(u64 key, 1177 struct bpf_attach_target_info *tgt_info); 1178 void bpf_trampoline_put(struct bpf_trampoline *tr); 1179 int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs); 1180 1181 /* 1182 * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn 1183 * indirection with a direct call to the bpf program. If the architecture does 1184 * not have STATIC_CALL, avoid a double-indirection. 1185 */ 1186 #ifdef CONFIG_HAVE_STATIC_CALL 1187 1188 #define __BPF_DISPATCHER_SC_INIT(_name) \ 1189 .sc_key = &STATIC_CALL_KEY(_name), \ 1190 .sc_tramp = STATIC_CALL_TRAMP_ADDR(_name), 1191 1192 #define __BPF_DISPATCHER_SC(name) \ 1193 DEFINE_STATIC_CALL(bpf_dispatcher_##name##_call, bpf_dispatcher_nop_func) 1194 1195 #define __BPF_DISPATCHER_CALL(name) \ 1196 static_call(bpf_dispatcher_##name##_call)(ctx, insnsi, bpf_func) 1197 1198 #define __BPF_DISPATCHER_UPDATE(_d, _new) \ 1199 __static_call_update((_d)->sc_key, (_d)->sc_tramp, (_new)) 1200 1201 #else 1202 #define __BPF_DISPATCHER_SC_INIT(name) 1203 #define __BPF_DISPATCHER_SC(name) 1204 #define __BPF_DISPATCHER_CALL(name) bpf_func(ctx, insnsi) 1205 #define __BPF_DISPATCHER_UPDATE(_d, _new) 1206 #endif 1207 1208 #define BPF_DISPATCHER_INIT(_name) { \ 1209 .mutex = __MUTEX_INITIALIZER(_name.mutex), \ 1210 .func = &_name##_func, \ 1211 .progs = {}, \ 1212 .num_progs = 0, \ 1213 .image = NULL, \ 1214 .image_off = 0, \ 1215 .ksym = { \ 1216 .name = #_name, \ 1217 .lnode = LIST_HEAD_INIT(_name.ksym.lnode), \ 1218 }, \ 1219 __BPF_DISPATCHER_SC_INIT(_name##_call) \ 1220 } 1221 1222 #define DEFINE_BPF_DISPATCHER(name) \ 1223 __BPF_DISPATCHER_SC(name); \ 1224 noinline __nocfi unsigned int bpf_dispatcher_##name##_func( \ 1225 const void *ctx, \ 1226 const struct bpf_insn *insnsi, \ 1227 bpf_func_t bpf_func) \ 1228 { \ 1229 return __BPF_DISPATCHER_CALL(name); \ 1230 } \ 1231 EXPORT_SYMBOL(bpf_dispatcher_##name##_func); \ 1232 struct bpf_dispatcher bpf_dispatcher_##name = \ 1233 BPF_DISPATCHER_INIT(bpf_dispatcher_##name); 1234 1235 #define DECLARE_BPF_DISPATCHER(name) \ 1236 unsigned int bpf_dispatcher_##name##_func( \ 1237 const void *ctx, \ 1238 const struct bpf_insn *insnsi, \ 1239 bpf_func_t bpf_func); \ 1240 extern struct bpf_dispatcher bpf_dispatcher_##name; 1241 1242 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func 1243 #define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name) 1244 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, 1245 struct bpf_prog *to); 1246 /* Called only from JIT-enabled code, so there's no need for stubs. */ 1247 void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym); 1248 void bpf_image_ksym_del(struct bpf_ksym *ksym); 1249 void bpf_ksym_add(struct bpf_ksym *ksym); 1250 void bpf_ksym_del(struct bpf_ksym *ksym); 1251 int bpf_jit_charge_modmem(u32 size); 1252 void bpf_jit_uncharge_modmem(u32 size); 1253 bool bpf_prog_has_trampoline(const struct bpf_prog *prog); 1254 #else 1255 static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link, 1256 struct bpf_trampoline *tr) 1257 { 1258 return -ENOTSUPP; 1259 } 1260 static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, 1261 struct bpf_trampoline *tr) 1262 { 1263 return -ENOTSUPP; 1264 } 1265 static inline struct bpf_trampoline *bpf_trampoline_get(u64 key, 1266 struct bpf_attach_target_info *tgt_info) 1267 { 1268 return ERR_PTR(-EOPNOTSUPP); 1269 } 1270 static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {} 1271 #define DEFINE_BPF_DISPATCHER(name) 1272 #define DECLARE_BPF_DISPATCHER(name) 1273 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nop_func 1274 #define BPF_DISPATCHER_PTR(name) NULL 1275 static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, 1276 struct bpf_prog *from, 1277 struct bpf_prog *to) {} 1278 static inline bool is_bpf_image_address(unsigned long address) 1279 { 1280 return false; 1281 } 1282 static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog) 1283 { 1284 return false; 1285 } 1286 #endif 1287 1288 struct bpf_func_info_aux { 1289 u16 linkage; 1290 bool unreliable; 1291 }; 1292 1293 enum bpf_jit_poke_reason { 1294 BPF_POKE_REASON_TAIL_CALL, 1295 }; 1296 1297 /* Descriptor of pokes pointing /into/ the JITed image. */ 1298 struct bpf_jit_poke_descriptor { 1299 void *tailcall_target; 1300 void *tailcall_bypass; 1301 void *bypass_addr; 1302 void *aux; 1303 union { 1304 struct { 1305 struct bpf_map *map; 1306 u32 key; 1307 } tail_call; 1308 }; 1309 bool tailcall_target_stable; 1310 u8 adj_off; 1311 u16 reason; 1312 u32 insn_idx; 1313 }; 1314 1315 /* reg_type info for ctx arguments */ 1316 struct bpf_ctx_arg_aux { 1317 u32 offset; 1318 enum bpf_reg_type reg_type; 1319 u32 btf_id; 1320 }; 1321 1322 struct btf_mod_pair { 1323 struct btf *btf; 1324 struct module *module; 1325 }; 1326 1327 struct bpf_kfunc_desc_tab; 1328 1329 struct bpf_prog_aux { 1330 atomic64_t refcnt; 1331 u32 used_map_cnt; 1332 u32 used_btf_cnt; 1333 u32 max_ctx_offset; 1334 u32 max_pkt_offset; 1335 u32 max_tp_access; 1336 u32 stack_depth; 1337 u32 id; 1338 u32 func_cnt; /* used by non-func prog as the number of func progs */ 1339 u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */ 1340 u32 attach_btf_id; /* in-kernel BTF type id to attach to */ 1341 u32 ctx_arg_info_size; 1342 u32 max_rdonly_access; 1343 u32 max_rdwr_access; 1344 struct btf *attach_btf; 1345 const struct bpf_ctx_arg_aux *ctx_arg_info; 1346 struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */ 1347 struct bpf_prog *dst_prog; 1348 struct bpf_trampoline *dst_trampoline; 1349 enum bpf_prog_type saved_dst_prog_type; 1350 enum bpf_attach_type saved_dst_attach_type; 1351 bool verifier_zext; /* Zero extensions has been inserted by verifier. */ 1352 bool dev_bound; /* Program is bound to the netdev. */ 1353 bool offload_requested; /* Program is bound and offloaded to the netdev. */ 1354 bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ 1355 bool func_proto_unreliable; 1356 bool sleepable; 1357 bool tail_call_reachable; 1358 bool xdp_has_frags; 1359 /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ 1360 const struct btf_type *attach_func_proto; 1361 /* function name for valid attach_btf_id */ 1362 const char *attach_func_name; 1363 struct bpf_prog **func; 1364 void *jit_data; /* JIT specific data. arch dependent */ 1365 struct bpf_jit_poke_descriptor *poke_tab; 1366 struct bpf_kfunc_desc_tab *kfunc_tab; 1367 struct bpf_kfunc_btf_tab *kfunc_btf_tab; 1368 u32 size_poke_tab; 1369 struct bpf_ksym ksym; 1370 const struct bpf_prog_ops *ops; 1371 struct bpf_map **used_maps; 1372 struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */ 1373 struct btf_mod_pair *used_btfs; 1374 struct bpf_prog *prog; 1375 struct user_struct *user; 1376 u64 load_time; /* ns since boottime */ 1377 u32 verified_insns; 1378 int cgroup_atype; /* enum cgroup_bpf_attach_type */ 1379 struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 1380 char name[BPF_OBJ_NAME_LEN]; 1381 #ifdef CONFIG_SECURITY 1382 void *security; 1383 #endif 1384 struct bpf_prog_offload *offload; 1385 struct btf *btf; 1386 struct bpf_func_info *func_info; 1387 struct bpf_func_info_aux *func_info_aux; 1388 /* bpf_line_info loaded from userspace. linfo->insn_off 1389 * has the xlated insn offset. 1390 * Both the main and sub prog share the same linfo. 1391 * The subprog can access its first linfo by 1392 * using the linfo_idx. 1393 */ 1394 struct bpf_line_info *linfo; 1395 /* jited_linfo is the jited addr of the linfo. It has a 1396 * one to one mapping to linfo: 1397 * jited_linfo[i] is the jited addr for the linfo[i]->insn_off. 1398 * Both the main and sub prog share the same jited_linfo. 1399 * The subprog can access its first jited_linfo by 1400 * using the linfo_idx. 1401 */ 1402 void **jited_linfo; 1403 u32 func_info_cnt; 1404 u32 nr_linfo; 1405 /* subprog can use linfo_idx to access its first linfo and 1406 * jited_linfo. 1407 * main prog always has linfo_idx == 0 1408 */ 1409 u32 linfo_idx; 1410 struct module *mod; 1411 u32 num_exentries; 1412 struct exception_table_entry *extable; 1413 union { 1414 struct work_struct work; 1415 struct rcu_head rcu; 1416 }; 1417 }; 1418 1419 struct bpf_prog { 1420 u16 pages; /* Number of allocated pages */ 1421 u16 jited:1, /* Is our filter JIT'ed? */ 1422 jit_requested:1,/* archs need to JIT the prog */ 1423 gpl_compatible:1, /* Is filter GPL compatible? */ 1424 cb_access:1, /* Is control block accessed? */ 1425 dst_needed:1, /* Do we need dst entry? */ 1426 blinding_requested:1, /* needs constant blinding */ 1427 blinded:1, /* Was blinded */ 1428 is_func:1, /* program is a bpf function */ 1429 kprobe_override:1, /* Do we override a kprobe? */ 1430 has_callchain_buf:1, /* callchain buffer allocated? */ 1431 enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */ 1432 call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */ 1433 call_get_func_ip:1, /* Do we call get_func_ip() */ 1434 tstamp_type_access:1; /* Accessed __sk_buff->tstamp_type */ 1435 enum bpf_prog_type type; /* Type of BPF program */ 1436 enum bpf_attach_type expected_attach_type; /* For some prog types */ 1437 u32 len; /* Number of filter blocks */ 1438 u32 jited_len; /* Size of jited insns in bytes */ 1439 u8 tag[BPF_TAG_SIZE]; 1440 struct bpf_prog_stats __percpu *stats; 1441 int __percpu *active; 1442 unsigned int (*bpf_func)(const void *ctx, 1443 const struct bpf_insn *insn); 1444 struct bpf_prog_aux *aux; /* Auxiliary fields */ 1445 struct sock_fprog_kern *orig_prog; /* Original BPF program */ 1446 /* Instructions for interpreter */ 1447 union { 1448 DECLARE_FLEX_ARRAY(struct sock_filter, insns); 1449 DECLARE_FLEX_ARRAY(struct bpf_insn, insnsi); 1450 }; 1451 }; 1452 1453 struct bpf_array_aux { 1454 /* Programs with direct jumps into programs part of this array. */ 1455 struct list_head poke_progs; 1456 struct bpf_map *map; 1457 struct mutex poke_mutex; 1458 struct work_struct work; 1459 }; 1460 1461 struct bpf_link { 1462 atomic64_t refcnt; 1463 u32 id; 1464 enum bpf_link_type type; 1465 const struct bpf_link_ops *ops; 1466 struct bpf_prog *prog; 1467 struct work_struct work; 1468 }; 1469 1470 struct bpf_link_ops { 1471 void (*release)(struct bpf_link *link); 1472 void (*dealloc)(struct bpf_link *link); 1473 int (*detach)(struct bpf_link *link); 1474 int (*update_prog)(struct bpf_link *link, struct bpf_prog *new_prog, 1475 struct bpf_prog *old_prog); 1476 void (*show_fdinfo)(const struct bpf_link *link, struct seq_file *seq); 1477 int (*fill_link_info)(const struct bpf_link *link, 1478 struct bpf_link_info *info); 1479 }; 1480 1481 struct bpf_tramp_link { 1482 struct bpf_link link; 1483 struct hlist_node tramp_hlist; 1484 u64 cookie; 1485 }; 1486 1487 struct bpf_shim_tramp_link { 1488 struct bpf_tramp_link link; 1489 struct bpf_trampoline *trampoline; 1490 }; 1491 1492 struct bpf_tracing_link { 1493 struct bpf_tramp_link link; 1494 enum bpf_attach_type attach_type; 1495 struct bpf_trampoline *trampoline; 1496 struct bpf_prog *tgt_prog; 1497 }; 1498 1499 struct bpf_link_primer { 1500 struct bpf_link *link; 1501 struct file *file; 1502 int fd; 1503 u32 id; 1504 }; 1505 1506 struct bpf_struct_ops_value; 1507 struct btf_member; 1508 1509 #define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64 1510 struct bpf_struct_ops { 1511 const struct bpf_verifier_ops *verifier_ops; 1512 int (*init)(struct btf *btf); 1513 int (*check_member)(const struct btf_type *t, 1514 const struct btf_member *member, 1515 const struct bpf_prog *prog); 1516 int (*init_member)(const struct btf_type *t, 1517 const struct btf_member *member, 1518 void *kdata, const void *udata); 1519 int (*reg)(void *kdata); 1520 void (*unreg)(void *kdata); 1521 const struct btf_type *type; 1522 const struct btf_type *value_type; 1523 const char *name; 1524 struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; 1525 u32 type_id; 1526 u32 value_id; 1527 }; 1528 1529 #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) 1530 #define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA)) 1531 const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id); 1532 void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log); 1533 bool bpf_struct_ops_get(const void *kdata); 1534 void bpf_struct_ops_put(const void *kdata); 1535 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, 1536 void *value); 1537 int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, 1538 struct bpf_tramp_link *link, 1539 const struct btf_func_model *model, 1540 void *image, void *image_end); 1541 static inline bool bpf_try_module_get(const void *data, struct module *owner) 1542 { 1543 if (owner == BPF_MODULE_OWNER) 1544 return bpf_struct_ops_get(data); 1545 else 1546 return try_module_get(owner); 1547 } 1548 static inline void bpf_module_put(const void *data, struct module *owner) 1549 { 1550 if (owner == BPF_MODULE_OWNER) 1551 bpf_struct_ops_put(data); 1552 else 1553 module_put(owner); 1554 } 1555 1556 #ifdef CONFIG_NET 1557 /* Define it here to avoid the use of forward declaration */ 1558 struct bpf_dummy_ops_state { 1559 int val; 1560 }; 1561 1562 struct bpf_dummy_ops { 1563 int (*test_1)(struct bpf_dummy_ops_state *cb); 1564 int (*test_2)(struct bpf_dummy_ops_state *cb, int a1, unsigned short a2, 1565 char a3, unsigned long a4); 1566 int (*test_sleepable)(struct bpf_dummy_ops_state *cb); 1567 }; 1568 1569 int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, 1570 union bpf_attr __user *uattr); 1571 #endif 1572 #else 1573 static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id) 1574 { 1575 return NULL; 1576 } 1577 static inline void bpf_struct_ops_init(struct btf *btf, 1578 struct bpf_verifier_log *log) 1579 { 1580 } 1581 static inline bool bpf_try_module_get(const void *data, struct module *owner) 1582 { 1583 return try_module_get(owner); 1584 } 1585 static inline void bpf_module_put(const void *data, struct module *owner) 1586 { 1587 module_put(owner); 1588 } 1589 static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, 1590 void *key, 1591 void *value) 1592 { 1593 return -EINVAL; 1594 } 1595 #endif 1596 1597 #if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM) 1598 int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, 1599 int cgroup_atype); 1600 void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog); 1601 #else 1602 static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, 1603 int cgroup_atype) 1604 { 1605 return -EOPNOTSUPP; 1606 } 1607 static inline void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog) 1608 { 1609 } 1610 #endif 1611 1612 struct bpf_array { 1613 struct bpf_map map; 1614 u32 elem_size; 1615 u32 index_mask; 1616 struct bpf_array_aux *aux; 1617 union { 1618 DECLARE_FLEX_ARRAY(char, value) __aligned(8); 1619 DECLARE_FLEX_ARRAY(void *, ptrs) __aligned(8); 1620 DECLARE_FLEX_ARRAY(void __percpu *, pptrs) __aligned(8); 1621 }; 1622 }; 1623 1624 #define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */ 1625 #define MAX_TAIL_CALL_CNT 33 1626 1627 /* Maximum number of loops for bpf_loop and bpf_iter_num. 1628 * It's enum to expose it (and thus make it discoverable) through BTF. 1629 */ 1630 enum { 1631 BPF_MAX_LOOPS = 8 * 1024 * 1024, 1632 }; 1633 1634 #define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \ 1635 BPF_F_RDONLY_PROG | \ 1636 BPF_F_WRONLY | \ 1637 BPF_F_WRONLY_PROG) 1638 1639 #define BPF_MAP_CAN_READ BIT(0) 1640 #define BPF_MAP_CAN_WRITE BIT(1) 1641 1642 /* Maximum number of user-producer ring buffer samples that can be drained in 1643 * a call to bpf_user_ringbuf_drain(). 1644 */ 1645 #define BPF_MAX_USER_RINGBUF_SAMPLES (128 * 1024) 1646 1647 static inline u32 bpf_map_flags_to_cap(struct bpf_map *map) 1648 { 1649 u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); 1650 1651 /* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is 1652 * not possible. 1653 */ 1654 if (access_flags & BPF_F_RDONLY_PROG) 1655 return BPF_MAP_CAN_READ; 1656 else if (access_flags & BPF_F_WRONLY_PROG) 1657 return BPF_MAP_CAN_WRITE; 1658 else 1659 return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE; 1660 } 1661 1662 static inline bool bpf_map_flags_access_ok(u32 access_flags) 1663 { 1664 return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) != 1665 (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); 1666 } 1667 1668 struct bpf_event_entry { 1669 struct perf_event *event; 1670 struct file *perf_file; 1671 struct file *map_file; 1672 struct rcu_head rcu; 1673 }; 1674 1675 static inline bool map_type_contains_progs(struct bpf_map *map) 1676 { 1677 return map->map_type == BPF_MAP_TYPE_PROG_ARRAY || 1678 map->map_type == BPF_MAP_TYPE_DEVMAP || 1679 map->map_type == BPF_MAP_TYPE_CPUMAP; 1680 } 1681 1682 bool bpf_prog_map_compatible(struct bpf_map *map, const struct bpf_prog *fp); 1683 int bpf_prog_calc_tag(struct bpf_prog *fp); 1684 1685 const struct bpf_func_proto *bpf_get_trace_printk_proto(void); 1686 const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void); 1687 1688 typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src, 1689 unsigned long off, unsigned long len); 1690 typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type, 1691 const struct bpf_insn *src, 1692 struct bpf_insn *dst, 1693 struct bpf_prog *prog, 1694 u32 *target_size); 1695 1696 u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, 1697 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy); 1698 1699 /* an array of programs to be executed under rcu_lock. 1700 * 1701 * Typical usage: 1702 * ret = bpf_prog_run_array(rcu_dereference(&bpf_prog_array), ctx, bpf_prog_run); 1703 * 1704 * the structure returned by bpf_prog_array_alloc() should be populated 1705 * with program pointers and the last pointer must be NULL. 1706 * The user has to keep refcnt on the program and make sure the program 1707 * is removed from the array before bpf_prog_put(). 1708 * The 'struct bpf_prog_array *' should only be replaced with xchg() 1709 * since other cpus are walking the array of pointers in parallel. 1710 */ 1711 struct bpf_prog_array_item { 1712 struct bpf_prog *prog; 1713 union { 1714 struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 1715 u64 bpf_cookie; 1716 }; 1717 }; 1718 1719 struct bpf_prog_array { 1720 struct rcu_head rcu; 1721 struct bpf_prog_array_item items[]; 1722 }; 1723 1724 struct bpf_empty_prog_array { 1725 struct bpf_prog_array hdr; 1726 struct bpf_prog *null_prog; 1727 }; 1728 1729 /* to avoid allocating empty bpf_prog_array for cgroups that 1730 * don't have bpf program attached use one global 'bpf_empty_prog_array' 1731 * It will not be modified the caller of bpf_prog_array_alloc() 1732 * (since caller requested prog_cnt == 0) 1733 * that pointer should be 'freed' by bpf_prog_array_free() 1734 */ 1735 extern struct bpf_empty_prog_array bpf_empty_prog_array; 1736 1737 struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags); 1738 void bpf_prog_array_free(struct bpf_prog_array *progs); 1739 /* Use when traversal over the bpf_prog_array uses tasks_trace rcu */ 1740 void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs); 1741 int bpf_prog_array_length(struct bpf_prog_array *progs); 1742 bool bpf_prog_array_is_empty(struct bpf_prog_array *array); 1743 int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs, 1744 __u32 __user *prog_ids, u32 cnt); 1745 1746 void bpf_prog_array_delete_safe(struct bpf_prog_array *progs, 1747 struct bpf_prog *old_prog); 1748 int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index); 1749 int bpf_prog_array_update_at(struct bpf_prog_array *array, int index, 1750 struct bpf_prog *prog); 1751 int bpf_prog_array_copy_info(struct bpf_prog_array *array, 1752 u32 *prog_ids, u32 request_cnt, 1753 u32 *prog_cnt); 1754 int bpf_prog_array_copy(struct bpf_prog_array *old_array, 1755 struct bpf_prog *exclude_prog, 1756 struct bpf_prog *include_prog, 1757 u64 bpf_cookie, 1758 struct bpf_prog_array **new_array); 1759 1760 struct bpf_run_ctx {}; 1761 1762 struct bpf_cg_run_ctx { 1763 struct bpf_run_ctx run_ctx; 1764 const struct bpf_prog_array_item *prog_item; 1765 int retval; 1766 }; 1767 1768 struct bpf_trace_run_ctx { 1769 struct bpf_run_ctx run_ctx; 1770 u64 bpf_cookie; 1771 }; 1772 1773 struct bpf_tramp_run_ctx { 1774 struct bpf_run_ctx run_ctx; 1775 u64 bpf_cookie; 1776 struct bpf_run_ctx *saved_run_ctx; 1777 }; 1778 1779 static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx) 1780 { 1781 struct bpf_run_ctx *old_ctx = NULL; 1782 1783 #ifdef CONFIG_BPF_SYSCALL 1784 old_ctx = current->bpf_ctx; 1785 current->bpf_ctx = new_ctx; 1786 #endif 1787 return old_ctx; 1788 } 1789 1790 static inline void bpf_reset_run_ctx(struct bpf_run_ctx *old_ctx) 1791 { 1792 #ifdef CONFIG_BPF_SYSCALL 1793 current->bpf_ctx = old_ctx; 1794 #endif 1795 } 1796 1797 /* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */ 1798 #define BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE (1 << 0) 1799 /* BPF program asks to set CN on the packet. */ 1800 #define BPF_RET_SET_CN (1 << 0) 1801 1802 typedef u32 (*bpf_prog_run_fn)(const struct bpf_prog *prog, const void *ctx); 1803 1804 static __always_inline u32 1805 bpf_prog_run_array(const struct bpf_prog_array *array, 1806 const void *ctx, bpf_prog_run_fn run_prog) 1807 { 1808 const struct bpf_prog_array_item *item; 1809 const struct bpf_prog *prog; 1810 struct bpf_run_ctx *old_run_ctx; 1811 struct bpf_trace_run_ctx run_ctx; 1812 u32 ret = 1; 1813 1814 RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "no rcu lock held"); 1815 1816 if (unlikely(!array)) 1817 return ret; 1818 1819 migrate_disable(); 1820 old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); 1821 item = &array->items[0]; 1822 while ((prog = READ_ONCE(item->prog))) { 1823 run_ctx.bpf_cookie = item->bpf_cookie; 1824 ret &= run_prog(prog, ctx); 1825 item++; 1826 } 1827 bpf_reset_run_ctx(old_run_ctx); 1828 migrate_enable(); 1829 return ret; 1830 } 1831 1832 /* Notes on RCU design for bpf_prog_arrays containing sleepable programs: 1833 * 1834 * We use the tasks_trace rcu flavor read section to protect the bpf_prog_array 1835 * overall. As a result, we must use the bpf_prog_array_free_sleepable 1836 * in order to use the tasks_trace rcu grace period. 1837 * 1838 * When a non-sleepable program is inside the array, we take the rcu read 1839 * section and disable preemption for that program alone, so it can access 1840 * rcu-protected dynamically sized maps. 1841 */ 1842 static __always_inline u32 1843 bpf_prog_run_array_sleepable(const struct bpf_prog_array __rcu *array_rcu, 1844 const void *ctx, bpf_prog_run_fn run_prog) 1845 { 1846 const struct bpf_prog_array_item *item; 1847 const struct bpf_prog *prog; 1848 const struct bpf_prog_array *array; 1849 struct bpf_run_ctx *old_run_ctx; 1850 struct bpf_trace_run_ctx run_ctx; 1851 u32 ret = 1; 1852 1853 might_fault(); 1854 1855 rcu_read_lock_trace(); 1856 migrate_disable(); 1857 1858 array = rcu_dereference_check(array_rcu, rcu_read_lock_trace_held()); 1859 if (unlikely(!array)) 1860 goto out; 1861 old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); 1862 item = &array->items[0]; 1863 while ((prog = READ_ONCE(item->prog))) { 1864 if (!prog->aux->sleepable) 1865 rcu_read_lock(); 1866 1867 run_ctx.bpf_cookie = item->bpf_cookie; 1868 ret &= run_prog(prog, ctx); 1869 item++; 1870 1871 if (!prog->aux->sleepable) 1872 rcu_read_unlock(); 1873 } 1874 bpf_reset_run_ctx(old_run_ctx); 1875 out: 1876 migrate_enable(); 1877 rcu_read_unlock_trace(); 1878 return ret; 1879 } 1880 1881 #ifdef CONFIG_BPF_SYSCALL 1882 DECLARE_PER_CPU(int, bpf_prog_active); 1883 extern struct mutex bpf_stats_enabled_mutex; 1884 1885 /* 1886 * Block execution of BPF programs attached to instrumentation (perf, 1887 * kprobes, tracepoints) to prevent deadlocks on map operations as any of 1888 * these events can happen inside a region which holds a map bucket lock 1889 * and can deadlock on it. 1890 */ 1891 static inline void bpf_disable_instrumentation(void) 1892 { 1893 migrate_disable(); 1894 this_cpu_inc(bpf_prog_active); 1895 } 1896 1897 static inline void bpf_enable_instrumentation(void) 1898 { 1899 this_cpu_dec(bpf_prog_active); 1900 migrate_enable(); 1901 } 1902 1903 extern const struct file_operations bpf_map_fops; 1904 extern const struct file_operations bpf_prog_fops; 1905 extern const struct file_operations bpf_iter_fops; 1906 1907 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 1908 extern const struct bpf_prog_ops _name ## _prog_ops; \ 1909 extern const struct bpf_verifier_ops _name ## _verifier_ops; 1910 #define BPF_MAP_TYPE(_id, _ops) \ 1911 extern const struct bpf_map_ops _ops; 1912 #define BPF_LINK_TYPE(_id, _name) 1913 #include <linux/bpf_types.h> 1914 #undef BPF_PROG_TYPE 1915 #undef BPF_MAP_TYPE 1916 #undef BPF_LINK_TYPE 1917 1918 extern const struct bpf_prog_ops bpf_offload_prog_ops; 1919 extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops; 1920 extern const struct bpf_verifier_ops xdp_analyzer_ops; 1921 1922 struct bpf_prog *bpf_prog_get(u32 ufd); 1923 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 1924 bool attach_drv); 1925 void bpf_prog_add(struct bpf_prog *prog, int i); 1926 void bpf_prog_sub(struct bpf_prog *prog, int i); 1927 void bpf_prog_inc(struct bpf_prog *prog); 1928 struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog); 1929 void bpf_prog_put(struct bpf_prog *prog); 1930 1931 void bpf_prog_free_id(struct bpf_prog *prog); 1932 void bpf_map_free_id(struct bpf_map *map); 1933 1934 struct btf_field *btf_record_find(const struct btf_record *rec, 1935 u32 offset, u32 field_mask); 1936 void btf_record_free(struct btf_record *rec); 1937 void bpf_map_free_record(struct bpf_map *map); 1938 struct btf_record *btf_record_dup(const struct btf_record *rec); 1939 bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b); 1940 void bpf_obj_free_timer(const struct btf_record *rec, void *obj); 1941 void bpf_obj_free_fields(const struct btf_record *rec, void *obj); 1942 1943 struct bpf_map *bpf_map_get(u32 ufd); 1944 struct bpf_map *bpf_map_get_with_uref(u32 ufd); 1945 struct bpf_map *__bpf_map_get(struct fd f); 1946 void bpf_map_inc(struct bpf_map *map); 1947 void bpf_map_inc_with_uref(struct bpf_map *map); 1948 struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map); 1949 void bpf_map_put_with_uref(struct bpf_map *map); 1950 void bpf_map_put(struct bpf_map *map); 1951 void *bpf_map_area_alloc(u64 size, int numa_node); 1952 void *bpf_map_area_mmapable_alloc(u64 size, int numa_node); 1953 void bpf_map_area_free(void *base); 1954 bool bpf_map_write_active(const struct bpf_map *map); 1955 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr); 1956 int generic_map_lookup_batch(struct bpf_map *map, 1957 const union bpf_attr *attr, 1958 union bpf_attr __user *uattr); 1959 int generic_map_update_batch(struct bpf_map *map, struct file *map_file, 1960 const union bpf_attr *attr, 1961 union bpf_attr __user *uattr); 1962 int generic_map_delete_batch(struct bpf_map *map, 1963 const union bpf_attr *attr, 1964 union bpf_attr __user *uattr); 1965 struct bpf_map *bpf_map_get_curr_or_next(u32 *id); 1966 struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id); 1967 1968 #ifdef CONFIG_MEMCG_KMEM 1969 void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, 1970 int node); 1971 void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags); 1972 void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size, 1973 gfp_t flags); 1974 void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, 1975 size_t align, gfp_t flags); 1976 #else 1977 static inline void * 1978 bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, 1979 int node) 1980 { 1981 return kmalloc_node(size, flags, node); 1982 } 1983 1984 static inline void * 1985 bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags) 1986 { 1987 return kzalloc(size, flags); 1988 } 1989 1990 static inline void * 1991 bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size, gfp_t flags) 1992 { 1993 return kvcalloc(n, size, flags); 1994 } 1995 1996 static inline void __percpu * 1997 bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, size_t align, 1998 gfp_t flags) 1999 { 2000 return __alloc_percpu_gfp(size, align, flags); 2001 } 2002 #endif 2003 2004 extern int sysctl_unprivileged_bpf_disabled; 2005 2006 static inline bool bpf_allow_ptr_leaks(void) 2007 { 2008 return perfmon_capable(); 2009 } 2010 2011 static inline bool bpf_allow_uninit_stack(void) 2012 { 2013 return perfmon_capable(); 2014 } 2015 2016 static inline bool bpf_bypass_spec_v1(void) 2017 { 2018 return perfmon_capable(); 2019 } 2020 2021 static inline bool bpf_bypass_spec_v4(void) 2022 { 2023 return perfmon_capable(); 2024 } 2025 2026 int bpf_map_new_fd(struct bpf_map *map, int flags); 2027 int bpf_prog_new_fd(struct bpf_prog *prog); 2028 2029 void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, 2030 const struct bpf_link_ops *ops, struct bpf_prog *prog); 2031 int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer); 2032 int bpf_link_settle(struct bpf_link_primer *primer); 2033 void bpf_link_cleanup(struct bpf_link_primer *primer); 2034 void bpf_link_inc(struct bpf_link *link); 2035 void bpf_link_put(struct bpf_link *link); 2036 int bpf_link_new_fd(struct bpf_link *link); 2037 struct file *bpf_link_new_file(struct bpf_link *link, int *reserved_fd); 2038 struct bpf_link *bpf_link_get_from_fd(u32 ufd); 2039 struct bpf_link *bpf_link_get_curr_or_next(u32 *id); 2040 2041 int bpf_obj_pin_user(u32 ufd, const char __user *pathname); 2042 int bpf_obj_get_user(const char __user *pathname, int flags); 2043 2044 #define BPF_ITER_FUNC_PREFIX "bpf_iter_" 2045 #define DEFINE_BPF_ITER_FUNC(target, args...) \ 2046 extern int bpf_iter_ ## target(args); \ 2047 int __init bpf_iter_ ## target(args) { return 0; } 2048 2049 /* 2050 * The task type of iterators. 2051 * 2052 * For BPF task iterators, they can be parameterized with various 2053 * parameters to visit only some of tasks. 2054 * 2055 * BPF_TASK_ITER_ALL (default) 2056 * Iterate over resources of every task. 2057 * 2058 * BPF_TASK_ITER_TID 2059 * Iterate over resources of a task/tid. 2060 * 2061 * BPF_TASK_ITER_TGID 2062 * Iterate over resources of every task of a process / task group. 2063 */ 2064 enum bpf_iter_task_type { 2065 BPF_TASK_ITER_ALL = 0, 2066 BPF_TASK_ITER_TID, 2067 BPF_TASK_ITER_TGID, 2068 }; 2069 2070 struct bpf_iter_aux_info { 2071 /* for map_elem iter */ 2072 struct bpf_map *map; 2073 2074 /* for cgroup iter */ 2075 struct { 2076 struct cgroup *start; /* starting cgroup */ 2077 enum bpf_cgroup_iter_order order; 2078 } cgroup; 2079 struct { 2080 enum bpf_iter_task_type type; 2081 u32 pid; 2082 } task; 2083 }; 2084 2085 typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog, 2086 union bpf_iter_link_info *linfo, 2087 struct bpf_iter_aux_info *aux); 2088 typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux); 2089 typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux, 2090 struct seq_file *seq); 2091 typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux, 2092 struct bpf_link_info *info); 2093 typedef const struct bpf_func_proto * 2094 (*bpf_iter_get_func_proto_t)(enum bpf_func_id func_id, 2095 const struct bpf_prog *prog); 2096 2097 enum bpf_iter_feature { 2098 BPF_ITER_RESCHED = BIT(0), 2099 }; 2100 2101 #define BPF_ITER_CTX_ARG_MAX 2 2102 struct bpf_iter_reg { 2103 const char *target; 2104 bpf_iter_attach_target_t attach_target; 2105 bpf_iter_detach_target_t detach_target; 2106 bpf_iter_show_fdinfo_t show_fdinfo; 2107 bpf_iter_fill_link_info_t fill_link_info; 2108 bpf_iter_get_func_proto_t get_func_proto; 2109 u32 ctx_arg_info_size; 2110 u32 feature; 2111 struct bpf_ctx_arg_aux ctx_arg_info[BPF_ITER_CTX_ARG_MAX]; 2112 const struct bpf_iter_seq_info *seq_info; 2113 }; 2114 2115 struct bpf_iter_meta { 2116 __bpf_md_ptr(struct seq_file *, seq); 2117 u64 session_id; 2118 u64 seq_num; 2119 }; 2120 2121 struct bpf_iter__bpf_map_elem { 2122 __bpf_md_ptr(struct bpf_iter_meta *, meta); 2123 __bpf_md_ptr(struct bpf_map *, map); 2124 __bpf_md_ptr(void *, key); 2125 __bpf_md_ptr(void *, value); 2126 }; 2127 2128 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info); 2129 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info); 2130 bool bpf_iter_prog_supported(struct bpf_prog *prog); 2131 const struct bpf_func_proto * 2132 bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); 2133 int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_prog *prog); 2134 int bpf_iter_new_fd(struct bpf_link *link); 2135 bool bpf_link_is_iter(struct bpf_link *link); 2136 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop); 2137 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx); 2138 void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux, 2139 struct seq_file *seq); 2140 int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux, 2141 struct bpf_link_info *info); 2142 2143 int map_set_for_each_callback_args(struct bpf_verifier_env *env, 2144 struct bpf_func_state *caller, 2145 struct bpf_func_state *callee); 2146 2147 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value); 2148 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value); 2149 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, 2150 u64 flags); 2151 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, 2152 u64 flags); 2153 2154 int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value); 2155 2156 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, 2157 void *key, void *value, u64 map_flags); 2158 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); 2159 int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, 2160 void *key, void *value, u64 map_flags); 2161 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); 2162 2163 int bpf_get_file_flag(int flags); 2164 int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size, 2165 size_t actual_size); 2166 2167 /* verify correctness of eBPF program */ 2168 int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr); 2169 2170 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 2171 void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); 2172 #endif 2173 2174 struct btf *bpf_get_btf_vmlinux(void); 2175 2176 /* Map specifics */ 2177 struct xdp_frame; 2178 struct sk_buff; 2179 struct bpf_dtab_netdev; 2180 struct bpf_cpu_map_entry; 2181 2182 void __dev_flush(void); 2183 int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf, 2184 struct net_device *dev_rx); 2185 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf, 2186 struct net_device *dev_rx); 2187 int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx, 2188 struct bpf_map *map, bool exclude_ingress); 2189 int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, 2190 struct bpf_prog *xdp_prog); 2191 int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, 2192 struct bpf_prog *xdp_prog, struct bpf_map *map, 2193 bool exclude_ingress); 2194 2195 void __cpu_map_flush(void); 2196 int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf, 2197 struct net_device *dev_rx); 2198 int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu, 2199 struct sk_buff *skb); 2200 2201 /* Return map's numa specified by userspace */ 2202 static inline int bpf_map_attr_numa_node(const union bpf_attr *attr) 2203 { 2204 return (attr->map_flags & BPF_F_NUMA_NODE) ? 2205 attr->numa_node : NUMA_NO_NODE; 2206 } 2207 2208 struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type); 2209 int array_map_alloc_check(union bpf_attr *attr); 2210 2211 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 2212 union bpf_attr __user *uattr); 2213 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 2214 union bpf_attr __user *uattr); 2215 int bpf_prog_test_run_tracing(struct bpf_prog *prog, 2216 const union bpf_attr *kattr, 2217 union bpf_attr __user *uattr); 2218 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 2219 const union bpf_attr *kattr, 2220 union bpf_attr __user *uattr); 2221 int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, 2222 const union bpf_attr *kattr, 2223 union bpf_attr __user *uattr); 2224 int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, 2225 const union bpf_attr *kattr, 2226 union bpf_attr __user *uattr); 2227 bool btf_ctx_access(int off, int size, enum bpf_access_type type, 2228 const struct bpf_prog *prog, 2229 struct bpf_insn_access_aux *info); 2230 2231 static inline bool bpf_tracing_ctx_access(int off, int size, 2232 enum bpf_access_type type) 2233 { 2234 if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) 2235 return false; 2236 if (type != BPF_READ) 2237 return false; 2238 if (off % size != 0) 2239 return false; 2240 return true; 2241 } 2242 2243 static inline bool bpf_tracing_btf_ctx_access(int off, int size, 2244 enum bpf_access_type type, 2245 const struct bpf_prog *prog, 2246 struct bpf_insn_access_aux *info) 2247 { 2248 if (!bpf_tracing_ctx_access(off, size, type)) 2249 return false; 2250 return btf_ctx_access(off, size, type, prog, info); 2251 } 2252 2253 int btf_struct_access(struct bpf_verifier_log *log, 2254 const struct bpf_reg_state *reg, 2255 int off, int size, enum bpf_access_type atype, 2256 u32 *next_btf_id, enum bpf_type_flag *flag); 2257 bool btf_struct_ids_match(struct bpf_verifier_log *log, 2258 const struct btf *btf, u32 id, int off, 2259 const struct btf *need_btf, u32 need_type_id, 2260 bool strict); 2261 2262 int btf_distill_func_proto(struct bpf_verifier_log *log, 2263 struct btf *btf, 2264 const struct btf_type *func_proto, 2265 const char *func_name, 2266 struct btf_func_model *m); 2267 2268 struct bpf_reg_state; 2269 int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, 2270 struct bpf_reg_state *regs); 2271 int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, 2272 struct bpf_reg_state *regs); 2273 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, 2274 struct bpf_reg_state *reg); 2275 int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog, 2276 struct btf *btf, const struct btf_type *t); 2277 2278 struct bpf_prog *bpf_prog_by_id(u32 id); 2279 struct bpf_link *bpf_link_by_id(u32 id); 2280 2281 const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id); 2282 void bpf_task_storage_free(struct task_struct *task); 2283 void bpf_cgrp_storage_free(struct cgroup *cgroup); 2284 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog); 2285 const struct btf_func_model * 2286 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 2287 const struct bpf_insn *insn); 2288 struct bpf_core_ctx { 2289 struct bpf_verifier_log *log; 2290 const struct btf *btf; 2291 }; 2292 2293 bool btf_nested_type_is_trusted(struct bpf_verifier_log *log, 2294 const struct bpf_reg_state *reg, 2295 int off, const char *suffix); 2296 2297 bool btf_type_ids_nocast_alias(struct bpf_verifier_log *log, 2298 const struct btf *reg_btf, u32 reg_id, 2299 const struct btf *arg_btf, u32 arg_id); 2300 2301 int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo, 2302 int relo_idx, void *insn); 2303 2304 static inline bool unprivileged_ebpf_enabled(void) 2305 { 2306 return !sysctl_unprivileged_bpf_disabled; 2307 } 2308 2309 /* Not all bpf prog type has the bpf_ctx. 2310 * For the bpf prog type that has initialized the bpf_ctx, 2311 * this function can be used to decide if a kernel function 2312 * is called by a bpf program. 2313 */ 2314 static inline bool has_current_bpf_ctx(void) 2315 { 2316 return !!current->bpf_ctx; 2317 } 2318 2319 void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog); 2320 2321 void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data, 2322 enum bpf_dynptr_type type, u32 offset, u32 size); 2323 void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr); 2324 void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr); 2325 #else /* !CONFIG_BPF_SYSCALL */ 2326 static inline struct bpf_prog *bpf_prog_get(u32 ufd) 2327 { 2328 return ERR_PTR(-EOPNOTSUPP); 2329 } 2330 2331 static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, 2332 enum bpf_prog_type type, 2333 bool attach_drv) 2334 { 2335 return ERR_PTR(-EOPNOTSUPP); 2336 } 2337 2338 static inline void bpf_prog_add(struct bpf_prog *prog, int i) 2339 { 2340 } 2341 2342 static inline void bpf_prog_sub(struct bpf_prog *prog, int i) 2343 { 2344 } 2345 2346 static inline void bpf_prog_put(struct bpf_prog *prog) 2347 { 2348 } 2349 2350 static inline void bpf_prog_inc(struct bpf_prog *prog) 2351 { 2352 } 2353 2354 static inline struct bpf_prog *__must_check 2355 bpf_prog_inc_not_zero(struct bpf_prog *prog) 2356 { 2357 return ERR_PTR(-EOPNOTSUPP); 2358 } 2359 2360 static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, 2361 const struct bpf_link_ops *ops, 2362 struct bpf_prog *prog) 2363 { 2364 } 2365 2366 static inline int bpf_link_prime(struct bpf_link *link, 2367 struct bpf_link_primer *primer) 2368 { 2369 return -EOPNOTSUPP; 2370 } 2371 2372 static inline int bpf_link_settle(struct bpf_link_primer *primer) 2373 { 2374 return -EOPNOTSUPP; 2375 } 2376 2377 static inline void bpf_link_cleanup(struct bpf_link_primer *primer) 2378 { 2379 } 2380 2381 static inline void bpf_link_inc(struct bpf_link *link) 2382 { 2383 } 2384 2385 static inline void bpf_link_put(struct bpf_link *link) 2386 { 2387 } 2388 2389 static inline int bpf_obj_get_user(const char __user *pathname, int flags) 2390 { 2391 return -EOPNOTSUPP; 2392 } 2393 2394 static inline void __dev_flush(void) 2395 { 2396 } 2397 2398 struct xdp_frame; 2399 struct bpf_dtab_netdev; 2400 struct bpf_cpu_map_entry; 2401 2402 static inline 2403 int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf, 2404 struct net_device *dev_rx) 2405 { 2406 return 0; 2407 } 2408 2409 static inline 2410 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf, 2411 struct net_device *dev_rx) 2412 { 2413 return 0; 2414 } 2415 2416 static inline 2417 int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx, 2418 struct bpf_map *map, bool exclude_ingress) 2419 { 2420 return 0; 2421 } 2422 2423 struct sk_buff; 2424 2425 static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, 2426 struct sk_buff *skb, 2427 struct bpf_prog *xdp_prog) 2428 { 2429 return 0; 2430 } 2431 2432 static inline 2433 int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, 2434 struct bpf_prog *xdp_prog, struct bpf_map *map, 2435 bool exclude_ingress) 2436 { 2437 return 0; 2438 } 2439 2440 static inline void __cpu_map_flush(void) 2441 { 2442 } 2443 2444 static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, 2445 struct xdp_frame *xdpf, 2446 struct net_device *dev_rx) 2447 { 2448 return 0; 2449 } 2450 2451 static inline int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu, 2452 struct sk_buff *skb) 2453 { 2454 return -EOPNOTSUPP; 2455 } 2456 2457 static inline struct bpf_prog *bpf_prog_get_type_path(const char *name, 2458 enum bpf_prog_type type) 2459 { 2460 return ERR_PTR(-EOPNOTSUPP); 2461 } 2462 2463 static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog, 2464 const union bpf_attr *kattr, 2465 union bpf_attr __user *uattr) 2466 { 2467 return -ENOTSUPP; 2468 } 2469 2470 static inline int bpf_prog_test_run_skb(struct bpf_prog *prog, 2471 const union bpf_attr *kattr, 2472 union bpf_attr __user *uattr) 2473 { 2474 return -ENOTSUPP; 2475 } 2476 2477 static inline int bpf_prog_test_run_tracing(struct bpf_prog *prog, 2478 const union bpf_attr *kattr, 2479 union bpf_attr __user *uattr) 2480 { 2481 return -ENOTSUPP; 2482 } 2483 2484 static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 2485 const union bpf_attr *kattr, 2486 union bpf_attr __user *uattr) 2487 { 2488 return -ENOTSUPP; 2489 } 2490 2491 static inline int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, 2492 const union bpf_attr *kattr, 2493 union bpf_attr __user *uattr) 2494 { 2495 return -ENOTSUPP; 2496 } 2497 2498 static inline void bpf_map_put(struct bpf_map *map) 2499 { 2500 } 2501 2502 static inline struct bpf_prog *bpf_prog_by_id(u32 id) 2503 { 2504 return ERR_PTR(-ENOTSUPP); 2505 } 2506 2507 static inline int btf_struct_access(struct bpf_verifier_log *log, 2508 const struct bpf_reg_state *reg, 2509 int off, int size, enum bpf_access_type atype, 2510 u32 *next_btf_id, enum bpf_type_flag *flag) 2511 { 2512 return -EACCES; 2513 } 2514 2515 static inline const struct bpf_func_proto * 2516 bpf_base_func_proto(enum bpf_func_id func_id) 2517 { 2518 return NULL; 2519 } 2520 2521 static inline void bpf_task_storage_free(struct task_struct *task) 2522 { 2523 } 2524 2525 static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) 2526 { 2527 return false; 2528 } 2529 2530 static inline const struct btf_func_model * 2531 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 2532 const struct bpf_insn *insn) 2533 { 2534 return NULL; 2535 } 2536 2537 static inline bool unprivileged_ebpf_enabled(void) 2538 { 2539 return false; 2540 } 2541 2542 static inline bool has_current_bpf_ctx(void) 2543 { 2544 return false; 2545 } 2546 2547 static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog) 2548 { 2549 } 2550 2551 static inline void bpf_cgrp_storage_free(struct cgroup *cgroup) 2552 { 2553 } 2554 2555 static inline void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data, 2556 enum bpf_dynptr_type type, u32 offset, u32 size) 2557 { 2558 } 2559 2560 static inline void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr) 2561 { 2562 } 2563 2564 static inline void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr) 2565 { 2566 } 2567 #endif /* CONFIG_BPF_SYSCALL */ 2568 2569 void __bpf_free_used_btfs(struct bpf_prog_aux *aux, 2570 struct btf_mod_pair *used_btfs, u32 len); 2571 2572 static inline struct bpf_prog *bpf_prog_get_type(u32 ufd, 2573 enum bpf_prog_type type) 2574 { 2575 return bpf_prog_get_type_dev(ufd, type, false); 2576 } 2577 2578 void __bpf_free_used_maps(struct bpf_prog_aux *aux, 2579 struct bpf_map **used_maps, u32 len); 2580 2581 bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool); 2582 2583 int bpf_prog_offload_compile(struct bpf_prog *prog); 2584 void bpf_prog_dev_bound_destroy(struct bpf_prog *prog); 2585 int bpf_prog_offload_info_fill(struct bpf_prog_info *info, 2586 struct bpf_prog *prog); 2587 2588 int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map); 2589 2590 int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value); 2591 int bpf_map_offload_update_elem(struct bpf_map *map, 2592 void *key, void *value, u64 flags); 2593 int bpf_map_offload_delete_elem(struct bpf_map *map, void *key); 2594 int bpf_map_offload_get_next_key(struct bpf_map *map, 2595 void *key, void *next_key); 2596 2597 bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map); 2598 2599 struct bpf_offload_dev * 2600 bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv); 2601 void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev); 2602 void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev); 2603 int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev, 2604 struct net_device *netdev); 2605 void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev, 2606 struct net_device *netdev); 2607 bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev); 2608 2609 void unpriv_ebpf_notify(int new_state); 2610 2611 #if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL) 2612 int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log, 2613 struct bpf_prog_aux *prog_aux); 2614 void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id); 2615 int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr); 2616 int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog); 2617 void bpf_dev_bound_netdev_unregister(struct net_device *dev); 2618 2619 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) 2620 { 2621 return aux->dev_bound; 2622 } 2623 2624 static inline bool bpf_prog_is_offloaded(const struct bpf_prog_aux *aux) 2625 { 2626 return aux->offload_requested; 2627 } 2628 2629 bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs); 2630 2631 static inline bool bpf_map_is_offloaded(struct bpf_map *map) 2632 { 2633 return unlikely(map->ops == &bpf_map_offload_ops); 2634 } 2635 2636 struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr); 2637 void bpf_map_offload_map_free(struct bpf_map *map); 2638 u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map); 2639 int bpf_prog_test_run_syscall(struct bpf_prog *prog, 2640 const union bpf_attr *kattr, 2641 union bpf_attr __user *uattr); 2642 2643 int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog); 2644 int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype); 2645 int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, u64 flags); 2646 int sock_map_bpf_prog_query(const union bpf_attr *attr, 2647 union bpf_attr __user *uattr); 2648 2649 void sock_map_unhash(struct sock *sk); 2650 void sock_map_destroy(struct sock *sk); 2651 void sock_map_close(struct sock *sk, long timeout); 2652 #else 2653 static inline int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log, 2654 struct bpf_prog_aux *prog_aux) 2655 { 2656 return -EOPNOTSUPP; 2657 } 2658 2659 static inline void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, 2660 u32 func_id) 2661 { 2662 return NULL; 2663 } 2664 2665 static inline int bpf_prog_dev_bound_init(struct bpf_prog *prog, 2666 union bpf_attr *attr) 2667 { 2668 return -EOPNOTSUPP; 2669 } 2670 2671 static inline int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, 2672 struct bpf_prog *old_prog) 2673 { 2674 return -EOPNOTSUPP; 2675 } 2676 2677 static inline void bpf_dev_bound_netdev_unregister(struct net_device *dev) 2678 { 2679 } 2680 2681 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) 2682 { 2683 return false; 2684 } 2685 2686 static inline bool bpf_prog_is_offloaded(struct bpf_prog_aux *aux) 2687 { 2688 return false; 2689 } 2690 2691 static inline bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs) 2692 { 2693 return false; 2694 } 2695 2696 static inline bool bpf_map_is_offloaded(struct bpf_map *map) 2697 { 2698 return false; 2699 } 2700 2701 static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) 2702 { 2703 return ERR_PTR(-EOPNOTSUPP); 2704 } 2705 2706 static inline void bpf_map_offload_map_free(struct bpf_map *map) 2707 { 2708 } 2709 2710 static inline u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map) 2711 { 2712 return 0; 2713 } 2714 2715 static inline int bpf_prog_test_run_syscall(struct bpf_prog *prog, 2716 const union bpf_attr *kattr, 2717 union bpf_attr __user *uattr) 2718 { 2719 return -ENOTSUPP; 2720 } 2721 2722 #ifdef CONFIG_BPF_SYSCALL 2723 static inline int sock_map_get_from_fd(const union bpf_attr *attr, 2724 struct bpf_prog *prog) 2725 { 2726 return -EINVAL; 2727 } 2728 2729 static inline int sock_map_prog_detach(const union bpf_attr *attr, 2730 enum bpf_prog_type ptype) 2731 { 2732 return -EOPNOTSUPP; 2733 } 2734 2735 static inline int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, 2736 u64 flags) 2737 { 2738 return -EOPNOTSUPP; 2739 } 2740 2741 static inline int sock_map_bpf_prog_query(const union bpf_attr *attr, 2742 union bpf_attr __user *uattr) 2743 { 2744 return -EINVAL; 2745 } 2746 #endif /* CONFIG_BPF_SYSCALL */ 2747 #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */ 2748 2749 #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) 2750 void bpf_sk_reuseport_detach(struct sock *sk); 2751 int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, 2752 void *value); 2753 int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key, 2754 void *value, u64 map_flags); 2755 #else 2756 static inline void bpf_sk_reuseport_detach(struct sock *sk) 2757 { 2758 } 2759 2760 #ifdef CONFIG_BPF_SYSCALL 2761 static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, 2762 void *key, void *value) 2763 { 2764 return -EOPNOTSUPP; 2765 } 2766 2767 static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, 2768 void *key, void *value, 2769 u64 map_flags) 2770 { 2771 return -EOPNOTSUPP; 2772 } 2773 #endif /* CONFIG_BPF_SYSCALL */ 2774 #endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */ 2775 2776 /* verifier prototypes for helper functions called from eBPF programs */ 2777 extern const struct bpf_func_proto bpf_map_lookup_elem_proto; 2778 extern const struct bpf_func_proto bpf_map_update_elem_proto; 2779 extern const struct bpf_func_proto bpf_map_delete_elem_proto; 2780 extern const struct bpf_func_proto bpf_map_push_elem_proto; 2781 extern const struct bpf_func_proto bpf_map_pop_elem_proto; 2782 extern const struct bpf_func_proto bpf_map_peek_elem_proto; 2783 extern const struct bpf_func_proto bpf_map_lookup_percpu_elem_proto; 2784 2785 extern const struct bpf_func_proto bpf_get_prandom_u32_proto; 2786 extern const struct bpf_func_proto bpf_get_smp_processor_id_proto; 2787 extern const struct bpf_func_proto bpf_get_numa_node_id_proto; 2788 extern const struct bpf_func_proto bpf_tail_call_proto; 2789 extern const struct bpf_func_proto bpf_ktime_get_ns_proto; 2790 extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto; 2791 extern const struct bpf_func_proto bpf_ktime_get_tai_ns_proto; 2792 extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto; 2793 extern const struct bpf_func_proto bpf_get_current_uid_gid_proto; 2794 extern const struct bpf_func_proto bpf_get_current_comm_proto; 2795 extern const struct bpf_func_proto bpf_get_stackid_proto; 2796 extern const struct bpf_func_proto bpf_get_stack_proto; 2797 extern const struct bpf_func_proto bpf_get_task_stack_proto; 2798 extern const struct bpf_func_proto bpf_get_stackid_proto_pe; 2799 extern const struct bpf_func_proto bpf_get_stack_proto_pe; 2800 extern const struct bpf_func_proto bpf_sock_map_update_proto; 2801 extern const struct bpf_func_proto bpf_sock_hash_update_proto; 2802 extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto; 2803 extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto; 2804 extern const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto; 2805 extern const struct bpf_func_proto bpf_msg_redirect_hash_proto; 2806 extern const struct bpf_func_proto bpf_msg_redirect_map_proto; 2807 extern const struct bpf_func_proto bpf_sk_redirect_hash_proto; 2808 extern const struct bpf_func_proto bpf_sk_redirect_map_proto; 2809 extern const struct bpf_func_proto bpf_spin_lock_proto; 2810 extern const struct bpf_func_proto bpf_spin_unlock_proto; 2811 extern const struct bpf_func_proto bpf_get_local_storage_proto; 2812 extern const struct bpf_func_proto bpf_strtol_proto; 2813 extern const struct bpf_func_proto bpf_strtoul_proto; 2814 extern const struct bpf_func_proto bpf_tcp_sock_proto; 2815 extern const struct bpf_func_proto bpf_jiffies64_proto; 2816 extern const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto; 2817 extern const struct bpf_func_proto bpf_event_output_data_proto; 2818 extern const struct bpf_func_proto bpf_ringbuf_output_proto; 2819 extern const struct bpf_func_proto bpf_ringbuf_reserve_proto; 2820 extern const struct bpf_func_proto bpf_ringbuf_submit_proto; 2821 extern const struct bpf_func_proto bpf_ringbuf_discard_proto; 2822 extern const struct bpf_func_proto bpf_ringbuf_query_proto; 2823 extern const struct bpf_func_proto bpf_ringbuf_reserve_dynptr_proto; 2824 extern const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto; 2825 extern const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto; 2826 extern const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto; 2827 extern const struct bpf_func_proto bpf_skc_to_tcp_sock_proto; 2828 extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto; 2829 extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto; 2830 extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto; 2831 extern const struct bpf_func_proto bpf_skc_to_unix_sock_proto; 2832 extern const struct bpf_func_proto bpf_skc_to_mptcp_sock_proto; 2833 extern const struct bpf_func_proto bpf_copy_from_user_proto; 2834 extern const struct bpf_func_proto bpf_snprintf_btf_proto; 2835 extern const struct bpf_func_proto bpf_snprintf_proto; 2836 extern const struct bpf_func_proto bpf_per_cpu_ptr_proto; 2837 extern const struct bpf_func_proto bpf_this_cpu_ptr_proto; 2838 extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto; 2839 extern const struct bpf_func_proto bpf_sock_from_file_proto; 2840 extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto; 2841 extern const struct bpf_func_proto bpf_task_storage_get_recur_proto; 2842 extern const struct bpf_func_proto bpf_task_storage_get_proto; 2843 extern const struct bpf_func_proto bpf_task_storage_delete_recur_proto; 2844 extern const struct bpf_func_proto bpf_task_storage_delete_proto; 2845 extern const struct bpf_func_proto bpf_for_each_map_elem_proto; 2846 extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto; 2847 extern const struct bpf_func_proto bpf_sk_setsockopt_proto; 2848 extern const struct bpf_func_proto bpf_sk_getsockopt_proto; 2849 extern const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto; 2850 extern const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto; 2851 extern const struct bpf_func_proto bpf_find_vma_proto; 2852 extern const struct bpf_func_proto bpf_loop_proto; 2853 extern const struct bpf_func_proto bpf_copy_from_user_task_proto; 2854 extern const struct bpf_func_proto bpf_set_retval_proto; 2855 extern const struct bpf_func_proto bpf_get_retval_proto; 2856 extern const struct bpf_func_proto bpf_user_ringbuf_drain_proto; 2857 extern const struct bpf_func_proto bpf_cgrp_storage_get_proto; 2858 extern const struct bpf_func_proto bpf_cgrp_storage_delete_proto; 2859 2860 const struct bpf_func_proto *tracing_prog_func_proto( 2861 enum bpf_func_id func_id, const struct bpf_prog *prog); 2862 2863 /* Shared helpers among cBPF and eBPF. */ 2864 void bpf_user_rnd_init_once(void); 2865 u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 2866 u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 2867 2868 #if defined(CONFIG_NET) 2869 bool bpf_sock_common_is_valid_access(int off, int size, 2870 enum bpf_access_type type, 2871 struct bpf_insn_access_aux *info); 2872 bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type, 2873 struct bpf_insn_access_aux *info); 2874 u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, 2875 const struct bpf_insn *si, 2876 struct bpf_insn *insn_buf, 2877 struct bpf_prog *prog, 2878 u32 *target_size); 2879 int bpf_dynptr_from_skb_rdonly(struct sk_buff *skb, u64 flags, 2880 struct bpf_dynptr_kern *ptr); 2881 #else 2882 static inline bool bpf_sock_common_is_valid_access(int off, int size, 2883 enum bpf_access_type type, 2884 struct bpf_insn_access_aux *info) 2885 { 2886 return false; 2887 } 2888 static inline bool bpf_sock_is_valid_access(int off, int size, 2889 enum bpf_access_type type, 2890 struct bpf_insn_access_aux *info) 2891 { 2892 return false; 2893 } 2894 static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, 2895 const struct bpf_insn *si, 2896 struct bpf_insn *insn_buf, 2897 struct bpf_prog *prog, 2898 u32 *target_size) 2899 { 2900 return 0; 2901 } 2902 static inline int bpf_dynptr_from_skb_rdonly(struct sk_buff *skb, u64 flags, 2903 struct bpf_dynptr_kern *ptr) 2904 { 2905 return -EOPNOTSUPP; 2906 } 2907 #endif 2908 2909 #ifdef CONFIG_INET 2910 struct sk_reuseport_kern { 2911 struct sk_buff *skb; 2912 struct sock *sk; 2913 struct sock *selected_sk; 2914 struct sock *migrating_sk; 2915 void *data_end; 2916 u32 hash; 2917 u32 reuseport_id; 2918 bool bind_inany; 2919 }; 2920 bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type, 2921 struct bpf_insn_access_aux *info); 2922 2923 u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, 2924 const struct bpf_insn *si, 2925 struct bpf_insn *insn_buf, 2926 struct bpf_prog *prog, 2927 u32 *target_size); 2928 2929 bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type, 2930 struct bpf_insn_access_aux *info); 2931 2932 u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, 2933 const struct bpf_insn *si, 2934 struct bpf_insn *insn_buf, 2935 struct bpf_prog *prog, 2936 u32 *target_size); 2937 #else 2938 static inline bool bpf_tcp_sock_is_valid_access(int off, int size, 2939 enum bpf_access_type type, 2940 struct bpf_insn_access_aux *info) 2941 { 2942 return false; 2943 } 2944 2945 static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, 2946 const struct bpf_insn *si, 2947 struct bpf_insn *insn_buf, 2948 struct bpf_prog *prog, 2949 u32 *target_size) 2950 { 2951 return 0; 2952 } 2953 static inline bool bpf_xdp_sock_is_valid_access(int off, int size, 2954 enum bpf_access_type type, 2955 struct bpf_insn_access_aux *info) 2956 { 2957 return false; 2958 } 2959 2960 static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, 2961 const struct bpf_insn *si, 2962 struct bpf_insn *insn_buf, 2963 struct bpf_prog *prog, 2964 u32 *target_size) 2965 { 2966 return 0; 2967 } 2968 #endif /* CONFIG_INET */ 2969 2970 enum bpf_text_poke_type { 2971 BPF_MOD_CALL, 2972 BPF_MOD_JUMP, 2973 }; 2974 2975 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, 2976 void *addr1, void *addr2); 2977 2978 void *bpf_arch_text_copy(void *dst, void *src, size_t len); 2979 int bpf_arch_text_invalidate(void *dst, size_t len); 2980 2981 struct btf_id_set; 2982 bool btf_id_set_contains(const struct btf_id_set *set, u32 id); 2983 2984 #define MAX_BPRINTF_VARARGS 12 2985 #define MAX_BPRINTF_BUF 1024 2986 2987 struct bpf_bprintf_data { 2988 u32 *bin_args; 2989 char *buf; 2990 bool get_bin_args; 2991 bool get_buf; 2992 }; 2993 2994 int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args, 2995 u32 num_args, struct bpf_bprintf_data *data); 2996 void bpf_bprintf_cleanup(struct bpf_bprintf_data *data); 2997 2998 #ifdef CONFIG_BPF_LSM 2999 void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype); 3000 void bpf_cgroup_atype_put(int cgroup_atype); 3001 #else 3002 static inline void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) {} 3003 static inline void bpf_cgroup_atype_put(int cgroup_atype) {} 3004 #endif /* CONFIG_BPF_LSM */ 3005 3006 struct key; 3007 3008 #ifdef CONFIG_KEYS 3009 struct bpf_key { 3010 struct key *key; 3011 bool has_ref; 3012 }; 3013 #endif /* CONFIG_KEYS */ 3014 3015 static inline bool type_is_alloc(u32 type) 3016 { 3017 return type & MEM_ALLOC; 3018 } 3019 3020 static inline gfp_t bpf_memcg_flags(gfp_t flags) 3021 { 3022 if (memcg_bpf_enabled()) 3023 return flags | __GFP_ACCOUNT; 3024 return flags; 3025 } 3026 3027 #endif /* _LINUX_BPF_H */ 3028