1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 3 */ 4 #ifndef _LINUX_BPF_H 5 #define _LINUX_BPF_H 1 6 7 #include <uapi/linux/bpf.h> 8 #include <uapi/linux/filter.h> 9 10 #include <linux/workqueue.h> 11 #include <linux/file.h> 12 #include <linux/percpu.h> 13 #include <linux/err.h> 14 #include <linux/rbtree_latch.h> 15 #include <linux/numa.h> 16 #include <linux/mm_types.h> 17 #include <linux/wait.h> 18 #include <linux/refcount.h> 19 #include <linux/mutex.h> 20 #include <linux/module.h> 21 #include <linux/kallsyms.h> 22 #include <linux/capability.h> 23 #include <linux/sched/mm.h> 24 #include <linux/slab.h> 25 #include <linux/percpu-refcount.h> 26 #include <linux/stddef.h> 27 #include <linux/bpfptr.h> 28 #include <linux/btf.h> 29 #include <linux/rcupdate_trace.h> 30 #include <linux/static_call.h> 31 #include <linux/memcontrol.h> 32 33 struct bpf_verifier_env; 34 struct bpf_verifier_log; 35 struct perf_event; 36 struct bpf_prog; 37 struct bpf_prog_aux; 38 struct bpf_map; 39 struct sock; 40 struct seq_file; 41 struct btf; 42 struct btf_type; 43 struct exception_table_entry; 44 struct seq_operations; 45 struct bpf_iter_aux_info; 46 struct bpf_local_storage; 47 struct bpf_local_storage_map; 48 struct kobject; 49 struct mem_cgroup; 50 struct module; 51 struct bpf_func_state; 52 struct ftrace_ops; 53 struct cgroup; 54 55 extern struct idr btf_idr; 56 extern spinlock_t btf_idr_lock; 57 extern struct kobject *btf_kobj; 58 extern struct bpf_mem_alloc bpf_global_ma, bpf_global_percpu_ma; 59 extern bool bpf_global_ma_set; 60 61 typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64); 62 typedef int (*bpf_iter_init_seq_priv_t)(void *private_data, 63 struct bpf_iter_aux_info *aux); 64 typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data); 65 typedef unsigned int (*bpf_func_t)(const void *, 66 const struct bpf_insn *); 67 struct bpf_iter_seq_info { 68 const struct seq_operations *seq_ops; 69 bpf_iter_init_seq_priv_t init_seq_private; 70 bpf_iter_fini_seq_priv_t fini_seq_private; 71 u32 seq_priv_size; 72 }; 73 74 /* map is generic key/value storage optionally accessible by eBPF programs */ 75 struct bpf_map_ops { 76 /* funcs callable from userspace (via syscall) */ 77 int (*map_alloc_check)(union bpf_attr *attr); 78 struct bpf_map *(*map_alloc)(union bpf_attr *attr); 79 void (*map_release)(struct bpf_map *map, struct file *map_file); 80 void (*map_free)(struct bpf_map *map); 81 int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key); 82 void (*map_release_uref)(struct bpf_map *map); 83 void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key); 84 int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr, 85 union bpf_attr __user *uattr); 86 int (*map_lookup_and_delete_elem)(struct bpf_map *map, void *key, 87 void *value, u64 flags); 88 int (*map_lookup_and_delete_batch)(struct bpf_map *map, 89 const union bpf_attr *attr, 90 union bpf_attr __user *uattr); 91 int (*map_update_batch)(struct bpf_map *map, struct file *map_file, 92 const union bpf_attr *attr, 93 union bpf_attr __user *uattr); 94 int (*map_delete_batch)(struct bpf_map *map, const union bpf_attr *attr, 95 union bpf_attr __user *uattr); 96 97 /* funcs callable from userspace and from eBPF programs */ 98 void *(*map_lookup_elem)(struct bpf_map *map, void *key); 99 long (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags); 100 long (*map_delete_elem)(struct bpf_map *map, void *key); 101 long (*map_push_elem)(struct bpf_map *map, void *value, u64 flags); 102 long (*map_pop_elem)(struct bpf_map *map, void *value); 103 long (*map_peek_elem)(struct bpf_map *map, void *value); 104 void *(*map_lookup_percpu_elem)(struct bpf_map *map, void *key, u32 cpu); 105 106 /* funcs called by prog_array and perf_event_array map */ 107 void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file, 108 int fd); 109 void (*map_fd_put_ptr)(void *ptr); 110 int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf); 111 u32 (*map_fd_sys_lookup_elem)(void *ptr); 112 void (*map_seq_show_elem)(struct bpf_map *map, void *key, 113 struct seq_file *m); 114 int (*map_check_btf)(const struct bpf_map *map, 115 const struct btf *btf, 116 const struct btf_type *key_type, 117 const struct btf_type *value_type); 118 119 /* Prog poke tracking helpers. */ 120 int (*map_poke_track)(struct bpf_map *map, struct bpf_prog_aux *aux); 121 void (*map_poke_untrack)(struct bpf_map *map, struct bpf_prog_aux *aux); 122 void (*map_poke_run)(struct bpf_map *map, u32 key, struct bpf_prog *old, 123 struct bpf_prog *new); 124 125 /* Direct value access helpers. */ 126 int (*map_direct_value_addr)(const struct bpf_map *map, 127 u64 *imm, u32 off); 128 int (*map_direct_value_meta)(const struct bpf_map *map, 129 u64 imm, u32 *off); 130 int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma); 131 __poll_t (*map_poll)(struct bpf_map *map, struct file *filp, 132 struct poll_table_struct *pts); 133 134 /* Functions called by bpf_local_storage maps */ 135 int (*map_local_storage_charge)(struct bpf_local_storage_map *smap, 136 void *owner, u32 size); 137 void (*map_local_storage_uncharge)(struct bpf_local_storage_map *smap, 138 void *owner, u32 size); 139 struct bpf_local_storage __rcu ** (*map_owner_storage_ptr)(void *owner); 140 141 /* Misc helpers.*/ 142 long (*map_redirect)(struct bpf_map *map, u64 key, u64 flags); 143 144 /* map_meta_equal must be implemented for maps that can be 145 * used as an inner map. It is a runtime check to ensure 146 * an inner map can be inserted to an outer map. 147 * 148 * Some properties of the inner map has been used during the 149 * verification time. When inserting an inner map at the runtime, 150 * map_meta_equal has to ensure the inserting map has the same 151 * properties that the verifier has used earlier. 152 */ 153 bool (*map_meta_equal)(const struct bpf_map *meta0, 154 const struct bpf_map *meta1); 155 156 157 int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env, 158 struct bpf_func_state *caller, 159 struct bpf_func_state *callee); 160 long (*map_for_each_callback)(struct bpf_map *map, 161 bpf_callback_t callback_fn, 162 void *callback_ctx, u64 flags); 163 164 u64 (*map_mem_usage)(const struct bpf_map *map); 165 166 /* BTF id of struct allocated by map_alloc */ 167 int *map_btf_id; 168 169 /* bpf_iter info used to open a seq_file */ 170 const struct bpf_iter_seq_info *iter_seq_info; 171 }; 172 173 enum { 174 /* Support at most 10 fields in a BTF type */ 175 BTF_FIELDS_MAX = 10, 176 }; 177 178 enum btf_field_type { 179 BPF_SPIN_LOCK = (1 << 0), 180 BPF_TIMER = (1 << 1), 181 BPF_KPTR_UNREF = (1 << 2), 182 BPF_KPTR_REF = (1 << 3), 183 BPF_KPTR_PERCPU = (1 << 4), 184 BPF_KPTR = BPF_KPTR_UNREF | BPF_KPTR_REF | BPF_KPTR_PERCPU, 185 BPF_LIST_HEAD = (1 << 5), 186 BPF_LIST_NODE = (1 << 6), 187 BPF_RB_ROOT = (1 << 7), 188 BPF_RB_NODE = (1 << 8), 189 BPF_GRAPH_NODE_OR_ROOT = BPF_LIST_NODE | BPF_LIST_HEAD | 190 BPF_RB_NODE | BPF_RB_ROOT, 191 BPF_REFCOUNT = (1 << 9), 192 }; 193 194 typedef void (*btf_dtor_kfunc_t)(void *); 195 196 struct btf_field_kptr { 197 struct btf *btf; 198 struct module *module; 199 /* dtor used if btf_is_kernel(btf), otherwise the type is 200 * program-allocated, dtor is NULL, and __bpf_obj_drop_impl is used 201 */ 202 btf_dtor_kfunc_t dtor; 203 u32 btf_id; 204 }; 205 206 struct btf_field_graph_root { 207 struct btf *btf; 208 u32 value_btf_id; 209 u32 node_offset; 210 struct btf_record *value_rec; 211 }; 212 213 struct btf_field { 214 u32 offset; 215 u32 size; 216 enum btf_field_type type; 217 union { 218 struct btf_field_kptr kptr; 219 struct btf_field_graph_root graph_root; 220 }; 221 }; 222 223 struct btf_record { 224 u32 cnt; 225 u32 field_mask; 226 int spin_lock_off; 227 int timer_off; 228 int refcount_off; 229 struct btf_field fields[]; 230 }; 231 232 /* Non-opaque version of bpf_rb_node in uapi/linux/bpf.h */ 233 struct bpf_rb_node_kern { 234 struct rb_node rb_node; 235 void *owner; 236 } __attribute__((aligned(8))); 237 238 /* Non-opaque version of bpf_list_node in uapi/linux/bpf.h */ 239 struct bpf_list_node_kern { 240 struct list_head list_head; 241 void *owner; 242 } __attribute__((aligned(8))); 243 244 struct bpf_map { 245 /* The first two cachelines with read-mostly members of which some 246 * are also accessed in fast-path (e.g. ops, max_entries). 247 */ 248 const struct bpf_map_ops *ops ____cacheline_aligned; 249 struct bpf_map *inner_map_meta; 250 #ifdef CONFIG_SECURITY 251 void *security; 252 #endif 253 enum bpf_map_type map_type; 254 u32 key_size; 255 u32 value_size; 256 u32 max_entries; 257 u64 map_extra; /* any per-map-type extra fields */ 258 u32 map_flags; 259 u32 id; 260 struct btf_record *record; 261 int numa_node; 262 u32 btf_key_type_id; 263 u32 btf_value_type_id; 264 u32 btf_vmlinux_value_type_id; 265 struct btf *btf; 266 #ifdef CONFIG_MEMCG_KMEM 267 struct obj_cgroup *objcg; 268 #endif 269 char name[BPF_OBJ_NAME_LEN]; 270 /* The 3rd and 4th cacheline with misc members to avoid false sharing 271 * particularly with refcounting. 272 */ 273 atomic64_t refcnt ____cacheline_aligned; 274 atomic64_t usercnt; 275 struct work_struct work; 276 struct mutex freeze_mutex; 277 atomic64_t writecnt; 278 /* 'Ownership' of program-containing map is claimed by the first program 279 * that is going to use this map or by the first program which FD is 280 * stored in the map to make sure that all callers and callees have the 281 * same prog type, JITed flag and xdp_has_frags flag. 282 */ 283 struct { 284 spinlock_t lock; 285 enum bpf_prog_type type; 286 bool jited; 287 bool xdp_has_frags; 288 } owner; 289 bool bypass_spec_v1; 290 bool frozen; /* write-once; write-protected by freeze_mutex */ 291 s64 __percpu *elem_count; 292 }; 293 294 static inline const char *btf_field_type_name(enum btf_field_type type) 295 { 296 switch (type) { 297 case BPF_SPIN_LOCK: 298 return "bpf_spin_lock"; 299 case BPF_TIMER: 300 return "bpf_timer"; 301 case BPF_KPTR_UNREF: 302 case BPF_KPTR_REF: 303 return "kptr"; 304 case BPF_KPTR_PERCPU: 305 return "percpu_kptr"; 306 case BPF_LIST_HEAD: 307 return "bpf_list_head"; 308 case BPF_LIST_NODE: 309 return "bpf_list_node"; 310 case BPF_RB_ROOT: 311 return "bpf_rb_root"; 312 case BPF_RB_NODE: 313 return "bpf_rb_node"; 314 case BPF_REFCOUNT: 315 return "bpf_refcount"; 316 default: 317 WARN_ON_ONCE(1); 318 return "unknown"; 319 } 320 } 321 322 static inline u32 btf_field_type_size(enum btf_field_type type) 323 { 324 switch (type) { 325 case BPF_SPIN_LOCK: 326 return sizeof(struct bpf_spin_lock); 327 case BPF_TIMER: 328 return sizeof(struct bpf_timer); 329 case BPF_KPTR_UNREF: 330 case BPF_KPTR_REF: 331 case BPF_KPTR_PERCPU: 332 return sizeof(u64); 333 case BPF_LIST_HEAD: 334 return sizeof(struct bpf_list_head); 335 case BPF_LIST_NODE: 336 return sizeof(struct bpf_list_node); 337 case BPF_RB_ROOT: 338 return sizeof(struct bpf_rb_root); 339 case BPF_RB_NODE: 340 return sizeof(struct bpf_rb_node); 341 case BPF_REFCOUNT: 342 return sizeof(struct bpf_refcount); 343 default: 344 WARN_ON_ONCE(1); 345 return 0; 346 } 347 } 348 349 static inline u32 btf_field_type_align(enum btf_field_type type) 350 { 351 switch (type) { 352 case BPF_SPIN_LOCK: 353 return __alignof__(struct bpf_spin_lock); 354 case BPF_TIMER: 355 return __alignof__(struct bpf_timer); 356 case BPF_KPTR_UNREF: 357 case BPF_KPTR_REF: 358 case BPF_KPTR_PERCPU: 359 return __alignof__(u64); 360 case BPF_LIST_HEAD: 361 return __alignof__(struct bpf_list_head); 362 case BPF_LIST_NODE: 363 return __alignof__(struct bpf_list_node); 364 case BPF_RB_ROOT: 365 return __alignof__(struct bpf_rb_root); 366 case BPF_RB_NODE: 367 return __alignof__(struct bpf_rb_node); 368 case BPF_REFCOUNT: 369 return __alignof__(struct bpf_refcount); 370 default: 371 WARN_ON_ONCE(1); 372 return 0; 373 } 374 } 375 376 static inline void bpf_obj_init_field(const struct btf_field *field, void *addr) 377 { 378 memset(addr, 0, field->size); 379 380 switch (field->type) { 381 case BPF_REFCOUNT: 382 refcount_set((refcount_t *)addr, 1); 383 break; 384 case BPF_RB_NODE: 385 RB_CLEAR_NODE((struct rb_node *)addr); 386 break; 387 case BPF_LIST_HEAD: 388 case BPF_LIST_NODE: 389 INIT_LIST_HEAD((struct list_head *)addr); 390 break; 391 case BPF_RB_ROOT: 392 /* RB_ROOT_CACHED 0-inits, no need to do anything after memset */ 393 case BPF_SPIN_LOCK: 394 case BPF_TIMER: 395 case BPF_KPTR_UNREF: 396 case BPF_KPTR_REF: 397 case BPF_KPTR_PERCPU: 398 break; 399 default: 400 WARN_ON_ONCE(1); 401 return; 402 } 403 } 404 405 static inline bool btf_record_has_field(const struct btf_record *rec, enum btf_field_type type) 406 { 407 if (IS_ERR_OR_NULL(rec)) 408 return false; 409 return rec->field_mask & type; 410 } 411 412 static inline void bpf_obj_init(const struct btf_record *rec, void *obj) 413 { 414 int i; 415 416 if (IS_ERR_OR_NULL(rec)) 417 return; 418 for (i = 0; i < rec->cnt; i++) 419 bpf_obj_init_field(&rec->fields[i], obj + rec->fields[i].offset); 420 } 421 422 /* 'dst' must be a temporary buffer and should not point to memory that is being 423 * used in parallel by a bpf program or bpf syscall, otherwise the access from 424 * the bpf program or bpf syscall may be corrupted by the reinitialization, 425 * leading to weird problems. Even 'dst' is newly-allocated from bpf memory 426 * allocator, it is still possible for 'dst' to be used in parallel by a bpf 427 * program or bpf syscall. 428 */ 429 static inline void check_and_init_map_value(struct bpf_map *map, void *dst) 430 { 431 bpf_obj_init(map->record, dst); 432 } 433 434 /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and 435 * forced to use 'long' read/writes to try to atomically copy long counters. 436 * Best-effort only. No barriers here, since it _will_ race with concurrent 437 * updates from BPF programs. Called from bpf syscall and mostly used with 438 * size 8 or 16 bytes, so ask compiler to inline it. 439 */ 440 static inline void bpf_long_memcpy(void *dst, const void *src, u32 size) 441 { 442 const long *lsrc = src; 443 long *ldst = dst; 444 445 size /= sizeof(long); 446 while (size--) 447 data_race(*ldst++ = *lsrc++); 448 } 449 450 /* copy everything but bpf_spin_lock, bpf_timer, and kptrs. There could be one of each. */ 451 static inline void bpf_obj_memcpy(struct btf_record *rec, 452 void *dst, void *src, u32 size, 453 bool long_memcpy) 454 { 455 u32 curr_off = 0; 456 int i; 457 458 if (IS_ERR_OR_NULL(rec)) { 459 if (long_memcpy) 460 bpf_long_memcpy(dst, src, round_up(size, 8)); 461 else 462 memcpy(dst, src, size); 463 return; 464 } 465 466 for (i = 0; i < rec->cnt; i++) { 467 u32 next_off = rec->fields[i].offset; 468 u32 sz = next_off - curr_off; 469 470 memcpy(dst + curr_off, src + curr_off, sz); 471 curr_off += rec->fields[i].size + sz; 472 } 473 memcpy(dst + curr_off, src + curr_off, size - curr_off); 474 } 475 476 static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) 477 { 478 bpf_obj_memcpy(map->record, dst, src, map->value_size, false); 479 } 480 481 static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src) 482 { 483 bpf_obj_memcpy(map->record, dst, src, map->value_size, true); 484 } 485 486 static inline void bpf_obj_memzero(struct btf_record *rec, void *dst, u32 size) 487 { 488 u32 curr_off = 0; 489 int i; 490 491 if (IS_ERR_OR_NULL(rec)) { 492 memset(dst, 0, size); 493 return; 494 } 495 496 for (i = 0; i < rec->cnt; i++) { 497 u32 next_off = rec->fields[i].offset; 498 u32 sz = next_off - curr_off; 499 500 memset(dst + curr_off, 0, sz); 501 curr_off += rec->fields[i].size + sz; 502 } 503 memset(dst + curr_off, 0, size - curr_off); 504 } 505 506 static inline void zero_map_value(struct bpf_map *map, void *dst) 507 { 508 bpf_obj_memzero(map->record, dst, map->value_size); 509 } 510 511 void copy_map_value_locked(struct bpf_map *map, void *dst, void *src, 512 bool lock_src); 513 void bpf_timer_cancel_and_free(void *timer); 514 void bpf_list_head_free(const struct btf_field *field, void *list_head, 515 struct bpf_spin_lock *spin_lock); 516 void bpf_rb_root_free(const struct btf_field *field, void *rb_root, 517 struct bpf_spin_lock *spin_lock); 518 519 520 int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size); 521 522 struct bpf_offload_dev; 523 struct bpf_offloaded_map; 524 525 struct bpf_map_dev_ops { 526 int (*map_get_next_key)(struct bpf_offloaded_map *map, 527 void *key, void *next_key); 528 int (*map_lookup_elem)(struct bpf_offloaded_map *map, 529 void *key, void *value); 530 int (*map_update_elem)(struct bpf_offloaded_map *map, 531 void *key, void *value, u64 flags); 532 int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key); 533 }; 534 535 struct bpf_offloaded_map { 536 struct bpf_map map; 537 struct net_device *netdev; 538 const struct bpf_map_dev_ops *dev_ops; 539 void *dev_priv; 540 struct list_head offloads; 541 }; 542 543 static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map) 544 { 545 return container_of(map, struct bpf_offloaded_map, map); 546 } 547 548 static inline bool bpf_map_offload_neutral(const struct bpf_map *map) 549 { 550 return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 551 } 552 553 static inline bool bpf_map_support_seq_show(const struct bpf_map *map) 554 { 555 return (map->btf_value_type_id || map->btf_vmlinux_value_type_id) && 556 map->ops->map_seq_show_elem; 557 } 558 559 int map_check_no_btf(const struct bpf_map *map, 560 const struct btf *btf, 561 const struct btf_type *key_type, 562 const struct btf_type *value_type); 563 564 bool bpf_map_meta_equal(const struct bpf_map *meta0, 565 const struct bpf_map *meta1); 566 567 extern const struct bpf_map_ops bpf_map_offload_ops; 568 569 /* bpf_type_flag contains a set of flags that are applicable to the values of 570 * arg_type, ret_type and reg_type. For example, a pointer value may be null, 571 * or a memory is read-only. We classify types into two categories: base types 572 * and extended types. Extended types are base types combined with a type flag. 573 * 574 * Currently there are no more than 32 base types in arg_type, ret_type and 575 * reg_types. 576 */ 577 #define BPF_BASE_TYPE_BITS 8 578 579 enum bpf_type_flag { 580 /* PTR may be NULL. */ 581 PTR_MAYBE_NULL = BIT(0 + BPF_BASE_TYPE_BITS), 582 583 /* MEM is read-only. When applied on bpf_arg, it indicates the arg is 584 * compatible with both mutable and immutable memory. 585 */ 586 MEM_RDONLY = BIT(1 + BPF_BASE_TYPE_BITS), 587 588 /* MEM points to BPF ring buffer reservation. */ 589 MEM_RINGBUF = BIT(2 + BPF_BASE_TYPE_BITS), 590 591 /* MEM is in user address space. */ 592 MEM_USER = BIT(3 + BPF_BASE_TYPE_BITS), 593 594 /* MEM is a percpu memory. MEM_PERCPU tags PTR_TO_BTF_ID. When tagged 595 * with MEM_PERCPU, PTR_TO_BTF_ID _cannot_ be directly accessed. In 596 * order to drop this tag, it must be passed into bpf_per_cpu_ptr() 597 * or bpf_this_cpu_ptr(), which will return the pointer corresponding 598 * to the specified cpu. 599 */ 600 MEM_PERCPU = BIT(4 + BPF_BASE_TYPE_BITS), 601 602 /* Indicates that the argument will be released. */ 603 OBJ_RELEASE = BIT(5 + BPF_BASE_TYPE_BITS), 604 605 /* PTR is not trusted. This is only used with PTR_TO_BTF_ID, to mark 606 * unreferenced and referenced kptr loaded from map value using a load 607 * instruction, so that they can only be dereferenced but not escape the 608 * BPF program into the kernel (i.e. cannot be passed as arguments to 609 * kfunc or bpf helpers). 610 */ 611 PTR_UNTRUSTED = BIT(6 + BPF_BASE_TYPE_BITS), 612 613 MEM_UNINIT = BIT(7 + BPF_BASE_TYPE_BITS), 614 615 /* DYNPTR points to memory local to the bpf program. */ 616 DYNPTR_TYPE_LOCAL = BIT(8 + BPF_BASE_TYPE_BITS), 617 618 /* DYNPTR points to a kernel-produced ringbuf record. */ 619 DYNPTR_TYPE_RINGBUF = BIT(9 + BPF_BASE_TYPE_BITS), 620 621 /* Size is known at compile time. */ 622 MEM_FIXED_SIZE = BIT(10 + BPF_BASE_TYPE_BITS), 623 624 /* MEM is of an allocated object of type in program BTF. This is used to 625 * tag PTR_TO_BTF_ID allocated using bpf_obj_new. 626 */ 627 MEM_ALLOC = BIT(11 + BPF_BASE_TYPE_BITS), 628 629 /* PTR was passed from the kernel in a trusted context, and may be 630 * passed to KF_TRUSTED_ARGS kfuncs or BPF helper functions. 631 * Confusingly, this is _not_ the opposite of PTR_UNTRUSTED above. 632 * PTR_UNTRUSTED refers to a kptr that was read directly from a map 633 * without invoking bpf_kptr_xchg(). What we really need to know is 634 * whether a pointer is safe to pass to a kfunc or BPF helper function. 635 * While PTR_UNTRUSTED pointers are unsafe to pass to kfuncs and BPF 636 * helpers, they do not cover all possible instances of unsafe 637 * pointers. For example, a pointer that was obtained from walking a 638 * struct will _not_ get the PTR_UNTRUSTED type modifier, despite the 639 * fact that it may be NULL, invalid, etc. This is due to backwards 640 * compatibility requirements, as this was the behavior that was first 641 * introduced when kptrs were added. The behavior is now considered 642 * deprecated, and PTR_UNTRUSTED will eventually be removed. 643 * 644 * PTR_TRUSTED, on the other hand, is a pointer that the kernel 645 * guarantees to be valid and safe to pass to kfuncs and BPF helpers. 646 * For example, pointers passed to tracepoint arguments are considered 647 * PTR_TRUSTED, as are pointers that are passed to struct_ops 648 * callbacks. As alluded to above, pointers that are obtained from 649 * walking PTR_TRUSTED pointers are _not_ trusted. For example, if a 650 * struct task_struct *task is PTR_TRUSTED, then accessing 651 * task->last_wakee will lose the PTR_TRUSTED modifier when it's stored 652 * in a BPF register. Similarly, pointers passed to certain programs 653 * types such as kretprobes are not guaranteed to be valid, as they may 654 * for example contain an object that was recently freed. 655 */ 656 PTR_TRUSTED = BIT(12 + BPF_BASE_TYPE_BITS), 657 658 /* MEM is tagged with rcu and memory access needs rcu_read_lock protection. */ 659 MEM_RCU = BIT(13 + BPF_BASE_TYPE_BITS), 660 661 /* Used to tag PTR_TO_BTF_ID | MEM_ALLOC references which are non-owning. 662 * Currently only valid for linked-list and rbtree nodes. If the nodes 663 * have a bpf_refcount_field, they must be tagged MEM_RCU as well. 664 */ 665 NON_OWN_REF = BIT(14 + BPF_BASE_TYPE_BITS), 666 667 /* DYNPTR points to sk_buff */ 668 DYNPTR_TYPE_SKB = BIT(15 + BPF_BASE_TYPE_BITS), 669 670 /* DYNPTR points to xdp_buff */ 671 DYNPTR_TYPE_XDP = BIT(16 + BPF_BASE_TYPE_BITS), 672 673 __BPF_TYPE_FLAG_MAX, 674 __BPF_TYPE_LAST_FLAG = __BPF_TYPE_FLAG_MAX - 1, 675 }; 676 677 #define DYNPTR_TYPE_FLAG_MASK (DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF | DYNPTR_TYPE_SKB \ 678 | DYNPTR_TYPE_XDP) 679 680 /* Max number of base types. */ 681 #define BPF_BASE_TYPE_LIMIT (1UL << BPF_BASE_TYPE_BITS) 682 683 /* Max number of all types. */ 684 #define BPF_TYPE_LIMIT (__BPF_TYPE_LAST_FLAG | (__BPF_TYPE_LAST_FLAG - 1)) 685 686 /* function argument constraints */ 687 enum bpf_arg_type { 688 ARG_DONTCARE = 0, /* unused argument in helper function */ 689 690 /* the following constraints used to prototype 691 * bpf_map_lookup/update/delete_elem() functions 692 */ 693 ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */ 694 ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */ 695 ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */ 696 697 /* Used to prototype bpf_memcmp() and other functions that access data 698 * on eBPF program stack 699 */ 700 ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ 701 702 ARG_CONST_SIZE, /* number of bytes accessed from memory */ 703 ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ 704 705 ARG_PTR_TO_CTX, /* pointer to context */ 706 ARG_ANYTHING, /* any (initialized) argument is ok */ 707 ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */ 708 ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */ 709 ARG_PTR_TO_INT, /* pointer to int */ 710 ARG_PTR_TO_LONG, /* pointer to long */ 711 ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */ 712 ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */ 713 ARG_PTR_TO_RINGBUF_MEM, /* pointer to dynamically reserved ringbuf memory */ 714 ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */ 715 ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */ 716 ARG_PTR_TO_PERCPU_BTF_ID, /* pointer to in-kernel percpu type */ 717 ARG_PTR_TO_FUNC, /* pointer to a bpf program function */ 718 ARG_PTR_TO_STACK, /* pointer to stack */ 719 ARG_PTR_TO_CONST_STR, /* pointer to a null terminated read-only string */ 720 ARG_PTR_TO_TIMER, /* pointer to bpf_timer */ 721 ARG_PTR_TO_KPTR, /* pointer to referenced kptr */ 722 ARG_PTR_TO_DYNPTR, /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */ 723 __BPF_ARG_TYPE_MAX, 724 725 /* Extended arg_types. */ 726 ARG_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MAP_VALUE, 727 ARG_PTR_TO_MEM_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MEM, 728 ARG_PTR_TO_CTX_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_CTX, 729 ARG_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_SOCKET, 730 ARG_PTR_TO_STACK_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_STACK, 731 ARG_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_BTF_ID, 732 /* pointer to memory does not need to be initialized, helper function must fill 733 * all bytes or clear them in error case. 734 */ 735 ARG_PTR_TO_UNINIT_MEM = MEM_UNINIT | ARG_PTR_TO_MEM, 736 /* Pointer to valid memory of size known at compile time. */ 737 ARG_PTR_TO_FIXED_SIZE_MEM = MEM_FIXED_SIZE | ARG_PTR_TO_MEM, 738 739 /* This must be the last entry. Its purpose is to ensure the enum is 740 * wide enough to hold the higher bits reserved for bpf_type_flag. 741 */ 742 __BPF_ARG_TYPE_LIMIT = BPF_TYPE_LIMIT, 743 }; 744 static_assert(__BPF_ARG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); 745 746 /* type of values returned from helper functions */ 747 enum bpf_return_type { 748 RET_INTEGER, /* function returns integer */ 749 RET_VOID, /* function doesn't return anything */ 750 RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */ 751 RET_PTR_TO_SOCKET, /* returns a pointer to a socket */ 752 RET_PTR_TO_TCP_SOCK, /* returns a pointer to a tcp_sock */ 753 RET_PTR_TO_SOCK_COMMON, /* returns a pointer to a sock_common */ 754 RET_PTR_TO_MEM, /* returns a pointer to memory */ 755 RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */ 756 RET_PTR_TO_BTF_ID, /* returns a pointer to a btf_id */ 757 __BPF_RET_TYPE_MAX, 758 759 /* Extended ret_types. */ 760 RET_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_MAP_VALUE, 761 RET_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCKET, 762 RET_PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_TCP_SOCK, 763 RET_PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCK_COMMON, 764 RET_PTR_TO_RINGBUF_MEM_OR_NULL = PTR_MAYBE_NULL | MEM_RINGBUF | RET_PTR_TO_MEM, 765 RET_PTR_TO_DYNPTR_MEM_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_MEM, 766 RET_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_BTF_ID, 767 RET_PTR_TO_BTF_ID_TRUSTED = PTR_TRUSTED | RET_PTR_TO_BTF_ID, 768 769 /* This must be the last entry. Its purpose is to ensure the enum is 770 * wide enough to hold the higher bits reserved for bpf_type_flag. 771 */ 772 __BPF_RET_TYPE_LIMIT = BPF_TYPE_LIMIT, 773 }; 774 static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); 775 776 /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs 777 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL 778 * instructions after verifying 779 */ 780 struct bpf_func_proto { 781 u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 782 bool gpl_only; 783 bool pkt_access; 784 bool might_sleep; 785 enum bpf_return_type ret_type; 786 union { 787 struct { 788 enum bpf_arg_type arg1_type; 789 enum bpf_arg_type arg2_type; 790 enum bpf_arg_type arg3_type; 791 enum bpf_arg_type arg4_type; 792 enum bpf_arg_type arg5_type; 793 }; 794 enum bpf_arg_type arg_type[5]; 795 }; 796 union { 797 struct { 798 u32 *arg1_btf_id; 799 u32 *arg2_btf_id; 800 u32 *arg3_btf_id; 801 u32 *arg4_btf_id; 802 u32 *arg5_btf_id; 803 }; 804 u32 *arg_btf_id[5]; 805 struct { 806 size_t arg1_size; 807 size_t arg2_size; 808 size_t arg3_size; 809 size_t arg4_size; 810 size_t arg5_size; 811 }; 812 size_t arg_size[5]; 813 }; 814 int *ret_btf_id; /* return value btf_id */ 815 bool (*allowed)(const struct bpf_prog *prog); 816 }; 817 818 /* bpf_context is intentionally undefined structure. Pointer to bpf_context is 819 * the first argument to eBPF programs. 820 * For socket filters: 'struct bpf_context *' == 'struct sk_buff *' 821 */ 822 struct bpf_context; 823 824 enum bpf_access_type { 825 BPF_READ = 1, 826 BPF_WRITE = 2 827 }; 828 829 /* types of values stored in eBPF registers */ 830 /* Pointer types represent: 831 * pointer 832 * pointer + imm 833 * pointer + (u16) var 834 * pointer + (u16) var + imm 835 * if (range > 0) then [ptr, ptr + range - off) is safe to access 836 * if (id > 0) means that some 'var' was added 837 * if (off > 0) means that 'imm' was added 838 */ 839 enum bpf_reg_type { 840 NOT_INIT = 0, /* nothing was written into register */ 841 SCALAR_VALUE, /* reg doesn't contain a valid pointer */ 842 PTR_TO_CTX, /* reg points to bpf_context */ 843 CONST_PTR_TO_MAP, /* reg points to struct bpf_map */ 844 PTR_TO_MAP_VALUE, /* reg points to map element value */ 845 PTR_TO_MAP_KEY, /* reg points to a map element key */ 846 PTR_TO_STACK, /* reg == frame_pointer + offset */ 847 PTR_TO_PACKET_META, /* skb->data - meta_len */ 848 PTR_TO_PACKET, /* reg points to skb->data */ 849 PTR_TO_PACKET_END, /* skb->data + headlen */ 850 PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */ 851 PTR_TO_SOCKET, /* reg points to struct bpf_sock */ 852 PTR_TO_SOCK_COMMON, /* reg points to sock_common */ 853 PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */ 854 PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */ 855 PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */ 856 /* PTR_TO_BTF_ID points to a kernel struct that does not need 857 * to be null checked by the BPF program. This does not imply the 858 * pointer is _not_ null and in practice this can easily be a null 859 * pointer when reading pointer chains. The assumption is program 860 * context will handle null pointer dereference typically via fault 861 * handling. The verifier must keep this in mind and can make no 862 * assumptions about null or non-null when doing branch analysis. 863 * Further, when passed into helpers the helpers can not, without 864 * additional context, assume the value is non-null. 865 */ 866 PTR_TO_BTF_ID, 867 /* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not 868 * been checked for null. Used primarily to inform the verifier 869 * an explicit null check is required for this struct. 870 */ 871 PTR_TO_MEM, /* reg points to valid memory region */ 872 PTR_TO_BUF, /* reg points to a read/write buffer */ 873 PTR_TO_FUNC, /* reg points to a bpf program function */ 874 CONST_PTR_TO_DYNPTR, /* reg points to a const struct bpf_dynptr */ 875 __BPF_REG_TYPE_MAX, 876 877 /* Extended reg_types. */ 878 PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | PTR_TO_MAP_VALUE, 879 PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCKET, 880 PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON, 881 PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | PTR_TO_TCP_SOCK, 882 PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | PTR_TO_BTF_ID, 883 884 /* This must be the last entry. Its purpose is to ensure the enum is 885 * wide enough to hold the higher bits reserved for bpf_type_flag. 886 */ 887 __BPF_REG_TYPE_LIMIT = BPF_TYPE_LIMIT, 888 }; 889 static_assert(__BPF_REG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); 890 891 /* The information passed from prog-specific *_is_valid_access 892 * back to the verifier. 893 */ 894 struct bpf_insn_access_aux { 895 enum bpf_reg_type reg_type; 896 union { 897 int ctx_field_size; 898 struct { 899 struct btf *btf; 900 u32 btf_id; 901 }; 902 }; 903 struct bpf_verifier_log *log; /* for verbose logs */ 904 }; 905 906 static inline void 907 bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size) 908 { 909 aux->ctx_field_size = size; 910 } 911 912 static bool bpf_is_ldimm64(const struct bpf_insn *insn) 913 { 914 return insn->code == (BPF_LD | BPF_IMM | BPF_DW); 915 } 916 917 static inline bool bpf_pseudo_func(const struct bpf_insn *insn) 918 { 919 return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC; 920 } 921 922 struct bpf_prog_ops { 923 int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, 924 union bpf_attr __user *uattr); 925 }; 926 927 struct bpf_reg_state; 928 struct bpf_verifier_ops { 929 /* return eBPF function prototype for verification */ 930 const struct bpf_func_proto * 931 (*get_func_proto)(enum bpf_func_id func_id, 932 const struct bpf_prog *prog); 933 934 /* return true if 'size' wide access at offset 'off' within bpf_context 935 * with 'type' (read or write) is allowed 936 */ 937 bool (*is_valid_access)(int off, int size, enum bpf_access_type type, 938 const struct bpf_prog *prog, 939 struct bpf_insn_access_aux *info); 940 int (*gen_prologue)(struct bpf_insn *insn, bool direct_write, 941 const struct bpf_prog *prog); 942 int (*gen_ld_abs)(const struct bpf_insn *orig, 943 struct bpf_insn *insn_buf); 944 u32 (*convert_ctx_access)(enum bpf_access_type type, 945 const struct bpf_insn *src, 946 struct bpf_insn *dst, 947 struct bpf_prog *prog, u32 *target_size); 948 int (*btf_struct_access)(struct bpf_verifier_log *log, 949 const struct bpf_reg_state *reg, 950 int off, int size); 951 }; 952 953 struct bpf_prog_offload_ops { 954 /* verifier basic callbacks */ 955 int (*insn_hook)(struct bpf_verifier_env *env, 956 int insn_idx, int prev_insn_idx); 957 int (*finalize)(struct bpf_verifier_env *env); 958 /* verifier optimization callbacks (called after .finalize) */ 959 int (*replace_insn)(struct bpf_verifier_env *env, u32 off, 960 struct bpf_insn *insn); 961 int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt); 962 /* program management callbacks */ 963 int (*prepare)(struct bpf_prog *prog); 964 int (*translate)(struct bpf_prog *prog); 965 void (*destroy)(struct bpf_prog *prog); 966 }; 967 968 struct bpf_prog_offload { 969 struct bpf_prog *prog; 970 struct net_device *netdev; 971 struct bpf_offload_dev *offdev; 972 void *dev_priv; 973 struct list_head offloads; 974 bool dev_state; 975 bool opt_failed; 976 void *jited_image; 977 u32 jited_len; 978 }; 979 980 enum bpf_cgroup_storage_type { 981 BPF_CGROUP_STORAGE_SHARED, 982 BPF_CGROUP_STORAGE_PERCPU, 983 __BPF_CGROUP_STORAGE_MAX 984 }; 985 986 #define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX 987 988 /* The longest tracepoint has 12 args. 989 * See include/trace/bpf_probe.h 990 */ 991 #define MAX_BPF_FUNC_ARGS 12 992 993 /* The maximum number of arguments passed through registers 994 * a single function may have. 995 */ 996 #define MAX_BPF_FUNC_REG_ARGS 5 997 998 /* The argument is a structure. */ 999 #define BTF_FMODEL_STRUCT_ARG BIT(0) 1000 1001 /* The argument is signed. */ 1002 #define BTF_FMODEL_SIGNED_ARG BIT(1) 1003 1004 struct btf_func_model { 1005 u8 ret_size; 1006 u8 ret_flags; 1007 u8 nr_args; 1008 u8 arg_size[MAX_BPF_FUNC_ARGS]; 1009 u8 arg_flags[MAX_BPF_FUNC_ARGS]; 1010 }; 1011 1012 /* Restore arguments before returning from trampoline to let original function 1013 * continue executing. This flag is used for fentry progs when there are no 1014 * fexit progs. 1015 */ 1016 #define BPF_TRAMP_F_RESTORE_REGS BIT(0) 1017 /* Call original function after fentry progs, but before fexit progs. 1018 * Makes sense for fentry/fexit, normal calls and indirect calls. 1019 */ 1020 #define BPF_TRAMP_F_CALL_ORIG BIT(1) 1021 /* Skip current frame and return to parent. Makes sense for fentry/fexit 1022 * programs only. Should not be used with normal calls and indirect calls. 1023 */ 1024 #define BPF_TRAMP_F_SKIP_FRAME BIT(2) 1025 /* Store IP address of the caller on the trampoline stack, 1026 * so it's available for trampoline's programs. 1027 */ 1028 #define BPF_TRAMP_F_IP_ARG BIT(3) 1029 /* Return the return value of fentry prog. Only used by bpf_struct_ops. */ 1030 #define BPF_TRAMP_F_RET_FENTRY_RET BIT(4) 1031 1032 /* Get original function from stack instead of from provided direct address. 1033 * Makes sense for trampolines with fexit or fmod_ret programs. 1034 */ 1035 #define BPF_TRAMP_F_ORIG_STACK BIT(5) 1036 1037 /* This trampoline is on a function with another ftrace_ops with IPMODIFY, 1038 * e.g., a live patch. This flag is set and cleared by ftrace call backs, 1039 */ 1040 #define BPF_TRAMP_F_SHARE_IPMODIFY BIT(6) 1041 1042 /* Indicate that current trampoline is in a tail call context. Then, it has to 1043 * cache and restore tail_call_cnt to avoid infinite tail call loop. 1044 */ 1045 #define BPF_TRAMP_F_TAIL_CALL_CTX BIT(7) 1046 1047 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 1048 * bytes on x86. 1049 */ 1050 enum { 1051 #if defined(__s390x__) 1052 BPF_MAX_TRAMP_LINKS = 27, 1053 #else 1054 BPF_MAX_TRAMP_LINKS = 38, 1055 #endif 1056 }; 1057 1058 struct bpf_tramp_links { 1059 struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS]; 1060 int nr_links; 1061 }; 1062 1063 struct bpf_tramp_run_ctx; 1064 1065 /* Different use cases for BPF trampoline: 1066 * 1. replace nop at the function entry (kprobe equivalent) 1067 * flags = BPF_TRAMP_F_RESTORE_REGS 1068 * fentry = a set of programs to run before returning from trampoline 1069 * 1070 * 2. replace nop at the function entry (kprobe + kretprobe equivalent) 1071 * flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME 1072 * orig_call = fentry_ip + MCOUNT_INSN_SIZE 1073 * fentry = a set of program to run before calling original function 1074 * fexit = a set of program to run after original function 1075 * 1076 * 3. replace direct call instruction anywhere in the function body 1077 * or assign a function pointer for indirect call (like tcp_congestion_ops->cong_avoid) 1078 * With flags = 0 1079 * fentry = a set of programs to run before returning from trampoline 1080 * With flags = BPF_TRAMP_F_CALL_ORIG 1081 * orig_call = original callback addr or direct function addr 1082 * fentry = a set of program to run before calling original function 1083 * fexit = a set of program to run after original function 1084 */ 1085 struct bpf_tramp_image; 1086 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end, 1087 const struct btf_func_model *m, u32 flags, 1088 struct bpf_tramp_links *tlinks, 1089 void *orig_call); 1090 u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog, 1091 struct bpf_tramp_run_ctx *run_ctx); 1092 void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start, 1093 struct bpf_tramp_run_ctx *run_ctx); 1094 void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr); 1095 void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr); 1096 typedef u64 (*bpf_trampoline_enter_t)(struct bpf_prog *prog, 1097 struct bpf_tramp_run_ctx *run_ctx); 1098 typedef void (*bpf_trampoline_exit_t)(struct bpf_prog *prog, u64 start, 1099 struct bpf_tramp_run_ctx *run_ctx); 1100 bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog); 1101 bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog); 1102 1103 struct bpf_ksym { 1104 unsigned long start; 1105 unsigned long end; 1106 char name[KSYM_NAME_LEN]; 1107 struct list_head lnode; 1108 struct latch_tree_node tnode; 1109 bool prog; 1110 }; 1111 1112 enum bpf_tramp_prog_type { 1113 BPF_TRAMP_FENTRY, 1114 BPF_TRAMP_FEXIT, 1115 BPF_TRAMP_MODIFY_RETURN, 1116 BPF_TRAMP_MAX, 1117 BPF_TRAMP_REPLACE, /* more than MAX */ 1118 }; 1119 1120 struct bpf_tramp_image { 1121 void *image; 1122 struct bpf_ksym ksym; 1123 struct percpu_ref pcref; 1124 void *ip_after_call; 1125 void *ip_epilogue; 1126 union { 1127 struct rcu_head rcu; 1128 struct work_struct work; 1129 }; 1130 }; 1131 1132 struct bpf_trampoline { 1133 /* hlist for trampoline_table */ 1134 struct hlist_node hlist; 1135 struct ftrace_ops *fops; 1136 /* serializes access to fields of this trampoline */ 1137 struct mutex mutex; 1138 refcount_t refcnt; 1139 u32 flags; 1140 u64 key; 1141 struct { 1142 struct btf_func_model model; 1143 void *addr; 1144 bool ftrace_managed; 1145 } func; 1146 /* if !NULL this is BPF_PROG_TYPE_EXT program that extends another BPF 1147 * program by replacing one of its functions. func.addr is the address 1148 * of the function it replaced. 1149 */ 1150 struct bpf_prog *extension_prog; 1151 /* list of BPF programs using this trampoline */ 1152 struct hlist_head progs_hlist[BPF_TRAMP_MAX]; 1153 /* Number of attached programs. A counter per kind. */ 1154 int progs_cnt[BPF_TRAMP_MAX]; 1155 /* Executable image of trampoline */ 1156 struct bpf_tramp_image *cur_image; 1157 struct module *mod; 1158 }; 1159 1160 struct bpf_attach_target_info { 1161 struct btf_func_model fmodel; 1162 long tgt_addr; 1163 struct module *tgt_mod; 1164 const char *tgt_name; 1165 const struct btf_type *tgt_type; 1166 }; 1167 1168 #define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */ 1169 1170 struct bpf_dispatcher_prog { 1171 struct bpf_prog *prog; 1172 refcount_t users; 1173 }; 1174 1175 struct bpf_dispatcher { 1176 /* dispatcher mutex */ 1177 struct mutex mutex; 1178 void *func; 1179 struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX]; 1180 int num_progs; 1181 void *image; 1182 void *rw_image; 1183 u32 image_off; 1184 struct bpf_ksym ksym; 1185 #ifdef CONFIG_HAVE_STATIC_CALL 1186 struct static_call_key *sc_key; 1187 void *sc_tramp; 1188 #endif 1189 }; 1190 1191 static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( 1192 const void *ctx, 1193 const struct bpf_insn *insnsi, 1194 bpf_func_t bpf_func) 1195 { 1196 return bpf_func(ctx, insnsi); 1197 } 1198 1199 /* the implementation of the opaque uapi struct bpf_dynptr */ 1200 struct bpf_dynptr_kern { 1201 void *data; 1202 /* Size represents the number of usable bytes of dynptr data. 1203 * If for example the offset is at 4 for a local dynptr whose data is 1204 * of type u64, the number of usable bytes is 4. 1205 * 1206 * The upper 8 bits are reserved. It is as follows: 1207 * Bits 0 - 23 = size 1208 * Bits 24 - 30 = dynptr type 1209 * Bit 31 = whether dynptr is read-only 1210 */ 1211 u32 size; 1212 u32 offset; 1213 } __aligned(8); 1214 1215 enum bpf_dynptr_type { 1216 BPF_DYNPTR_TYPE_INVALID, 1217 /* Points to memory that is local to the bpf program */ 1218 BPF_DYNPTR_TYPE_LOCAL, 1219 /* Underlying data is a ringbuf record */ 1220 BPF_DYNPTR_TYPE_RINGBUF, 1221 /* Underlying data is a sk_buff */ 1222 BPF_DYNPTR_TYPE_SKB, 1223 /* Underlying data is a xdp_buff */ 1224 BPF_DYNPTR_TYPE_XDP, 1225 }; 1226 1227 int bpf_dynptr_check_size(u32 size); 1228 u32 __bpf_dynptr_size(const struct bpf_dynptr_kern *ptr); 1229 1230 #ifdef CONFIG_BPF_JIT 1231 int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr); 1232 int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr); 1233 struct bpf_trampoline *bpf_trampoline_get(u64 key, 1234 struct bpf_attach_target_info *tgt_info); 1235 void bpf_trampoline_put(struct bpf_trampoline *tr); 1236 int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs); 1237 1238 /* 1239 * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn 1240 * indirection with a direct call to the bpf program. If the architecture does 1241 * not have STATIC_CALL, avoid a double-indirection. 1242 */ 1243 #ifdef CONFIG_HAVE_STATIC_CALL 1244 1245 #define __BPF_DISPATCHER_SC_INIT(_name) \ 1246 .sc_key = &STATIC_CALL_KEY(_name), \ 1247 .sc_tramp = STATIC_CALL_TRAMP_ADDR(_name), 1248 1249 #define __BPF_DISPATCHER_SC(name) \ 1250 DEFINE_STATIC_CALL(bpf_dispatcher_##name##_call, bpf_dispatcher_nop_func) 1251 1252 #define __BPF_DISPATCHER_CALL(name) \ 1253 static_call(bpf_dispatcher_##name##_call)(ctx, insnsi, bpf_func) 1254 1255 #define __BPF_DISPATCHER_UPDATE(_d, _new) \ 1256 __static_call_update((_d)->sc_key, (_d)->sc_tramp, (_new)) 1257 1258 #else 1259 #define __BPF_DISPATCHER_SC_INIT(name) 1260 #define __BPF_DISPATCHER_SC(name) 1261 #define __BPF_DISPATCHER_CALL(name) bpf_func(ctx, insnsi) 1262 #define __BPF_DISPATCHER_UPDATE(_d, _new) 1263 #endif 1264 1265 #define BPF_DISPATCHER_INIT(_name) { \ 1266 .mutex = __MUTEX_INITIALIZER(_name.mutex), \ 1267 .func = &_name##_func, \ 1268 .progs = {}, \ 1269 .num_progs = 0, \ 1270 .image = NULL, \ 1271 .image_off = 0, \ 1272 .ksym = { \ 1273 .name = #_name, \ 1274 .lnode = LIST_HEAD_INIT(_name.ksym.lnode), \ 1275 }, \ 1276 __BPF_DISPATCHER_SC_INIT(_name##_call) \ 1277 } 1278 1279 #define DEFINE_BPF_DISPATCHER(name) \ 1280 __BPF_DISPATCHER_SC(name); \ 1281 noinline __nocfi unsigned int bpf_dispatcher_##name##_func( \ 1282 const void *ctx, \ 1283 const struct bpf_insn *insnsi, \ 1284 bpf_func_t bpf_func) \ 1285 { \ 1286 return __BPF_DISPATCHER_CALL(name); \ 1287 } \ 1288 EXPORT_SYMBOL(bpf_dispatcher_##name##_func); \ 1289 struct bpf_dispatcher bpf_dispatcher_##name = \ 1290 BPF_DISPATCHER_INIT(bpf_dispatcher_##name); 1291 1292 #define DECLARE_BPF_DISPATCHER(name) \ 1293 unsigned int bpf_dispatcher_##name##_func( \ 1294 const void *ctx, \ 1295 const struct bpf_insn *insnsi, \ 1296 bpf_func_t bpf_func); \ 1297 extern struct bpf_dispatcher bpf_dispatcher_##name; 1298 1299 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func 1300 #define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name) 1301 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, 1302 struct bpf_prog *to); 1303 /* Called only from JIT-enabled code, so there's no need for stubs. */ 1304 void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym); 1305 void bpf_image_ksym_del(struct bpf_ksym *ksym); 1306 void bpf_ksym_add(struct bpf_ksym *ksym); 1307 void bpf_ksym_del(struct bpf_ksym *ksym); 1308 int bpf_jit_charge_modmem(u32 size); 1309 void bpf_jit_uncharge_modmem(u32 size); 1310 bool bpf_prog_has_trampoline(const struct bpf_prog *prog); 1311 #else 1312 static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link, 1313 struct bpf_trampoline *tr) 1314 { 1315 return -ENOTSUPP; 1316 } 1317 static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, 1318 struct bpf_trampoline *tr) 1319 { 1320 return -ENOTSUPP; 1321 } 1322 static inline struct bpf_trampoline *bpf_trampoline_get(u64 key, 1323 struct bpf_attach_target_info *tgt_info) 1324 { 1325 return NULL; 1326 } 1327 static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {} 1328 #define DEFINE_BPF_DISPATCHER(name) 1329 #define DECLARE_BPF_DISPATCHER(name) 1330 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nop_func 1331 #define BPF_DISPATCHER_PTR(name) NULL 1332 static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, 1333 struct bpf_prog *from, 1334 struct bpf_prog *to) {} 1335 static inline bool is_bpf_image_address(unsigned long address) 1336 { 1337 return false; 1338 } 1339 static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog) 1340 { 1341 return false; 1342 } 1343 #endif 1344 1345 struct bpf_func_info_aux { 1346 u16 linkage; 1347 bool unreliable; 1348 }; 1349 1350 enum bpf_jit_poke_reason { 1351 BPF_POKE_REASON_TAIL_CALL, 1352 }; 1353 1354 /* Descriptor of pokes pointing /into/ the JITed image. */ 1355 struct bpf_jit_poke_descriptor { 1356 void *tailcall_target; 1357 void *tailcall_bypass; 1358 void *bypass_addr; 1359 void *aux; 1360 union { 1361 struct { 1362 struct bpf_map *map; 1363 u32 key; 1364 } tail_call; 1365 }; 1366 bool tailcall_target_stable; 1367 u8 adj_off; 1368 u16 reason; 1369 u32 insn_idx; 1370 }; 1371 1372 /* reg_type info for ctx arguments */ 1373 struct bpf_ctx_arg_aux { 1374 u32 offset; 1375 enum bpf_reg_type reg_type; 1376 u32 btf_id; 1377 }; 1378 1379 struct btf_mod_pair { 1380 struct btf *btf; 1381 struct module *module; 1382 }; 1383 1384 struct bpf_kfunc_desc_tab; 1385 1386 struct bpf_prog_aux { 1387 atomic64_t refcnt; 1388 u32 used_map_cnt; 1389 u32 used_btf_cnt; 1390 u32 max_ctx_offset; 1391 u32 max_pkt_offset; 1392 u32 max_tp_access; 1393 u32 stack_depth; 1394 u32 id; 1395 u32 func_cnt; /* used by non-func prog as the number of func progs */ 1396 u32 real_func_cnt; /* includes hidden progs, only used for JIT and freeing progs */ 1397 u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */ 1398 u32 attach_btf_id; /* in-kernel BTF type id to attach to */ 1399 u32 ctx_arg_info_size; 1400 u32 max_rdonly_access; 1401 u32 max_rdwr_access; 1402 struct btf *attach_btf; 1403 const struct bpf_ctx_arg_aux *ctx_arg_info; 1404 struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */ 1405 struct bpf_prog *dst_prog; 1406 struct bpf_trampoline *dst_trampoline; 1407 enum bpf_prog_type saved_dst_prog_type; 1408 enum bpf_attach_type saved_dst_attach_type; 1409 bool verifier_zext; /* Zero extensions has been inserted by verifier. */ 1410 bool dev_bound; /* Program is bound to the netdev. */ 1411 bool offload_requested; /* Program is bound and offloaded to the netdev. */ 1412 bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ 1413 bool func_proto_unreliable; 1414 bool sleepable; 1415 bool tail_call_reachable; 1416 bool xdp_has_frags; 1417 bool exception_cb; 1418 bool exception_boundary; 1419 /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ 1420 const struct btf_type *attach_func_proto; 1421 /* function name for valid attach_btf_id */ 1422 const char *attach_func_name; 1423 struct bpf_prog **func; 1424 void *jit_data; /* JIT specific data. arch dependent */ 1425 struct bpf_jit_poke_descriptor *poke_tab; 1426 struct bpf_kfunc_desc_tab *kfunc_tab; 1427 struct bpf_kfunc_btf_tab *kfunc_btf_tab; 1428 u32 size_poke_tab; 1429 struct bpf_ksym ksym; 1430 const struct bpf_prog_ops *ops; 1431 struct bpf_map **used_maps; 1432 struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */ 1433 struct btf_mod_pair *used_btfs; 1434 struct bpf_prog *prog; 1435 struct user_struct *user; 1436 u64 load_time; /* ns since boottime */ 1437 u32 verified_insns; 1438 int cgroup_atype; /* enum cgroup_bpf_attach_type */ 1439 struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 1440 char name[BPF_OBJ_NAME_LEN]; 1441 unsigned int (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp); 1442 #ifdef CONFIG_SECURITY 1443 void *security; 1444 #endif 1445 struct bpf_prog_offload *offload; 1446 struct btf *btf; 1447 struct bpf_func_info *func_info; 1448 struct bpf_func_info_aux *func_info_aux; 1449 /* bpf_line_info loaded from userspace. linfo->insn_off 1450 * has the xlated insn offset. 1451 * Both the main and sub prog share the same linfo. 1452 * The subprog can access its first linfo by 1453 * using the linfo_idx. 1454 */ 1455 struct bpf_line_info *linfo; 1456 /* jited_linfo is the jited addr of the linfo. It has a 1457 * one to one mapping to linfo: 1458 * jited_linfo[i] is the jited addr for the linfo[i]->insn_off. 1459 * Both the main and sub prog share the same jited_linfo. 1460 * The subprog can access its first jited_linfo by 1461 * using the linfo_idx. 1462 */ 1463 void **jited_linfo; 1464 u32 func_info_cnt; 1465 u32 nr_linfo; 1466 /* subprog can use linfo_idx to access its first linfo and 1467 * jited_linfo. 1468 * main prog always has linfo_idx == 0 1469 */ 1470 u32 linfo_idx; 1471 struct module *mod; 1472 u32 num_exentries; 1473 struct exception_table_entry *extable; 1474 union { 1475 struct work_struct work; 1476 struct rcu_head rcu; 1477 }; 1478 }; 1479 1480 struct bpf_prog { 1481 u16 pages; /* Number of allocated pages */ 1482 u16 jited:1, /* Is our filter JIT'ed? */ 1483 jit_requested:1,/* archs need to JIT the prog */ 1484 gpl_compatible:1, /* Is filter GPL compatible? */ 1485 cb_access:1, /* Is control block accessed? */ 1486 dst_needed:1, /* Do we need dst entry? */ 1487 blinding_requested:1, /* needs constant blinding */ 1488 blinded:1, /* Was blinded */ 1489 is_func:1, /* program is a bpf function */ 1490 kprobe_override:1, /* Do we override a kprobe? */ 1491 has_callchain_buf:1, /* callchain buffer allocated? */ 1492 enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */ 1493 call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */ 1494 call_get_func_ip:1, /* Do we call get_func_ip() */ 1495 tstamp_type_access:1; /* Accessed __sk_buff->tstamp_type */ 1496 enum bpf_prog_type type; /* Type of BPF program */ 1497 enum bpf_attach_type expected_attach_type; /* For some prog types */ 1498 u32 len; /* Number of filter blocks */ 1499 u32 jited_len; /* Size of jited insns in bytes */ 1500 u8 tag[BPF_TAG_SIZE]; 1501 struct bpf_prog_stats __percpu *stats; 1502 int __percpu *active; 1503 unsigned int (*bpf_func)(const void *ctx, 1504 const struct bpf_insn *insn); 1505 struct bpf_prog_aux *aux; /* Auxiliary fields */ 1506 struct sock_fprog_kern *orig_prog; /* Original BPF program */ 1507 /* Instructions for interpreter */ 1508 union { 1509 DECLARE_FLEX_ARRAY(struct sock_filter, insns); 1510 DECLARE_FLEX_ARRAY(struct bpf_insn, insnsi); 1511 }; 1512 }; 1513 1514 struct bpf_array_aux { 1515 /* Programs with direct jumps into programs part of this array. */ 1516 struct list_head poke_progs; 1517 struct bpf_map *map; 1518 struct mutex poke_mutex; 1519 struct work_struct work; 1520 }; 1521 1522 struct bpf_link { 1523 atomic64_t refcnt; 1524 u32 id; 1525 enum bpf_link_type type; 1526 const struct bpf_link_ops *ops; 1527 struct bpf_prog *prog; 1528 struct work_struct work; 1529 }; 1530 1531 struct bpf_link_ops { 1532 void (*release)(struct bpf_link *link); 1533 void (*dealloc)(struct bpf_link *link); 1534 int (*detach)(struct bpf_link *link); 1535 int (*update_prog)(struct bpf_link *link, struct bpf_prog *new_prog, 1536 struct bpf_prog *old_prog); 1537 void (*show_fdinfo)(const struct bpf_link *link, struct seq_file *seq); 1538 int (*fill_link_info)(const struct bpf_link *link, 1539 struct bpf_link_info *info); 1540 int (*update_map)(struct bpf_link *link, struct bpf_map *new_map, 1541 struct bpf_map *old_map); 1542 }; 1543 1544 struct bpf_tramp_link { 1545 struct bpf_link link; 1546 struct hlist_node tramp_hlist; 1547 u64 cookie; 1548 }; 1549 1550 struct bpf_shim_tramp_link { 1551 struct bpf_tramp_link link; 1552 struct bpf_trampoline *trampoline; 1553 }; 1554 1555 struct bpf_tracing_link { 1556 struct bpf_tramp_link link; 1557 enum bpf_attach_type attach_type; 1558 struct bpf_trampoline *trampoline; 1559 struct bpf_prog *tgt_prog; 1560 }; 1561 1562 struct bpf_link_primer { 1563 struct bpf_link *link; 1564 struct file *file; 1565 int fd; 1566 u32 id; 1567 }; 1568 1569 struct bpf_struct_ops_value; 1570 struct btf_member; 1571 1572 #define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64 1573 /** 1574 * struct bpf_struct_ops - A structure of callbacks allowing a subsystem to 1575 * define a BPF_MAP_TYPE_STRUCT_OPS map type composed 1576 * of BPF_PROG_TYPE_STRUCT_OPS progs. 1577 * @verifier_ops: A structure of callbacks that are invoked by the verifier 1578 * when determining whether the struct_ops progs in the 1579 * struct_ops map are valid. 1580 * @init: A callback that is invoked a single time, and before any other 1581 * callback, to initialize the structure. A nonzero return value means 1582 * the subsystem could not be initialized. 1583 * @check_member: When defined, a callback invoked by the verifier to allow 1584 * the subsystem to determine if an entry in the struct_ops map 1585 * is valid. A nonzero return value means that the map is 1586 * invalid and should be rejected by the verifier. 1587 * @init_member: A callback that is invoked for each member of the struct_ops 1588 * map to allow the subsystem to initialize the member. A nonzero 1589 * value means the member could not be initialized. This callback 1590 * is exclusive with the @type, @type_id, @value_type, and 1591 * @value_id fields. 1592 * @reg: A callback that is invoked when the struct_ops map has been 1593 * initialized and is being attached to. Zero means the struct_ops map 1594 * has been successfully registered and is live. A nonzero return value 1595 * means the struct_ops map could not be registered. 1596 * @unreg: A callback that is invoked when the struct_ops map should be 1597 * unregistered. 1598 * @update: A callback that is invoked when the live struct_ops map is being 1599 * updated to contain new values. This callback is only invoked when 1600 * the struct_ops map is loaded with BPF_F_LINK. If not defined, the 1601 * it is assumed that the struct_ops map cannot be updated. 1602 * @validate: A callback that is invoked after all of the members have been 1603 * initialized. This callback should perform static checks on the 1604 * map, meaning that it should either fail or succeed 1605 * deterministically. A struct_ops map that has been validated may 1606 * not necessarily succeed in being registered if the call to @reg 1607 * fails. For example, a valid struct_ops map may be loaded, but 1608 * then fail to be registered due to there being another active 1609 * struct_ops map on the system in the subsystem already. For this 1610 * reason, if this callback is not defined, the check is skipped as 1611 * the struct_ops map will have final verification performed in 1612 * @reg. 1613 * @type: BTF type. 1614 * @value_type: Value type. 1615 * @name: The name of the struct bpf_struct_ops object. 1616 * @func_models: Func models 1617 * @type_id: BTF type id. 1618 * @value_id: BTF value id. 1619 */ 1620 struct bpf_struct_ops { 1621 const struct bpf_verifier_ops *verifier_ops; 1622 int (*init)(struct btf *btf); 1623 int (*check_member)(const struct btf_type *t, 1624 const struct btf_member *member, 1625 const struct bpf_prog *prog); 1626 int (*init_member)(const struct btf_type *t, 1627 const struct btf_member *member, 1628 void *kdata, const void *udata); 1629 int (*reg)(void *kdata); 1630 void (*unreg)(void *kdata); 1631 int (*update)(void *kdata, void *old_kdata); 1632 int (*validate)(void *kdata); 1633 const struct btf_type *type; 1634 const struct btf_type *value_type; 1635 const char *name; 1636 struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; 1637 u32 type_id; 1638 u32 value_id; 1639 }; 1640 1641 #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) 1642 #define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA)) 1643 const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id); 1644 void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log); 1645 bool bpf_struct_ops_get(const void *kdata); 1646 void bpf_struct_ops_put(const void *kdata); 1647 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, 1648 void *value); 1649 int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, 1650 struct bpf_tramp_link *link, 1651 const struct btf_func_model *model, 1652 void *image, void *image_end); 1653 static inline bool bpf_try_module_get(const void *data, struct module *owner) 1654 { 1655 if (owner == BPF_MODULE_OWNER) 1656 return bpf_struct_ops_get(data); 1657 else 1658 return try_module_get(owner); 1659 } 1660 static inline void bpf_module_put(const void *data, struct module *owner) 1661 { 1662 if (owner == BPF_MODULE_OWNER) 1663 bpf_struct_ops_put(data); 1664 else 1665 module_put(owner); 1666 } 1667 int bpf_struct_ops_link_create(union bpf_attr *attr); 1668 1669 #ifdef CONFIG_NET 1670 /* Define it here to avoid the use of forward declaration */ 1671 struct bpf_dummy_ops_state { 1672 int val; 1673 }; 1674 1675 struct bpf_dummy_ops { 1676 int (*test_1)(struct bpf_dummy_ops_state *cb); 1677 int (*test_2)(struct bpf_dummy_ops_state *cb, int a1, unsigned short a2, 1678 char a3, unsigned long a4); 1679 int (*test_sleepable)(struct bpf_dummy_ops_state *cb); 1680 }; 1681 1682 int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, 1683 union bpf_attr __user *uattr); 1684 #endif 1685 #else 1686 static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id) 1687 { 1688 return NULL; 1689 } 1690 static inline void bpf_struct_ops_init(struct btf *btf, 1691 struct bpf_verifier_log *log) 1692 { 1693 } 1694 static inline bool bpf_try_module_get(const void *data, struct module *owner) 1695 { 1696 return try_module_get(owner); 1697 } 1698 static inline void bpf_module_put(const void *data, struct module *owner) 1699 { 1700 module_put(owner); 1701 } 1702 static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, 1703 void *key, 1704 void *value) 1705 { 1706 return -EINVAL; 1707 } 1708 static inline int bpf_struct_ops_link_create(union bpf_attr *attr) 1709 { 1710 return -EOPNOTSUPP; 1711 } 1712 1713 #endif 1714 1715 #if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM) 1716 int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, 1717 int cgroup_atype); 1718 void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog); 1719 #else 1720 static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, 1721 int cgroup_atype) 1722 { 1723 return -EOPNOTSUPP; 1724 } 1725 static inline void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog) 1726 { 1727 } 1728 #endif 1729 1730 struct bpf_array { 1731 struct bpf_map map; 1732 u32 elem_size; 1733 u32 index_mask; 1734 struct bpf_array_aux *aux; 1735 union { 1736 DECLARE_FLEX_ARRAY(char, value) __aligned(8); 1737 DECLARE_FLEX_ARRAY(void *, ptrs) __aligned(8); 1738 DECLARE_FLEX_ARRAY(void __percpu *, pptrs) __aligned(8); 1739 }; 1740 }; 1741 1742 #define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */ 1743 #define MAX_TAIL_CALL_CNT 33 1744 1745 /* Maximum number of loops for bpf_loop and bpf_iter_num. 1746 * It's enum to expose it (and thus make it discoverable) through BTF. 1747 */ 1748 enum { 1749 BPF_MAX_LOOPS = 8 * 1024 * 1024, 1750 }; 1751 1752 #define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \ 1753 BPF_F_RDONLY_PROG | \ 1754 BPF_F_WRONLY | \ 1755 BPF_F_WRONLY_PROG) 1756 1757 #define BPF_MAP_CAN_READ BIT(0) 1758 #define BPF_MAP_CAN_WRITE BIT(1) 1759 1760 /* Maximum number of user-producer ring buffer samples that can be drained in 1761 * a call to bpf_user_ringbuf_drain(). 1762 */ 1763 #define BPF_MAX_USER_RINGBUF_SAMPLES (128 * 1024) 1764 1765 static inline u32 bpf_map_flags_to_cap(struct bpf_map *map) 1766 { 1767 u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); 1768 1769 /* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is 1770 * not possible. 1771 */ 1772 if (access_flags & BPF_F_RDONLY_PROG) 1773 return BPF_MAP_CAN_READ; 1774 else if (access_flags & BPF_F_WRONLY_PROG) 1775 return BPF_MAP_CAN_WRITE; 1776 else 1777 return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE; 1778 } 1779 1780 static inline bool bpf_map_flags_access_ok(u32 access_flags) 1781 { 1782 return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) != 1783 (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); 1784 } 1785 1786 struct bpf_event_entry { 1787 struct perf_event *event; 1788 struct file *perf_file; 1789 struct file *map_file; 1790 struct rcu_head rcu; 1791 }; 1792 1793 static inline bool map_type_contains_progs(struct bpf_map *map) 1794 { 1795 return map->map_type == BPF_MAP_TYPE_PROG_ARRAY || 1796 map->map_type == BPF_MAP_TYPE_DEVMAP || 1797 map->map_type == BPF_MAP_TYPE_CPUMAP; 1798 } 1799 1800 bool bpf_prog_map_compatible(struct bpf_map *map, const struct bpf_prog *fp); 1801 int bpf_prog_calc_tag(struct bpf_prog *fp); 1802 1803 const struct bpf_func_proto *bpf_get_trace_printk_proto(void); 1804 const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void); 1805 1806 typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src, 1807 unsigned long off, unsigned long len); 1808 typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type, 1809 const struct bpf_insn *src, 1810 struct bpf_insn *dst, 1811 struct bpf_prog *prog, 1812 u32 *target_size); 1813 1814 u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, 1815 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy); 1816 1817 /* an array of programs to be executed under rcu_lock. 1818 * 1819 * Typical usage: 1820 * ret = bpf_prog_run_array(rcu_dereference(&bpf_prog_array), ctx, bpf_prog_run); 1821 * 1822 * the structure returned by bpf_prog_array_alloc() should be populated 1823 * with program pointers and the last pointer must be NULL. 1824 * The user has to keep refcnt on the program and make sure the program 1825 * is removed from the array before bpf_prog_put(). 1826 * The 'struct bpf_prog_array *' should only be replaced with xchg() 1827 * since other cpus are walking the array of pointers in parallel. 1828 */ 1829 struct bpf_prog_array_item { 1830 struct bpf_prog *prog; 1831 union { 1832 struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 1833 u64 bpf_cookie; 1834 }; 1835 }; 1836 1837 struct bpf_prog_array { 1838 struct rcu_head rcu; 1839 struct bpf_prog_array_item items[]; 1840 }; 1841 1842 struct bpf_empty_prog_array { 1843 struct bpf_prog_array hdr; 1844 struct bpf_prog *null_prog; 1845 }; 1846 1847 /* to avoid allocating empty bpf_prog_array for cgroups that 1848 * don't have bpf program attached use one global 'bpf_empty_prog_array' 1849 * It will not be modified the caller of bpf_prog_array_alloc() 1850 * (since caller requested prog_cnt == 0) 1851 * that pointer should be 'freed' by bpf_prog_array_free() 1852 */ 1853 extern struct bpf_empty_prog_array bpf_empty_prog_array; 1854 1855 struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags); 1856 void bpf_prog_array_free(struct bpf_prog_array *progs); 1857 /* Use when traversal over the bpf_prog_array uses tasks_trace rcu */ 1858 void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs); 1859 int bpf_prog_array_length(struct bpf_prog_array *progs); 1860 bool bpf_prog_array_is_empty(struct bpf_prog_array *array); 1861 int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs, 1862 __u32 __user *prog_ids, u32 cnt); 1863 1864 void bpf_prog_array_delete_safe(struct bpf_prog_array *progs, 1865 struct bpf_prog *old_prog); 1866 int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index); 1867 int bpf_prog_array_update_at(struct bpf_prog_array *array, int index, 1868 struct bpf_prog *prog); 1869 int bpf_prog_array_copy_info(struct bpf_prog_array *array, 1870 u32 *prog_ids, u32 request_cnt, 1871 u32 *prog_cnt); 1872 int bpf_prog_array_copy(struct bpf_prog_array *old_array, 1873 struct bpf_prog *exclude_prog, 1874 struct bpf_prog *include_prog, 1875 u64 bpf_cookie, 1876 struct bpf_prog_array **new_array); 1877 1878 struct bpf_run_ctx {}; 1879 1880 struct bpf_cg_run_ctx { 1881 struct bpf_run_ctx run_ctx; 1882 const struct bpf_prog_array_item *prog_item; 1883 int retval; 1884 }; 1885 1886 struct bpf_trace_run_ctx { 1887 struct bpf_run_ctx run_ctx; 1888 u64 bpf_cookie; 1889 bool is_uprobe; 1890 }; 1891 1892 struct bpf_tramp_run_ctx { 1893 struct bpf_run_ctx run_ctx; 1894 u64 bpf_cookie; 1895 struct bpf_run_ctx *saved_run_ctx; 1896 }; 1897 1898 static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx) 1899 { 1900 struct bpf_run_ctx *old_ctx = NULL; 1901 1902 #ifdef CONFIG_BPF_SYSCALL 1903 old_ctx = current->bpf_ctx; 1904 current->bpf_ctx = new_ctx; 1905 #endif 1906 return old_ctx; 1907 } 1908 1909 static inline void bpf_reset_run_ctx(struct bpf_run_ctx *old_ctx) 1910 { 1911 #ifdef CONFIG_BPF_SYSCALL 1912 current->bpf_ctx = old_ctx; 1913 #endif 1914 } 1915 1916 /* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */ 1917 #define BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE (1 << 0) 1918 /* BPF program asks to set CN on the packet. */ 1919 #define BPF_RET_SET_CN (1 << 0) 1920 1921 typedef u32 (*bpf_prog_run_fn)(const struct bpf_prog *prog, const void *ctx); 1922 1923 static __always_inline u32 1924 bpf_prog_run_array(const struct bpf_prog_array *array, 1925 const void *ctx, bpf_prog_run_fn run_prog) 1926 { 1927 const struct bpf_prog_array_item *item; 1928 const struct bpf_prog *prog; 1929 struct bpf_run_ctx *old_run_ctx; 1930 struct bpf_trace_run_ctx run_ctx; 1931 u32 ret = 1; 1932 1933 RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "no rcu lock held"); 1934 1935 if (unlikely(!array)) 1936 return ret; 1937 1938 run_ctx.is_uprobe = false; 1939 1940 migrate_disable(); 1941 old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); 1942 item = &array->items[0]; 1943 while ((prog = READ_ONCE(item->prog))) { 1944 run_ctx.bpf_cookie = item->bpf_cookie; 1945 ret &= run_prog(prog, ctx); 1946 item++; 1947 } 1948 bpf_reset_run_ctx(old_run_ctx); 1949 migrate_enable(); 1950 return ret; 1951 } 1952 1953 /* Notes on RCU design for bpf_prog_arrays containing sleepable programs: 1954 * 1955 * We use the tasks_trace rcu flavor read section to protect the bpf_prog_array 1956 * overall. As a result, we must use the bpf_prog_array_free_sleepable 1957 * in order to use the tasks_trace rcu grace period. 1958 * 1959 * When a non-sleepable program is inside the array, we take the rcu read 1960 * section and disable preemption for that program alone, so it can access 1961 * rcu-protected dynamically sized maps. 1962 */ 1963 static __always_inline u32 1964 bpf_prog_run_array_uprobe(const struct bpf_prog_array __rcu *array_rcu, 1965 const void *ctx, bpf_prog_run_fn run_prog) 1966 { 1967 const struct bpf_prog_array_item *item; 1968 const struct bpf_prog *prog; 1969 const struct bpf_prog_array *array; 1970 struct bpf_run_ctx *old_run_ctx; 1971 struct bpf_trace_run_ctx run_ctx; 1972 u32 ret = 1; 1973 1974 might_fault(); 1975 1976 rcu_read_lock_trace(); 1977 migrate_disable(); 1978 1979 run_ctx.is_uprobe = true; 1980 1981 array = rcu_dereference_check(array_rcu, rcu_read_lock_trace_held()); 1982 if (unlikely(!array)) 1983 goto out; 1984 old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); 1985 item = &array->items[0]; 1986 while ((prog = READ_ONCE(item->prog))) { 1987 if (!prog->aux->sleepable) 1988 rcu_read_lock(); 1989 1990 run_ctx.bpf_cookie = item->bpf_cookie; 1991 ret &= run_prog(prog, ctx); 1992 item++; 1993 1994 if (!prog->aux->sleepable) 1995 rcu_read_unlock(); 1996 } 1997 bpf_reset_run_ctx(old_run_ctx); 1998 out: 1999 migrate_enable(); 2000 rcu_read_unlock_trace(); 2001 return ret; 2002 } 2003 2004 #ifdef CONFIG_BPF_SYSCALL 2005 DECLARE_PER_CPU(int, bpf_prog_active); 2006 extern struct mutex bpf_stats_enabled_mutex; 2007 2008 /* 2009 * Block execution of BPF programs attached to instrumentation (perf, 2010 * kprobes, tracepoints) to prevent deadlocks on map operations as any of 2011 * these events can happen inside a region which holds a map bucket lock 2012 * and can deadlock on it. 2013 */ 2014 static inline void bpf_disable_instrumentation(void) 2015 { 2016 migrate_disable(); 2017 this_cpu_inc(bpf_prog_active); 2018 } 2019 2020 static inline void bpf_enable_instrumentation(void) 2021 { 2022 this_cpu_dec(bpf_prog_active); 2023 migrate_enable(); 2024 } 2025 2026 extern const struct file_operations bpf_map_fops; 2027 extern const struct file_operations bpf_prog_fops; 2028 extern const struct file_operations bpf_iter_fops; 2029 2030 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 2031 extern const struct bpf_prog_ops _name ## _prog_ops; \ 2032 extern const struct bpf_verifier_ops _name ## _verifier_ops; 2033 #define BPF_MAP_TYPE(_id, _ops) \ 2034 extern const struct bpf_map_ops _ops; 2035 #define BPF_LINK_TYPE(_id, _name) 2036 #include <linux/bpf_types.h> 2037 #undef BPF_PROG_TYPE 2038 #undef BPF_MAP_TYPE 2039 #undef BPF_LINK_TYPE 2040 2041 extern const struct bpf_prog_ops bpf_offload_prog_ops; 2042 extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops; 2043 extern const struct bpf_verifier_ops xdp_analyzer_ops; 2044 2045 struct bpf_prog *bpf_prog_get(u32 ufd); 2046 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 2047 bool attach_drv); 2048 void bpf_prog_add(struct bpf_prog *prog, int i); 2049 void bpf_prog_sub(struct bpf_prog *prog, int i); 2050 void bpf_prog_inc(struct bpf_prog *prog); 2051 struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog); 2052 void bpf_prog_put(struct bpf_prog *prog); 2053 2054 void bpf_prog_free_id(struct bpf_prog *prog); 2055 void bpf_map_free_id(struct bpf_map *map); 2056 2057 struct btf_field *btf_record_find(const struct btf_record *rec, 2058 u32 offset, u32 field_mask); 2059 void btf_record_free(struct btf_record *rec); 2060 void bpf_map_free_record(struct bpf_map *map); 2061 struct btf_record *btf_record_dup(const struct btf_record *rec); 2062 bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b); 2063 void bpf_obj_free_timer(const struct btf_record *rec, void *obj); 2064 void bpf_obj_free_fields(const struct btf_record *rec, void *obj); 2065 void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu); 2066 2067 struct bpf_map *bpf_map_get(u32 ufd); 2068 struct bpf_map *bpf_map_get_with_uref(u32 ufd); 2069 struct bpf_map *__bpf_map_get(struct fd f); 2070 void bpf_map_inc(struct bpf_map *map); 2071 void bpf_map_inc_with_uref(struct bpf_map *map); 2072 struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref); 2073 struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map); 2074 void bpf_map_put_with_uref(struct bpf_map *map); 2075 void bpf_map_put(struct bpf_map *map); 2076 void *bpf_map_area_alloc(u64 size, int numa_node); 2077 void *bpf_map_area_mmapable_alloc(u64 size, int numa_node); 2078 void bpf_map_area_free(void *base); 2079 bool bpf_map_write_active(const struct bpf_map *map); 2080 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr); 2081 int generic_map_lookup_batch(struct bpf_map *map, 2082 const union bpf_attr *attr, 2083 union bpf_attr __user *uattr); 2084 int generic_map_update_batch(struct bpf_map *map, struct file *map_file, 2085 const union bpf_attr *attr, 2086 union bpf_attr __user *uattr); 2087 int generic_map_delete_batch(struct bpf_map *map, 2088 const union bpf_attr *attr, 2089 union bpf_attr __user *uattr); 2090 struct bpf_map *bpf_map_get_curr_or_next(u32 *id); 2091 struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id); 2092 2093 #ifdef CONFIG_MEMCG_KMEM 2094 void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, 2095 int node); 2096 void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags); 2097 void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size, 2098 gfp_t flags); 2099 void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, 2100 size_t align, gfp_t flags); 2101 #else 2102 static inline void * 2103 bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, 2104 int node) 2105 { 2106 return kmalloc_node(size, flags, node); 2107 } 2108 2109 static inline void * 2110 bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags) 2111 { 2112 return kzalloc(size, flags); 2113 } 2114 2115 static inline void * 2116 bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size, gfp_t flags) 2117 { 2118 return kvcalloc(n, size, flags); 2119 } 2120 2121 static inline void __percpu * 2122 bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, size_t align, 2123 gfp_t flags) 2124 { 2125 return __alloc_percpu_gfp(size, align, flags); 2126 } 2127 #endif 2128 2129 static inline int 2130 bpf_map_init_elem_count(struct bpf_map *map) 2131 { 2132 size_t size = sizeof(*map->elem_count), align = size; 2133 gfp_t flags = GFP_USER | __GFP_NOWARN; 2134 2135 map->elem_count = bpf_map_alloc_percpu(map, size, align, flags); 2136 if (!map->elem_count) 2137 return -ENOMEM; 2138 2139 return 0; 2140 } 2141 2142 static inline void 2143 bpf_map_free_elem_count(struct bpf_map *map) 2144 { 2145 free_percpu(map->elem_count); 2146 } 2147 2148 static inline void bpf_map_inc_elem_count(struct bpf_map *map) 2149 { 2150 this_cpu_inc(*map->elem_count); 2151 } 2152 2153 static inline void bpf_map_dec_elem_count(struct bpf_map *map) 2154 { 2155 this_cpu_dec(*map->elem_count); 2156 } 2157 2158 extern int sysctl_unprivileged_bpf_disabled; 2159 2160 static inline bool bpf_allow_ptr_leaks(void) 2161 { 2162 return perfmon_capable(); 2163 } 2164 2165 static inline bool bpf_allow_uninit_stack(void) 2166 { 2167 return perfmon_capable(); 2168 } 2169 2170 static inline bool bpf_bypass_spec_v1(void) 2171 { 2172 return cpu_mitigations_off() || perfmon_capable(); 2173 } 2174 2175 static inline bool bpf_bypass_spec_v4(void) 2176 { 2177 return cpu_mitigations_off() || perfmon_capable(); 2178 } 2179 2180 int bpf_map_new_fd(struct bpf_map *map, int flags); 2181 int bpf_prog_new_fd(struct bpf_prog *prog); 2182 2183 void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, 2184 const struct bpf_link_ops *ops, struct bpf_prog *prog); 2185 int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer); 2186 int bpf_link_settle(struct bpf_link_primer *primer); 2187 void bpf_link_cleanup(struct bpf_link_primer *primer); 2188 void bpf_link_inc(struct bpf_link *link); 2189 void bpf_link_put(struct bpf_link *link); 2190 int bpf_link_new_fd(struct bpf_link *link); 2191 struct bpf_link *bpf_link_get_from_fd(u32 ufd); 2192 struct bpf_link *bpf_link_get_curr_or_next(u32 *id); 2193 2194 int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname); 2195 int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags); 2196 2197 #define BPF_ITER_FUNC_PREFIX "bpf_iter_" 2198 #define DEFINE_BPF_ITER_FUNC(target, args...) \ 2199 extern int bpf_iter_ ## target(args); \ 2200 int __init bpf_iter_ ## target(args) { return 0; } 2201 2202 /* 2203 * The task type of iterators. 2204 * 2205 * For BPF task iterators, they can be parameterized with various 2206 * parameters to visit only some of tasks. 2207 * 2208 * BPF_TASK_ITER_ALL (default) 2209 * Iterate over resources of every task. 2210 * 2211 * BPF_TASK_ITER_TID 2212 * Iterate over resources of a task/tid. 2213 * 2214 * BPF_TASK_ITER_TGID 2215 * Iterate over resources of every task of a process / task group. 2216 */ 2217 enum bpf_iter_task_type { 2218 BPF_TASK_ITER_ALL = 0, 2219 BPF_TASK_ITER_TID, 2220 BPF_TASK_ITER_TGID, 2221 }; 2222 2223 struct bpf_iter_aux_info { 2224 /* for map_elem iter */ 2225 struct bpf_map *map; 2226 2227 /* for cgroup iter */ 2228 struct { 2229 struct cgroup *start; /* starting cgroup */ 2230 enum bpf_cgroup_iter_order order; 2231 } cgroup; 2232 struct { 2233 enum bpf_iter_task_type type; 2234 u32 pid; 2235 } task; 2236 }; 2237 2238 typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog, 2239 union bpf_iter_link_info *linfo, 2240 struct bpf_iter_aux_info *aux); 2241 typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux); 2242 typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux, 2243 struct seq_file *seq); 2244 typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux, 2245 struct bpf_link_info *info); 2246 typedef const struct bpf_func_proto * 2247 (*bpf_iter_get_func_proto_t)(enum bpf_func_id func_id, 2248 const struct bpf_prog *prog); 2249 2250 enum bpf_iter_feature { 2251 BPF_ITER_RESCHED = BIT(0), 2252 }; 2253 2254 #define BPF_ITER_CTX_ARG_MAX 2 2255 struct bpf_iter_reg { 2256 const char *target; 2257 bpf_iter_attach_target_t attach_target; 2258 bpf_iter_detach_target_t detach_target; 2259 bpf_iter_show_fdinfo_t show_fdinfo; 2260 bpf_iter_fill_link_info_t fill_link_info; 2261 bpf_iter_get_func_proto_t get_func_proto; 2262 u32 ctx_arg_info_size; 2263 u32 feature; 2264 struct bpf_ctx_arg_aux ctx_arg_info[BPF_ITER_CTX_ARG_MAX]; 2265 const struct bpf_iter_seq_info *seq_info; 2266 }; 2267 2268 struct bpf_iter_meta { 2269 __bpf_md_ptr(struct seq_file *, seq); 2270 u64 session_id; 2271 u64 seq_num; 2272 }; 2273 2274 struct bpf_iter__bpf_map_elem { 2275 __bpf_md_ptr(struct bpf_iter_meta *, meta); 2276 __bpf_md_ptr(struct bpf_map *, map); 2277 __bpf_md_ptr(void *, key); 2278 __bpf_md_ptr(void *, value); 2279 }; 2280 2281 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info); 2282 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info); 2283 bool bpf_iter_prog_supported(struct bpf_prog *prog); 2284 const struct bpf_func_proto * 2285 bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); 2286 int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_prog *prog); 2287 int bpf_iter_new_fd(struct bpf_link *link); 2288 bool bpf_link_is_iter(struct bpf_link *link); 2289 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop); 2290 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx); 2291 void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux, 2292 struct seq_file *seq); 2293 int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux, 2294 struct bpf_link_info *info); 2295 2296 int map_set_for_each_callback_args(struct bpf_verifier_env *env, 2297 struct bpf_func_state *caller, 2298 struct bpf_func_state *callee); 2299 2300 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value); 2301 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value); 2302 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, 2303 u64 flags); 2304 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, 2305 u64 flags); 2306 2307 int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value); 2308 2309 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, 2310 void *key, void *value, u64 map_flags); 2311 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); 2312 int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, 2313 void *key, void *value, u64 map_flags); 2314 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); 2315 2316 int bpf_get_file_flag(int flags); 2317 int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size, 2318 size_t actual_size); 2319 2320 /* verify correctness of eBPF program */ 2321 int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size); 2322 2323 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 2324 void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); 2325 #endif 2326 2327 struct btf *bpf_get_btf_vmlinux(void); 2328 2329 /* Map specifics */ 2330 struct xdp_frame; 2331 struct sk_buff; 2332 struct bpf_dtab_netdev; 2333 struct bpf_cpu_map_entry; 2334 2335 void __dev_flush(void); 2336 int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf, 2337 struct net_device *dev_rx); 2338 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf, 2339 struct net_device *dev_rx); 2340 int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx, 2341 struct bpf_map *map, bool exclude_ingress); 2342 int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, 2343 struct bpf_prog *xdp_prog); 2344 int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, 2345 struct bpf_prog *xdp_prog, struct bpf_map *map, 2346 bool exclude_ingress); 2347 2348 void __cpu_map_flush(void); 2349 int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf, 2350 struct net_device *dev_rx); 2351 int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu, 2352 struct sk_buff *skb); 2353 2354 /* Return map's numa specified by userspace */ 2355 static inline int bpf_map_attr_numa_node(const union bpf_attr *attr) 2356 { 2357 return (attr->map_flags & BPF_F_NUMA_NODE) ? 2358 attr->numa_node : NUMA_NO_NODE; 2359 } 2360 2361 struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type); 2362 int array_map_alloc_check(union bpf_attr *attr); 2363 2364 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 2365 union bpf_attr __user *uattr); 2366 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 2367 union bpf_attr __user *uattr); 2368 int bpf_prog_test_run_tracing(struct bpf_prog *prog, 2369 const union bpf_attr *kattr, 2370 union bpf_attr __user *uattr); 2371 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 2372 const union bpf_attr *kattr, 2373 union bpf_attr __user *uattr); 2374 int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, 2375 const union bpf_attr *kattr, 2376 union bpf_attr __user *uattr); 2377 int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, 2378 const union bpf_attr *kattr, 2379 union bpf_attr __user *uattr); 2380 int bpf_prog_test_run_nf(struct bpf_prog *prog, 2381 const union bpf_attr *kattr, 2382 union bpf_attr __user *uattr); 2383 bool btf_ctx_access(int off, int size, enum bpf_access_type type, 2384 const struct bpf_prog *prog, 2385 struct bpf_insn_access_aux *info); 2386 2387 static inline bool bpf_tracing_ctx_access(int off, int size, 2388 enum bpf_access_type type) 2389 { 2390 if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) 2391 return false; 2392 if (type != BPF_READ) 2393 return false; 2394 if (off % size != 0) 2395 return false; 2396 return true; 2397 } 2398 2399 static inline bool bpf_tracing_btf_ctx_access(int off, int size, 2400 enum bpf_access_type type, 2401 const struct bpf_prog *prog, 2402 struct bpf_insn_access_aux *info) 2403 { 2404 if (!bpf_tracing_ctx_access(off, size, type)) 2405 return false; 2406 return btf_ctx_access(off, size, type, prog, info); 2407 } 2408 2409 int btf_struct_access(struct bpf_verifier_log *log, 2410 const struct bpf_reg_state *reg, 2411 int off, int size, enum bpf_access_type atype, 2412 u32 *next_btf_id, enum bpf_type_flag *flag, const char **field_name); 2413 bool btf_struct_ids_match(struct bpf_verifier_log *log, 2414 const struct btf *btf, u32 id, int off, 2415 const struct btf *need_btf, u32 need_type_id, 2416 bool strict); 2417 2418 int btf_distill_func_proto(struct bpf_verifier_log *log, 2419 struct btf *btf, 2420 const struct btf_type *func_proto, 2421 const char *func_name, 2422 struct btf_func_model *m); 2423 2424 struct bpf_reg_state; 2425 int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, 2426 struct bpf_reg_state *regs); 2427 int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, 2428 struct bpf_reg_state *regs); 2429 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, 2430 struct bpf_reg_state *reg, bool is_ex_cb); 2431 int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog, 2432 struct btf *btf, const struct btf_type *t); 2433 const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt, 2434 int comp_idx, const char *tag_key); 2435 2436 struct bpf_prog *bpf_prog_by_id(u32 id); 2437 struct bpf_link *bpf_link_by_id(u32 id); 2438 2439 const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id); 2440 void bpf_task_storage_free(struct task_struct *task); 2441 void bpf_cgrp_storage_free(struct cgroup *cgroup); 2442 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog); 2443 const struct btf_func_model * 2444 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 2445 const struct bpf_insn *insn); 2446 int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, 2447 u16 btf_fd_idx, u8 **func_addr); 2448 2449 struct bpf_core_ctx { 2450 struct bpf_verifier_log *log; 2451 const struct btf *btf; 2452 }; 2453 2454 bool btf_nested_type_is_trusted(struct bpf_verifier_log *log, 2455 const struct bpf_reg_state *reg, 2456 const char *field_name, u32 btf_id, const char *suffix); 2457 2458 bool btf_type_ids_nocast_alias(struct bpf_verifier_log *log, 2459 const struct btf *reg_btf, u32 reg_id, 2460 const struct btf *arg_btf, u32 arg_id); 2461 2462 int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo, 2463 int relo_idx, void *insn); 2464 2465 static inline bool unprivileged_ebpf_enabled(void) 2466 { 2467 return !sysctl_unprivileged_bpf_disabled; 2468 } 2469 2470 /* Not all bpf prog type has the bpf_ctx. 2471 * For the bpf prog type that has initialized the bpf_ctx, 2472 * this function can be used to decide if a kernel function 2473 * is called by a bpf program. 2474 */ 2475 static inline bool has_current_bpf_ctx(void) 2476 { 2477 return !!current->bpf_ctx; 2478 } 2479 2480 void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog); 2481 2482 void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data, 2483 enum bpf_dynptr_type type, u32 offset, u32 size); 2484 void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr); 2485 void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr); 2486 2487 bool dev_check_flush(void); 2488 bool cpu_map_check_flush(void); 2489 #else /* !CONFIG_BPF_SYSCALL */ 2490 static inline struct bpf_prog *bpf_prog_get(u32 ufd) 2491 { 2492 return ERR_PTR(-EOPNOTSUPP); 2493 } 2494 2495 static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, 2496 enum bpf_prog_type type, 2497 bool attach_drv) 2498 { 2499 return ERR_PTR(-EOPNOTSUPP); 2500 } 2501 2502 static inline void bpf_prog_add(struct bpf_prog *prog, int i) 2503 { 2504 } 2505 2506 static inline void bpf_prog_sub(struct bpf_prog *prog, int i) 2507 { 2508 } 2509 2510 static inline void bpf_prog_put(struct bpf_prog *prog) 2511 { 2512 } 2513 2514 static inline void bpf_prog_inc(struct bpf_prog *prog) 2515 { 2516 } 2517 2518 static inline struct bpf_prog *__must_check 2519 bpf_prog_inc_not_zero(struct bpf_prog *prog) 2520 { 2521 return ERR_PTR(-EOPNOTSUPP); 2522 } 2523 2524 static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, 2525 const struct bpf_link_ops *ops, 2526 struct bpf_prog *prog) 2527 { 2528 } 2529 2530 static inline int bpf_link_prime(struct bpf_link *link, 2531 struct bpf_link_primer *primer) 2532 { 2533 return -EOPNOTSUPP; 2534 } 2535 2536 static inline int bpf_link_settle(struct bpf_link_primer *primer) 2537 { 2538 return -EOPNOTSUPP; 2539 } 2540 2541 static inline void bpf_link_cleanup(struct bpf_link_primer *primer) 2542 { 2543 } 2544 2545 static inline void bpf_link_inc(struct bpf_link *link) 2546 { 2547 } 2548 2549 static inline void bpf_link_put(struct bpf_link *link) 2550 { 2551 } 2552 2553 static inline int bpf_obj_get_user(const char __user *pathname, int flags) 2554 { 2555 return -EOPNOTSUPP; 2556 } 2557 2558 static inline void __dev_flush(void) 2559 { 2560 } 2561 2562 struct xdp_frame; 2563 struct bpf_dtab_netdev; 2564 struct bpf_cpu_map_entry; 2565 2566 static inline 2567 int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf, 2568 struct net_device *dev_rx) 2569 { 2570 return 0; 2571 } 2572 2573 static inline 2574 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf, 2575 struct net_device *dev_rx) 2576 { 2577 return 0; 2578 } 2579 2580 static inline 2581 int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx, 2582 struct bpf_map *map, bool exclude_ingress) 2583 { 2584 return 0; 2585 } 2586 2587 struct sk_buff; 2588 2589 static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, 2590 struct sk_buff *skb, 2591 struct bpf_prog *xdp_prog) 2592 { 2593 return 0; 2594 } 2595 2596 static inline 2597 int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, 2598 struct bpf_prog *xdp_prog, struct bpf_map *map, 2599 bool exclude_ingress) 2600 { 2601 return 0; 2602 } 2603 2604 static inline void __cpu_map_flush(void) 2605 { 2606 } 2607 2608 static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, 2609 struct xdp_frame *xdpf, 2610 struct net_device *dev_rx) 2611 { 2612 return 0; 2613 } 2614 2615 static inline int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu, 2616 struct sk_buff *skb) 2617 { 2618 return -EOPNOTSUPP; 2619 } 2620 2621 static inline struct bpf_prog *bpf_prog_get_type_path(const char *name, 2622 enum bpf_prog_type type) 2623 { 2624 return ERR_PTR(-EOPNOTSUPP); 2625 } 2626 2627 static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog, 2628 const union bpf_attr *kattr, 2629 union bpf_attr __user *uattr) 2630 { 2631 return -ENOTSUPP; 2632 } 2633 2634 static inline int bpf_prog_test_run_skb(struct bpf_prog *prog, 2635 const union bpf_attr *kattr, 2636 union bpf_attr __user *uattr) 2637 { 2638 return -ENOTSUPP; 2639 } 2640 2641 static inline int bpf_prog_test_run_tracing(struct bpf_prog *prog, 2642 const union bpf_attr *kattr, 2643 union bpf_attr __user *uattr) 2644 { 2645 return -ENOTSUPP; 2646 } 2647 2648 static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 2649 const union bpf_attr *kattr, 2650 union bpf_attr __user *uattr) 2651 { 2652 return -ENOTSUPP; 2653 } 2654 2655 static inline int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, 2656 const union bpf_attr *kattr, 2657 union bpf_attr __user *uattr) 2658 { 2659 return -ENOTSUPP; 2660 } 2661 2662 static inline void bpf_map_put(struct bpf_map *map) 2663 { 2664 } 2665 2666 static inline struct bpf_prog *bpf_prog_by_id(u32 id) 2667 { 2668 return ERR_PTR(-ENOTSUPP); 2669 } 2670 2671 static inline int btf_struct_access(struct bpf_verifier_log *log, 2672 const struct bpf_reg_state *reg, 2673 int off, int size, enum bpf_access_type atype, 2674 u32 *next_btf_id, enum bpf_type_flag *flag, 2675 const char **field_name) 2676 { 2677 return -EACCES; 2678 } 2679 2680 static inline const struct bpf_func_proto * 2681 bpf_base_func_proto(enum bpf_func_id func_id) 2682 { 2683 return NULL; 2684 } 2685 2686 static inline void bpf_task_storage_free(struct task_struct *task) 2687 { 2688 } 2689 2690 static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) 2691 { 2692 return false; 2693 } 2694 2695 static inline const struct btf_func_model * 2696 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 2697 const struct bpf_insn *insn) 2698 { 2699 return NULL; 2700 } 2701 2702 static inline int 2703 bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, 2704 u16 btf_fd_idx, u8 **func_addr) 2705 { 2706 return -ENOTSUPP; 2707 } 2708 2709 static inline bool unprivileged_ebpf_enabled(void) 2710 { 2711 return false; 2712 } 2713 2714 static inline bool has_current_bpf_ctx(void) 2715 { 2716 return false; 2717 } 2718 2719 static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog) 2720 { 2721 } 2722 2723 static inline void bpf_cgrp_storage_free(struct cgroup *cgroup) 2724 { 2725 } 2726 2727 static inline void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data, 2728 enum bpf_dynptr_type type, u32 offset, u32 size) 2729 { 2730 } 2731 2732 static inline void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr) 2733 { 2734 } 2735 2736 static inline void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr) 2737 { 2738 } 2739 #endif /* CONFIG_BPF_SYSCALL */ 2740 2741 static __always_inline int 2742 bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr) 2743 { 2744 int ret = -EFAULT; 2745 2746 if (IS_ENABLED(CONFIG_BPF_EVENTS)) 2747 ret = copy_from_kernel_nofault(dst, unsafe_ptr, size); 2748 if (unlikely(ret < 0)) 2749 memset(dst, 0, size); 2750 return ret; 2751 } 2752 2753 void __bpf_free_used_btfs(struct bpf_prog_aux *aux, 2754 struct btf_mod_pair *used_btfs, u32 len); 2755 2756 static inline struct bpf_prog *bpf_prog_get_type(u32 ufd, 2757 enum bpf_prog_type type) 2758 { 2759 return bpf_prog_get_type_dev(ufd, type, false); 2760 } 2761 2762 void __bpf_free_used_maps(struct bpf_prog_aux *aux, 2763 struct bpf_map **used_maps, u32 len); 2764 2765 bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool); 2766 2767 int bpf_prog_offload_compile(struct bpf_prog *prog); 2768 void bpf_prog_dev_bound_destroy(struct bpf_prog *prog); 2769 int bpf_prog_offload_info_fill(struct bpf_prog_info *info, 2770 struct bpf_prog *prog); 2771 2772 int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map); 2773 2774 int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value); 2775 int bpf_map_offload_update_elem(struct bpf_map *map, 2776 void *key, void *value, u64 flags); 2777 int bpf_map_offload_delete_elem(struct bpf_map *map, void *key); 2778 int bpf_map_offload_get_next_key(struct bpf_map *map, 2779 void *key, void *next_key); 2780 2781 bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map); 2782 2783 struct bpf_offload_dev * 2784 bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv); 2785 void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev); 2786 void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev); 2787 int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev, 2788 struct net_device *netdev); 2789 void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev, 2790 struct net_device *netdev); 2791 bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev); 2792 2793 void unpriv_ebpf_notify(int new_state); 2794 2795 #if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL) 2796 int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log, 2797 struct bpf_prog_aux *prog_aux); 2798 void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id); 2799 int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr); 2800 int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog); 2801 void bpf_dev_bound_netdev_unregister(struct net_device *dev); 2802 2803 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) 2804 { 2805 return aux->dev_bound; 2806 } 2807 2808 static inline bool bpf_prog_is_offloaded(const struct bpf_prog_aux *aux) 2809 { 2810 return aux->offload_requested; 2811 } 2812 2813 bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs); 2814 2815 static inline bool bpf_map_is_offloaded(struct bpf_map *map) 2816 { 2817 return unlikely(map->ops == &bpf_map_offload_ops); 2818 } 2819 2820 struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr); 2821 void bpf_map_offload_map_free(struct bpf_map *map); 2822 u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map); 2823 int bpf_prog_test_run_syscall(struct bpf_prog *prog, 2824 const union bpf_attr *kattr, 2825 union bpf_attr __user *uattr); 2826 2827 int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog); 2828 int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype); 2829 int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, u64 flags); 2830 int sock_map_bpf_prog_query(const union bpf_attr *attr, 2831 union bpf_attr __user *uattr); 2832 2833 void sock_map_unhash(struct sock *sk); 2834 void sock_map_destroy(struct sock *sk); 2835 void sock_map_close(struct sock *sk, long timeout); 2836 #else 2837 static inline int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log, 2838 struct bpf_prog_aux *prog_aux) 2839 { 2840 return -EOPNOTSUPP; 2841 } 2842 2843 static inline void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, 2844 u32 func_id) 2845 { 2846 return NULL; 2847 } 2848 2849 static inline int bpf_prog_dev_bound_init(struct bpf_prog *prog, 2850 union bpf_attr *attr) 2851 { 2852 return -EOPNOTSUPP; 2853 } 2854 2855 static inline int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, 2856 struct bpf_prog *old_prog) 2857 { 2858 return -EOPNOTSUPP; 2859 } 2860 2861 static inline void bpf_dev_bound_netdev_unregister(struct net_device *dev) 2862 { 2863 } 2864 2865 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) 2866 { 2867 return false; 2868 } 2869 2870 static inline bool bpf_prog_is_offloaded(struct bpf_prog_aux *aux) 2871 { 2872 return false; 2873 } 2874 2875 static inline bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs) 2876 { 2877 return false; 2878 } 2879 2880 static inline bool bpf_map_is_offloaded(struct bpf_map *map) 2881 { 2882 return false; 2883 } 2884 2885 static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) 2886 { 2887 return ERR_PTR(-EOPNOTSUPP); 2888 } 2889 2890 static inline void bpf_map_offload_map_free(struct bpf_map *map) 2891 { 2892 } 2893 2894 static inline u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map) 2895 { 2896 return 0; 2897 } 2898 2899 static inline int bpf_prog_test_run_syscall(struct bpf_prog *prog, 2900 const union bpf_attr *kattr, 2901 union bpf_attr __user *uattr) 2902 { 2903 return -ENOTSUPP; 2904 } 2905 2906 #ifdef CONFIG_BPF_SYSCALL 2907 static inline int sock_map_get_from_fd(const union bpf_attr *attr, 2908 struct bpf_prog *prog) 2909 { 2910 return -EINVAL; 2911 } 2912 2913 static inline int sock_map_prog_detach(const union bpf_attr *attr, 2914 enum bpf_prog_type ptype) 2915 { 2916 return -EOPNOTSUPP; 2917 } 2918 2919 static inline int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, 2920 u64 flags) 2921 { 2922 return -EOPNOTSUPP; 2923 } 2924 2925 static inline int sock_map_bpf_prog_query(const union bpf_attr *attr, 2926 union bpf_attr __user *uattr) 2927 { 2928 return -EINVAL; 2929 } 2930 #endif /* CONFIG_BPF_SYSCALL */ 2931 #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */ 2932 2933 static __always_inline void 2934 bpf_prog_inc_misses_counters(const struct bpf_prog_array *array) 2935 { 2936 const struct bpf_prog_array_item *item; 2937 struct bpf_prog *prog; 2938 2939 if (unlikely(!array)) 2940 return; 2941 2942 item = &array->items[0]; 2943 while ((prog = READ_ONCE(item->prog))) { 2944 bpf_prog_inc_misses_counter(prog); 2945 item++; 2946 } 2947 } 2948 2949 #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) 2950 void bpf_sk_reuseport_detach(struct sock *sk); 2951 int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, 2952 void *value); 2953 int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key, 2954 void *value, u64 map_flags); 2955 #else 2956 static inline void bpf_sk_reuseport_detach(struct sock *sk) 2957 { 2958 } 2959 2960 #ifdef CONFIG_BPF_SYSCALL 2961 static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, 2962 void *key, void *value) 2963 { 2964 return -EOPNOTSUPP; 2965 } 2966 2967 static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, 2968 void *key, void *value, 2969 u64 map_flags) 2970 { 2971 return -EOPNOTSUPP; 2972 } 2973 #endif /* CONFIG_BPF_SYSCALL */ 2974 #endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */ 2975 2976 /* verifier prototypes for helper functions called from eBPF programs */ 2977 extern const struct bpf_func_proto bpf_map_lookup_elem_proto; 2978 extern const struct bpf_func_proto bpf_map_update_elem_proto; 2979 extern const struct bpf_func_proto bpf_map_delete_elem_proto; 2980 extern const struct bpf_func_proto bpf_map_push_elem_proto; 2981 extern const struct bpf_func_proto bpf_map_pop_elem_proto; 2982 extern const struct bpf_func_proto bpf_map_peek_elem_proto; 2983 extern const struct bpf_func_proto bpf_map_lookup_percpu_elem_proto; 2984 2985 extern const struct bpf_func_proto bpf_get_prandom_u32_proto; 2986 extern const struct bpf_func_proto bpf_get_smp_processor_id_proto; 2987 extern const struct bpf_func_proto bpf_get_numa_node_id_proto; 2988 extern const struct bpf_func_proto bpf_tail_call_proto; 2989 extern const struct bpf_func_proto bpf_ktime_get_ns_proto; 2990 extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto; 2991 extern const struct bpf_func_proto bpf_ktime_get_tai_ns_proto; 2992 extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto; 2993 extern const struct bpf_func_proto bpf_get_current_uid_gid_proto; 2994 extern const struct bpf_func_proto bpf_get_current_comm_proto; 2995 extern const struct bpf_func_proto bpf_get_stackid_proto; 2996 extern const struct bpf_func_proto bpf_get_stack_proto; 2997 extern const struct bpf_func_proto bpf_get_task_stack_proto; 2998 extern const struct bpf_func_proto bpf_get_stackid_proto_pe; 2999 extern const struct bpf_func_proto bpf_get_stack_proto_pe; 3000 extern const struct bpf_func_proto bpf_sock_map_update_proto; 3001 extern const struct bpf_func_proto bpf_sock_hash_update_proto; 3002 extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto; 3003 extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto; 3004 extern const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto; 3005 extern const struct bpf_func_proto bpf_msg_redirect_hash_proto; 3006 extern const struct bpf_func_proto bpf_msg_redirect_map_proto; 3007 extern const struct bpf_func_proto bpf_sk_redirect_hash_proto; 3008 extern const struct bpf_func_proto bpf_sk_redirect_map_proto; 3009 extern const struct bpf_func_proto bpf_spin_lock_proto; 3010 extern const struct bpf_func_proto bpf_spin_unlock_proto; 3011 extern const struct bpf_func_proto bpf_get_local_storage_proto; 3012 extern const struct bpf_func_proto bpf_strtol_proto; 3013 extern const struct bpf_func_proto bpf_strtoul_proto; 3014 extern const struct bpf_func_proto bpf_tcp_sock_proto; 3015 extern const struct bpf_func_proto bpf_jiffies64_proto; 3016 extern const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto; 3017 extern const struct bpf_func_proto bpf_event_output_data_proto; 3018 extern const struct bpf_func_proto bpf_ringbuf_output_proto; 3019 extern const struct bpf_func_proto bpf_ringbuf_reserve_proto; 3020 extern const struct bpf_func_proto bpf_ringbuf_submit_proto; 3021 extern const struct bpf_func_proto bpf_ringbuf_discard_proto; 3022 extern const struct bpf_func_proto bpf_ringbuf_query_proto; 3023 extern const struct bpf_func_proto bpf_ringbuf_reserve_dynptr_proto; 3024 extern const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto; 3025 extern const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto; 3026 extern const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto; 3027 extern const struct bpf_func_proto bpf_skc_to_tcp_sock_proto; 3028 extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto; 3029 extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto; 3030 extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto; 3031 extern const struct bpf_func_proto bpf_skc_to_unix_sock_proto; 3032 extern const struct bpf_func_proto bpf_skc_to_mptcp_sock_proto; 3033 extern const struct bpf_func_proto bpf_copy_from_user_proto; 3034 extern const struct bpf_func_proto bpf_snprintf_btf_proto; 3035 extern const struct bpf_func_proto bpf_snprintf_proto; 3036 extern const struct bpf_func_proto bpf_per_cpu_ptr_proto; 3037 extern const struct bpf_func_proto bpf_this_cpu_ptr_proto; 3038 extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto; 3039 extern const struct bpf_func_proto bpf_sock_from_file_proto; 3040 extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto; 3041 extern const struct bpf_func_proto bpf_task_storage_get_recur_proto; 3042 extern const struct bpf_func_proto bpf_task_storage_get_proto; 3043 extern const struct bpf_func_proto bpf_task_storage_delete_recur_proto; 3044 extern const struct bpf_func_proto bpf_task_storage_delete_proto; 3045 extern const struct bpf_func_proto bpf_for_each_map_elem_proto; 3046 extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto; 3047 extern const struct bpf_func_proto bpf_sk_setsockopt_proto; 3048 extern const struct bpf_func_proto bpf_sk_getsockopt_proto; 3049 extern const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto; 3050 extern const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto; 3051 extern const struct bpf_func_proto bpf_find_vma_proto; 3052 extern const struct bpf_func_proto bpf_loop_proto; 3053 extern const struct bpf_func_proto bpf_copy_from_user_task_proto; 3054 extern const struct bpf_func_proto bpf_set_retval_proto; 3055 extern const struct bpf_func_proto bpf_get_retval_proto; 3056 extern const struct bpf_func_proto bpf_user_ringbuf_drain_proto; 3057 extern const struct bpf_func_proto bpf_cgrp_storage_get_proto; 3058 extern const struct bpf_func_proto bpf_cgrp_storage_delete_proto; 3059 3060 const struct bpf_func_proto *tracing_prog_func_proto( 3061 enum bpf_func_id func_id, const struct bpf_prog *prog); 3062 3063 /* Shared helpers among cBPF and eBPF. */ 3064 void bpf_user_rnd_init_once(void); 3065 u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 3066 u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 3067 3068 #if defined(CONFIG_NET) 3069 bool bpf_sock_common_is_valid_access(int off, int size, 3070 enum bpf_access_type type, 3071 struct bpf_insn_access_aux *info); 3072 bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type, 3073 struct bpf_insn_access_aux *info); 3074 u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, 3075 const struct bpf_insn *si, 3076 struct bpf_insn *insn_buf, 3077 struct bpf_prog *prog, 3078 u32 *target_size); 3079 int bpf_dynptr_from_skb_rdonly(struct sk_buff *skb, u64 flags, 3080 struct bpf_dynptr_kern *ptr); 3081 #else 3082 static inline bool bpf_sock_common_is_valid_access(int off, int size, 3083 enum bpf_access_type type, 3084 struct bpf_insn_access_aux *info) 3085 { 3086 return false; 3087 } 3088 static inline bool bpf_sock_is_valid_access(int off, int size, 3089 enum bpf_access_type type, 3090 struct bpf_insn_access_aux *info) 3091 { 3092 return false; 3093 } 3094 static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, 3095 const struct bpf_insn *si, 3096 struct bpf_insn *insn_buf, 3097 struct bpf_prog *prog, 3098 u32 *target_size) 3099 { 3100 return 0; 3101 } 3102 static inline int bpf_dynptr_from_skb_rdonly(struct sk_buff *skb, u64 flags, 3103 struct bpf_dynptr_kern *ptr) 3104 { 3105 return -EOPNOTSUPP; 3106 } 3107 #endif 3108 3109 #ifdef CONFIG_INET 3110 struct sk_reuseport_kern { 3111 struct sk_buff *skb; 3112 struct sock *sk; 3113 struct sock *selected_sk; 3114 struct sock *migrating_sk; 3115 void *data_end; 3116 u32 hash; 3117 u32 reuseport_id; 3118 bool bind_inany; 3119 }; 3120 bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type, 3121 struct bpf_insn_access_aux *info); 3122 3123 u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, 3124 const struct bpf_insn *si, 3125 struct bpf_insn *insn_buf, 3126 struct bpf_prog *prog, 3127 u32 *target_size); 3128 3129 bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type, 3130 struct bpf_insn_access_aux *info); 3131 3132 u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, 3133 const struct bpf_insn *si, 3134 struct bpf_insn *insn_buf, 3135 struct bpf_prog *prog, 3136 u32 *target_size); 3137 #else 3138 static inline bool bpf_tcp_sock_is_valid_access(int off, int size, 3139 enum bpf_access_type type, 3140 struct bpf_insn_access_aux *info) 3141 { 3142 return false; 3143 } 3144 3145 static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, 3146 const struct bpf_insn *si, 3147 struct bpf_insn *insn_buf, 3148 struct bpf_prog *prog, 3149 u32 *target_size) 3150 { 3151 return 0; 3152 } 3153 static inline bool bpf_xdp_sock_is_valid_access(int off, int size, 3154 enum bpf_access_type type, 3155 struct bpf_insn_access_aux *info) 3156 { 3157 return false; 3158 } 3159 3160 static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, 3161 const struct bpf_insn *si, 3162 struct bpf_insn *insn_buf, 3163 struct bpf_prog *prog, 3164 u32 *target_size) 3165 { 3166 return 0; 3167 } 3168 #endif /* CONFIG_INET */ 3169 3170 enum bpf_text_poke_type { 3171 BPF_MOD_CALL, 3172 BPF_MOD_JUMP, 3173 }; 3174 3175 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, 3176 void *addr1, void *addr2); 3177 3178 void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke, 3179 struct bpf_prog *new, struct bpf_prog *old); 3180 3181 void *bpf_arch_text_copy(void *dst, void *src, size_t len); 3182 int bpf_arch_text_invalidate(void *dst, size_t len); 3183 3184 struct btf_id_set; 3185 bool btf_id_set_contains(const struct btf_id_set *set, u32 id); 3186 3187 #define MAX_BPRINTF_VARARGS 12 3188 #define MAX_BPRINTF_BUF 1024 3189 3190 struct bpf_bprintf_data { 3191 u32 *bin_args; 3192 char *buf; 3193 bool get_bin_args; 3194 bool get_buf; 3195 }; 3196 3197 int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args, 3198 u32 num_args, struct bpf_bprintf_data *data); 3199 void bpf_bprintf_cleanup(struct bpf_bprintf_data *data); 3200 3201 #ifdef CONFIG_BPF_LSM 3202 void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype); 3203 void bpf_cgroup_atype_put(int cgroup_atype); 3204 #else 3205 static inline void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) {} 3206 static inline void bpf_cgroup_atype_put(int cgroup_atype) {} 3207 #endif /* CONFIG_BPF_LSM */ 3208 3209 struct key; 3210 3211 #ifdef CONFIG_KEYS 3212 struct bpf_key { 3213 struct key *key; 3214 bool has_ref; 3215 }; 3216 #endif /* CONFIG_KEYS */ 3217 3218 static inline bool type_is_alloc(u32 type) 3219 { 3220 return type & MEM_ALLOC; 3221 } 3222 3223 static inline gfp_t bpf_memcg_flags(gfp_t flags) 3224 { 3225 if (memcg_bpf_enabled()) 3226 return flags | __GFP_ACCOUNT; 3227 return flags; 3228 } 3229 3230 static inline bool bpf_is_subprog(const struct bpf_prog *prog) 3231 { 3232 return prog->aux->func_idx != 0; 3233 } 3234 3235 #endif /* _LINUX_BPF_H */ 3236