1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 3 */ 4 #ifndef _LINUX_BPF_H 5 #define _LINUX_BPF_H 1 6 7 #include <uapi/linux/bpf.h> 8 9 #include <linux/workqueue.h> 10 #include <linux/file.h> 11 #include <linux/percpu.h> 12 #include <linux/err.h> 13 #include <linux/rbtree_latch.h> 14 #include <linux/numa.h> 15 #include <linux/mm_types.h> 16 #include <linux/wait.h> 17 #include <linux/refcount.h> 18 #include <linux/mutex.h> 19 #include <linux/module.h> 20 #include <linux/kallsyms.h> 21 #include <linux/capability.h> 22 #include <linux/sched/mm.h> 23 #include <linux/slab.h> 24 #include <linux/percpu-refcount.h> 25 26 struct bpf_verifier_env; 27 struct bpf_verifier_log; 28 struct perf_event; 29 struct bpf_prog; 30 struct bpf_prog_aux; 31 struct bpf_map; 32 struct sock; 33 struct seq_file; 34 struct btf; 35 struct btf_type; 36 struct exception_table_entry; 37 struct seq_operations; 38 struct bpf_iter_aux_info; 39 struct bpf_local_storage; 40 struct bpf_local_storage_map; 41 struct kobject; 42 struct mem_cgroup; 43 struct module; 44 struct bpf_func_state; 45 46 extern struct idr btf_idr; 47 extern spinlock_t btf_idr_lock; 48 extern struct kobject *btf_kobj; 49 50 typedef int (*bpf_iter_init_seq_priv_t)(void *private_data, 51 struct bpf_iter_aux_info *aux); 52 typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data); 53 struct bpf_iter_seq_info { 54 const struct seq_operations *seq_ops; 55 bpf_iter_init_seq_priv_t init_seq_private; 56 bpf_iter_fini_seq_priv_t fini_seq_private; 57 u32 seq_priv_size; 58 }; 59 60 /* map is generic key/value storage optionally accessible by eBPF programs */ 61 struct bpf_map_ops { 62 /* funcs callable from userspace (via syscall) */ 63 int (*map_alloc_check)(union bpf_attr *attr); 64 struct bpf_map *(*map_alloc)(union bpf_attr *attr); 65 void (*map_release)(struct bpf_map *map, struct file *map_file); 66 void (*map_free)(struct bpf_map *map); 67 int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key); 68 void (*map_release_uref)(struct bpf_map *map); 69 void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key); 70 int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr, 71 union bpf_attr __user *uattr); 72 int (*map_lookup_and_delete_batch)(struct bpf_map *map, 73 const union bpf_attr *attr, 74 union bpf_attr __user *uattr); 75 int (*map_update_batch)(struct bpf_map *map, const union bpf_attr *attr, 76 union bpf_attr __user *uattr); 77 int (*map_delete_batch)(struct bpf_map *map, const union bpf_attr *attr, 78 union bpf_attr __user *uattr); 79 80 /* funcs callable from userspace and from eBPF programs */ 81 void *(*map_lookup_elem)(struct bpf_map *map, void *key); 82 int (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags); 83 int (*map_delete_elem)(struct bpf_map *map, void *key); 84 int (*map_push_elem)(struct bpf_map *map, void *value, u64 flags); 85 int (*map_pop_elem)(struct bpf_map *map, void *value); 86 int (*map_peek_elem)(struct bpf_map *map, void *value); 87 88 /* funcs called by prog_array and perf_event_array map */ 89 void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file, 90 int fd); 91 void (*map_fd_put_ptr)(void *ptr); 92 int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf); 93 u32 (*map_fd_sys_lookup_elem)(void *ptr); 94 void (*map_seq_show_elem)(struct bpf_map *map, void *key, 95 struct seq_file *m); 96 int (*map_check_btf)(const struct bpf_map *map, 97 const struct btf *btf, 98 const struct btf_type *key_type, 99 const struct btf_type *value_type); 100 101 /* Prog poke tracking helpers. */ 102 int (*map_poke_track)(struct bpf_map *map, struct bpf_prog_aux *aux); 103 void (*map_poke_untrack)(struct bpf_map *map, struct bpf_prog_aux *aux); 104 void (*map_poke_run)(struct bpf_map *map, u32 key, struct bpf_prog *old, 105 struct bpf_prog *new); 106 107 /* Direct value access helpers. */ 108 int (*map_direct_value_addr)(const struct bpf_map *map, 109 u64 *imm, u32 off); 110 int (*map_direct_value_meta)(const struct bpf_map *map, 111 u64 imm, u32 *off); 112 int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma); 113 __poll_t (*map_poll)(struct bpf_map *map, struct file *filp, 114 struct poll_table_struct *pts); 115 116 /* Functions called by bpf_local_storage maps */ 117 int (*map_local_storage_charge)(struct bpf_local_storage_map *smap, 118 void *owner, u32 size); 119 void (*map_local_storage_uncharge)(struct bpf_local_storage_map *smap, 120 void *owner, u32 size); 121 struct bpf_local_storage __rcu ** (*map_owner_storage_ptr)(void *owner); 122 123 /* Misc helpers.*/ 124 int (*map_redirect)(struct bpf_map *map, u32 ifindex, u64 flags); 125 126 /* map_meta_equal must be implemented for maps that can be 127 * used as an inner map. It is a runtime check to ensure 128 * an inner map can be inserted to an outer map. 129 * 130 * Some properties of the inner map has been used during the 131 * verification time. When inserting an inner map at the runtime, 132 * map_meta_equal has to ensure the inserting map has the same 133 * properties that the verifier has used earlier. 134 */ 135 bool (*map_meta_equal)(const struct bpf_map *meta0, 136 const struct bpf_map *meta1); 137 138 139 int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env, 140 struct bpf_func_state *caller, 141 struct bpf_func_state *callee); 142 int (*map_for_each_callback)(struct bpf_map *map, void *callback_fn, 143 void *callback_ctx, u64 flags); 144 145 /* BTF name and id of struct allocated by map_alloc */ 146 const char * const map_btf_name; 147 int *map_btf_id; 148 149 /* bpf_iter info used to open a seq_file */ 150 const struct bpf_iter_seq_info *iter_seq_info; 151 }; 152 153 struct bpf_map { 154 /* The first two cachelines with read-mostly members of which some 155 * are also accessed in fast-path (e.g. ops, max_entries). 156 */ 157 const struct bpf_map_ops *ops ____cacheline_aligned; 158 struct bpf_map *inner_map_meta; 159 #ifdef CONFIG_SECURITY 160 void *security; 161 #endif 162 enum bpf_map_type map_type; 163 u32 key_size; 164 u32 value_size; 165 u32 max_entries; 166 u32 map_flags; 167 int spin_lock_off; /* >=0 valid offset, <0 error */ 168 u32 id; 169 int numa_node; 170 u32 btf_key_type_id; 171 u32 btf_value_type_id; 172 struct btf *btf; 173 #ifdef CONFIG_MEMCG_KMEM 174 struct mem_cgroup *memcg; 175 #endif 176 char name[BPF_OBJ_NAME_LEN]; 177 u32 btf_vmlinux_value_type_id; 178 bool bypass_spec_v1; 179 bool frozen; /* write-once; write-protected by freeze_mutex */ 180 /* 22 bytes hole */ 181 182 /* The 3rd and 4th cacheline with misc members to avoid false sharing 183 * particularly with refcounting. 184 */ 185 atomic64_t refcnt ____cacheline_aligned; 186 atomic64_t usercnt; 187 struct work_struct work; 188 struct mutex freeze_mutex; 189 u64 writecnt; /* writable mmap cnt; protected by freeze_mutex */ 190 }; 191 192 static inline bool map_value_has_spin_lock(const struct bpf_map *map) 193 { 194 return map->spin_lock_off >= 0; 195 } 196 197 static inline void check_and_init_map_lock(struct bpf_map *map, void *dst) 198 { 199 if (likely(!map_value_has_spin_lock(map))) 200 return; 201 *(struct bpf_spin_lock *)(dst + map->spin_lock_off) = 202 (struct bpf_spin_lock){}; 203 } 204 205 /* copy everything but bpf_spin_lock */ 206 static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) 207 { 208 if (unlikely(map_value_has_spin_lock(map))) { 209 u32 off = map->spin_lock_off; 210 211 memcpy(dst, src, off); 212 memcpy(dst + off + sizeof(struct bpf_spin_lock), 213 src + off + sizeof(struct bpf_spin_lock), 214 map->value_size - off - sizeof(struct bpf_spin_lock)); 215 } else { 216 memcpy(dst, src, map->value_size); 217 } 218 } 219 void copy_map_value_locked(struct bpf_map *map, void *dst, void *src, 220 bool lock_src); 221 int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size); 222 223 struct bpf_offload_dev; 224 struct bpf_offloaded_map; 225 226 struct bpf_map_dev_ops { 227 int (*map_get_next_key)(struct bpf_offloaded_map *map, 228 void *key, void *next_key); 229 int (*map_lookup_elem)(struct bpf_offloaded_map *map, 230 void *key, void *value); 231 int (*map_update_elem)(struct bpf_offloaded_map *map, 232 void *key, void *value, u64 flags); 233 int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key); 234 }; 235 236 struct bpf_offloaded_map { 237 struct bpf_map map; 238 struct net_device *netdev; 239 const struct bpf_map_dev_ops *dev_ops; 240 void *dev_priv; 241 struct list_head offloads; 242 }; 243 244 static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map) 245 { 246 return container_of(map, struct bpf_offloaded_map, map); 247 } 248 249 static inline bool bpf_map_offload_neutral(const struct bpf_map *map) 250 { 251 return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; 252 } 253 254 static inline bool bpf_map_support_seq_show(const struct bpf_map *map) 255 { 256 return (map->btf_value_type_id || map->btf_vmlinux_value_type_id) && 257 map->ops->map_seq_show_elem; 258 } 259 260 int map_check_no_btf(const struct bpf_map *map, 261 const struct btf *btf, 262 const struct btf_type *key_type, 263 const struct btf_type *value_type); 264 265 bool bpf_map_meta_equal(const struct bpf_map *meta0, 266 const struct bpf_map *meta1); 267 268 extern const struct bpf_map_ops bpf_map_offload_ops; 269 270 /* function argument constraints */ 271 enum bpf_arg_type { 272 ARG_DONTCARE = 0, /* unused argument in helper function */ 273 274 /* the following constraints used to prototype 275 * bpf_map_lookup/update/delete_elem() functions 276 */ 277 ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */ 278 ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */ 279 ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */ 280 ARG_PTR_TO_UNINIT_MAP_VALUE, /* pointer to valid memory used to store a map value */ 281 ARG_PTR_TO_MAP_VALUE_OR_NULL, /* pointer to stack used as map value or NULL */ 282 283 /* the following constraints used to prototype bpf_memcmp() and other 284 * functions that access data on eBPF program stack 285 */ 286 ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ 287 ARG_PTR_TO_MEM_OR_NULL, /* pointer to valid memory or NULL */ 288 ARG_PTR_TO_UNINIT_MEM, /* pointer to memory does not need to be initialized, 289 * helper function must fill all bytes or clear 290 * them in error case. 291 */ 292 293 ARG_CONST_SIZE, /* number of bytes accessed from memory */ 294 ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ 295 296 ARG_PTR_TO_CTX, /* pointer to context */ 297 ARG_PTR_TO_CTX_OR_NULL, /* pointer to context or NULL */ 298 ARG_ANYTHING, /* any (initialized) argument is ok */ 299 ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */ 300 ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */ 301 ARG_PTR_TO_INT, /* pointer to int */ 302 ARG_PTR_TO_LONG, /* pointer to long */ 303 ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */ 304 ARG_PTR_TO_SOCKET_OR_NULL, /* pointer to bpf_sock (fullsock) or NULL */ 305 ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */ 306 ARG_PTR_TO_ALLOC_MEM, /* pointer to dynamically allocated memory */ 307 ARG_PTR_TO_ALLOC_MEM_OR_NULL, /* pointer to dynamically allocated memory or NULL */ 308 ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */ 309 ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */ 310 ARG_PTR_TO_PERCPU_BTF_ID, /* pointer to in-kernel percpu type */ 311 ARG_PTR_TO_FUNC, /* pointer to a bpf program function */ 312 ARG_PTR_TO_STACK_OR_NULL, /* pointer to stack or NULL */ 313 ARG_PTR_TO_CONST_STR, /* pointer to a null terminated read-only string */ 314 __BPF_ARG_TYPE_MAX, 315 }; 316 317 /* type of values returned from helper functions */ 318 enum bpf_return_type { 319 RET_INTEGER, /* function returns integer */ 320 RET_VOID, /* function doesn't return anything */ 321 RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */ 322 RET_PTR_TO_MAP_VALUE_OR_NULL, /* returns a pointer to map elem value or NULL */ 323 RET_PTR_TO_SOCKET_OR_NULL, /* returns a pointer to a socket or NULL */ 324 RET_PTR_TO_TCP_SOCK_OR_NULL, /* returns a pointer to a tcp_sock or NULL */ 325 RET_PTR_TO_SOCK_COMMON_OR_NULL, /* returns a pointer to a sock_common or NULL */ 326 RET_PTR_TO_ALLOC_MEM_OR_NULL, /* returns a pointer to dynamically allocated memory or NULL */ 327 RET_PTR_TO_BTF_ID_OR_NULL, /* returns a pointer to a btf_id or NULL */ 328 RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL, /* returns a pointer to a valid memory or a btf_id or NULL */ 329 RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */ 330 RET_PTR_TO_BTF_ID, /* returns a pointer to a btf_id */ 331 }; 332 333 /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs 334 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL 335 * instructions after verifying 336 */ 337 struct bpf_func_proto { 338 u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 339 bool gpl_only; 340 bool pkt_access; 341 enum bpf_return_type ret_type; 342 union { 343 struct { 344 enum bpf_arg_type arg1_type; 345 enum bpf_arg_type arg2_type; 346 enum bpf_arg_type arg3_type; 347 enum bpf_arg_type arg4_type; 348 enum bpf_arg_type arg5_type; 349 }; 350 enum bpf_arg_type arg_type[5]; 351 }; 352 union { 353 struct { 354 u32 *arg1_btf_id; 355 u32 *arg2_btf_id; 356 u32 *arg3_btf_id; 357 u32 *arg4_btf_id; 358 u32 *arg5_btf_id; 359 }; 360 u32 *arg_btf_id[5]; 361 }; 362 int *ret_btf_id; /* return value btf_id */ 363 bool (*allowed)(const struct bpf_prog *prog); 364 }; 365 366 /* bpf_context is intentionally undefined structure. Pointer to bpf_context is 367 * the first argument to eBPF programs. 368 * For socket filters: 'struct bpf_context *' == 'struct sk_buff *' 369 */ 370 struct bpf_context; 371 372 enum bpf_access_type { 373 BPF_READ = 1, 374 BPF_WRITE = 2 375 }; 376 377 /* types of values stored in eBPF registers */ 378 /* Pointer types represent: 379 * pointer 380 * pointer + imm 381 * pointer + (u16) var 382 * pointer + (u16) var + imm 383 * if (range > 0) then [ptr, ptr + range - off) is safe to access 384 * if (id > 0) means that some 'var' was added 385 * if (off > 0) means that 'imm' was added 386 */ 387 enum bpf_reg_type { 388 NOT_INIT = 0, /* nothing was written into register */ 389 SCALAR_VALUE, /* reg doesn't contain a valid pointer */ 390 PTR_TO_CTX, /* reg points to bpf_context */ 391 CONST_PTR_TO_MAP, /* reg points to struct bpf_map */ 392 PTR_TO_MAP_VALUE, /* reg points to map element value */ 393 PTR_TO_MAP_VALUE_OR_NULL,/* points to map elem value or NULL */ 394 PTR_TO_STACK, /* reg == frame_pointer + offset */ 395 PTR_TO_PACKET_META, /* skb->data - meta_len */ 396 PTR_TO_PACKET, /* reg points to skb->data */ 397 PTR_TO_PACKET_END, /* skb->data + headlen */ 398 PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */ 399 PTR_TO_SOCKET, /* reg points to struct bpf_sock */ 400 PTR_TO_SOCKET_OR_NULL, /* reg points to struct bpf_sock or NULL */ 401 PTR_TO_SOCK_COMMON, /* reg points to sock_common */ 402 PTR_TO_SOCK_COMMON_OR_NULL, /* reg points to sock_common or NULL */ 403 PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */ 404 PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */ 405 PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */ 406 PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */ 407 /* PTR_TO_BTF_ID points to a kernel struct that does not need 408 * to be null checked by the BPF program. This does not imply the 409 * pointer is _not_ null and in practice this can easily be a null 410 * pointer when reading pointer chains. The assumption is program 411 * context will handle null pointer dereference typically via fault 412 * handling. The verifier must keep this in mind and can make no 413 * assumptions about null or non-null when doing branch analysis. 414 * Further, when passed into helpers the helpers can not, without 415 * additional context, assume the value is non-null. 416 */ 417 PTR_TO_BTF_ID, 418 /* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not 419 * been checked for null. Used primarily to inform the verifier 420 * an explicit null check is required for this struct. 421 */ 422 PTR_TO_BTF_ID_OR_NULL, 423 PTR_TO_MEM, /* reg points to valid memory region */ 424 PTR_TO_MEM_OR_NULL, /* reg points to valid memory region or NULL */ 425 PTR_TO_RDONLY_BUF, /* reg points to a readonly buffer */ 426 PTR_TO_RDONLY_BUF_OR_NULL, /* reg points to a readonly buffer or NULL */ 427 PTR_TO_RDWR_BUF, /* reg points to a read/write buffer */ 428 PTR_TO_RDWR_BUF_OR_NULL, /* reg points to a read/write buffer or NULL */ 429 PTR_TO_PERCPU_BTF_ID, /* reg points to a percpu kernel variable */ 430 PTR_TO_FUNC, /* reg points to a bpf program function */ 431 PTR_TO_MAP_KEY, /* reg points to a map element key */ 432 __BPF_REG_TYPE_MAX, 433 }; 434 435 /* The information passed from prog-specific *_is_valid_access 436 * back to the verifier. 437 */ 438 struct bpf_insn_access_aux { 439 enum bpf_reg_type reg_type; 440 union { 441 int ctx_field_size; 442 struct { 443 struct btf *btf; 444 u32 btf_id; 445 }; 446 }; 447 struct bpf_verifier_log *log; /* for verbose logs */ 448 }; 449 450 static inline void 451 bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size) 452 { 453 aux->ctx_field_size = size; 454 } 455 456 struct bpf_prog_ops { 457 int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, 458 union bpf_attr __user *uattr); 459 }; 460 461 struct bpf_verifier_ops { 462 /* return eBPF function prototype for verification */ 463 const struct bpf_func_proto * 464 (*get_func_proto)(enum bpf_func_id func_id, 465 const struct bpf_prog *prog); 466 467 /* return true if 'size' wide access at offset 'off' within bpf_context 468 * with 'type' (read or write) is allowed 469 */ 470 bool (*is_valid_access)(int off, int size, enum bpf_access_type type, 471 const struct bpf_prog *prog, 472 struct bpf_insn_access_aux *info); 473 int (*gen_prologue)(struct bpf_insn *insn, bool direct_write, 474 const struct bpf_prog *prog); 475 int (*gen_ld_abs)(const struct bpf_insn *orig, 476 struct bpf_insn *insn_buf); 477 u32 (*convert_ctx_access)(enum bpf_access_type type, 478 const struct bpf_insn *src, 479 struct bpf_insn *dst, 480 struct bpf_prog *prog, u32 *target_size); 481 int (*btf_struct_access)(struct bpf_verifier_log *log, 482 const struct btf *btf, 483 const struct btf_type *t, int off, int size, 484 enum bpf_access_type atype, 485 u32 *next_btf_id); 486 bool (*check_kfunc_call)(u32 kfunc_btf_id); 487 }; 488 489 struct bpf_prog_offload_ops { 490 /* verifier basic callbacks */ 491 int (*insn_hook)(struct bpf_verifier_env *env, 492 int insn_idx, int prev_insn_idx); 493 int (*finalize)(struct bpf_verifier_env *env); 494 /* verifier optimization callbacks (called after .finalize) */ 495 int (*replace_insn)(struct bpf_verifier_env *env, u32 off, 496 struct bpf_insn *insn); 497 int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt); 498 /* program management callbacks */ 499 int (*prepare)(struct bpf_prog *prog); 500 int (*translate)(struct bpf_prog *prog); 501 void (*destroy)(struct bpf_prog *prog); 502 }; 503 504 struct bpf_prog_offload { 505 struct bpf_prog *prog; 506 struct net_device *netdev; 507 struct bpf_offload_dev *offdev; 508 void *dev_priv; 509 struct list_head offloads; 510 bool dev_state; 511 bool opt_failed; 512 void *jited_image; 513 u32 jited_len; 514 }; 515 516 enum bpf_cgroup_storage_type { 517 BPF_CGROUP_STORAGE_SHARED, 518 BPF_CGROUP_STORAGE_PERCPU, 519 __BPF_CGROUP_STORAGE_MAX 520 }; 521 522 #define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX 523 524 /* The longest tracepoint has 12 args. 525 * See include/trace/bpf_probe.h 526 */ 527 #define MAX_BPF_FUNC_ARGS 12 528 529 /* The maximum number of arguments passed through registers 530 * a single function may have. 531 */ 532 #define MAX_BPF_FUNC_REG_ARGS 5 533 534 struct btf_func_model { 535 u8 ret_size; 536 u8 nr_args; 537 u8 arg_size[MAX_BPF_FUNC_ARGS]; 538 }; 539 540 /* Restore arguments before returning from trampoline to let original function 541 * continue executing. This flag is used for fentry progs when there are no 542 * fexit progs. 543 */ 544 #define BPF_TRAMP_F_RESTORE_REGS BIT(0) 545 /* Call original function after fentry progs, but before fexit progs. 546 * Makes sense for fentry/fexit, normal calls and indirect calls. 547 */ 548 #define BPF_TRAMP_F_CALL_ORIG BIT(1) 549 /* Skip current frame and return to parent. Makes sense for fentry/fexit 550 * programs only. Should not be used with normal calls and indirect calls. 551 */ 552 #define BPF_TRAMP_F_SKIP_FRAME BIT(2) 553 554 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 555 * bytes on x86. Pick a number to fit into BPF_IMAGE_SIZE / 2 556 */ 557 #define BPF_MAX_TRAMP_PROGS 38 558 559 struct bpf_tramp_progs { 560 struct bpf_prog *progs[BPF_MAX_TRAMP_PROGS]; 561 int nr_progs; 562 }; 563 564 /* Different use cases for BPF trampoline: 565 * 1. replace nop at the function entry (kprobe equivalent) 566 * flags = BPF_TRAMP_F_RESTORE_REGS 567 * fentry = a set of programs to run before returning from trampoline 568 * 569 * 2. replace nop at the function entry (kprobe + kretprobe equivalent) 570 * flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME 571 * orig_call = fentry_ip + MCOUNT_INSN_SIZE 572 * fentry = a set of program to run before calling original function 573 * fexit = a set of program to run after original function 574 * 575 * 3. replace direct call instruction anywhere in the function body 576 * or assign a function pointer for indirect call (like tcp_congestion_ops->cong_avoid) 577 * With flags = 0 578 * fentry = a set of programs to run before returning from trampoline 579 * With flags = BPF_TRAMP_F_CALL_ORIG 580 * orig_call = original callback addr or direct function addr 581 * fentry = a set of program to run before calling original function 582 * fexit = a set of program to run after original function 583 */ 584 struct bpf_tramp_image; 585 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end, 586 const struct btf_func_model *m, u32 flags, 587 struct bpf_tramp_progs *tprogs, 588 void *orig_call); 589 /* these two functions are called from generated trampoline */ 590 u64 notrace __bpf_prog_enter(struct bpf_prog *prog); 591 void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start); 592 u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog); 593 void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start); 594 void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr); 595 void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr); 596 597 struct bpf_ksym { 598 unsigned long start; 599 unsigned long end; 600 char name[KSYM_NAME_LEN]; 601 struct list_head lnode; 602 struct latch_tree_node tnode; 603 bool prog; 604 }; 605 606 enum bpf_tramp_prog_type { 607 BPF_TRAMP_FENTRY, 608 BPF_TRAMP_FEXIT, 609 BPF_TRAMP_MODIFY_RETURN, 610 BPF_TRAMP_MAX, 611 BPF_TRAMP_REPLACE, /* more than MAX */ 612 }; 613 614 struct bpf_tramp_image { 615 void *image; 616 struct bpf_ksym ksym; 617 struct percpu_ref pcref; 618 void *ip_after_call; 619 void *ip_epilogue; 620 union { 621 struct rcu_head rcu; 622 struct work_struct work; 623 }; 624 }; 625 626 struct bpf_trampoline { 627 /* hlist for trampoline_table */ 628 struct hlist_node hlist; 629 /* serializes access to fields of this trampoline */ 630 struct mutex mutex; 631 refcount_t refcnt; 632 u64 key; 633 struct { 634 struct btf_func_model model; 635 void *addr; 636 bool ftrace_managed; 637 } func; 638 /* if !NULL this is BPF_PROG_TYPE_EXT program that extends another BPF 639 * program by replacing one of its functions. func.addr is the address 640 * of the function it replaced. 641 */ 642 struct bpf_prog *extension_prog; 643 /* list of BPF programs using this trampoline */ 644 struct hlist_head progs_hlist[BPF_TRAMP_MAX]; 645 /* Number of attached programs. A counter per kind. */ 646 int progs_cnt[BPF_TRAMP_MAX]; 647 /* Executable image of trampoline */ 648 struct bpf_tramp_image *cur_image; 649 u64 selector; 650 struct module *mod; 651 }; 652 653 struct bpf_attach_target_info { 654 struct btf_func_model fmodel; 655 long tgt_addr; 656 const char *tgt_name; 657 const struct btf_type *tgt_type; 658 }; 659 660 #define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */ 661 662 struct bpf_dispatcher_prog { 663 struct bpf_prog *prog; 664 refcount_t users; 665 }; 666 667 struct bpf_dispatcher { 668 /* dispatcher mutex */ 669 struct mutex mutex; 670 void *func; 671 struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX]; 672 int num_progs; 673 void *image; 674 u32 image_off; 675 struct bpf_ksym ksym; 676 }; 677 678 static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( 679 const void *ctx, 680 const struct bpf_insn *insnsi, 681 unsigned int (*bpf_func)(const void *, 682 const struct bpf_insn *)) 683 { 684 return bpf_func(ctx, insnsi); 685 } 686 #ifdef CONFIG_BPF_JIT 687 int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr); 688 int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr); 689 struct bpf_trampoline *bpf_trampoline_get(u64 key, 690 struct bpf_attach_target_info *tgt_info); 691 void bpf_trampoline_put(struct bpf_trampoline *tr); 692 #define BPF_DISPATCHER_INIT(_name) { \ 693 .mutex = __MUTEX_INITIALIZER(_name.mutex), \ 694 .func = &_name##_func, \ 695 .progs = {}, \ 696 .num_progs = 0, \ 697 .image = NULL, \ 698 .image_off = 0, \ 699 .ksym = { \ 700 .name = #_name, \ 701 .lnode = LIST_HEAD_INIT(_name.ksym.lnode), \ 702 }, \ 703 } 704 705 #define DEFINE_BPF_DISPATCHER(name) \ 706 noinline __nocfi unsigned int bpf_dispatcher_##name##_func( \ 707 const void *ctx, \ 708 const struct bpf_insn *insnsi, \ 709 unsigned int (*bpf_func)(const void *, \ 710 const struct bpf_insn *)) \ 711 { \ 712 return bpf_func(ctx, insnsi); \ 713 } \ 714 EXPORT_SYMBOL(bpf_dispatcher_##name##_func); \ 715 struct bpf_dispatcher bpf_dispatcher_##name = \ 716 BPF_DISPATCHER_INIT(bpf_dispatcher_##name); 717 #define DECLARE_BPF_DISPATCHER(name) \ 718 unsigned int bpf_dispatcher_##name##_func( \ 719 const void *ctx, \ 720 const struct bpf_insn *insnsi, \ 721 unsigned int (*bpf_func)(const void *, \ 722 const struct bpf_insn *)); \ 723 extern struct bpf_dispatcher bpf_dispatcher_##name; 724 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func 725 #define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name) 726 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, 727 struct bpf_prog *to); 728 /* Called only from JIT-enabled code, so there's no need for stubs. */ 729 void *bpf_jit_alloc_exec_page(void); 730 void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym); 731 void bpf_image_ksym_del(struct bpf_ksym *ksym); 732 void bpf_ksym_add(struct bpf_ksym *ksym); 733 void bpf_ksym_del(struct bpf_ksym *ksym); 734 int bpf_jit_charge_modmem(u32 pages); 735 void bpf_jit_uncharge_modmem(u32 pages); 736 #else 737 static inline int bpf_trampoline_link_prog(struct bpf_prog *prog, 738 struct bpf_trampoline *tr) 739 { 740 return -ENOTSUPP; 741 } 742 static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog, 743 struct bpf_trampoline *tr) 744 { 745 return -ENOTSUPP; 746 } 747 static inline struct bpf_trampoline *bpf_trampoline_get(u64 key, 748 struct bpf_attach_target_info *tgt_info) 749 { 750 return ERR_PTR(-EOPNOTSUPP); 751 } 752 static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {} 753 #define DEFINE_BPF_DISPATCHER(name) 754 #define DECLARE_BPF_DISPATCHER(name) 755 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nop_func 756 #define BPF_DISPATCHER_PTR(name) NULL 757 static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, 758 struct bpf_prog *from, 759 struct bpf_prog *to) {} 760 static inline bool is_bpf_image_address(unsigned long address) 761 { 762 return false; 763 } 764 #endif 765 766 struct bpf_func_info_aux { 767 u16 linkage; 768 bool unreliable; 769 }; 770 771 enum bpf_jit_poke_reason { 772 BPF_POKE_REASON_TAIL_CALL, 773 }; 774 775 /* Descriptor of pokes pointing /into/ the JITed image. */ 776 struct bpf_jit_poke_descriptor { 777 void *tailcall_target; 778 void *tailcall_bypass; 779 void *bypass_addr; 780 union { 781 struct { 782 struct bpf_map *map; 783 u32 key; 784 } tail_call; 785 }; 786 bool tailcall_target_stable; 787 u8 adj_off; 788 u16 reason; 789 u32 insn_idx; 790 }; 791 792 /* reg_type info for ctx arguments */ 793 struct bpf_ctx_arg_aux { 794 u32 offset; 795 enum bpf_reg_type reg_type; 796 u32 btf_id; 797 }; 798 799 struct btf_mod_pair { 800 struct btf *btf; 801 struct module *module; 802 }; 803 804 struct bpf_kfunc_desc_tab; 805 806 struct bpf_prog_aux { 807 atomic64_t refcnt; 808 u32 used_map_cnt; 809 u32 used_btf_cnt; 810 u32 max_ctx_offset; 811 u32 max_pkt_offset; 812 u32 max_tp_access; 813 u32 stack_depth; 814 u32 id; 815 u32 func_cnt; /* used by non-func prog as the number of func progs */ 816 u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */ 817 u32 attach_btf_id; /* in-kernel BTF type id to attach to */ 818 u32 ctx_arg_info_size; 819 u32 max_rdonly_access; 820 u32 max_rdwr_access; 821 struct btf *attach_btf; 822 const struct bpf_ctx_arg_aux *ctx_arg_info; 823 struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */ 824 struct bpf_prog *dst_prog; 825 struct bpf_trampoline *dst_trampoline; 826 enum bpf_prog_type saved_dst_prog_type; 827 enum bpf_attach_type saved_dst_attach_type; 828 bool verifier_zext; /* Zero extensions has been inserted by verifier. */ 829 bool offload_requested; 830 bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ 831 bool func_proto_unreliable; 832 bool sleepable; 833 bool tail_call_reachable; 834 struct hlist_node tramp_hlist; 835 /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ 836 const struct btf_type *attach_func_proto; 837 /* function name for valid attach_btf_id */ 838 const char *attach_func_name; 839 struct bpf_prog **func; 840 void *jit_data; /* JIT specific data. arch dependent */ 841 struct bpf_jit_poke_descriptor *poke_tab; 842 struct bpf_kfunc_desc_tab *kfunc_tab; 843 u32 size_poke_tab; 844 struct bpf_ksym ksym; 845 const struct bpf_prog_ops *ops; 846 struct bpf_map **used_maps; 847 struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */ 848 struct btf_mod_pair *used_btfs; 849 struct bpf_prog *prog; 850 struct user_struct *user; 851 u64 load_time; /* ns since boottime */ 852 struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 853 char name[BPF_OBJ_NAME_LEN]; 854 #ifdef CONFIG_SECURITY 855 void *security; 856 #endif 857 struct bpf_prog_offload *offload; 858 struct btf *btf; 859 struct bpf_func_info *func_info; 860 struct bpf_func_info_aux *func_info_aux; 861 /* bpf_line_info loaded from userspace. linfo->insn_off 862 * has the xlated insn offset. 863 * Both the main and sub prog share the same linfo. 864 * The subprog can access its first linfo by 865 * using the linfo_idx. 866 */ 867 struct bpf_line_info *linfo; 868 /* jited_linfo is the jited addr of the linfo. It has a 869 * one to one mapping to linfo: 870 * jited_linfo[i] is the jited addr for the linfo[i]->insn_off. 871 * Both the main and sub prog share the same jited_linfo. 872 * The subprog can access its first jited_linfo by 873 * using the linfo_idx. 874 */ 875 void **jited_linfo; 876 u32 func_info_cnt; 877 u32 nr_linfo; 878 /* subprog can use linfo_idx to access its first linfo and 879 * jited_linfo. 880 * main prog always has linfo_idx == 0 881 */ 882 u32 linfo_idx; 883 u32 num_exentries; 884 struct exception_table_entry *extable; 885 union { 886 struct work_struct work; 887 struct rcu_head rcu; 888 }; 889 }; 890 891 struct bpf_array_aux { 892 /* 'Ownership' of prog array is claimed by the first program that 893 * is going to use this map or by the first program which FD is 894 * stored in the map to make sure that all callers and callees have 895 * the same prog type and JITed flag. 896 */ 897 enum bpf_prog_type type; 898 bool jited; 899 /* Programs with direct jumps into programs part of this array. */ 900 struct list_head poke_progs; 901 struct bpf_map *map; 902 struct mutex poke_mutex; 903 struct work_struct work; 904 }; 905 906 struct bpf_link { 907 atomic64_t refcnt; 908 u32 id; 909 enum bpf_link_type type; 910 const struct bpf_link_ops *ops; 911 struct bpf_prog *prog; 912 struct work_struct work; 913 }; 914 915 struct bpf_link_ops { 916 void (*release)(struct bpf_link *link); 917 void (*dealloc)(struct bpf_link *link); 918 int (*detach)(struct bpf_link *link); 919 int (*update_prog)(struct bpf_link *link, struct bpf_prog *new_prog, 920 struct bpf_prog *old_prog); 921 void (*show_fdinfo)(const struct bpf_link *link, struct seq_file *seq); 922 int (*fill_link_info)(const struct bpf_link *link, 923 struct bpf_link_info *info); 924 }; 925 926 struct bpf_link_primer { 927 struct bpf_link *link; 928 struct file *file; 929 int fd; 930 u32 id; 931 }; 932 933 struct bpf_struct_ops_value; 934 struct btf_member; 935 936 #define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64 937 struct bpf_struct_ops { 938 const struct bpf_verifier_ops *verifier_ops; 939 int (*init)(struct btf *btf); 940 int (*check_member)(const struct btf_type *t, 941 const struct btf_member *member); 942 int (*init_member)(const struct btf_type *t, 943 const struct btf_member *member, 944 void *kdata, const void *udata); 945 int (*reg)(void *kdata); 946 void (*unreg)(void *kdata); 947 const struct btf_type *type; 948 const struct btf_type *value_type; 949 const char *name; 950 struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; 951 u32 type_id; 952 u32 value_id; 953 }; 954 955 #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) 956 #define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA)) 957 const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id); 958 void bpf_struct_ops_init(struct btf *btf, struct bpf_verifier_log *log); 959 bool bpf_struct_ops_get(const void *kdata); 960 void bpf_struct_ops_put(const void *kdata); 961 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, 962 void *value); 963 static inline bool bpf_try_module_get(const void *data, struct module *owner) 964 { 965 if (owner == BPF_MODULE_OWNER) 966 return bpf_struct_ops_get(data); 967 else 968 return try_module_get(owner); 969 } 970 static inline void bpf_module_put(const void *data, struct module *owner) 971 { 972 if (owner == BPF_MODULE_OWNER) 973 bpf_struct_ops_put(data); 974 else 975 module_put(owner); 976 } 977 #else 978 static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id) 979 { 980 return NULL; 981 } 982 static inline void bpf_struct_ops_init(struct btf *btf, 983 struct bpf_verifier_log *log) 984 { 985 } 986 static inline bool bpf_try_module_get(const void *data, struct module *owner) 987 { 988 return try_module_get(owner); 989 } 990 static inline void bpf_module_put(const void *data, struct module *owner) 991 { 992 module_put(owner); 993 } 994 static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, 995 void *key, 996 void *value) 997 { 998 return -EINVAL; 999 } 1000 #endif 1001 1002 struct bpf_array { 1003 struct bpf_map map; 1004 u32 elem_size; 1005 u32 index_mask; 1006 struct bpf_array_aux *aux; 1007 union { 1008 char value[0] __aligned(8); 1009 void *ptrs[0] __aligned(8); 1010 void __percpu *pptrs[0] __aligned(8); 1011 }; 1012 }; 1013 1014 #define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */ 1015 #define MAX_TAIL_CALL_CNT 32 1016 1017 #define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \ 1018 BPF_F_RDONLY_PROG | \ 1019 BPF_F_WRONLY | \ 1020 BPF_F_WRONLY_PROG) 1021 1022 #define BPF_MAP_CAN_READ BIT(0) 1023 #define BPF_MAP_CAN_WRITE BIT(1) 1024 1025 static inline u32 bpf_map_flags_to_cap(struct bpf_map *map) 1026 { 1027 u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); 1028 1029 /* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is 1030 * not possible. 1031 */ 1032 if (access_flags & BPF_F_RDONLY_PROG) 1033 return BPF_MAP_CAN_READ; 1034 else if (access_flags & BPF_F_WRONLY_PROG) 1035 return BPF_MAP_CAN_WRITE; 1036 else 1037 return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE; 1038 } 1039 1040 static inline bool bpf_map_flags_access_ok(u32 access_flags) 1041 { 1042 return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) != 1043 (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); 1044 } 1045 1046 struct bpf_event_entry { 1047 struct perf_event *event; 1048 struct file *perf_file; 1049 struct file *map_file; 1050 struct rcu_head rcu; 1051 }; 1052 1053 bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp); 1054 int bpf_prog_calc_tag(struct bpf_prog *fp); 1055 1056 const struct bpf_func_proto *bpf_get_trace_printk_proto(void); 1057 1058 typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src, 1059 unsigned long off, unsigned long len); 1060 typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type, 1061 const struct bpf_insn *src, 1062 struct bpf_insn *dst, 1063 struct bpf_prog *prog, 1064 u32 *target_size); 1065 1066 u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, 1067 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy); 1068 1069 /* an array of programs to be executed under rcu_lock. 1070 * 1071 * Typical usage: 1072 * ret = BPF_PROG_RUN_ARRAY(&bpf_prog_array, ctx, BPF_PROG_RUN); 1073 * 1074 * the structure returned by bpf_prog_array_alloc() should be populated 1075 * with program pointers and the last pointer must be NULL. 1076 * The user has to keep refcnt on the program and make sure the program 1077 * is removed from the array before bpf_prog_put(). 1078 * The 'struct bpf_prog_array *' should only be replaced with xchg() 1079 * since other cpus are walking the array of pointers in parallel. 1080 */ 1081 struct bpf_prog_array_item { 1082 struct bpf_prog *prog; 1083 struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; 1084 }; 1085 1086 struct bpf_prog_array { 1087 struct rcu_head rcu; 1088 struct bpf_prog_array_item items[]; 1089 }; 1090 1091 struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags); 1092 void bpf_prog_array_free(struct bpf_prog_array *progs); 1093 int bpf_prog_array_length(struct bpf_prog_array *progs); 1094 bool bpf_prog_array_is_empty(struct bpf_prog_array *array); 1095 int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs, 1096 __u32 __user *prog_ids, u32 cnt); 1097 1098 void bpf_prog_array_delete_safe(struct bpf_prog_array *progs, 1099 struct bpf_prog *old_prog); 1100 int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index); 1101 int bpf_prog_array_update_at(struct bpf_prog_array *array, int index, 1102 struct bpf_prog *prog); 1103 int bpf_prog_array_copy_info(struct bpf_prog_array *array, 1104 u32 *prog_ids, u32 request_cnt, 1105 u32 *prog_cnt); 1106 int bpf_prog_array_copy(struct bpf_prog_array *old_array, 1107 struct bpf_prog *exclude_prog, 1108 struct bpf_prog *include_prog, 1109 struct bpf_prog_array **new_array); 1110 1111 /* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */ 1112 #define BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE (1 << 0) 1113 /* BPF program asks to set CN on the packet. */ 1114 #define BPF_RET_SET_CN (1 << 0) 1115 1116 /* For BPF_PROG_RUN_ARRAY_FLAGS and __BPF_PROG_RUN_ARRAY, 1117 * if bpf_cgroup_storage_set() failed, the rest of programs 1118 * will not execute. This should be a really rare scenario 1119 * as it requires BPF_CGROUP_STORAGE_NEST_MAX number of 1120 * preemptions all between bpf_cgroup_storage_set() and 1121 * bpf_cgroup_storage_unset() on the same cpu. 1122 */ 1123 #define BPF_PROG_RUN_ARRAY_FLAGS(array, ctx, func, ret_flags) \ 1124 ({ \ 1125 struct bpf_prog_array_item *_item; \ 1126 struct bpf_prog *_prog; \ 1127 struct bpf_prog_array *_array; \ 1128 u32 _ret = 1; \ 1129 u32 func_ret; \ 1130 migrate_disable(); \ 1131 rcu_read_lock(); \ 1132 _array = rcu_dereference(array); \ 1133 _item = &_array->items[0]; \ 1134 while ((_prog = READ_ONCE(_item->prog))) { \ 1135 if (unlikely(bpf_cgroup_storage_set(_item->cgroup_storage))) \ 1136 break; \ 1137 func_ret = func(_prog, ctx); \ 1138 _ret &= (func_ret & 1); \ 1139 *(ret_flags) |= (func_ret >> 1); \ 1140 bpf_cgroup_storage_unset(); \ 1141 _item++; \ 1142 } \ 1143 rcu_read_unlock(); \ 1144 migrate_enable(); \ 1145 _ret; \ 1146 }) 1147 1148 #define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null, set_cg_storage) \ 1149 ({ \ 1150 struct bpf_prog_array_item *_item; \ 1151 struct bpf_prog *_prog; \ 1152 struct bpf_prog_array *_array; \ 1153 u32 _ret = 1; \ 1154 migrate_disable(); \ 1155 rcu_read_lock(); \ 1156 _array = rcu_dereference(array); \ 1157 if (unlikely(check_non_null && !_array))\ 1158 goto _out; \ 1159 _item = &_array->items[0]; \ 1160 while ((_prog = READ_ONCE(_item->prog))) { \ 1161 if (!set_cg_storage) { \ 1162 _ret &= func(_prog, ctx); \ 1163 } else { \ 1164 if (unlikely(bpf_cgroup_storage_set(_item->cgroup_storage))) \ 1165 break; \ 1166 _ret &= func(_prog, ctx); \ 1167 bpf_cgroup_storage_unset(); \ 1168 } \ 1169 _item++; \ 1170 } \ 1171 _out: \ 1172 rcu_read_unlock(); \ 1173 migrate_enable(); \ 1174 _ret; \ 1175 }) 1176 1177 /* To be used by __cgroup_bpf_run_filter_skb for EGRESS BPF progs 1178 * so BPF programs can request cwr for TCP packets. 1179 * 1180 * Current cgroup skb programs can only return 0 or 1 (0 to drop the 1181 * packet. This macro changes the behavior so the low order bit 1182 * indicates whether the packet should be dropped (0) or not (1) 1183 * and the next bit is a congestion notification bit. This could be 1184 * used by TCP to call tcp_enter_cwr() 1185 * 1186 * Hence, new allowed return values of CGROUP EGRESS BPF programs are: 1187 * 0: drop packet 1188 * 1: keep packet 1189 * 2: drop packet and cn 1190 * 3: keep packet and cn 1191 * 1192 * This macro then converts it to one of the NET_XMIT or an error 1193 * code that is then interpreted as drop packet (and no cn): 1194 * 0: NET_XMIT_SUCCESS skb should be transmitted 1195 * 1: NET_XMIT_DROP skb should be dropped and cn 1196 * 2: NET_XMIT_CN skb should be transmitted and cn 1197 * 3: -EPERM skb should be dropped 1198 */ 1199 #define BPF_PROG_CGROUP_INET_EGRESS_RUN_ARRAY(array, ctx, func) \ 1200 ({ \ 1201 u32 _flags = 0; \ 1202 bool _cn; \ 1203 u32 _ret; \ 1204 _ret = BPF_PROG_RUN_ARRAY_FLAGS(array, ctx, func, &_flags); \ 1205 _cn = _flags & BPF_RET_SET_CN; \ 1206 if (_ret) \ 1207 _ret = (_cn ? NET_XMIT_CN : NET_XMIT_SUCCESS); \ 1208 else \ 1209 _ret = (_cn ? NET_XMIT_DROP : -EPERM); \ 1210 _ret; \ 1211 }) 1212 1213 #define BPF_PROG_RUN_ARRAY(array, ctx, func) \ 1214 __BPF_PROG_RUN_ARRAY(array, ctx, func, false, true) 1215 1216 #define BPF_PROG_RUN_ARRAY_CHECK(array, ctx, func) \ 1217 __BPF_PROG_RUN_ARRAY(array, ctx, func, true, false) 1218 1219 #ifdef CONFIG_BPF_SYSCALL 1220 DECLARE_PER_CPU(int, bpf_prog_active); 1221 extern struct mutex bpf_stats_enabled_mutex; 1222 1223 /* 1224 * Block execution of BPF programs attached to instrumentation (perf, 1225 * kprobes, tracepoints) to prevent deadlocks on map operations as any of 1226 * these events can happen inside a region which holds a map bucket lock 1227 * and can deadlock on it. 1228 * 1229 * Use the preemption safe inc/dec variants on RT because migrate disable 1230 * is preemptible on RT and preemption in the middle of the RMW operation 1231 * might lead to inconsistent state. Use the raw variants for non RT 1232 * kernels as migrate_disable() maps to preempt_disable() so the slightly 1233 * more expensive save operation can be avoided. 1234 */ 1235 static inline void bpf_disable_instrumentation(void) 1236 { 1237 migrate_disable(); 1238 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 1239 this_cpu_inc(bpf_prog_active); 1240 else 1241 __this_cpu_inc(bpf_prog_active); 1242 } 1243 1244 static inline void bpf_enable_instrumentation(void) 1245 { 1246 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 1247 this_cpu_dec(bpf_prog_active); 1248 else 1249 __this_cpu_dec(bpf_prog_active); 1250 migrate_enable(); 1251 } 1252 1253 extern const struct file_operations bpf_map_fops; 1254 extern const struct file_operations bpf_prog_fops; 1255 extern const struct file_operations bpf_iter_fops; 1256 1257 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ 1258 extern const struct bpf_prog_ops _name ## _prog_ops; \ 1259 extern const struct bpf_verifier_ops _name ## _verifier_ops; 1260 #define BPF_MAP_TYPE(_id, _ops) \ 1261 extern const struct bpf_map_ops _ops; 1262 #define BPF_LINK_TYPE(_id, _name) 1263 #include <linux/bpf_types.h> 1264 #undef BPF_PROG_TYPE 1265 #undef BPF_MAP_TYPE 1266 #undef BPF_LINK_TYPE 1267 1268 extern const struct bpf_prog_ops bpf_offload_prog_ops; 1269 extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops; 1270 extern const struct bpf_verifier_ops xdp_analyzer_ops; 1271 1272 struct bpf_prog *bpf_prog_get(u32 ufd); 1273 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, 1274 bool attach_drv); 1275 void bpf_prog_add(struct bpf_prog *prog, int i); 1276 void bpf_prog_sub(struct bpf_prog *prog, int i); 1277 void bpf_prog_inc(struct bpf_prog *prog); 1278 struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog); 1279 void bpf_prog_put(struct bpf_prog *prog); 1280 1281 void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock); 1282 void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock); 1283 1284 struct bpf_map *bpf_map_get(u32 ufd); 1285 struct bpf_map *bpf_map_get_with_uref(u32 ufd); 1286 struct bpf_map *__bpf_map_get(struct fd f); 1287 void bpf_map_inc(struct bpf_map *map); 1288 void bpf_map_inc_with_uref(struct bpf_map *map); 1289 struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map); 1290 void bpf_map_put_with_uref(struct bpf_map *map); 1291 void bpf_map_put(struct bpf_map *map); 1292 void *bpf_map_area_alloc(u64 size, int numa_node); 1293 void *bpf_map_area_mmapable_alloc(u64 size, int numa_node); 1294 void bpf_map_area_free(void *base); 1295 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr); 1296 int generic_map_lookup_batch(struct bpf_map *map, 1297 const union bpf_attr *attr, 1298 union bpf_attr __user *uattr); 1299 int generic_map_update_batch(struct bpf_map *map, 1300 const union bpf_attr *attr, 1301 union bpf_attr __user *uattr); 1302 int generic_map_delete_batch(struct bpf_map *map, 1303 const union bpf_attr *attr, 1304 union bpf_attr __user *uattr); 1305 struct bpf_map *bpf_map_get_curr_or_next(u32 *id); 1306 struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id); 1307 1308 #ifdef CONFIG_MEMCG_KMEM 1309 void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, 1310 int node); 1311 void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags); 1312 void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, 1313 size_t align, gfp_t flags); 1314 #else 1315 static inline void * 1316 bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, 1317 int node) 1318 { 1319 return kmalloc_node(size, flags, node); 1320 } 1321 1322 static inline void * 1323 bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags) 1324 { 1325 return kzalloc(size, flags); 1326 } 1327 1328 static inline void __percpu * 1329 bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, size_t align, 1330 gfp_t flags) 1331 { 1332 return __alloc_percpu_gfp(size, align, flags); 1333 } 1334 #endif 1335 1336 extern int sysctl_unprivileged_bpf_disabled; 1337 1338 static inline bool bpf_allow_ptr_leaks(void) 1339 { 1340 return perfmon_capable(); 1341 } 1342 1343 static inline bool bpf_allow_uninit_stack(void) 1344 { 1345 return perfmon_capable(); 1346 } 1347 1348 static inline bool bpf_allow_ptr_to_map_access(void) 1349 { 1350 return perfmon_capable(); 1351 } 1352 1353 static inline bool bpf_bypass_spec_v1(void) 1354 { 1355 return perfmon_capable(); 1356 } 1357 1358 static inline bool bpf_bypass_spec_v4(void) 1359 { 1360 return perfmon_capable(); 1361 } 1362 1363 int bpf_map_new_fd(struct bpf_map *map, int flags); 1364 int bpf_prog_new_fd(struct bpf_prog *prog); 1365 1366 void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, 1367 const struct bpf_link_ops *ops, struct bpf_prog *prog); 1368 int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer); 1369 int bpf_link_settle(struct bpf_link_primer *primer); 1370 void bpf_link_cleanup(struct bpf_link_primer *primer); 1371 void bpf_link_inc(struct bpf_link *link); 1372 void bpf_link_put(struct bpf_link *link); 1373 int bpf_link_new_fd(struct bpf_link *link); 1374 struct file *bpf_link_new_file(struct bpf_link *link, int *reserved_fd); 1375 struct bpf_link *bpf_link_get_from_fd(u32 ufd); 1376 1377 int bpf_obj_pin_user(u32 ufd, const char __user *pathname); 1378 int bpf_obj_get_user(const char __user *pathname, int flags); 1379 1380 #define BPF_ITER_FUNC_PREFIX "bpf_iter_" 1381 #define DEFINE_BPF_ITER_FUNC(target, args...) \ 1382 extern int bpf_iter_ ## target(args); \ 1383 int __init bpf_iter_ ## target(args) { return 0; } 1384 1385 struct bpf_iter_aux_info { 1386 struct bpf_map *map; 1387 }; 1388 1389 typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog, 1390 union bpf_iter_link_info *linfo, 1391 struct bpf_iter_aux_info *aux); 1392 typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux); 1393 typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux, 1394 struct seq_file *seq); 1395 typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux, 1396 struct bpf_link_info *info); 1397 1398 enum bpf_iter_feature { 1399 BPF_ITER_RESCHED = BIT(0), 1400 }; 1401 1402 #define BPF_ITER_CTX_ARG_MAX 2 1403 struct bpf_iter_reg { 1404 const char *target; 1405 bpf_iter_attach_target_t attach_target; 1406 bpf_iter_detach_target_t detach_target; 1407 bpf_iter_show_fdinfo_t show_fdinfo; 1408 bpf_iter_fill_link_info_t fill_link_info; 1409 u32 ctx_arg_info_size; 1410 u32 feature; 1411 struct bpf_ctx_arg_aux ctx_arg_info[BPF_ITER_CTX_ARG_MAX]; 1412 const struct bpf_iter_seq_info *seq_info; 1413 }; 1414 1415 struct bpf_iter_meta { 1416 __bpf_md_ptr(struct seq_file *, seq); 1417 u64 session_id; 1418 u64 seq_num; 1419 }; 1420 1421 struct bpf_iter__bpf_map_elem { 1422 __bpf_md_ptr(struct bpf_iter_meta *, meta); 1423 __bpf_md_ptr(struct bpf_map *, map); 1424 __bpf_md_ptr(void *, key); 1425 __bpf_md_ptr(void *, value); 1426 }; 1427 1428 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info); 1429 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info); 1430 bool bpf_iter_prog_supported(struct bpf_prog *prog); 1431 int bpf_iter_link_attach(const union bpf_attr *attr, struct bpf_prog *prog); 1432 int bpf_iter_new_fd(struct bpf_link *link); 1433 bool bpf_link_is_iter(struct bpf_link *link); 1434 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop); 1435 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx); 1436 void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux, 1437 struct seq_file *seq); 1438 int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux, 1439 struct bpf_link_info *info); 1440 1441 int map_set_for_each_callback_args(struct bpf_verifier_env *env, 1442 struct bpf_func_state *caller, 1443 struct bpf_func_state *callee); 1444 1445 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value); 1446 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value); 1447 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, 1448 u64 flags); 1449 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, 1450 u64 flags); 1451 1452 int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value); 1453 1454 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, 1455 void *key, void *value, u64 map_flags); 1456 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); 1457 int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, 1458 void *key, void *value, u64 map_flags); 1459 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); 1460 1461 int bpf_get_file_flag(int flags); 1462 int bpf_check_uarg_tail_zero(void __user *uaddr, size_t expected_size, 1463 size_t actual_size); 1464 1465 /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and 1466 * forced to use 'long' read/writes to try to atomically copy long counters. 1467 * Best-effort only. No barriers here, since it _will_ race with concurrent 1468 * updates from BPF programs. Called from bpf syscall and mostly used with 1469 * size 8 or 16 bytes, so ask compiler to inline it. 1470 */ 1471 static inline void bpf_long_memcpy(void *dst, const void *src, u32 size) 1472 { 1473 const long *lsrc = src; 1474 long *ldst = dst; 1475 1476 size /= sizeof(long); 1477 while (size--) 1478 *ldst++ = *lsrc++; 1479 } 1480 1481 /* verify correctness of eBPF program */ 1482 int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, 1483 union bpf_attr __user *uattr); 1484 1485 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 1486 void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); 1487 #endif 1488 1489 struct btf *bpf_get_btf_vmlinux(void); 1490 1491 /* Map specifics */ 1492 struct xdp_buff; 1493 struct sk_buff; 1494 struct bpf_dtab_netdev; 1495 struct bpf_cpu_map_entry; 1496 1497 void __dev_flush(void); 1498 int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp, 1499 struct net_device *dev_rx); 1500 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, 1501 struct net_device *dev_rx); 1502 int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, 1503 struct bpf_prog *xdp_prog); 1504 bool dev_map_can_have_prog(struct bpf_map *map); 1505 1506 void __cpu_map_flush(void); 1507 int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_buff *xdp, 1508 struct net_device *dev_rx); 1509 bool cpu_map_prog_allowed(struct bpf_map *map); 1510 1511 /* Return map's numa specified by userspace */ 1512 static inline int bpf_map_attr_numa_node(const union bpf_attr *attr) 1513 { 1514 return (attr->map_flags & BPF_F_NUMA_NODE) ? 1515 attr->numa_node : NUMA_NO_NODE; 1516 } 1517 1518 struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type); 1519 int array_map_alloc_check(union bpf_attr *attr); 1520 1521 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 1522 union bpf_attr __user *uattr); 1523 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 1524 union bpf_attr __user *uattr); 1525 int bpf_prog_test_run_tracing(struct bpf_prog *prog, 1526 const union bpf_attr *kattr, 1527 union bpf_attr __user *uattr); 1528 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 1529 const union bpf_attr *kattr, 1530 union bpf_attr __user *uattr); 1531 int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, 1532 const union bpf_attr *kattr, 1533 union bpf_attr __user *uattr); 1534 int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, 1535 const union bpf_attr *kattr, 1536 union bpf_attr __user *uattr); 1537 bool bpf_prog_test_check_kfunc_call(u32 kfunc_id); 1538 bool btf_ctx_access(int off, int size, enum bpf_access_type type, 1539 const struct bpf_prog *prog, 1540 struct bpf_insn_access_aux *info); 1541 int btf_struct_access(struct bpf_verifier_log *log, const struct btf *btf, 1542 const struct btf_type *t, int off, int size, 1543 enum bpf_access_type atype, 1544 u32 *next_btf_id); 1545 bool btf_struct_ids_match(struct bpf_verifier_log *log, 1546 const struct btf *btf, u32 id, int off, 1547 const struct btf *need_btf, u32 need_type_id); 1548 1549 int btf_distill_func_proto(struct bpf_verifier_log *log, 1550 struct btf *btf, 1551 const struct btf_type *func_proto, 1552 const char *func_name, 1553 struct btf_func_model *m); 1554 1555 struct bpf_reg_state; 1556 int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, 1557 struct bpf_reg_state *regs); 1558 int btf_check_kfunc_arg_match(struct bpf_verifier_env *env, 1559 const struct btf *btf, u32 func_id, 1560 struct bpf_reg_state *regs); 1561 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, 1562 struct bpf_reg_state *reg); 1563 int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog, 1564 struct btf *btf, const struct btf_type *t); 1565 1566 struct bpf_prog *bpf_prog_by_id(u32 id); 1567 struct bpf_link *bpf_link_by_id(u32 id); 1568 1569 const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id); 1570 void bpf_task_storage_free(struct task_struct *task); 1571 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog); 1572 const struct btf_func_model * 1573 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 1574 const struct bpf_insn *insn); 1575 #else /* !CONFIG_BPF_SYSCALL */ 1576 static inline struct bpf_prog *bpf_prog_get(u32 ufd) 1577 { 1578 return ERR_PTR(-EOPNOTSUPP); 1579 } 1580 1581 static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, 1582 enum bpf_prog_type type, 1583 bool attach_drv) 1584 { 1585 return ERR_PTR(-EOPNOTSUPP); 1586 } 1587 1588 static inline void bpf_prog_add(struct bpf_prog *prog, int i) 1589 { 1590 } 1591 1592 static inline void bpf_prog_sub(struct bpf_prog *prog, int i) 1593 { 1594 } 1595 1596 static inline void bpf_prog_put(struct bpf_prog *prog) 1597 { 1598 } 1599 1600 static inline void bpf_prog_inc(struct bpf_prog *prog) 1601 { 1602 } 1603 1604 static inline struct bpf_prog *__must_check 1605 bpf_prog_inc_not_zero(struct bpf_prog *prog) 1606 { 1607 return ERR_PTR(-EOPNOTSUPP); 1608 } 1609 1610 static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, 1611 const struct bpf_link_ops *ops, 1612 struct bpf_prog *prog) 1613 { 1614 } 1615 1616 static inline int bpf_link_prime(struct bpf_link *link, 1617 struct bpf_link_primer *primer) 1618 { 1619 return -EOPNOTSUPP; 1620 } 1621 1622 static inline int bpf_link_settle(struct bpf_link_primer *primer) 1623 { 1624 return -EOPNOTSUPP; 1625 } 1626 1627 static inline void bpf_link_cleanup(struct bpf_link_primer *primer) 1628 { 1629 } 1630 1631 static inline void bpf_link_inc(struct bpf_link *link) 1632 { 1633 } 1634 1635 static inline void bpf_link_put(struct bpf_link *link) 1636 { 1637 } 1638 1639 static inline int bpf_obj_get_user(const char __user *pathname, int flags) 1640 { 1641 return -EOPNOTSUPP; 1642 } 1643 1644 static inline bool dev_map_can_have_prog(struct bpf_map *map) 1645 { 1646 return false; 1647 } 1648 1649 static inline void __dev_flush(void) 1650 { 1651 } 1652 1653 struct xdp_buff; 1654 struct bpf_dtab_netdev; 1655 struct bpf_cpu_map_entry; 1656 1657 static inline 1658 int dev_xdp_enqueue(struct net_device *dev, struct xdp_buff *xdp, 1659 struct net_device *dev_rx) 1660 { 1661 return 0; 1662 } 1663 1664 static inline 1665 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_buff *xdp, 1666 struct net_device *dev_rx) 1667 { 1668 return 0; 1669 } 1670 1671 struct sk_buff; 1672 1673 static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, 1674 struct sk_buff *skb, 1675 struct bpf_prog *xdp_prog) 1676 { 1677 return 0; 1678 } 1679 1680 static inline void __cpu_map_flush(void) 1681 { 1682 } 1683 1684 static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, 1685 struct xdp_buff *xdp, 1686 struct net_device *dev_rx) 1687 { 1688 return 0; 1689 } 1690 1691 static inline bool cpu_map_prog_allowed(struct bpf_map *map) 1692 { 1693 return false; 1694 } 1695 1696 static inline struct bpf_prog *bpf_prog_get_type_path(const char *name, 1697 enum bpf_prog_type type) 1698 { 1699 return ERR_PTR(-EOPNOTSUPP); 1700 } 1701 1702 static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog, 1703 const union bpf_attr *kattr, 1704 union bpf_attr __user *uattr) 1705 { 1706 return -ENOTSUPP; 1707 } 1708 1709 static inline int bpf_prog_test_run_skb(struct bpf_prog *prog, 1710 const union bpf_attr *kattr, 1711 union bpf_attr __user *uattr) 1712 { 1713 return -ENOTSUPP; 1714 } 1715 1716 static inline int bpf_prog_test_run_tracing(struct bpf_prog *prog, 1717 const union bpf_attr *kattr, 1718 union bpf_attr __user *uattr) 1719 { 1720 return -ENOTSUPP; 1721 } 1722 1723 static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, 1724 const union bpf_attr *kattr, 1725 union bpf_attr __user *uattr) 1726 { 1727 return -ENOTSUPP; 1728 } 1729 1730 static inline int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, 1731 const union bpf_attr *kattr, 1732 union bpf_attr __user *uattr) 1733 { 1734 return -ENOTSUPP; 1735 } 1736 1737 static inline bool bpf_prog_test_check_kfunc_call(u32 kfunc_id) 1738 { 1739 return false; 1740 } 1741 1742 static inline void bpf_map_put(struct bpf_map *map) 1743 { 1744 } 1745 1746 static inline struct bpf_prog *bpf_prog_by_id(u32 id) 1747 { 1748 return ERR_PTR(-ENOTSUPP); 1749 } 1750 1751 static inline const struct bpf_func_proto * 1752 bpf_base_func_proto(enum bpf_func_id func_id) 1753 { 1754 return NULL; 1755 } 1756 1757 static inline void bpf_task_storage_free(struct task_struct *task) 1758 { 1759 } 1760 1761 static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) 1762 { 1763 return false; 1764 } 1765 1766 static inline const struct btf_func_model * 1767 bpf_jit_find_kfunc_model(const struct bpf_prog *prog, 1768 const struct bpf_insn *insn) 1769 { 1770 return NULL; 1771 } 1772 #endif /* CONFIG_BPF_SYSCALL */ 1773 1774 void __bpf_free_used_btfs(struct bpf_prog_aux *aux, 1775 struct btf_mod_pair *used_btfs, u32 len); 1776 1777 static inline struct bpf_prog *bpf_prog_get_type(u32 ufd, 1778 enum bpf_prog_type type) 1779 { 1780 return bpf_prog_get_type_dev(ufd, type, false); 1781 } 1782 1783 void __bpf_free_used_maps(struct bpf_prog_aux *aux, 1784 struct bpf_map **used_maps, u32 len); 1785 1786 bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool); 1787 1788 int bpf_prog_offload_compile(struct bpf_prog *prog); 1789 void bpf_prog_offload_destroy(struct bpf_prog *prog); 1790 int bpf_prog_offload_info_fill(struct bpf_prog_info *info, 1791 struct bpf_prog *prog); 1792 1793 int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map); 1794 1795 int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value); 1796 int bpf_map_offload_update_elem(struct bpf_map *map, 1797 void *key, void *value, u64 flags); 1798 int bpf_map_offload_delete_elem(struct bpf_map *map, void *key); 1799 int bpf_map_offload_get_next_key(struct bpf_map *map, 1800 void *key, void *next_key); 1801 1802 bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map); 1803 1804 struct bpf_offload_dev * 1805 bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv); 1806 void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev); 1807 void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev); 1808 int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev, 1809 struct net_device *netdev); 1810 void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev, 1811 struct net_device *netdev); 1812 bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev); 1813 1814 #if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL) 1815 int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr); 1816 1817 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) 1818 { 1819 return aux->offload_requested; 1820 } 1821 1822 static inline bool bpf_map_is_dev_bound(struct bpf_map *map) 1823 { 1824 return unlikely(map->ops == &bpf_map_offload_ops); 1825 } 1826 1827 struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr); 1828 void bpf_map_offload_map_free(struct bpf_map *map); 1829 #else 1830 static inline int bpf_prog_offload_init(struct bpf_prog *prog, 1831 union bpf_attr *attr) 1832 { 1833 return -EOPNOTSUPP; 1834 } 1835 1836 static inline bool bpf_prog_is_dev_bound(struct bpf_prog_aux *aux) 1837 { 1838 return false; 1839 } 1840 1841 static inline bool bpf_map_is_dev_bound(struct bpf_map *map) 1842 { 1843 return false; 1844 } 1845 1846 static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) 1847 { 1848 return ERR_PTR(-EOPNOTSUPP); 1849 } 1850 1851 static inline void bpf_map_offload_map_free(struct bpf_map *map) 1852 { 1853 } 1854 #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */ 1855 1856 #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) 1857 int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog); 1858 int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype); 1859 int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, u64 flags); 1860 void sock_map_unhash(struct sock *sk); 1861 void sock_map_close(struct sock *sk, long timeout); 1862 1863 void bpf_sk_reuseport_detach(struct sock *sk); 1864 int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, 1865 void *value); 1866 int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key, 1867 void *value, u64 map_flags); 1868 #else 1869 static inline void bpf_sk_reuseport_detach(struct sock *sk) 1870 { 1871 } 1872 1873 #ifdef CONFIG_BPF_SYSCALL 1874 static inline int sock_map_get_from_fd(const union bpf_attr *attr, 1875 struct bpf_prog *prog) 1876 { 1877 return -EINVAL; 1878 } 1879 1880 static inline int sock_map_prog_detach(const union bpf_attr *attr, 1881 enum bpf_prog_type ptype) 1882 { 1883 return -EOPNOTSUPP; 1884 } 1885 1886 static inline int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, 1887 u64 flags) 1888 { 1889 return -EOPNOTSUPP; 1890 } 1891 1892 static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, 1893 void *key, void *value) 1894 { 1895 return -EOPNOTSUPP; 1896 } 1897 1898 static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, 1899 void *key, void *value, 1900 u64 map_flags) 1901 { 1902 return -EOPNOTSUPP; 1903 } 1904 #endif /* CONFIG_BPF_SYSCALL */ 1905 #endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */ 1906 1907 /* verifier prototypes for helper functions called from eBPF programs */ 1908 extern const struct bpf_func_proto bpf_map_lookup_elem_proto; 1909 extern const struct bpf_func_proto bpf_map_update_elem_proto; 1910 extern const struct bpf_func_proto bpf_map_delete_elem_proto; 1911 extern const struct bpf_func_proto bpf_map_push_elem_proto; 1912 extern const struct bpf_func_proto bpf_map_pop_elem_proto; 1913 extern const struct bpf_func_proto bpf_map_peek_elem_proto; 1914 1915 extern const struct bpf_func_proto bpf_get_prandom_u32_proto; 1916 extern const struct bpf_func_proto bpf_get_smp_processor_id_proto; 1917 extern const struct bpf_func_proto bpf_get_numa_node_id_proto; 1918 extern const struct bpf_func_proto bpf_tail_call_proto; 1919 extern const struct bpf_func_proto bpf_ktime_get_ns_proto; 1920 extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto; 1921 extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto; 1922 extern const struct bpf_func_proto bpf_get_current_uid_gid_proto; 1923 extern const struct bpf_func_proto bpf_get_current_comm_proto; 1924 extern const struct bpf_func_proto bpf_get_stackid_proto; 1925 extern const struct bpf_func_proto bpf_get_stack_proto; 1926 extern const struct bpf_func_proto bpf_get_task_stack_proto; 1927 extern const struct bpf_func_proto bpf_get_stackid_proto_pe; 1928 extern const struct bpf_func_proto bpf_get_stack_proto_pe; 1929 extern const struct bpf_func_proto bpf_sock_map_update_proto; 1930 extern const struct bpf_func_proto bpf_sock_hash_update_proto; 1931 extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto; 1932 extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto; 1933 extern const struct bpf_func_proto bpf_msg_redirect_hash_proto; 1934 extern const struct bpf_func_proto bpf_msg_redirect_map_proto; 1935 extern const struct bpf_func_proto bpf_sk_redirect_hash_proto; 1936 extern const struct bpf_func_proto bpf_sk_redirect_map_proto; 1937 extern const struct bpf_func_proto bpf_spin_lock_proto; 1938 extern const struct bpf_func_proto bpf_spin_unlock_proto; 1939 extern const struct bpf_func_proto bpf_get_local_storage_proto; 1940 extern const struct bpf_func_proto bpf_strtol_proto; 1941 extern const struct bpf_func_proto bpf_strtoul_proto; 1942 extern const struct bpf_func_proto bpf_tcp_sock_proto; 1943 extern const struct bpf_func_proto bpf_jiffies64_proto; 1944 extern const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto; 1945 extern const struct bpf_func_proto bpf_event_output_data_proto; 1946 extern const struct bpf_func_proto bpf_ringbuf_output_proto; 1947 extern const struct bpf_func_proto bpf_ringbuf_reserve_proto; 1948 extern const struct bpf_func_proto bpf_ringbuf_submit_proto; 1949 extern const struct bpf_func_proto bpf_ringbuf_discard_proto; 1950 extern const struct bpf_func_proto bpf_ringbuf_query_proto; 1951 extern const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto; 1952 extern const struct bpf_func_proto bpf_skc_to_tcp_sock_proto; 1953 extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto; 1954 extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto; 1955 extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto; 1956 extern const struct bpf_func_proto bpf_copy_from_user_proto; 1957 extern const struct bpf_func_proto bpf_snprintf_btf_proto; 1958 extern const struct bpf_func_proto bpf_snprintf_proto; 1959 extern const struct bpf_func_proto bpf_per_cpu_ptr_proto; 1960 extern const struct bpf_func_proto bpf_this_cpu_ptr_proto; 1961 extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto; 1962 extern const struct bpf_func_proto bpf_sock_from_file_proto; 1963 extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto; 1964 extern const struct bpf_func_proto bpf_task_storage_get_proto; 1965 extern const struct bpf_func_proto bpf_task_storage_delete_proto; 1966 extern const struct bpf_func_proto bpf_for_each_map_elem_proto; 1967 1968 const struct bpf_func_proto *bpf_tracing_func_proto( 1969 enum bpf_func_id func_id, const struct bpf_prog *prog); 1970 1971 const struct bpf_func_proto *tracing_prog_func_proto( 1972 enum bpf_func_id func_id, const struct bpf_prog *prog); 1973 1974 /* Shared helpers among cBPF and eBPF. */ 1975 void bpf_user_rnd_init_once(void); 1976 u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 1977 u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 1978 1979 #if defined(CONFIG_NET) 1980 bool bpf_sock_common_is_valid_access(int off, int size, 1981 enum bpf_access_type type, 1982 struct bpf_insn_access_aux *info); 1983 bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type, 1984 struct bpf_insn_access_aux *info); 1985 u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, 1986 const struct bpf_insn *si, 1987 struct bpf_insn *insn_buf, 1988 struct bpf_prog *prog, 1989 u32 *target_size); 1990 #else 1991 static inline bool bpf_sock_common_is_valid_access(int off, int size, 1992 enum bpf_access_type type, 1993 struct bpf_insn_access_aux *info) 1994 { 1995 return false; 1996 } 1997 static inline bool bpf_sock_is_valid_access(int off, int size, 1998 enum bpf_access_type type, 1999 struct bpf_insn_access_aux *info) 2000 { 2001 return false; 2002 } 2003 static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, 2004 const struct bpf_insn *si, 2005 struct bpf_insn *insn_buf, 2006 struct bpf_prog *prog, 2007 u32 *target_size) 2008 { 2009 return 0; 2010 } 2011 #endif 2012 2013 #ifdef CONFIG_INET 2014 struct sk_reuseport_kern { 2015 struct sk_buff *skb; 2016 struct sock *sk; 2017 struct sock *selected_sk; 2018 void *data_end; 2019 u32 hash; 2020 u32 reuseport_id; 2021 bool bind_inany; 2022 }; 2023 bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type, 2024 struct bpf_insn_access_aux *info); 2025 2026 u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, 2027 const struct bpf_insn *si, 2028 struct bpf_insn *insn_buf, 2029 struct bpf_prog *prog, 2030 u32 *target_size); 2031 2032 bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type, 2033 struct bpf_insn_access_aux *info); 2034 2035 u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, 2036 const struct bpf_insn *si, 2037 struct bpf_insn *insn_buf, 2038 struct bpf_prog *prog, 2039 u32 *target_size); 2040 #else 2041 static inline bool bpf_tcp_sock_is_valid_access(int off, int size, 2042 enum bpf_access_type type, 2043 struct bpf_insn_access_aux *info) 2044 { 2045 return false; 2046 } 2047 2048 static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, 2049 const struct bpf_insn *si, 2050 struct bpf_insn *insn_buf, 2051 struct bpf_prog *prog, 2052 u32 *target_size) 2053 { 2054 return 0; 2055 } 2056 static inline bool bpf_xdp_sock_is_valid_access(int off, int size, 2057 enum bpf_access_type type, 2058 struct bpf_insn_access_aux *info) 2059 { 2060 return false; 2061 } 2062 2063 static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, 2064 const struct bpf_insn *si, 2065 struct bpf_insn *insn_buf, 2066 struct bpf_prog *prog, 2067 u32 *target_size) 2068 { 2069 return 0; 2070 } 2071 #endif /* CONFIG_INET */ 2072 2073 enum bpf_text_poke_type { 2074 BPF_MOD_CALL, 2075 BPF_MOD_JUMP, 2076 }; 2077 2078 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, 2079 void *addr1, void *addr2); 2080 2081 struct btf_id_set; 2082 bool btf_id_set_contains(const struct btf_id_set *set, u32 id); 2083 2084 int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args, 2085 u32 **bin_buf, u32 num_args); 2086 void bpf_bprintf_cleanup(void); 2087 2088 #endif /* _LINUX_BPF_H */ 2089