Lines Matching refs:btf
252 struct btf { struct
272 struct btf *base_btf; argument
313 struct btf *btf; member
419 const struct btf *btf; member
458 void (*show)(const struct btf *btf, const struct btf_type *t,
529 bool btf_is_vmlinux(const struct btf *btf) in btf_is_vmlinux() argument
531 return btf->kernel_btf && !btf->base_btf; in btf_is_vmlinux()
534 u32 btf_nr_types(const struct btf *btf) in btf_nr_types() argument
538 while (btf) { in btf_nr_types()
539 total += btf->nr_types; in btf_nr_types()
540 btf = btf->base_btf; in btf_nr_types()
546 s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind) in btf_find_by_name_kind() argument
552 total = btf_nr_types(btf); in btf_find_by_name_kind()
554 t = btf_type_by_id(btf, i); in btf_find_by_name_kind()
558 tname = btf_name_by_offset(btf, t->name_off); in btf_find_by_name_kind()
566 s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p) in bpf_find_btf_id()
568 struct btf *btf; in bpf_find_btf_id() local
572 btf = bpf_get_btf_vmlinux(); in bpf_find_btf_id()
573 if (IS_ERR(btf)) in bpf_find_btf_id()
574 return PTR_ERR(btf); in bpf_find_btf_id()
575 if (!btf) in bpf_find_btf_id()
578 ret = btf_find_by_name_kind(btf, name, kind); in bpf_find_btf_id()
583 btf_get(btf); in bpf_find_btf_id()
584 *btf_p = btf; in bpf_find_btf_id()
590 idr_for_each_entry(&btf_idr, btf, id) { in bpf_find_btf_id()
591 if (!btf_is_module(btf)) in bpf_find_btf_id()
596 btf_get(btf); in bpf_find_btf_id()
598 ret = btf_find_by_name_kind(btf, name, kind); in bpf_find_btf_id()
600 *btf_p = btf; in bpf_find_btf_id()
603 btf_put(btf); in bpf_find_btf_id()
611 const struct btf_type *btf_type_skip_modifiers(const struct btf *btf, in btf_type_skip_modifiers() argument
614 const struct btf_type *t = btf_type_by_id(btf, id); in btf_type_skip_modifiers()
618 t = btf_type_by_id(btf, t->type); in btf_type_skip_modifiers()
627 const struct btf_type *btf_type_resolve_ptr(const struct btf *btf, in btf_type_resolve_ptr() argument
632 t = btf_type_skip_modifiers(btf, id, NULL); in btf_type_resolve_ptr()
636 return btf_type_skip_modifiers(btf, t->type, res_id); in btf_type_resolve_ptr()
639 const struct btf_type *btf_type_resolve_func_ptr(const struct btf *btf, in btf_type_resolve_func_ptr() argument
644 ptype = btf_type_resolve_ptr(btf, id, res_id); in btf_type_resolve_func_ptr()
755 static bool btf_name_offset_valid(const struct btf *btf, u32 offset) in btf_name_offset_valid() argument
760 while (offset < btf->start_str_off) in btf_name_offset_valid()
761 btf = btf->base_btf; in btf_name_offset_valid()
763 offset -= btf->start_str_off; in btf_name_offset_valid()
764 return offset < btf->hdr.str_len; in btf_name_offset_valid()
777 const char *btf_str_by_offset(const struct btf *btf, u32 offset) in btf_str_by_offset() argument
779 while (offset < btf->start_str_off) in btf_str_by_offset()
780 btf = btf->base_btf; in btf_str_by_offset()
782 offset -= btf->start_str_off; in btf_str_by_offset()
783 if (offset < btf->hdr.str_len) in btf_str_by_offset()
784 return &btf->strings[offset]; in btf_str_by_offset()
789 static bool btf_name_valid_identifier(const struct btf *btf, u32 offset) in btf_name_valid_identifier() argument
792 const char *src = btf_str_by_offset(btf, offset); in btf_name_valid_identifier()
811 static bool btf_name_valid_section(const struct btf *btf, u32 offset) in btf_name_valid_section() argument
814 const char *src = btf_str_by_offset(btf, offset); in btf_name_valid_section()
831 static const char *__btf_name_by_offset(const struct btf *btf, u32 offset) in __btf_name_by_offset() argument
838 name = btf_str_by_offset(btf, offset); in __btf_name_by_offset()
842 const char *btf_name_by_offset(const struct btf *btf, u32 offset) in btf_name_by_offset() argument
844 return btf_str_by_offset(btf, offset); in btf_name_by_offset()
847 const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id) in btf_type_by_id() argument
849 while (type_id < btf->start_id) in btf_type_by_id()
850 btf = btf->base_btf; in btf_type_by_id()
852 type_id -= btf->start_id; in btf_type_by_id()
853 if (type_id >= btf->nr_types) in btf_type_by_id()
855 return btf->types[type_id]; in btf_type_by_id()
886 bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s, in btf_member_is_reg_int() argument
895 t = btf_type_id_size(btf, &id, NULL); in btf_member_is_reg_int()
924 static const struct btf_type *btf_type_skip_qualifiers(const struct btf *btf, in btf_type_skip_qualifiers() argument
927 const struct btf_type *t = btf_type_by_id(btf, id); in btf_type_skip_qualifiers()
931 t = btf_type_by_id(btf, t->type); in btf_type_skip_qualifiers()
977 member = btf_name_by_offset(show->btf, m->name_off); in btf_show_name()
989 t = btf_type_by_id(show->btf, id); in btf_show_name()
1020 name = btf_name_by_offset(show->btf, in btf_show_name()
1047 t = btf_type_skip_qualifiers(show->btf, id); in btf_show_name()
1054 name = btf_name_by_offset(show->btf, t->name_off); in btf_show_name()
1244 rt = btf_resolve_size(show->btf, t, &size); in btf_show_obj_safe()
1436 struct btf *btf = env->btf; in __btf_verifier_log_type() local
1451 if (env->btf->base_btf && IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) in __btf_verifier_log_type()
1458 __btf_name_by_offset(btf, t->name_off), in __btf_verifier_log_type()
1486 struct btf *btf = env->btf; in btf_verifier_log_member() local
1497 if (env->btf->base_btf && IS_ENABLED(CONFIG_MODULE_ALLOW_BTF_MISMATCH)) in btf_verifier_log_member()
1513 __btf_name_by_offset(btf, member->name_off), in btf_verifier_log_member()
1519 __btf_name_by_offset(btf, member->name_off), in btf_verifier_log_member()
1564 const struct btf *btf = env->btf; in btf_verifier_log_hdr() local
1572 hdr = &btf->hdr; in btf_verifier_log_hdr()
1586 struct btf *btf = env->btf; in btf_add_type() local
1588 if (btf->types_size == btf->nr_types) { in btf_add_type()
1594 if (btf->start_id + btf->types_size == BTF_MAX_TYPE) { in btf_add_type()
1599 expand_by = max_t(u32, btf->types_size >> 2, 16); in btf_add_type()
1601 btf->types_size + expand_by); in btf_add_type()
1608 if (btf->nr_types == 0) { in btf_add_type()
1609 if (!btf->base_btf) { in btf_add_type()
1612 btf->nr_types++; in btf_add_type()
1615 memcpy(new_types, btf->types, in btf_add_type()
1616 sizeof(*btf->types) * btf->nr_types); in btf_add_type()
1619 kvfree(btf->types); in btf_add_type()
1620 btf->types = new_types; in btf_add_type()
1621 btf->types_size = new_size; in btf_add_type()
1624 btf->types[btf->nr_types++] = t; in btf_add_type()
1629 static int btf_alloc_id(struct btf *btf) in btf_alloc_id() argument
1635 id = idr_alloc_cyclic(&btf_idr, btf, 1, INT_MAX, GFP_ATOMIC); in btf_alloc_id()
1637 btf->id = id; in btf_alloc_id()
1647 static void btf_free_id(struct btf *btf) in btf_free_id() argument
1661 idr_remove(&btf_idr, btf->id); in btf_free_id()
1665 static void btf_free_kfunc_set_tab(struct btf *btf) in btf_free_kfunc_set_tab() argument
1667 struct btf_kfunc_set_tab *tab = btf->kfunc_set_tab; in btf_free_kfunc_set_tab()
1675 btf->kfunc_set_tab = NULL; in btf_free_kfunc_set_tab()
1678 static void btf_free_dtor_kfunc_tab(struct btf *btf) in btf_free_dtor_kfunc_tab() argument
1680 struct btf_id_dtor_kfunc_tab *tab = btf->dtor_kfunc_tab; in btf_free_dtor_kfunc_tab()
1685 btf->dtor_kfunc_tab = NULL; in btf_free_dtor_kfunc_tab()
1699 static void btf_free_struct_meta_tab(struct btf *btf) in btf_free_struct_meta_tab() argument
1701 struct btf_struct_metas *tab = btf->struct_meta_tab; in btf_free_struct_meta_tab()
1704 btf->struct_meta_tab = NULL; in btf_free_struct_meta_tab()
1707 static void btf_free_struct_ops_tab(struct btf *btf) in btf_free_struct_ops_tab() argument
1709 struct btf_struct_ops_tab *tab = btf->struct_ops_tab; in btf_free_struct_ops_tab()
1719 btf->struct_ops_tab = NULL; in btf_free_struct_ops_tab()
1722 static void btf_free(struct btf *btf) in btf_free() argument
1724 btf_free_struct_meta_tab(btf); in btf_free()
1725 btf_free_dtor_kfunc_tab(btf); in btf_free()
1726 btf_free_kfunc_set_tab(btf); in btf_free()
1727 btf_free_struct_ops_tab(btf); in btf_free()
1728 kvfree(btf->types); in btf_free()
1729 kvfree(btf->resolved_sizes); in btf_free()
1730 kvfree(btf->resolved_ids); in btf_free()
1734 if (!btf_is_vmlinux(btf)) in btf_free()
1735 kvfree(btf->data); in btf_free()
1736 kvfree(btf->base_id_map); in btf_free()
1737 kfree(btf); in btf_free()
1742 struct btf *btf = container_of(rcu, struct btf, rcu); in btf_free_rcu() local
1744 btf_free(btf); in btf_free_rcu()
1747 const char *btf_get_name(const struct btf *btf) in btf_get_name() argument
1749 return btf->name; in btf_get_name()
1752 void btf_get(struct btf *btf) in btf_get() argument
1754 refcount_inc(&btf->refcnt); in btf_get()
1757 void btf_put(struct btf *btf) in btf_put() argument
1759 if (btf && refcount_dec_and_test(&btf->refcnt)) { in btf_put()
1760 btf_free_id(btf); in btf_put()
1761 call_rcu(&btf->rcu, btf_free_rcu); in btf_put()
1765 struct btf *btf_base_btf(const struct btf *btf) in btf_base_btf() argument
1767 return btf->base_btf; in btf_base_btf()
1770 const struct btf_header *btf_header(const struct btf *btf) in btf_header() argument
1772 return &btf->hdr; in btf_header()
1775 void btf_set_base_btf(struct btf *btf, const struct btf *base_btf) in btf_set_base_btf() argument
1777 btf->base_btf = (struct btf *)base_btf; in btf_set_base_btf()
1778 btf->start_id = btf_nr_types(base_btf); in btf_set_base_btf()
1779 btf->start_str_off = base_btf->hdr.str_len; in btf_set_base_btf()
1784 struct btf *btf = env->btf; in env_resolve_init() local
1785 u32 nr_types = btf->nr_types; in env_resolve_init()
1805 btf->resolved_sizes = resolved_sizes; in env_resolve_init()
1806 btf->resolved_ids = resolved_ids; in env_resolve_init()
1853 if (type_id < env->btf->start_id) in env_type_is_resolved()
1856 return env->visit_states[type_id - env->btf->start_id] == RESOLVED; in env_type_is_resolved()
1862 const struct btf *btf = env->btf; in env_stack_push() local
1868 if (type_id < btf->start_id in env_stack_push()
1869 || env->visit_states[type_id - btf->start_id] != NOT_VISITED) in env_stack_push()
1872 env->visit_states[type_id - btf->start_id] = VISITED; in env_stack_push()
1900 struct btf *btf = env->btf; in env_stack_pop_resolved() local
1902 type_id -= btf->start_id; /* adjust to local type id */ in env_stack_pop_resolved()
1903 btf->resolved_sizes[type_id] = resolved_size; in env_stack_pop_resolved()
1904 btf->resolved_ids[type_id] = resolved_type_id; in env_stack_pop_resolved()
1934 __btf_resolve_size(const struct btf *btf, const struct btf_type *type, in __btf_resolve_size() argument
1965 type = btf_type_by_id(btf, type->type); in __btf_resolve_size()
1975 type = btf_type_by_id(btf, array->type); in __btf_resolve_size()
2004 btf_resolve_size(const struct btf *btf, const struct btf_type *type, in btf_resolve_size() argument
2007 return __btf_resolve_size(btf, type, type_size, NULL, NULL, NULL, NULL); in btf_resolve_size()
2010 static u32 btf_resolved_type_id(const struct btf *btf, u32 type_id) in btf_resolved_type_id() argument
2012 while (type_id < btf->start_id) in btf_resolved_type_id()
2013 btf = btf->base_btf; in btf_resolved_type_id()
2015 return btf->resolved_ids[type_id - btf->start_id]; in btf_resolved_type_id()
2019 static const struct btf_type *btf_type_id_resolve(const struct btf *btf, in btf_type_id_resolve() argument
2022 *type_id = btf_resolved_type_id(btf, *type_id); in btf_type_id_resolve()
2023 return btf_type_by_id(btf, *type_id); in btf_type_id_resolve()
2026 static u32 btf_resolved_type_size(const struct btf *btf, u32 type_id) in btf_resolved_type_size() argument
2028 while (type_id < btf->start_id) in btf_resolved_type_size()
2029 btf = btf->base_btf; in btf_resolved_type_size()
2031 return btf->resolved_sizes[type_id - btf->start_id]; in btf_resolved_type_size()
2034 const struct btf_type *btf_type_id_size(const struct btf *btf, in btf_type_id_size() argument
2041 size_type = btf_type_by_id(btf, size_type_id); in btf_type_id_size()
2048 size = btf_resolved_type_size(btf, size_type_id); in btf_type_id_size()
2056 size_type_id = btf_resolved_type_id(btf, size_type_id); in btf_type_id_size()
2057 size_type = btf_type_by_id(btf, size_type_id); in btf_type_id_size()
2063 size = btf_resolved_type_size(btf, size_type_id); in btf_type_id_size()
2126 static void btf_df_show(const struct btf *btf, const struct btf_type *t, in btf_df_show() argument
2394 static void btf_int_bits_show(const struct btf *btf, in btf_int_bits_show() argument
2413 static void btf_int_show(const struct btf *btf, const struct btf_type *t, in btf_int_show() argument
2429 btf_int_bits_show(btf, t, safe_data, bits_offset, show); in btf_int_show()
2476 btf_int_bits_show(btf, t, safe_data, bits_offset, show); in btf_int_show()
2500 struct btf *btf = env->btf; in btf_modifier_check_member() local
2502 resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL); in btf_modifier_check_member()
2525 struct btf *btf = env->btf; in btf_modifier_check_kflag_member() local
2527 resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL); in btf_modifier_check_kflag_member()
2594 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_ref_type_check_meta()
2599 value = btf_name_by_offset(env->btf, t->name_off); in btf_ref_type_check_meta()
2622 struct btf *btf = env->btf; in btf_modifier_resolve() local
2624 next_type = btf_type_by_id(btf, next_type_id); in btf_modifier_resolve()
2640 if (!btf_type_id_size(btf, &next_type_id, NULL)) { in btf_modifier_resolve()
2642 next_type = btf_type_id_resolve(btf, &next_type_id); in btf_modifier_resolve()
2664 struct btf *btf = env->btf; in btf_var_resolve() local
2666 next_type = btf_type_by_id(btf, next_type_id); in btf_var_resolve()
2681 resolved_type = btf_type_id_resolve(btf, &resolved_type_id); in btf_var_resolve()
2694 if (!btf_type_id_size(btf, &next_type_id, NULL)) { in btf_var_resolve()
2710 struct btf *btf = env->btf; in btf_ptr_resolve() local
2712 next_type = btf_type_by_id(btf, next_type_id); in btf_ptr_resolve()
2735 resolved_type = btf_type_id_resolve(btf, &resolved_type_id); in btf_ptr_resolve()
2744 if (!btf_type_id_size(btf, &next_type_id, NULL)) { in btf_ptr_resolve()
2746 next_type = btf_type_id_resolve(btf, &next_type_id); in btf_ptr_resolve()
2761 static void btf_modifier_show(const struct btf *btf, in btf_modifier_show() argument
2766 if (btf->resolved_ids) in btf_modifier_show()
2767 t = btf_type_id_resolve(btf, &type_id); in btf_modifier_show()
2769 t = btf_type_skip_modifiers(btf, type_id, NULL); in btf_modifier_show()
2771 btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); in btf_modifier_show()
2774 static void btf_var_show(const struct btf *btf, const struct btf_type *t, in btf_var_show() argument
2778 t = btf_type_id_resolve(btf, &type_id); in btf_var_show()
2780 btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); in btf_var_show()
2783 static void btf_ptr_show(const struct btf *btf, const struct btf_type *t, in btf_ptr_show() argument
2841 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_fwd_check_meta()
2874 struct btf *btf = env->btf; in btf_array_check_member() local
2883 btf_type_id_size(btf, &array_type_id, &array_size); in btf_array_check_member()
2954 struct btf *btf = env->btf; in btf_array_resolve() local
2959 index_type = btf_type_by_id(btf, index_type_id); in btf_array_resolve()
2970 index_type = btf_type_id_size(btf, &index_type_id, NULL); in btf_array_resolve()
2979 elem_type = btf_type_by_id(btf, elem_type_id); in btf_array_resolve()
2991 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size); in btf_array_resolve()
3022 static void __btf_array_show(const struct btf *btf, const struct btf_type *t, in __btf_array_show() argument
3033 elem_type = btf_type_skip_modifiers(btf, elem_type_id, NULL); in __btf_array_show()
3062 elem_ops->show(btf, elem_type, elem_type_id, data, in __btf_array_show()
3075 static void btf_array_show(const struct btf *btf, const struct btf_type *t, in btf_array_show() argument
3091 __btf_array_show(btf, t, type_id, data, bits_offset, show); in btf_array_show()
3105 __btf_array_show(btf, t, type_id, data, bits_offset, show); in btf_array_show()
3149 struct btf *btf = env->btf; in btf_struct_check_meta() local
3164 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_struct_check_meta()
3173 if (!btf_name_offset_valid(btf, member->name_off)) { in btf_struct_check_meta()
3182 !btf_name_valid_identifier(btf, member->name_off)) { in btf_struct_check_meta()
3245 last_member_type = btf_type_by_id(env->btf, in btf_struct_resolve()
3261 const struct btf_type *member_type = btf_type_by_id(env->btf, in btf_struct_resolve()
3319 static int btf_find_struct(const struct btf *btf, const struct btf_type *t, in btf_find_struct() argument
3332 static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, in btf_find_kptr() argument
3342 t = btf_type_by_id(btf, t->type); in btf_find_kptr()
3346 t = btf_type_by_id(btf, t->type); in btf_find_kptr()
3351 if (btf_type_is_type_tag(btf_type_by_id(btf, t->type))) in btf_find_kptr()
3353 tag_value = __btf_name_by_offset(btf, t->name_off); in btf_find_kptr()
3369 t = btf_type_skip_modifiers(btf, t->type, &res_id); in btf_find_kptr()
3380 int btf_find_next_decl_tag(const struct btf *btf, const struct btf_type *pt, in btf_find_next_decl_tag() argument
3386 for (i = last_id + 1, n = btf_nr_types(btf); i < n; i++) { in btf_find_next_decl_tag()
3387 const struct btf_type *t = btf_type_by_id(btf, i); in btf_find_next_decl_tag()
3391 if (pt != btf_type_by_id(btf, t->type)) in btf_find_next_decl_tag()
3395 if (strncmp(__btf_name_by_offset(btf, t->name_off), tag_key, len)) in btf_find_next_decl_tag()
3402 const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt, in btf_find_decl_tag_value() argument
3409 id = btf_find_next_decl_tag(btf, pt, comp_idx, tag_key, 0); in btf_find_decl_tag_value()
3413 t = btf_type_by_id(btf, id); in btf_find_decl_tag_value()
3415 value = __btf_name_by_offset(btf, t->name_off) + len; in btf_find_decl_tag_value()
3418 id = btf_find_next_decl_tag(btf, pt, comp_idx, tag_key, id); in btf_find_decl_tag_value()
3426 btf_find_graph_root(const struct btf *btf, const struct btf_type *pt, in btf_find_graph_root() argument
3439 value_type = btf_find_decl_tag_value(btf, pt, comp_idx, "contains:"); in btf_find_graph_root()
3448 id = btf_find_by_name_kind(btf, value_type, BTF_KIND_STRUCT); in btf_find_graph_root()
3468 static int btf_get_field_type(const struct btf *btf, const struct btf_type *var_type, in btf_get_field_type() argument
3473 const char *name = __btf_name_by_offset(btf, var_type->name_off); in btf_get_field_type()
3576 static int btf_find_struct_field(const struct btf *btf,
3587 static int btf_find_nested_struct(const struct btf *btf, const struct btf_type *t, in btf_find_nested_struct() argument
3598 ret = btf_find_struct_field(btf, t, field_mask, info, info_cnt, level); in btf_find_nested_struct()
3620 static int btf_find_field_one(const struct btf *btf, in btf_find_field_one() argument
3640 var_type = btf_type_by_id(btf, array->type); in btf_find_field_one()
3647 field_type = btf_get_field_type(btf, var_type, in btf_find_field_one()
3654 ret = btf_find_nested_struct(btf, var_type, off, nelems, field_mask, in btf_find_field_one()
3677 ret = btf_find_struct(btf, var_type, off, sz, field_type, in btf_find_field_one()
3686 ret = btf_find_kptr(btf, var_type, off, sz, in btf_find_field_one()
3693 ret = btf_find_graph_root(btf, var, var_type, in btf_find_field_one()
3716 static int btf_find_struct_field(const struct btf *btf, in btf_find_struct_field() argument
3726 const struct btf_type *member_type = btf_type_by_id(btf, in btf_find_struct_field()
3735 ret = btf_find_field_one(btf, t, member_type, i, in btf_find_struct_field()
3746 static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t, in btf_find_datasec_var() argument
3755 const struct btf_type *var = btf_type_by_id(btf, vsi->type); in btf_find_datasec_var()
3756 const struct btf_type *var_type = btf_type_by_id(btf, var->type); in btf_find_datasec_var()
3759 ret = btf_find_field_one(btf, var, var_type, -1, off, vsi->size, in btf_find_datasec_var()
3770 static int btf_find_field(const struct btf *btf, const struct btf_type *t, in btf_find_field() argument
3775 return btf_find_struct_field(btf, t, field_mask, info, info_cnt, 0); in btf_find_field()
3777 return btf_find_datasec_var(btf, t, field_mask, info, info_cnt, 0); in btf_find_field()
3782 static int btf_parse_kptr(const struct btf *btf, struct btf_field *field, in btf_parse_kptr() argument
3790 struct btf *kptr_btf; in btf_parse_kptr()
3797 t = btf_type_by_id(btf, info->kptr.type_id); in btf_parse_kptr()
3798 id = bpf_find_btf_id(__btf_name_by_offset(btf, t->name_off), BTF_INFO_KIND(t->info), in btf_parse_kptr()
3802 WARN_ON_ONCE(btf_is_kernel(btf)); in btf_parse_kptr()
3809 kptr_btf = (struct btf *)btf; in btf_parse_kptr()
3862 field->kptr.btf = kptr_btf; in btf_parse_kptr()
3872 static int btf_parse_graph_root(const struct btf *btf, in btf_parse_graph_root() argument
3883 t = btf_type_by_id(btf, info->graph_root.value_btf_id); in btf_parse_graph_root()
3890 __btf_name_by_offset(btf, member->name_off))) in btf_parse_graph_root()
3895 n = btf_type_by_id(btf, member->type); in btf_parse_graph_root()
3898 if (strcmp(node_type_name, __btf_name_by_offset(btf, n->name_off))) in btf_parse_graph_root()
3907 field->graph_root.btf = (struct btf *)btf; in btf_parse_graph_root()
3916 static int btf_parse_list_head(const struct btf *btf, struct btf_field *field, in btf_parse_list_head() argument
3919 return btf_parse_graph_root(btf, field, info, "bpf_list_node", in btf_parse_list_head()
3923 static int btf_parse_rb_root(const struct btf *btf, struct btf_field *field, in btf_parse_rb_root() argument
3926 return btf_parse_graph_root(btf, field, info, "bpf_rb_node", in btf_parse_rb_root()
3942 struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type *t, in btf_parse_fields() argument
3950 ret = btf_find_field(btf, t, field_mask, info_arr, ARRAY_SIZE(info_arr)); in btf_parse_fields()
4017 ret = btf_parse_kptr(btf, &rec->fields[i], &info_arr[i]); in btf_parse_fields()
4022 ret = btf_parse_list_head(btf, &rec->fields[i], &info_arr[i]); in btf_parse_fields()
4027 ret = btf_parse_rb_root(btf, &rec->fields[i], &info_arr[i]); in btf_parse_fields()
4070 int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec) in btf_check_and_fixup_fields() argument
4093 if (btf_is_kernel(rec->fields[i].kptr.btf)) in btf_check_and_fixup_fields()
4095 t = btf_type_by_id(rec->fields[i].kptr.btf, in btf_check_and_fixup_fields()
4107 meta = btf_find_struct_meta(btf, btf_id); in btf_check_and_fixup_fields()
4158 static void __btf_struct_show(const struct btf *btf, const struct btf_type *t, in __btf_struct_show() argument
4171 const struct btf_type *member_type = btf_type_by_id(btf, in __btf_struct_show()
4195 ops->show(btf, member_type, member->type, in __btf_struct_show()
4205 static void btf_struct_show(const struct btf *btf, const struct btf_type *t, in btf_struct_show() argument
4221 __btf_struct_show(btf, t, type_id, data, bits_offset, show); in btf_struct_show()
4236 __btf_struct_show(btf, t, type_id, data, bits_offset, show); in btf_struct_show()
4313 struct btf *btf = env->btf; in btf_enum_check_meta() local
4335 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_enum_check_meta()
4343 if (!btf_name_offset_valid(btf, enums[i].name_off)) { in btf_enum_check_meta()
4351 !btf_name_valid_identifier(btf, enums[i].name_off)) { in btf_enum_check_meta()
4360 __btf_name_by_offset(btf, enums[i].name_off), in btf_enum_check_meta()
4373 static void btf_enum_show(const struct btf *btf, const struct btf_type *t, in btf_enum_show() argument
4393 __btf_name_by_offset(btf, in btf_enum_show()
4421 struct btf *btf = env->btf; in btf_enum64_check_meta() local
4443 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_enum64_check_meta()
4451 if (!btf_name_offset_valid(btf, enums[i].name_off)) { in btf_enum64_check_meta()
4459 !btf_name_valid_identifier(btf, enums[i].name_off)) { in btf_enum64_check_meta()
4469 __btf_name_by_offset(btf, enums[i].name_off), in btf_enum64_check_meta()
4476 static void btf_enum64_show(const struct btf *btf, const struct btf_type *t, in btf_enum64_show() argument
4496 __btf_name_by_offset(btf, in btf_enum64_show()
4566 __btf_name_by_offset(env->btf, in btf_func_proto_log()
4570 __btf_name_by_offset(env->btf, in btf_func_proto_log()
4578 __btf_name_by_offset(env->btf, in btf_func_proto_log()
4611 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_func_check_meta()
4680 !btf_name_valid_identifier(env->btf, t->name_off)) { in btf_var_check_meta()
4746 !btf_name_valid_section(env->btf, t->name_off)) { in btf_datasec_check_meta()
4796 struct btf *btf = env->btf; in btf_datasec_resolve() local
4802 const struct btf_type *var_type = btf_type_by_id(env->btf, in btf_datasec_resolve()
4817 if (!btf_type_id_size(btf, &type_id, &type_size)) { in btf_datasec_resolve()
4838 static void btf_datasec_show(const struct btf *btf, in btf_datasec_show() argument
4851 __btf_name_by_offset(btf, t->name_off)); in btf_datasec_show()
4853 var = btf_type_by_id(btf, vsi->type); in btf_datasec_show()
4856 btf_type_ops(var)->show(btf, var, vsi->type, in btf_datasec_show()
4962 value = btf_name_by_offset(env->btf, t->name_off); in btf_decl_tag_check_meta()
4990 struct btf *btf = env->btf; in btf_decl_tag_resolve() local
4994 next_type = btf_type_by_id(btf, next_type_id); in btf_decl_tag_resolve()
5015 next_type = btf_type_by_id(btf, next_type->type); in btf_decl_tag_resolve()
5050 const struct btf *btf; in btf_func_proto_check() local
5054 btf = env->btf; in btf_func_proto_check()
5062 ret_type = btf_type_by_id(btf, ret_type_id); in btf_func_proto_check()
5081 if (!btf_type_id_size(btf, &ret_type_id, NULL)) { in btf_func_proto_check()
5105 arg_type = btf_type_by_id(btf, arg_type_id); in btf_func_proto_check()
5117 (!btf_name_offset_valid(btf, args[i].name_off) || in btf_func_proto_check()
5118 !btf_name_valid_identifier(btf, args[i].name_off))) { in btf_func_proto_check()
5131 if (!btf_type_id_size(btf, &arg_type_id, NULL)) { in btf_func_proto_check()
5145 const struct btf *btf; in btf_func_check() local
5148 btf = env->btf; in btf_func_check()
5149 proto_type = btf_type_by_id(btf, t->type); in btf_func_check()
5217 if (!btf_name_offset_valid(env->btf, t->name_off)) { in btf_check_meta()
5234 struct btf *btf = env->btf; in btf_check_all_metas() local
5238 hdr = &btf->hdr; in btf_check_all_metas()
5239 cur = btf->nohdr_data + hdr->type_off; in btf_check_all_metas()
5242 env->log_type_id = btf->base_btf ? btf->start_id : 1; in btf_check_all_metas()
5263 struct btf *btf = env->btf; in btf_resolve_valid() local
5269 return !btf_resolved_type_id(btf, type_id) && in btf_resolve_valid()
5270 !btf_resolved_type_size(btf, type_id); in btf_resolve_valid()
5273 return btf_resolved_type_id(btf, type_id) && in btf_resolve_valid()
5274 !btf_resolved_type_size(btf, type_id); in btf_resolve_valid()
5278 t = btf_type_id_resolve(btf, &type_id); in btf_resolve_valid()
5291 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size); in btf_resolve_valid()
5294 btf_resolved_type_size(btf, type_id)); in btf_resolve_valid()
5335 struct btf *btf = env->btf; in btf_check_all_types() local
5345 for (i = btf->base_btf ? 0 : 1; i < btf->nr_types; i++) { in btf_check_all_types()
5346 type_id = btf->start_id + i; in btf_check_all_types()
5347 t = btf_type_by_id(btf, type_id); in btf_check_all_types()
5369 const struct btf_header *hdr = &env->btf->hdr; in btf_parse_type_sec()
5378 if (!env->btf->base_btf && !hdr->type_len) { in btf_parse_type_sec()
5393 struct btf *btf = env->btf; in btf_parse_str_sec() local
5396 hdr = &btf->hdr; in btf_parse_str_sec()
5397 start = btf->nohdr_data + hdr->str_off; in btf_parse_str_sec()
5400 if (end != btf->data + btf->data_size) { in btf_parse_str_sec()
5405 btf->strings = start; in btf_parse_str_sec()
5407 if (btf->base_btf && !hdr->str_len) in btf_parse_str_sec()
5413 if (!btf->base_btf && start[0]) { in btf_parse_str_sec()
5440 const struct btf *btf; in btf_check_sec_info() local
5442 btf = env->btf; in btf_check_sec_info()
5443 hdr = &btf->hdr; in btf_check_sec_info()
5491 struct btf *btf; in btf_parse_hdr() local
5493 btf = env->btf; in btf_parse_hdr()
5494 btf_data_size = btf->data_size; in btf_parse_hdr()
5501 hdr = btf->data; in btf_parse_hdr()
5509 if (hdr_len > sizeof(btf->hdr)) { in btf_parse_hdr()
5510 u8 *expected_zero = btf->data + sizeof(btf->hdr); in btf_parse_hdr()
5511 u8 *end = btf->data + hdr_len; in btf_parse_hdr()
5521 hdr_copy = min_t(u32, hdr_len, sizeof(btf->hdr)); in btf_parse_hdr()
5522 memcpy(&btf->hdr, btf->data, hdr_copy); in btf_parse_hdr()
5524 hdr = &btf->hdr; in btf_parse_hdr()
5543 if (!btf->base_btf && btf_data_size == hdr->hdr_len) { in btf_parse_hdr()
5561 btf_parse_struct_metas(struct bpf_verifier_log *log, struct btf *btf) in btf_parse_struct_metas() argument
5582 id = btf_find_by_name_kind(btf, alloc_obj_fields[i], BTF_KIND_STRUCT); in btf_parse_struct_metas()
5596 n = btf_nr_types(btf); in btf_parse_struct_metas()
5603 t = btf_type_by_id(btf, i); in btf_parse_struct_metas()
5609 ret = btf_find_kptr(btf, t, 0, 0, &tmp, BPF_KPTR); in btf_parse_struct_metas()
5637 t = btf_type_by_id(btf, i); in btf_parse_struct_metas()
5662 …record = btf_parse_fields(btf, t, BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK | BPF_LIST_HEAD | BPF_LIST_NOD… in btf_parse_struct_metas()
5682 struct btf_struct_meta *btf_find_struct_meta(const struct btf *btf, u32 btf_id) in btf_find_struct_meta() argument
5687 tab = btf->struct_meta_tab; in btf_find_struct_meta()
5694 struct btf *btf, int start_id) in btf_check_type_tags() argument
5699 n = btf_nr_types(btf); in btf_check_type_tags()
5705 t = btf_type_by_id(btf, i); in btf_check_type_tags()
5731 t = btf_type_by_id(btf, cur_id); in btf_check_type_tags()
5755 static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) in btf_parse()
5757 bpfptr_t btf_data = make_bpfptr(attr->btf, uattr.is_kernel); in btf_parse()
5761 struct btf *btf = NULL; in btf_parse() local
5780 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN); in btf_parse()
5781 if (!btf) { in btf_parse()
5785 env->btf = btf; in btf_parse()
5793 btf->data = data; in btf_parse()
5794 btf->data_size = attr->btf_size; in btf_parse()
5805 btf->nohdr_data = btf->data + btf->hdr.hdr_len; in btf_parse()
5815 err = btf_check_type_tags(env, btf, 1); in btf_parse()
5819 struct_meta_tab = btf_parse_struct_metas(&env->log, btf); in btf_parse()
5824 btf->struct_meta_tab = struct_meta_tab; in btf_parse()
5830 err = btf_check_and_fixup_fields(btf, struct_meta_tab->types[i].record); in btf_parse()
5841 refcount_set(&btf->refcnt, 1); in btf_parse()
5842 return btf; in btf_parse()
5845 btf_free_struct_meta_tab(btf); in btf_parse()
5853 if (btf) in btf_parse()
5854 btf_free(btf); in btf_parse()
5860 extern struct btf *btf_vmlinux;
5933 bool btf_is_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf, in btf_is_prog_ctx_type() argument
5940 t = btf_type_by_id(btf, t->type); in btf_is_prog_ctx_type()
5947 t = btf_type_by_id(btf, t->type); in btf_is_prog_ctx_type()
5950 tname = btf_name_by_offset(btf, t->name_off); in btf_is_prog_ctx_type()
5957 t = btf_type_by_id(btf, t->type); in btf_is_prog_ctx_type()
5966 tname = btf_name_by_offset(btf, t->name_off); in btf_is_prog_ctx_type()
6019 static int btf_validate_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf, in btf_validate_prog_ctx_type() argument
6031 t = btf_type_by_id(btf, t->type); in btf_validate_prog_ctx_type()
6036 t = btf_type_by_id(btf, t->type); in btf_validate_prog_ctx_type()
6039 tname = btf_name_by_offset(btf, t->name_off); in btf_validate_prog_ctx_type()
6047 t = btf_type_by_id(btf, t->type); in btf_validate_prog_ctx_type()
6053 tname = btf_name_by_offset(btf, t->name_off); in btf_validate_prog_ctx_type()
6154 struct btf *btf, in btf_translate_to_vmlinux() argument
6159 if (!btf_is_prog_ctx_type(log, btf, t, prog_type, arg)) in btf_translate_to_vmlinux()
6187 static struct btf *btf_parse_base(struct btf_verifier_env *env, const char *name, in BTF_ID()
6190 struct btf *btf = NULL; in BTF_ID() local
6196 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN); in BTF_ID()
6197 if (!btf) { in BTF_ID()
6201 env->btf = btf; in BTF_ID()
6203 btf->data = data; in BTF_ID()
6204 btf->data_size = data_size; in BTF_ID()
6205 btf->kernel_btf = true; in BTF_ID()
6206 snprintf(btf->name, sizeof(btf->name), "%s", name); in BTF_ID()
6212 btf->nohdr_data = btf->data + btf->hdr.hdr_len; in BTF_ID()
6222 err = btf_check_type_tags(env, btf, 1); in BTF_ID()
6226 refcount_set(&btf->refcnt, 1); in BTF_ID()
6228 return btf; in BTF_ID()
6231 if (btf) { in BTF_ID()
6232 kvfree(btf->types); in BTF_ID()
6233 kfree(btf); in BTF_ID()
6238 struct btf *btf_parse_vmlinux(void) in btf_parse_vmlinux()
6242 struct btf *btf; in btf_parse_vmlinux() local
6251 btf = btf_parse_base(env, "vmlinux", __start_BTF, __stop_BTF - __start_BTF); in btf_parse_vmlinux()
6252 if (IS_ERR(btf)) in btf_parse_vmlinux()
6256 bpf_ctx_convert.t = btf_type_by_id(btf, bpf_ctx_convert_btf_id[0]); in btf_parse_vmlinux()
6257 err = btf_alloc_id(btf); in btf_parse_vmlinux()
6259 btf_free(btf); in btf_parse_vmlinux()
6260 btf = ERR_PTR(err); in btf_parse_vmlinux()
6264 return btf; in btf_parse_vmlinux()
6271 static __u32 btf_relocate_id(const struct btf *btf, __u32 id) in btf_relocate_id() argument
6273 if (!btf->base_btf || !btf->base_id_map) in btf_relocate_id()
6275 return btf->base_id_map[id]; in btf_relocate_id()
6280 static struct btf *btf_parse_module(const char *module_name, const void *data, in btf_parse_module()
6284 struct btf *btf = NULL, *vmlinux_btf, *base_btf = NULL; in btf_parse_module() local
6312 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN); in btf_parse_module()
6313 if (!btf) { in btf_parse_module()
6317 env->btf = btf; in btf_parse_module()
6319 btf->base_btf = base_btf; in btf_parse_module()
6320 btf->start_id = base_btf->nr_types; in btf_parse_module()
6321 btf->start_str_off = base_btf->hdr.str_len; in btf_parse_module()
6322 btf->kernel_btf = true; in btf_parse_module()
6323 snprintf(btf->name, sizeof(btf->name), "%s", module_name); in btf_parse_module()
6325 btf->data = kvmemdup(data, data_size, GFP_KERNEL | __GFP_NOWARN); in btf_parse_module()
6326 if (!btf->data) { in btf_parse_module()
6330 btf->data_size = data_size; in btf_parse_module()
6336 btf->nohdr_data = btf->data + btf->hdr.hdr_len; in btf_parse_module()
6346 err = btf_check_type_tags(env, btf, btf_nr_types(base_btf)); in btf_parse_module()
6351 err = btf_relocate(btf, vmlinux_btf, &btf->base_id_map); in btf_parse_module()
6359 refcount_set(&btf->refcnt, 1); in btf_parse_module()
6360 return btf; in btf_parse_module()
6366 if (btf) { in btf_parse_module()
6367 kvfree(btf->data); in btf_parse_module()
6368 kvfree(btf->types); in btf_parse_module()
6369 kfree(btf); in btf_parse_module()
6376 struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog) in bpf_prog_get_target_btf()
6381 return tgt_prog->aux->btf; in bpf_prog_get_target_btf()
6386 static bool is_int_ptr(struct btf *btf, const struct btf_type *t) in is_int_ptr() argument
6389 t = btf_type_skip_modifiers(btf, t->type, NULL); in is_int_ptr()
6394 static u32 get_ctx_arg_idx(struct btf *btf, const struct btf_type *func_proto, in get_ctx_arg_idx() argument
6408 t = btf_type_skip_modifiers(btf, args[i].type, NULL); in get_ctx_arg_idx()
6414 t = btf_type_skip_modifiers(btf, func_proto->type, NULL); in get_ctx_arg_idx()
6438 int btf_ctx_arg_offset(const struct btf *btf, const struct btf_type *func_proto, in btf_ctx_arg_offset() argument
6448 t = btf_type_by_id(btf, args[i].type); in btf_ctx_arg_offset()
6449 t = btf_resolve_size(btf, t, &sz); in btf_ctx_arg_offset()
6660 struct btf *btf = bpf_prog_get_target_btf(prog); in btf_ctx_access() local
6674 arg = get_ctx_arg_idx(btf, t, off); in btf_ctx_access()
6715 t = btf_type_by_id(btf, t->type); in btf_ctx_access()
6724 t = btf_type_skip_modifiers(btf, t->type, NULL); in btf_ctx_access()
6741 t = btf_type_by_id(btf, args[arg].type); in btf_ctx_access()
6746 t = btf_type_by_id(btf, t->type); in btf_ctx_access()
6754 __btf_name_by_offset(btf, t->name_off), in btf_ctx_access()
6786 if (is_int_ptr(btf, t)) in btf_ctx_access()
6800 info->btf = ctx_arg_info->btf ? : btf_vmlinux; in btf_ctx_access()
6811 if (btf_param_match_suffix(btf, &args[arg], "__nullable")) in btf_ctx_access()
6815 struct btf *btf = prog->aux->attach_btf; in btf_ctx_access() local
6820 t = btf_type_by_id(btf, prog->aux->attach_btf_id); in btf_ctx_access()
6823 tname = btf_name_by_offset(btf, t->name_off); in btf_ctx_access()
6843 if (i == ARRAY_SIZE(raw_tp_null_args) && btf_is_module(btf)) in btf_ctx_access()
6855 ret = btf_translate_to_vmlinux(log, btf, t, tgt_type, arg); in btf_ctx_access()
6857 info->btf = btf_vmlinux; in btf_ctx_access()
6865 info->btf = btf; in btf_ctx_access()
6867 t = btf_type_by_id(btf, t->type); in btf_ctx_access()
6870 tag_value = __btf_name_by_offset(btf, t->name_off); in btf_ctx_access()
6880 t = btf_type_by_id(btf, t->type); in btf_ctx_access()
6890 __btf_name_by_offset(btf, t->name_off)); in btf_ctx_access()
6911 static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, in btf_struct_walk() argument
6924 t = btf_type_skip_modifiers(btf, t->type, NULL); in btf_struct_walk()
6925 tname = __btf_name_by_offset(btf, t->name_off); in btf_struct_walk()
6950 mtype = btf_type_skip_modifiers(btf, member->type, in btf_struct_walk()
6964 t = btf_type_skip_modifiers(btf, array_elem->type, in btf_struct_walk()
7026 mtype = btf_type_by_id(btf, member->type); in btf_struct_walk()
7027 mname = __btf_name_by_offset(btf, member->name_off); in btf_struct_walk()
7029 mtype = __btf_resolve_size(btf, mtype, &msize, in btf_struct_walk()
7127 t = btf_type_by_id(btf, mtype->type); in btf_struct_walk()
7129 tag_value = __btf_name_by_offset(btf, t->name_off); in btf_struct_walk()
7141 stype = btf_type_skip_modifiers(btf, mtype->type, &id); in btf_struct_walk()
7176 const struct btf *btf = reg->btf; in btf_struct_access() local
7187 meta = btf_find_struct_meta(btf, id); in btf_struct_access()
7204 t = btf_type_by_id(btf, id); in btf_struct_access()
7206 err = btf_struct_walk(log, btf, t, off, size, &id, &tmp_flag, field_name); in btf_struct_access()
7228 t = btf_type_by_id(btf, id); in btf_struct_access()
7249 bool btf_types_are_same(const struct btf *btf1, u32 id1, in btf_types_are_same()
7250 const struct btf *btf2, u32 id2) in btf_types_are_same()
7260 const struct btf *btf, u32 id, int off, in btf_struct_ids_match() argument
7261 const struct btf *need_btf, u32 need_type_id, in btf_struct_ids_match()
7269 if (off == 0 && btf_types_are_same(btf, id, need_btf, need_type_id)) in btf_struct_ids_match()
7278 type = btf_type_by_id(btf, id); in btf_struct_ids_match()
7281 err = btf_struct_walk(log, btf, type, off, 1, &id, &flag, NULL); in btf_struct_ids_match()
7290 if (!btf_types_are_same(btf, id, need_btf, need_type_id)) { in btf_struct_ids_match()
7298 static int __get_type_size(struct btf *btf, u32 btf_id, in __get_type_size() argument
7303 *ret_type = btf_type_by_id(btf, 0); in __get_type_size()
7307 t = btf_type_by_id(btf, btf_id); in __get_type_size()
7309 t = btf_type_by_id(btf, t->type); in __get_type_size()
7334 struct btf *btf, in btf_distill_func_proto() argument
7365 ret = __get_type_size(btf, func->type, &t); in btf_distill_func_proto()
7382 ret = __get_type_size(btf, args[i].type, &t); in btf_distill_func_proto()
7413 struct btf *btf1, const struct btf_type *t1, in btf_check_func_type_match()
7414 struct btf *btf2, const struct btf_type *t2) in btf_check_func_type_match()
7524 struct btf *btf2, const struct btf_type *t2) in btf_check_type_match()
7526 struct btf *btf1 = prog->aux->btf; in btf_check_type_match()
7546 static bool btf_is_dynptr_ptr(const struct btf *btf, const struct btf_type *t) in btf_is_dynptr_ptr() argument
7550 t = btf_type_by_id(btf, t->type); /* skip PTR */ in btf_is_dynptr_ptr()
7553 t = btf_type_by_id(btf, t->type); in btf_is_dynptr_ptr()
7558 name = btf_str_by_offset(btf, t->name_off); in btf_is_dynptr_ptr()
7571 const struct btf *btf; member
7582 const struct btf *btf, const struct btf_type *t) in btf_get_ptr_to_btf_id() argument
7586 .btf = btf, in btf_get_ptr_to_btf_id()
7594 t = btf_type_by_id(btf, t->type); in btf_get_ptr_to_btf_id()
7597 t = btf_type_by_id(btf, t->type); in btf_get_ptr_to_btf_id()
7605 arg_idx, btf_type_str(t), __btf_name_by_offset(btf, t->name_off), in btf_get_ptr_to_btf_id()
7611 arg_idx, btf_type_str(t), __btf_name_by_offset(btf, t->name_off), in btf_get_ptr_to_btf_id()
7616 if (btf_is_module(cc->cands[0].btf)) { in btf_get_ptr_to_btf_id()
7618 arg_idx, btf_type_str(t), __btf_name_by_offset(btf, t->name_off)); in btf_get_ptr_to_btf_id()
7655 struct btf *btf = prog->aux->btf; in btf_prepare_func_args() local
7677 fn_t = btf_type_by_id(btf, btf_id); in btf_prepare_func_args()
7686 tname = btf_name_by_offset(btf, fn_t->name_off); in btf_prepare_func_args()
7695 t = btf_type_by_id(btf, fn_t->type); in btf_prepare_func_args()
7710 t = btf_type_by_id(btf, t->type); in btf_prepare_func_args()
7712 t = btf_type_by_id(btf, t->type); in btf_prepare_func_args()
7731 while ((id = btf_find_next_decl_tag(btf, fn_t, i, "arg:", id)) > 0) { in btf_prepare_func_args()
7732 const struct btf_type *tag_t = btf_type_by_id(btf, id); in btf_prepare_func_args()
7733 const char *tag = __btf_name_by_offset(btf, tag_t->name_off) + 4; in btf_prepare_func_args()
7761 t = btf_type_by_id(btf, args[i].type); in btf_prepare_func_args()
7763 t = btf_type_by_id(btf, t->type); in btf_prepare_func_args()
7767 if ((tags & ARG_TAG_CTX) || btf_is_prog_ctx_type(log, btf, t, prog_type, i)) { in btf_prepare_func_args()
7773 btf_validate_prog_ctx_type(log, btf, t, i, prog_type, in btf_prepare_func_args()
7779 if (btf_is_dynptr_ptr(btf, t)) { in btf_prepare_func_args()
7795 kern_type_id = btf_get_ptr_to_btf_id(log, i, btf, t); in btf_prepare_func_args()
7821 t = btf_type_skip_modifiers(btf, t->type, NULL); in btf_prepare_func_args()
7822 ref_t = btf_resolve_size(btf, t, &mem_size); in btf_prepare_func_args()
7825 i, btf_type_str(t), btf_name_by_offset(btf, t->name_off), in btf_prepare_func_args()
7859 static void btf_type_show(const struct btf *btf, u32 type_id, void *obj, in btf_type_show() argument
7862 const struct btf_type *t = btf_type_by_id(btf, type_id); in btf_type_show()
7864 show->btf = btf; in btf_type_show()
7868 btf_type_ops(t)->show(btf, t, type_id, obj, 0, show); in btf_type_show()
7877 int btf_type_seq_show_flags(const struct btf *btf, u32 type_id, in btf_type_seq_show_flags() argument
7886 btf_type_show(btf, type_id, obj, &sseq); in btf_type_seq_show_flags()
7891 void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj, in btf_type_seq_show() argument
7894 (void) btf_type_seq_show_flags(btf, type_id, obj, m, in btf_type_seq_show()
7927 int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj, in btf_type_snprintf_show() argument
7938 btf_type_show(btf, type_id, obj, (struct btf_show *)&ssnprintf); in btf_type_snprintf_show()
7951 const struct btf *btf = filp->private_data; in bpf_btf_show_fdinfo() local
7953 seq_printf(m, "btf_id:\t%u\n", btf->id); in bpf_btf_show_fdinfo()
7970 static int __btf_new_fd(struct btf *btf) in __btf_new_fd() argument
7972 return anon_inode_getfd("btf", &btf_fops, btf, O_RDONLY | O_CLOEXEC); in __btf_new_fd()
7977 struct btf *btf; in btf_new_fd() local
7980 btf = btf_parse(attr, uattr, uattr_size); in btf_new_fd()
7981 if (IS_ERR(btf)) in btf_new_fd()
7982 return PTR_ERR(btf); in btf_new_fd()
7984 ret = btf_alloc_id(btf); in btf_new_fd()
7986 btf_free(btf); in btf_new_fd()
7996 ret = __btf_new_fd(btf); in btf_new_fd()
7998 btf_put(btf); in btf_new_fd()
8003 struct btf *btf_get_by_fd(int fd) in btf_get_by_fd()
8005 struct btf *btf; in btf_get_by_fd() local
8008 btf = __btf_get_by_fd(f); in btf_get_by_fd()
8009 if (!IS_ERR(btf)) in btf_get_by_fd()
8010 refcount_inc(&btf->refcnt); in btf_get_by_fd()
8012 return btf; in btf_get_by_fd()
8015 int btf_get_info_by_fd(const struct btf *btf, in btf_get_info_by_fd() argument
8035 info.id = btf->id; in btf_get_info_by_fd()
8036 ubtf = u64_to_user_ptr(info.btf); in btf_get_info_by_fd()
8037 btf_copy = min_t(u32, btf->data_size, info.btf_size); in btf_get_info_by_fd()
8038 if (copy_to_user(ubtf, btf->data, btf_copy)) in btf_get_info_by_fd()
8040 info.btf_size = btf->data_size; in btf_get_info_by_fd()
8042 info.kernel_btf = btf->kernel_btf; in btf_get_info_by_fd()
8049 name_len = strlen(btf->name); in btf_get_info_by_fd()
8054 if (copy_to_user(uname, btf->name, name_len + 1)) in btf_get_info_by_fd()
8059 if (copy_to_user(uname, btf->name, uname_len - 1)) in btf_get_info_by_fd()
8077 struct btf *btf; in btf_get_fd_by_id() local
8081 btf = idr_find(&btf_idr, id); in btf_get_fd_by_id()
8082 if (!btf || !refcount_inc_not_zero(&btf->refcnt)) in btf_get_fd_by_id()
8083 btf = ERR_PTR(-ENOENT); in btf_get_fd_by_id()
8086 if (IS_ERR(btf)) in btf_get_fd_by_id()
8087 return PTR_ERR(btf); in btf_get_fd_by_id()
8089 fd = __btf_new_fd(btf); in btf_get_fd_by_id()
8091 btf_put(btf); in btf_get_fd_by_id()
8096 u32 btf_obj_id(const struct btf *btf) in btf_obj_id() argument
8098 return btf->id; in btf_obj_id()
8101 bool btf_is_kernel(const struct btf *btf) in btf_is_kernel() argument
8103 return btf->kernel_btf; in btf_is_kernel()
8106 bool btf_is_module(const struct btf *btf) in btf_is_module() argument
8108 return btf->kernel_btf && strcmp(btf->name, "vmlinux") != 0; in btf_is_module()
8119 struct btf *btf; member
8127 static void purge_cand_cache(struct btf *btf);
8134 struct btf *btf; in btf_module_notify() local
8149 btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size, in btf_module_notify()
8151 if (IS_ERR(btf)) { in btf_module_notify()
8155 mod->name, PTR_ERR(btf)); in btf_module_notify()
8156 err = PTR_ERR(btf); in btf_module_notify()
8162 err = btf_alloc_id(btf); in btf_module_notify()
8164 btf_free(btf); in btf_module_notify()
8172 btf_mod->btf = btf; in btf_module_notify()
8184 attr->attr.name = btf->name; in btf_module_notify()
8186 attr->size = btf->data_size; in btf_module_notify()
8187 attr->private = btf->data; in btf_module_notify()
8223 purge_cand_cache(btf_mod->btf); in btf_module_notify()
8224 btf_put(btf_mod->btf); in btf_module_notify()
8249 struct module *btf_try_get_module(const struct btf *btf) in btf_try_get_module() argument
8257 if (btf_mod->btf != btf) in btf_try_get_module()
8279 static struct btf *btf_get_module_btf(const struct module *module) in btf_get_module_btf()
8284 struct btf *btf = NULL; in btf_get_module_btf() local
8287 btf = bpf_get_btf_vmlinux(); in btf_get_module_btf()
8288 if (!IS_ERR_OR_NULL(btf)) in btf_get_module_btf()
8289 btf_get(btf); in btf_get_module_btf()
8290 return btf; in btf_get_module_btf()
8299 btf_get(btf_mod->btf); in btf_get_module_btf()
8300 btf = btf_mod->btf; in btf_get_module_btf()
8306 return btf; in btf_get_module_btf()
8322 struct btf *btf = NULL; in BPF_CALL_4() local
8332 ret = bpf_find_btf_id(name, kind, &btf); in BPF_CALL_4()
8333 if (ret > 0 && btf_is_module(btf)) { in BPF_CALL_4()
8334 btf_obj_fd = __btf_new_fd(btf); in BPF_CALL_4()
8336 btf_put(btf); in BPF_CALL_4()
8342 btf_put(btf); in BPF_CALL_4()
8365 int btf_check_iter_arg(struct btf *btf, const struct btf_type *func, int arg_idx) in BTF_ID_LIST_GLOBAL()
8376 t = btf_type_skip_modifiers(btf, arg->type, NULL); in BTF_ID_LIST_GLOBAL()
8379 t = btf_type_skip_modifiers(btf, t->type, &btf_id); in BTF_ID_LIST_GLOBAL()
8383 name = btf_name_by_offset(btf, t->name_off); in BTF_ID_LIST_GLOBAL()
8390 static int btf_check_iter_kfuncs(struct btf *btf, const char *func_name, in btf_check_iter_kfuncs() argument
8409 btf_id = btf_check_iter_arg(btf, func, 0); in btf_check_iter_kfuncs()
8416 t = btf_type_by_id(btf, btf_id); in btf_check_iter_kfuncs()
8423 iter_name = btf_name_by_offset(btf, t->name_off) + sizeof(ITER_PREFIX) - 1; in btf_check_iter_kfuncs()
8441 t = btf_type_skip_modifiers(btf, func->type, NULL); in btf_check_iter_kfuncs()
8448 t = btf_type_by_id(btf, func->type); in btf_check_iter_kfuncs()
8456 static int btf_check_kfunc_protos(struct btf *btf, u32 func_id, u32 func_flags) in btf_check_kfunc_protos() argument
8463 func = btf_type_by_id(btf, func_id); in btf_check_kfunc_protos()
8468 func_name = btf_name_by_offset(btf, func->name_off); in btf_check_kfunc_protos()
8472 func = btf_type_by_id(btf, func->type); in btf_check_kfunc_protos()
8477 err = btf_check_iter_kfuncs(btf, func_name, func, func_flags); in btf_check_kfunc_protos()
8487 static int btf_populate_kfunc_set(struct btf *btf, enum btf_kfunc_hook hook, in btf_populate_kfunc_set() argument
8492 bool vmlinux_set = !btf_is_module(btf); in btf_populate_kfunc_set()
8507 tab = btf->kfunc_set_tab; in btf_populate_kfunc_set()
8530 btf->kfunc_set_tab = tab; in btf_populate_kfunc_set()
8582 set->pairs[i].id = btf_relocate_id(btf, set->pairs[i].id); in btf_populate_kfunc_set()
8594 btf_free_kfunc_set_tab(btf); in btf_populate_kfunc_set()
8598 static u32 *__btf_kfunc_id_set_contains(const struct btf *btf, in __btf_kfunc_id_set_contains() argument
8609 if (!btf->kfunc_set_tab) in __btf_kfunc_id_set_contains()
8611 hook_filter = &btf->kfunc_set_tab->hook_filters[hook]; in __btf_kfunc_id_set_contains()
8616 set = btf->kfunc_set_tab->sets[hook]; in __btf_kfunc_id_set_contains()
8679 u32 *btf_kfunc_id_set_contains(const struct btf *btf, in btf_kfunc_id_set_contains() argument
8687 kfunc_flags = __btf_kfunc_id_set_contains(btf, BTF_KFUNC_HOOK_COMMON, kfunc_btf_id, prog); in btf_kfunc_id_set_contains()
8692 return __btf_kfunc_id_set_contains(btf, hook, kfunc_btf_id, prog); in btf_kfunc_id_set_contains()
8695 u32 *btf_kfunc_is_modify_return(const struct btf *btf, u32 kfunc_btf_id, in btf_kfunc_is_modify_return() argument
8698 return __btf_kfunc_id_set_contains(btf, BTF_KFUNC_HOOK_FMODRET, kfunc_btf_id, prog); in btf_kfunc_is_modify_return()
8704 struct btf *btf; in __register_btf_kfunc_id_set() local
8707 btf = btf_get_module_btf(kset->owner); in __register_btf_kfunc_id_set()
8708 if (!btf) in __register_btf_kfunc_id_set()
8710 if (IS_ERR(btf)) in __register_btf_kfunc_id_set()
8711 return PTR_ERR(btf); in __register_btf_kfunc_id_set()
8714 ret = btf_check_kfunc_protos(btf, btf_relocate_id(btf, kset->set->pairs[i].id), in __register_btf_kfunc_id_set()
8720 ret = btf_populate_kfunc_set(btf, hook, kset); in __register_btf_kfunc_id_set()
8723 btf_put(btf); in __register_btf_kfunc_id_set()
8753 s32 btf_find_dtor_kfunc(struct btf *btf, u32 btf_id) in btf_find_dtor_kfunc() argument
8755 struct btf_id_dtor_kfunc_tab *tab = btf->dtor_kfunc_tab; in btf_find_dtor_kfunc()
8770 static int btf_check_dtor_kfuncs(struct btf *btf, const struct btf_id_dtor_kfunc *dtors, u32 cnt) in btf_check_dtor_kfuncs() argument
8778 dtor_btf_id = btf_relocate_id(btf, dtors[i].kfunc_btf_id); in btf_check_dtor_kfuncs()
8780 dtor_func = btf_type_by_id(btf, dtor_btf_id); in btf_check_dtor_kfuncs()
8784 dtor_func_proto = btf_type_by_id(btf, dtor_func->type); in btf_check_dtor_kfuncs()
8789 t = btf_type_by_id(btf, dtor_func_proto->type); in btf_check_dtor_kfuncs()
8797 t = btf_type_by_id(btf, args[0].type); in btf_check_dtor_kfuncs()
8812 struct btf *btf; in register_btf_id_dtor_kfuncs() local
8816 btf = btf_get_module_btf(owner); in register_btf_id_dtor_kfuncs()
8817 if (!btf) in register_btf_id_dtor_kfuncs()
8819 if (IS_ERR(btf)) in register_btf_id_dtor_kfuncs()
8820 return PTR_ERR(btf); in register_btf_id_dtor_kfuncs()
8829 ret = btf_check_dtor_kfuncs(btf, dtors, add_cnt); in register_btf_id_dtor_kfuncs()
8833 tab = btf->dtor_kfunc_tab; in register_btf_id_dtor_kfuncs()
8835 if (WARN_ON_ONCE(tab && btf_is_module(btf))) { in register_btf_id_dtor_kfuncs()
8851 tab = krealloc(btf->dtor_kfunc_tab, in register_btf_id_dtor_kfuncs()
8859 if (!btf->dtor_kfunc_tab) in register_btf_id_dtor_kfuncs()
8861 btf->dtor_kfunc_tab = tab; in register_btf_id_dtor_kfuncs()
8867 tab->dtors[i].btf_id = btf_relocate_id(btf, tab->dtors[i].btf_id); in register_btf_id_dtor_kfuncs()
8868 tab->dtors[i].kfunc_btf_id = btf_relocate_id(btf, tab->dtors[i].kfunc_btf_id); in register_btf_id_dtor_kfuncs()
8877 btf_free_dtor_kfunc_tab(btf); in register_btf_id_dtor_kfuncs()
8878 btf_put(btf); in register_btf_id_dtor_kfuncs()
8904 int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, in bpf_core_types_are_compat()
8905 const struct btf *targ_btf, __u32 targ_id) in bpf_core_types_are_compat()
8913 int bpf_core_types_match(const struct btf *local_btf, u32 local_id, in bpf_core_types_match()
8914 const struct btf *targ_btf, u32 targ_id) in bpf_core_types_match()
9043 static void __purge_cand_cache(struct btf *btf, struct bpf_cand_cache **cache, in __purge_cand_cache() argument
9053 if (!btf) { in __purge_cand_cache()
9066 if (cc->cands[j].btf == btf) { in __purge_cand_cache()
9075 static void purge_cand_cache(struct btf *btf) in purge_cand_cache() argument
9078 __purge_cand_cache(btf, module_cand_cache, MODULE_CAND_CACHE_SIZE); in purge_cand_cache()
9084 bpf_core_add_cands(struct bpf_cand_cache *cands, const struct btf *targ_btf, in bpf_core_add_cands()
9125 cands->cands[cands->cnt].btf = targ_btf; in bpf_core_add_cands()
9136 const struct btf *local_btf = ctx->btf; in bpf_core_find_cands()
9138 const struct btf *main_btf; in bpf_core_find_cands()
9140 struct btf *mod_btf; in bpf_core_find_cands()
9237 type = btf_type_by_id(ctx->btf, relo->type_id); in bpf_core_apply()
9268 cands.cands[i].btf = cc->cands[i].btf; in bpf_core_apply()
9279 err = bpf_core_calc_relo_insn((void *)ctx->log, relo, relo_idx, ctx->btf, &cands, specs, in bpf_core_apply()
9302 struct btf *btf = reg->btf; in btf_nested_type_is_trusted() local
9310 walk_type = btf_type_by_id(btf, reg->btf_id); in btf_nested_type_is_trusted()
9314 tname = btf_name_by_offset(btf, walk_type->name_off); in btf_nested_type_is_trusted()
9320 safe_id = btf_find_by_name_kind(btf, safe_tname, BTF_INFO_KIND(walk_type->info)); in btf_nested_type_is_trusted()
9324 safe_type = btf_type_by_id(btf, safe_id); in btf_nested_type_is_trusted()
9329 const char *m_name = __btf_name_by_offset(btf, member->name_off); in btf_nested_type_is_trusted()
9330 const struct btf_type *mtype = btf_type_by_id(btf, member->type); in btf_nested_type_is_trusted()
9336 btf_type_skip_modifiers(btf, mtype->type, &id); in btf_nested_type_is_trusted()
9346 const struct btf *reg_btf, u32 reg_id, in btf_type_ids_nocast_alias()
9347 const struct btf *arg_btf, u32 arg_id) in btf_type_ids_nocast_alias()
9402 btf_add_struct_ops(struct btf *btf, struct bpf_struct_ops *st_ops, in btf_add_struct_ops() argument
9408 tab = btf->struct_ops_tab; in btf_add_struct_ops()
9415 btf->struct_ops_tab = tab; in btf_add_struct_ops()
9431 btf->struct_ops_tab = tab; in btf_add_struct_ops()
9434 tab->ops[btf->struct_ops_tab->cnt].st_ops = st_ops; in btf_add_struct_ops()
9436 err = bpf_struct_ops_desc_init(&tab->ops[btf->struct_ops_tab->cnt], btf, log); in btf_add_struct_ops()
9440 btf->struct_ops_tab->cnt++; in btf_add_struct_ops()
9446 bpf_struct_ops_find_value(struct btf *btf, u32 value_id) in bpf_struct_ops_find_value() argument
9454 if (!btf->struct_ops_tab) in bpf_struct_ops_find_value()
9457 cnt = btf->struct_ops_tab->cnt; in bpf_struct_ops_find_value()
9458 st_ops_list = btf->struct_ops_tab->ops; in bpf_struct_ops_find_value()
9468 bpf_struct_ops_find(struct btf *btf, u32 type_id) in bpf_struct_ops_find() argument
9476 if (!btf->struct_ops_tab) in bpf_struct_ops_find()
9479 cnt = btf->struct_ops_tab->cnt; in bpf_struct_ops_find()
9480 st_ops_list = btf->struct_ops_tab->ops; in bpf_struct_ops_find()
9492 struct btf *btf; in __register_bpf_struct_ops() local
9495 btf = btf_get_module_btf(st_ops->owner); in __register_bpf_struct_ops()
9496 if (!btf) in __register_bpf_struct_ops()
9498 if (IS_ERR(btf)) in __register_bpf_struct_ops()
9499 return PTR_ERR(btf); in __register_bpf_struct_ops()
9509 err = btf_add_struct_ops(btf, st_ops, log); in __register_bpf_struct_ops()
9513 btf_put(btf); in __register_bpf_struct_ops()
9520 bool btf_param_match_suffix(const struct btf *btf, in btf_param_match_suffix() argument
9528 param_name = btf_name_by_offset(btf, arg->name_off); in btf_param_match_suffix()