Lines Matching refs:prog
75 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);
757 void bpf_program__unload(struct bpf_program *prog) in bpf_program__unload() argument
759 if (!prog) in bpf_program__unload()
762 zclose(prog->fd); in bpf_program__unload()
764 zfree(&prog->func_info); in bpf_program__unload()
765 zfree(&prog->line_info); in bpf_program__unload()
768 static void bpf_program__exit(struct bpf_program *prog) in bpf_program__exit() argument
770 if (!prog) in bpf_program__exit()
773 bpf_program__unload(prog); in bpf_program__exit()
774 zfree(&prog->name); in bpf_program__exit()
775 zfree(&prog->sec_name); in bpf_program__exit()
776 zfree(&prog->insns); in bpf_program__exit()
777 zfree(&prog->reloc_desc); in bpf_program__exit()
779 prog->nr_reloc = 0; in bpf_program__exit()
780 prog->insns_cnt = 0; in bpf_program__exit()
781 prog->sec_idx = -1; in bpf_program__exit()
805 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog, in bpf_object__init_prog() argument
815 memset(prog, 0, sizeof(*prog)); in bpf_object__init_prog()
816 prog->obj = obj; in bpf_object__init_prog()
818 prog->sec_idx = sec_idx; in bpf_object__init_prog()
819 prog->sec_insn_off = sec_off / BPF_INSN_SZ; in bpf_object__init_prog()
820 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ; in bpf_object__init_prog()
822 prog->insns_cnt = prog->sec_insn_cnt; in bpf_object__init_prog()
824 prog->type = BPF_PROG_TYPE_UNSPEC; in bpf_object__init_prog()
825 prog->fd = -1; in bpf_object__init_prog()
826 prog->exception_cb_idx = -1; in bpf_object__init_prog()
833 prog->autoload = false; in bpf_object__init_prog()
837 prog->autoload = true; in bpf_object__init_prog()
840 prog->autoattach = true; in bpf_object__init_prog()
843 prog->log_level = obj->log_level; in bpf_object__init_prog()
845 prog->sec_name = strdup(sec_name); in bpf_object__init_prog()
846 if (!prog->sec_name) in bpf_object__init_prog()
849 prog->name = strdup(name); in bpf_object__init_prog()
850 if (!prog->name) in bpf_object__init_prog()
853 prog->insns = malloc(insn_data_sz); in bpf_object__init_prog()
854 if (!prog->insns) in bpf_object__init_prog()
856 memcpy(prog->insns, insn_data, insn_data_sz); in bpf_object__init_prog()
861 bpf_program__exit(prog); in bpf_object__init_prog()
870 struct bpf_program *prog, *progs; in bpf_object__add_programs() local
926 prog = &progs[nr_progs]; in bpf_object__add_programs()
928 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name, in bpf_object__add_programs()
934 prog->sym_global = true; in bpf_object__add_programs()
941 if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN in bpf_object__add_programs()
943 prog->mark_btf_static = true; in bpf_object__add_programs()
954 struct bpf_program *prog = obj->programs; in bpf_object_bswap_progs() local
958 for (p = 0; p < obj->nr_programs; p++, prog++) { in bpf_object_bswap_progs()
959 insn = prog->insns; in bpf_object_bswap_progs()
960 for (i = 0; i < prog->insns_cnt; i++, insn++) in bpf_object_bswap_progs()
1075 const struct bpf_program *prog) in is_valid_st_ops_program() argument
1080 if (&obj->programs[i] == prog) in is_valid_st_ops_program()
1081 return prog->type == BPF_PROG_TYPE_STRUCT_OPS; in is_valid_st_ops_program()
1094 struct bpf_program *prog, *slot_prog; in bpf_object_adjust_struct_ops_autoload() local
1102 prog = &obj->programs[i]; in bpf_object_adjust_struct_ops_autoload()
1103 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) in bpf_object_adjust_struct_ops_autoload()
1117 if (prog != slot_prog) in bpf_object_adjust_struct_ops_autoload()
1126 prog->autoload = should_load; in bpf_object_adjust_struct_ops_autoload()
1179 struct bpf_program *prog; in bpf_map__init_kern_struct_ops() local
1244 prog = *(void **)mdata; in bpf_map__init_kern_struct_ops()
1249 if (st_ops->progs[i] && st_ops->progs[i] != prog) in bpf_map__init_kern_struct_ops()
1253 st_ops->progs[i] = prog; in bpf_map__init_kern_struct_ops()
1254 if (!prog) in bpf_map__init_kern_struct_ops()
1257 if (!is_valid_st_ops_program(obj, prog)) { in bpf_map__init_kern_struct_ops()
1278 prog->attach_btf_obj_fd = mod_btf->fd; in bpf_map__init_kern_struct_ops()
1283 if (!prog->attach_btf_id) { in bpf_map__init_kern_struct_ops()
1284 prog->attach_btf_id = kern_type_id; in bpf_map__init_kern_struct_ops()
1285 prog->expected_attach_type = kern_member_idx; in bpf_map__init_kern_struct_ops()
1293 if (prog->attach_btf_id != kern_type_id) { in bpf_map__init_kern_struct_ops()
1295 map->name, mname, prog->name, prog->sec_name, prog->type, in bpf_map__init_kern_struct_ops()
1296 prog->attach_btf_id, kern_type_id); in bpf_map__init_kern_struct_ops()
1299 if (prog->expected_attach_type != kern_member_idx) { in bpf_map__init_kern_struct_ops()
1301 map->name, mname, prog->name, prog->sec_name, prog->type, in bpf_map__init_kern_struct_ops()
1302 prog->expected_attach_type, kern_member_idx); in bpf_map__init_kern_struct_ops()
1309 map->name, mname, prog->name, moff, in bpf_map__init_kern_struct_ops()
3449 static bool prog_needs_vmlinux_btf(struct bpf_program *prog) in prog_needs_vmlinux_btf() argument
3451 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS || in prog_needs_vmlinux_btf()
3452 prog->type == BPF_PROG_TYPE_LSM) in prog_needs_vmlinux_btf()
3458 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd) in prog_needs_vmlinux_btf()
3471 struct bpf_program *prog; in obj_needs_vmlinux_btf() local
3490 bpf_object__for_each_program(prog, obj) { in obj_needs_vmlinux_btf()
3491 if (!prog->autoload) in obj_needs_vmlinux_btf()
3493 if (prog_needs_vmlinux_btf(prog)) in obj_needs_vmlinux_btf()
3553 struct bpf_program *prog = &obj->programs[i]; in bpf_object__sanitize_and_load_btf() local
3558 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog)) in bpf_object__sanitize_and_load_btf()
3568 if (strcmp(name, prog->name) != 0) in bpf_object__sanitize_and_load_btf()
4435 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog) in prog_is_subprog() argument
4437 return prog->sec_idx == obj->efile.text_shndx; in prog_is_subprog()
4444 struct bpf_program *prog; in bpf_object__find_program_by_name() local
4446 bpf_object__for_each_program(prog, obj) { in bpf_object__find_program_by_name()
4447 if (prog_is_subprog(obj, prog)) in bpf_object__find_program_by_name()
4449 if (!strcmp(prog->name, name)) in bpf_object__find_program_by_name()
4450 return prog; in bpf_object__find_program_by_name()
4492 static int bpf_program__record_reloc(struct bpf_program *prog, in bpf_program__record_reloc() argument
4497 struct bpf_insn *insn = &prog->insns[insn_idx]; in bpf_program__record_reloc()
4498 size_t map_idx, nr_maps = prog->obj->nr_maps; in bpf_program__record_reloc()
4499 struct bpf_object *obj = prog->obj; in bpf_program__record_reloc()
4507 prog->name, sym_name, insn_idx, insn->code); in bpf_program__record_reloc()
4523 prog->name, sym_name, sym_idx); in bpf_program__record_reloc()
4527 prog->name, i, ext->name, ext->sym_idx, insn_idx); in bpf_program__record_reloc()
4540 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name); in bpf_program__record_reloc()
4547 prog->name, sym_name, sym_sec_name); in bpf_program__record_reloc()
4552 prog->name, sym_name, (size_t)sym->st_value); in bpf_program__record_reloc()
4563 prog->name, sym_name, shdr_idx); in bpf_program__record_reloc()
4574 prog->name, sym_name, (size_t)sym->st_value, insn->imm); in bpf_program__record_reloc()
4600 prog->name, sym_name, sym_sec_name); in bpf_program__record_reloc()
4610 prog->name, map_idx, map->name, map->sec_idx, in bpf_program__record_reloc()
4616 prog->name, sym_sec_name, (size_t)sym->st_value); in bpf_program__record_reloc()
4629 prog->name, sym_sec_name); in bpf_program__record_reloc()
4637 prog->name, map_idx, map->name, map->sec_idx, in bpf_program__record_reloc()
4643 prog->name, sym_sec_name); in bpf_program__record_reloc()
4654 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx) in prog_contains_insn() argument
4656 return insn_idx >= prog->sec_insn_off && in prog_contains_insn()
4657 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt; in prog_contains_insn()
4664 struct bpf_program *prog; in find_prog_by_sec_insn() local
4671 prog = &obj->programs[m]; in find_prog_by_sec_insn()
4673 if (prog->sec_idx < sec_idx || in find_prog_by_sec_insn()
4674 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx)) in find_prog_by_sec_insn()
4682 prog = &obj->programs[l]; in find_prog_by_sec_insn()
4683 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx)) in find_prog_by_sec_insn()
4684 return prog; in find_prog_by_sec_insn()
4693 struct bpf_program *prog; in bpf_object__collect_prog_relos() local
4763 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); in bpf_object__collect_prog_relos()
4764 if (!prog) { in bpf_object__collect_prog_relos()
4770 relos = libbpf_reallocarray(prog->reloc_desc, in bpf_object__collect_prog_relos()
4771 prog->nr_reloc + 1, sizeof(*relos)); in bpf_object__collect_prog_relos()
4774 prog->reloc_desc = relos; in bpf_object__collect_prog_relos()
4777 insn_idx -= prog->sec_insn_off; in bpf_object__collect_prog_relos()
4778 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc], in bpf_object__collect_prog_relos()
4783 prog->nr_reloc++; in bpf_object__collect_prog_relos()
5838 static int record_relo_core(struct bpf_program *prog, in record_relo_core() argument
5843 relos = libbpf_reallocarray(prog->reloc_desc, in record_relo_core()
5844 prog->nr_reloc + 1, sizeof(*relos)); in record_relo_core()
5847 relo = &relos[prog->nr_reloc]; in record_relo_core()
5851 prog->reloc_desc = relos; in record_relo_core()
5852 prog->nr_reloc++; in record_relo_core()
5856 static const struct bpf_core_relo *find_relo_core(struct bpf_program *prog, int insn_idx) in find_relo_core() argument
5861 for (i = 0; i < prog->nr_reloc; i++) { in find_relo_core()
5862 relo = &prog->reloc_desc[i]; in find_relo_core()
5872 static int bpf_core_resolve_relo(struct bpf_program *prog, in bpf_core_resolve_relo() argument
5881 const char *prog_name = prog->name; in bpf_core_resolve_relo()
5897 cands = bpf_core_find_cands(prog->obj, local_btf, local_id); in bpf_core_resolve_relo()
5924 struct bpf_program *prog; in bpf_object__relocate_core() local
5965 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); in bpf_object__relocate_core()
5966 if (!prog) { in bpf_object__relocate_core()
5982 if (!prog->autoload) in bpf_object__relocate_core()
5989 insn_idx = insn_idx - prog->sec_insn_off; in bpf_object__relocate_core()
5990 if (insn_idx >= prog->insns_cnt) in bpf_object__relocate_core()
5992 insn = &prog->insns[insn_idx]; in bpf_object__relocate_core()
5994 err = record_relo_core(prog, rec, insn_idx); in bpf_object__relocate_core()
5997 prog->name, i, errstr(err)); in bpf_object__relocate_core()
6001 if (prog->obj->gen_loader) in bpf_object__relocate_core()
6004 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res); in bpf_object__relocate_core()
6007 prog->name, i, errstr(err)); in bpf_object__relocate_core()
6011 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res); in bpf_object__relocate_core()
6014 prog->name, i, insn_idx, errstr(err)); in bpf_object__relocate_core()
6038 static void poison_map_ldimm64(struct bpf_program *prog, int relo_idx, in poison_map_ldimm64() argument
6045 prog->name, relo_idx, insn_idx, map_idx, map->name); in poison_map_ldimm64()
6068 static void poison_kfunc_call(struct bpf_program *prog, int relo_idx, in poison_kfunc_call() argument
6073 prog->name, relo_idx, insn_idx, ext->name); in poison_kfunc_call()
6094 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) in bpf_object__relocate_data() argument
6098 for (i = 0; i < prog->nr_reloc; i++) { in bpf_object__relocate_data()
6099 struct reloc_desc *relo = &prog->reloc_desc[i]; in bpf_object__relocate_data()
6100 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; in bpf_object__relocate_data()
6114 poison_map_ldimm64(prog, i, relo->insn_idx, insn, in bpf_object__relocate_data()
6128 poison_map_ldimm64(prog, i, relo->insn_idx, insn, in bpf_object__relocate_data()
6161 poison_kfunc_call(prog, i, relo->insn_idx, insn, in bpf_object__relocate_data()
6168 prog->name, i); in bpf_object__relocate_data()
6181 prog->name, i, relo->type); in bpf_object__relocate_data()
6190 const struct bpf_program *prog, in adjust_prog_btf_ext_info() argument
6205 if (prog->sec_idx != sec_idx) in adjust_prog_btf_ext_info()
6211 if (insn_off < prog->sec_insn_off) in adjust_prog_btf_ext_info()
6213 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt) in adjust_prog_btf_ext_info()
6242 off_adj = prog->sub_insn_off - prog->sec_insn_off; in adjust_prog_btf_ext_info()
6260 const struct bpf_program *prog) in reloc_prog_func_and_line_info() argument
6273 if (main_prog != prog && !main_prog->func_info) in reloc_prog_func_and_line_info()
6276 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info, in reloc_prog_func_and_line_info()
6283 prog->name, errstr(err)); in reloc_prog_func_and_line_info()
6291 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name); in reloc_prog_func_and_line_info()
6296 prog->name); in reloc_prog_func_and_line_info()
6301 if (main_prog != prog && !main_prog->line_info) in reloc_prog_func_and_line_info()
6304 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info, in reloc_prog_func_and_line_info()
6311 prog->name, errstr(err)); in reloc_prog_func_and_line_info()
6319 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name); in reloc_prog_func_and_line_info()
6324 prog->name); in reloc_prog_func_and_line_info()
6339 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx) in find_prog_insn_relo() argument
6341 if (!prog->nr_reloc) in find_prog_insn_relo()
6343 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc, in find_prog_insn_relo()
6344 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx); in find_prog_insn_relo()
6410 struct bpf_program *prog) in bpf_object__reloc_code() argument
6418 err = reloc_prog_func_and_line_info(obj, main_prog, prog); in bpf_object__reloc_code()
6422 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) { in bpf_object__reloc_code()
6423 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; in bpf_object__reloc_code()
6427 relo = find_prog_insn_relo(prog, insn_idx); in bpf_object__reloc_code()
6435 prog->name, insn_idx, relo->type); in bpf_object__reloc_code()
6459 prog->name, insn_idx); in bpf_object__reloc_code()
6468 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1; in bpf_object__reloc_code()
6475 prog->name); in bpf_object__reloc_code()
6501 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; in bpf_object__reloc_code()
6508 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1; in bpf_object__reloc_code()
6511 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off); in bpf_object__reloc_code()
6599 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog) in bpf_object__relocate_calls() argument
6615 err = bpf_object__reloc_code(obj, prog, prog); in bpf_object__relocate_calls()
6625 struct bpf_program *prog; in bpf_object__free_relocs() local
6630 prog = &obj->programs[i]; in bpf_object__free_relocs()
6631 zfree(&prog->reloc_desc); in bpf_object__free_relocs()
6632 prog->nr_reloc = 0; in bpf_object__free_relocs()
6665 static int bpf_prog_assign_exc_cb(struct bpf_object *obj, struct bpf_program *prog) in bpf_prog_assign_exc_cb() argument
6690 prog->name); in bpf_prog_assign_exc_cb()
6693 if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off)) != 0) in bpf_prog_assign_exc_cb()
6699 if (prog->exception_cb_idx >= 0) { in bpf_prog_assign_exc_cb()
6700 prog->exception_cb_idx = -1; in bpf_prog_assign_exc_cb()
6707 prog->name); in bpf_prog_assign_exc_cb()
6724 prog->name, subprog->name); in bpf_prog_assign_exc_cb()
6728 if (prog->exception_cb_idx >= 0) { in bpf_prog_assign_exc_cb()
6730 prog->name, subprog->name); in bpf_prog_assign_exc_cb()
6733 prog->exception_cb_idx = j; in bpf_prog_assign_exc_cb()
6737 if (prog->exception_cb_idx >= 0) in bpf_prog_assign_exc_cb()
6740 pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name); in bpf_prog_assign_exc_cb()
6787 static bool need_func_arg_type_fixup(const struct btf *btf, const struct bpf_program *prog, in need_func_arg_type_fixup() argument
6806 (prog->type == BPF_PROG_TYPE_KPROBE || prog->type == BPF_PROG_TYPE_PERF_EVENT)) { in need_func_arg_type_fixup()
6825 switch (prog->type) { in need_func_arg_type_fixup()
6854 prog->name, subprog_name, arg_idx, ctx_name); in need_func_arg_type_fixup()
6858 static int clone_func_btf_info(struct btf *btf, int orig_fn_id, struct bpf_program *prog) in clone_func_btf_info() argument
6910 fn_id = btf__add_func(btf, prog->name, linkage, fn_proto_id); in clone_func_btf_info()
6927 static int bpf_program_fixup_func_info(struct bpf_object *obj, struct bpf_program *prog) in bpf_program_fixup_func_info() argument
6940 if (!obj->btf_ext || !prog->func_info) in bpf_program_fixup_func_info()
6951 if (global_ctx_map[i].prog_type != prog->type) in bpf_program_fixup_func_info()
6960 orig_ids = calloc(prog->func_info_cnt, sizeof(*orig_ids)); in bpf_program_fixup_func_info()
6963 for (i = 0; i < prog->func_info_cnt; i++) { in bpf_program_fixup_func_info()
6964 func_rec = prog->func_info + prog->func_info_rec_size * i; in bpf_program_fixup_func_info()
6993 for (rec_idx = 0; rec_idx < prog->func_info_cnt; rec_idx++) { in bpf_program_fixup_func_info()
6995 func_rec = prog->func_info + prog->func_info_rec_size * rec_idx; in bpf_program_fixup_func_info()
7012 if (!need_func_arg_type_fixup(btf, prog, fn_name, arg_idx, p->type, ctx_name)) in bpf_program_fixup_func_info()
7019 fn_id = clone_func_btf_info(btf, orig_fn_id, prog); in bpf_program_fixup_func_info()
7068 struct bpf_program *prog; in bpf_object__relocate() local
7090 prog = &obj->programs[i]; in bpf_object__relocate()
7091 for (j = 0; j < prog->nr_reloc; j++) { in bpf_object__relocate()
7092 struct reloc_desc *relo = &prog->reloc_desc[j]; in bpf_object__relocate()
7093 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; in bpf_object__relocate()
7109 prog = &obj->programs[i]; in bpf_object__relocate()
7113 if (prog_is_subprog(obj, prog)) in bpf_object__relocate()
7115 if (!prog->autoload) in bpf_object__relocate()
7118 err = bpf_object__relocate_calls(obj, prog); in bpf_object__relocate()
7121 prog->name, errstr(err)); in bpf_object__relocate()
7125 err = bpf_prog_assign_exc_cb(obj, prog); in bpf_object__relocate()
7129 if (prog->exception_cb_idx >= 0) { in bpf_object__relocate()
7130 struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx]; in bpf_object__relocate()
7138 err = bpf_object__append_subprog_code(obj, prog, subprog); in bpf_object__relocate()
7141 err = bpf_object__reloc_code(obj, prog, subprog); in bpf_object__relocate()
7148 prog = &obj->programs[i]; in bpf_object__relocate()
7149 if (prog_is_subprog(obj, prog)) in bpf_object__relocate()
7151 if (!prog->autoload) in bpf_object__relocate()
7155 err = bpf_object__relocate_data(obj, prog); in bpf_object__relocate()
7158 prog->name, errstr(err)); in bpf_object__relocate()
7163 err = bpf_program_fixup_func_info(obj, prog); in bpf_object__relocate()
7166 prog->name, errstr(err)); in bpf_object__relocate()
7363 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog) in bpf_object__sanitize_prog() argument
7365 struct bpf_insn *insn = prog->insns; in bpf_object__sanitize_prog()
7372 for (i = 0; i < prog->insns_cnt; i++, insn++) { in bpf_object__sanitize_prog()
7398 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
7402 static int libbpf_prepare_prog_load(struct bpf_program *prog, in libbpf_prepare_prog_load() argument
7408 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE)) in libbpf_prepare_prog_load()
7414 if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) in libbpf_prepare_prog_load()
7418 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) { in libbpf_prepare_prog_load()
7423 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI; in libbpf_prepare_prog_load()
7427 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) { in libbpf_prepare_prog_load()
7431 attach_name = strchr(prog->sec_name, '/'); in libbpf_prepare_prog_load()
7443 prog->name); in libbpf_prepare_prog_load()
7448 err = libbpf_find_attach_btf_id(prog, attach_name, &btf_obj_fd, &btf_type_id); in libbpf_prepare_prog_load()
7453 prog->attach_btf_obj_fd = btf_obj_fd; in libbpf_prepare_prog_load()
7454 prog->attach_btf_id = btf_type_id; in libbpf_prepare_prog_load()
7467 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz);
7469 static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog, in bpf_object_load_prog() argument
7478 __u32 log_level = prog->log_level; in bpf_object_load_prog()
7484 switch (prog->type) { in bpf_object_load_prog()
7491 prog->name, prog->sec_name); in bpf_object_load_prog()
7494 if (prog->attach_btf_id == 0) { in bpf_object_load_prog()
7496 prog->name); in bpf_object_load_prog()
7508 prog_name = prog->name; in bpf_object_load_prog()
7509 load_attr.attach_prog_fd = prog->attach_prog_fd; in bpf_object_load_prog()
7510 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd; in bpf_object_load_prog()
7511 load_attr.attach_btf_id = prog->attach_btf_id; in bpf_object_load_prog()
7513 load_attr.prog_ifindex = prog->prog_ifindex; in bpf_object_load_prog()
7514 load_attr.expected_attach_type = prog->expected_attach_type; in bpf_object_load_prog()
7519 load_attr.func_info = prog->func_info; in bpf_object_load_prog()
7520 load_attr.func_info_rec_size = prog->func_info_rec_size; in bpf_object_load_prog()
7521 load_attr.func_info_cnt = prog->func_info_cnt; in bpf_object_load_prog()
7522 load_attr.line_info = prog->line_info; in bpf_object_load_prog()
7523 load_attr.line_info_rec_size = prog->line_info_rec_size; in bpf_object_load_prog()
7524 load_attr.line_info_cnt = prog->line_info_cnt; in bpf_object_load_prog()
7527 load_attr.prog_flags = prog->prog_flags; in bpf_object_load_prog()
7535 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) { in bpf_object_load_prog()
7536 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie); in bpf_object_load_prog()
7539 prog->name, errstr(err)); in bpf_object_load_prog()
7542 insns = prog->insns; in bpf_object_load_prog()
7543 insns_cnt = prog->insns_cnt; in bpf_object_load_prog()
7547 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name, in bpf_object_load_prog()
7549 prog - obj->programs); in bpf_object_load_prog()
7561 if (prog->log_buf) { in bpf_object_load_prog()
7562 log_buf = prog->log_buf; in bpf_object_load_prog()
7563 log_buf_size = prog->log_size; in bpf_object_load_prog()
7586 ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr); in bpf_object_load_prog()
7590 prog->name, log_buf); in bpf_object_load_prog()
7598 map = &prog->obj->maps[i]; in bpf_object_load_prog()
7604 prog->name, map->real_name, errstr(errno)); in bpf_object_load_prog()
7632 fixup_verifier_log(prog, log_buf, log_buf_size); in bpf_object_load_prog()
7634 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, errstr(errno)); in bpf_object_load_prog()
7639 prog->name, log_buf); in bpf_object_load_prog()
7699 static void fixup_log_failed_core_relo(struct bpf_program *prog, in fixup_log_failed_core_relo() argument
7721 relo = find_relo_core(prog, insn_idx); in fixup_log_failed_core_relo()
7725 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec); in fixup_log_failed_core_relo()
7738 static void fixup_log_missing_map_load(struct bpf_program *prog, in fixup_log_missing_map_load() argument
7750 struct bpf_object *obj = prog->obj; in fixup_log_missing_map_load()
7771 static void fixup_log_missing_kfunc_call(struct bpf_program *prog, in fixup_log_missing_kfunc_call() argument
7783 struct bpf_object *obj = prog->obj; in fixup_log_missing_kfunc_call()
7804 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz) in fixup_verifier_log() argument
7829 fixup_log_failed_core_relo(prog, buf, buf_sz, log_sz, in fixup_verifier_log()
7838 fixup_log_missing_map_load(prog, buf, buf_sz, log_sz, in fixup_verifier_log()
7847 fixup_log_missing_kfunc_call(prog, buf, buf_sz, log_sz, in fixup_verifier_log()
7854 static int bpf_program_record_relos(struct bpf_program *prog) in bpf_program_record_relos() argument
7856 struct bpf_object *obj = prog->obj; in bpf_program_record_relos()
7859 for (i = 0; i < prog->nr_reloc; i++) { in bpf_program_record_relos()
7860 struct reloc_desc *relo = &prog->reloc_desc[i]; in bpf_program_record_relos()
7900 struct bpf_program *prog; in bpf_object__load_progs() local
7905 prog = &obj->programs[i]; in bpf_object__load_progs()
7906 if (prog_is_subprog(obj, prog)) in bpf_object__load_progs()
7908 if (!prog->autoload) { in bpf_object__load_progs()
7909 pr_debug("prog '%s': skipped loading\n", prog->name); in bpf_object__load_progs()
7912 prog->log_level |= log_level; in bpf_object__load_progs()
7915 bpf_program_record_relos(prog); in bpf_object__load_progs()
7917 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt, in bpf_object__load_progs()
7918 obj->license, obj->kern_version, &prog->fd); in bpf_object__load_progs()
7920 pr_warn("prog '%s': failed to load: %s\n", prog->name, errstr(err)); in bpf_object__load_progs()
7931 struct bpf_program *prog; in bpf_object_prepare_progs() local
7936 prog = &obj->programs[i]; in bpf_object_prepare_progs()
7937 err = bpf_object__sanitize_prog(obj, prog); in bpf_object_prepare_progs()
7948 struct bpf_program *prog; in bpf_object_init_progs() local
7951 bpf_object__for_each_program(prog, obj) { in bpf_object_init_progs()
7952 prog->sec_def = find_sec_def(prog->sec_name); in bpf_object_init_progs()
7953 if (!prog->sec_def) { in bpf_object_init_progs()
7956 prog->name, prog->sec_name); in bpf_object_init_progs()
7960 prog->type = prog->sec_def->prog_type; in bpf_object_init_progs()
7961 prog->expected_attach_type = prog->sec_def->expected_attach_type; in bpf_object_init_progs()
7966 if (prog->sec_def->prog_setup_fn) { in bpf_object_init_progs()
7967 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie); in bpf_object_init_progs()
7970 prog->name, errstr(err)); in bpf_object_init_progs()
8527 struct bpf_program *prog = st_ops->progs[i]; in bpf_map_prepare_vdata() local
8531 if (!prog) in bpf_map_prepare_vdata()
8534 prog_fd = bpf_program__fd(prog); in bpf_map_prepare_vdata()
8733 int bpf_program__pin(struct bpf_program *prog, const char *path) in bpf_program__pin() argument
8737 if (prog->fd < 0) { in bpf_program__pin()
8738 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name); in bpf_program__pin()
8750 if (bpf_obj_pin(prog->fd, path)) { in bpf_program__pin()
8752 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, errstr(err)); in bpf_program__pin()
8756 pr_debug("prog '%s': pinned at '%s'\n", prog->name, path); in bpf_program__pin()
8760 int bpf_program__unpin(struct bpf_program *prog, const char *path) in bpf_program__unpin() argument
8764 if (prog->fd < 0) { in bpf_program__unpin()
8765 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name); in bpf_program__unpin()
8777 pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path); in bpf_program__unpin()
8999 struct bpf_program *prog; in bpf_object__pin_programs() local
9011 bpf_object__for_each_program(prog, obj) { in bpf_object__pin_programs()
9012 err = pathname_concat(buf, sizeof(buf), path, prog->name); in bpf_object__pin_programs()
9016 err = bpf_program__pin(prog, buf); in bpf_object__pin_programs()
9024 while ((prog = bpf_object__prev_program(obj, prog))) { in bpf_object__pin_programs()
9025 if (pathname_concat(buf, sizeof(buf), path, prog->name)) in bpf_object__pin_programs()
9028 bpf_program__unpin(prog, buf); in bpf_object__pin_programs()
9036 struct bpf_program *prog; in bpf_object__unpin_programs() local
9042 bpf_object__for_each_program(prog, obj) { in bpf_object__unpin_programs()
9045 err = pathname_concat(buf, sizeof(buf), path, prog->name); in bpf_object__unpin_programs()
9049 err = bpf_program__unpin(prog, buf); in bpf_object__unpin_programs()
9254 struct bpf_program *prog = prev; in bpf_object__next_program() local
9257 prog = __bpf_program__iter(prog, obj, true); in bpf_object__next_program()
9258 } while (prog && prog_is_subprog(obj, prog)); in bpf_object__next_program()
9260 return prog; in bpf_object__next_program()
9266 struct bpf_program *prog = next; in bpf_object__prev_program() local
9269 prog = __bpf_program__iter(prog, obj, false); in bpf_object__prev_program()
9270 } while (prog && prog_is_subprog(obj, prog)); in bpf_object__prev_program()
9272 return prog; in bpf_object__prev_program()
9275 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) in bpf_program__set_ifindex() argument
9277 prog->prog_ifindex = ifindex; in bpf_program__set_ifindex()
9280 const char *bpf_program__name(const struct bpf_program *prog) in bpf_program__name() argument
9282 return prog->name; in bpf_program__name()
9285 const char *bpf_program__section_name(const struct bpf_program *prog) in bpf_program__section_name() argument
9287 return prog->sec_name; in bpf_program__section_name()
9290 bool bpf_program__autoload(const struct bpf_program *prog) in bpf_program__autoload() argument
9292 return prog->autoload; in bpf_program__autoload()
9295 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload) in bpf_program__set_autoload() argument
9297 if (prog->obj->state >= OBJ_LOADED) in bpf_program__set_autoload()
9300 prog->autoload = autoload; in bpf_program__set_autoload()
9304 bool bpf_program__autoattach(const struct bpf_program *prog) in bpf_program__autoattach() argument
9306 return prog->autoattach; in bpf_program__autoattach()
9309 void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach) in bpf_program__set_autoattach() argument
9311 prog->autoattach = autoattach; in bpf_program__set_autoattach()
9314 const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog) in bpf_program__insns() argument
9316 return prog->insns; in bpf_program__insns()
9319 size_t bpf_program__insn_cnt(const struct bpf_program *prog) in bpf_program__insn_cnt() argument
9321 return prog->insns_cnt; in bpf_program__insn_cnt()
9324 int bpf_program__set_insns(struct bpf_program *prog, in bpf_program__set_insns() argument
9329 if (prog->obj->state >= OBJ_LOADED) in bpf_program__set_insns()
9332 insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns)); in bpf_program__set_insns()
9335 pr_warn("prog '%s': failed to realloc prog code\n", prog->name); in bpf_program__set_insns()
9340 prog->insns = insns; in bpf_program__set_insns()
9341 prog->insns_cnt = new_insn_cnt; in bpf_program__set_insns()
9345 int bpf_program__fd(const struct bpf_program *prog) in bpf_program__fd() argument
9347 if (!prog) in bpf_program__fd()
9350 if (prog->fd < 0) in bpf_program__fd()
9353 return prog->fd; in bpf_program__fd()
9357 enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog);
9359 enum bpf_prog_type bpf_program__type(const struct bpf_program *prog) in bpf_program__type() argument
9361 return prog->type; in bpf_program__type()
9370 int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) in bpf_program__set_type() argument
9372 if (prog->obj->state >= OBJ_LOADED) in bpf_program__set_type()
9376 if (prog->type == type) in bpf_program__set_type()
9379 prog->type = type; in bpf_program__set_type()
9387 if (prog->sec_def != &custom_fallback_def) in bpf_program__set_type()
9388 prog->sec_def = NULL; in bpf_program__set_type()
9393 enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog);
9395 enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog) in bpf_program__expected_attach_type() argument
9397 return prog->expected_attach_type; in bpf_program__expected_attach_type()
9400 int bpf_program__set_expected_attach_type(struct bpf_program *prog, in bpf_program__set_expected_attach_type() argument
9403 if (prog->obj->state >= OBJ_LOADED) in bpf_program__set_expected_attach_type()
9406 prog->expected_attach_type = type; in bpf_program__set_expected_attach_type()
9410 __u32 bpf_program__flags(const struct bpf_program *prog) in bpf_program__flags() argument
9412 return prog->prog_flags; in bpf_program__flags()
9415 int bpf_program__set_flags(struct bpf_program *prog, __u32 flags) in bpf_program__set_flags() argument
9417 if (prog->obj->state >= OBJ_LOADED) in bpf_program__set_flags()
9420 prog->prog_flags = flags; in bpf_program__set_flags()
9424 __u32 bpf_program__log_level(const struct bpf_program *prog) in bpf_program__log_level() argument
9426 return prog->log_level; in bpf_program__log_level()
9429 int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level) in bpf_program__set_log_level() argument
9431 if (prog->obj->state >= OBJ_LOADED) in bpf_program__set_log_level()
9434 prog->log_level = log_level; in bpf_program__set_log_level()
9438 const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size) in bpf_program__log_buf() argument
9440 *log_size = prog->log_size; in bpf_program__log_buf()
9441 return prog->log_buf; in bpf_program__log_buf()
9444 int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size) in bpf_program__set_log_buf() argument
9448 if (prog->log_size > UINT_MAX) in bpf_program__set_log_buf()
9450 if (prog->obj->state >= OBJ_LOADED) in bpf_program__set_log_buf()
9453 prog->log_buf = log_buf; in bpf_program__set_log_buf()
9454 prog->log_size = log_size; in bpf_program__set_log_buf()
9467 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9468 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9469 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9470 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9471 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9472 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9473 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9474 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9475 static int attach_kprobe_session(const struct bpf_program *prog, long cookie, struct bpf_link **lin…
9476 static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9477 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9478 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9850 struct bpf_program *prog; in bpf_object__collect_st_ops_relos() local
9923 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx); in bpf_object__collect_st_ops_relos()
9924 if (!prog) { in bpf_object__collect_st_ops_relos()
9931 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) { in bpf_object__collect_st_ops_relos()
9933 map->name, prog->name); in bpf_object__collect_st_ops_relos()
9937 st_ops->progs[member_idx] = prog; in bpf_object__collect_st_ops_relos()
9945 *((struct bpf_program **)(st_ops->data + moff)) = prog; in bpf_object__collect_st_ops_relos()
10117 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name, in libbpf_find_attach_btf_id() argument
10120 enum bpf_attach_type attach_type = prog->expected_attach_type; in libbpf_find_attach_btf_id()
10121 __u32 attach_prog_fd = prog->attach_prog_fd; in libbpf_find_attach_btf_id()
10125 if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) { in libbpf_find_attach_btf_id()
10127 pr_warn("prog '%s': attach program FD is not set\n", prog->name); in libbpf_find_attach_btf_id()
10130 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd, prog->obj->token_fd); in libbpf_find_attach_btf_id()
10133 prog->name, attach_prog_fd, attach_name, errstr(err)); in libbpf_find_attach_btf_id()
10142 if (prog->obj->gen_loader) { in libbpf_find_attach_btf_id()
10143 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type); in libbpf_find_attach_btf_id()
10147 err = find_kernel_btf_id(prog->obj, attach_name, in libbpf_find_attach_btf_id()
10153 prog->name, attach_name, errstr(err)); in libbpf_find_attach_btf_id()
10689 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog) in bpf_link__update_program() argument
10692 int prog_fd = bpf_program__fd(prog); in bpf_link__update_program()
10696 prog->name); in bpf_link__update_program()
10876 struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd, in bpf_program__attach_perf_event_opts() argument
10888 prog->name, pfd); in bpf_program__attach_perf_event_opts()
10891 prog_fd = bpf_program__fd(prog); in bpf_program__attach_perf_event_opts()
10894 prog->name); in bpf_program__attach_perf_event_opts()
10906 if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) { in bpf_program__attach_perf_event_opts()
10914 prog->name, pfd, errstr(err)); in bpf_program__attach_perf_event_opts()
10920 pr_warn("prog '%s': user context value is not supported\n", prog->name); in bpf_program__attach_perf_event_opts()
10928 prog->name, pfd, errstr(err)); in bpf_program__attach_perf_event_opts()
10931 prog->name, pfd); in bpf_program__attach_perf_event_opts()
10939 prog->name, pfd, errstr(err)); in bpf_program__attach_perf_event_opts()
10951 struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd) in bpf_program__attach_perf_event() argument
10953 return bpf_program__attach_perf_event_opts(prog, pfd, NULL); in bpf_program__attach_perf_event()
11270 bpf_program__attach_kprobe_opts(const struct bpf_program *prog, in bpf_program__attach_kprobe_opts() argument
11302 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) in bpf_program__attach_kprobe_opts()
11331 prog->name, retprobe ? "kretprobe" : "kprobe", in bpf_program__attach_kprobe_opts()
11336 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); in bpf_program__attach_kprobe_opts()
11341 prog->name, retprobe ? "kretprobe" : "kprobe", in bpf_program__attach_kprobe_opts()
11364 struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog, in bpf_program__attach_kprobe() argument
11372 return bpf_program__attach_kprobe_opts(prog, func_name, &opts); in bpf_program__attach_kprobe()
11375 struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog, in bpf_program__attach_ksyscall() argument
11385 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) { in bpf_program__attach_ksyscall()
11400 return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts); in bpf_program__attach_ksyscall()
11614 bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, in bpf_program__attach_kprobe_multi_opts() argument
11634 prog_fd = bpf_program__fd(prog); in bpf_program__attach_kprobe_multi_opts()
11637 prog->name); in bpf_program__attach_kprobe_multi_opts()
11668 prog->name, pattern, res.cnt); in bpf_program__attach_kprobe_multi_opts()
11702 prog->name, errstr(err)); in bpf_program__attach_kprobe_multi_opts()
11715 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_kprobe() argument
11726 if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0) in attach_kprobe()
11729 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/"); in attach_kprobe()
11731 func_name = prog->sec_name + sizeof("kretprobe/") - 1; in attach_kprobe()
11733 func_name = prog->sec_name + sizeof("kprobe/") - 1; in attach_kprobe()
11747 *link = bpf_program__attach_kprobe_opts(prog, func, &opts); in attach_kprobe()
11752 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_ksyscall() argument
11760 if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0) in attach_ksyscall()
11763 opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/"); in attach_ksyscall()
11765 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1; in attach_ksyscall()
11767 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1; in attach_ksyscall()
11769 *link = bpf_program__attach_ksyscall(prog, syscall_name, &opts); in attach_ksyscall()
11773 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_kprobe_multi() argument
11783 if (strcmp(prog->sec_name, "kprobe.multi") == 0 || in attach_kprobe_multi()
11784 strcmp(prog->sec_name, "kretprobe.multi") == 0) in attach_kprobe_multi()
11787 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/"); in attach_kprobe_multi()
11789 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1; in attach_kprobe_multi()
11791 spec = prog->sec_name + sizeof("kprobe.multi/") - 1; in attach_kprobe_multi()
11799 *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts); in attach_kprobe_multi()
11804 static int attach_kprobe_session(const struct bpf_program *prog, long cookie, in attach_kprobe_session() argument
11815 if (strcmp(prog->sec_name, "kprobe.session") == 0) in attach_kprobe_session()
11818 spec = prog->sec_name + sizeof("kprobe.session/") - 1; in attach_kprobe_session()
11825 *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts); in attach_kprobe_session()
11830 static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_uprobe_multi() argument
11838 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]", in attach_uprobe_multi()
11849 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts); in attach_uprobe_multi()
11853 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name, in attach_uprobe_multi()
11854 prog->sec_name); in attach_uprobe_multi()
12088 bpf_program__attach_uprobe_multi(const struct bpf_program *prog, in bpf_program__attach_uprobe_multi() argument
12109 prog_fd = bpf_program__fd(prog); in bpf_program__attach_uprobe_multi()
12112 prog->name); in bpf_program__attach_uprobe_multi()
12159 prog->name, path, errstr(err)); in bpf_program__attach_uprobe_multi()
12202 prog->name, errstr(err)); in bpf_program__attach_uprobe_multi()
12216 bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, in bpf_program__attach_uprobe_opts() argument
12254 prog->name, binary_path, errstr(err)); in bpf_program__attach_uprobe_opts()
12287 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) in bpf_program__attach_uprobe_opts()
12318 prog->name, retprobe ? "uretprobe" : "uprobe", in bpf_program__attach_uprobe_opts()
12324 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); in bpf_program__attach_uprobe_opts()
12329 prog->name, retprobe ? "uretprobe" : "uprobe", in bpf_program__attach_uprobe_opts()
12361 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_uprobe() argument
12370 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]", in attach_uprobe()
12379 prog->name, prog->sec_name); in attach_uprobe()
12397 prog->name); in attach_uprobe()
12401 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts); in attach_uprobe()
12405 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name, in attach_uprobe()
12406 prog->sec_name); in attach_uprobe()
12416 struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog, in bpf_program__attach_uprobe() argument
12423 return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts); in bpf_program__attach_uprobe()
12426 struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog, in bpf_program__attach_usdt() argument
12432 struct bpf_object *obj = prog->obj; in bpf_program__attach_usdt()
12440 if (bpf_program__fd(prog) < 0) { in bpf_program__attach_usdt()
12442 prog->name); in bpf_program__attach_usdt()
12453 prog->name, binary_path, errstr(err)); in bpf_program__attach_usdt()
12471 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path, in bpf_program__attach_usdt()
12479 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_usdt() argument
12485 sec_name = bpf_program__section_name(prog); in attach_usdt()
12498 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path, in attach_usdt()
12558 struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *prog, in bpf_program__attach_tracepoint_opts() argument
12575 prog->name, tp_category, tp_name, in bpf_program__attach_tracepoint_opts()
12579 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); in bpf_program__attach_tracepoint_opts()
12584 prog->name, tp_category, tp_name, in bpf_program__attach_tracepoint_opts()
12591 struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog, in bpf_program__attach_tracepoint() argument
12595 return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL); in bpf_program__attach_tracepoint()
12598 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_tp() argument
12605 if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0) in attach_tp()
12608 sec_name = strdup(prog->sec_name); in attach_tp()
12613 if (str_has_pfx(prog->sec_name, "tp/")) in attach_tp()
12625 *link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name); in attach_tp()
12631 bpf_program__attach_raw_tracepoint_opts(const struct bpf_program *prog, in bpf_program__attach_raw_tracepoint_opts() argument
12642 prog_fd = bpf_program__fd(prog); in bpf_program__attach_raw_tracepoint_opts()
12644 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program__attach_raw_tracepoint_opts()
12660 prog->name, tp_name, errstr(pfd)); in bpf_program__attach_raw_tracepoint_opts()
12667 struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog, in bpf_program__attach_raw_tracepoint() argument
12670 return bpf_program__attach_raw_tracepoint_opts(prog, tp_name, NULL); in bpf_program__attach_raw_tracepoint()
12673 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_raw_tp() argument
12689 if (!str_has_pfx(prog->sec_name, prefixes[i])) in attach_raw_tp()
12694 if (prog->sec_name[pfx_len] == '\0') in attach_raw_tp()
12697 if (prog->sec_name[pfx_len] != '/') in attach_raw_tp()
12700 tp_name = prog->sec_name + pfx_len + 1; in attach_raw_tp()
12706 prog->name, prog->sec_name); in attach_raw_tp()
12710 *link = bpf_program__attach_raw_tracepoint(prog, tp_name); in attach_raw_tp()
12715 static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog, in bpf_program__attach_btf_id() argument
12725 prog_fd = bpf_program__fd(prog); in bpf_program__attach_btf_id()
12727 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program__attach_btf_id()
12738 pfd = bpf_link_create(prog_fd, 0, bpf_program__expected_attach_type(prog), &link_opts); in bpf_program__attach_btf_id()
12743 prog->name, errstr(pfd)); in bpf_program__attach_btf_id()
12750 struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog) in bpf_program__attach_trace() argument
12752 return bpf_program__attach_btf_id(prog, NULL); in bpf_program__attach_trace()
12755 struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog, in bpf_program__attach_trace_opts() argument
12758 return bpf_program__attach_btf_id(prog, opts); in bpf_program__attach_trace_opts()
12761 struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog) in bpf_program__attach_lsm() argument
12763 return bpf_program__attach_btf_id(prog, NULL); in bpf_program__attach_lsm()
12766 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_trace() argument
12768 *link = bpf_program__attach_trace(prog); in attach_trace()
12772 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_lsm() argument
12774 *link = bpf_program__attach_lsm(prog); in attach_lsm()
12779 bpf_program_attach_fd(const struct bpf_program *prog, in bpf_program_attach_fd() argument
12787 prog_fd = bpf_program__fd(prog); in bpf_program_attach_fd()
12789 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program_attach_fd()
12798 attach_type = bpf_program__expected_attach_type(prog); in bpf_program_attach_fd()
12804 prog->name, target_name, in bpf_program_attach_fd()
12813 bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd) in bpf_program__attach_cgroup() argument
12815 return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", NULL); in bpf_program__attach_cgroup()
12819 bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd) in bpf_program__attach_netns() argument
12821 return bpf_program_attach_fd(prog, netns_fd, "netns", NULL); in bpf_program__attach_netns()
12825 bpf_program__attach_sockmap(const struct bpf_program *prog, int map_fd) in bpf_program__attach_sockmap() argument
12827 return bpf_program_attach_fd(prog, map_fd, "sockmap", NULL); in bpf_program__attach_sockmap()
12830 struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex) in bpf_program__attach_xdp() argument
12833 return bpf_program_attach_fd(prog, ifindex, "xdp", NULL); in bpf_program__attach_xdp()
12837 bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex, in bpf_program__attach_tcx() argument
12853 prog->name); in bpf_program__attach_tcx()
12858 prog->name); in bpf_program__attach_tcx()
12868 return bpf_program_attach_fd(prog, ifindex, "tcx", &link_create_opts); in bpf_program__attach_tcx()
12872 bpf_program__attach_netkit(const struct bpf_program *prog, int ifindex, in bpf_program__attach_netkit() argument
12888 prog->name); in bpf_program__attach_netkit()
12893 prog->name); in bpf_program__attach_netkit()
12902 return bpf_program_attach_fd(prog, ifindex, "netkit", &link_create_opts); in bpf_program__attach_netkit()
12905 struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog, in bpf_program__attach_freplace() argument
12913 prog->name); in bpf_program__attach_freplace()
12917 if (prog->type != BPF_PROG_TYPE_EXT) { in bpf_program__attach_freplace()
12919 prog->name); in bpf_program__attach_freplace()
12926 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd, prog->obj->token_fd); in bpf_program__attach_freplace()
12932 return bpf_program_attach_fd(prog, target_fd, "freplace", in bpf_program__attach_freplace()
12938 return bpf_program__attach_trace(prog); in bpf_program__attach_freplace()
12943 bpf_program__attach_iter(const struct bpf_program *prog, in bpf_program__attach_iter() argument
12957 prog_fd = bpf_program__fd(prog); in bpf_program__attach_iter()
12959 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program__attach_iter()
12974 prog->name, errstr(link_fd)); in bpf_program__attach_iter()
12981 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link) in attach_iter() argument
12983 *link = bpf_program__attach_iter(prog, NULL); in attach_iter()
12987 struct bpf_link *bpf_program__attach_netfilter(const struct bpf_program *prog, in bpf_program__attach_netfilter() argument
12997 prog_fd = bpf_program__fd(prog); in bpf_program__attach_netfilter()
12999 pr_warn("prog '%s': can't attach before loaded\n", prog->name); in bpf_program__attach_netfilter()
13019 prog->name, errstr(link_fd)); in bpf_program__attach_netfilter()
13027 struct bpf_link *bpf_program__attach(const struct bpf_program *prog) in bpf_program__attach() argument
13032 if (!prog->sec_def || !prog->sec_def->prog_attach_fn) in bpf_program__attach()
13035 if (bpf_program__fd(prog) < 0) { in bpf_program__attach()
13037 prog->name); in bpf_program__attach()
13041 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link); in bpf_program__attach()
13725 int bpf_program__set_attach_target(struct bpf_program *prog, in bpf_program__set_attach_target() argument
13731 if (!prog || attach_prog_fd < 0) in bpf_program__set_attach_target()
13734 if (prog->obj->state >= OBJ_LOADED) in bpf_program__set_attach_target()
13741 prog->attach_prog_fd = attach_prog_fd; in bpf_program__set_attach_target()
13747 attach_prog_fd, prog->obj->token_fd); in bpf_program__set_attach_target()
13755 err = bpf_object__load_vmlinux_btf(prog->obj, true); in bpf_program__set_attach_target()
13758 err = find_kernel_btf_id(prog->obj, attach_func_name, in bpf_program__set_attach_target()
13759 prog->expected_attach_type, in bpf_program__set_attach_target()
13765 prog->attach_btf_id = btf_id; in bpf_program__set_attach_target()
13766 prog->attach_btf_obj_fd = btf_obj_fd; in bpf_program__set_attach_target()
13767 prog->attach_prog_fd = attach_prog_fd; in bpf_program__set_attach_target()
13907 struct bpf_program **prog = prog_skel->prog; in populate_skeleton_progs() local
13910 *prog = bpf_object__find_program_by_name(obj, name); in populate_skeleton_progs()
13911 if (!*prog) { in populate_skeleton_progs()
14048 struct bpf_program *prog = *prog_skel->prog; in bpf_object__attach_skeleton() local
14051 if (!prog->autoload || !prog->autoattach) in bpf_object__attach_skeleton()
14055 if (!prog->sec_def || !prog->sec_def->prog_attach_fn) in bpf_object__attach_skeleton()
14062 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link); in bpf_object__attach_skeleton()
14065 bpf_program__name(prog), errstr(err)); in bpf_object__attach_skeleton()