107 bool bpf_prog_has_trampoline(const struct bpf_prog *prog)  in bpf_prog_has_trampoline()  argument
109 	enum bpf_attach_type eatype = prog->expected_attach_type;  in bpf_prog_has_trampoline()
110 	enum bpf_prog_type ptype = prog->type;  in bpf_prog_has_trampoline()
251 			*ip_arg |= link->link.prog->call_get_func_ip;  in bpf_trampoline_get_progs()
509 static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)  in bpf_attach_type_to_tramp()  argument
511 	switch (prog->expected_attach_type) {  in bpf_attach_type_to_tramp()
519 		if (!prog->aux->attach_func_proto->type)  in bpf_attach_type_to_tramp()
558 	kind = bpf_attach_type_to_tramp(link->link.prog);  in __bpf_trampoline_link_prog()
575 		tr->extension_prog = link->link.prog;  in __bpf_trampoline_link_prog()
577 					  link->link.prog->bpf_func);  in __bpf_trampoline_link_prog()
585 		if (link_exiting->link.prog != link->link.prog)  in __bpf_trampoline_link_prog()
620 	kind = bpf_attach_type_to_tramp(link->link.prog);  in __bpf_trampoline_unlink_prog()
675 static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog,  in cgroup_shim_alloc()  argument
696 	p->aux->attach_func_proto = prog->aux->attach_func_proto;  in cgroup_shim_alloc()
697 	p->aux->attach_btf_id = prog->aux->attach_btf_id;  in cgroup_shim_alloc()
698 	p->aux->attach_btf = prog->aux->attach_btf;  in cgroup_shim_alloc()
718 			struct bpf_prog *p = link->link.prog;  in cgroup_shim_find()
728 int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,  in bpf_trampoline_link_cgroup_shim()  argument
738 	err = bpf_check_attach_target(NULL, prog, NULL,  in bpf_trampoline_link_cgroup_shim()
739 				      prog->aux->attach_btf_id,  in bpf_trampoline_link_cgroup_shim()
744 	key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,  in bpf_trampoline_link_cgroup_shim()
745 					 prog->aux->attach_btf_id);  in bpf_trampoline_link_cgroup_shim()
747 	bpf_lsm_find_cgroup_shim(prog, &bpf_func);  in bpf_trampoline_link_cgroup_shim()
766 	shim_link = cgroup_shim_alloc(prog, bpf_func, cgroup_atype);  in bpf_trampoline_link_cgroup_shim()
794 void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)  in bpf_trampoline_unlink_cgroup_shim()  argument
801 	key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,  in bpf_trampoline_unlink_cgroup_shim()
802 					 prog->aux->attach_btf_id);  in bpf_trampoline_unlink_cgroup_shim()
804 	bpf_lsm_find_cgroup_shim(prog, &bpf_func);  in bpf_trampoline_unlink_cgroup_shim()
897 static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)  in __bpf_prog_enter_recur()  argument
905 	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {  in __bpf_prog_enter_recur()
906 		bpf_prog_inc_misses_counter(prog);  in __bpf_prog_enter_recur()
907 		if (prog->aux->recursion_detected)  in __bpf_prog_enter_recur()
908 			prog->aux->recursion_detected(prog);  in __bpf_prog_enter_recur()
914 static void notrace update_prog_stats(struct bpf_prog *prog,  in update_prog_stats()  argument
929 		stats = this_cpu_ptr(prog->stats);  in update_prog_stats()
937 static void notrace __bpf_prog_exit_recur(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit_recur()  argument
943 	update_prog_stats(prog, start);  in __bpf_prog_exit_recur()
944 	this_cpu_dec(*(prog->active));  in __bpf_prog_exit_recur()
949 static u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,  in __bpf_prog_enter_lsm_cgroup()  argument
964 static void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit_lsm_cgroup()  argument
974 u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,  in __bpf_prog_enter_sleepable_recur()  argument
983 	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {  in __bpf_prog_enter_sleepable_recur()
984 		bpf_prog_inc_misses_counter(prog);  in __bpf_prog_enter_sleepable_recur()
985 		if (prog->aux->recursion_detected)  in __bpf_prog_enter_sleepable_recur()
986 			prog->aux->recursion_detected(prog);  in __bpf_prog_enter_sleepable_recur()
992 void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit_sleepable_recur()  argument
997 	update_prog_stats(prog, start);  in __bpf_prog_exit_sleepable_recur()
998 	this_cpu_dec(*(prog->active));  in __bpf_prog_exit_sleepable_recur()
1003 static u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog,  in __bpf_prog_enter_sleepable()  argument
1015 static void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit_sleepable()  argument
1020 	update_prog_stats(prog, start);  in __bpf_prog_exit_sleepable()
1025 static u64 notrace __bpf_prog_enter(struct bpf_prog *prog,  in __bpf_prog_enter()  argument
1037 static void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit()  argument
1043 	update_prog_stats(prog, start);  in __bpf_prog_exit()
1058 bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog)  in bpf_trampoline_enter()  argument
1060 	bool sleepable = prog->sleepable;  in bpf_trampoline_enter()
1062 	if (bpf_prog_check_recur(prog))  in bpf_trampoline_enter()
1066 	if (resolve_prog_type(prog) == BPF_PROG_TYPE_LSM &&  in bpf_trampoline_enter()
1067 	    prog->expected_attach_type == BPF_LSM_CGROUP)  in bpf_trampoline_enter()
1073 bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog)  in bpf_trampoline_exit()  argument
1075 	bool sleepable = prog->sleepable;  in bpf_trampoline_exit()
1077 	if (bpf_prog_check_recur(prog))  in bpf_trampoline_exit()
1081 	if (resolve_prog_type(prog) == BPF_PROG_TYPE_LSM &&  in bpf_trampoline_exit()
1082 	    prog->expected_attach_type == BPF_LSM_CGROUP)  in bpf_trampoline_exit()