Lines Matching refs:subprog
425 static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog) in subprog_is_global() argument
429 return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL; in subprog_is_global()
432 static const char *subprog_name(const struct bpf_verifier_env *env, int subprog) in subprog_name() argument
439 info = &env->prog->aux->func_info[subprog]; in subprog_name()
443 static void mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog) in mark_subprog_exc_cb() argument
445 struct bpf_subprog_info *info = subprog_info(env, subprog); in mark_subprog_exc_cb()
452 static bool subprog_is_exc_cb(struct bpf_verifier_env *env, int subprog) in subprog_is_exc_cb() argument
454 return subprog_info(env, subprog)->is_exception_cb; in subprog_is_exc_cb()
2787 int subprog, bool is_sleepable) in push_async_cb() argument
2826 subprog /* subprog number within this prog */); in push_async_cb()
3326 struct bpf_subprog_info *subprog = env->subprog_info; in add_subprog_and_kfunc() local
3378 subprog[env->subprog_cnt].start = insn_cnt; in add_subprog_and_kfunc()
3382 verbose(env, "func#%d @%d\n", i, subprog[i].start); in add_subprog_and_kfunc()
3399 struct bpf_subprog_info *subprog = env->subprog_info; in check_subprogs() local
3404 subprog_start = subprog[cur_subprog].start; in check_subprogs()
3405 subprog_end = subprog[cur_subprog + 1].start; in check_subprogs()
3412 subprog[cur_subprog].has_tail_call = true; in check_subprogs()
3413 subprog[cur_subprog].tail_call_reachable = true; in check_subprogs()
3417 subprog[cur_subprog].has_ld_abs = true; in check_subprogs()
3442 subprog_end = subprog[cur_subprog + 1].start; in check_subprogs()
4266 int subprog_insn_idx, subprog; in backtrack_insn() local
4269 subprog = find_subprog(env, subprog_insn_idx); in backtrack_insn()
4270 if (subprog < 0) in backtrack_insn()
4273 if (subprog_is_global(env, subprog)) { in backtrack_insn()
4971 struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno]; in check_fastcall_stack_contract() local
4975 if (subprog->fastcall_stack_off <= off || aux[insn_idx].fastcall_pattern) in check_fastcall_stack_contract()
4982 subprog->fastcall_stack_off = S16_MIN; in check_fastcall_stack_contract()
4986 for (i = subprog->start; i < (subprog + 1)->start; ++i) { in check_fastcall_stack_contract()
6466 struct bpf_subprog_info *subprog = env->subprog_info; in check_max_stack_depth_subprog() local
6474 i = subprog[idx].start; in check_max_stack_depth_subprog()
6476 subprog[idx].priv_stack_mode = NO_PRIV_STACK; in check_max_stack_depth_subprog()
6497 if (idx && subprog[idx].has_tail_call && depth >= 256) { in check_max_stack_depth_subprog()
6504 subprog_depth = round_up_stack_depth(env, subprog[idx].stack_depth); in check_max_stack_depth_subprog()
6510 if (subprog[idx].priv_stack_mode == PRIV_STACK_UNKNOWN && in check_max_stack_depth_subprog()
6512 subprog[idx].priv_stack_mode = PRIV_STACK_ADAPTIVE; in check_max_stack_depth_subprog()
6515 if (subprog[idx].priv_stack_mode == PRIV_STACK_ADAPTIVE) { in check_max_stack_depth_subprog()
6530 subprog_end = subprog[idx + 1].start; in check_max_stack_depth_subprog()
6539 if (subprog[idx].is_cb) in check_max_stack_depth_subprog()
6542 if (subprog[ret_prog[c]].is_cb) { in check_max_stack_depth_subprog()
6569 if (subprog[sidx].is_async_cb) { in check_max_stack_depth_subprog()
6570 if (subprog[sidx].has_tail_call) { in check_max_stack_depth_subprog()
6577 if (subprog[sidx].is_exception_cb) { in check_max_stack_depth_subprog()
6585 subprog[idx].priv_stack_mode = NO_PRIV_STACK; in check_max_stack_depth_subprog()
6587 if (subprog[idx].has_tail_call) in check_max_stack_depth_subprog()
6605 if (subprog[ret_prog[j]].is_exception_cb) { in check_max_stack_depth_subprog()
6609 subprog[ret_prog[j]].tail_call_reachable = true; in check_max_stack_depth_subprog()
6611 if (subprog[0].tail_call_reachable) in check_max_stack_depth_subprog()
6619 if (subprog[idx].priv_stack_mode != PRIV_STACK_ADAPTIVE) in check_max_stack_depth_subprog()
6620 depth -= round_up_stack_depth(env, subprog[idx].stack_depth); in check_max_stack_depth_subprog()
6675 int start = idx + insn->imm + 1, subprog; in get_callee_stack_depth() local
6677 subprog = find_subprog(env, start); in get_callee_stack_depth()
6678 if (subprog < 0) { in get_callee_stack_depth()
6683 return env->subprog_info[subprog].stack_depth; in get_callee_stack_depth()
10274 static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int callsite, in setup_func_entry() argument
10307 subprog /* subprog number within this prog */); in setup_func_entry()
10323 static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, in btf_check_func_arg_match() argument
10327 struct bpf_subprog_info *sub = subprog_info(env, subprog); in btf_check_func_arg_match()
10332 ret = btf_prepare_func_args(env, subprog); in btf_check_func_arg_match()
10419 static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, in btf_check_subprog_call() argument
10430 btf_id = prog->aux->func_info[subprog].type_id; in btf_check_subprog_call()
10434 if (prog->aux->func_info_aux[subprog].unreliable) in btf_check_subprog_call()
10437 err = btf_check_func_arg_match(env, subprog, btf, regs); in btf_check_subprog_call()
10443 prog->aux->func_info_aux[subprog].unreliable = true; in btf_check_subprog_call()
10448 int insn_idx, int subprog, in push_callback_call() argument
10456 err = btf_check_subprog_call(env, subprog, caller->regs); in push_callback_call()
10464 env->subprog_info[subprog].is_cb = true; in push_callback_call()
10481 env->subprog_info[subprog].is_async_cb = true; in push_callback_call()
10482 async_cb = push_async_cb(env, env->subprog_info[subprog].start, in push_callback_call()
10483 insn_idx, subprog, in push_callback_call()
10501 callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false); in push_callback_call()
10505 err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb, in push_callback_call()
10521 int err, subprog, target_insn; in check_func_call() local
10524 subprog = find_subprog(env, target_insn); in check_func_call()
10525 if (subprog < 0) { in check_func_call()
10531 err = btf_check_subprog_call(env, subprog, caller->regs); in check_func_call()
10534 if (subprog_is_global(env, subprog)) { in check_func_call()
10535 const char *sub_name = subprog_name(env, subprog); in check_func_call()
10543 if (env->subprog_info[subprog].might_sleep && in check_func_call()
10554 subprog, sub_name); in check_func_call()
10559 subprog, sub_name); in check_func_call()
10560 if (env->subprog_info[subprog].changes_pkt_data) in check_func_call()
10563 subprog_aux(env, subprog)->called = true; in check_func_call()
10577 err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state); in check_func_call()
10584 *insn_idx = env->subprog_info[subprog].start - 1; in check_func_call()
17005 struct bpf_subprog_info *subprog; in mark_subprog_changes_pkt_data() local
17007 subprog = find_containing_subprog(env, off); in mark_subprog_changes_pkt_data()
17008 subprog->changes_pkt_data = true; in mark_subprog_changes_pkt_data()
17013 struct bpf_subprog_info *subprog; in mark_subprog_might_sleep() local
17015 subprog = find_containing_subprog(env, off); in mark_subprog_might_sleep()
17016 subprog->might_sleep = true; in mark_subprog_might_sleep()
17332 struct bpf_subprog_info *subprog, in mark_fastcall_pattern_for_call() argument
17414 subprog->keep_fastcall_stack = 1; in mark_fastcall_pattern_for_call()
17415 subprog->fastcall_stack_off = min(subprog->fastcall_stack_off, off); in mark_fastcall_pattern_for_call()
17420 struct bpf_subprog_info *subprog = env->subprog_info; in mark_fastcall_patterns() local
17425 for (s = 0; s < env->subprog_cnt; ++s, ++subprog) { in mark_fastcall_patterns()
17428 for (i = subprog->start; i < (subprog + 1)->start; ++i) { in mark_fastcall_patterns()
17436 for (i = subprog->start; i < (subprog + 1)->start; ++i) { in mark_fastcall_patterns()
17440 mark_fastcall_pattern_for_call(env, subprog, i, lowest_off); in mark_fastcall_patterns()
21025 int i, j, subprog_start, subprog_end = 0, len, subprog; in jit_subprogs() local
21042 subprog = find_subprog(env, i + insn->imm + 1); in jit_subprogs()
21043 if (subprog < 0) { in jit_subprogs()
21051 insn->off = subprog; in jit_subprogs()
21173 subprog = insn->off; in jit_subprogs()
21174 insn[0].imm = (u32)(long)func[subprog]->bpf_func; in jit_subprogs()
21175 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32; in jit_subprogs()
21180 subprog = insn->off; in jit_subprogs()
21181 insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func); in jit_subprogs()
21237 subprog = find_subprog(env, i + insn->off + 1); in jit_subprogs()
21238 insn->imm = subprog; in jit_subprogs()
22600 struct bpf_subprog_info *subprog = env->subprog_info; in remove_fastcall_spills_fills() local
22618 if ((subprog + 1)->start == i + 1) { in remove_fastcall_spills_fills()
22619 if (modified && !subprog->keep_fastcall_stack) in remove_fastcall_spills_fills()
22620 subprog->stack_depth = -subprog->fastcall_stack_off; in remove_fastcall_spills_fills()
22621 subprog++; in remove_fastcall_spills_fills()
22657 static int do_check_common(struct bpf_verifier_env *env, int subprog) in do_check_common() argument
22660 struct bpf_subprog_info *sub = subprog_info(env, subprog); in do_check_common()
22684 subprog); in do_check_common()
22685 state->first_insn_idx = env->subprog_info[subprog].start; in do_check_common()
22689 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { in do_check_common()
22690 const char *sub_name = subprog_name(env, subprog); in do_check_common()
22694 verbose(env, "Validating %s() func#%d...\n", sub_name, subprog); in do_check_common()
22695 ret = btf_prepare_func_args(env, subprog); in do_check_common()
22699 if (subprog_is_exc_cb(env, subprog)) { in do_check_common()
22770 if (!subprog && env->prog->type == BPF_PROG_TYPE_STRUCT_OPS) { in do_check_common()
23059 int ret = 0, subprog = -1, i; in bpf_check_attach_target() local
23100 subprog = i; in bpf_check_attach_target()
23103 if (subprog == -1) { in bpf_check_attach_target()
23107 if (aux->func && aux->func[subprog]->aux->exception_cb) { in bpf_check_attach_target()
23113 conservative = aux->func_info_aux[subprog].unreliable; in bpf_check_attach_target()
23126 ? aux->func[subprog]->aux->changes_pkt_data in bpf_check_attach_target()
23135 ? aux->func[subprog]->aux->might_sleep in bpf_check_attach_target()
23293 if (subprog == 0) in bpf_check_attach_target()
23296 addr = (long) tgt_prog->aux->func[subprog]->bpf_func; in bpf_check_attach_target()