Lines Matching refs:env
199 static int acquire_reference(struct bpf_verifier_env *env, int insn_idx);
201 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id);
202 static void invalidate_non_owning_refs(struct bpf_verifier_env *env);
203 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env);
204 static int ref_set_non_owning(struct bpf_verifier_env *env,
206 static void specialize_kfunc(struct bpf_verifier_env *env,
360 struct bpf_verifier_env *env = private_data; in verbose() local
363 if (!bpf_verifier_log_needed(&env->log)) in verbose()
367 bpf_verifier_vlog(&env->log, fmt, args); in verbose()
371 static void verbose_invalid_scalar(struct bpf_verifier_env *env, in verbose_invalid_scalar() argument
378 verbose(env, "%s the register %s has", ctx, reg_name); in verbose_invalid_scalar()
380 verbose(env, " smin=%lld", reg->smin_value); in verbose_invalid_scalar()
384 verbose(env, " smax=%lld", reg->smax_value); in verbose_invalid_scalar()
388 verbose(env, " unknown scalar value"); in verbose_invalid_scalar()
389 verbose(env, " should have been in [%d, %d]\n", range.minval, range.maxval); in verbose_invalid_scalar()
425 static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog) in subprog_is_global() argument
427 struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux; in subprog_is_global()
432 static const char *subprog_name(const struct bpf_verifier_env *env, int subprog) in subprog_name() argument
436 if (!env->prog->aux->func_info) in subprog_name()
439 info = &env->prog->aux->func_info[subprog]; in subprog_name()
440 return btf_type_name(env->prog->aux->btf, info->type_id); in subprog_name()
443 static void mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog) in mark_subprog_exc_cb() argument
445 struct bpf_subprog_info *info = subprog_info(env, subprog); in mark_subprog_exc_cb()
452 static bool subprog_is_exc_cb(struct bpf_verifier_env *env, int subprog) in subprog_is_exc_cb() argument
454 return subprog_info(env, subprog)->is_exception_cb; in subprog_is_exc_cb()
547 static bool is_may_goto_insn_at(struct bpf_verifier_env *env, int insn_idx) in is_may_goto_insn_at() argument
549 return is_may_goto_insn(&env->prog->insnsi[insn_idx]); in is_may_goto_insn_at()
594 static struct bpf_func_state *func(struct bpf_verifier_env *env, in func() argument
597 struct bpf_verifier_state *cur = env->cur_state; in func()
616 static int stack_slot_obj_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in stack_slot_obj_get_spi() argument
622 verbose(env, "%s has to be at a constant offset\n", obj_kind); in stack_slot_obj_get_spi()
628 verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off); in stack_slot_obj_get_spi()
634 verbose(env, "cannot pass in %s at an offset=%d\n", obj_kind, off); in stack_slot_obj_get_spi()
638 if (!is_spi_bounds_valid(func(env, reg), spi, nr_slots)) in stack_slot_obj_get_spi()
643 static int dynptr_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in dynptr_get_spi() argument
645 return stack_slot_obj_get_spi(env, reg, "dynptr", BPF_DYNPTR_NR_SLOTS); in dynptr_get_spi()
648 static int iter_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots) in iter_get_spi() argument
650 return stack_slot_obj_get_spi(env, reg, "iter", nr_slots); in iter_get_spi()
653 static int irq_flag_get_spi(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in irq_flag_get_spi() argument
655 return stack_slot_obj_get_spi(env, reg, "irq_flag", 1); in irq_flag_get_spi()
699 static void __mark_reg_not_init(const struct bpf_verifier_env *env,
702 static void mark_dynptr_stack_regs(struct bpf_verifier_env *env, in mark_dynptr_stack_regs() argument
707 int id = ++env->id_gen; in mark_dynptr_stack_regs()
713 static void mark_dynptr_cb_reg(struct bpf_verifier_env *env, in mark_dynptr_cb_reg() argument
717 __mark_dynptr_reg(reg, type, true, ++env->id_gen); in mark_dynptr_cb_reg()
720 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env,
723 static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in mark_stack_slots_dynptr() argument
726 struct bpf_func_state *state = func(env, reg); in mark_stack_slots_dynptr()
730 spi = dynptr_get_spi(env, reg); in mark_stack_slots_dynptr()
743 err = destroy_if_dynptr_stack_slot(env, state, spi); in mark_stack_slots_dynptr()
746 err = destroy_if_dynptr_stack_slot(env, state, spi - 1); in mark_stack_slots_dynptr()
759 mark_dynptr_stack_regs(env, &state->stack[spi].spilled_ptr, in mark_stack_slots_dynptr()
769 id = acquire_reference(env, insn_idx); in mark_stack_slots_dynptr()
784 static void invalidate_dynptr(struct bpf_verifier_env *env, struct bpf_func_state *state, int spi) in invalidate_dynptr() argument
793 __mark_reg_not_init(env, &state->stack[spi].spilled_ptr); in invalidate_dynptr()
794 __mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr); in invalidate_dynptr()
821 static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in unmark_stack_slots_dynptr() argument
823 struct bpf_func_state *state = func(env, reg); in unmark_stack_slots_dynptr()
826 spi = dynptr_get_spi(env, reg); in unmark_stack_slots_dynptr()
831 invalidate_dynptr(env, state, spi); in unmark_stack_slots_dynptr()
845 WARN_ON_ONCE(release_reference(env, ref_obj_id)); in unmark_stack_slots_dynptr()
857 verbose(env, "verifier internal error: misconfigured ref_obj_id\n"); in unmark_stack_slots_dynptr()
861 invalidate_dynptr(env, state, i); in unmark_stack_slots_dynptr()
867 static void __mark_reg_unknown(const struct bpf_verifier_env *env,
870 static void mark_reg_invalid(const struct bpf_verifier_env *env, struct bpf_reg_state *reg) in mark_reg_invalid() argument
872 if (!env->allow_ptr_leaks) in mark_reg_invalid()
873 __mark_reg_not_init(env, reg); in mark_reg_invalid()
875 __mark_reg_unknown(env, reg); in mark_reg_invalid()
878 static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, in destroy_if_dynptr_stack_slot() argument
898 verbose(env, "cannot overwrite referenced dynptr\n"); in destroy_if_dynptr_stack_slot()
902 mark_stack_slot_scratched(env, spi); in destroy_if_dynptr_stack_slot()
903 mark_stack_slot_scratched(env, spi - 1); in destroy_if_dynptr_stack_slot()
913 bpf_for_each_reg_in_vstate(env->cur_state, fstate, dreg, ({ in destroy_if_dynptr_stack_slot()
918 mark_reg_invalid(env, dreg); in destroy_if_dynptr_stack_slot()
924 __mark_reg_not_init(env, &state->stack[spi].spilled_ptr); in destroy_if_dynptr_stack_slot()
925 __mark_reg_not_init(env, &state->stack[spi - 1].spilled_ptr); in destroy_if_dynptr_stack_slot()
934 static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in is_dynptr_reg_valid_uninit() argument
941 spi = dynptr_get_spi(env, reg); in is_dynptr_reg_valid_uninit()
962 static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in is_dynptr_reg_valid_init() argument
964 struct bpf_func_state *state = func(env, reg); in is_dynptr_reg_valid_init()
976 spi = dynptr_get_spi(env, reg); in is_dynptr_reg_valid_init()
991 static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in is_dynptr_type_expected() argument
994 struct bpf_func_state *state = func(env, reg); in is_dynptr_type_expected()
1006 spi = dynptr_get_spi(env, reg); in is_dynptr_type_expected()
1015 static bool in_rcu_cs(struct bpf_verifier_env *env);
1019 static int mark_stack_slots_iter(struct bpf_verifier_env *env, in mark_stack_slots_iter() argument
1024 struct bpf_func_state *state = func(env, reg); in mark_stack_slots_iter()
1027 spi = iter_get_spi(env, reg, nr_slots); in mark_stack_slots_iter()
1031 id = acquire_reference(env, insn_idx); in mark_stack_slots_iter()
1042 if (in_rcu_cs(env)) in mark_stack_slots_iter()
1057 mark_stack_slot_scratched(env, spi - i); in mark_stack_slots_iter()
1063 static int unmark_stack_slots_iter(struct bpf_verifier_env *env, in unmark_stack_slots_iter() argument
1066 struct bpf_func_state *state = func(env, reg); in unmark_stack_slots_iter()
1069 spi = iter_get_spi(env, reg, nr_slots); in unmark_stack_slots_iter()
1078 WARN_ON_ONCE(release_reference(env, st->ref_obj_id)); in unmark_stack_slots_iter()
1080 __mark_reg_not_init(env, st); in unmark_stack_slots_iter()
1088 mark_stack_slot_scratched(env, spi - i); in unmark_stack_slots_iter()
1094 static bool is_iter_reg_valid_uninit(struct bpf_verifier_env *env, in is_iter_reg_valid_uninit() argument
1097 struct bpf_func_state *state = func(env, reg); in is_iter_reg_valid_uninit()
1104 spi = iter_get_spi(env, reg, nr_slots); in is_iter_reg_valid_uninit()
1121 static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in is_iter_reg_valid_init() argument
1124 struct bpf_func_state *state = func(env, reg); in is_iter_reg_valid_init()
1127 spi = iter_get_spi(env, reg, nr_slots); in is_iter_reg_valid_init()
1153 static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx);
1156 static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env, in mark_stack_slot_irq_flag() argument
1161 struct bpf_func_state *state = func(env, reg); in mark_stack_slot_irq_flag()
1166 spi = irq_flag_get_spi(env, reg); in mark_stack_slot_irq_flag()
1170 id = acquire_irq_state(env, insn_idx); in mark_stack_slot_irq_flag()
1186 mark_stack_slot_scratched(env, spi); in mark_stack_slot_irq_flag()
1190 static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in unmark_stack_slot_irq_flag() argument
1193 struct bpf_func_state *state = func(env, reg); in unmark_stack_slot_irq_flag()
1198 spi = irq_flag_get_spi(env, reg); in unmark_stack_slot_irq_flag()
1209 verbose(env, "irq flag acquired by %s kfuncs cannot be restored with %s kfuncs\n", in unmark_stack_slot_irq_flag()
1214 err = release_irq_state(env->cur_state, st->ref_obj_id); in unmark_stack_slot_irq_flag()
1219 for (int i = 0; i < env->cur_state->acquired_refs; i++) { in unmark_stack_slot_irq_flag()
1220 if (env->cur_state->refs[i].id == env->cur_state->active_irq_id) { in unmark_stack_slot_irq_flag()
1221 insn_idx = env->cur_state->refs[i].insn_idx; in unmark_stack_slot_irq_flag()
1226 verbose(env, "cannot restore irq state out of order, expected id=%d acquired at insn_idx=%d\n", in unmark_stack_slot_irq_flag()
1227 env->cur_state->active_irq_id, insn_idx); in unmark_stack_slot_irq_flag()
1231 __mark_reg_not_init(env, st); in unmark_stack_slot_irq_flag()
1239 mark_stack_slot_scratched(env, spi); in unmark_stack_slot_irq_flag()
1243 static bool is_irq_flag_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in is_irq_flag_reg_valid_uninit() argument
1245 struct bpf_func_state *state = func(env, reg); in is_irq_flag_reg_valid_uninit()
1253 spi = irq_flag_get_spi(env, reg); in is_irq_flag_reg_valid_uninit()
1267 static int is_irq_flag_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in is_irq_flag_reg_valid_init() argument
1269 struct bpf_func_state *state = func(env, reg); in is_irq_flag_reg_valid_init()
1274 spi = irq_flag_get_spi(env, reg); in is_irq_flag_reg_valid_init()
1345 static void mark_stack_slot_misc(struct bpf_verifier_env *env, u8 *stype) in mark_stack_slot_misc() argument
1463 static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size) in grow_stack_state() argument
1481 if (env->subprog_info[state->subprogno].stack_depth < size) in grow_stack_state()
1482 env->subprog_info[state->subprogno].stack_depth = size; in grow_stack_state()
1492 static struct bpf_reference_state *acquire_reference_state(struct bpf_verifier_env *env, int insn_i… in acquire_reference_state() argument
1494 struct bpf_verifier_state *state = env->cur_state; in acquire_reference_state()
1506 static int acquire_reference(struct bpf_verifier_env *env, int insn_idx) in acquire_reference() argument
1510 s = acquire_reference_state(env, insn_idx); in acquire_reference()
1514 s->id = ++env->id_gen; in acquire_reference()
1518 static int acquire_lock_state(struct bpf_verifier_env *env, int insn_idx, enum ref_state_type type, in acquire_lock_state() argument
1521 struct bpf_verifier_state *state = env->cur_state; in acquire_lock_state()
1524 s = acquire_reference_state(env, insn_idx); in acquire_lock_state()
1537 static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx) in acquire_irq_state() argument
1539 struct bpf_verifier_state *state = env->cur_state; in acquire_irq_state()
1542 s = acquire_reference_state(env, insn_idx); in acquire_irq_state()
1546 s->id = ++env->id_gen; in acquire_irq_state()
1645 static void update_peak_states(struct bpf_verifier_env *env) in update_peak_states() argument
1649 cur_states = env->explored_states_size + env->free_list_size; in update_peak_states()
1650 env->peak_states = max(env->peak_states, cur_states); in update_peak_states()
1700 static void maybe_free_verifier_state(struct bpf_verifier_env *env, in maybe_free_verifier_state() argument
1714 env->free_list_size--; in maybe_free_verifier_state()
1774 static u32 state_htab_size(struct bpf_verifier_env *env) in state_htab_size() argument
1776 return env->prog->len; in state_htab_size()
1779 static struct list_head *explored_state(struct bpf_verifier_env *env, int idx) in explored_state() argument
1781 struct bpf_verifier_state *cur = env->cur_state; in explored_state()
1784 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; in explored_state()
1919 static struct bpf_verifier_state *get_loop_entry(struct bpf_verifier_env *env, in get_loop_entry() argument
1928 verbose(env, "verifier bug: infinite loop in get_loop_entry()\n"); in get_loop_entry()
1936 static void update_loop_entry(struct bpf_verifier_env *env, in update_loop_entry() argument
1948 maybe_free_verifier_state(env, state_loop_entry_as_list(cur)); in update_loop_entry()
1955 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st) in update_branch_counts() argument
1969 update_loop_entry(env, st->parent, st->loop_entry); in update_branch_counts()
1982 maybe_free_verifier_state(env, sl); in update_branch_counts()
1988 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, in pop_stack() argument
1991 struct bpf_verifier_state *cur = env->cur_state; in pop_stack()
1992 struct bpf_verifier_stack_elem *elem, *head = env->head; in pop_stack()
1995 if (env->head == NULL) in pop_stack()
2004 bpf_vlog_reset(&env->log, head->log_pos); in pop_stack()
2012 env->head = elem; in pop_stack()
2013 env->stack_size--; in pop_stack()
2017 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, in push_stack() argument
2021 struct bpf_verifier_state *cur = env->cur_state; in push_stack()
2031 elem->next = env->head; in push_stack()
2032 elem->log_pos = env->log.end_pos; in push_stack()
2033 env->head = elem; in push_stack()
2034 env->stack_size++; in push_stack()
2039 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { in push_stack()
2040 verbose(env, "The sequence of %d jumps is too complex.\n", in push_stack()
2041 env->stack_size); in push_stack()
2058 free_verifier_state(env->cur_state, true); in push_stack()
2059 env->cur_state = NULL; in push_stack()
2061 while (!pop_stack(env, NULL, NULL, false)); in push_stack()
2115 static void __mark_reg_const_zero(const struct bpf_verifier_env *env, struct bpf_reg_state *reg) in __mark_reg_const_zero() argument
2122 reg->precise = !env->bpf_capable; in __mark_reg_const_zero()
2125 static void mark_reg_known_zero(struct bpf_verifier_env *env, in mark_reg_known_zero() argument
2129 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); in mark_reg_known_zero()
2132 __mark_reg_not_init(env, regs + regno); in mark_reg_known_zero()
2575 static int reg_bounds_sanity_check(struct bpf_verifier_env *env, in reg_bounds_sanity_check() argument
2612 verbose(env, "REG INVARIANTS VIOLATION (%s): %s u64=[%#llx, %#llx] " in reg_bounds_sanity_check()
2619 if (env->test_reg_invariants) in reg_bounds_sanity_check()
2669 static void __mark_reg_unknown(const struct bpf_verifier_env *env, in __mark_reg_unknown() argument
2673 reg->precise = !env->bpf_capable; in __mark_reg_unknown()
2676 static void mark_reg_unknown(struct bpf_verifier_env *env, in mark_reg_unknown() argument
2680 verbose(env, "mark_reg_unknown(regs, %u)\n", regno); in mark_reg_unknown()
2683 __mark_reg_not_init(env, regs + regno); in mark_reg_unknown()
2686 __mark_reg_unknown(env, regs + regno); in mark_reg_unknown()
2689 static int __mark_reg_s32_range(struct bpf_verifier_env *env, in __mark_reg_s32_range() argument
2705 return reg_bounds_sanity_check(env, reg, "s32_range"); in __mark_reg_s32_range()
2708 static void __mark_reg_not_init(const struct bpf_verifier_env *env, in __mark_reg_not_init() argument
2711 __mark_reg_unknown(env, reg); in __mark_reg_not_init()
2715 static void mark_reg_not_init(struct bpf_verifier_env *env, in mark_reg_not_init() argument
2719 verbose(env, "mark_reg_not_init(regs, %u)\n", regno); in mark_reg_not_init()
2722 __mark_reg_not_init(env, regs + regno); in mark_reg_not_init()
2725 __mark_reg_not_init(env, regs + regno); in mark_reg_not_init()
2728 static void mark_btf_ld_reg(struct bpf_verifier_env *env, in mark_btf_ld_reg() argument
2735 mark_reg_unknown(env, regs, regno); in mark_btf_ld_reg()
2738 mark_reg_known_zero(env, regs, regno); in mark_btf_ld_reg()
2743 regs[regno].id = ++env->id_gen; in mark_btf_ld_reg()
2747 static void init_reg_state(struct bpf_verifier_env *env, in init_reg_state() argument
2754 mark_reg_not_init(env, regs, i); in init_reg_state()
2762 mark_reg_known_zero(env, regs, BPF_REG_FP); in init_reg_state()
2772 static void init_func_state(struct bpf_verifier_env *env, in init_func_state() argument
2780 init_reg_state(env, state); in init_func_state()
2781 mark_verifier_state_scratched(env); in init_func_state()
2785 static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, in push_async_cb() argument
2798 elem->next = env->head; in push_async_cb()
2799 elem->log_pos = env->log.end_pos; in push_async_cb()
2800 env->head = elem; in push_async_cb()
2801 env->stack_size++; in push_async_cb()
2802 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { in push_async_cb()
2803 verbose(env, in push_async_cb()
2805 env->stack_size); in push_async_cb()
2818 elem->st.insn_hist_start = env->cur_state->insn_hist_end; in push_async_cb()
2823 init_func_state(env, frame, in push_async_cb()
2830 free_verifier_state(env->cur_state, true); in push_async_cb()
2831 env->cur_state = NULL; in push_async_cb()
2833 while (!pop_stack(env, NULL, NULL, false)); in push_async_cb()
2851 static struct bpf_subprog_info *find_containing_subprog(struct bpf_verifier_env *env, int off) in find_containing_subprog() argument
2853 struct bpf_subprog_info *vals = env->subprog_info; in find_containing_subprog()
2856 if (off >= env->prog->len || off < 0 || env->subprog_cnt == 0) in find_containing_subprog()
2860 r = env->subprog_cnt - 1; in find_containing_subprog()
2872 static int find_subprog(struct bpf_verifier_env *env, int off) in find_subprog() argument
2876 p = find_containing_subprog(env, off); in find_subprog()
2879 return p - env->subprog_info; in find_subprog()
2882 static int add_subprog(struct bpf_verifier_env *env, int off) in add_subprog() argument
2884 int insn_cnt = env->prog->len; in add_subprog()
2888 verbose(env, "call to invalid destination\n"); in add_subprog()
2891 ret = find_subprog(env, off); in add_subprog()
2894 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { in add_subprog()
2895 verbose(env, "too many subprograms\n"); in add_subprog()
2899 env->subprog_info[env->subprog_cnt++].start = off; in add_subprog()
2900 sort(env->subprog_info, env->subprog_cnt, in add_subprog()
2901 sizeof(env->subprog_info[0]), cmp_subprogs, NULL); in add_subprog()
2902 return env->subprog_cnt - 1; in add_subprog()
2905 static int bpf_find_exception_callback_insn_off(struct bpf_verifier_env *env) in bpf_find_exception_callback_insn_off() argument
2907 struct bpf_prog_aux *aux = env->prog->aux; in bpf_find_exception_callback_insn_off()
2921 verbose(env, "invalid btf id for main subprog in func_info\n"); in bpf_find_exception_callback_insn_off()
2932 verbose(env, "multiple exception callback tags for main subprog\n"); in bpf_find_exception_callback_insn_off()
2938 verbose(env, "exception callback '%s' could not be found in BTF\n", name); in bpf_find_exception_callback_insn_off()
2944 verbose(env, "exception callback '%s' must have global linkage\n", name); in bpf_find_exception_callback_insn_off()
2956 verbose(env, "invalid exception callback insn_off in func_info: 0\n"); in bpf_find_exception_callback_insn_off()
2961 verbose(env, "exception callback type id not found in func_info\n"); in bpf_find_exception_callback_insn_off()
3043 static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env, in __find_kfunc_desc_btf() argument
3053 tab = env->prog->aux->kfunc_btf_tab; in __find_kfunc_desc_btf()
3058 verbose(env, "too many different module BTFs\n"); in __find_kfunc_desc_btf()
3062 if (bpfptr_is_null(env->fd_array)) { in __find_kfunc_desc_btf()
3063 verbose(env, "kfunc offset > 0 without fd_array is invalid\n"); in __find_kfunc_desc_btf()
3067 if (copy_from_bpfptr_offset(&btf_fd, env->fd_array, in __find_kfunc_desc_btf()
3074 verbose(env, "invalid module BTF fd specified\n"); in __find_kfunc_desc_btf()
3079 verbose(env, "BTF fd for kfunc is not a module BTF\n"); in __find_kfunc_desc_btf()
3119 static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) in find_kfunc_desc_btf() argument
3126 verbose(env, "negative offset disallowed for kernel module function call\n"); in find_kfunc_desc_btf()
3130 return __find_kfunc_desc_btf(env, offset); in find_kfunc_desc_btf()
3135 static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset) in add_kfunc_call() argument
3148 prog_aux = env->prog->aux; in add_kfunc_call()
3153 verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n"); in add_kfunc_call()
3157 if (!env->prog->jit_requested) { in add_kfunc_call()
3158 verbose(env, "JIT is required for calling kernel function\n"); in add_kfunc_call()
3163 verbose(env, "JIT does not support calling kernel function\n"); in add_kfunc_call()
3167 if (!env->prog->gpl_compatible) { in add_kfunc_call()
3168 verbose(env, "cannot call kernel function from non-GPL compatible program\n"); in add_kfunc_call()
3194 desc_btf = find_kfunc_desc_btf(env, offset); in add_kfunc_call()
3196 verbose(env, "failed to find BTF for kernel function\n"); in add_kfunc_call()
3200 if (find_kfunc_desc(env->prog, func_id, offset)) in add_kfunc_call()
3204 verbose(env, "too many different kernel function calls\n"); in add_kfunc_call()
3210 verbose(env, "kernel btf_id %u is not a function\n", in add_kfunc_call()
3216 verbose(env, "kernel function btf_id %u does not have a valid func_proto\n", in add_kfunc_call()
3224 verbose(env, "cannot find address for kernel function %s\n", in add_kfunc_call()
3228 specialize_kfunc(env, func_id, offset, &addr); in add_kfunc_call()
3236 verbose(env, "address of kernel function %s is out of range\n", in add_kfunc_call()
3243 err = bpf_dev_bound_kfunc_check(&env->log, prog_aux); in add_kfunc_call()
3253 err = btf_distill_func_proto(&env->log, desc_btf, in add_kfunc_call()
3309 static int add_kfunc_in_insns(struct bpf_verifier_env *env, in add_kfunc_in_insns() argument
3316 ret = add_kfunc_call(env, insn->imm, insn->off); in add_kfunc_in_insns()
3324 static int add_subprog_and_kfunc(struct bpf_verifier_env *env) in add_subprog_and_kfunc() argument
3326 struct bpf_subprog_info *subprog = env->subprog_info; in add_subprog_and_kfunc()
3327 int i, ret, insn_cnt = env->prog->len, ex_cb_insn; in add_subprog_and_kfunc()
3328 struct bpf_insn *insn = env->prog->insnsi; in add_subprog_and_kfunc()
3331 ret = add_subprog(env, 0); in add_subprog_and_kfunc()
3340 if (!env->bpf_capable) { in add_subprog_and_kfunc()
3341 …verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_AD… in add_subprog_and_kfunc()
3346 ret = add_subprog(env, i + insn->imm + 1); in add_subprog_and_kfunc()
3348 ret = add_kfunc_call(env, insn->imm, insn->off); in add_subprog_and_kfunc()
3354 ret = bpf_find_exception_callback_insn_off(env); in add_subprog_and_kfunc()
3363 ret = add_subprog(env, ex_cb_insn); in add_subprog_and_kfunc()
3366 for (i = 1; i < env->subprog_cnt; i++) { in add_subprog_and_kfunc()
3367 if (env->subprog_info[i].start != ex_cb_insn) in add_subprog_and_kfunc()
3369 env->exception_callback_subprog = i; in add_subprog_and_kfunc()
3370 mark_subprog_exc_cb(env, i); in add_subprog_and_kfunc()
3378 subprog[env->subprog_cnt].start = insn_cnt; in add_subprog_and_kfunc()
3380 if (env->log.level & BPF_LOG_LEVEL2) in add_subprog_and_kfunc()
3381 for (i = 0; i < env->subprog_cnt; i++) in add_subprog_and_kfunc()
3382 verbose(env, "func#%d @%d\n", i, subprog[i].start); in add_subprog_and_kfunc()
3396 static int check_subprogs(struct bpf_verifier_env *env) in check_subprogs() argument
3399 struct bpf_subprog_info *subprog = env->subprog_info; in check_subprogs()
3400 struct bpf_insn *insn = env->prog->insnsi; in check_subprogs()
3401 int insn_cnt = env->prog->len; in check_subprogs()
3424 verbose(env, "jump out of range from insn %d to %d\n", i, off); in check_subprogs()
3436 verbose(env, "last insn is not an exit or jmp\n"); in check_subprogs()
3441 if (cur_subprog < env->subprog_cnt) in check_subprogs()
3451 static int mark_reg_read(struct bpf_verifier_env *env, in mark_reg_read() argument
3463 verbose(env, "verifier BUG type %s var_off %lld off %d\n", in mark_reg_read()
3464 reg_type_str(env, parent->type), in mark_reg_read()
3494 if (env->longest_mark_read_walk < cnt) in mark_reg_read()
3495 env->longest_mark_read_walk = cnt; in mark_reg_read()
3499 static int mark_stack_slot_obj_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in mark_stack_slot_obj_read() argument
3502 struct bpf_func_state *state = func(env, reg); in mark_stack_slot_obj_read()
3508 err = mark_reg_read(env, st, st->parent, REG_LIVE_READ64); in mark_stack_slot_obj_read()
3512 mark_stack_slot_scratched(env, spi - i); in mark_stack_slot_obj_read()
3517 static int mark_dynptr_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in mark_dynptr_read() argument
3527 spi = dynptr_get_spi(env, reg); in mark_dynptr_read()
3534 return mark_stack_slot_obj_read(env, reg, spi, BPF_DYNPTR_NR_SLOTS); in mark_dynptr_read()
3537 static int mark_iter_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in mark_iter_read() argument
3540 return mark_stack_slot_obj_read(env, reg, spi, nr_slots); in mark_iter_read()
3543 static int mark_irq_flag_read(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in mark_irq_flag_read() argument
3547 spi = irq_flag_get_spi(env, reg); in mark_irq_flag_read()
3550 return mark_stack_slot_obj_read(env, reg, spi, 1); in mark_irq_flag_read()
3557 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, in is_reg64() argument
3668 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn) in insn_has_def32() argument
3675 return !is_reg64(env, insn, dst_reg, NULL, DST_OP); in insn_has_def32()
3678 static void mark_insn_zext(struct bpf_verifier_env *env, in mark_insn_zext() argument
3686 env->insn_aux_data[def_idx - 1].zext_dst = true; in mark_insn_zext()
3691 static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno, in __check_reg_arg() argument
3694 struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; in __check_reg_arg()
3699 verbose(env, "R%d is invalid\n", regno); in __check_reg_arg()
3703 mark_reg_scratched(env, regno); in __check_reg_arg()
3706 rw64 = is_reg64(env, insn, regno, reg, t); in __check_reg_arg()
3710 verbose(env, "R%d !read_ok\n", regno); in __check_reg_arg()
3718 mark_insn_zext(env, reg); in __check_reg_arg()
3720 return mark_reg_read(env, reg, reg->parent, in __check_reg_arg()
3725 verbose(env, "frame pointer is read only\n"); in __check_reg_arg()
3729 reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; in __check_reg_arg()
3731 mark_reg_unknown(env, regs, regno); in __check_reg_arg()
3736 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, in check_reg_arg() argument
3739 struct bpf_verifier_state *vstate = env->cur_state; in check_reg_arg()
3742 return __check_reg_arg(env, state->regs, regno, t); in check_reg_arg()
3760 static void mark_jmp_point(struct bpf_verifier_env *env, int idx) in mark_jmp_point() argument
3762 env->insn_aux_data[idx].jmp_point = true; in mark_jmp_point()
3765 static bool is_jmp_point(struct bpf_verifier_env *env, int insn_idx) in is_jmp_point() argument
3767 return env->insn_aux_data[insn_idx].jmp_point; in is_jmp_point()
3849 static int push_insn_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, in push_insn_history() argument
3856 if (env->cur_hist_ent) { in push_insn_history()
3860 WARN_ONCE((env->cur_hist_ent->flags & insn_flags) && in push_insn_history()
3861 (env->cur_hist_ent->flags & insn_flags) != insn_flags, in push_insn_history()
3863 env->insn_idx, env->cur_hist_ent->flags, insn_flags); in push_insn_history()
3864 env->cur_hist_ent->flags |= insn_flags; in push_insn_history()
3865 WARN_ONCE(env->cur_hist_ent->linked_regs != 0, in push_insn_history()
3867 env->insn_idx, env->cur_hist_ent->linked_regs); in push_insn_history()
3868 env->cur_hist_ent->linked_regs = linked_regs; in push_insn_history()
3872 if (cur->insn_hist_end + 1 > env->insn_hist_cap) { in push_insn_history()
3874 p = kvrealloc(env->insn_hist, alloc_size, GFP_USER); in push_insn_history()
3877 env->insn_hist = p; in push_insn_history()
3878 env->insn_hist_cap = alloc_size / sizeof(*p); in push_insn_history()
3881 p = &env->insn_hist[cur->insn_hist_end]; in push_insn_history()
3882 p->idx = env->insn_idx; in push_insn_history()
3883 p->prev_idx = env->prev_insn_idx; in push_insn_history()
3888 env->cur_hist_ent = p; in push_insn_history()
3893 static struct bpf_insn_hist_entry *get_insn_hist_entry(struct bpf_verifier_env *env, in get_insn_hist_entry() argument
3896 if (hist_end > hist_start && env->insn_hist[hist_end - 1].idx == insn_idx) in get_insn_hist_entry()
3897 return &env->insn_hist[hist_end - 1]; in get_insn_hist_entry()
3914 static int get_prev_insn_idx(const struct bpf_verifier_env *env, in get_prev_insn_idx() argument
3924 if (cnt == 1 && env->insn_hist[hist_start].idx == insn_idx) in get_prev_insn_idx()
3928 if (cnt && env->insn_hist[hist_end - 1].idx == insn_idx) { in get_prev_insn_idx()
3930 return env->insn_hist[hist_end - 1].prev_idx; in get_prev_insn_idx()
3952 static void verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn) in verbose_insn() argument
3957 .private_data = env, in verbose_insn()
3960 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); in verbose_insn()
3970 struct bpf_verifier_env *env = bt->env; in bt_reset() local
3973 bt->env = env; in bt_reset()
3990 verbose(bt->env, "BUG subprog enter from frame %d\n", bt->frame); in bt_subprog_enter()
4001 verbose(bt->env, "BUG subprog exit from frame 0\n"); in bt_subprog_exit()
4149 static bool calls_callback(struct bpf_verifier_env *env, int insn_idx);
4160 static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, in backtrack_insn() argument
4163 struct bpf_insn *insn = env->prog->insnsi + idx; in backtrack_insn()
4173 if (env->log.level & BPF_LOG_LEVEL2) { in backtrack_insn()
4174 fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt)); in backtrack_insn()
4175 verbose(env, "mark_precise: frame%d: regs=%s ", in backtrack_insn()
4176 bt->frame, env->tmp_str_buf); in backtrack_insn()
4177 fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt)); in backtrack_insn()
4178 verbose(env, "stack=%s before ", env->tmp_str_buf); in backtrack_insn()
4179 verbose(env, "%d: ", idx); in backtrack_insn()
4180 verbose_insn(env, insn); in backtrack_insn()
4269 subprog = find_subprog(env, subprog_insn_idx); in backtrack_insn()
4273 if (subprog_is_global(env, subprog)) { in backtrack_insn()
4286 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); in backtrack_insn()
4301 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); in backtrack_insn()
4309 verbose(env, "BUG stack slots %llx\n", bt_stack_mask(bt)); in backtrack_insn()
4333 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); in backtrack_insn()
4338 verbose(env, "BUG stack slots %llx\n", bt_stack_mask(bt)); in backtrack_insn()
4361 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); in backtrack_insn()
4376 if (subseq_idx >= 0 && calls_callback(env, subseq_idx)) in backtrack_insn()
4380 verbose(env, "BUG regs %x\n", bt_reg_mask(bt)); in backtrack_insn()
4394 bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) && in backtrack_insn()
4496 static void mark_all_scalars_precise(struct bpf_verifier_env *env, in mark_all_scalars_precise() argument
4503 if (env->log.level & BPF_LOG_LEVEL2) { in mark_all_scalars_precise()
4504 verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n", in mark_all_scalars_precise()
4522 if (env->log.level & BPF_LOG_LEVEL2) { in mark_all_scalars_precise()
4523 verbose(env, "force_precise: frame%d: forcing r%d to be precise\n", in mark_all_scalars_precise()
4534 if (env->log.level & BPF_LOG_LEVEL2) { in mark_all_scalars_precise()
4535 verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n", in mark_all_scalars_precise()
4543 static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_verifier_state *st) in mark_all_scalars_imprecise() argument
4655 static int __mark_chain_precision(struct bpf_verifier_env *env, int regno) in __mark_chain_precision() argument
4657 struct backtrack_state *bt = &env->bt; in __mark_chain_precision()
4658 struct bpf_verifier_state *st = env->cur_state; in __mark_chain_precision()
4660 int last_idx = env->insn_idx; in __mark_chain_precision()
4667 if (!env->bpf_capable) in __mark_chain_precision()
4671 bt_init(bt, env->cur_state->curframe); in __mark_chain_precision()
4696 if (env->log.level & BPF_LOG_LEVEL2) { in __mark_chain_precision()
4697 verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n", in __mark_chain_precision()
4722 verbose(env, "BUG backtracking func entry subprog %d reg_mask %x stack_mask %llx\n", in __mark_chain_precision()
4733 hist = get_insn_hist_entry(env, hist_start, hist_end, i); in __mark_chain_precision()
4734 err = backtrack_insn(env, i, subseq_idx, hist, bt); in __mark_chain_precision()
4737 mark_all_scalars_precise(env, env->cur_state); in __mark_chain_precision()
4750 i = get_prev_insn_idx(env, st, i, hist_start, &hist_end); in __mark_chain_precision()
4753 if (i >= env->prog->len) { in __mark_chain_precision()
4760 verbose(env, "BUG backtracking idx %d\n", i); in __mark_chain_precision()
4787 verbose(env, "BUG backtracking (stack slot %d, total slots %d)\n", in __mark_chain_precision()
4803 if (env->log.level & BPF_LOG_LEVEL2) { in __mark_chain_precision()
4804 fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, in __mark_chain_precision()
4806 verbose(env, "mark_precise: frame%d: parent state regs=%s ", in __mark_chain_precision()
4807 fr, env->tmp_str_buf); in __mark_chain_precision()
4808 fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, in __mark_chain_precision()
4810 verbose(env, "stack=%s: ", env->tmp_str_buf); in __mark_chain_precision()
4811 print_verifier_state(env, st, fr, true); in __mark_chain_precision()
4828 mark_all_scalars_precise(env, env->cur_state); in __mark_chain_precision()
4835 int mark_chain_precision(struct bpf_verifier_env *env, int regno) in mark_chain_precision() argument
4837 return __mark_chain_precision(env, regno); in mark_chain_precision()
4843 static int mark_chain_precision_batch(struct bpf_verifier_env *env) in mark_chain_precision_batch() argument
4845 return __mark_chain_precision(env, -1); in mark_chain_precision_batch()
4903 static void assign_scalar_id_before_mov(struct bpf_verifier_env *env, in assign_scalar_id_before_mov() argument
4924 src_reg->id = ++env->id_gen; in assign_scalar_id_before_mov()
4938 static void save_register_state(struct bpf_verifier_env *env, in save_register_state() argument
4954 mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]); in save_register_state()
4968 static void check_fastcall_stack_contract(struct bpf_verifier_env *env, in check_fastcall_stack_contract() argument
4971 struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno]; in check_fastcall_stack_contract()
4972 struct bpf_insn_aux_data *aux = env->insn_aux_data; in check_fastcall_stack_contract()
4995 static int check_stack_write_fixed_off(struct bpf_verifier_env *env, in check_stack_write_fixed_off() argument
5003 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; in check_stack_write_fixed_off()
5010 if (!env->allow_ptr_leaks && in check_stack_write_fixed_off()
5014 verbose(env, "attempt to corrupt spilled pointer on stack\n"); in check_stack_write_fixed_off()
5018 cur = env->cur_state->frame[env->cur_state->curframe]; in check_stack_write_fixed_off()
5021 if (!env->bypass_spec_v4) { in check_stack_write_fixed_off()
5034 env->insn_aux_data[insn_idx].sanitize_stack_spill = true; in check_stack_write_fixed_off()
5037 err = destroy_if_dynptr_stack_slot(env, state, spi); in check_stack_write_fixed_off()
5041 check_fastcall_stack_contract(env, state, insn_idx, off); in check_stack_write_fixed_off()
5042 mark_stack_slot_scratched(env, spi); in check_stack_write_fixed_off()
5043 if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) { in check_stack_write_fixed_off()
5049 assign_scalar_id_before_mov(env, reg); in check_stack_write_fixed_off()
5050 save_register_state(env, state, spi, reg, size); in check_stack_write_fixed_off()
5055 env->bpf_capable) { in check_stack_write_fixed_off()
5056 struct bpf_reg_state *tmp_reg = &env->fake_reg[0]; in check_stack_write_fixed_off()
5061 save_register_state(env, state, spi, tmp_reg, size); in check_stack_write_fixed_off()
5065 verbose_linfo(env, insn_idx, "; "); in check_stack_write_fixed_off()
5066 verbose(env, "invalid size of register spill\n"); in check_stack_write_fixed_off()
5070 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); in check_stack_write_fixed_off()
5073 save_register_state(env, state, spi, reg, size); in check_stack_write_fixed_off()
5104 err = mark_chain_precision(env, value_regno); in check_stack_write_fixed_off()
5117 return push_insn_history(env, env->cur_state, insn_flags, 0); in check_stack_write_fixed_off()
5140 static int check_stack_write_var_off(struct bpf_verifier_env *env, in check_stack_write_var_off() argument
5150 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; in check_stack_write_var_off()
5157 cur = env->cur_state->frame[env->cur_state->curframe]; in check_stack_write_var_off()
5171 err = destroy_if_dynptr_stack_slot(env, state, spi); in check_stack_write_var_off()
5176 check_fastcall_stack_contract(env, state, insn_idx, min_off); in check_stack_write_var_off()
5185 mark_stack_slot_scratched(env, spi); in check_stack_write_var_off()
5187 if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) { in check_stack_write_var_off()
5199 verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", in check_stack_write_var_off()
5234 if (*stype == STACK_INVALID && !env->allow_uninit_stack) { in check_stack_write_var_off()
5235 verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", in check_stack_write_var_off()
5243 err = mark_chain_precision(env, value_regno); in check_stack_write_var_off()
5258 static void mark_reg_stack_read(struct bpf_verifier_env *env, in mark_reg_stack_read() argument
5263 struct bpf_verifier_state *vstate = env->cur_state; in mark_reg_stack_read()
5272 mark_stack_slot_scratched(env, spi); in mark_reg_stack_read()
5282 __mark_reg_const_zero(env, &state->regs[dst_regno]); in mark_reg_stack_read()
5285 mark_reg_unknown(env, state->regs, dst_regno); in mark_reg_stack_read()
5299 static int check_stack_read_fixed_off(struct bpf_verifier_env *env, in check_stack_read_fixed_off() argument
5304 struct bpf_verifier_state *vstate = env->cur_state; in check_stack_read_fixed_off()
5314 mark_stack_slot_scratched(env, spi); in check_stack_read_fixed_off()
5315 check_fastcall_stack_contract(env, state, env->insn_idx, off); in check_stack_read_fixed_off()
5325 verbose_linfo(env, env->insn_idx, "; "); in check_stack_read_fixed_off()
5326 verbose(env, "invalid size of register fill\n"); in check_stack_read_fixed_off()
5330 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in check_stack_read_fixed_off()
5364 if (type == STACK_INVALID && env->allow_uninit_stack) in check_stack_read_fixed_off()
5366 verbose(env, "invalid read from stack off %d+%d size %d\n", in check_stack_read_fixed_off()
5373 __mark_reg_const_zero(env, &state->regs[dst_regno]); in check_stack_read_fixed_off()
5377 __mark_reg_const_zero(env, &state->regs[dst_regno]); in check_stack_read_fixed_off()
5380 mark_reg_unknown(env, state->regs, dst_regno); in check_stack_read_fixed_off()
5393 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { in check_stack_read_fixed_off()
5400 verbose(env, "leaking pointer from stack off %d\n", in check_stack_read_fixed_off()
5404 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in check_stack_read_fixed_off()
5412 if (type == STACK_INVALID && env->allow_uninit_stack) in check_stack_read_fixed_off()
5414 verbose(env, "invalid read from stack off %d+%d size %d\n", in check_stack_read_fixed_off()
5418 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in check_stack_read_fixed_off()
5420 mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); in check_stack_read_fixed_off()
5424 return push_insn_history(env, env->cur_state, insn_flags, 0); in check_stack_read_fixed_off()
5433 static int check_stack_range_initialized(struct bpf_verifier_env *env,
5439 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) in reg_state() argument
5441 return cur_regs(env) + regno; in reg_state()
5457 static int check_stack_read_var_off(struct bpf_verifier_env *env, in check_stack_read_var_off() argument
5461 struct bpf_reg_state *reg = reg_state(env, ptr_regno); in check_stack_read_var_off()
5462 struct bpf_func_state *ptr_state = func(env, reg); in check_stack_read_var_off()
5468 err = check_stack_range_initialized(env, ptr_regno, off, size, in check_stack_read_var_off()
5475 mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); in check_stack_read_var_off()
5476 check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off); in check_stack_read_var_off()
5489 static int check_stack_read(struct bpf_verifier_env *env, in check_stack_read() argument
5493 struct bpf_reg_state *reg = reg_state(env, ptr_regno); in check_stack_read()
5494 struct bpf_func_state *state = func(env, reg); in check_stack_read()
5507 …verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=… in check_stack_read()
5522 err = check_stack_read_fixed_off(env, state, off, size, in check_stack_read()
5529 err = check_stack_read_var_off(env, ptr_regno, off, size, in check_stack_read()
5546 static int check_stack_write(struct bpf_verifier_env *env, in check_stack_write() argument
5550 struct bpf_reg_state *reg = reg_state(env, ptr_regno); in check_stack_write()
5551 struct bpf_func_state *state = func(env, reg); in check_stack_write()
5556 err = check_stack_write_fixed_off(env, state, off, size, in check_stack_write()
5562 err = check_stack_write_var_off(env, state, in check_stack_write()
5569 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, in check_map_access_type() argument
5572 struct bpf_reg_state *regs = cur_regs(env); in check_map_access_type()
5577 verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", in check_map_access_type()
5583 verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", in check_map_access_type()
5592 static int __check_mem_access(struct bpf_verifier_env *env, int regno, in __check_mem_access() argument
5602 reg = &cur_regs(env)[regno]; in __check_mem_access()
5605 verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n", in __check_mem_access()
5609 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", in __check_mem_access()
5615 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", in __check_mem_access()
5620 verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", in __check_mem_access()
5628 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, in check_mem_region_access() argument
5632 struct bpf_verifier_state *vstate = env->cur_state; in check_mem_region_access()
5651 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_mem_region_access()
5655 err = __check_mem_access(env, regno, reg->smin_value + off, size, in check_mem_region_access()
5658 verbose(env, "R%d min value is outside of the allowed memory range\n", in check_mem_region_access()
5668 verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n", in check_mem_region_access()
5672 err = __check_mem_access(env, regno, reg->umax_value + off, size, in check_mem_region_access()
5675 verbose(env, "R%d max value is outside of the allowed memory range\n", in check_mem_region_access()
5683 static int __check_ptr_off_reg(struct bpf_verifier_env *env, in __check_ptr_off_reg() argument
5692 verbose(env, "negative offset %s ptr R%d off=%d disallowed\n", in __check_ptr_off_reg()
5693 reg_type_str(env, reg->type), regno, reg->off); in __check_ptr_off_reg()
5698 verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n", in __check_ptr_off_reg()
5699 reg_type_str(env, reg->type), regno, reg->off); in __check_ptr_off_reg()
5707 verbose(env, "variable %s access var_off=%s disallowed\n", in __check_ptr_off_reg()
5708 reg_type_str(env, reg->type), tn_buf); in __check_ptr_off_reg()
5715 static int check_ptr_off_reg(struct bpf_verifier_env *env, in check_ptr_off_reg() argument
5718 return __check_ptr_off_reg(env, reg, regno, false); in check_ptr_off_reg()
5721 static int map_kptr_match_type(struct bpf_verifier_env *env, in map_kptr_match_type() argument
5753 if (__check_ptr_off_reg(env, reg, regno, true)) in map_kptr_match_type()
5780 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, in map_kptr_match_type()
5786 verbose(env, "invalid kptr access, R%d type=%s%s ", regno, in map_kptr_match_type()
5787 reg_type_str(env, reg->type), reg_name); in map_kptr_match_type()
5788 verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name); in map_kptr_match_type()
5790 verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED), in map_kptr_match_type()
5793 verbose(env, "\n"); in map_kptr_match_type()
5797 static bool in_sleepable(struct bpf_verifier_env *env) in in_sleepable() argument
5799 return env->prog->sleepable || in in_sleepable()
5800 (env->cur_state && env->cur_state->in_sleepable); in in_sleepable()
5806 static bool in_rcu_cs(struct bpf_verifier_env *env) in in_rcu_cs() argument
5808 return env->cur_state->active_rcu_lock || in in_rcu_cs()
5809 env->cur_state->active_locks || in in_rcu_cs()
5810 !in_sleepable(env); in in_rcu_cs()
5858 static u32 btf_ld_kptr_type(struct bpf_verifier_env *env, struct btf_field *kptr_field) in btf_ld_kptr_type() argument
5864 if (rcu_safe_kptr(kptr_field) && in_rcu_cs(env)) { in btf_ld_kptr_type()
5881 static int mark_uptr_ld_reg(struct bpf_verifier_env *env, u32 regno, in mark_uptr_ld_reg() argument
5888 mark_reg_known_zero(env, cur_regs(env), regno); in mark_uptr_ld_reg()
5889 reg = reg_state(env, regno); in mark_uptr_ld_reg()
5892 reg->id = ++env->id_gen; in mark_uptr_ld_reg()
5897 static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno, in check_map_kptr_access() argument
5901 struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; in check_map_kptr_access()
5913 verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n"); in check_map_kptr_access()
5922 verbose(env, "store to referenced kptr disallowed\n"); in check_map_kptr_access()
5926 verbose(env, "store to uptr disallowed\n"); in check_map_kptr_access()
5932 return mark_uptr_ld_reg(env, value_regno, kptr_field); in check_map_kptr_access()
5937 mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, kptr_field->kptr.btf, in check_map_kptr_access()
5938 kptr_field->kptr.btf_id, btf_ld_kptr_type(env, kptr_field)); in check_map_kptr_access()
5940 val_reg = reg_state(env, value_regno); in check_map_kptr_access()
5942 map_kptr_match_type(env, kptr_field, val_reg, value_regno)) in check_map_kptr_access()
5946 verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n", in check_map_kptr_access()
5951 verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n"); in check_map_kptr_access()
5958 static int check_map_access(struct bpf_verifier_env *env, u32 regno, in check_map_access() argument
5962 struct bpf_verifier_state *vstate = env->cur_state; in check_map_access()
5969 err = check_mem_region_access(env, regno, off, size, map->value_size, in check_map_access()
5993 verbose(env, "%s cannot be accessed indirectly by helper\n", in check_map_access()
5998 verbose(env, "%s access cannot have variable offset\n", in check_map_access()
6003 verbose(env, "%s access misaligned expected=%u off=%llu\n", in check_map_access()
6009 verbose(env, "%s access size must be BPF_DW\n", in check_map_access()
6015 verbose(env, "%s cannot be accessed directly by load/store\n", in check_map_access()
6026 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, in may_access_direct_pkt_data() argument
6030 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); in may_access_direct_pkt_data()
6054 env->seen_direct_write = true; in may_access_direct_pkt_data()
6059 env->seen_direct_write = true; in may_access_direct_pkt_data()
6068 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, in check_packet_access() argument
6071 struct bpf_reg_state *regs = cur_regs(env); in check_packet_access()
6084 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_packet_access()
6090 __check_mem_access(env, regno, off, size, reg->range, in check_packet_access()
6093 verbose(env, "R%d offset is outside of the packet\n", regno); in check_packet_access()
6103 env->prog->aux->max_pkt_offset = in check_packet_access()
6104 max_t(u32, env->prog->aux->max_pkt_offset, in check_packet_access()
6111 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, in check_ctx_access() argument
6114 if (env->ops->is_valid_access && in check_ctx_access()
6115 env->ops->is_valid_access(off, size, t, env->prog, info)) { in check_ctx_access()
6125 !find_reference_state(env->cur_state, info->ref_obj_id)) { in check_ctx_access()
6126 verbose(env, "invalid bpf_context access off=%d. Reference may already be released\n", in check_ctx_access()
6131 env->insn_aux_data[insn_idx].ctx_field_size = info->ctx_field_size; in check_ctx_access()
6134 if (env->prog->aux->max_ctx_offset < off + size) in check_ctx_access()
6135 env->prog->aux->max_ctx_offset = off + size; in check_ctx_access()
6139 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); in check_ctx_access()
6143 static int check_flow_keys_access(struct bpf_verifier_env *env, int off, in check_flow_keys_access() argument
6148 verbose(env, "invalid access to flow keys off=%d size=%d\n", in check_flow_keys_access()
6155 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, in check_sock_access() argument
6159 struct bpf_reg_state *regs = cur_regs(env); in check_sock_access()
6165 …verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n… in check_sock_access()
6189 env->insn_aux_data[insn_idx].ctx_field_size = in check_sock_access()
6194 verbose(env, "R%d invalid %s access off=%d size=%d\n", in check_sock_access()
6195 regno, reg_type_str(env, reg->type), off, size); in check_sock_access()
6200 static bool is_pointer_value(struct bpf_verifier_env *env, int regno) in is_pointer_value() argument
6202 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); in is_pointer_value()
6205 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) in is_ctx_reg() argument
6207 const struct bpf_reg_state *reg = reg_state(env, regno); in is_ctx_reg()
6212 static bool is_sk_reg(struct bpf_verifier_env *env, int regno) in is_sk_reg() argument
6214 const struct bpf_reg_state *reg = reg_state(env, regno); in is_sk_reg()
6219 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) in is_pkt_reg() argument
6221 const struct bpf_reg_state *reg = reg_state(env, regno); in is_pkt_reg()
6226 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) in is_flow_key_reg() argument
6228 const struct bpf_reg_state *reg = reg_state(env, regno); in is_flow_key_reg()
6234 static bool is_arena_reg(struct bpf_verifier_env *env, int regno) in is_arena_reg() argument
6236 const struct bpf_reg_state *reg = reg_state(env, regno); in is_arena_reg()
6244 static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno, in atomic_ptr_type_ok() argument
6247 if (is_ctx_reg(env, regno)) in atomic_ptr_type_ok()
6249 if (is_pkt_reg(env, regno)) in atomic_ptr_type_ok()
6251 if (is_flow_key_reg(env, regno)) in atomic_ptr_type_ok()
6253 if (is_sk_reg(env, regno)) in atomic_ptr_type_ok()
6255 if (is_arena_reg(env, regno)) in atomic_ptr_type_ok()
6304 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, in check_pkt_ptr_alignment() argument
6330 verbose(env, in check_pkt_ptr_alignment()
6339 static int check_generic_ptr_alignment(struct bpf_verifier_env *env, in check_generic_ptr_alignment() argument
6355 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", in check_generic_ptr_alignment()
6363 static int check_ptr_alignment(struct bpf_verifier_env *env, in check_ptr_alignment() argument
6367 bool strict = env->strict_alignment || strict_alignment_once; in check_ptr_alignment()
6376 return check_pkt_ptr_alignment(env, reg, off, size, strict); in check_ptr_alignment()
6414 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, in check_ptr_alignment()
6446 static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth) in round_up_stack_depth() argument
6448 if (env->prog->jit_requested) in round_up_stack_depth()
6463 static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, in check_max_stack_depth_subprog() argument
6466 struct bpf_subprog_info *subprog = env->subprog_info; in check_max_stack_depth_subprog()
6467 struct bpf_insn *insn = env->prog->insnsi; in check_max_stack_depth_subprog()
6498 verbose(env, in check_max_stack_depth_subprog()
6504 subprog_depth = round_up_stack_depth(env, subprog[idx].stack_depth); in check_max_stack_depth_subprog()
6517 verbose(env, "stack size of subprog %d is %d. Too large\n", in check_max_stack_depth_subprog()
6524 verbose(env, "combined stack size of %d calls is %d. Too large\n", in check_max_stack_depth_subprog()
6549 verbose(env, in check_max_stack_depth_subprog()
6563 sidx = find_subprog(env, next_insn); in check_max_stack_depth_subprog()
6571 verbose(env, "verifier bug. subprog has tail_call and async cb\n"); in check_max_stack_depth_subprog()
6578 verbose(env, "insn %d cannot call exception cb directly\n", i); in check_max_stack_depth_subprog()
6592 verbose(env, "the call stack of %d frames is too deep !\n", in check_max_stack_depth_subprog()
6606 verbose(env, "cannot tail call within exception cb\n"); in check_max_stack_depth_subprog()
6612 env->prog->aux->tail_call_reachable = true; in check_max_stack_depth_subprog()
6620 depth -= round_up_stack_depth(env, subprog[idx].stack_depth); in check_max_stack_depth_subprog()
6627 static int check_max_stack_depth(struct bpf_verifier_env *env) in check_max_stack_depth() argument
6630 struct bpf_subprog_info *si = env->subprog_info; in check_max_stack_depth()
6634 for (int i = 0; i < env->subprog_cnt; i++) { in check_max_stack_depth()
6642 priv_stack_mode = bpf_enable_priv_stack(env->prog); in check_max_stack_depth()
6652 for (int i = env->subprog_cnt - 1; i >= 0; i--) { in check_max_stack_depth()
6655 ret = check_max_stack_depth_subprog(env, i, priv_stack_supported); in check_max_stack_depth()
6661 for (int i = 0; i < env->subprog_cnt; i++) { in check_max_stack_depth()
6663 env->prog->aux->jits_use_priv_stack = true; in check_max_stack_depth()
6672 static int get_callee_stack_depth(struct bpf_verifier_env *env, in get_callee_stack_depth() argument
6677 subprog = find_subprog(env, start); in get_callee_stack_depth()
6683 return env->subprog_info[subprog].stack_depth; in get_callee_stack_depth()
6687 static int __check_buffer_access(struct bpf_verifier_env *env, in __check_buffer_access() argument
6693 verbose(env, in __check_buffer_access()
6702 verbose(env, in __check_buffer_access()
6711 static int check_tp_buffer_access(struct bpf_verifier_env *env, in check_tp_buffer_access() argument
6717 err = __check_buffer_access(env, "tracepoint", reg, regno, off, size); in check_tp_buffer_access()
6721 if (off + size > env->prog->aux->max_tp_access) in check_tp_buffer_access()
6722 env->prog->aux->max_tp_access = off + size; in check_tp_buffer_access()
6727 static int check_buffer_access(struct bpf_verifier_env *env, in check_buffer_access() argument
6736 err = __check_buffer_access(env, buf_info, reg, regno, off, size); in check_buffer_access()
7050 static bool type_is_rcu(struct bpf_verifier_env *env, in type_is_rcu() argument
7058 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu"); in type_is_rcu()
7061 static bool type_is_rcu_or_null(struct bpf_verifier_env *env, in type_is_rcu_or_null() argument
7069 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_rcu_or_null"); in type_is_rcu_or_null()
7072 static bool type_is_trusted(struct bpf_verifier_env *env, in type_is_trusted() argument
7082 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted"); in type_is_trusted()
7085 static bool type_is_trusted_or_null(struct bpf_verifier_env *env, in type_is_trusted_or_null() argument
7091 return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, in type_is_trusted_or_null()
7095 static int check_ptr_to_btf_access(struct bpf_verifier_env *env, in check_ptr_to_btf_access() argument
7109 if (!env->allow_ptr_leaks) { in check_ptr_to_btf_access()
7110 verbose(env, in check_ptr_to_btf_access()
7115 if (!env->prog->gpl_compatible && btf_is_kernel(reg->btf)) { in check_ptr_to_btf_access()
7116 verbose(env, in check_ptr_to_btf_access()
7122 verbose(env, in check_ptr_to_btf_access()
7131 verbose(env, in check_ptr_to_btf_access()
7138 verbose(env, in check_ptr_to_btf_access()
7145 verbose(env, in check_ptr_to_btf_access()
7151 if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) { in check_ptr_to_btf_access()
7153 verbose(env, "verifier internal error: reg->btf must be kernel btf\n"); in check_ptr_to_btf_access()
7156 ret = env->ops->btf_struct_access(&env->log, reg, off, size); in check_ptr_to_btf_access()
7163 verbose(env, "only read is supported\n"); in check_ptr_to_btf_access()
7169 verbose(env, "verifier internal error: ref_obj_id for allocated object must be non-zero\n"); in check_ptr_to_btf_access()
7173 ret = btf_struct_access(&env->log, reg, off, size, atype, &btf_id, &flag, &field_name); in check_ptr_to_btf_access()
7201 if (type_is_trusted(env, reg, field_name, btf_id)) { in check_ptr_to_btf_access()
7203 } else if (type_is_trusted_or_null(env, reg, field_name, btf_id)) { in check_ptr_to_btf_access()
7205 } else if (in_rcu_cs(env) && !type_may_be_null(reg->type)) { in check_ptr_to_btf_access()
7206 if (type_is_rcu(env, reg, field_name, btf_id)) { in check_ptr_to_btf_access()
7210 type_is_rcu_or_null(env, reg, field_name, btf_id)) { in check_ptr_to_btf_access()
7215 if (type_is_rcu_or_null(env, reg, field_name, btf_id) && in check_ptr_to_btf_access()
7240 mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); in check_ptr_to_btf_access()
7245 static int check_ptr_to_map_access(struct bpf_verifier_env *env, in check_ptr_to_map_access() argument
7261 verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n"); in check_ptr_to_map_access()
7266 verbose(env, "map_ptr access not supported for map type %d\n", in check_ptr_to_map_access()
7274 if (!env->allow_ptr_leaks) { in check_ptr_to_map_access()
7275 verbose(env, in check_ptr_to_map_access()
7282 verbose(env, "R%d is %s invalid negative access: off=%d\n", in check_ptr_to_map_access()
7288 verbose(env, "only read from %s is supported\n", tname); in check_ptr_to_map_access()
7294 mark_btf_ld_reg(env, &map_reg, 0, PTR_TO_BTF_ID, btf_vmlinux, *map->ops->map_btf_id, 0); in check_ptr_to_map_access()
7295 ret = btf_struct_access(&env->log, &map_reg, off, size, atype, &btf_id, &flag, NULL); in check_ptr_to_map_access()
7300 mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag); in check_ptr_to_map_access()
7311 static int check_stack_slot_within_bounds(struct bpf_verifier_env *env, in check_stack_slot_within_bounds() argument
7318 if (t == BPF_WRITE || env->allow_uninit_stack) in check_stack_slot_within_bounds()
7334 struct bpf_verifier_env *env, in check_stack_access_within_bounds() argument
7338 struct bpf_reg_state *regs = cur_regs(env); in check_stack_access_within_bounds()
7340 struct bpf_func_state *state = func(env, reg); in check_stack_access_within_bounds()
7356 verbose(env, "invalid unbounded variable-offset%s stack R%d\n", in check_stack_access_within_bounds()
7364 err = check_stack_slot_within_bounds(env, min_off, state, type); in check_stack_access_within_bounds()
7375 verbose(env, "invalid%s stack R%d off=%d size=%d\n", in check_stack_access_within_bounds()
7381 verbose(env, "invalid variable-offset%s stack R%d var_off=%s off=%d size=%d\n", in check_stack_access_within_bounds()
7390 return grow_stack_state(env, state, -min_off /* size */); in check_stack_access_within_bounds()
7410 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, in check_mem_access() argument
7414 struct bpf_reg_state *regs = cur_regs(env); in check_mem_access()
7423 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); in check_mem_access()
7432 verbose(env, "write to change key R%d not allowed\n", regno); in check_mem_access()
7436 err = check_mem_region_access(env, regno, off, size, in check_mem_access()
7441 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7446 is_pointer_value(env, value_regno)) { in check_mem_access()
7447 verbose(env, "R%d leaks addr into map\n", value_regno); in check_mem_access()
7450 err = check_map_access_type(env, regno, off, size, t); in check_mem_access()
7453 err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT); in check_mem_access()
7460 err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field); in check_mem_access()
7479 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7486 verbose(env, "R%d invalid mem access '%s'\n", regno, in check_mem_access()
7487 reg_type_str(env, reg->type)); in check_mem_access()
7492 verbose(env, "R%d cannot write into %s\n", in check_mem_access()
7493 regno, reg_type_str(env, reg->type)); in check_mem_access()
7498 is_pointer_value(env, value_regno)) { in check_mem_access()
7499 verbose(env, "R%d leaks addr into mem\n", value_regno); in check_mem_access()
7503 err = check_mem_region_access(env, regno, off, size, in check_mem_access()
7506 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7512 .log = &env->log, in check_mem_access()
7516 is_pointer_value(env, value_regno)) { in check_mem_access()
7517 verbose(env, "R%d leaks addr into ctx\n", value_regno); in check_mem_access()
7521 err = check_ptr_off_reg(env, reg, regno); in check_mem_access()
7525 err = check_ctx_access(env, insn_idx, off, size, t, &info); in check_mem_access()
7527 verbose_linfo(env, insn_idx, "; "); in check_mem_access()
7534 if (info.is_retval && get_func_retval_range(env->prog, &range)) { in check_mem_access()
7535 err = __mark_reg_s32_range(env, regs, value_regno, in check_mem_access()
7540 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7543 mark_reg_known_zero(env, regs, in check_mem_access()
7546 regs[value_regno].id = ++env->id_gen; in check_mem_access()
7564 err = check_stack_access_within_bounds(env, regno, off, size, t); in check_mem_access()
7569 err = check_stack_read(env, regno, off, size, in check_mem_access()
7572 err = check_stack_write(env, regno, off, size, in check_mem_access()
7575 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { in check_mem_access()
7576 verbose(env, "cannot write into packet\n"); in check_mem_access()
7580 is_pointer_value(env, value_regno)) { in check_mem_access()
7581 verbose(env, "R%d leaks addr into packet\n", in check_mem_access()
7585 err = check_packet_access(env, regno, off, size, false); in check_mem_access()
7587 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7590 is_pointer_value(env, value_regno)) { in check_mem_access()
7591 verbose(env, "R%d leaks addr into flow keys\n", in check_mem_access()
7596 err = check_flow_keys_access(env, off, size); in check_mem_access()
7598 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7601 verbose(env, "R%d cannot write into %s\n", in check_mem_access()
7602 regno, reg_type_str(env, reg->type)); in check_mem_access()
7605 err = check_sock_access(env, insn_idx, regno, off, size, t); in check_mem_access()
7607 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7609 err = check_tp_buffer_access(env, reg, regno, off, size); in check_mem_access()
7611 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7614 err = check_ptr_to_btf_access(env, regs, regno, off, size, t, in check_mem_access()
7617 err = check_ptr_to_map_access(env, regs, regno, off, size, t, in check_mem_access()
7625 verbose(env, "R%d cannot write into %s\n", in check_mem_access()
7626 regno, reg_type_str(env, reg->type)); in check_mem_access()
7629 max_access = &env->prog->aux->max_rdonly_access; in check_mem_access()
7631 max_access = &env->prog->aux->max_rdwr_access; in check_mem_access()
7634 err = check_buffer_access(env, reg, regno, off, size, false, in check_mem_access()
7638 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7641 mark_reg_unknown(env, regs, value_regno); in check_mem_access()
7643 verbose(env, "R%d invalid mem access '%s'\n", regno, in check_mem_access()
7644 reg_type_str(env, reg->type)); in check_mem_access()
7659 static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,
7662 static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, in check_load_mem() argument
7666 struct bpf_reg_state *regs = cur_regs(env); in check_load_mem()
7671 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_load_mem()
7676 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_load_mem()
7685 err = check_mem_access(env, env->insn_idx, insn->src_reg, insn->off, in check_load_mem()
7688 err = err ?: save_aux_ptr_type(env, src_reg_type, in check_load_mem()
7690 err = err ?: reg_bounds_sanity_check(env, ®s[insn->dst_reg], ctx); in check_load_mem()
7695 static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, in check_store_reg() argument
7698 struct bpf_reg_state *regs = cur_regs(env); in check_store_reg()
7703 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_store_reg()
7708 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_store_reg()
7715 err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off, in check_store_reg()
7718 err = err ?: save_aux_ptr_type(env, dst_reg_type, false); in check_store_reg()
7723 static int check_atomic_rmw(struct bpf_verifier_env *env, in check_atomic_rmw() argument
7730 verbose(env, "invalid atomic operand size\n"); in check_atomic_rmw()
7735 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_atomic_rmw()
7740 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_atomic_rmw()
7748 err = check_reg_arg(env, aux_reg, SRC_OP); in check_atomic_rmw()
7752 if (is_pointer_value(env, aux_reg)) { in check_atomic_rmw()
7753 verbose(env, "R%d leaks addr into mem\n", aux_reg); in check_atomic_rmw()
7758 if (is_pointer_value(env, insn->src_reg)) { in check_atomic_rmw()
7759 verbose(env, "R%d leaks addr into mem\n", insn->src_reg); in check_atomic_rmw()
7763 if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) { in check_atomic_rmw()
7764 verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", in check_atomic_rmw()
7766 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); in check_atomic_rmw()
7777 err = check_reg_arg(env, load_reg, DST_OP); in check_atomic_rmw()
7790 err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off, in check_atomic_rmw()
7793 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in check_atomic_rmw()
7799 if (is_arena_reg(env, insn->dst_reg)) { in check_atomic_rmw()
7800 err = save_aux_ptr_type(env, PTR_TO_ARENA, false); in check_atomic_rmw()
7805 err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off, in check_atomic_rmw()
7812 static int check_atomic_load(struct bpf_verifier_env *env, in check_atomic_load() argument
7817 err = check_load_mem(env, insn, true, false, false, "atomic_load"); in check_atomic_load()
7821 if (!atomic_ptr_type_ok(env, insn->src_reg, insn)) { in check_atomic_load()
7822 verbose(env, "BPF_ATOMIC loads from R%d %s is not allowed\n", in check_atomic_load()
7824 reg_type_str(env, reg_state(env, insn->src_reg)->type)); in check_atomic_load()
7831 static int check_atomic_store(struct bpf_verifier_env *env, in check_atomic_store() argument
7836 err = check_store_reg(env, insn, true); in check_atomic_store()
7840 if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) { in check_atomic_store()
7841 verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", in check_atomic_store()
7843 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); in check_atomic_store()
7850 static int check_atomic(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_atomic() argument
7863 return check_atomic_rmw(env, insn); in check_atomic()
7866 verbose(env, in check_atomic()
7870 return check_atomic_load(env, insn); in check_atomic()
7873 verbose(env, in check_atomic()
7877 return check_atomic_store(env, insn); in check_atomic()
7879 verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", in check_atomic()
7896 struct bpf_verifier_env *env, int regno, int off, in check_stack_range_initialized() argument
7900 struct bpf_reg_state *reg = reg_state(env, regno); in check_stack_range_initialized()
7901 struct bpf_func_state *state = func(env, reg); in check_stack_range_initialized()
7909 verbose(env, "invalid zero-sized read\n"); in check_stack_range_initialized()
7916 err = check_stack_access_within_bounds(env, regno, off, access_size, type); in check_stack_range_initialized()
7929 if (!env->bypass_spec_v1) { in check_stack_range_initialized()
7933 verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n", in check_stack_range_initialized()
7972 verbose(env, "potential write to dynptr at off=%d disallowed\n", i); in check_stack_range_initialized()
7987 verbose(env, "verifier bug: allocated_stack too small\n"); in check_stack_range_initialized()
7995 (*stype == STACK_INVALID && env->allow_uninit_stack)) { in check_stack_range_initialized()
8005 env->allow_ptr_leaks)) { in check_stack_range_initialized()
8007 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); in check_stack_range_initialized()
8015 verbose(env, "invalid read from stack R%d off %d+%d size %d\n", in check_stack_range_initialized()
8021 verbose(env, "invalid read from stack R%d var_off %s+%d size %d\n", in check_stack_range_initialized()
8029 mark_reg_read(env, &state->stack[spi].spilled_ptr, in check_stack_range_initialized()
8041 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, in check_helper_mem_access() argument
8046 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_helper_mem_access()
8052 return check_packet_access(env, regno, reg->off, access_size, in check_helper_mem_access()
8056 verbose(env, "R%d cannot write into %s\n", regno, in check_helper_mem_access()
8057 reg_type_str(env, reg->type)); in check_helper_mem_access()
8060 return check_mem_region_access(env, regno, reg->off, access_size, in check_helper_mem_access()
8063 if (check_map_access_type(env, regno, reg->off, access_size, access_type)) in check_helper_mem_access()
8065 return check_map_access(env, regno, reg->off, access_size, in check_helper_mem_access()
8070 verbose(env, "R%d cannot write into %s\n", regno, in check_helper_mem_access()
8071 reg_type_str(env, reg->type)); in check_helper_mem_access()
8075 return check_mem_region_access(env, regno, reg->off, in check_helper_mem_access()
8081 verbose(env, "R%d cannot write into %s\n", regno, in check_helper_mem_access()
8082 reg_type_str(env, reg->type)); in check_helper_mem_access()
8086 max_access = &env->prog->aux->max_rdonly_access; in check_helper_mem_access()
8088 max_access = &env->prog->aux->max_rdwr_access; in check_helper_mem_access()
8090 return check_buffer_access(env, reg, regno, reg->off, in check_helper_mem_access()
8095 env, in check_helper_mem_access()
8099 return check_ptr_to_btf_access(env, regs, regno, reg->off, in check_helper_mem_access()
8107 if (!env->ops->convert_ctx_access) { in check_helper_mem_access()
8114 return check_mem_access(env, env->insn_idx, regno, offset, BPF_B, in check_helper_mem_access()
8125 verbose(env, "R%d type=%s ", regno, in check_helper_mem_access()
8126 reg_type_str(env, reg->type)); in check_helper_mem_access()
8127 verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK)); in check_helper_mem_access()
8138 static int check_mem_size_reg(struct bpf_verifier_env *env, in check_mem_size_reg() argument
8165 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", in check_mem_size_reg()
8171 verbose(env, "R%d invalid zero-sized read: u64=[%lld,%lld]\n", in check_mem_size_reg()
8177 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", in check_mem_size_reg()
8181 err = check_helper_mem_access(env, regno - 1, reg->umax_value, in check_mem_size_reg()
8184 err = mark_chain_precision(env, regno); in check_mem_size_reg()
8188 static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in check_mem_reg() argument
8207 err = check_helper_mem_access(env, regno, mem_size, BPF_READ, true, NULL); in check_mem_reg()
8208 err = err ?: check_helper_mem_access(env, regno, mem_size, BPF_WRITE, true, NULL); in check_mem_reg()
8216 static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, in check_kfunc_mem_size_reg() argument
8219 struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1]; in check_kfunc_mem_size_reg()
8234 err = check_mem_size_reg(env, reg, regno, BPF_READ, true, &meta); in check_kfunc_mem_size_reg()
8235 err = err ?: check_mem_size_reg(env, reg, regno, BPF_WRITE, true, &meta); in check_kfunc_mem_size_reg()
8271 static int process_spin_lock(struct bpf_verifier_env *env, int regno, int flags) in process_spin_lock() argument
8275 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_spin_lock()
8276 struct bpf_verifier_state *cur = env->cur_state; in process_spin_lock()
8287 verbose(env, in process_spin_lock()
8295 verbose(env, in process_spin_lock()
8306 verbose(env, "%s '%s' has no valid %s_lock\n", map ? "map" : "local", in process_spin_lock()
8312 verbose(env, "off %lld doesn't point to 'struct %s_lock' that is at %d\n", in process_spin_lock()
8326 if (find_lock_state(env->cur_state, REF_TYPE_LOCK, 0, NULL)) { in process_spin_lock()
8327 verbose(env, in process_spin_lock()
8332 if (find_lock_state(env->cur_state, REF_TYPE_RES_LOCK | REF_TYPE_RES_LOCK_IRQ, reg->id, ptr)) { in process_spin_lock()
8333 verbose(env, "Acquiring the same lock again, AA deadlock detected\n"); in process_spin_lock()
8344 err = acquire_lock_state(env, env->insn_idx, type, reg->id, ptr); in process_spin_lock()
8346 verbose(env, "Failed to acquire lock state\n"); in process_spin_lock()
8359 verbose(env, "%s_unlock without taking a lock\n", lock_str); in process_spin_lock()
8370 verbose(env, "%s_unlock of different lock\n", lock_str); in process_spin_lock()
8374 verbose(env, "%s_unlock cannot be out of order\n", lock_str); in process_spin_lock()
8378 verbose(env, "%s_unlock of different lock\n", lock_str); in process_spin_lock()
8382 invalidate_non_owning_refs(env); in process_spin_lock()
8387 static int process_timer_func(struct bpf_verifier_env *env, int regno, in process_timer_func() argument
8390 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_timer_func()
8396 verbose(env, in process_timer_func()
8402 verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n", in process_timer_func()
8407 verbose(env, "map '%s' has no valid bpf_timer\n", map->name); in process_timer_func()
8411 verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n", in process_timer_func()
8416 verbose(env, "verifier bug. Two map pointers in a timer helper\n"); in process_timer_func()
8424 static int process_wq_func(struct bpf_verifier_env *env, int regno, in process_wq_func() argument
8427 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_wq_func()
8432 verbose(env, "off %lld doesn't point to 'struct bpf_wq' that is at %d\n", in process_wq_func()
8441 static int process_kptr_func(struct bpf_verifier_env *env, int regno, in process_kptr_func() argument
8444 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_kptr_func()
8455 verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n", in process_kptr_func()
8464 verbose(env, in process_kptr_func()
8471 verbose(env, "R%d has no valid kptr\n", regno); in process_kptr_func()
8478 verbose(env, "off=%d doesn't point to kptr\n", kptr_off); in process_kptr_func()
8482 verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off); in process_kptr_func()
8514 static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx, in process_dynptr_func() argument
8517 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_dynptr_func()
8521 verbose(env, in process_dynptr_func()
8531 verbose(env, "verifier internal error: misconfigured dynptr helper type flags\n"); in process_dynptr_func()
8553 if (!is_dynptr_reg_valid_uninit(env, reg)) { in process_dynptr_func()
8554 verbose(env, "Dynptr has to be an uninitialized dynptr\n"); in process_dynptr_func()
8560 err = check_mem_access(env, insn_idx, regno, in process_dynptr_func()
8566 err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, clone_ref_obj_id); in process_dynptr_func()
8570 verbose(env, "cannot pass pointer to const bpf_dynptr, the helper mutates it\n"); in process_dynptr_func()
8574 if (!is_dynptr_reg_valid_init(env, reg)) { in process_dynptr_func()
8575 verbose(env, in process_dynptr_func()
8582 if (!is_dynptr_type_expected(env, reg, arg_type & ~MEM_RDONLY)) { in process_dynptr_func()
8583 verbose(env, in process_dynptr_func()
8589 err = mark_dynptr_read(env, reg); in process_dynptr_func()
8594 static u32 iter_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int spi) in iter_ref_obj_id() argument
8596 struct bpf_func_state *state = func(env, reg); in iter_ref_obj_id()
8634 static int process_iter_arg(struct bpf_verifier_env *env, int regno, int insn_idx, in process_iter_arg() argument
8637 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_iter_arg()
8642 verbose(env, "arg#%d expected pointer to an iterator on stack\n", regno - 1); in process_iter_arg()
8654 verbose(env, "expected valid iter pointer as arg #%d\n", regno - 1); in process_iter_arg()
8662 if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) { in process_iter_arg()
8663 verbose(env, "expected uninitialized iter_%s as arg #%d\n", in process_iter_arg()
8669 err = check_mem_access(env, insn_idx, regno, in process_iter_arg()
8675 err = mark_stack_slots_iter(env, meta, reg, insn_idx, meta->btf, btf_id, nr_slots); in process_iter_arg()
8682 err = is_iter_reg_valid_init(env, reg, meta->btf, btf_id, nr_slots); in process_iter_arg()
8687 verbose(env, "expected an initialized iter_%s as arg #%d\n", in process_iter_arg()
8691 verbose(env, "expected an RCU CS when using %s\n", meta->func_name); in process_iter_arg()
8697 spi = iter_get_spi(env, reg, nr_slots); in process_iter_arg()
8701 err = mark_iter_read(env, reg, spi, nr_slots); in process_iter_arg()
8708 meta->ref_obj_id = iter_ref_obj_id(env, reg, spi); in process_iter_arg()
8711 err = unmark_stack_slots_iter(env, reg, nr_slots); in process_iter_arg()
8723 static struct bpf_verifier_state *find_prev_entry(struct bpf_verifier_env *env, in find_prev_entry() argument
8732 head = explored_state(env, insn_idx); in find_prev_entry()
8747 static void reset_idmap_scratch(struct bpf_verifier_env *env);
8752 static void maybe_widen_reg(struct bpf_verifier_env *env, in maybe_widen_reg() argument
8762 __mark_reg_unknown(env, rcur); in maybe_widen_reg()
8765 static int widen_imprecise_scalars(struct bpf_verifier_env *env, in widen_imprecise_scalars() argument
8772 reset_idmap_scratch(env); in widen_imprecise_scalars()
8778 maybe_widen_reg(env, in widen_imprecise_scalars()
8781 &env->idmap_scratch); in widen_imprecise_scalars()
8788 maybe_widen_reg(env, in widen_imprecise_scalars()
8791 &env->idmap_scratch); in widen_imprecise_scalars()
8884 static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx, in process_iter_next_call() argument
8887 struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st; in process_iter_next_call()
8897 verbose(env, "verifier internal error: unexpected iterator state %d (%s)\n", in process_iter_next_call()
8908 verbose(env, "bug: bad parent state for iter next call"); in process_iter_next_call()
8915 prev_st = find_prev_entry(env, cur_st->parent, insn_idx); in process_iter_next_call()
8917 queued_st = push_stack(env, insn_idx + 1, insn_idx, false); in process_iter_next_call()
8925 widen_imprecise_scalars(env, prev_st, queued_st); in process_iter_next_call()
8934 __mark_reg_const_zero(env, &cur_fr->regs[BPF_REG_0]); in process_iter_next_call()
8961 static int resolve_map_arg_type(struct bpf_verifier_env *env, in resolve_map_arg_type() argument
8967 verbose(env, "invalid map_ptr to access map->type\n"); in resolve_map_arg_type()
8977 verbose(env, "invalid arg_type for sockmap/sockhash\n"); in resolve_map_arg_type()
9102 static int check_reg_type(struct bpf_verifier_env *env, u32 regno, in check_reg_type() argument
9107 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_reg_type()
9114 verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); in check_reg_type()
9152 verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type)); in check_reg_type()
9154 verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); in check_reg_type()
9155 verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); in check_reg_type()
9164 verbose(env, in check_reg_type()
9167 regno, reg_type_str(env, reg->type)); in check_reg_type()
9190 verbose(env, "Possibly NULL pointer passed to helper arg%d\n", regno); in check_reg_type()
9196 verbose(env, "verifier internal error: missing arg compatible BTF ID\n"); in check_reg_type()
9203 if (map_kptr_match_type(env, meta->kptr_field, reg, regno)) in check_reg_type()
9207 verbose(env, "verifier internal error:"); in check_reg_type()
9208 verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n", in check_reg_type()
9213 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, in check_reg_type()
9216 verbose(env, "R%d is of type %s but %s is expected\n", in check_reg_type()
9228 verbose(env, "verifier internal error: unimplemented handling of MEM_ALLOC\n"); in check_reg_type()
9233 if (map_kptr_match_type(env, meta->kptr_field, reg, regno)) in check_reg_type()
9243 verbose(env, "verifier internal error: invalid PTR_TO_BTF_ID register for type match\n"); in check_reg_type()
9266 static int check_func_arg_reg_off(struct bpf_verifier_env *env, in check_func_arg_reg_off() argument
9293 verbose(env, "R%d must have zero offset when passed to release func or trusted arg to kfunc\n", in check_func_arg_reg_off()
9297 return __check_ptr_off_reg(env, reg, regno, false); in check_func_arg_reg_off()
9331 return __check_ptr_off_reg(env, reg, regno, true); in check_func_arg_reg_off()
9333 return __check_ptr_off_reg(env, reg, regno, false); in check_func_arg_reg_off()
9337 static struct bpf_reg_state *get_dynptr_arg_reg(struct bpf_verifier_env *env, in get_dynptr_arg_reg() argument
9347 verbose(env, "verifier internal error: multiple dynptr args\n"); in get_dynptr_arg_reg()
9354 verbose(env, "verifier internal error: no dynptr arg found\n"); in get_dynptr_arg_reg()
9359 static int dynptr_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in dynptr_id() argument
9361 struct bpf_func_state *state = func(env, reg); in dynptr_id()
9366 spi = dynptr_get_spi(env, reg); in dynptr_id()
9372 static int dynptr_ref_obj_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in dynptr_ref_obj_id() argument
9374 struct bpf_func_state *state = func(env, reg); in dynptr_ref_obj_id()
9379 spi = dynptr_get_spi(env, reg); in dynptr_ref_obj_id()
9385 static enum bpf_dynptr_type dynptr_get_type(struct bpf_verifier_env *env, in dynptr_get_type() argument
9388 struct bpf_func_state *state = func(env, reg); in dynptr_get_type()
9396 verbose(env, "verifier internal error: invalid spi when querying dynptr type\n"); in dynptr_get_type()
9403 static int check_reg_const_str(struct bpf_verifier_env *env, in check_reg_const_str() argument
9416 verbose(env, "R%d does not point to a readonly map'\n", regno); in check_reg_const_str()
9421 verbose(env, "R%d is not a constant address'\n", regno); in check_reg_const_str()
9426 verbose(env, "no direct value access support for this map type\n"); in check_reg_const_str()
9430 err = check_map_access(env, regno, reg->off, in check_reg_const_str()
9439 verbose(env, "direct value access on string failed\n"); in check_reg_const_str()
9445 verbose(env, "string is not zero-terminated\n"); in check_reg_const_str()
9452 static int get_constant_map_key(struct bpf_verifier_env *env, in get_constant_map_key() argument
9457 struct bpf_func_state *state = func(env, key); in get_constant_map_key()
9466 if (!env->bpf_capable) in get_constant_map_key()
9503 bt_set_frame_slot(&env->bt, key->frameno, spi); in get_constant_map_key()
9504 err = mark_chain_precision_batch(env); in get_constant_map_key()
9514 static int check_func_arg(struct bpf_verifier_env *env, u32 arg, in check_func_arg() argument
9520 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in check_func_arg()
9530 err = check_reg_arg(env, regno, SRC_OP); in check_func_arg()
9535 if (is_pointer_value(env, regno)) { in check_func_arg()
9536 verbose(env, "R%d leaks addr into helper function\n", in check_func_arg()
9544 !may_access_direct_pkt_data(env, meta, BPF_READ)) { in check_func_arg()
9545 verbose(env, "helper access to the packet is not allowed\n"); in check_func_arg()
9550 err = resolve_map_arg_type(env, meta, &arg_type); in check_func_arg()
9566 err = check_reg_type(env, regno, arg_type, arg_btf_id, meta); in check_func_arg()
9570 err = check_func_arg_reg_off(env, reg, regno, arg_type); in check_func_arg()
9577 struct bpf_func_state *state = func(env, reg); in check_func_arg()
9586 spi = dynptr_get_spi(env, reg); in check_func_arg()
9588 verbose(env, "arg %d is an unacquired reference\n", regno); in check_func_arg()
9592 verbose(env, "cannot release unowned const bpf_dynptr\n"); in check_func_arg()
9596 verbose(env, "R%d must be referenced when passed to release function\n", in check_func_arg()
9601 verbose(env, "verifier internal error: more than one release argument\n"); in check_func_arg()
9609 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", in check_func_arg()
9635 verbose(env, in check_func_arg()
9655 verbose(env, "invalid map_ptr to access map->key\n"); in check_func_arg()
9659 err = check_helper_mem_access(env, regno, key_size, BPF_READ, false, NULL); in check_func_arg()
9663 err = get_constant_map_key(env, reg, key_size, &meta->const_map_key); in check_func_arg()
9682 verbose(env, "invalid map_ptr to access map->value\n"); in check_func_arg()
9686 err = check_helper_mem_access(env, regno, meta->map_ptr->value_size, in check_func_arg()
9692 verbose(env, "Helper has invalid btf_id in R%d\n", regno); in check_func_arg()
9699 if (in_rbtree_lock_required_cb(env)) { in check_func_arg()
9700 verbose(env, "can't spin_{lock,unlock} in rbtree cb\n"); in check_func_arg()
9704 err = process_spin_lock(env, regno, PROCESS_SPIN_LOCK); in check_func_arg()
9708 err = process_spin_lock(env, regno, 0); in check_func_arg()
9712 verbose(env, "verifier internal error\n"); in check_func_arg()
9717 err = process_timer_func(env, regno, meta); in check_func_arg()
9730 err = check_helper_mem_access(env, regno, fn->arg_size[arg], in check_func_arg()
9736 err = check_ptr_alignment(env, reg, 0, fn->arg_size[arg], true); in check_func_arg()
9740 err = check_mem_size_reg(env, reg, regno, in check_func_arg()
9746 err = check_mem_size_reg(env, reg, regno, in check_func_arg()
9752 err = process_dynptr_func(env, regno, insn_idx, arg_type, 0); in check_func_arg()
9758 verbose(env, "R%d is not a known constant'\n", in check_func_arg()
9763 err = mark_chain_precision(env, regno); in check_func_arg()
9769 err = check_reg_const_str(env, reg, regno); in check_func_arg()
9775 err = process_kptr_func(env, regno, meta); in check_func_arg()
9784 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) in may_update_sockmap() argument
9786 enum bpf_attach_type eatype = env->prog->expected_attach_type; in may_update_sockmap()
9787 enum bpf_prog_type type = resolve_prog_type(env->prog); in may_update_sockmap()
9818 verbose(env, "cannot update sockmap in this context\n"); in may_update_sockmap()
9822 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env) in allow_tail_call_in_subprogs() argument
9824 return env->prog->jit_requested && in allow_tail_call_in_subprogs()
9828 static int check_map_func_compatibility(struct bpf_verifier_env *env, in check_map_func_compatibility() argument
9904 !may_update_sockmap(env, func_id)) in check_map_func_compatibility()
9913 !may_update_sockmap(env, func_id)) in check_map_func_compatibility()
9965 if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) { in check_map_func_compatibility()
9966 verbose(env, "mixing of tail_calls and bpf-to-bpf calls is not supported\n"); in check_map_func_compatibility()
10074 verbose(env, "cannot pass map_type %d into func %s#%d\n", in check_map_func_compatibility()
10166 static void clear_all_pkt_pointers(struct bpf_verifier_env *env) in clear_all_pkt_pointers() argument
10171 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ in clear_all_pkt_pointers()
10173 mark_reg_invalid(env, reg); in clear_all_pkt_pointers()
10223 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id) in release_reference() argument
10225 struct bpf_verifier_state *vstate = env->cur_state; in release_reference()
10236 mark_reg_invalid(env, reg); in release_reference()
10242 static void invalidate_non_owning_refs(struct bpf_verifier_env *env) in invalidate_non_owning_refs() argument
10247 bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ in invalidate_non_owning_refs()
10249 mark_reg_invalid(env, reg); in invalidate_non_owning_refs()
10253 static void clear_caller_saved_regs(struct bpf_verifier_env *env, in clear_caller_saved_regs() argument
10260 mark_reg_not_init(env, regs, caller_saved[i]); in clear_caller_saved_regs()
10261 __check_reg_arg(env, regs, caller_saved[i], DST_OP_NO_MARK); in clear_caller_saved_regs()
10265 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,
10270 static int set_callee_state(struct bpf_verifier_env *env,
10274 static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int callsite, in setup_func_entry() argument
10282 verbose(env, "the call stack of %d frames is too deep\n", in setup_func_entry()
10288 verbose(env, "verifier bug. Frame %d already allocated\n", in setup_func_entry()
10303 init_func_state(env, callee, in setup_func_entry()
10308 err = set_callee_state_cb(env, caller, callee, callsite); in setup_func_entry()
10323 static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, in btf_check_func_arg_match() argument
10327 struct bpf_subprog_info *sub = subprog_info(env, subprog); in btf_check_func_arg_match()
10328 struct bpf_verifier_log *log = &env->log; in btf_check_func_arg_match()
10332 ret = btf_prepare_func_args(env, subprog); in btf_check_func_arg_match()
10350 ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE); in btf_check_func_arg_match()
10361 ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE); in btf_check_func_arg_match()
10364 if (check_mem_reg(env, reg, regno, arg->mem_size)) in btf_check_func_arg_match()
10383 ret = check_func_arg_reg_off(env, reg, regno, ARG_PTR_TO_DYNPTR); in btf_check_func_arg_match()
10387 ret = process_dynptr_func(env, regno, -1, arg->arg_type, 0); in btf_check_func_arg_match()
10398 err = check_reg_type(env, regno, arg->arg_type, &arg->btf_id, &meta); in btf_check_func_arg_match()
10399 err = err ?: check_func_arg_reg_off(env, reg, regno, arg->arg_type); in btf_check_func_arg_match()
10419 static int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, in btf_check_subprog_call() argument
10422 struct bpf_prog *prog = env->prog; in btf_check_subprog_call()
10437 err = btf_check_func_arg_match(env, subprog, btf, regs); in btf_check_subprog_call()
10447 static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in push_callback_call() argument
10451 struct bpf_verifier_state *state = env->cur_state, *callback_state; in push_callback_call()
10456 err = btf_check_subprog_call(env, subprog, caller->regs); in push_callback_call()
10464 env->subprog_info[subprog].is_cb = true; in push_callback_call()
10467 verbose(env, "verifier bug: kfunc %s#%d not marked as callback-calling\n", in push_callback_call()
10472 verbose(env, "verifier bug: helper %s#%d not marked as callback-calling\n", in push_callback_call()
10481 env->subprog_info[subprog].is_async_cb = true; in push_callback_call()
10482 async_cb = push_async_cb(env, env->subprog_info[subprog].start, in push_callback_call()
10491 err = set_callee_state_cb(env, caller, callee, insn_idx); in push_callback_call()
10501 callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false); in push_callback_call()
10505 err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb, in push_callback_call()
10516 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in check_func_call() argument
10519 struct bpf_verifier_state *state = env->cur_state; in check_func_call()
10524 subprog = find_subprog(env, target_insn); in check_func_call()
10526 verbose(env, "verifier bug. No program starts at insn %d\n", target_insn); in check_func_call()
10531 err = btf_check_subprog_call(env, subprog, caller->regs); in check_func_call()
10534 if (subprog_is_global(env, subprog)) { in check_func_call()
10535 const char *sub_name = subprog_name(env, subprog); in check_func_call()
10537 if (env->cur_state->active_locks) { in check_func_call()
10538 verbose(env, "global function calls are not allowed while holding a lock,\n" in check_func_call()
10543 if (env->subprog_info[subprog].might_sleep && in check_func_call()
10544 (env->cur_state->active_rcu_lock || env->cur_state->active_preempt_locks || in check_func_call()
10545 env->cur_state->active_irq_id || !in_sleepable(env))) { in check_func_call()
10546 verbose(env, "global functions that may sleep are not allowed in non-sleepable context,\n" in check_func_call()
10553 verbose(env, "Caller passes invalid args into func#%d ('%s')\n", in check_func_call()
10558 verbose(env, "Func#%d ('%s') is global and assumed valid.\n", in check_func_call()
10560 if (env->subprog_info[subprog].changes_pkt_data) in check_func_call()
10561 clear_all_pkt_pointers(env); in check_func_call()
10563 subprog_aux(env, subprog)->called = true; in check_func_call()
10564 clear_caller_saved_regs(env, caller->regs); in check_func_call()
10567 mark_reg_unknown(env, caller->regs, BPF_REG_0); in check_func_call()
10577 err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state); in check_func_call()
10581 clear_caller_saved_regs(env, caller->regs); in check_func_call()
10584 *insn_idx = env->subprog_info[subprog].start - 1; in check_func_call()
10586 if (env->log.level & BPF_LOG_LEVEL) { in check_func_call()
10587 verbose(env, "caller:\n"); in check_func_call()
10588 print_verifier_state(env, state, caller->frameno, true); in check_func_call()
10589 verbose(env, "callee:\n"); in check_func_call()
10590 print_verifier_state(env, state, state->curframe, true); in check_func_call()
10596 int map_set_for_each_callback_args(struct bpf_verifier_env *env, in map_set_for_each_callback_args() argument
10619 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in map_set_for_each_callback_args()
10623 static int set_callee_state(struct bpf_verifier_env *env, in set_callee_state() argument
10637 static int set_map_elem_callback_state(struct bpf_verifier_env *env, in set_map_elem_callback_state() argument
10642 struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx]; in set_map_elem_callback_state()
10650 verbose(env, "callback function not allowed for map\n"); in set_map_elem_callback_state()
10654 err = map->ops->map_set_for_each_callback_args(env, caller, callee); in set_map_elem_callback_state()
10663 static int set_loop_callback_state(struct bpf_verifier_env *env, in set_loop_callback_state() argument
10676 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); in set_loop_callback_state()
10677 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_loop_callback_state()
10678 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_loop_callback_state()
10685 static int set_timer_callback_state(struct bpf_verifier_env *env, in set_timer_callback_state() argument
10708 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_timer_callback_state()
10709 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_timer_callback_state()
10715 static int set_find_vma_callback_state(struct bpf_verifier_env *env, in set_find_vma_callback_state() argument
10736 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_find_vma_callback_state()
10737 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_find_vma_callback_state()
10743 static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env, in set_user_ringbuf_callback_state() argument
10752 __mark_reg_not_init(env, &callee->regs[BPF_REG_0]); in set_user_ringbuf_callback_state()
10753 mark_dynptr_cb_reg(env, &callee->regs[BPF_REG_1], BPF_DYNPTR_TYPE_LOCAL); in set_user_ringbuf_callback_state()
10757 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); in set_user_ringbuf_callback_state()
10758 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_user_ringbuf_callback_state()
10759 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_user_ringbuf_callback_state()
10766 static int set_rbtree_add_callback_state(struct bpf_verifier_env *env, in set_rbtree_add_callback_state() argument
10786 ref_set_non_owning(env, &callee->regs[BPF_REG_1]); in set_rbtree_add_callback_state()
10788 ref_set_non_owning(env, &callee->regs[BPF_REG_2]); in set_rbtree_add_callback_state()
10790 __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); in set_rbtree_add_callback_state()
10791 __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); in set_rbtree_add_callback_state()
10792 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); in set_rbtree_add_callback_state()
10804 static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env) in in_rbtree_lock_required_cb() argument
10806 struct bpf_verifier_state *state = env->cur_state; in in_rbtree_lock_required_cb()
10807 struct bpf_insn *insn = env->prog->insnsi; in in_rbtree_lock_required_cb()
10832 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) in prepare_func_exit() argument
10834 struct bpf_verifier_state *state = env->cur_state, *prev_st; in prepare_func_exit()
10849 verbose(env, "cannot return stack pointer to the caller\n"); in prepare_func_exit()
10856 verbose(env, "R0 not a scalar value\n"); in prepare_func_exit()
10861 err = mark_reg_read(env, r0, r0->parent, REG_LIVE_READ64); in prepare_func_exit()
10862 err = err ?: mark_chain_precision(env, BPF_REG_0); in prepare_func_exit()
10868 verbose_invalid_scalar(env, r0, callee->callback_ret_range, in prepare_func_exit()
10872 if (!calls_callback(env, callee->callsite)) { in prepare_func_exit()
10873 verbose(env, "BUG: in callback at %d, callsite %d !calls_callback\n", in prepare_func_exit()
10892 if (env->log.level & BPF_LOG_LEVEL) { in prepare_func_exit()
10893 verbose(env, "returning from callee:\n"); in prepare_func_exit()
10894 print_verifier_state(env, state, callee->frameno, true); in prepare_func_exit()
10895 verbose(env, "to caller at %d:\n", *insn_idx); in prepare_func_exit()
10896 print_verifier_state(env, state, caller->frameno, true); in prepare_func_exit()
10914 prev_st = in_callback_fn ? find_prev_entry(env, state, *insn_idx) : NULL; in prepare_func_exit()
10916 err = widen_imprecise_scalars(env, prev_st, state); in prepare_func_exit()
10923 static int do_refine_retval_range(struct bpf_verifier_env *env, in do_refine_retval_range() argument
10958 return reg_bounds_sanity_check(env, ret_reg, "retval"); in do_refine_retval_range()
10962 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, in record_func_map() argument
10965 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; in record_func_map()
10981 verbose(env, "kernel subsystem misconfigured verifier\n"); in record_func_map()
10994 verbose(env, "write into map forbidden\n"); in record_func_map()
11008 record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, in record_func_key() argument
11011 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; in record_func_key()
11012 struct bpf_reg_state *regs = cur_regs(env), *reg; in record_func_key()
11020 verbose(env, "kernel subsystem misconfigured verifier\n"); in record_func_key()
11033 err = mark_chain_precision(env, BPF_REG_3); in record_func_key()
11044 static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exit) in check_reference_leak() argument
11046 struct bpf_verifier_state *state = env->cur_state; in check_reference_leak()
11047 enum bpf_prog_type type = resolve_prog_type(env->prog); in check_reference_leak()
11048 struct bpf_reg_state *reg = reg_state(env, BPF_REG_0); in check_reference_leak()
11052 if (!exception_exit && cur_func(env)->frameno) in check_reference_leak()
11064 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", in check_reference_leak()
11071 static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit, bool check_lock, … in check_resource_leak() argument
11075 if (check_lock && env->cur_state->active_locks) { in check_resource_leak()
11076 verbose(env, "%s cannot be used inside bpf_spin_lock-ed region\n", prefix); in check_resource_leak()
11080 err = check_reference_leak(env, exception_exit); in check_resource_leak()
11082 verbose(env, "%s would lead to reference leak\n", prefix); in check_resource_leak()
11086 if (check_lock && env->cur_state->active_irq_id) { in check_resource_leak()
11087 verbose(env, "%s cannot be used inside bpf_local_irq_save-ed region\n", prefix); in check_resource_leak()
11091 if (check_lock && env->cur_state->active_rcu_lock) { in check_resource_leak()
11092 verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix); in check_resource_leak()
11096 if (check_lock && env->cur_state->active_preempt_locks) { in check_resource_leak()
11097 verbose(env, "%s cannot be used inside bpf_preempt_disable-ed region\n", prefix); in check_resource_leak()
11104 static int check_bpf_snprintf_call(struct bpf_verifier_env *env, in check_bpf_snprintf_call() argument
11127 verbose(env, "verifier bug\n"); in check_bpf_snprintf_call()
11137 verbose(env, "Invalid format string\n"); in check_bpf_snprintf_call()
11142 static int check_get_func_ip(struct bpf_verifier_env *env) in check_get_func_ip() argument
11144 enum bpf_prog_type type = resolve_prog_type(env->prog); in check_get_func_ip()
11148 if (!bpf_prog_has_trampoline(env->prog)) { in check_get_func_ip()
11149 verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n", in check_get_func_ip()
11158 verbose(env, "func %s#%d not supported for program type %d\n", in check_get_func_ip()
11163 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) in cur_aux() argument
11165 return &env->insn_aux_data[env->insn_idx]; in cur_aux()
11168 static bool loop_flag_is_zero(struct bpf_verifier_env *env) in loop_flag_is_zero() argument
11170 struct bpf_reg_state *regs = cur_regs(env); in loop_flag_is_zero()
11175 mark_chain_precision(env, BPF_REG_4); in loop_flag_is_zero()
11180 static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno) in update_loop_inline_state() argument
11182 struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state; in update_loop_inline_state()
11186 state->fit_for_inline = loop_flag_is_zero(env); in update_loop_inline_state()
11194 state->fit_for_inline = (loop_flag_is_zero(env) && in update_loop_inline_state()
11213 static int get_helper_proto(struct bpf_verifier_env *env, int func_id, in get_helper_proto() argument
11219 if (!env->ops->get_func_proto) in get_helper_proto()
11222 *ptr = env->ops->get_func_proto(func_id, env->prog); in get_helper_proto()
11226 static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in check_helper_call() argument
11229 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); in check_helper_call()
11242 err = get_helper_proto(env, insn->imm, &fn); in check_helper_call()
11244 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), func_id); in check_helper_call()
11249 verbose(env, "program of this type cannot use helper %s#%d\n", in check_helper_call()
11255 if (!env->prog->gpl_compatible && fn->gpl_only) { in check_helper_call()
11256 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); in check_helper_call()
11260 if (fn->allowed && !fn->allowed(env->prog)) { in check_helper_call()
11261 verbose(env, "helper call is not allowed in probe\n"); in check_helper_call()
11265 if (!in_sleepable(env) && fn->might_sleep) { in check_helper_call()
11266 verbose(env, "helper call might sleep in a non-sleepable prog\n"); in check_helper_call()
11273 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", in check_helper_call()
11283 verbose(env, "kernel subsystem misconfigured func %s#%d\n", in check_helper_call()
11288 if (env->cur_state->active_rcu_lock) { in check_helper_call()
11290 verbose(env, "sleepable helper %s#%d in rcu_read_lock region\n", in check_helper_call()
11295 if (in_sleepable(env) && is_storage_get_function(func_id)) in check_helper_call()
11296 env->insn_aux_data[insn_idx].storage_get_func_atomic = true; in check_helper_call()
11299 if (env->cur_state->active_preempt_locks) { in check_helper_call()
11301 verbose(env, "sleepable helper %s#%d in non-preemptible region\n", in check_helper_call()
11306 if (in_sleepable(env) && is_storage_get_function(func_id)) in check_helper_call()
11307 env->insn_aux_data[insn_idx].storage_get_func_atomic = true; in check_helper_call()
11310 if (env->cur_state->active_irq_id) { in check_helper_call()
11312 verbose(env, "sleepable helper %s#%d in IRQ-disabled region\n", in check_helper_call()
11317 if (in_sleepable(env) && is_storage_get_function(func_id)) in check_helper_call()
11318 env->insn_aux_data[insn_idx].storage_get_func_atomic = true; in check_helper_call()
11324 err = check_func_arg(env, i, &meta, fn, insn_idx); in check_helper_call()
11329 err = record_func_map(env, &meta, func_id, insn_idx); in check_helper_call()
11333 err = record_func_key(env, &meta, func_id, insn_idx); in check_helper_call()
11341 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, in check_helper_call()
11347 regs = cur_regs(env); in check_helper_call()
11357 verbose(env, "verifier internal error: CONST_PTR_TO_DYNPTR cannot be released\n"); in check_helper_call()
11360 err = unmark_stack_slots_dynptr(env, ®s[meta.release_regno]); in check_helper_call()
11363 bool in_rcu = in_rcu_cs(env); in check_helper_call()
11367 err = release_reference_nomark(env->cur_state, ref_obj_id); in check_helper_call()
11369 bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ in check_helper_call()
11376 mark_reg_invalid(env, reg); in check_helper_call()
11382 err = release_reference(env, meta.ref_obj_id); in check_helper_call()
11390 verbose(env, "func %s#%d reference has not been acquired before\n", in check_helper_call()
11398 err = check_resource_leak(env, false, true, "tail_call"); in check_helper_call()
11407 verbose(env, "get_local_storage() doesn't support non-zero flags\n"); in check_helper_call()
11412 err = push_callback_call(env, insn, insn_idx, meta.subprogno, in check_helper_call()
11416 err = push_callback_call(env, insn, insn_idx, meta.subprogno, in check_helper_call()
11420 err = push_callback_call(env, insn, insn_idx, meta.subprogno, in check_helper_call()
11424 err = check_bpf_snprintf_call(env, regs); in check_helper_call()
11427 update_loop_inline_state(env, meta.subprogno); in check_helper_call()
11431 err = mark_chain_precision(env, BPF_REG_1); in check_helper_call()
11434 if (cur_func(env)->callback_depth < regs[BPF_REG_1].umax_value) { in check_helper_call()
11435 err = push_callback_call(env, insn, insn_idx, meta.subprogno, in check_helper_call()
11438 cur_func(env)->callback_depth = 0; in check_helper_call()
11439 if (env->log.level & BPF_LOG_LEVEL2) in check_helper_call()
11440 verbose(env, "frame%d bpf_loop iteration limit reached\n", in check_helper_call()
11441 env->cur_state->curframe); in check_helper_call()
11446 verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n", in check_helper_call()
11447 reg_type_str(env, regs[BPF_REG_1].type)); in check_helper_call()
11453 env->prog->expected_attach_type == BPF_LSM_CGROUP) { in check_helper_call()
11454 if (!env->prog->aux->attach_func_proto->type) { in check_helper_call()
11458 verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); in check_helper_call()
11468 reg = get_dynptr_arg_reg(env, fn, regs); in check_helper_call()
11474 verbose(env, "verifier internal error: meta.dynptr_id already set\n"); in check_helper_call()
11478 verbose(env, "verifier internal error: meta.ref_obj_id already set\n"); in check_helper_call()
11482 id = dynptr_id(env, reg); in check_helper_call()
11484 verbose(env, "verifier internal error: failed to obtain dynptr id\n"); in check_helper_call()
11488 ref_obj_id = dynptr_ref_obj_id(env, reg); in check_helper_call()
11490 verbose(env, "verifier internal error: failed to obtain dynptr ref_obj_id\n"); in check_helper_call()
11504 reg = get_dynptr_arg_reg(env, fn, regs); in check_helper_call()
11508 dynptr_type = dynptr_get_type(env, reg); in check_helper_call()
11529 verbose(env, "Helper has invalid btf/btf_id in R1\n"); in check_helper_call()
11533 env->insn_aux_data[insn_idx].call_with_percpu_alloc_ptr = true; in check_helper_call()
11538 err = push_callback_call(env, insn, insn_idx, meta.subprogno, in check_helper_call()
11548 mark_reg_not_init(env, regs, caller_saved[i]); in check_helper_call()
11549 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); in check_helper_call()
11562 mark_reg_unknown(env, regs, BPF_REG_0); in check_helper_call()
11569 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
11575 verbose(env, in check_helper_call()
11591 regs[BPF_REG_0].id = ++env->id_gen; in check_helper_call()
11595 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
11599 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
11603 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
11607 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
11615 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
11626 verbose(env, "unable to resolve the size of type '%s': %ld\n", in check_helper_call()
11655 mark_reg_known_zero(env, regs, BPF_REG_0); in check_helper_call()
11667 verbose(env, "verifier internal error:"); in check_helper_call()
11668 verbose(env, "func %s has non-overwritten BPF_PTR_POISON return type\n", in check_helper_call()
11676 verbose(env, "invalid return type %u of func %s#%d\n", in check_helper_call()
11686 verbose(env, "unknown return type %u of func %s#%d\n", in check_helper_call()
11692 regs[BPF_REG_0].id = ++env->id_gen; in check_helper_call()
11695 verbose(env, "verifier internal error: func %s#%d sets ref_obj_id more than once\n", in check_helper_call()
11707 int id = acquire_reference(env, insn_idx); in check_helper_call()
11717 err = do_refine_retval_range(env, regs, fn->ret_type, func_id, &meta); in check_helper_call()
11721 err = check_map_func_compatibility(env, meta.map_ptr, func_id); in check_helper_call()
11727 !env->prog->has_callchain_buf) { in check_helper_call()
11738 verbose(env, err_str, func_id_name(func_id), func_id); in check_helper_call()
11742 env->prog->has_callchain_buf = true; in check_helper_call()
11746 env->prog->call_get_stack = true; in check_helper_call()
11749 if (check_get_func_ip(env)) in check_helper_call()
11751 env->prog->call_get_func_ip = true; in check_helper_call()
11755 clear_all_pkt_pointers(env); in check_helper_call()
11762 static void __mark_btf_func_reg_size(struct bpf_verifier_env *env, struct bpf_reg_state *regs, in __mark_btf_func_reg_size() argument
11771 DEF_NOT_SUBREG : env->insn_idx + 1; in __mark_btf_func_reg_size()
11775 mark_insn_zext(env, reg); in __mark_btf_func_reg_size()
11776 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); in __mark_btf_func_reg_size()
11778 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32); in __mark_btf_func_reg_size()
11783 static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno, in mark_btf_func_reg_size() argument
11786 return __mark_btf_func_reg_size(env, cur_regs(env), regno, reg_size); in mark_btf_func_reg_size()
11990 static bool is_kfunc_arg_callback(struct bpf_verifier_env *env, const struct btf *btf, in is_kfunc_arg_callback() argument
12003 static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, in __btf_type_is_scalar_struct() argument
12020 verbose(env, "max struct nesting depth exceeded\n"); in __btf_type_is_scalar_struct()
12023 if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) in __btf_type_is_scalar_struct()
12229 get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, in get_kfunc_ptr_arg_type() argument
12236 struct bpf_reg_state *regs = cur_regs(env); in get_kfunc_ptr_arg_type()
12248 if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), argno)) in get_kfunc_ptr_arg_type()
12295 verbose(env, "kernel function %s args#%d pointer type %s %s is not supported\n", in get_kfunc_ptr_arg_type()
12302 if (is_kfunc_arg_callback(env, meta->btf, &args[argno])) in get_kfunc_ptr_arg_type()
12315 if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) && in get_kfunc_ptr_arg_type()
12317 verbose(env, "arg#%d pointer type %s %s must point to %sscalar, or struct with scalar\n", in get_kfunc_ptr_arg_type()
12324 static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, in process_kf_arg_ptr_to_btf_id() argument
12372 btf_type_ids_nocast_alias(&env->log, reg_btf, reg_ref_id, meta->btf, ref_id)) in process_kf_arg_ptr_to_btf_id()
12381 …struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->off, meta->btf, ref_id, st… in process_kf_arg_ptr_to_btf_id()
12388 …verbose(env, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\… in process_kf_arg_ptr_to_btf_id()
12396 static int process_irq_flag(struct bpf_verifier_env *env, int regno, in process_irq_flag() argument
12399 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; in process_irq_flag()
12414 verbose(env, "verifier internal error: unknown irq flags kfunc\n"); in process_irq_flag()
12419 if (!is_irq_flag_reg_valid_uninit(env, reg)) { in process_irq_flag()
12420 verbose(env, "expected uninitialized irq flag as arg#%d\n", regno - 1); in process_irq_flag()
12424 err = check_mem_access(env, env->insn_idx, regno, 0, BPF_DW, BPF_WRITE, -1, false, false); in process_irq_flag()
12428 err = mark_stack_slot_irq_flag(env, meta, reg, env->insn_idx, kfunc_class); in process_irq_flag()
12432 err = is_irq_flag_reg_valid_init(env, reg); in process_irq_flag()
12434 verbose(env, "expected an initialized irq flag as arg#%d\n", regno - 1); in process_irq_flag()
12438 err = mark_irq_flag_read(env, reg); in process_irq_flag()
12442 err = unmark_stack_slot_irq_flag(env, reg, kfunc_class); in process_irq_flag()
12450 static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in ref_set_non_owning() argument
12454 if (!env->cur_state->active_locks) { in ref_set_non_owning()
12455 verbose(env, "verifier internal error: ref_set_non_owning w/o active lock\n"); in ref_set_non_owning()
12460 verbose(env, "verifier internal error: NON_OWN_REF already set\n"); in ref_set_non_owning()
12471 static int ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 ref_obj_id) in ref_convert_owning_non_owning() argument
12473 struct bpf_verifier_state *state = env->cur_state; in ref_convert_owning_non_owning()
12479 verbose(env, "verifier internal error: ref_obj_id is zero for " in ref_convert_owning_non_owning()
12491 bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ in ref_convert_owning_non_owning()
12494 ref_set_non_owning(env, reg); in ref_convert_owning_non_owning()
12500 verbose(env, "verifier internal error: ref state missing for ref_obj_id\n"); in ref_convert_owning_non_owning()
12548 static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg) in check_reg_allocation_locked() argument
12562 verbose(env, "verifier internal error: unknown reg type for lock check\n"); in check_reg_allocation_locked()
12567 if (!env->cur_state->active_locks) in check_reg_allocation_locked()
12569 s = find_lock_state(env->cur_state, REF_TYPE_LOCK_MASK, id, ptr); in check_reg_allocation_locked()
12571 verbose(env, "held lock and object are not in the same allocation\n"); in check_reg_allocation_locked()
12651 static bool check_kfunc_is_graph_root_api(struct bpf_verifier_env *env, in check_kfunc_is_graph_root_api() argument
12665 verbose(env, "verifier internal error: unexpected graph root argument type %s\n", in check_kfunc_is_graph_root_api()
12671 verbose(env, "verifier internal error: %s head arg for unknown kfunc\n", in check_kfunc_is_graph_root_api()
12676 static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env, in check_kfunc_is_graph_node_api() argument
12692 verbose(env, "verifier internal error: unexpected graph node argument type %s\n", in check_kfunc_is_graph_node_api()
12698 verbose(env, "verifier internal error: %s node arg for unknown kfunc\n", in check_kfunc_is_graph_node_api()
12704 __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env, in __process_kf_arg_ptr_to_graph_root() argument
12716 verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n"); in __process_kf_arg_ptr_to_graph_root()
12720 if (!check_kfunc_is_graph_root_api(env, head_field_type, meta->func_id)) in __process_kf_arg_ptr_to_graph_root()
12725 verbose(env, in __process_kf_arg_ptr_to_graph_root()
12735 verbose(env, "%s not found at offset=%u\n", head_type_name, head_off); in __process_kf_arg_ptr_to_graph_root()
12740 if (check_reg_allocation_locked(env, reg)) { in __process_kf_arg_ptr_to_graph_root()
12741 verbose(env, "bpf_spin_lock at off=%d must be held for %s\n", in __process_kf_arg_ptr_to_graph_root()
12747 verbose(env, "verifier internal error: repeating %s arg\n", head_type_name); in __process_kf_arg_ptr_to_graph_root()
12754 static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env, in process_kf_arg_ptr_to_list_head() argument
12758 return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_LIST_HEAD, in process_kf_arg_ptr_to_list_head()
12762 static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env, in process_kf_arg_ptr_to_rbtree_root() argument
12766 return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_RB_ROOT, in process_kf_arg_ptr_to_rbtree_root()
12771 __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env, in __process_kf_arg_ptr_to_graph_node() argument
12784 verbose(env, "verifier internal error: unexpected btf mismatch in kfunc call\n"); in __process_kf_arg_ptr_to_graph_node()
12788 if (!check_kfunc_is_graph_node_api(env, node_field_type, meta->func_id)) in __process_kf_arg_ptr_to_graph_node()
12793 verbose(env, in __process_kf_arg_ptr_to_graph_node()
12802 verbose(env, "%s not found at offset=%u\n", node_type_name, node_off); in __process_kf_arg_ptr_to_graph_node()
12810 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf, in __process_kf_arg_ptr_to_graph_node()
12812 verbose(env, "operation on %s expects arg#1 %s at offset=%d " in __process_kf_arg_ptr_to_graph_node()
12825 verbose(env, "arg#1 offset=%d, but expected %s at offset=%d in struct %s\n", in __process_kf_arg_ptr_to_graph_node()
12835 static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env, in process_kf_arg_ptr_to_list_node() argument
12839 return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta, in process_kf_arg_ptr_to_list_node()
12844 static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env, in process_kf_arg_ptr_to_rbtree_node() argument
12848 return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta, in process_kf_arg_ptr_to_rbtree_node()
12859 static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env) in check_css_task_iter_allowlist() argument
12861 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); in check_css_task_iter_allowlist()
12867 if (env->prog->expected_attach_type == BPF_TRACE_ITER) in check_css_task_iter_allowlist()
12871 return in_sleepable(env); in check_css_task_iter_allowlist()
12875 static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, in check_kfunc_args() argument
12888 verbose(env, "Function %s has %d > %d args\n", func_name, nargs, in check_kfunc_args()
12897 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[i + 1]; in check_kfunc_args()
12911 verbose(env, "R%d is not a scalar\n", regno); in check_kfunc_args()
12917 verbose(env, "verifier internal error: only one constant argument permitted\n"); in check_kfunc_args()
12921 verbose(env, "R%d must be a known constant\n", regno); in check_kfunc_args()
12924 ret = mark_chain_precision(env, regno); in check_kfunc_args()
12938 verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc"); in check_kfunc_args()
12943 verbose(env, "R%d is not a const\n", regno); in check_kfunc_args()
12948 ret = mark_chain_precision(env, regno); in check_kfunc_args()
12956 verbose(env, "Unrecognized arg#%d type %s\n", i, btf_type_str(t)); in check_kfunc_args()
12963 verbose(env, "Possibly NULL pointer passed to trusted arg%d\n", i); in check_kfunc_args()
12969 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", in check_kfunc_args()
12982 kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs); in check_kfunc_args()
12991 verbose(env, "pointer in R%d isn't map pointer\n", regno); in check_kfunc_args()
13009 verbose(env, in check_kfunc_args()
13025 verbose(env, "R%d must be referenced or trusted\n", regno); in check_kfunc_args()
13029 verbose(env, "R%d must be a rcu pointer\n", regno); in check_kfunc_args()
13057 ret = check_func_arg_reg_off(env, reg, regno, arg_type); in check_kfunc_args()
13064 verbose(env, "arg#%d expected pointer to ctx, but got %s\n", in check_kfunc_args()
13065 i, reg_type_str(env, reg->type)); in check_kfunc_args()
13070 ret = get_kern_ctx_btf_id(&env->log, resolve_prog_type(env->prog)); in check_kfunc_args()
13079 verbose(env, "arg#%d expected for bpf_obj_drop_impl()\n", i); in check_kfunc_args()
13084 verbose(env, "arg#%d expected for bpf_percpu_obj_drop_impl()\n", i); in check_kfunc_args()
13088 verbose(env, "arg#%d expected pointer to allocated object\n", i); in check_kfunc_args()
13092 verbose(env, "allocated object must be referenced\n"); in check_kfunc_args()
13120 verbose(env, "verifier internal error: no dynptr type for parent of clone\n"); in check_kfunc_args()
13127 verbose(env, "verifier internal error: missing ref obj id for parent of clone\n"); in check_kfunc_args()
13132 ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type, clone_ref_obj_id); in check_kfunc_args()
13137 int id = dynptr_id(env, reg); in check_kfunc_args()
13140 verbose(env, "verifier internal error: failed to obtain dynptr id\n"); in check_kfunc_args()
13144 meta->initialized_dynptr.type = dynptr_get_type(env, reg); in check_kfunc_args()
13145 meta->initialized_dynptr.ref_obj_id = dynptr_ref_obj_id(env, reg); in check_kfunc_args()
13152 if (!check_css_task_iter_allowlist(env)) { in check_kfunc_args()
13153 verbose(env, "css_task_iter is only allowed in bpf_lsm, bpf_iter and sleepable progs\n"); in check_kfunc_args()
13157 ret = process_iter_arg(env, regno, insn_idx, meta); in check_kfunc_args()
13164 verbose(env, "arg#%d expected pointer to map value or allocated object\n", i); in check_kfunc_args()
13168 verbose(env, "allocated object must be referenced\n"); in check_kfunc_args()
13171 ret = process_kf_arg_ptr_to_list_head(env, reg, regno, meta); in check_kfunc_args()
13178 verbose(env, "arg#%d expected pointer to map value or allocated object\n", i); in check_kfunc_args()
13182 verbose(env, "allocated object must be referenced\n"); in check_kfunc_args()
13185 ret = process_kf_arg_ptr_to_rbtree_root(env, reg, regno, meta); in check_kfunc_args()
13191 verbose(env, "arg#%d expected pointer to allocated object\n", i); in check_kfunc_args()
13195 verbose(env, "allocated object must be referenced\n"); in check_kfunc_args()
13198 ret = process_kf_arg_ptr_to_list_node(env, reg, regno, meta); in check_kfunc_args()
13205 verbose(env, "rbtree_remove node input must be non-owning ref\n"); in check_kfunc_args()
13208 if (in_rbtree_lock_required_cb(env)) { in check_kfunc_args()
13209 verbose(env, "rbtree_remove not allowed in rbtree cb\n"); in check_kfunc_args()
13214 verbose(env, "arg#%d expected pointer to allocated object\n", i); in check_kfunc_args()
13218 verbose(env, "allocated object must be referenced\n"); in check_kfunc_args()
13223 ret = process_kf_arg_ptr_to_rbtree_node(env, reg, regno, meta); in check_kfunc_args()
13238 verbose(env, "arg#%d is %s ", i, reg_type_str(env, reg->type)); in check_kfunc_args()
13239 verbose(env, "expected %s or socket\n", in check_kfunc_args()
13240 reg_type_str(env, base_type(reg->type) | in check_kfunc_args()
13244 ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i); in check_kfunc_args()
13251 verbose(env, "arg#%d reference type('%s %s') size cannot be determined: %ld\n", in check_kfunc_args()
13255 ret = check_mem_reg(env, reg, regno, type_size); in check_kfunc_args()
13267 ret = check_kfunc_mem_size_reg(env, size_reg, regno + 1); in check_kfunc_args()
13269 verbose(env, "arg#%d arg#%d memory, len pair leads to invalid memory access\n", i, i + 1); in check_kfunc_args()
13276 verbose(env, "verifier internal error: only one constant argument permitted\n"); in check_kfunc_args()
13280 verbose(env, "R%d must be a known constant\n", regno + 1); in check_kfunc_args()
13293 verbose(env, "arg%d expected pointer to func\n", i); in check_kfunc_args()
13300 verbose(env, "arg#%d is neither owning or non-owning ref\n", i); in check_kfunc_args()
13308 verbose(env, "verifier internal error: Couldn't find btf_record\n"); in check_kfunc_args()
13313 verbose(env, "arg#%d doesn't point to a type with bpf_refcount field\n", i); in check_kfunc_args()
13322 verbose(env, "arg#%d doesn't point to a const string\n", i); in check_kfunc_args()
13325 ret = check_reg_const_str(env, reg, regno); in check_kfunc_args()
13331 verbose(env, "arg#%d doesn't point to a map value\n", i); in check_kfunc_args()
13334 ret = process_wq_func(env, regno, meta); in check_kfunc_args()
13340 verbose(env, "arg#%d doesn't point to an irq flag on stack\n", i); in check_kfunc_args()
13343 ret = process_irq_flag(env, regno, meta); in check_kfunc_args()
13352 verbose(env, "arg#%d doesn't point to map value or allocated object\n", i); in check_kfunc_args()
13364 ret = process_spin_lock(env, regno, flags); in check_kfunc_args()
13373 verbose(env, "release kernel function %s expects refcounted PTR_TO_BTF_ID\n", in check_kfunc_args()
13381 static int fetch_kfunc_meta(struct bpf_verifier_env *env, in fetch_kfunc_meta() argument
13397 desc_btf = find_kfunc_desc_btf(env, insn->off); in fetch_kfunc_meta()
13408 kfunc_flags = btf_kfunc_id_set_contains(desc_btf, func_id, env->prog); in fetch_kfunc_meta()
13423 static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name);
13425 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in check_kfunc_call() argument
13430 struct bpf_reg_state *regs = cur_regs(env); in check_kfunc_call()
13444 err = fetch_kfunc_meta(env, insn, &meta, &func_name); in check_kfunc_call()
13446 verbose(env, "calling kernel function %s is not allowed\n", func_name); in check_kfunc_call()
13450 insn_aux = &env->insn_aux_data[insn_idx]; in check_kfunc_call()
13460 branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); in check_kfunc_call()
13462 verbose(env, "failed to push state for failed lock acquisition\n"); in check_kfunc_call()
13470 mark_reg_not_init(env, regs, caller_saved[i]); in check_kfunc_call()
13472 mark_reg_unknown(env, regs, BPF_REG_0); in check_kfunc_call()
13473 err = __mark_reg_s32_range(env, regs, BPF_REG_0, -MAX_ERRNO, -1); in check_kfunc_call()
13475 verbose(env, "failed to mark s32 range for retval in forked state for lock\n"); in check_kfunc_call()
13478 __mark_btf_func_reg_size(env, regs, BPF_REG_0, sizeof(u32)); in check_kfunc_call()
13482 verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n"); in check_kfunc_call()
13487 if (sleepable && !in_sleepable(env)) { in check_kfunc_call()
13488 verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); in check_kfunc_call()
13493 err = check_kfunc_args(env, &meta, insn_idx); in check_kfunc_call()
13498 err = push_callback_call(env, insn, insn_idx, meta.subprogno, in check_kfunc_call()
13501 verbose(env, "kfunc %s#%d failed callback verification\n", in check_kfunc_call()
13513 err = push_callback_call(env, insn, insn_idx, meta.subprogno, in check_kfunc_call()
13516 verbose(env, "kfunc %s#%d failed callback verification\n", in check_kfunc_call()
13528 if (env->cur_state->active_rcu_lock) { in check_kfunc_call()
13533 if (in_rbtree_lock_required_cb(env) && (rcu_lock || rcu_unlock)) { in check_kfunc_call()
13534 verbose(env, "Calling bpf_rcu_read_{lock,unlock} in unnecessary rbtree callback\n"); in check_kfunc_call()
13539 verbose(env, "nested rcu read lock (kernel function %s)\n", func_name); in check_kfunc_call()
13542 bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, clear_mask, ({ in check_kfunc_call()
13548 env->cur_state->active_rcu_lock = false; in check_kfunc_call()
13550 verbose(env, "kernel func %s is sleepable within rcu_read_lock region\n", func_name); in check_kfunc_call()
13554 env->cur_state->active_rcu_lock = true; in check_kfunc_call()
13556 verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name); in check_kfunc_call()
13560 if (env->cur_state->active_preempt_locks) { in check_kfunc_call()
13562 env->cur_state->active_preempt_locks++; in check_kfunc_call()
13564 env->cur_state->active_preempt_locks--; in check_kfunc_call()
13566 verbose(env, "kernel func %s is sleepable within non-preemptible region\n", func_name); in check_kfunc_call()
13570 env->cur_state->active_preempt_locks++; in check_kfunc_call()
13572 verbose(env, "unmatched attempt to enable preemption (kernel function %s)\n", func_name); in check_kfunc_call()
13576 if (env->cur_state->active_irq_id && sleepable) { in check_kfunc_call()
13577 verbose(env, "kernel func %s is sleepable within IRQ-disabled region\n", func_name); in check_kfunc_call()
13585 err = release_reference(env, regs[meta.release_regno].ref_obj_id); in check_kfunc_call()
13587 verbose(env, "kfunc %s#%d reference has not been acquired before\n", in check_kfunc_call()
13599 err = ref_convert_owning_non_owning(env, release_ref_obj_id); in check_kfunc_call()
13601 verbose(env, "kfunc %s#%d conversion of owning ref to non-owning failed\n", in check_kfunc_call()
13606 err = release_reference(env, release_ref_obj_id); in check_kfunc_call()
13608 verbose(env, "kfunc %s#%d reference has not been acquired before\n", in check_kfunc_call()
13616 verbose(env, "JIT does not support calling kfunc %s#%d\n", in check_kfunc_call()
13620 env->seen_exception = true; in check_kfunc_call()
13625 if (!env->exception_callback_subprog) { in check_kfunc_call()
13626 err = check_return_code(env, BPF_REG_1, "R1"); in check_kfunc_call()
13633 mark_reg_not_init(env, regs, caller_saved[i]); in check_kfunc_call()
13644 verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n"); in check_kfunc_call()
13650 mark_reg_unknown(env, regs, BPF_REG_0); in check_kfunc_call()
13653 __mark_reg_const_zero(env, ®s[BPF_REG_0]); in check_kfunc_call()
13654 mark_btf_func_reg_size(env, BPF_REG_0, t->size); in check_kfunc_call()
13669 verbose(env, "local type ID argument must be in range [0, U32_MAX]\n"); in check_kfunc_call()
13673 ret_btf = env->prog->aux->btf; in check_kfunc_call()
13678 verbose(env, "bpf_obj_new/bpf_percpu_obj_new requires prog BTF\n"); in check_kfunc_call()
13684 verbose(env, "bpf_obj_new/bpf_percpu_obj_new type ID argument must be of a struct\n"); in check_kfunc_call()
13690 verbose(env, "bpf_percpu_obj_new type size (%d) is greater than %d\n", in check_kfunc_call()
13719 if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { in check_kfunc_call()
13720 verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); in check_kfunc_call()
13725 verbose(env, "bpf_percpu_obj_new type ID argument must not contain special fields\n"); in check_kfunc_call()
13730 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
13740 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
13759 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
13766 verbose(env, in check_kfunc_call()
13771 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
13779 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
13782 verbose(env, "verifier internal error: bpf_dynptr_slice(_rdwr) no constant size\n"); in check_kfunc_call()
13795 if (!may_access_direct_pkt_data(env, NULL, BPF_WRITE)) { in check_kfunc_call()
13796 verbose(env, "the prog does not allow writes to packet data\n"); in check_kfunc_call()
13802 verbose(env, "verifier internal error: no dynptr id\n"); in check_kfunc_call()
13812 verbose(env, "kernel function %s unhandled dynamic return type\n", in check_kfunc_call()
13818 mark_reg_unknown(env, regs, BPF_REG_0); in check_kfunc_call()
13831 verbose(env, in check_kfunc_call()
13839 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
13850 mark_reg_known_zero(env, regs, BPF_REG_0); in check_kfunc_call()
13861 cur_iter = get_iter_from_state(env->cur_state, &meta); in check_kfunc_call()
13873 regs[BPF_REG_0].id = ++env->id_gen; in check_kfunc_call()
13875 mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *)); in check_kfunc_call()
13877 int id = acquire_reference(env, insn_idx); in check_kfunc_call()
13885 ref_set_non_owning(env, ®s[BPF_REG_0]); in check_kfunc_call()
13889 regs[BPF_REG_0].id = ++env->id_gen; in check_kfunc_call()
13908 mark_btf_func_reg_size(env, regno, sizeof(void *)); in check_kfunc_call()
13911 mark_btf_func_reg_size(env, regno, t->size); in check_kfunc_call()
13915 err = process_iter_next_call(env, insn_idx, &meta); in check_kfunc_call()
13923 static bool check_reg_sane_offset(struct bpf_verifier_env *env, in check_reg_sane_offset() argument
13932 verbose(env, "math between %s pointer and %lld is not allowed\n", in check_reg_sane_offset()
13933 reg_type_str(env, type), val); in check_reg_sane_offset()
13938 verbose(env, "%s pointer offset %d is not allowed\n", in check_reg_sane_offset()
13939 reg_type_str(env, type), reg->off); in check_reg_sane_offset()
13944 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", in check_reg_sane_offset()
13945 reg_type_str(env, type)); in check_reg_sane_offset()
13950 verbose(env, "value %lld makes %s pointer be out of bounds\n", in check_reg_sane_offset()
13951 smin, reg_type_str(env, type)); in check_reg_sane_offset()
13997 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, in can_skip_alu_sanitation() argument
14000 return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K; in can_skip_alu_sanitation()
14020 static int sanitize_val_alu(struct bpf_verifier_env *env, in sanitize_val_alu() argument
14023 struct bpf_insn_aux_data *aux = cur_aux(env); in sanitize_val_alu()
14025 if (can_skip_alu_sanitation(env, insn)) in sanitize_val_alu()
14042 sanitize_speculative_path(struct bpf_verifier_env *env, in sanitize_speculative_path() argument
14049 branch = push_stack(env, next_idx, curr_idx, true); in sanitize_speculative_path()
14053 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
14055 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
14056 mark_reg_unknown(env, regs, insn->src_reg); in sanitize_speculative_path()
14062 static int sanitize_ptr_alu(struct bpf_verifier_env *env, in sanitize_ptr_alu() argument
14070 struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux; in sanitize_ptr_alu()
14071 struct bpf_verifier_state *vstate = env->cur_state; in sanitize_ptr_alu()
14081 if (can_skip_alu_sanitation(env, insn)) in sanitize_ptr_alu()
14120 env->explore_alu_limits = true; in sanitize_ptr_alu()
14151 ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1, in sanitize_ptr_alu()
14152 env->insn_idx); in sanitize_ptr_alu()
14158 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) in sanitize_mark_insn_seen() argument
14160 struct bpf_verifier_state *vstate = env->cur_state; in sanitize_mark_insn_seen()
14168 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; in sanitize_mark_insn_seen()
14171 static int sanitize_err(struct bpf_verifier_env *env, in sanitize_err() argument
14182 verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", in sanitize_err()
14186 verbose(env, "R%d has pointer with unsupported alu operation, %s\n", in sanitize_err()
14190 verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", in sanitize_err()
14194 verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", in sanitize_err()
14198 verbose(env, "R%d could not be pushed for speculative verification, %s\n", in sanitize_err()
14202 verbose(env, "verifier internal error: unknown reason (%d)\n", in sanitize_err()
14221 struct bpf_verifier_env *env, in check_stack_access_for_ptr_arithmetic() argument
14230 verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", in check_stack_access_for_ptr_arithmetic()
14236 verbose(env, "R%d stack pointer arithmetic goes out of range, " in check_stack_access_for_ptr_arithmetic()
14244 static int sanitize_check_bounds(struct bpf_verifier_env *env, in sanitize_check_bounds() argument
14253 if (env->bypass_spec_v1) in sanitize_check_bounds()
14258 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, in sanitize_check_bounds()
14263 if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) { in sanitize_check_bounds()
14264 verbose(env, "R%d pointer arithmetic of map value goes out of range, " in sanitize_check_bounds()
14281 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, in adjust_ptr_min_max_vals() argument
14286 struct bpf_verifier_state *vstate = env->cur_state; in adjust_ptr_min_max_vals()
14306 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
14312 if (opcode == BPF_SUB && env->allow_ptr_leaks) { in adjust_ptr_min_max_vals()
14313 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
14317 verbose(env, in adjust_ptr_min_max_vals()
14324 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", in adjust_ptr_min_max_vals()
14325 dst, reg_type_str(env, ptr_reg->type)); in adjust_ptr_min_max_vals()
14353 verbose(env, "R%d pointer arithmetic on %s prohibited\n", in adjust_ptr_min_max_vals()
14354 dst, reg_type_str(env, ptr_reg->type)); in adjust_ptr_min_max_vals()
14364 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || in adjust_ptr_min_max_vals()
14365 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) in adjust_ptr_min_max_vals()
14372 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
14375 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
14418 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
14426 verbose(env, "R%d tried to subtract pointer from scalar\n", in adjust_ptr_min_max_vals()
14435 verbose(env, "R%d subtraction from stack pointer prohibited\n", in adjust_ptr_min_max_vals()
14474 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
14484 verbose(env, "R%d bitwise operator %s on pointer prohibited\n", in adjust_ptr_min_max_vals()
14489 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", in adjust_ptr_min_max_vals()
14494 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) in adjust_ptr_min_max_vals()
14497 if (sanitize_check_bounds(env, insn, dst_reg) < 0) in adjust_ptr_min_max_vals()
14500 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
14503 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
15080 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, in adjust_scalar_min_max_vals() argument
15090 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
15095 ret = sanitize_val_alu(env, insn); in adjust_scalar_min_max_vals()
15097 return sanitize_err(env, insn, ret, NULL, NULL); in adjust_scalar_min_max_vals()
15177 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, in adjust_reg_min_max_vals() argument
15180 struct bpf_verifier_state *vstate = env->cur_state; in adjust_reg_min_max_vals()
15192 struct bpf_insn_aux_data *aux = cur_aux(env); in adjust_reg_min_max_vals()
15216 if (opcode == BPF_SUB && env->allow_ptr_leaks) { in adjust_reg_min_max_vals()
15217 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_reg_min_max_vals()
15220 verbose(env, "R%d pointer %s pointer prohibited\n", in adjust_reg_min_max_vals()
15229 err = mark_chain_precision(env, insn->dst_reg); in adjust_reg_min_max_vals()
15232 return adjust_ptr_min_max_vals(env, insn, in adjust_reg_min_max_vals()
15237 err = mark_chain_precision(env, insn->src_reg); in adjust_reg_min_max_vals()
15240 return adjust_ptr_min_max_vals(env, insn, in adjust_reg_min_max_vals()
15244 err = mark_chain_precision(env, insn->src_reg); in adjust_reg_min_max_vals()
15256 return adjust_ptr_min_max_vals(env, insn, in adjust_reg_min_max_vals()
15262 print_verifier_state(env, vstate, vstate->curframe, true); in adjust_reg_min_max_vals()
15263 verbose(env, "verifier internal error: unexpected ptr_reg\n"); in adjust_reg_min_max_vals()
15267 print_verifier_state(env, vstate, vstate->curframe, true); in adjust_reg_min_max_vals()
15268 verbose(env, "verifier internal error: no src_reg\n"); in adjust_reg_min_max_vals()
15271 err = adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); in adjust_reg_min_max_vals()
15283 if (env->bpf_capable && in adjust_reg_min_max_vals()
15312 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_alu_op() argument
15314 struct bpf_reg_state *regs = cur_regs(env); in check_alu_op()
15323 verbose(env, "BPF_NEG uses reserved fields\n"); in check_alu_op()
15331 verbose(env, "BPF_END uses reserved fields\n"); in check_alu_op()
15337 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
15341 if (is_pointer_value(env, insn->dst_reg)) { in check_alu_op()
15342 verbose(env, "R%d pointer arithmetic prohibited\n", in check_alu_op()
15348 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_alu_op()
15358 verbose(env, "BPF_MOV uses reserved fields\n"); in check_alu_op()
15363 verbose(env, "addr_space_cast insn can only convert between address space 1 and 0\n"); in check_alu_op()
15366 if (!env->prog->aux->arena) { in check_alu_op()
15367 … verbose(env, "addr_space_cast insn can only be used in a program that has an associated arena\n"); in check_alu_op()
15373 verbose(env, "BPF_MOV uses reserved fields\n"); in check_alu_op()
15379 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_alu_op()
15384 verbose(env, "BPF_MOV uses reserved fields\n"); in check_alu_op()
15390 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
15401 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
15405 dst_reg->subreg_def = env->insn_idx + 1; in check_alu_op()
15411 assign_scalar_id_before_mov(env, src_reg); in check_alu_op()
15417 if (is_pointer_value(env, insn->src_reg)) { in check_alu_op()
15418 verbose(env, in check_alu_op()
15427 assign_scalar_id_before_mov(env, src_reg); in check_alu_op()
15435 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
15440 if (is_pointer_value(env, insn->src_reg)) { in check_alu_op()
15441 verbose(env, in check_alu_op()
15450 assign_scalar_id_before_mov(env, src_reg); in check_alu_op()
15459 dst_reg->subreg_def = env->insn_idx + 1; in check_alu_op()
15465 assign_scalar_id_before_mov(env, src_reg); in check_alu_op()
15470 dst_reg->subreg_def = env->insn_idx + 1; in check_alu_op()
15474 mark_reg_unknown(env, regs, in check_alu_op()
15485 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
15497 verbose(env, "invalid BPF_ALU opcode %x\n", opcode); in check_alu_op()
15505 verbose(env, "BPF_ALU uses reserved fields\n"); in check_alu_op()
15509 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_alu_op()
15515 verbose(env, "BPF_ALU uses reserved fields\n"); in check_alu_op()
15521 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
15527 verbose(env, "div by zero\n"); in check_alu_op()
15536 verbose(env, "invalid shift %d\n", insn->imm); in check_alu_op()
15542 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
15543 err = err ?: adjust_reg_min_max_vals(env, insn); in check_alu_op()
15548 return reg_bounds_sanity_check(env, ®s[insn->dst_reg], "alu"); in check_alu_op()
16069 static int reg_set_min_max(struct bpf_verifier_env *env, in reg_set_min_max() argument
16095 err = reg_bounds_sanity_check(env, true_reg1, "true_reg1"); in reg_set_min_max()
16096 err = err ?: reg_bounds_sanity_check(env, true_reg2, "true_reg2"); in reg_set_min_max()
16097 err = err ?: reg_bounds_sanity_check(env, false_reg1, "false_reg1"); in reg_set_min_max()
16098 err = err ?: reg_bounds_sanity_check(env, false_reg2, "false_reg2"); in reg_set_min_max()
16370 static int check_cond_jmp_op(struct bpf_verifier_env *env, in check_cond_jmp_op() argument
16373 struct bpf_verifier_state *this_branch = env->cur_state; in check_cond_jmp_op()
16386 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); in check_cond_jmp_op()
16391 struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st; in check_cond_jmp_op()
16397 verbose(env, "invalid may_goto imm %d\n", insn->imm); in check_cond_jmp_op()
16400 prev_st = find_prev_entry(env, cur_st->parent, idx); in check_cond_jmp_op()
16403 queued_st = push_stack(env, idx + 1, idx, false); in check_cond_jmp_op()
16409 widen_imprecise_scalars(env, prev_st, queued_st); in check_cond_jmp_op()
16415 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_cond_jmp_op()
16422 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); in check_cond_jmp_op()
16427 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_cond_jmp_op()
16433 is_pointer_value(env, insn->src_reg)) { in check_cond_jmp_op()
16434 verbose(env, "R%d pointer comparison prohibited\n", in check_cond_jmp_op()
16440 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); in check_cond_jmp_op()
16443 src_reg = &env->fake_reg[0]; in check_cond_jmp_op()
16456 err = mark_chain_precision(env, insn->dst_reg); in check_cond_jmp_op()
16459 err = mark_chain_precision(env, insn->src_reg); in check_cond_jmp_op()
16469 if (!env->bypass_spec_v1 && in check_cond_jmp_op()
16470 !sanitize_speculative_path(env, insn, *insn_idx + 1, in check_cond_jmp_op()
16473 if (env->log.level & BPF_LOG_LEVEL) in check_cond_jmp_op()
16474 print_insn_state(env, this_branch, this_branch->curframe); in check_cond_jmp_op()
16482 if (!env->bypass_spec_v1 && in check_cond_jmp_op()
16483 !sanitize_speculative_path(env, insn, in check_cond_jmp_op()
16487 if (env->log.level & BPF_LOG_LEVEL) in check_cond_jmp_op()
16488 print_insn_state(env, this_branch, this_branch->curframe); in check_cond_jmp_op()
16502 err = push_insn_history(env, this_branch, 0, linked_regs_pack(&linked_regs)); in check_cond_jmp_op()
16507 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, in check_cond_jmp_op()
16514 err = reg_set_min_max(env, in check_cond_jmp_op()
16523 memcpy(&env->fake_reg[1], &env->fake_reg[0], in check_cond_jmp_op()
16524 sizeof(env->fake_reg[0])); in check_cond_jmp_op()
16525 err = reg_set_min_max(env, in check_cond_jmp_op()
16527 &env->fake_reg[0], in check_cond_jmp_op()
16528 dst_reg, &env->fake_reg[1], in check_cond_jmp_op()
16599 is_pointer_value(env, insn->dst_reg)) { in check_cond_jmp_op()
16600 verbose(env, "R%d pointer comparison prohibited\n", in check_cond_jmp_op()
16604 if (env->log.level & BPF_LOG_LEVEL) in check_cond_jmp_op()
16605 print_insn_state(env, this_branch, this_branch->curframe); in check_cond_jmp_op()
16610 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_ld_imm() argument
16612 struct bpf_insn_aux_data *aux = cur_aux(env); in check_ld_imm()
16613 struct bpf_reg_state *regs = cur_regs(env); in check_ld_imm()
16619 verbose(env, "invalid BPF_LD_IMM insn\n"); in check_ld_imm()
16623 verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); in check_ld_imm()
16627 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_ld_imm()
16644 mark_reg_known_zero(env, regs, insn->dst_reg); in check_ld_imm()
16657 verbose(env, "bpf verifier is misconfigured\n"); in check_ld_imm()
16664 struct bpf_prog_aux *aux = env->prog->aux; in check_ld_imm()
16665 u32 subprogno = find_subprog(env, in check_ld_imm()
16666 env->insn_idx + insn->imm + 1); in check_ld_imm()
16669 verbose(env, "missing btf func_info\n"); in check_ld_imm()
16673 verbose(env, "callback function not static\n"); in check_ld_imm()
16682 map = env->used_maps[aux->map_index]; in check_ld_imm()
16688 __mark_reg_unknown(env, dst_reg); in check_ld_imm()
16699 verbose(env, "bpf verifier is misconfigured\n"); in check_ld_imm()
16733 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) in check_ld_abs() argument
16735 struct bpf_reg_state *regs = cur_regs(env); in check_ld_abs()
16740 if (!may_access_skb(resolve_prog_type(env->prog))) { in check_ld_abs()
16741 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); in check_ld_abs()
16745 if (!env->ops->gen_ld_abs) { in check_ld_abs()
16746 verbose(env, "bpf verifier is misconfigured\n"); in check_ld_abs()
16753 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); in check_ld_abs()
16758 err = check_reg_arg(env, ctx_reg, SRC_OP); in check_ld_abs()
16766 err = check_resource_leak(env, false, true, "BPF_LD_[ABS|IND]"); in check_ld_abs()
16771 verbose(env, in check_ld_abs()
16778 err = check_reg_arg(env, insn->src_reg, SRC_OP); in check_ld_abs()
16783 err = check_ptr_off_reg(env, ®s[ctx_reg], ctx_reg); in check_ld_abs()
16789 mark_reg_not_init(env, regs, caller_saved[i]); in check_ld_abs()
16790 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); in check_ld_abs()
16797 mark_reg_unknown(env, regs, BPF_REG_0); in check_ld_abs()
16799 regs[BPF_REG_0].subreg_def = env->insn_idx + 1; in check_ld_abs()
16803 static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name) in check_return_code() argument
16807 const struct bpf_prog *prog = env->prog; in check_return_code()
16808 struct bpf_reg_state *reg = reg_state(env, regno); in check_return_code()
16810 enum bpf_prog_type prog_type = resolve_prog_type(env->prog); in check_return_code()
16812 struct bpf_func_state *frame = env->cur_state->frame[0]; in check_return_code()
16843 return __check_ptr_off_reg(env, reg, regno, false); in check_return_code()
16856 err = check_reg_arg(env, regno, SRC_OP); in check_return_code()
16860 if (is_pointer_value(env, regno)) { in check_return_code()
16861 verbose(env, "R%d leaks addr as return value\n", regno); in check_return_code()
16874 verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n", in check_return_code()
16875 regno, reg_type_str(env, reg->type)); in check_return_code()
16883 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || in check_return_code()
16884 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG || in check_return_code()
16885 env->prog->expected_attach_type == BPF_CGROUP_UNIX_RECVMSG || in check_return_code()
16886 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME || in check_return_code()
16887 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME || in check_return_code()
16888 env->prog->expected_attach_type == BPF_CGROUP_UNIX_GETPEERNAME || in check_return_code()
16889 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME || in check_return_code()
16890 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME || in check_return_code()
16891 env->prog->expected_attach_type == BPF_CGROUP_UNIX_GETSOCKNAME) in check_return_code()
16893 if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND || in check_return_code()
16894 env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND) in check_return_code()
16898 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { in check_return_code()
16910 if (!env->prog->aux->attach_btf_id) in check_return_code()
16915 switch (env->prog->expected_attach_type) { in check_return_code()
16930 switch (env->prog->expected_attach_type) { in check_return_code()
16944 if (env->prog->expected_attach_type != BPF_LSM_CGROUP) { in check_return_code()
16946 if (!get_func_retval_range(env->prog, &range)) in check_return_code()
16952 } else if (!env->prog->aux->attach_func_proto->type) { in check_return_code()
16978 verbose(env, "%s the register R%d is not a known value (%s)\n", in check_return_code()
16979 exit_ctx, regno, reg_type_str(env, reg->type)); in check_return_code()
16983 err = mark_chain_precision(env, regno); in check_return_code()
16988 verbose_invalid_scalar(env, reg, range, exit_ctx, reg_name); in check_return_code()
16993 verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n"); in check_return_code()
16999 env->prog->enforce_expected_attach_type = 1; in check_return_code()
17003 static void mark_subprog_changes_pkt_data(struct bpf_verifier_env *env, int off) in mark_subprog_changes_pkt_data() argument
17007 subprog = find_containing_subprog(env, off); in mark_subprog_changes_pkt_data()
17011 static void mark_subprog_might_sleep(struct bpf_verifier_env *env, int off) in mark_subprog_might_sleep() argument
17015 subprog = find_containing_subprog(env, off); in mark_subprog_might_sleep()
17025 static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w) in merge_callee_effects() argument
17029 caller = find_containing_subprog(env, t); in merge_callee_effects()
17030 callee = find_containing_subprog(env, w); in merge_callee_effects()
17075 static void mark_prune_point(struct bpf_verifier_env *env, int idx) in mark_prune_point() argument
17077 env->insn_aux_data[idx].prune_point = true; in mark_prune_point()
17080 static bool is_prune_point(struct bpf_verifier_env *env, int insn_idx) in is_prune_point() argument
17082 return env->insn_aux_data[insn_idx].prune_point; in is_prune_point()
17085 static void mark_force_checkpoint(struct bpf_verifier_env *env, int idx) in mark_force_checkpoint() argument
17087 env->insn_aux_data[idx].force_checkpoint = true; in mark_force_checkpoint()
17090 static bool is_force_checkpoint(struct bpf_verifier_env *env, int insn_idx) in is_force_checkpoint() argument
17092 return env->insn_aux_data[insn_idx].force_checkpoint; in is_force_checkpoint()
17095 static void mark_calls_callback(struct bpf_verifier_env *env, int idx) in mark_calls_callback() argument
17097 env->insn_aux_data[idx].calls_callback = true; in mark_calls_callback()
17100 static bool calls_callback(struct bpf_verifier_env *env, int insn_idx) in calls_callback() argument
17102 return env->insn_aux_data[insn_idx].calls_callback; in calls_callback()
17115 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) in push_insn() argument
17117 int *insn_stack = env->cfg.insn_stack; in push_insn()
17118 int *insn_state = env->cfg.insn_state; in push_insn()
17126 if (w < 0 || w >= env->prog->len) { in push_insn()
17127 verbose_linfo(env, t, "%d: ", t); in push_insn()
17128 verbose(env, "jump out of range from insn %d to %d\n", t, w); in push_insn()
17134 mark_prune_point(env, w); in push_insn()
17135 mark_jmp_point(env, w); in push_insn()
17142 if (env->cfg.cur_stack >= env->prog->len) in push_insn()
17144 insn_stack[env->cfg.cur_stack++] = w; in push_insn()
17147 if (env->bpf_capable) in push_insn()
17149 verbose_linfo(env, t, "%d: ", t); in push_insn()
17150 verbose_linfo(env, w, "%d: ", w); in push_insn()
17151 verbose(env, "back-edge from insn %d to %d\n", t, w); in push_insn()
17157 verbose(env, "insn state internal bug\n"); in push_insn()
17164 struct bpf_verifier_env *env, in visit_func_call_insn() argument
17171 ret = push_insn(t, t + insn_sz, FALLTHROUGH, env); in visit_func_call_insn()
17175 mark_prune_point(env, t + insn_sz); in visit_func_call_insn()
17177 mark_jmp_point(env, t + insn_sz); in visit_func_call_insn()
17181 mark_prune_point(env, t); in visit_func_call_insn()
17182 merge_callee_effects(env, t, w); in visit_func_call_insn()
17183 ret = push_insn(t, w, BRANCH, env); in visit_func_call_insn()
17195 static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm) in verifier_inlines_helper_call() argument
17200 return env->prog->jit_requested && bpf_jit_supports_percpu_insn(); in verifier_inlines_helper_call()
17216 static bool get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, in get_call_summary() argument
17225 if (get_helper_proto(env, call->imm, &fn) < 0) in get_call_summary()
17229 (verifier_inlines_helper_call(env, call->imm) || in get_call_summary()
17244 err = fetch_kfunc_meta(env, call, &meta, NULL); in get_call_summary()
17331 static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, in mark_fastcall_pattern_for_call() argument
17335 struct bpf_insn *insns = env->prog->insnsi, *stx, *ldx; in mark_fastcall_pattern_for_call()
17336 struct bpf_insn *call = &env->prog->insnsi[insn_idx]; in mark_fastcall_pattern_for_call()
17343 if (!get_call_summary(env, call, &cs)) in mark_fastcall_pattern_for_call()
17366 if (insn_idx - i < 0 || insn_idx + i >= env->prog->len) in mark_fastcall_pattern_for_call()
17389 env->insn_aux_data[insn_idx - i].fastcall_pattern = 1; in mark_fastcall_pattern_for_call()
17390 env->insn_aux_data[insn_idx + i].fastcall_pattern = 1; in mark_fastcall_pattern_for_call()
17412 env->insn_aux_data[insn_idx].fastcall_spills_num = i - 1; in mark_fastcall_pattern_for_call()
17418 static int mark_fastcall_patterns(struct bpf_verifier_env *env) in mark_fastcall_patterns() argument
17420 struct bpf_subprog_info *subprog = env->subprog_info; in mark_fastcall_patterns()
17425 for (s = 0; s < env->subprog_cnt; ++s, ++subprog) { in mark_fastcall_patterns()
17429 insn = env->prog->insnsi + i; in mark_fastcall_patterns()
17437 insn = env->prog->insnsi + i; in mark_fastcall_patterns()
17440 mark_fastcall_pattern_for_call(env, subprog, i, lowest_off); in mark_fastcall_patterns()
17451 static int visit_insn(int t, struct bpf_verifier_env *env) in visit_insn() argument
17453 struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t]; in visit_insn()
17457 return visit_func_call_insn(t, insns, env, true); in visit_insn()
17463 return push_insn(t, t + insn_sz, FALLTHROUGH, env); in visit_insn()
17477 mark_prune_point(env, t); in visit_insn()
17488 mark_calls_callback(env, t); in visit_insn()
17489 mark_force_checkpoint(env, t); in visit_insn()
17490 mark_prune_point(env, t); in visit_insn()
17491 mark_jmp_point(env, t); in visit_insn()
17496 ret = get_helper_proto(env, insn->imm, &fp); in visit_insn()
17503 mark_subprog_might_sleep(env, t); in visit_insn()
17505 mark_subprog_changes_pkt_data(env, t); in visit_insn()
17509 ret = fetch_kfunc_meta(env, insn, &meta, NULL); in visit_insn()
17511 mark_prune_point(env, t); in visit_insn()
17523 mark_force_checkpoint(env, t); in visit_insn()
17531 mark_subprog_might_sleep(env, t); in visit_insn()
17533 return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL); in visit_insn()
17545 ret = push_insn(t, t + off + 1, FALLTHROUGH, env); in visit_insn()
17549 mark_prune_point(env, t + off + 1); in visit_insn()
17550 mark_jmp_point(env, t + off + 1); in visit_insn()
17556 mark_prune_point(env, t); in visit_insn()
17558 mark_force_checkpoint(env, t); in visit_insn()
17560 ret = push_insn(t, t + 1, FALLTHROUGH, env); in visit_insn()
17564 return push_insn(t, t + insn->off + 1, BRANCH, env); in visit_insn()
17571 static int check_cfg(struct bpf_verifier_env *env) in check_cfg() argument
17573 int insn_cnt = env->prog->len; in check_cfg()
17577 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); in check_cfg()
17581 insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); in check_cfg()
17587 insn_postorder = env->cfg.insn_postorder = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); in check_cfg()
17594 ex_insn_beg = env->exception_callback_subprog in check_cfg()
17595 ? env->subprog_info[env->exception_callback_subprog].start in check_cfg()
17600 env->cfg.cur_stack = 1; in check_cfg()
17603 while (env->cfg.cur_stack > 0) { in check_cfg()
17604 int t = insn_stack[env->cfg.cur_stack - 1]; in check_cfg()
17606 ret = visit_insn(t, env); in check_cfg()
17610 env->cfg.cur_stack--; in check_cfg()
17611 insn_postorder[env->cfg.cur_postorder++] = t; in check_cfg()
17617 verbose(env, "visit_insn internal bug\n"); in check_cfg()
17624 if (env->cfg.cur_stack < 0) { in check_cfg()
17625 verbose(env, "pop stack internal bug\n"); in check_cfg()
17633 env->cfg.cur_stack = 1; in check_cfg()
17638 struct bpf_insn *insn = &env->prog->insnsi[i]; in check_cfg()
17641 verbose(env, "unreachable insn %d\n", i); in check_cfg()
17647 verbose(env, "jump into the middle of ldimm64 insn %d\n", i); in check_cfg()
17655 env->prog->aux->changes_pkt_data = env->subprog_info[0].changes_pkt_data; in check_cfg()
17656 env->prog->aux->might_sleep = env->subprog_info[0].might_sleep; in check_cfg()
17661 env->cfg.insn_state = env->cfg.insn_stack = NULL; in check_cfg()
17665 static int check_abnormal_return(struct bpf_verifier_env *env) in check_abnormal_return() argument
17669 for (i = 1; i < env->subprog_cnt; i++) { in check_abnormal_return()
17670 if (env->subprog_info[i].has_ld_abs) { in check_abnormal_return()
17671 verbose(env, "LD_ABS is not allowed in subprogs without BTF\n"); in check_abnormal_return()
17674 if (env->subprog_info[i].has_tail_call) { in check_abnormal_return()
17675 verbose(env, "tail_call is not allowed in subprogs without BTF\n"); in check_abnormal_return()
17686 static int check_btf_func_early(struct bpf_verifier_env *env, in check_btf_func_early() argument
17702 if (check_abnormal_return(env)) in check_btf_func_early()
17711 verbose(env, "invalid func info rec size %u\n", urec_size); in check_btf_func_early()
17715 prog = env->prog; in check_btf_func_early()
17729 verbose(env, "nonzero tailing record in func info"); in check_btf_func_early()
17750 verbose(env, in check_btf_func_early()
17756 verbose(env, in check_btf_func_early()
17765 verbose(env, "invalid type id %d in func info", in check_btf_func_early()
17788 static int check_btf_func(struct bpf_verifier_env *env, in check_btf_func() argument
17804 if (check_abnormal_return(env)) in check_btf_func()
17808 if (nfuncs != env->subprog_cnt) { in check_btf_func()
17809 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); in check_btf_func()
17815 prog = env->prog; in check_btf_func()
17829 if (env->subprog_info[i].start != krecord[i].insn_off) { in check_btf_func()
17830 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); in check_btf_func()
17843 if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { in check_btf_func()
17844 verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); in check_btf_func()
17847 if (i && !scalar_return && env->subprog_info[i].has_tail_call) { in check_btf_func()
17848 verbose(env, "tail_call is only allowed in functions that return 'int'.\n"); in check_btf_func()
17863 static void adjust_btf_func(struct bpf_verifier_env *env) in adjust_btf_func() argument
17865 struct bpf_prog_aux *aux = env->prog->aux; in adjust_btf_func()
17872 for (i = 0; i < env->subprog_cnt - env->hidden_subprog_cnt; i++) in adjust_btf_func()
17873 aux->func_info[i].insn_off = env->subprog_info[i].start; in adjust_btf_func()
17879 static int check_btf_line(struct bpf_verifier_env *env, in check_btf_line() argument
17911 prog = env->prog; in check_btf_line()
17915 sub = env->subprog_info; in check_btf_line()
17923 verbose(env, "nonzero tailing record in line_info"); in check_btf_line()
17950 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", in check_btf_line()
17958 verbose(env, in check_btf_line()
17967 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); in check_btf_line()
17972 if (s != env->subprog_cnt) { in check_btf_line()
17977 verbose(env, "missing bpf_line_info for func#%u\n", s); in check_btf_line()
17987 if (s != env->subprog_cnt) { in check_btf_line()
17988 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", in check_btf_line()
17989 env->subprog_cnt - s, s); in check_btf_line()
18007 static int check_core_relo(struct bpf_verifier_env *env, in check_core_relo() argument
18013 struct bpf_prog *prog = env->prog; in check_core_relo()
18016 .log = &env->log, in check_core_relo()
18046 verbose(env, "nonzero tailing record in core_relo"); in check_core_relo()
18061 verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n", in check_core_relo()
18076 static int check_btf_info_early(struct bpf_verifier_env *env, in check_btf_info_early() argument
18084 if (check_abnormal_return(env)) in check_btf_info_early()
18096 env->prog->aux->btf = btf; in check_btf_info_early()
18098 err = check_btf_func_early(env, attr, uattr); in check_btf_info_early()
18104 static int check_btf_info(struct bpf_verifier_env *env, in check_btf_info() argument
18111 if (check_abnormal_return(env)) in check_btf_info()
18116 err = check_btf_func(env, attr, uattr); in check_btf_info()
18120 err = check_btf_line(env, attr, uattr); in check_btf_info()
18124 err = check_core_relo(env, attr, uattr); in check_btf_info()
18196 static void clean_func_state(struct bpf_verifier_env *env, in clean_func_state() argument
18210 __mark_reg_not_init(env, &st->regs[i]); in clean_func_state()
18218 __mark_reg_not_init(env, &st->stack[i].spilled_ptr); in clean_func_state()
18225 static void clean_verifier_state(struct bpf_verifier_env *env, in clean_verifier_state() argument
18235 clean_func_state(env, st->frame[i]); in clean_verifier_state()
18270 static void clean_live_states(struct bpf_verifier_env *env, int insn, in clean_live_states() argument
18277 head = explored_state(env, insn); in clean_live_states()
18282 loop_entry = get_loop_entry(env, &sl->state); in clean_live_states()
18288 clean_verifier_state(env, &sl->state); in clean_live_states()
18308 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, in regsafe() argument
18350 if (env->explore_alu_limits) { in regsafe()
18446 static bool is_stack_all_misc(struct bpf_verifier_env *env, in is_stack_all_misc() argument
18453 (stack->slot_type[i] == STACK_INVALID && env->allow_uninit_stack)) in is_stack_all_misc()
18461 static struct bpf_reg_state *scalar_reg_for_stack(struct bpf_verifier_env *env, in scalar_reg_for_stack() argument
18467 if (is_stack_all_misc(env, stack)) in scalar_reg_for_stack()
18473 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, in stacksafe() argument
18504 if (env->allow_uninit_stack && in stacksafe()
18519 old_reg = scalar_reg_for_stack(env, &old->stack[spi]); in stacksafe()
18520 cur_reg = scalar_reg_for_stack(env, &cur->stack[spi]); in stacksafe()
18522 if (!regsafe(env, old_reg, cur_reg, idmap, exact)) in stacksafe()
18558 if (!regsafe(env, &old->stack[spi].spilled_ptr, in stacksafe()
18678 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old, in func_states_equal() argument
18681 u16 live_regs = env->insn_aux_data[insn_idx].live_regs_before; in func_states_equal()
18689 !regsafe(env, &old->regs[i], &cur->regs[i], in func_states_equal()
18690 &env->idmap_scratch, exact)) in func_states_equal()
18693 if (!stacksafe(env, old, cur, &env->idmap_scratch, exact)) in func_states_equal()
18699 static void reset_idmap_scratch(struct bpf_verifier_env *env) in reset_idmap_scratch() argument
18701 env->idmap_scratch.tmp_id_gen = env->id_gen; in reset_idmap_scratch()
18702 memset(&env->idmap_scratch.map, 0, sizeof(env->idmap_scratch.map)); in reset_idmap_scratch()
18705 static bool states_equal(struct bpf_verifier_env *env, in states_equal() argument
18716 reset_idmap_scratch(env); in states_equal()
18727 if (!refsafe(old, cur, &env->idmap_scratch)) in states_equal()
18735 ? env->insn_idx in states_equal()
18739 if (!func_states_equal(env, old->frame[i], cur->frame[i], insn_idx, exact)) in states_equal()
18748 static int propagate_liveness_reg(struct bpf_verifier_env *env, in propagate_liveness_reg() argument
18767 err = mark_reg_read(env, reg, parent_reg, flag); in propagate_liveness_reg()
18781 static int propagate_liveness(struct bpf_verifier_env *env, in propagate_liveness() argument
18803 err = propagate_liveness_reg(env, &state_reg[i], in propagate_liveness()
18808 mark_insn_zext(env, &parent_reg[i]); in propagate_liveness()
18816 err = propagate_liveness_reg(env, state_reg, in propagate_liveness()
18828 static int propagate_precision(struct bpf_verifier_env *env, in propagate_precision() argument
18845 if (env->log.level & BPF_LOG_LEVEL2) { in propagate_precision()
18847 verbose(env, "frame %d: propagating r%d", fr, i); in propagate_precision()
18849 verbose(env, ",r%d", i); in propagate_precision()
18851 bt_set_frame_reg(&env->bt, fr, i); in propagate_precision()
18863 if (env->log.level & BPF_LOG_LEVEL2) { in propagate_precision()
18865 verbose(env, "frame %d: propagating fp%d", in propagate_precision()
18868 verbose(env, ",fp%d", (-i - 1) * BPF_REG_SIZE); in propagate_precision()
18870 bt_set_frame_slot(&env->bt, fr, i); in propagate_precision()
18874 verbose(env, "\n"); in propagate_precision()
18877 err = mark_chain_precision_batch(env); in propagate_precision()
18902 static bool is_iter_next_insn(struct bpf_verifier_env *env, int insn_idx) in is_iter_next_insn() argument
18904 return env->insn_aux_data[insn_idx].is_iter_next; in is_iter_next_insn()
18989 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) in is_state_visited() argument
18993 struct bpf_verifier_state *cur = env->cur_state, *new, *loop_entry; in is_state_visited()
18998 force_new_state = env->test_state_freq || is_force_checkpoint(env, insn_idx) || in is_state_visited()
19011 if (env->jmps_processed - env->prev_jmps_processed >= 2 && in is_state_visited()
19012 env->insn_processed - env->prev_insn_processed >= 8) in is_state_visited()
19015 clean_live_states(env, insn_idx, cur); in is_state_visited()
19017 head = explored_state(env, insn_idx); in is_state_visited()
19078 if (is_iter_next_insn(env, insn_idx)) { in is_state_visited()
19079 if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) { in is_state_visited()
19094 iter_state = &func(env, iter_reg)->stack[spi].spilled_ptr; in is_state_visited()
19096 update_loop_entry(env, cur, &sl->state); in is_state_visited()
19102 if (is_may_goto_insn_at(env, insn_idx)) { in is_state_visited()
19104 states_equal(env, &sl->state, cur, RANGE_WITHIN)) { in is_state_visited()
19105 update_loop_entry(env, cur, &sl->state); in is_state_visited()
19109 if (calls_callback(env, insn_idx)) { in is_state_visited()
19110 if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) in is_state_visited()
19116 states_equal(env, &sl->state, cur, EXACT) && in is_state_visited()
19120 verbose_linfo(env, insn_idx, "; "); in is_state_visited()
19121 verbose(env, "infinite loop detected at insn %d\n", insn_idx); in is_state_visited()
19122 verbose(env, "cur state:"); in is_state_visited()
19123 print_verifier_state(env, cur, cur->curframe, true); in is_state_visited()
19124 verbose(env, "old state:"); in is_state_visited()
19125 print_verifier_state(env, &sl->state, cur->curframe, true); in is_state_visited()
19142 env->jmps_processed - env->prev_jmps_processed < 20 && in is_state_visited()
19143 env->insn_processed - env->prev_insn_processed < 100) in is_state_visited()
19172 loop_entry = get_loop_entry(env, &sl->state); in is_state_visited()
19176 if (states_equal(env, &sl->state, cur, force_exact ? RANGE_WITHIN : NOT_EXACT)) { in is_state_visited()
19178 update_loop_entry(env, cur, loop_entry); in is_state_visited()
19191 err = propagate_liveness(env, &sl->state, cur); in is_state_visited()
19198 if (is_jmp_point(env, env->insn_idx)) in is_state_visited()
19199 err = err ? : push_insn_history(env, cur, 0, 0); in is_state_visited()
19200 err = err ? : propagate_precision(env, &sl->state); in is_state_visited()
19222 n = is_force_checkpoint(env, insn_idx) && sl->state.branches > 0 ? 64 : 3; in is_state_visited()
19229 list_add(&sl->node, &env->free_list); in is_state_visited()
19230 env->free_list_size++; in is_state_visited()
19231 env->explored_states_size--; in is_state_visited()
19232 maybe_free_verifier_state(env, sl); in is_state_visited()
19236 if (env->max_states_per_insn < states_cnt) in is_state_visited()
19237 env->max_states_per_insn = states_cnt; in is_state_visited()
19239 if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) in is_state_visited()
19257 env->total_states++; in is_state_visited()
19258 env->explored_states_size++; in is_state_visited()
19259 update_peak_states(env); in is_state_visited()
19260 env->prev_jmps_processed = env->jmps_processed; in is_state_visited()
19261 env->prev_insn_processed = env->insn_processed; in is_state_visited()
19264 if (env->bpf_capable) in is_state_visited()
19265 mark_all_scalars_imprecise(env, cur); in is_state_visited()
19354 static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, in save_aux_ptr_type() argument
19357 enum bpf_reg_type *prev_type = &env->insn_aux_data[env->insn_idx].ptr_type; in save_aux_ptr_type()
19384 verbose(env, "same insn cannot be used with different pointers\n"); in save_aux_ptr_type()
19392 static int do_check(struct bpf_verifier_env *env) in do_check() argument
19394 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); in do_check()
19395 struct bpf_verifier_state *state = env->cur_state; in do_check()
19396 struct bpf_insn *insns = env->prog->insnsi; in do_check()
19398 int insn_cnt = env->prog->len; in do_check()
19409 env->cur_hist_ent = NULL; in do_check()
19411 env->prev_insn_idx = prev_insn_idx; in do_check()
19412 if (env->insn_idx >= insn_cnt) { in do_check()
19413 verbose(env, "invalid insn idx %d insn_cnt %d\n", in do_check()
19414 env->insn_idx, insn_cnt); in do_check()
19418 insn = &insns[env->insn_idx]; in do_check()
19421 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { in do_check()
19422 verbose(env, in do_check()
19424 env->insn_processed); in do_check()
19428 state->last_insn_idx = env->prev_insn_idx; in do_check()
19430 if (is_prune_point(env, env->insn_idx)) { in do_check()
19431 err = is_state_visited(env, env->insn_idx); in do_check()
19436 if (env->log.level & BPF_LOG_LEVEL) { in do_check()
19438 verbose(env, "\nfrom %d to %d%s: safe\n", in do_check()
19439 env->prev_insn_idx, env->insn_idx, in do_check()
19440 env->cur_state->speculative ? in do_check()
19443 verbose(env, "%d: safe\n", env->insn_idx); in do_check()
19449 if (is_jmp_point(env, env->insn_idx)) { in do_check()
19450 err = push_insn_history(env, state, 0, 0); in do_check()
19461 if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) { in do_check()
19462 verbose(env, "\nfrom %d to %d%s:", in do_check()
19463 env->prev_insn_idx, env->insn_idx, in do_check()
19464 env->cur_state->speculative ? in do_check()
19466 print_verifier_state(env, state, state->curframe, true); in do_check()
19470 if (env->log.level & BPF_LOG_LEVEL) { in do_check()
19471 if (verifier_state_scratched(env)) in do_check()
19472 print_insn_state(env, state, state->curframe); in do_check()
19474 verbose_linfo(env, env->insn_idx, "; "); in do_check()
19475 env->prev_log_pos = env->log.end_pos; in do_check()
19476 verbose(env, "%d: ", env->insn_idx); in do_check()
19477 verbose_insn(env, insn); in do_check()
19478 env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos; in do_check()
19479 env->prev_log_pos = env->log.end_pos; in do_check()
19482 if (bpf_prog_is_offloaded(env->prog->aux)) { in do_check()
19483 err = bpf_prog_offload_verify_insn(env, env->insn_idx, in do_check()
19484 env->prev_insn_idx); in do_check()
19489 regs = cur_regs(env); in do_check()
19490 sanitize_mark_insn_seen(env); in do_check()
19491 prev_insn_idx = env->insn_idx; in do_check()
19494 err = check_alu_op(env, insn); in do_check()
19504 err = check_load_mem(env, insn, false, is_ldsx, true, in do_check()
19510 err = check_atomic(env, insn); in do_check()
19513 env->insn_idx++; in do_check()
19518 verbose(env, "BPF_STX uses reserved fields\n"); in do_check()
19522 err = check_store_reg(env, insn, false); in do_check()
19530 verbose(env, "BPF_ST uses reserved fields\n"); in do_check()
19534 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
19541 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in do_check()
19547 err = save_aux_ptr_type(env, dst_reg_type, false); in do_check()
19553 env->jmps_processed++; in do_check()
19563 verbose(env, "BPF_CALL uses reserved fields\n"); in do_check()
19567 if (env->cur_state->active_locks) { in do_check()
19571 verbose(env, "function calls are not allowed while holding a lock\n"); in do_check()
19576 err = check_func_call(env, insn, &env->insn_idx); in do_check()
19578 err = check_kfunc_call(env, insn, &env->insn_idx); in do_check()
19584 err = check_helper_call(env, insn, &env->insn_idx); in do_check()
19589 mark_reg_scratched(env, BPF_REG_0); in do_check()
19596 verbose(env, "BPF_JA uses reserved fields\n"); in do_check()
19601 env->insn_idx += insn->off + 1; in do_check()
19603 env->insn_idx += insn->imm + 1; in do_check()
19612 verbose(env, "BPF_EXIT uses reserved fields\n"); in do_check()
19622 err = check_resource_leak(env, exception_exit, !env->cur_state->curframe, in do_check()
19642 err = prepare_func_exit(env, &env->insn_idx); in do_check()
19649 err = check_return_code(env, BPF_REG_0, "R0"); in do_check()
19653 mark_verifier_state_scratched(env); in do_check()
19654 update_branch_counts(env, env->cur_state); in do_check()
19655 err = pop_stack(env, &prev_insn_idx, in do_check()
19656 &env->insn_idx, pop_log); in do_check()
19662 if (WARN_ON_ONCE(env->cur_state->loop_entry)) { in do_check()
19663 verbose(env, "verifier bug: env->cur_state->loop_entry != NULL\n"); in do_check()
19670 err = check_cond_jmp_op(env, insn, &env->insn_idx); in do_check()
19678 err = check_ld_abs(env, insn); in do_check()
19683 err = check_ld_imm(env, insn); in do_check()
19687 env->insn_idx++; in do_check()
19688 sanitize_mark_insn_seen(env); in do_check()
19690 verbose(env, "invalid BPF_LD mode\n"); in do_check()
19694 verbose(env, "unknown insn class %d\n", class); in do_check()
19698 env->insn_idx++; in do_check()
19740 static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf) in __add_used_btf() argument
19746 for (i = 0; i < env->used_btf_cnt; i++) in __add_used_btf()
19747 if (env->used_btfs[i].btf == btf) in __add_used_btf()
19750 if (env->used_btf_cnt >= MAX_USED_BTFS) in __add_used_btf()
19755 btf_mod = &env->used_btfs[env->used_btf_cnt]; in __add_used_btf()
19768 return env->used_btf_cnt++; in __add_used_btf()
19772 static int __check_pseudo_btf_id(struct bpf_verifier_env *env, in __check_pseudo_btf_id() argument
19789 verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); in __check_pseudo_btf_id()
19794 verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR or KIND_FUNC\n", id); in __check_pseudo_btf_id()
19801 verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", in __check_pseudo_btf_id()
19840 verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", in __check_pseudo_btf_id()
19855 static int check_pseudo_btf_id(struct bpf_verifier_env *env, in check_pseudo_btf_id() argument
19869 verbose(env, "invalid module BTF object FD specified.\n"); in check_pseudo_btf_id()
19874 …verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"… in check_pseudo_btf_id()
19880 err = __check_pseudo_btf_id(env, insn, aux, btf); in check_pseudo_btf_id()
19884 err = __add_used_btf(env, btf); in check_pseudo_btf_id()
19910 static int check_map_prog_compatibility(struct bpf_verifier_env *env, in check_map_prog_compatibility() argument
19920 verbose(env, "tracing progs cannot use bpf_{list_head,rb_root} yet\n"); in check_map_prog_compatibility()
19927 verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); in check_map_prog_compatibility()
19932 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); in check_map_prog_compatibility()
19939 verbose(env, "tracing progs cannot use bpf_timer yet\n"); in check_map_prog_compatibility()
19946 verbose(env, "tracing progs cannot use bpf_wq yet\n"); in check_map_prog_compatibility()
19953 verbose(env, "offload device mismatch between prog and map\n"); in check_map_prog_compatibility()
19958 verbose(env, "bpf_struct_ops map cannot be used in prog\n"); in check_map_prog_compatibility()
19983 verbose(env, in check_map_prog_compatibility()
19989 bpf_cgroup_storage_assign(env->prog->aux, map)) { in check_map_prog_compatibility()
19990 verbose(env, "only one cgroup storage of each type is allowed\n"); in check_map_prog_compatibility()
19995 if (env->prog->aux->arena) { in check_map_prog_compatibility()
19996 verbose(env, "Only one arena per program\n"); in check_map_prog_compatibility()
19999 if (!env->allow_ptr_leaks || !env->bpf_capable) { in check_map_prog_compatibility()
20000 verbose(env, "CAP_BPF and CAP_PERFMON are required to use arena\n"); in check_map_prog_compatibility()
20003 if (!env->prog->jit_requested) { in check_map_prog_compatibility()
20004 verbose(env, "JIT is required to use arena\n"); in check_map_prog_compatibility()
20008 verbose(env, "JIT doesn't support arena\n"); in check_map_prog_compatibility()
20011 env->prog->aux->arena = (void *)map; in check_map_prog_compatibility()
20012 if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) { in check_map_prog_compatibility()
20013 verbose(env, "arena's user address must be set via map_extra or mmap()\n"); in check_map_prog_compatibility()
20021 static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) in __add_used_map() argument
20026 for (i = 0; i < env->used_map_cnt; i++) in __add_used_map()
20027 if (env->used_maps[i] == map) in __add_used_map()
20030 if (env->used_map_cnt >= MAX_USED_MAPS) { in __add_used_map()
20031 verbose(env, "The total number of maps per program has reached the limit of %u\n", in __add_used_map()
20036 err = check_map_prog_compatibility(env, map, env->prog); in __add_used_map()
20040 if (env->prog->sleepable) in __add_used_map()
20050 env->used_maps[env->used_map_cnt++] = map; in __add_used_map()
20052 return env->used_map_cnt - 1; in __add_used_map()
20059 static int add_used_map(struct bpf_verifier_env *env, int fd) in add_used_map() argument
20066 verbose(env, "fd %d is not pointing to valid bpf_map\n", fd); in add_used_map()
20070 return __add_used_map(env, map); in add_used_map()
20080 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) in resolve_pseudo_ldimm64() argument
20082 struct bpf_insn *insn = env->prog->insnsi; in resolve_pseudo_ldimm64()
20083 int insn_cnt = env->prog->len; in resolve_pseudo_ldimm64()
20086 err = bpf_prog_calc_tag(env->prog); in resolve_pseudo_ldimm64()
20094 verbose(env, "BPF_LDX uses reserved fields\n"); in resolve_pseudo_ldimm64()
20108 verbose(env, "invalid bpf_ld_imm64 insn\n"); in resolve_pseudo_ldimm64()
20117 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
20118 err = check_pseudo_btf_id(env, insn, aux); in resolve_pseudo_ldimm64()
20125 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
20143 verbose(env, "unrecognized bpf_ld_imm64 insn\n"); in resolve_pseudo_ldimm64()
20150 if (bpfptr_is_null(env->fd_array)) { in resolve_pseudo_ldimm64()
20151 verbose(env, "fd_idx without fd_array is invalid\n"); in resolve_pseudo_ldimm64()
20154 if (copy_from_bpfptr_offset(&fd, env->fd_array, in resolve_pseudo_ldimm64()
20164 map_idx = add_used_map(env, fd); in resolve_pseudo_ldimm64()
20167 map = env->used_maps[map_idx]; in resolve_pseudo_ldimm64()
20169 aux = &env->insn_aux_data[i]; in resolve_pseudo_ldimm64()
20179 verbose(env, "direct value offset of %u is not allowed\n", off); in resolve_pseudo_ldimm64()
20184 verbose(env, "no direct value access support for this map type\n"); in resolve_pseudo_ldimm64()
20190 verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", in resolve_pseudo_ldimm64()
20210 verbose(env, "unknown opcode %02x\n", insn->code); in resolve_pseudo_ldimm64()
20223 static void release_maps(struct bpf_verifier_env *env) in release_maps() argument
20225 __bpf_free_used_maps(env->prog->aux, env->used_maps, in release_maps()
20226 env->used_map_cnt); in release_maps()
20230 static void release_btfs(struct bpf_verifier_env *env) in release_btfs() argument
20232 __bpf_free_used_btfs(env->used_btfs, env->used_btf_cnt); in release_btfs()
20236 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) in convert_pseudo_ld_imm64() argument
20238 struct bpf_insn *insn = env->prog->insnsi; in convert_pseudo_ld_imm64()
20239 int insn_cnt = env->prog->len; in convert_pseudo_ld_imm64()
20255 static void adjust_insn_aux_data(struct bpf_verifier_env *env, in adjust_insn_aux_data() argument
20259 struct bpf_insn_aux_data *old_data = env->insn_aux_data; in adjust_insn_aux_data()
20269 old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1); in adjust_insn_aux_data()
20281 new_data[i].zext_dst = insn_has_def32(env, insn + i); in adjust_insn_aux_data()
20283 env->insn_aux_data = new_data; in adjust_insn_aux_data()
20287 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) in adjust_subprog_starts() argument
20294 for (i = 0; i <= env->subprog_cnt; i++) { in adjust_subprog_starts()
20295 if (env->subprog_info[i].start <= off) in adjust_subprog_starts()
20297 env->subprog_info[i].start += len - 1; in adjust_subprog_starts()
20315 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, in bpf_patch_insn_data() argument
20322 new_data = vzalloc(array_size(env->prog->len + len - 1, in bpf_patch_insn_data()
20328 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); in bpf_patch_insn_data()
20331 verbose(env, in bpf_patch_insn_data()
20333 env->insn_aux_data[off].orig_idx); in bpf_patch_insn_data()
20337 adjust_insn_aux_data(env, new_data, new_prog, off, len); in bpf_patch_insn_data()
20338 adjust_subprog_starts(env, off, len); in bpf_patch_insn_data()
20381 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, in adjust_subprog_starts_after_remove() argument
20387 for (i = 0; i < env->subprog_cnt; i++) in adjust_subprog_starts_after_remove()
20388 if (env->subprog_info[i].start >= off) in adjust_subprog_starts_after_remove()
20391 for (j = i; j < env->subprog_cnt; j++) in adjust_subprog_starts_after_remove()
20392 if (env->subprog_info[j].start >= off + cnt) in adjust_subprog_starts_after_remove()
20397 if (env->subprog_info[j].start != off + cnt) in adjust_subprog_starts_after_remove()
20401 struct bpf_prog_aux *aux = env->prog->aux; in adjust_subprog_starts_after_remove()
20405 move = env->subprog_cnt + 1 - j; in adjust_subprog_starts_after_remove()
20407 memmove(env->subprog_info + i, in adjust_subprog_starts_after_remove()
20408 env->subprog_info + j, in adjust_subprog_starts_after_remove()
20409 sizeof(*env->subprog_info) * move); in adjust_subprog_starts_after_remove()
20410 env->subprog_cnt -= j - i; in adjust_subprog_starts_after_remove()
20426 if (env->subprog_info[i].start == off) in adjust_subprog_starts_after_remove()
20431 for (; i <= env->subprog_cnt; i++) in adjust_subprog_starts_after_remove()
20432 env->subprog_info[i].start -= cnt; in adjust_subprog_starts_after_remove()
20437 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, in bpf_adj_linfo_after_remove() argument
20440 struct bpf_prog *prog = env->prog; in bpf_adj_linfo_after_remove()
20487 for (i = 0; i <= env->subprog_cnt; i++) in bpf_adj_linfo_after_remove()
20488 if (env->subprog_info[i].linfo_idx > l_off) { in bpf_adj_linfo_after_remove()
20492 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) in bpf_adj_linfo_after_remove()
20493 env->subprog_info[i].linfo_idx -= l_cnt; in bpf_adj_linfo_after_remove()
20495 env->subprog_info[i].linfo_idx = l_off; in bpf_adj_linfo_after_remove()
20501 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) in verifier_remove_insns() argument
20503 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in verifier_remove_insns()
20504 unsigned int orig_prog_len = env->prog->len; in verifier_remove_insns()
20507 if (bpf_prog_is_offloaded(env->prog->aux)) in verifier_remove_insns()
20508 bpf_prog_offload_remove_insns(env, off, cnt); in verifier_remove_insns()
20510 err = bpf_remove_insns(env->prog, off, cnt); in verifier_remove_insns()
20514 err = adjust_subprog_starts_after_remove(env, off, cnt); in verifier_remove_insns()
20518 err = bpf_adj_linfo_after_remove(env, off, cnt); in verifier_remove_insns()
20539 static void sanitize_dead_code(struct bpf_verifier_env *env) in sanitize_dead_code() argument
20541 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in sanitize_dead_code()
20543 struct bpf_insn *insn = env->prog->insnsi; in sanitize_dead_code()
20544 const int insn_cnt = env->prog->len; in sanitize_dead_code()
20569 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) in opt_hard_wire_dead_code_branches() argument
20571 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in opt_hard_wire_dead_code_branches()
20573 struct bpf_insn *insn = env->prog->insnsi; in opt_hard_wire_dead_code_branches()
20574 const int insn_cnt = env->prog->len; in opt_hard_wire_dead_code_branches()
20588 if (bpf_prog_is_offloaded(env->prog->aux)) in opt_hard_wire_dead_code_branches()
20589 bpf_prog_offload_replace_insn(env, i, &ja); in opt_hard_wire_dead_code_branches()
20595 static int opt_remove_dead_code(struct bpf_verifier_env *env) in opt_remove_dead_code() argument
20597 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; in opt_remove_dead_code()
20598 int insn_cnt = env->prog->len; in opt_remove_dead_code()
20610 err = verifier_remove_insns(env, i, j); in opt_remove_dead_code()
20613 insn_cnt = env->prog->len; in opt_remove_dead_code()
20622 static int opt_remove_nops(struct bpf_verifier_env *env) in opt_remove_nops() argument
20624 struct bpf_insn *insn = env->prog->insnsi; in opt_remove_nops()
20625 int insn_cnt = env->prog->len; in opt_remove_nops()
20636 err = verifier_remove_insns(env, i, 1); in opt_remove_nops()
20647 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, in opt_subreg_zext_lo32_rnd_hi32() argument
20651 struct bpf_insn_aux_data *aux = env->insn_aux_data; in opt_subreg_zext_lo32_rnd_hi32()
20652 int i, patch_len, delta = 0, len = env->prog->len; in opt_subreg_zext_lo32_rnd_hi32()
20653 struct bpf_insn *insns = env->prog->insnsi; in opt_subreg_zext_lo32_rnd_hi32()
20685 if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) { in opt_subreg_zext_lo32_rnd_hi32()
20724 verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n"); in opt_subreg_zext_lo32_rnd_hi32()
20734 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); in opt_subreg_zext_lo32_rnd_hi32()
20737 env->prog = new_prog; in opt_subreg_zext_lo32_rnd_hi32()
20739 aux = env->insn_aux_data; in opt_subreg_zext_lo32_rnd_hi32()
20751 static int convert_ctx_accesses(struct bpf_verifier_env *env) in convert_ctx_accesses() argument
20753 struct bpf_subprog_info *subprogs = env->subprog_info; in convert_ctx_accesses()
20754 const struct bpf_verifier_ops *ops = env->ops; in convert_ctx_accesses()
20756 const int insn_cnt = env->prog->len; in convert_ctx_accesses()
20757 struct bpf_insn *epilogue_buf = env->epilogue_buf; in convert_ctx_accesses()
20758 struct bpf_insn *insn_buf = env->insn_buf; in convert_ctx_accesses()
20767 epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog, in convert_ctx_accesses()
20770 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
20778 insn_buf[cnt++] = env->prog->insnsi[0]; in convert_ctx_accesses()
20779 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); in convert_ctx_accesses()
20782 env->prog = new_prog; in convert_ctx_accesses()
20785 ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1); in convert_ctx_accesses()
20791 if (ops->gen_prologue || env->seen_direct_write) { in convert_ctx_accesses()
20793 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
20796 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, in convert_ctx_accesses()
20797 env->prog); in convert_ctx_accesses()
20799 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
20802 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); in convert_ctx_accesses()
20806 env->prog = new_prog; in convert_ctx_accesses()
20809 ret = add_kfunc_in_insns(env, insn_buf, cnt - 1); in convert_ctx_accesses()
20816 WARN_ON(adjust_jmp_off(env->prog, 0, delta)); in convert_ctx_accesses()
20818 if (bpf_prog_is_offloaded(env->prog->aux)) in convert_ctx_accesses()
20821 insn = env->prog->insnsi + delta; in convert_ctx_accesses()
20848 env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) { in convert_ctx_accesses()
20850 env->prog->aux->num_exentries++; in convert_ctx_accesses()
20876 env->insn_aux_data[i + delta].sanitize_stack_spill) { in convert_ctx_accesses()
20883 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); in convert_ctx_accesses()
20888 env->prog = new_prog; in convert_ctx_accesses()
20893 switch ((int)env->insn_aux_data[i + delta].ptr_type) { in convert_ctx_accesses()
20925 env->prog->aux->num_exentries++; in convert_ctx_accesses()
20930 verbose(env, "sign extending loads from arena are not supported yet\n"); in convert_ctx_accesses()
20934 env->prog->aux->num_exentries++; in convert_ctx_accesses()
20940 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; in convert_ctx_accesses()
20956 verbose(env, "bpf verifier narrow ctx access misconfigured\n"); in convert_ctx_accesses()
20971 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, in convert_ctx_accesses()
20975 verbose(env, "bpf verifier is misconfigured\n"); in convert_ctx_accesses()
20983 verbose(env, "bpf verifier narrow ctx load misconfigured\n"); in convert_ctx_accesses()
21008 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in convert_ctx_accesses()
21015 env->prog = new_prog; in convert_ctx_accesses()
21022 static int jit_subprogs(struct bpf_verifier_env *env) in jit_subprogs() argument
21024 struct bpf_prog *prog = env->prog, **func, *tmp; in jit_subprogs()
21031 if (env->subprog_cnt <= 1) in jit_subprogs()
21042 subprog = find_subprog(env, i + insn->imm + 1); in jit_subprogs()
21055 env->insn_aux_data[i].call_imm = insn->imm; in jit_subprogs()
21078 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); in jit_subprogs()
21082 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
21084 subprog_end = env->subprog_info[i + 1].start; in jit_subprogs()
21121 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; in jit_subprogs()
21122 if (env->subprog_info[i].priv_stack_mode == PRIV_STACK_ADAPTIVE) in jit_subprogs()
21132 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; in jit_subprogs()
21151 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; in jit_subprogs()
21152 func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb; in jit_subprogs()
21153 func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data; in jit_subprogs()
21154 func[i]->aux->might_sleep = env->subprog_info[i].might_sleep; in jit_subprogs()
21156 func[i]->aux->exception_boundary = env->seen_exception; in jit_subprogs()
21169 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
21196 func[i]->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; in jit_subprogs()
21197 func[i]->aux->real_func_cnt = env->subprog_cnt; in jit_subprogs()
21199 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
21203 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); in jit_subprogs()
21214 for (i = 1; i < env->subprog_cnt; i++) { in jit_subprogs()
21220 for (i = 1; i < env->subprog_cnt; i++) in jit_subprogs()
21229 insn[0].imm = env->insn_aux_data[i].call_imm; in jit_subprogs()
21236 insn->off = env->insn_aux_data[i].call_imm; in jit_subprogs()
21237 subprog = find_subprog(env, i + insn->off + 1); in jit_subprogs()
21247 prog->aux->func_cnt = env->subprog_cnt - env->hidden_subprog_cnt; in jit_subprogs()
21248 prog->aux->real_func_cnt = env->subprog_cnt; in jit_subprogs()
21249 prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func; in jit_subprogs()
21266 for (i = 0; i < env->subprog_cnt; i++) { in jit_subprogs()
21281 insn->imm = env->insn_aux_data[i].call_imm; in jit_subprogs()
21287 static int fixup_call_args(struct bpf_verifier_env *env) in fixup_call_args() argument
21290 struct bpf_prog *prog = env->prog; in fixup_call_args()
21297 if (env->prog->jit_requested && in fixup_call_args()
21298 !bpf_prog_is_offloaded(env->prog->aux)) { in fixup_call_args()
21299 err = jit_subprogs(env); in fixup_call_args()
21307 verbose(env, "calling kernel functions are not allowed in non-JITed programs\n"); in fixup_call_args()
21310 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { in fixup_call_args()
21314 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); in fixup_call_args()
21322 verbose(env, "callbacks are not allowed in non-JITed programs\n"); in fixup_call_args()
21328 depth = get_callee_stack_depth(env, insn, i); in fixup_call_args()
21339 static void specialize_kfunc(struct bpf_verifier_env *env, in specialize_kfunc() argument
21342 struct bpf_prog *prog = env->prog; in specialize_kfunc()
21360 seen_direct_write = env->seen_direct_write; in specialize_kfunc()
21361 is_rdonly = !may_access_direct_pkt_data(env, NULL, BPF_WRITE); in specialize_kfunc()
21369 env->seen_direct_write = seen_direct_write; in specialize_kfunc()
21398 static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, in fixup_kfunc_call() argument
21404 verbose(env, "invalid kernel function call not eliminated in verifier pass\n"); in fixup_kfunc_call()
21414 desc = find_kfunc_desc(env->prog, insn->imm, insn->off); in fixup_kfunc_call()
21416 verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n", in fixup_kfunc_call()
21427 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; in fixup_kfunc_call()
21429 u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size; in fixup_kfunc_call()
21432 verbose(env, "verifier internal error: NULL kptr_struct_meta expected at insn_idx %d\n", in fixup_kfunc_call()
21445 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; in fixup_kfunc_call()
21449 verbose(env, "verifier internal error: NULL kptr_struct_meta expected at insn_idx %d\n", in fixup_kfunc_call()
21456 verbose(env, "verifier internal error: kptr_struct_meta expected at insn_idx %d\n", in fixup_kfunc_call()
21468 struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta; in fixup_kfunc_call()
21479 verbose(env, "verifier internal error: kptr_struct_meta expected at insn_idx %d\n", in fixup_kfunc_call()
21484 __fixup_collection_insert_kfunc(&env->insn_aux_data[insn_idx], struct_meta_reg, in fixup_kfunc_call()
21491 struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(BPF_REG_4, (long)env->prog->aux) }; in fixup_kfunc_call()
21502 static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len) in add_hidden_subprog() argument
21504 struct bpf_subprog_info *info = env->subprog_info; in add_hidden_subprog()
21505 int cnt = env->subprog_cnt; in add_hidden_subprog()
21509 if (env->hidden_subprog_cnt) { in add_hidden_subprog()
21510 verbose(env, "verifier internal error: only one hidden subprog supported\n"); in add_hidden_subprog()
21517 prog = bpf_patch_insn_data(env, env->prog->len - 1, patch, len); in add_hidden_subprog()
21520 env->prog = prog; in add_hidden_subprog()
21523 env->subprog_cnt++; in add_hidden_subprog()
21524 env->hidden_subprog_cnt++; in add_hidden_subprog()
21531 static int do_misc_fixups(struct bpf_verifier_env *env) in do_misc_fixups() argument
21533 struct bpf_prog *prog = env->prog; in do_misc_fixups()
21541 struct bpf_insn *insn_buf = env->insn_buf; in do_misc_fixups()
21545 struct bpf_subprog_info *subprogs = env->subprog_info; in do_misc_fixups()
21549 if (env->seen_exception && !env->exception_callback_subprog) { in do_misc_fixups()
21551 env->prog->insnsi[insn_cnt - 1], in do_misc_fixups()
21556 ret = add_hidden_subprog(env, patch, ARRAY_SIZE(patch)); in do_misc_fixups()
21559 prog = env->prog; in do_misc_fixups()
21562 env->exception_callback_subprog = env->subprog_cnt - 1; in do_misc_fixups()
21564 mark_subprog_exc_cb(env, env->exception_callback_subprog); in do_misc_fixups()
21570 (((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) { in do_misc_fixups()
21580 if (env->insn_aux_data[i + delta].needs_zext) in do_misc_fixups()
21605 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); in do_misc_fixups()
21610 env->prog = prog = new_prog; in do_misc_fixups()
21700 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); in do_misc_fixups()
21705 env->prog = prog = new_prog; in do_misc_fixups()
21730 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21735 env->prog = prog = new_prog; in do_misc_fixups()
21744 cnt = env->ops->gen_ld_abs(insn, insn_buf); in do_misc_fixups()
21746 verbose(env, "bpf verifier is misconfigured\n"); in do_misc_fixups()
21750 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21755 env->prog = prog = new_prog; in do_misc_fixups()
21769 aux = &env->insn_aux_data[i + delta]; in do_misc_fixups()
21803 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21808 env->prog = prog = new_prog; in do_misc_fixups()
21848 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21853 env->prog = prog = new_prog; in do_misc_fixups()
21869 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21874 env->prog = prog = new_prog; in do_misc_fixups()
21884 ret = fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt); in do_misc_fixups()
21890 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21895 env->prog = prog = new_prog; in do_misc_fixups()
21917 if (!allow_tail_call_in_subprogs(env)) in do_misc_fixups()
21929 aux = &env->insn_aux_data[i + delta]; in do_misc_fixups()
21930 if (env->bpf_capable && !prog->blinding_requested && in do_misc_fixups()
21944 verbose(env, "adding tail call poke descriptor failed\n"); in do_misc_fixups()
21962 verbose(env, "tail_call abusing map_ptr\n"); in do_misc_fixups()
21975 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
21980 env->prog = prog = new_prog; in do_misc_fixups()
22008 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
22013 env->prog = prog = new_prog; in do_misc_fixups()
22019 if (!in_sleepable(env) || in do_misc_fixups()
22020 env->insn_aux_data[i + delta].storage_get_func_atomic) in do_misc_fixups()
22027 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
22032 env->prog = prog = new_prog; in do_misc_fixups()
22038 if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) { in do_misc_fixups()
22046 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
22051 env->prog = prog = new_prog; in do_misc_fixups()
22070 aux = &env->insn_aux_data[i + delta]; in do_misc_fixups()
22082 verbose(env, "bpf verifier is misconfigured\n"); in do_misc_fixups()
22086 new_prog = bpf_patch_insn_data(env, i + delta, in do_misc_fixups()
22092 env->prog = prog = new_prog; in do_misc_fixups()
22169 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, in do_misc_fixups()
22175 env->prog = prog = new_prog; in do_misc_fixups()
22183 verifier_inlines_helper_call(env, insn->imm)) { in do_misc_fixups()
22198 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
22203 env->prog = prog = new_prog; in do_misc_fixups()
22223 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
22228 env->prog = prog = new_prog; in do_misc_fixups()
22251 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
22256 env->prog = prog = new_prog; in do_misc_fixups()
22267 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); in do_misc_fixups()
22271 env->prog = prog = new_prog; in do_misc_fixups()
22282 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); in do_misc_fixups()
22286 env->prog = prog = new_prog; in do_misc_fixups()
22337 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
22342 env->prog = prog = new_prog; in do_misc_fixups()
22355 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); in do_misc_fixups()
22360 env->prog = prog = new_prog; in do_misc_fixups()
22365 fn = env->ops->get_func_proto(insn->imm, env->prog); in do_misc_fixups()
22370 verbose(env, in do_misc_fixups()
22383 verbose(env, "stack size %d(extra %d) is too large\n", in do_misc_fixups()
22395 env->prog->aux->stack_depth = subprogs[0].stack_depth; in do_misc_fixups()
22396 for (i = 0; i < env->subprog_cnt; i++) { in do_misc_fixups()
22406 verbose(env, "verifier bug: stack_slots supports may_goto only\n"); in do_misc_fixups()
22421 insn_buf[cnt++] = env->prog->insnsi[subprog_start]; in do_misc_fixups()
22423 new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt); in do_misc_fixups()
22426 env->prog = prog = new_prog; in do_misc_fixups()
22433 WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta)); in do_misc_fixups()
22442 verbose(env, "bpf verifier is misconfigured\n"); in do_misc_fixups()
22448 verbose(env, "tracking tail call prog failed\n"); in do_misc_fixups()
22453 sort_kfunc_descs_by_imm_off(env->prog); in do_misc_fixups()
22458 static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env, in inline_bpf_loop() argument
22471 struct bpf_insn *insn_buf = env->insn_buf; in inline_bpf_loop()
22520 new_prog = bpf_patch_insn_data(env, position, insn_buf, cnt); in inline_bpf_loop()
22525 callback_start = env->subprog_info[callback_subprogno].start; in inline_bpf_loop()
22550 static int optimize_bpf_loop(struct bpf_verifier_env *env) in optimize_bpf_loop() argument
22552 struct bpf_subprog_info *subprogs = env->subprog_info; in optimize_bpf_loop()
22554 struct bpf_insn *insn = env->prog->insnsi; in optimize_bpf_loop()
22555 int insn_cnt = env->prog->len; in optimize_bpf_loop()
22562 &env->insn_aux_data[i + delta].loop_inline_state; in optimize_bpf_loop()
22568 new_prog = inline_bpf_loop(env, in optimize_bpf_loop()
22577 env->prog = new_prog; in optimize_bpf_loop()
22590 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; in optimize_bpf_loop()
22598 static int remove_fastcall_spills_fills(struct bpf_verifier_env *env) in remove_fastcall_spills_fills() argument
22600 struct bpf_subprog_info *subprog = env->subprog_info; in remove_fastcall_spills_fills()
22601 struct bpf_insn_aux_data *aux = env->insn_aux_data; in remove_fastcall_spills_fills()
22602 struct bpf_insn *insn = env->prog->insnsi; in remove_fastcall_spills_fills()
22603 int insn_cnt = env->prog->len; in remove_fastcall_spills_fills()
22629 static void free_states(struct bpf_verifier_env *env) in free_states() argument
22635 list_for_each_safe(pos, tmp, &env->free_list) { in free_states()
22640 INIT_LIST_HEAD(&env->free_list); in free_states()
22642 if (!env->explored_states) in free_states()
22645 for (i = 0; i < state_htab_size(env); i++) { in free_states()
22646 head = &env->explored_states[i]; in free_states()
22653 INIT_LIST_HEAD(&env->explored_states[i]); in free_states()
22657 static int do_check_common(struct bpf_verifier_env *env, int subprog) in do_check_common() argument
22659 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); in do_check_common()
22660 struct bpf_subprog_info *sub = subprog_info(env, subprog); in do_check_common()
22661 struct bpf_prog_aux *aux = env->prog->aux; in do_check_common()
22666 env->prev_linfo = NULL; in do_check_common()
22667 env->pass_cnt++; in do_check_common()
22680 env->cur_state = state; in do_check_common()
22681 init_func_state(env, state->frame[0], in do_check_common()
22685 state->first_insn_idx = env->subprog_info[subprog].start; in do_check_common()
22689 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { in do_check_common()
22690 const char *sub_name = subprog_name(env, subprog); in do_check_common()
22694 verbose(env, "Validating %s() func#%d...\n", sub_name, subprog); in do_check_common()
22695 ret = btf_prepare_func_args(env, subprog); in do_check_common()
22699 if (subprog_is_exc_cb(env, subprog)) { in do_check_common()
22706 verbose(env, "exception cb only supports single integer argument\n"); in do_check_common()
22717 mark_reg_known_zero(env, regs, i); in do_check_common()
22720 mark_reg_unknown(env, regs, i); in do_check_common()
22723 __mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen); in do_check_common()
22728 mark_reg_known_zero(env, regs, i); in do_check_common()
22730 reg->id = ++env->id_gen; in do_check_common()
22739 mark_reg_known_zero(env, regs, i); in do_check_common()
22742 reg->id = ++env->id_gen; in do_check_common()
22745 mark_reg_unknown(env, regs, i); in do_check_common()
22758 if (env->prog->aux->func_info_aux) { in do_check_common()
22759 ret = btf_prepare_func_args(env, 0); in do_check_common()
22761 env->prog->aux->func_info_aux[0].unreliable = true; in do_check_common()
22766 mark_reg_known_zero(env, regs, BPF_REG_1); in do_check_common()
22770 if (!subprog && env->prog->type == BPF_PROG_TYPE_STRUCT_OPS) { in do_check_common()
22773 acquire_reference(env, 0) : 0; in do_check_common()
22776 ret = do_check(env); in do_check_common()
22781 if (env->cur_state) { in do_check_common()
22782 free_verifier_state(env->cur_state, true); in do_check_common()
22783 env->cur_state = NULL; in do_check_common()
22785 while (!pop_stack(env, NULL, NULL, false)); in do_check_common()
22787 bpf_vlog_reset(&env->log, 0); in do_check_common()
22788 free_states(env); in do_check_common()
22812 static int do_check_subprogs(struct bpf_verifier_env *env) in do_check_subprogs() argument
22814 struct bpf_prog_aux *aux = env->prog->aux; in do_check_subprogs()
22822 if (env->exception_callback_subprog) in do_check_subprogs()
22823 subprog_aux(env, env->exception_callback_subprog)->called = true; in do_check_subprogs()
22827 for (i = 1; i < env->subprog_cnt; i++) { in do_check_subprogs()
22828 if (!subprog_is_global(env, i)) in do_check_subprogs()
22831 sub_aux = subprog_aux(env, i); in do_check_subprogs()
22835 env->insn_idx = env->subprog_info[i].start; in do_check_subprogs()
22836 WARN_ON_ONCE(env->insn_idx == 0); in do_check_subprogs()
22837 ret = do_check_common(env, i); in do_check_subprogs()
22840 } else if (env->log.level & BPF_LOG_LEVEL) { in do_check_subprogs()
22841 verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n", in do_check_subprogs()
22842 i, subprog_name(env, i)); in do_check_subprogs()
22862 static int do_check_main(struct bpf_verifier_env *env) in do_check_main() argument
22866 env->insn_idx = 0; in do_check_main()
22867 ret = do_check_common(env, 0); in do_check_main()
22869 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; in do_check_main()
22874 static void print_verification_stats(struct bpf_verifier_env *env) in print_verification_stats() argument
22878 if (env->log.level & BPF_LOG_STATS) { in print_verification_stats()
22879 verbose(env, "verification time %lld usec\n", in print_verification_stats()
22880 div_u64(env->verification_time, 1000)); in print_verification_stats()
22881 verbose(env, "stack depth "); in print_verification_stats()
22882 for (i = 0; i < env->subprog_cnt; i++) { in print_verification_stats()
22883 u32 depth = env->subprog_info[i].stack_depth; in print_verification_stats()
22885 verbose(env, "%d", depth); in print_verification_stats()
22886 if (i + 1 < env->subprog_cnt) in print_verification_stats()
22887 verbose(env, "+"); in print_verification_stats()
22889 verbose(env, "\n"); in print_verification_stats()
22891 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " in print_verification_stats()
22893 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, in print_verification_stats()
22894 env->max_states_per_insn, env->total_states, in print_verification_stats()
22895 env->peak_states, env->longest_mark_read_walk); in print_verification_stats()
22907 static int check_struct_ops_btf_id(struct bpf_verifier_env *env) in check_struct_ops_btf_id() argument
22913 struct bpf_prog *prog = env->prog; in check_struct_ops_btf_id()
22921 verbose(env, "struct ops programs must have a GPL compatible license\n"); in check_struct_ops_btf_id()
22931 env->attach_btf_mod = btf_try_get_module(btf); in check_struct_ops_btf_id()
22932 if (!env->attach_btf_mod) { in check_struct_ops_btf_id()
22933 verbose(env, "struct_ops module %s is not found\n", in check_struct_ops_btf_id()
22942 verbose(env, "attach_btf_id %u is not a supported struct\n", in check_struct_ops_btf_id()
22951 verbose(env, "attach to invalid member idx %u of struct %s\n", in check_struct_ops_btf_id()
22961 verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", in check_struct_ops_btf_id()
22969 verbose(env, "attach to unsupported member %s of struct %s\n", in check_struct_ops_btf_id()
22978 verbose(env, "attach to unsupported member %s of struct %s\n", in check_struct_ops_btf_id()
22985 verbose(env, "Private stack not supported by jit\n"); in check_struct_ops_btf_id()
22999 for (i = 0; i < env->subprog_cnt; i++) { in check_struct_ops_btf_id()
23000 if (has_refcounted_arg && env->subprog_info[i].has_tail_call) { in check_struct_ops_btf_id()
23001 verbose(env, "program with __ref argument cannot tail call\n"); in check_struct_ops_btf_id()
23011 env->ops = st_ops->verifier_ops; in check_struct_ops_btf_id()
23443 static int check_attach_btf_id(struct bpf_verifier_env *env) in check_attach_btf_id() argument
23445 struct bpf_prog *prog = env->prog; in check_attach_btf_id()
23457 verbose(env, "Syscall programs can only be sleepable\n"); in check_attach_btf_id()
23462 …verbose(env, "Only fentry/fexit/fmod_ret, lsm, iter, uprobe, and struct_ops programs can be sleepa… in check_attach_btf_id()
23467 return check_struct_ops_btf_id(env); in check_attach_btf_id()
23474 ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); in check_attach_btf_id()
23483 env->ops = bpf_verifier_ops[tgt_prog->type]; in check_attach_btf_id()
23505 ret = bpf_lsm_verify_prog(&env->log, prog); in check_attach_btf_id()
23514 verbose(env, "Attaching fexit/fmod_ret to __noreturn functions is rejected.\n"); in check_attach_btf_id()
23546 static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd) in add_fd_from_fd_array() argument
23555 err = __add_used_map(env, map); in add_fd_from_fd_array()
23563 err = __add_used_btf(env, btf); in add_fd_from_fd_array()
23569 verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd); in add_fd_from_fd_array()
23573 static int process_fd_array(struct bpf_verifier_env *env, union bpf_attr *attr, bpfptr_t uattr) in process_fd_array() argument
23580 env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel); in process_fd_array()
23592 verbose(env, "fd_array_cnt is too big (%u)\n", attr->fd_array_cnt); in process_fd_array()
23597 if (copy_from_bpfptr_offset(&fd, env->fd_array, i * size, size)) in process_fd_array()
23600 ret = add_fd_from_fd_array(env, fd); in process_fd_array()
23680 static void compute_insn_live_regs(struct bpf_verifier_env *env, in compute_insn_live_regs() argument
23793 if (get_call_summary(env, insn, &cs)) in compute_insn_live_regs()
23817 static int compute_live_registers(struct bpf_verifier_env *env) in compute_live_registers() argument
23819 struct bpf_insn_aux_data *insn_aux = env->insn_aux_data; in compute_live_registers()
23820 struct bpf_insn *insns = env->prog->insnsi; in compute_live_registers()
23822 int insn_cnt = env->prog->len; in compute_live_registers()
23852 compute_insn_live_regs(env, &insns[i], &state[i]); in compute_live_registers()
23857 for (i = 0; i < env->cfg.cur_postorder; ++i) { in compute_live_registers()
23858 int insn_idx = env->cfg.insn_postorder[i]; in compute_live_registers()
23865 succ_num = insn_successors(env->prog, insn_idx, succ); in compute_live_registers()
23880 if (env->log.level & BPF_LOG_LEVEL2) { in compute_live_registers()
23881 verbose(env, "Live regs before insn:\n"); in compute_live_registers()
23883 verbose(env, "%3d: ", i); in compute_live_registers()
23886 verbose(env, "%d", j); in compute_live_registers()
23888 verbose(env, "."); in compute_live_registers()
23889 verbose(env, " "); in compute_live_registers()
23890 verbose_insn(env, &insns[i]); in compute_live_registers()
23898 kvfree(env->cfg.insn_postorder); in compute_live_registers()
23899 env->cfg.insn_postorder = NULL; in compute_live_registers()
23900 env->cfg.cur_postorder = 0; in compute_live_registers()
23907 struct bpf_verifier_env *env; in bpf_check() local
23919 env = kvzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); in bpf_check()
23920 if (!env) in bpf_check()
23923 env->bt.env = env; in bpf_check()
23926 env->insn_aux_data = in bpf_check()
23929 if (!env->insn_aux_data) in bpf_check()
23932 env->insn_aux_data[i].orig_idx = i; in bpf_check()
23933 env->prog = *prog; in bpf_check()
23934 env->ops = bpf_verifier_ops[env->prog->type]; in bpf_check()
23936 env->allow_ptr_leaks = bpf_allow_ptr_leaks(env->prog->aux->token); in bpf_check()
23937 env->allow_uninit_stack = bpf_allow_uninit_stack(env->prog->aux->token); in bpf_check()
23938 env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token); in bpf_check()
23939 env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token); in bpf_check()
23940 env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF); in bpf_check()
23951 ret = bpf_vlog_init(&env->log, attr->log_level, in bpf_check()
23957 ret = process_fd_array(env, attr, uattr); in bpf_check()
23961 mark_verifier_state_clean(env); in bpf_check()
23965 verbose(env, "in-kernel BTF is malformed\n"); in bpf_check()
23970 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); in bpf_check()
23972 env->strict_alignment = true; in bpf_check()
23974 env->strict_alignment = false; in bpf_check()
23977 env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; in bpf_check()
23978 env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS; in bpf_check()
23980 env->explored_states = kvcalloc(state_htab_size(env), in bpf_check()
23984 if (!env->explored_states) in bpf_check()
23987 for (i = 0; i < state_htab_size(env); i++) in bpf_check()
23988 INIT_LIST_HEAD(&env->explored_states[i]); in bpf_check()
23989 INIT_LIST_HEAD(&env->free_list); in bpf_check()
23991 ret = check_btf_info_early(env, attr, uattr); in bpf_check()
23995 ret = add_subprog_and_kfunc(env); in bpf_check()
23999 ret = check_subprogs(env); in bpf_check()
24003 ret = check_btf_info(env, attr, uattr); in bpf_check()
24007 ret = resolve_pseudo_ldimm64(env); in bpf_check()
24011 if (bpf_prog_is_offloaded(env->prog->aux)) { in bpf_check()
24012 ret = bpf_prog_offload_verifier_prep(env->prog); in bpf_check()
24017 ret = check_cfg(env); in bpf_check()
24021 ret = check_attach_btf_id(env); in bpf_check()
24025 ret = compute_live_registers(env); in bpf_check()
24029 ret = mark_fastcall_patterns(env); in bpf_check()
24033 ret = do_check_main(env); in bpf_check()
24034 ret = ret ?: do_check_subprogs(env); in bpf_check()
24036 if (ret == 0 && bpf_prog_is_offloaded(env->prog->aux)) in bpf_check()
24037 ret = bpf_prog_offload_finalize(env); in bpf_check()
24040 kvfree(env->explored_states); in bpf_check()
24046 ret = remove_fastcall_spills_fills(env); in bpf_check()
24049 ret = check_max_stack_depth(env); in bpf_check()
24053 ret = optimize_bpf_loop(env); in bpf_check()
24057 opt_hard_wire_dead_code_branches(env); in bpf_check()
24059 ret = opt_remove_dead_code(env); in bpf_check()
24061 ret = opt_remove_nops(env); in bpf_check()
24064 sanitize_dead_code(env); in bpf_check()
24069 ret = convert_ctx_accesses(env); in bpf_check()
24072 ret = do_misc_fixups(env); in bpf_check()
24077 if (ret == 0 && !bpf_prog_is_offloaded(env->prog->aux)) { in bpf_check()
24078 ret = opt_subreg_zext_lo32_rnd_hi32(env, attr); in bpf_check()
24079 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret in bpf_check()
24084 ret = fixup_call_args(env); in bpf_check()
24086 env->verification_time = ktime_get_ns() - start_time; in bpf_check()
24087 print_verification_stats(env); in bpf_check()
24088 env->prog->aux->verified_insns = env->insn_processed; in bpf_check()
24091 err = bpf_vlog_finalize(&env->log, &log_true_size); in bpf_check()
24105 if (env->used_map_cnt) { in bpf_check()
24107 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, in bpf_check()
24108 sizeof(env->used_maps[0]), in bpf_check()
24111 if (!env->prog->aux->used_maps) { in bpf_check()
24116 memcpy(env->prog->aux->used_maps, env->used_maps, in bpf_check()
24117 sizeof(env->used_maps[0]) * env->used_map_cnt); in bpf_check()
24118 env->prog->aux->used_map_cnt = env->used_map_cnt; in bpf_check()
24120 if (env->used_btf_cnt) { in bpf_check()
24122 env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt, in bpf_check()
24123 sizeof(env->used_btfs[0]), in bpf_check()
24125 if (!env->prog->aux->used_btfs) { in bpf_check()
24130 memcpy(env->prog->aux->used_btfs, env->used_btfs, in bpf_check()
24131 sizeof(env->used_btfs[0]) * env->used_btf_cnt); in bpf_check()
24132 env->prog->aux->used_btf_cnt = env->used_btf_cnt; in bpf_check()
24134 if (env->used_map_cnt || env->used_btf_cnt) { in bpf_check()
24138 convert_pseudo_ld_imm64(env); in bpf_check()
24141 adjust_btf_func(env); in bpf_check()
24144 if (!env->prog->aux->used_maps) in bpf_check()
24148 release_maps(env); in bpf_check()
24149 if (!env->prog->aux->used_btfs) in bpf_check()
24150 release_btfs(env); in bpf_check()
24155 if (env->prog->type == BPF_PROG_TYPE_EXT) in bpf_check()
24156 env->prog->expected_attach_type = 0; in bpf_check()
24158 *prog = env->prog; in bpf_check()
24160 module_put(env->attach_btf_mod); in bpf_check()
24164 vfree(env->insn_aux_data); in bpf_check()
24165 kvfree(env->insn_hist); in bpf_check()
24167 kvfree(env->cfg.insn_postorder); in bpf_check()
24168 kvfree(env); in bpf_check()