182 	int insn_idx;  member
199 static int acquire_reference(struct bpf_verifier_env *env, int insn_idx);
547 static bool is_may_goto_insn_at(struct bpf_verifier_env *env, int insn_idx)  in is_may_goto_insn_at()  argument
549 	return is_may_goto_insn(&env->prog->insnsi[insn_idx]);  in is_may_goto_insn_at()
724 				   enum bpf_arg_type arg_type, int insn_idx, int clone_ref_obj_id)  in mark_stack_slots_dynptr()  argument
769 			id = acquire_reference(env, insn_idx);  in mark_stack_slots_dynptr()
1021 				 struct bpf_reg_state *reg, int insn_idx,  in mark_stack_slots_iter()  argument
1031 	id = acquire_reference(env, insn_idx);  in mark_stack_slots_iter()
1153 static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx);
1158 				     struct bpf_reg_state *reg, int insn_idx,  in mark_stack_slot_irq_flag()  argument
1170 	id = acquire_irq_state(env, insn_idx);  in mark_stack_slot_irq_flag()
1217 		int insn_idx = 0;  in unmark_stack_slot_irq_flag()  local
1221 				insn_idx = env->cur_state->refs[i].insn_idx;  in unmark_stack_slot_irq_flag()
1227 			env->cur_state->active_irq_id, insn_idx);  in unmark_stack_slot_irq_flag()
1492 …tic struct bpf_reference_state *acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)  in acquire_reference_state()  argument
1501 	state->refs[new_ofs].insn_idx = insn_idx;  in acquire_reference_state()
1506 static int acquire_reference(struct bpf_verifier_env *env, int insn_idx)  in acquire_reference()  argument
1510 	s = acquire_reference_state(env, insn_idx);  in acquire_reference()
1518 static int acquire_lock_state(struct bpf_verifier_env *env, int insn_idx, enum ref_state_type type,  in acquire_lock_state()  argument
1524 	s = acquire_reference_state(env, insn_idx);  in acquire_lock_state()
1537 static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx)  in acquire_irq_state()  argument
1542 	s = acquire_reference_state(env, insn_idx);  in acquire_irq_state()
1989 		     int *insn_idx, bool pop_log)  in pop_stack()  argument
2005 	if (insn_idx)  in pop_stack()
2006 		*insn_idx = head->insn_idx;  in pop_stack()
2018 					     int insn_idx, int prev_insn_idx,  in push_stack()  argument
2029 	elem->insn_idx = insn_idx;  in push_stack()
2786 						int insn_idx, int prev_insn_idx,  in push_async_cb()  argument
2796 	elem->insn_idx = insn_idx;  in push_async_cb()
3694 	struct bpf_insn *insn = env->prog->insnsi + env->insn_idx;  in __check_reg_arg()
3729 		reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1;  in __check_reg_arg()
3765 static bool is_jmp_point(struct bpf_verifier_env *env, int insn_idx)  in is_jmp_point()  argument
3767 	return env->insn_aux_data[insn_idx].jmp_point;  in is_jmp_point()
3863 			  env->insn_idx, env->cur_hist_ent->flags, insn_flags);  in push_insn_history()
3867 			  env->insn_idx, env->cur_hist_ent->linked_regs);  in push_insn_history()
3882 	p->idx = env->insn_idx;  in push_insn_history()
3894 						       u32 hist_start, u32 hist_end, int insn_idx)  in get_insn_hist_entry()  argument
3896 	if (hist_end > hist_start && env->insn_hist[hist_end - 1].idx == insn_idx)  in get_insn_hist_entry()
3916 			     int insn_idx, u32 hist_start, u32 *hist_endp)  in get_prev_insn_idx()  argument
3921 	if (insn_idx == st->first_insn_idx) {  in get_prev_insn_idx()
3924 		if (cnt == 1 && env->insn_hist[hist_start].idx == insn_idx)  in get_prev_insn_idx()
3928 	if (cnt && env->insn_hist[hist_end - 1].idx == insn_idx) {  in get_prev_insn_idx()
3932 		return insn_idx - 1;  in get_prev_insn_idx()
4149 static bool calls_callback(struct bpf_verifier_env *env, int insn_idx);
4660 	int last_idx = env->insn_idx;  in __mark_chain_precision()
4969 					  struct bpf_func_state *state, int insn_idx, int off)  in check_fastcall_stack_contract()  argument
4975 	if (subprog->fastcall_stack_off <= off || aux[insn_idx].fastcall_pattern)  in check_fastcall_stack_contract()
4999 				       int insn_idx)  in check_stack_write_fixed_off()  argument
5003 	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];  in check_stack_write_fixed_off()
5034 			env->insn_aux_data[insn_idx].sanitize_stack_spill = true;  in check_stack_write_fixed_off()
5041 	check_fastcall_stack_contract(env, state, insn_idx, off);  in check_stack_write_fixed_off()
5065 			verbose_linfo(env, insn_idx, "; ");  in check_stack_write_fixed_off()
5144 				     int value_regno, int insn_idx)  in check_stack_write_var_off()  argument
5150 	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];  in check_stack_write_var_off()
5176 	check_fastcall_stack_contract(env, state, insn_idx, min_off);  in check_stack_write_var_off()
5200 				insn_idx, i);  in check_stack_write_var_off()
5236 					insn_idx, i);  in check_stack_write_var_off()
5315 	check_fastcall_stack_contract(env, state, env->insn_idx, off);  in check_stack_read_fixed_off()
5325 				verbose_linfo(env, env->insn_idx, "; ");  in check_stack_read_fixed_off()
5476 	check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off);  in check_stack_read_var_off()
5548 			     int value_regno, int insn_idx)  in check_stack_write()  argument
5557 						  value_regno, insn_idx);  in check_stack_write()
5564 						value_regno, insn_idx);  in check_stack_write()
5898 				 int value_regno, int insn_idx,  in check_map_kptr_access()  argument
5901 	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];  in check_map_kptr_access()
6111 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,  in check_ctx_access()  argument
6131 			env->insn_aux_data[insn_idx].ctx_field_size = info->ctx_field_size;  in check_ctx_access()
6155 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,  in check_sock_access()  argument
6189 		env->insn_aux_data[insn_idx].ctx_field_size =  in check_sock_access()
7410 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno,  in check_mem_access()  argument
7460 			err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field);  in check_mem_access()
7525 		err = check_ctx_access(env, insn_idx, off, size, t, &info);  in check_mem_access()
7527 			verbose_linfo(env, insn_idx, "; ");  in check_mem_access()
7573 						value_regno, insn_idx);  in check_mem_access()
7605 		err = check_sock_access(env, insn_idx, regno, off, size, t);  in check_mem_access()
7685 	err = check_mem_access(env, env->insn_idx, insn->src_reg, insn->off,  in check_load_mem()
7715 	err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off,  in check_store_reg()
7790 	err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off,  in check_atomic_rmw()
7793 		err = check_mem_access(env, env->insn_idx, insn->dst_reg,  in check_atomic_rmw()
7805 	err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off,  in check_atomic_rmw()
8114 			return check_mem_access(env, env->insn_idx, regno, offset, BPF_B,  in check_helper_mem_access()
8344 		err = acquire_lock_state(env, env->insn_idx, type, reg->id, ptr);  in process_spin_lock()
8514 static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx,  in process_dynptr_func()  argument
8560 			err = check_mem_access(env, insn_idx, regno,  in process_dynptr_func()
8566 		err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, clone_ref_obj_id);  in process_dynptr_func()
8634 static int process_iter_arg(struct bpf_verifier_env *env, int regno, int insn_idx,  in process_iter_arg()  argument
8669 			err = check_mem_access(env, insn_idx, regno,  in process_iter_arg()
8675 		err = mark_stack_slots_iter(env, meta, reg, insn_idx, meta->btf, btf_id, nr_slots);  in process_iter_arg()
8725 						  int insn_idx)  in find_prev_entry()  argument
8732 	head = explored_state(env, insn_idx);  in find_prev_entry()
8739 		if (st->insn_idx == insn_idx && st->branches && same_callsites(st, cur) &&  in find_prev_entry()
8884 static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx,  in process_iter_next_call()  argument
8906 		if (!cur_st->parent || cur_st->parent->insn_idx != insn_idx ||  in process_iter_next_call()
8915 		prev_st = find_prev_entry(env, cur_st->parent, insn_idx);  in process_iter_next_call()
8917 		queued_st = push_stack(env, insn_idx + 1, insn_idx, false);  in process_iter_next_call()
9517 			  int insn_idx)  in check_func_arg()  argument
9752 		err = process_dynptr_func(env, regno, insn_idx, arg_type, 0);  in check_func_arg()
10268 				   int insn_idx);
10272 			    struct bpf_func_state *callee, int insn_idx);
10448 			      int insn_idx, int subprog,  in push_callback_call()  argument
10483 					 insn_idx, subprog,  in push_callback_call()
10491 		err = set_callee_state_cb(env, caller, callee, insn_idx);  in push_callback_call()
10501 	callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false);  in push_callback_call()
10505 	err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb,  in push_callback_call()
10517 			   int *insn_idx)  in check_func_call()  argument
10523 	target_insn = *insn_idx + insn->imm + 1;  in check_func_call()
10577 	err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state);  in check_func_call()
10584 	*insn_idx = env->subprog_info[subprog].start - 1;  in check_func_call()
10625 			    struct bpf_func_state *callee, int insn_idx)  in set_callee_state()  argument
10640 				       int insn_idx)  in set_map_elem_callback_state()  argument
10642 	struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx];  in set_map_elem_callback_state()
10666 				   int insn_idx)  in set_loop_callback_state()  argument
10688 				    int insn_idx)  in set_timer_callback_state()  argument
10718 				       int insn_idx)  in set_find_vma_callback_state()  argument
10746 					   int insn_idx)  in set_user_ringbuf_callback_state()  argument
10769 					 int insn_idx)  in set_rbtree_add_callback_state()  argument
10832 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)  in prepare_func_exit()  argument
10874 				*insn_idx, callee->callsite);  in prepare_func_exit()
10888 		*insn_idx = callee->callsite;  in prepare_func_exit()
10890 		*insn_idx = callee->callsite + 1;  in prepare_func_exit()
10895 		verbose(env, "to caller at %d:\n", *insn_idx);  in prepare_func_exit()
10914 	prev_st = in_callback_fn ? find_prev_entry(env, state, *insn_idx) : NULL;  in prepare_func_exit()
10963 		int func_id, int insn_idx)  in record_func_map()  argument
10965 	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];  in record_func_map()
11009 		int func_id, int insn_idx)  in record_func_key()  argument
11011 	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];  in record_func_key()
11065 			state->refs[i].id, state->refs[i].insn_idx);  in check_reference_leak()
11165 	return &env->insn_aux_data[env->insn_idx];  in cur_aux()
11236 	int insn_idx = *insn_idx_p;  in check_helper_call()  local
11296 			env->insn_aux_data[insn_idx].storage_get_func_atomic = true;  in check_helper_call()
11307 			env->insn_aux_data[insn_idx].storage_get_func_atomic = true;  in check_helper_call()
11318 			env->insn_aux_data[insn_idx].storage_get_func_atomic = true;  in check_helper_call()
11324 		err = check_func_arg(env, i, &meta, fn, insn_idx);  in check_helper_call()
11329 	err = record_func_map(env, &meta, func_id, insn_idx);  in check_helper_call()
11333 	err = record_func_key(env, &meta, func_id, insn_idx);  in check_helper_call()
11341 		err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B,  in check_helper_call()
11412 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,  in check_helper_call()
11416 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,  in check_helper_call()
11420 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,  in check_helper_call()
11435 			err = push_callback_call(env, insn, insn_idx, meta.subprogno,  in check_helper_call()
11533 			env->insn_aux_data[insn_idx].call_with_percpu_alloc_ptr = true;  in check_helper_call()
11538 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,  in check_helper_call()
11707 		int id = acquire_reference(env, insn_idx);  in check_helper_call()
11771 			DEF_NOT_SUBREG : env->insn_idx + 1;  in __mark_btf_func_reg_size()
12424 		err = check_mem_access(env, env->insn_idx, regno, 0, BPF_DW, BPF_WRITE, -1, false, false);  in process_irq_flag()
12428 		err = mark_stack_slot_irq_flag(env, meta, reg, env->insn_idx, kfunc_class);  in process_irq_flag()
12876 			    int insn_idx)  in check_kfunc_args()  argument
13132 			ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type, clone_ref_obj_id);  in check_kfunc_args()
13157 			ret = process_iter_arg(env, regno, insn_idx, meta);  in check_kfunc_args()
13435 	int err, insn_idx = *insn_idx_p;  in check_kfunc_call()  local
13450 	insn_aux = &env->insn_aux_data[insn_idx];  in check_kfunc_call()
13460 		branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false);  in check_kfunc_call()
13493 	err = check_kfunc_args(env, &meta, insn_idx);  in check_kfunc_call()
13498 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,  in check_kfunc_call()
13513 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,  in check_kfunc_call()
13877 			int id = acquire_reference(env, insn_idx);  in check_kfunc_call()
13915 		err = process_iter_next_call(env, insn_idx, &meta);  in check_kfunc_call()
14151 	ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1,  in sanitize_ptr_alu()
14152 					env->insn_idx);  in sanitize_ptr_alu()
14168 		env->insn_aux_data[env->insn_idx].seen = env->pass_cnt;  in sanitize_mark_insn_seen()
15405 						dst_reg->subreg_def = env->insn_idx + 1;  in check_alu_op()
15459 						dst_reg->subreg_def = env->insn_idx + 1;  in check_alu_op()
15470 						dst_reg->subreg_def = env->insn_idx + 1;  in check_alu_op()
16371 			     struct bpf_insn *insn, int *insn_idx)  in check_cond_jmp_op()  argument
16392 		int idx = *insn_idx;  in check_cond_jmp_op()
16410 		*insn_idx += insn->off;  in check_cond_jmp_op()
16470 		    !sanitize_speculative_path(env, insn, *insn_idx + 1,  in check_cond_jmp_op()
16471 					       *insn_idx))  in check_cond_jmp_op()
16475 		*insn_idx += insn->off;  in check_cond_jmp_op()
16484 					       *insn_idx + insn->off + 1,  in check_cond_jmp_op()
16485 					       *insn_idx))  in check_cond_jmp_op()
16507 	other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx,  in check_cond_jmp_op()
16666 					     env->insn_idx + insn->imm + 1);  in check_ld_imm()
16799 	regs[BPF_REG_0].subreg_def = env->insn_idx + 1;  in check_ld_abs()
17080 static bool is_prune_point(struct bpf_verifier_env *env, int insn_idx)  in is_prune_point()  argument
17082 	return env->insn_aux_data[insn_idx].prune_point;  in is_prune_point()
17090 static bool is_force_checkpoint(struct bpf_verifier_env *env, int insn_idx)  in is_force_checkpoint()  argument
17092 	return env->insn_aux_data[insn_idx].force_checkpoint;  in is_force_checkpoint()
17100 static bool calls_callback(struct bpf_verifier_env *env, int insn_idx)  in calls_callback()  argument
17102 	return env->insn_aux_data[insn_idx].calls_callback;  in calls_callback()
17333 					   int insn_idx, s16 lowest_off)  in mark_fastcall_pattern_for_call()  argument
17336 	struct bpf_insn *call = &env->prog->insnsi[insn_idx];  in mark_fastcall_pattern_for_call()
17366 		if (insn_idx - i < 0 || insn_idx + i >= env->prog->len)  in mark_fastcall_pattern_for_call()
17368 		stx = &insns[insn_idx - i];  in mark_fastcall_pattern_for_call()
17369 		ldx = &insns[insn_idx + i];  in mark_fastcall_pattern_for_call()
17389 		env->insn_aux_data[insn_idx - i].fastcall_pattern = 1;  in mark_fastcall_pattern_for_call()
17390 		env->insn_aux_data[insn_idx + i].fastcall_pattern = 1;  in mark_fastcall_pattern_for_call()
17412 		env->insn_aux_data[insn_idx].fastcall_spills_num = i - 1;  in mark_fastcall_pattern_for_call()
18285 		if (sl->state.insn_idx != insn ||  in clean_live_states()
18679 			      struct bpf_func_state *cur, u32 insn_idx, enum exact_level exact)  in func_states_equal()  argument
18681 	u16 live_regs = env->insn_aux_data[insn_idx].live_regs_before;  in func_states_equal()
18710 	u32 insn_idx;  in states_equal()  local
18734 		insn_idx = i == old->curframe  in states_equal()
18735 			   ? env->insn_idx  in states_equal()
18739 		if (!func_states_equal(env, old->frame[i], cur->frame[i], insn_idx, exact))  in states_equal()
18902 static bool is_iter_next_insn(struct bpf_verifier_env *env, int insn_idx)  in is_iter_next_insn()  argument
18904 	return env->insn_aux_data[insn_idx].is_iter_next;  in is_iter_next_insn()
18989 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)  in is_state_visited()  argument
18998 	force_new_state = env->test_state_freq || is_force_checkpoint(env, insn_idx) ||  in is_state_visited()
19015 	clean_live_states(env, insn_idx, cur);  in is_state_visited()
19017 	head = explored_state(env, insn_idx);  in is_state_visited()
19021 		if (sl->state.insn_idx != insn_idx)  in is_state_visited()
19078 			if (is_iter_next_insn(env, insn_idx)) {  in is_state_visited()
19102 			if (is_may_goto_insn_at(env, insn_idx)) {  in is_state_visited()
19109 			if (calls_callback(env, insn_idx)) {  in is_state_visited()
19120 				verbose_linfo(env, insn_idx, "; ");  in is_state_visited()
19121 				verbose(env, "infinite loop detected at insn %d\n", insn_idx);  in is_state_visited()
19198 			if (is_jmp_point(env, env->insn_idx))  in is_state_visited()
19222 		n = is_force_checkpoint(env, insn_idx) && sl->state.branches > 0 ? 64 : 3;  in is_state_visited()
19275 	new->insn_idx = insn_idx;  in is_state_visited()
19277 		  "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx);  in is_state_visited()
19280 	cur->first_insn_idx = insn_idx;  in is_state_visited()
19357 	enum bpf_reg_type *prev_type = &env->insn_aux_data[env->insn_idx].ptr_type;  in save_aux_ptr_type()
19412 		if (env->insn_idx >= insn_cnt) {  in do_check()
19414 				env->insn_idx, insn_cnt);  in do_check()
19418 		insn = &insns[env->insn_idx];  in do_check()
19430 		if (is_prune_point(env, env->insn_idx)) {  in do_check()
19431 			err = is_state_visited(env, env->insn_idx);  in do_check()
19439 							env->prev_insn_idx, env->insn_idx,  in do_check()
19443 						verbose(env, "%d: safe\n", env->insn_idx);  in do_check()
19449 		if (is_jmp_point(env, env->insn_idx)) {  in do_check()
19463 				env->prev_insn_idx, env->insn_idx,  in do_check()
19474 			verbose_linfo(env, env->insn_idx, "; ");  in do_check()
19476 			verbose(env, "%d: ", env->insn_idx);  in do_check()
19483 			err = bpf_prog_offload_verify_insn(env, env->insn_idx,  in do_check()
19491 		prev_insn_idx = env->insn_idx;  in do_check()
19513 				env->insn_idx++;  in do_check()
19541 			err = check_mem_access(env, env->insn_idx, insn->dst_reg,  in do_check()
19576 					err = check_func_call(env, insn, &env->insn_idx);  in do_check()
19578 					err = check_kfunc_call(env, insn, &env->insn_idx);  in do_check()
19584 					err = check_helper_call(env, insn, &env->insn_idx);  in do_check()
19601 					env->insn_idx += insn->off + 1;  in do_check()
19603 					env->insn_idx += insn->imm + 1;  in do_check()
19642 					err = prepare_func_exit(env, &env->insn_idx);  in do_check()
19656 						&env->insn_idx, pop_log);  in do_check()
19670 				err = check_cond_jmp_op(env, insn, &env->insn_idx);  in do_check()
19687 				env->insn_idx++;  in do_check()
19698 		env->insn_idx++;  in do_check()
20309 		if (desc->insn_idx <= off)  in adjust_poke_descs()
20311 		desc->insn_idx += len - 1;  in adjust_poke_descs()
21115 			if (poke->insn_idx < subprog_end &&  in jit_subprogs()
21116 			    poke->insn_idx >= subprog_start)  in jit_subprogs()
21399 			    struct bpf_insn *insn_buf, int insn_idx, int *cnt)  in fixup_kfunc_call()  argument
21427 		struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;  in fixup_kfunc_call()
21429 		u64 obj_new_size = env->insn_aux_data[insn_idx].obj_new_size;  in fixup_kfunc_call()
21433 				insn_idx);  in fixup_kfunc_call()
21445 		struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;  in fixup_kfunc_call()
21450 				insn_idx);  in fixup_kfunc_call()
21457 				insn_idx);  in fixup_kfunc_call()
21468 		struct btf_struct_meta *kptr_struct_meta = env->insn_aux_data[insn_idx].kptr_struct_meta;  in fixup_kfunc_call()
21480 				insn_idx);  in fixup_kfunc_call()
21484 		__fixup_collection_insert_kfunc(&env->insn_aux_data[insn_idx], struct_meta_reg,  in fixup_kfunc_call()
21939 					.insn_idx = i + delta,  in do_misc_fixups()
22835 		env->insn_idx = env->subprog_info[i].start;  in do_check_subprogs()
22836 		WARN_ON_ONCE(env->insn_idx == 0);  in do_check_subprogs()
22866 	env->insn_idx = 0;  in do_check_main()
23858 			int insn_idx = env->cfg.insn_postorder[i];  in compute_live_registers()  local
23859 			struct insn_live_regs *live = &state[insn_idx];  in compute_live_registers()
23865 			succ_num = insn_successors(env->prog, insn_idx, succ);  in compute_live_registers()