Lines Matching refs:dst_reg
3663 return insn->dst_reg; in insn_def_regno()
3670 int dst_reg = insn_def_regno(insn); in insn_has_def32() local
3672 if (dst_reg == -1) in insn_has_def32()
3675 return !is_reg64(env, insn, dst_reg, NULL, DST_OP); in insn_has_def32()
4167 u32 dreg = insn->dst_reg; in backtrack_insn()
7676 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_load_mem()
7686 BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, in check_load_mem()
7690 err = err ?: reg_bounds_sanity_check(env, ®s[insn->dst_reg], ctx); in check_load_mem()
7708 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_store_reg()
7712 dst_reg_type = regs[insn->dst_reg].type; in check_store_reg()
7715 err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off, in check_store_reg()
7740 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_atomic_rmw()
7763 if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) { in check_atomic_rmw()
7765 insn->dst_reg, in check_atomic_rmw()
7766 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); in check_atomic_rmw()
7790 err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off, in check_atomic_rmw()
7793 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in check_atomic_rmw()
7799 if (is_arena_reg(env, insn->dst_reg)) { in check_atomic_rmw()
7805 err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off, in check_atomic_rmw()
7840 if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) { in check_atomic_store()
7842 insn->dst_reg, in check_atomic_store()
7843 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); in check_atomic_store()
14053 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
14055 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
14066 struct bpf_reg_state *dst_reg, in sanitize_ptr_alu() argument
14074 bool ptr_is_dst_reg = ptr_reg == dst_reg; in sanitize_ptr_alu()
14148 tmp = *dst_reg; in sanitize_ptr_alu()
14149 copy_register_state(dst_reg, ptr_reg); in sanitize_ptr_alu()
14154 *dst_reg = tmp; in sanitize_ptr_alu()
14174 const struct bpf_reg_state *dst_reg) in sanitize_err() argument
14178 u32 dst = insn->dst_reg, src = insn->src_reg; in sanitize_err()
14183 off_reg == dst_reg ? dst : src, err); in sanitize_err()
14187 off_reg == dst_reg ? src : dst, err); in sanitize_err()
14246 const struct bpf_reg_state *dst_reg) in sanitize_check_bounds() argument
14248 u32 dst = insn->dst_reg; in sanitize_check_bounds()
14256 switch (dst_reg->type) { in sanitize_check_bounds()
14258 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, in sanitize_check_bounds()
14259 dst_reg->off + dst_reg->var_off.value)) in sanitize_check_bounds()
14263 if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) { in sanitize_check_bounds()
14288 struct bpf_reg_state *regs = state->regs, *dst_reg; in adjust_ptr_min_max_vals() local
14296 u32 dst = insn->dst_reg; in adjust_ptr_min_max_vals()
14299 dst_reg = ®s[dst]; in adjust_ptr_min_max_vals()
14306 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
14313 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
14361 dst_reg->type = ptr_reg->type; in adjust_ptr_min_max_vals()
14362 dst_reg->id = ptr_reg->id; in adjust_ptr_min_max_vals()
14369 __mark_reg32_unbounded(dst_reg); in adjust_ptr_min_max_vals()
14372 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
14375 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
14386 dst_reg->smin_value = smin_ptr; in adjust_ptr_min_max_vals()
14387 dst_reg->smax_value = smax_ptr; in adjust_ptr_min_max_vals()
14388 dst_reg->umin_value = umin_ptr; in adjust_ptr_min_max_vals()
14389 dst_reg->umax_value = umax_ptr; in adjust_ptr_min_max_vals()
14390 dst_reg->var_off = ptr_reg->var_off; in adjust_ptr_min_max_vals()
14391 dst_reg->off = ptr_reg->off + smin_val; in adjust_ptr_min_max_vals()
14392 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
14404 if (check_add_overflow(smin_ptr, smin_val, &dst_reg->smin_value) || in adjust_ptr_min_max_vals()
14405 check_add_overflow(smax_ptr, smax_val, &dst_reg->smax_value)) { in adjust_ptr_min_max_vals()
14406 dst_reg->smin_value = S64_MIN; in adjust_ptr_min_max_vals()
14407 dst_reg->smax_value = S64_MAX; in adjust_ptr_min_max_vals()
14409 if (check_add_overflow(umin_ptr, umin_val, &dst_reg->umin_value) || in adjust_ptr_min_max_vals()
14410 check_add_overflow(umax_ptr, umax_val, &dst_reg->umax_value)) { in adjust_ptr_min_max_vals()
14411 dst_reg->umin_value = 0; in adjust_ptr_min_max_vals()
14412 dst_reg->umax_value = U64_MAX; in adjust_ptr_min_max_vals()
14414 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); in adjust_ptr_min_max_vals()
14415 dst_reg->off = ptr_reg->off; in adjust_ptr_min_max_vals()
14416 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
14418 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
14420 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); in adjust_ptr_min_max_vals()
14424 if (dst_reg == off_reg) { in adjust_ptr_min_max_vals()
14442 dst_reg->smin_value = smin_ptr; in adjust_ptr_min_max_vals()
14443 dst_reg->smax_value = smax_ptr; in adjust_ptr_min_max_vals()
14444 dst_reg->umin_value = umin_ptr; in adjust_ptr_min_max_vals()
14445 dst_reg->umax_value = umax_ptr; in adjust_ptr_min_max_vals()
14446 dst_reg->var_off = ptr_reg->var_off; in adjust_ptr_min_max_vals()
14447 dst_reg->id = ptr_reg->id; in adjust_ptr_min_max_vals()
14448 dst_reg->off = ptr_reg->off - smin_val; in adjust_ptr_min_max_vals()
14449 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
14455 if (check_sub_overflow(smin_ptr, smax_val, &dst_reg->smin_value) || in adjust_ptr_min_max_vals()
14456 check_sub_overflow(smax_ptr, smin_val, &dst_reg->smax_value)) { in adjust_ptr_min_max_vals()
14458 dst_reg->smin_value = S64_MIN; in adjust_ptr_min_max_vals()
14459 dst_reg->smax_value = S64_MAX; in adjust_ptr_min_max_vals()
14463 dst_reg->umin_value = 0; in adjust_ptr_min_max_vals()
14464 dst_reg->umax_value = U64_MAX; in adjust_ptr_min_max_vals()
14467 dst_reg->umin_value = umin_ptr - umax_val; in adjust_ptr_min_max_vals()
14468 dst_reg->umax_value = umax_ptr - umin_val; in adjust_ptr_min_max_vals()
14470 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); in adjust_ptr_min_max_vals()
14471 dst_reg->off = ptr_reg->off; in adjust_ptr_min_max_vals()
14472 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
14474 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
14477 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); in adjust_ptr_min_max_vals()
14494 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) in adjust_ptr_min_max_vals()
14496 reg_bounds_sync(dst_reg); in adjust_ptr_min_max_vals()
14497 if (sanitize_check_bounds(env, insn, dst_reg) < 0) in adjust_ptr_min_max_vals()
14500 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
14503 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
14509 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, in scalar32_min_max_add() argument
14512 s32 *dst_smin = &dst_reg->s32_min_value; in scalar32_min_max_add()
14513 s32 *dst_smax = &dst_reg->s32_max_value; in scalar32_min_max_add()
14514 u32 *dst_umin = &dst_reg->u32_min_value; in scalar32_min_max_add()
14515 u32 *dst_umax = &dst_reg->u32_max_value; in scalar32_min_max_add()
14529 static void scalar_min_max_add(struct bpf_reg_state *dst_reg, in scalar_min_max_add() argument
14532 s64 *dst_smin = &dst_reg->smin_value; in scalar_min_max_add()
14533 s64 *dst_smax = &dst_reg->smax_value; in scalar_min_max_add()
14534 u64 *dst_umin = &dst_reg->umin_value; in scalar_min_max_add()
14535 u64 *dst_umax = &dst_reg->umax_value; in scalar_min_max_add()
14549 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, in scalar32_min_max_sub() argument
14552 s32 *dst_smin = &dst_reg->s32_min_value; in scalar32_min_max_sub()
14553 s32 *dst_smax = &dst_reg->s32_max_value; in scalar32_min_max_sub()
14563 if (dst_reg->u32_min_value < umax_val) { in scalar32_min_max_sub()
14565 dst_reg->u32_min_value = 0; in scalar32_min_max_sub()
14566 dst_reg->u32_max_value = U32_MAX; in scalar32_min_max_sub()
14569 dst_reg->u32_min_value -= umax_val; in scalar32_min_max_sub()
14570 dst_reg->u32_max_value -= umin_val; in scalar32_min_max_sub()
14574 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, in scalar_min_max_sub() argument
14577 s64 *dst_smin = &dst_reg->smin_value; in scalar_min_max_sub()
14578 s64 *dst_smax = &dst_reg->smax_value; in scalar_min_max_sub()
14588 if (dst_reg->umin_value < umax_val) { in scalar_min_max_sub()
14590 dst_reg->umin_value = 0; in scalar_min_max_sub()
14591 dst_reg->umax_value = U64_MAX; in scalar_min_max_sub()
14594 dst_reg->umin_value -= umax_val; in scalar_min_max_sub()
14595 dst_reg->umax_value -= umin_val; in scalar_min_max_sub()
14599 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, in scalar32_min_max_mul() argument
14602 s32 *dst_smin = &dst_reg->s32_min_value; in scalar32_min_max_mul()
14603 s32 *dst_smax = &dst_reg->s32_max_value; in scalar32_min_max_mul()
14604 u32 *dst_umin = &dst_reg->u32_min_value; in scalar32_min_max_mul()
14605 u32 *dst_umax = &dst_reg->u32_max_value; in scalar32_min_max_mul()
14627 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, in scalar_min_max_mul() argument
14630 s64 *dst_smin = &dst_reg->smin_value; in scalar_min_max_mul()
14631 s64 *dst_smax = &dst_reg->smax_value; in scalar_min_max_mul()
14632 u64 *dst_umin = &dst_reg->umin_value; in scalar_min_max_mul()
14633 u64 *dst_umax = &dst_reg->umax_value; in scalar_min_max_mul()
14655 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, in scalar32_min_max_and() argument
14659 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); in scalar32_min_max_and()
14660 struct tnum var32_off = tnum_subreg(dst_reg->var_off); in scalar32_min_max_and()
14664 __mark_reg32_known(dst_reg, var32_off.value); in scalar32_min_max_and()
14671 dst_reg->u32_min_value = var32_off.value; in scalar32_min_max_and()
14672 dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val); in scalar32_min_max_and()
14677 if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) { in scalar32_min_max_and()
14678 dst_reg->s32_min_value = dst_reg->u32_min_value; in scalar32_min_max_and()
14679 dst_reg->s32_max_value = dst_reg->u32_max_value; in scalar32_min_max_and()
14681 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_and()
14682 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_and()
14686 static void scalar_min_max_and(struct bpf_reg_state *dst_reg, in scalar_min_max_and() argument
14690 bool dst_known = tnum_is_const(dst_reg->var_off); in scalar_min_max_and()
14694 __mark_reg_known(dst_reg, dst_reg->var_off.value); in scalar_min_max_and()
14701 dst_reg->umin_value = dst_reg->var_off.value; in scalar_min_max_and()
14702 dst_reg->umax_value = min(dst_reg->umax_value, umax_val); in scalar_min_max_and()
14707 if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) { in scalar_min_max_and()
14708 dst_reg->smin_value = dst_reg->umin_value; in scalar_min_max_and()
14709 dst_reg->smax_value = dst_reg->umax_value; in scalar_min_max_and()
14711 dst_reg->smin_value = S64_MIN; in scalar_min_max_and()
14712 dst_reg->smax_value = S64_MAX; in scalar_min_max_and()
14715 __update_reg_bounds(dst_reg); in scalar_min_max_and()
14718 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, in scalar32_min_max_or() argument
14722 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); in scalar32_min_max_or()
14723 struct tnum var32_off = tnum_subreg(dst_reg->var_off); in scalar32_min_max_or()
14727 __mark_reg32_known(dst_reg, var32_off.value); in scalar32_min_max_or()
14734 dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val); in scalar32_min_max_or()
14735 dst_reg->u32_max_value = var32_off.value | var32_off.mask; in scalar32_min_max_or()
14740 if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) { in scalar32_min_max_or()
14741 dst_reg->s32_min_value = dst_reg->u32_min_value; in scalar32_min_max_or()
14742 dst_reg->s32_max_value = dst_reg->u32_max_value; in scalar32_min_max_or()
14744 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_or()
14745 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_or()
14749 static void scalar_min_max_or(struct bpf_reg_state *dst_reg, in scalar_min_max_or() argument
14753 bool dst_known = tnum_is_const(dst_reg->var_off); in scalar_min_max_or()
14757 __mark_reg_known(dst_reg, dst_reg->var_off.value); in scalar_min_max_or()
14764 dst_reg->umin_value = max(dst_reg->umin_value, umin_val); in scalar_min_max_or()
14765 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; in scalar_min_max_or()
14770 if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) { in scalar_min_max_or()
14771 dst_reg->smin_value = dst_reg->umin_value; in scalar_min_max_or()
14772 dst_reg->smax_value = dst_reg->umax_value; in scalar_min_max_or()
14774 dst_reg->smin_value = S64_MIN; in scalar_min_max_or()
14775 dst_reg->smax_value = S64_MAX; in scalar_min_max_or()
14778 __update_reg_bounds(dst_reg); in scalar_min_max_or()
14781 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, in scalar32_min_max_xor() argument
14785 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); in scalar32_min_max_xor()
14786 struct tnum var32_off = tnum_subreg(dst_reg->var_off); in scalar32_min_max_xor()
14789 __mark_reg32_known(dst_reg, var32_off.value); in scalar32_min_max_xor()
14794 dst_reg->u32_min_value = var32_off.value; in scalar32_min_max_xor()
14795 dst_reg->u32_max_value = var32_off.value | var32_off.mask; in scalar32_min_max_xor()
14800 if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) { in scalar32_min_max_xor()
14801 dst_reg->s32_min_value = dst_reg->u32_min_value; in scalar32_min_max_xor()
14802 dst_reg->s32_max_value = dst_reg->u32_max_value; in scalar32_min_max_xor()
14804 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_xor()
14805 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_xor()
14809 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, in scalar_min_max_xor() argument
14813 bool dst_known = tnum_is_const(dst_reg->var_off); in scalar_min_max_xor()
14817 __mark_reg_known(dst_reg, dst_reg->var_off.value); in scalar_min_max_xor()
14822 dst_reg->umin_value = dst_reg->var_off.value; in scalar_min_max_xor()
14823 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; in scalar_min_max_xor()
14828 if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) { in scalar_min_max_xor()
14829 dst_reg->smin_value = dst_reg->umin_value; in scalar_min_max_xor()
14830 dst_reg->smax_value = dst_reg->umax_value; in scalar_min_max_xor()
14832 dst_reg->smin_value = S64_MIN; in scalar_min_max_xor()
14833 dst_reg->smax_value = S64_MAX; in scalar_min_max_xor()
14836 __update_reg_bounds(dst_reg); in scalar_min_max_xor()
14839 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, in __scalar32_min_max_lsh() argument
14845 dst_reg->s32_min_value = S32_MIN; in __scalar32_min_max_lsh()
14846 dst_reg->s32_max_value = S32_MAX; in __scalar32_min_max_lsh()
14848 if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) { in __scalar32_min_max_lsh()
14849 dst_reg->u32_min_value = 0; in __scalar32_min_max_lsh()
14850 dst_reg->u32_max_value = U32_MAX; in __scalar32_min_max_lsh()
14852 dst_reg->u32_min_value <<= umin_val; in __scalar32_min_max_lsh()
14853 dst_reg->u32_max_value <<= umax_val; in __scalar32_min_max_lsh()
14857 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, in scalar32_min_max_lsh() argument
14863 struct tnum subreg = tnum_subreg(dst_reg->var_off); in scalar32_min_max_lsh()
14865 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); in scalar32_min_max_lsh()
14866 dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val)); in scalar32_min_max_lsh()
14871 __mark_reg64_unbounded(dst_reg); in scalar32_min_max_lsh()
14872 __update_reg32_bounds(dst_reg); in scalar32_min_max_lsh()
14875 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, in __scalar64_min_max_lsh() argument
14885 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0) in __scalar64_min_max_lsh()
14886 dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; in __scalar64_min_max_lsh()
14888 dst_reg->smax_value = S64_MAX; in __scalar64_min_max_lsh()
14890 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0) in __scalar64_min_max_lsh()
14891 dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; in __scalar64_min_max_lsh()
14893 dst_reg->smin_value = S64_MIN; in __scalar64_min_max_lsh()
14896 if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { in __scalar64_min_max_lsh()
14897 dst_reg->umin_value = 0; in __scalar64_min_max_lsh()
14898 dst_reg->umax_value = U64_MAX; in __scalar64_min_max_lsh()
14900 dst_reg->umin_value <<= umin_val; in __scalar64_min_max_lsh()
14901 dst_reg->umax_value <<= umax_val; in __scalar64_min_max_lsh()
14905 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, in scalar_min_max_lsh() argument
14912 __scalar64_min_max_lsh(dst_reg, umin_val, umax_val); in scalar_min_max_lsh()
14913 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); in scalar_min_max_lsh()
14915 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); in scalar_min_max_lsh()
14917 __update_reg_bounds(dst_reg); in scalar_min_max_lsh()
14920 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, in scalar32_min_max_rsh() argument
14923 struct tnum subreg = tnum_subreg(dst_reg->var_off); in scalar32_min_max_rsh()
14941 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_rsh()
14942 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_rsh()
14944 dst_reg->var_off = tnum_rshift(subreg, umin_val); in scalar32_min_max_rsh()
14945 dst_reg->u32_min_value >>= umax_val; in scalar32_min_max_rsh()
14946 dst_reg->u32_max_value >>= umin_val; in scalar32_min_max_rsh()
14948 __mark_reg64_unbounded(dst_reg); in scalar32_min_max_rsh()
14949 __update_reg32_bounds(dst_reg); in scalar32_min_max_rsh()
14952 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, in scalar_min_max_rsh() argument
14972 dst_reg->smin_value = S64_MIN; in scalar_min_max_rsh()
14973 dst_reg->smax_value = S64_MAX; in scalar_min_max_rsh()
14974 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); in scalar_min_max_rsh()
14975 dst_reg->umin_value >>= umax_val; in scalar_min_max_rsh()
14976 dst_reg->umax_value >>= umin_val; in scalar_min_max_rsh()
14982 __mark_reg32_unbounded(dst_reg); in scalar_min_max_rsh()
14983 __update_reg_bounds(dst_reg); in scalar_min_max_rsh()
14986 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, in scalar32_min_max_arsh() argument
14994 dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val); in scalar32_min_max_arsh()
14995 dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val); in scalar32_min_max_arsh()
14997 dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32); in scalar32_min_max_arsh()
15002 dst_reg->u32_min_value = 0; in scalar32_min_max_arsh()
15003 dst_reg->u32_max_value = U32_MAX; in scalar32_min_max_arsh()
15005 __mark_reg64_unbounded(dst_reg); in scalar32_min_max_arsh()
15006 __update_reg32_bounds(dst_reg); in scalar32_min_max_arsh()
15009 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, in scalar_min_max_arsh() argument
15017 dst_reg->smin_value >>= umin_val; in scalar_min_max_arsh()
15018 dst_reg->smax_value >>= umin_val; in scalar_min_max_arsh()
15020 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64); in scalar_min_max_arsh()
15025 dst_reg->umin_value = 0; in scalar_min_max_arsh()
15026 dst_reg->umax_value = U64_MAX; in scalar_min_max_arsh()
15032 __mark_reg32_unbounded(dst_reg); in scalar_min_max_arsh()
15033 __update_reg_bounds(dst_reg); in scalar_min_max_arsh()
15082 struct bpf_reg_state *dst_reg, in adjust_scalar_min_max_vals() argument
15090 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
15116 scalar32_min_max_add(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
15117 scalar_min_max_add(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
15118 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
15121 scalar32_min_max_sub(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
15122 scalar_min_max_sub(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
15123 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
15126 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
15127 scalar32_min_max_mul(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
15128 scalar_min_max_mul(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
15131 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
15132 scalar32_min_max_and(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
15133 scalar_min_max_and(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
15136 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
15137 scalar32_min_max_or(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
15138 scalar_min_max_or(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
15141 dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
15142 scalar32_min_max_xor(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
15143 scalar_min_max_xor(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
15147 scalar32_min_max_lsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
15149 scalar_min_max_lsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
15153 scalar32_min_max_rsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
15155 scalar_min_max_rsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
15159 scalar32_min_max_arsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
15161 scalar_min_max_arsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
15169 zext_32_to_64(dst_reg); in adjust_scalar_min_max_vals()
15170 reg_bounds_sync(dst_reg); in adjust_scalar_min_max_vals()
15182 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; in adjust_reg_min_max_vals() local
15188 dst_reg = ®s[insn->dst_reg]; in adjust_reg_min_max_vals()
15191 if (dst_reg->type == PTR_TO_ARENA) { in adjust_reg_min_max_vals()
15205 if (dst_reg->type != SCALAR_VALUE) in adjust_reg_min_max_vals()
15206 ptr_reg = dst_reg; in adjust_reg_min_max_vals()
15211 if (dst_reg->type != SCALAR_VALUE) { in adjust_reg_min_max_vals()
15217 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_reg_min_max_vals()
15221 insn->dst_reg, in adjust_reg_min_max_vals()
15229 err = mark_chain_precision(env, insn->dst_reg); in adjust_reg_min_max_vals()
15233 src_reg, dst_reg); in adjust_reg_min_max_vals()
15241 dst_reg, src_reg); in adjust_reg_min_max_vals()
15242 } else if (dst_reg->precise) { in adjust_reg_min_max_vals()
15271 err = adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); in adjust_reg_min_max_vals()
15285 dst_reg->id && is_reg_const(src_reg, false)) { in adjust_reg_min_max_vals()
15288 if ((dst_reg->id & BPF_ADD_CONST) || in adjust_reg_min_max_vals()
15295 dst_reg->off = 0; in adjust_reg_min_max_vals()
15296 dst_reg->id = 0; in adjust_reg_min_max_vals()
15298 dst_reg->id |= BPF_ADD_CONST; in adjust_reg_min_max_vals()
15299 dst_reg->off = val; in adjust_reg_min_max_vals()
15306 dst_reg->id = 0; in adjust_reg_min_max_vals()
15337 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
15341 if (is_pointer_value(env, insn->dst_reg)) { in check_alu_op()
15343 insn->dst_reg); in check_alu_op()
15348 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_alu_op()
15390 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
15396 struct bpf_reg_state *dst_reg = regs + insn->dst_reg; in check_alu_op() local
15401 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
15403 dst_reg->type = PTR_TO_ARENA; in check_alu_op()
15405 dst_reg->subreg_def = env->insn_idx + 1; in check_alu_op()
15412 copy_register_state(dst_reg, src_reg); in check_alu_op()
15413 dst_reg->live |= REG_LIVE_WRITTEN; in check_alu_op()
15414 dst_reg->subreg_def = DEF_NOT_SUBREG; in check_alu_op()
15428 copy_register_state(dst_reg, src_reg); in check_alu_op()
15430 dst_reg->id = 0; in check_alu_op()
15431 coerce_reg_to_size_sx(dst_reg, insn->off >> 3); in check_alu_op()
15432 dst_reg->live |= REG_LIVE_WRITTEN; in check_alu_op()
15433 dst_reg->subreg_def = DEF_NOT_SUBREG; in check_alu_op()
15435 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
15451 copy_register_state(dst_reg, src_reg); in check_alu_op()
15457 dst_reg->id = 0; in check_alu_op()
15458 dst_reg->live |= REG_LIVE_WRITTEN; in check_alu_op()
15459 dst_reg->subreg_def = env->insn_idx + 1; in check_alu_op()
15466 copy_register_state(dst_reg, src_reg); in check_alu_op()
15468 dst_reg->id = 0; in check_alu_op()
15469 dst_reg->live |= REG_LIVE_WRITTEN; in check_alu_op()
15470 dst_reg->subreg_def = env->insn_idx + 1; in check_alu_op()
15471 coerce_subreg_to_size_sx(dst_reg, insn->off >> 3); in check_alu_op()
15475 insn->dst_reg); in check_alu_op()
15477 zext_32_to_64(dst_reg); in check_alu_op()
15478 reg_bounds_sync(dst_reg); in check_alu_op()
15485 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
15486 regs[insn->dst_reg].type = SCALAR_VALUE; in check_alu_op()
15488 __mark_reg_known(regs + insn->dst_reg, in check_alu_op()
15491 __mark_reg_known(regs + insn->dst_reg, in check_alu_op()
15521 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
15542 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
15548 return reg_bounds_sanity_check(env, ®s[insn->dst_reg], "alu"); in check_alu_op()
15552 struct bpf_reg_state *dst_reg, in find_good_pkt_pointers() argument
15560 if (dst_reg->off < 0 || in find_good_pkt_pointers()
15561 (dst_reg->off == 0 && range_right_open)) in find_good_pkt_pointers()
15565 if (dst_reg->umax_value > MAX_PACKET_OFF || in find_good_pkt_pointers()
15566 dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) in find_good_pkt_pointers()
15572 new_range = dst_reg->off; in find_good_pkt_pointers()
15624 if (reg->type == type && reg->id == dst_reg->id) in find_good_pkt_pointers()
15782 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, in is_pkt_ptr_branch_taken() argument
15789 pkt = dst_reg; in is_pkt_ptr_branch_taken()
15790 } else if (dst_reg->type == PTR_TO_PACKET_END) { in is_pkt_ptr_branch_taken()
16173 struct bpf_reg_state *dst_reg, in try_match_pkt_pointers() argument
16187 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
16189 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
16192 find_good_pkt_pointers(this_branch, dst_reg, in try_match_pkt_pointers()
16193 dst_reg->type, false); in try_match_pkt_pointers()
16194 mark_pkt_end(other_branch, insn->dst_reg, true); in try_match_pkt_pointers()
16195 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
16197 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
16208 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
16210 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
16213 find_good_pkt_pointers(other_branch, dst_reg, in try_match_pkt_pointers()
16214 dst_reg->type, true); in try_match_pkt_pointers()
16215 mark_pkt_end(this_branch, insn->dst_reg, false); in try_match_pkt_pointers()
16216 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
16218 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
16229 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
16231 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
16234 find_good_pkt_pointers(this_branch, dst_reg, in try_match_pkt_pointers()
16235 dst_reg->type, true); in try_match_pkt_pointers()
16236 mark_pkt_end(other_branch, insn->dst_reg, false); in try_match_pkt_pointers()
16237 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
16239 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
16250 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
16252 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
16255 find_good_pkt_pointers(other_branch, dst_reg, in try_match_pkt_pointers()
16256 dst_reg->type, false); in try_match_pkt_pointers()
16257 mark_pkt_end(this_branch, insn->dst_reg, true); in try_match_pkt_pointers()
16258 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
16260 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
16376 struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; in check_cond_jmp_op() local
16396 insn->dst_reg || insn->imm) { in check_cond_jmp_op()
16415 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_cond_jmp_op()
16419 dst_reg = ®s[insn->dst_reg]; in check_cond_jmp_op()
16432 if (!(reg_is_pkt_pointer_any(dst_reg) && reg_is_pkt_pointer_any(src_reg)) && in check_cond_jmp_op()
16450 pred = is_branch_taken(dst_reg, src_reg, opcode, is_jmp32); in check_cond_jmp_op()
16455 if (!__is_pointer_value(false, dst_reg)) in check_cond_jmp_op()
16456 err = mark_chain_precision(env, insn->dst_reg); in check_cond_jmp_op()
16499 if (dst_reg->type == SCALAR_VALUE && dst_reg->id) in check_cond_jmp_op()
16500 collect_linked_regs(this_branch, dst_reg->id, &linked_regs); in check_cond_jmp_op()
16515 &other_branch_regs[insn->dst_reg], in check_cond_jmp_op()
16517 dst_reg, src_reg, opcode, is_jmp32); in check_cond_jmp_op()
16526 &other_branch_regs[insn->dst_reg], in check_cond_jmp_op()
16528 dst_reg, &env->fake_reg[1], in check_cond_jmp_op()
16540 if (dst_reg->type == SCALAR_VALUE && dst_reg->id && in check_cond_jmp_op()
16541 !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) { in check_cond_jmp_op()
16542 sync_linked_regs(this_branch, dst_reg, &linked_regs); in check_cond_jmp_op()
16543 sync_linked_regs(other_branch, &other_branch_regs[insn->dst_reg], &linked_regs); in check_cond_jmp_op()
16559 __is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) && in check_cond_jmp_op()
16560 type_may_be_null(src_reg->type) != type_may_be_null(dst_reg->type) && in check_cond_jmp_op()
16562 base_type(dst_reg->type) != PTR_TO_BTF_ID) { in check_cond_jmp_op()
16579 mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]); in check_cond_jmp_op()
16589 type_may_be_null(dst_reg->type)) { in check_cond_jmp_op()
16593 mark_ptr_or_null_regs(this_branch, insn->dst_reg, in check_cond_jmp_op()
16595 mark_ptr_or_null_regs(other_branch, insn->dst_reg, in check_cond_jmp_op()
16597 } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], in check_cond_jmp_op()
16599 is_pointer_value(env, insn->dst_reg)) { in check_cond_jmp_op()
16601 insn->dst_reg); in check_cond_jmp_op()
16614 struct bpf_reg_state *dst_reg; in check_ld_imm() local
16627 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_ld_imm()
16631 dst_reg = ®s[insn->dst_reg]; in check_ld_imm()
16635 dst_reg->type = SCALAR_VALUE; in check_ld_imm()
16636 __mark_reg_known(®s[insn->dst_reg], imm); in check_ld_imm()
16644 mark_reg_known_zero(env, regs, insn->dst_reg); in check_ld_imm()
16647 dst_reg->type = aux->btf_var.reg_type; in check_ld_imm()
16648 switch (base_type(dst_reg->type)) { in check_ld_imm()
16650 dst_reg->mem_size = aux->btf_var.mem_size; in check_ld_imm()
16653 dst_reg->btf = aux->btf_var.btf; in check_ld_imm()
16654 dst_reg->btf_id = aux->btf_var.btf_id; in check_ld_imm()
16677 dst_reg->type = PTR_TO_FUNC; in check_ld_imm()
16678 dst_reg->subprogno = subprogno; in check_ld_imm()
16683 dst_reg->map_ptr = map; in check_ld_imm()
16688 __mark_reg_unknown(env, dst_reg); in check_ld_imm()
16691 dst_reg->type = PTR_TO_MAP_VALUE; in check_ld_imm()
16692 dst_reg->off = aux->map_off; in check_ld_imm()
16697 dst_reg->type = CONST_PTR_TO_MAP; in check_ld_imm()
16750 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || in check_ld_abs()
17373 stx->dst_reg != BPF_REG_10 || in mark_fastcall_pattern_for_call()
17377 if (stx->src_reg != ldx->dst_reg) in mark_fastcall_pattern_for_call()
17431 insn->dst_reg != BPF_REG_10) in mark_fastcall_patterns()
19534 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
19538 dst_reg_type = regs[insn->dst_reg].type; in do_check()
19541 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in do_check()
19561 insn->dst_reg != BPF_REG_0 || in do_check()
19593 insn->dst_reg != BPF_REG_0 || in do_check()
19610 insn->dst_reg != BPF_REG_0 || in do_check()
20106 insn[1].dst_reg != 0 || insn[1].src_reg != 0 || in resolve_pseudo_ldimm64()
20700 rnd_hi32_patch[3].dst_reg = load_reg; in opt_subreg_zext_lo32_rnd_hi32()
20729 zext_patch[1].dst_reg = load_reg; in opt_subreg_zext_lo32_rnd_hi32()
20989 insn->dst_reg, in convert_ctx_accesses()
20991 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, in convert_ctx_accesses()
20996 insn->dst_reg, in convert_ctx_accesses()
20998 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, in convert_ctx_accesses()
21004 insn->dst_reg, insn->dst_reg, in convert_ctx_accesses()
21595 BPF_NEG | BPF_K, insn->dst_reg, in do_misc_fixups()
21599 BPF_MOV32_IMM(insn->dst_reg, 0), in do_misc_fixups()
21630 BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), in do_misc_fixups()
21641 BPF_MOV32_REG(insn->dst_reg, insn->dst_reg), in do_misc_fixups()
21659 BPF_MOV | BPF_K, insn->dst_reg, in do_misc_fixups()
21663 BPF_NEG | BPF_K, insn->dst_reg, in do_misc_fixups()
21681 BPF_MOV32_IMM(insn->dst_reg, 0), in do_misc_fixups()
21685 BPF_MOV32_REG(insn->dst_reg, insn->dst_reg), in do_misc_fixups()
21727 *patch++ = BPF_MOV64_IMM(insn->dst_reg, 0); in do_misc_fixups()
21779 off_reg = issrc ? insn->src_reg : insn->dst_reg; in do_misc_fixups()
21793 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); in do_misc_fixups()
23689 u16 dst = BIT(insn->dst_reg); in compute_insn_live_regs()