Lines Matching refs:fpl
3935 static bool cache_fplookup_is_mp(struct cache_fpl *fpl);
3936 static int cache_fplookup_cross_mount(struct cache_fpl *fpl);
3937 static int cache_fplookup_partial_setup(struct cache_fpl *fpl);
3938 static int cache_fplookup_skip_slashes(struct cache_fpl *fpl);
3939 static int cache_fplookup_trailingslash(struct cache_fpl *fpl);
3940 static void cache_fpl_pathlen_dec(struct cache_fpl *fpl);
3941 static void cache_fpl_pathlen_inc(struct cache_fpl *fpl);
3942 static void cache_fpl_pathlen_add(struct cache_fpl *fpl, size_t n);
3943 static void cache_fpl_pathlen_sub(struct cache_fpl *fpl, size_t n);
3957 cache_fpl_handle_root(struct cache_fpl *fpl) in cache_fpl_handle_root() argument
3962 ndp = fpl->ndp; in cache_fpl_handle_root()
3963 cnp = fpl->cnp; in cache_fpl_handle_root()
3967 cache_fpl_pathlen_dec(fpl); in cache_fpl_handle_root()
3972 cache_fpl_pathlen_dec(fpl); in cache_fpl_handle_root()
3980 cache_fpl_checkpoint_outer(struct cache_fpl *fpl) in cache_fpl_checkpoint_outer() argument
3983 fpl->snd_outer.ni_pathlen = fpl->ndp->ni_pathlen; in cache_fpl_checkpoint_outer()
3984 fpl->snd_outer.cn_flags = fpl->ndp->ni_cnd.cn_flags; in cache_fpl_checkpoint_outer()
3988 cache_fpl_checkpoint(struct cache_fpl *fpl) in cache_fpl_checkpoint() argument
3992 fpl->snd.cn_nameptr = fpl->ndp->ni_cnd.cn_nameptr; in cache_fpl_checkpoint()
3993 fpl->snd.ni_pathlen = fpl->debug.ni_pathlen; in cache_fpl_checkpoint()
3998 cache_fpl_restore_partial(struct cache_fpl *fpl) in cache_fpl_restore_partial() argument
4001 fpl->ndp->ni_cnd.cn_flags = fpl->snd_outer.cn_flags; in cache_fpl_restore_partial()
4003 fpl->debug.ni_pathlen = fpl->snd.ni_pathlen; in cache_fpl_restore_partial()
4008 cache_fpl_restore_abort(struct cache_fpl *fpl) in cache_fpl_restore_abort() argument
4011 cache_fpl_restore_partial(fpl); in cache_fpl_restore_abort()
4015 fpl->ndp->ni_resflags = 0; in cache_fpl_restore_abort()
4016 fpl->ndp->ni_cnd.cn_nameptr = fpl->ndp->ni_cnd.cn_pnbuf; in cache_fpl_restore_abort()
4017 fpl->ndp->ni_pathlen = fpl->snd_outer.ni_pathlen; in cache_fpl_restore_abort()
4021 #define cache_fpl_smr_assert_entered(fpl) ({ \ argument
4022 struct cache_fpl *_fpl = (fpl); \
4026 #define cache_fpl_smr_assert_not_entered(fpl) ({ \ argument
4027 struct cache_fpl *_fpl = (fpl); \
4032 cache_fpl_assert_status(struct cache_fpl *fpl) in cache_fpl_assert_status() argument
4035 switch (fpl->status) { in cache_fpl_assert_status()
4047 #define cache_fpl_smr_assert_entered(fpl) do { } while (0) argument
4048 #define cache_fpl_smr_assert_not_entered(fpl) do { } while (0) argument
4049 #define cache_fpl_assert_status(fpl) do { } while (0) argument
4052 #define cache_fpl_smr_enter_initial(fpl) ({ \ argument
4053 struct cache_fpl *_fpl = (fpl); \
4058 #define cache_fpl_smr_enter(fpl) ({ \ argument
4059 struct cache_fpl *_fpl = (fpl); \
4065 #define cache_fpl_smr_exit(fpl) ({ \ argument
4066 struct cache_fpl *_fpl = (fpl); \
4073 cache_fpl_aborted_early_impl(struct cache_fpl *fpl, int line) in cache_fpl_aborted_early_impl() argument
4076 if (fpl->status != CACHE_FPL_STATUS_UNSET) { in cache_fpl_aborted_early_impl()
4077 KASSERT(fpl->status == CACHE_FPL_STATUS_PARTIAL, in cache_fpl_aborted_early_impl()
4079 __func__, fpl->status, line, fpl->line)); in cache_fpl_aborted_early_impl()
4081 cache_fpl_smr_assert_not_entered(fpl); in cache_fpl_aborted_early_impl()
4082 fpl->status = CACHE_FPL_STATUS_ABORTED; in cache_fpl_aborted_early_impl()
4083 fpl->line = line; in cache_fpl_aborted_early_impl()
4090 cache_fpl_aborted_impl(struct cache_fpl *fpl, int line) in cache_fpl_aborted_impl() argument
4095 ndp = fpl->ndp; in cache_fpl_aborted_impl()
4096 cnp = fpl->cnp; in cache_fpl_aborted_impl()
4098 if (fpl->status != CACHE_FPL_STATUS_UNSET) { in cache_fpl_aborted_impl()
4099 KASSERT(fpl->status == CACHE_FPL_STATUS_PARTIAL, in cache_fpl_aborted_impl()
4101 __func__, fpl->status, line, fpl->line)); in cache_fpl_aborted_impl()
4103 fpl->status = CACHE_FPL_STATUS_ABORTED; in cache_fpl_aborted_impl()
4104 fpl->line = line; in cache_fpl_aborted_impl()
4105 if (fpl->in_smr) in cache_fpl_aborted_impl()
4106 cache_fpl_smr_exit(fpl); in cache_fpl_aborted_impl()
4107 cache_fpl_restore_abort(fpl); in cache_fpl_aborted_impl()
4113 fpl->status = CACHE_FPL_STATUS_DESTROYED; in cache_fpl_aborted_impl()
4122 cache_fpl_partial_impl(struct cache_fpl *fpl, int line) in cache_fpl_partial_impl() argument
4125 KASSERT(fpl->status == CACHE_FPL_STATUS_UNSET, in cache_fpl_partial_impl()
4127 __func__, line, fpl->status, fpl->line)); in cache_fpl_partial_impl()
4128 cache_fpl_smr_assert_entered(fpl); in cache_fpl_partial_impl()
4129 fpl->status = CACHE_FPL_STATUS_PARTIAL; in cache_fpl_partial_impl()
4130 fpl->line = line; in cache_fpl_partial_impl()
4131 return (cache_fplookup_partial_setup(fpl)); in cache_fpl_partial_impl()
4137 cache_fpl_handled_impl(struct cache_fpl *fpl, int line) in cache_fpl_handled_impl() argument
4140 KASSERT(fpl->status == CACHE_FPL_STATUS_UNSET, in cache_fpl_handled_impl()
4142 __func__, line, fpl->status, fpl->line)); in cache_fpl_handled_impl()
4143 cache_fpl_smr_assert_not_entered(fpl); in cache_fpl_handled_impl()
4144 fpl->status = CACHE_FPL_STATUS_HANDLED; in cache_fpl_handled_impl()
4145 fpl->line = line; in cache_fpl_handled_impl()
4152 cache_fpl_handled_error_impl(struct cache_fpl *fpl, int error, int line) in cache_fpl_handled_error_impl() argument
4155 KASSERT(fpl->status == CACHE_FPL_STATUS_UNSET, in cache_fpl_handled_error_impl()
4157 __func__, line, fpl->status, fpl->line)); in cache_fpl_handled_error_impl()
4160 cache_fpl_smr_assert_not_entered(fpl); in cache_fpl_handled_error_impl()
4161 fpl->status = CACHE_FPL_STATUS_HANDLED; in cache_fpl_handled_error_impl()
4162 fpl->line = line; in cache_fpl_handled_error_impl()
4163 fpl->dvp = NULL; in cache_fpl_handled_error_impl()
4164 fpl->tvp = NULL; in cache_fpl_handled_error_impl()
4165 fpl->savename = false; in cache_fpl_handled_error_impl()
4172 cache_fpl_terminated(struct cache_fpl *fpl) in cache_fpl_terminated() argument
4175 return (fpl->status != CACHE_FPL_STATUS_UNSET); in cache_fpl_terminated()
4198 cache_fpl_istrailingslash(struct cache_fpl *fpl) in cache_fpl_istrailingslash() argument
4201 MPASS(fpl->nulchar > fpl->cnp->cn_pnbuf); in cache_fpl_istrailingslash()
4202 return (*(fpl->nulchar - 1) == '/'); in cache_fpl_istrailingslash()
4216 cache_can_fplookup(struct cache_fpl *fpl) in cache_can_fplookup() argument
4222 ndp = fpl->ndp; in cache_can_fplookup()
4223 cnp = fpl->cnp; in cache_can_fplookup()
4227 cache_fpl_aborted_early(fpl); in cache_can_fplookup()
4231 cache_fpl_aborted_early(fpl); in cache_can_fplookup()
4235 cache_fpl_aborted_early(fpl); in cache_can_fplookup()
4239 cache_fpl_aborted_early(fpl); in cache_can_fplookup()
4243 cache_fpl_aborted_early(fpl); in cache_can_fplookup()
4250 cache_fplookup_dirfd(struct cache_fpl *fpl, struct vnode **vpp) in cache_fplookup_dirfd() argument
4257 ndp = fpl->ndp; in cache_fplookup_dirfd()
4258 cnp = fpl->cnp; in cache_fplookup_dirfd()
4262 return (cache_fpl_aborted(fpl)); in cache_fplookup_dirfd()
4264 fpl->fsearch = fsearch; in cache_fplookup_dirfd()
4267 cache_fpl_smr_exit(fpl); in cache_fplookup_dirfd()
4268 return (cache_fpl_handled_error(fpl, ENOTDIR)); in cache_fplookup_dirfd()
4275 cache_fplookup_negative_promote(struct cache_fpl *fpl, struct namecache *oncp, in cache_fplookup_negative_promote() argument
4281 cnp = fpl->cnp; in cache_fplookup_negative_promote()
4282 dvp = fpl->dvp; in cache_fplookup_negative_promote()
4284 cache_fpl_smr_exit(fpl); in cache_fplookup_negative_promote()
4286 return (cache_fpl_handled_error(fpl, ENOENT)); in cache_fplookup_negative_promote()
4288 return (cache_fpl_aborted(fpl)); in cache_fplookup_negative_promote()
4295 cache_fplookup_partial_setup(struct cache_fpl *fpl) in cache_fplookup_partial_setup() argument
4304 ndp = fpl->ndp; in cache_fplookup_partial_setup()
4305 cnp = fpl->cnp; in cache_fplookup_partial_setup()
4306 pwd = *(fpl->pwd); in cache_fplookup_partial_setup()
4307 dvp = fpl->dvp; in cache_fplookup_partial_setup()
4308 dvp_seqc = fpl->dvp_seqc; in cache_fplookup_partial_setup()
4311 return (cache_fpl_aborted(fpl)); in cache_fplookup_partial_setup()
4325 cache_fpl_smr_exit(fpl); in cache_fplookup_partial_setup()
4328 return (cache_fpl_aborted(fpl)); in cache_fplookup_partial_setup()
4335 return (cache_fpl_aborted(fpl)); in cache_fplookup_partial_setup()
4338 cache_fpl_restore_partial(fpl); in cache_fplookup_partial_setup()
4340 if (cnp->cn_nameptr != fpl->snd.cn_nameptr) { in cache_fplookup_partial_setup()
4342 cnp->cn_nameptr, fpl->snd.cn_nameptr, cnp->cn_pnbuf); in cache_fplookup_partial_setup()
4360 cache_fpl_pathlen_dec(fpl); in cache_fplookup_partial_setup()
4364 ndp->ni_pathlen = fpl->nulchar - cnp->cn_nameptr + 1; in cache_fplookup_partial_setup()
4366 if (ndp->ni_pathlen != fpl->debug.ni_pathlen) { in cache_fplookup_partial_setup()
4368 __func__, ndp->ni_pathlen, fpl->debug.ni_pathlen, fpl->nulchar, in cache_fplookup_partial_setup()
4376 cache_fplookup_final_child(struct cache_fpl *fpl, enum vgetstate tvs) in cache_fplookup_final_child() argument
4383 cnp = fpl->cnp; in cache_fplookup_final_child()
4384 tvp = fpl->tvp; in cache_fplookup_final_child()
4385 tvp_seqc = fpl->tvp_seqc; in cache_fplookup_final_child()
4393 return (cache_fpl_aborted(fpl)); in cache_fplookup_final_child()
4404 return (cache_fpl_aborted(fpl)); in cache_fplookup_final_child()
4407 return (cache_fpl_handled(fpl)); in cache_fplookup_final_child()
4414 cache_fplookup_final_modifying(struct cache_fpl *fpl) in cache_fplookup_final_modifying() argument
4425 ndp = fpl->ndp; in cache_fplookup_final_modifying()
4426 cnp = fpl->cnp; in cache_fplookup_final_modifying()
4427 dvp = fpl->dvp; in cache_fplookup_final_modifying()
4428 dvp_seqc = fpl->dvp_seqc; in cache_fplookup_final_modifying()
4449 if (cache_fpl_istrailingslash(fpl)) { in cache_fplookup_final_modifying()
4450 return (cache_fpl_partial(fpl)); in cache_fplookup_final_modifying()
4455 return (cache_fpl_aborted(fpl)); in cache_fplookup_final_modifying()
4459 cache_fpl_smr_exit(fpl); in cache_fplookup_final_modifying()
4465 return (cache_fpl_aborted(fpl)); in cache_fplookup_final_modifying()
4467 return (cache_fpl_handled_error(fpl, EROFS)); in cache_fplookup_final_modifying()
4470 if (fpl->tvp != NULL && (cnp->cn_flags & FAILIFEXISTS) != 0) { in cache_fplookup_final_modifying()
4471 cache_fpl_smr_exit(fpl); in cache_fplookup_final_modifying()
4472 return (cache_fpl_handled_error(fpl, EEXIST)); in cache_fplookup_final_modifying()
4490 cache_fpl_smr_exit(fpl); in cache_fplookup_final_modifying()
4492 return (cache_fpl_aborted(fpl)); in cache_fplookup_final_modifying()
4498 return (cache_fpl_aborted(fpl)); in cache_fplookup_final_modifying()
4504 return (cache_fpl_aborted(fpl)); in cache_fplookup_final_modifying()
4522 return (cache_fpl_handled_error(fpl, error)); in cache_fplookup_final_modifying()
4525 return (cache_fpl_aborted(fpl)); in cache_fplookup_final_modifying()
4528 fpl->tvp = tvp; in cache_fplookup_final_modifying()
4529 fpl->savename = (cnp->cn_flags & SAVENAME) != 0; in cache_fplookup_final_modifying()
4536 fpl->savename = true; in cache_fplookup_final_modifying()
4542 return (cache_fpl_handled(fpl)); in cache_fplookup_final_modifying()
4555 return (cache_fpl_aborted(fpl)); in cache_fplookup_final_modifying()
4566 return (cache_fpl_aborted(fpl)); in cache_fplookup_final_modifying()
4574 if (__predict_false(cache_fplookup_is_mp(fpl))) { in cache_fplookup_final_modifying()
4577 return (cache_fpl_aborted(fpl)); in cache_fplookup_final_modifying()
4583 return (cache_fpl_handled_error(fpl, EEXIST)); in cache_fplookup_final_modifying()
4598 fpl->savename = true; in cache_fplookup_final_modifying()
4601 return (cache_fpl_handled(fpl)); in cache_fplookup_final_modifying()
4605 cache_fplookup_modifying(struct cache_fpl *fpl) in cache_fplookup_modifying() argument
4609 ndp = fpl->ndp; in cache_fplookup_modifying()
4612 return (cache_fpl_partial(fpl)); in cache_fplookup_modifying()
4614 return (cache_fplookup_final_modifying(fpl)); in cache_fplookup_modifying()
4618 cache_fplookup_final_withparent(struct cache_fpl *fpl) in cache_fplookup_final_withparent() argument
4626 cnp = fpl->cnp; in cache_fplookup_final_withparent()
4627 dvp = fpl->dvp; in cache_fplookup_final_withparent()
4628 dvp_seqc = fpl->dvp_seqc; in cache_fplookup_final_withparent()
4629 tvp = fpl->tvp; in cache_fplookup_final_withparent()
4638 return (cache_fpl_aborted(fpl)); in cache_fplookup_final_withparent()
4642 cache_fpl_smr_exit(fpl); in cache_fplookup_final_withparent()
4644 return (cache_fpl_aborted(fpl)); in cache_fplookup_final_withparent()
4647 cache_fpl_smr_exit(fpl); in cache_fplookup_final_withparent()
4653 return (cache_fpl_aborted(fpl)); in cache_fplookup_final_withparent()
4665 return (cache_fpl_aborted(fpl)); in cache_fplookup_final_withparent()
4668 error = cache_fplookup_final_child(fpl, tvs); in cache_fplookup_final_withparent()
4670 MPASS(fpl->status == CACHE_FPL_STATUS_ABORTED || in cache_fplookup_final_withparent()
4671 fpl->status == CACHE_FPL_STATUS_DESTROYED); in cache_fplookup_final_withparent()
4679 MPASS(fpl->status == CACHE_FPL_STATUS_HANDLED); in cache_fplookup_final_withparent()
4684 cache_fplookup_final(struct cache_fpl *fpl) in cache_fplookup_final() argument
4691 cnp = fpl->cnp; in cache_fplookup_final()
4692 dvp = fpl->dvp; in cache_fplookup_final()
4693 dvp_seqc = fpl->dvp_seqc; in cache_fplookup_final()
4694 tvp = fpl->tvp; in cache_fplookup_final()
4699 return (cache_fplookup_final_modifying(fpl)); in cache_fplookup_final()
4703 return (cache_fplookup_final_withparent(fpl)); in cache_fplookup_final()
4707 return (cache_fpl_partial(fpl)); in cache_fplookup_final()
4711 cache_fpl_smr_exit(fpl); in cache_fplookup_final()
4713 return (cache_fpl_aborted(fpl)); in cache_fplookup_final()
4716 cache_fpl_smr_exit(fpl); in cache_fplookup_final()
4717 return (cache_fplookup_final_child(fpl, tvs)); in cache_fplookup_final()
4726 cache_fplookup_degenerate(struct cache_fpl *fpl) in cache_fplookup_degenerate() argument
4736 fpl->tvp = fpl->dvp; in cache_fplookup_degenerate()
4737 fpl->tvp_seqc = fpl->dvp_seqc; in cache_fplookup_degenerate()
4739 cnp = fpl->cnp; in cache_fplookup_degenerate()
4740 dvp = fpl->dvp; in cache_fplookup_degenerate()
4751 cache_fpl_smr_exit(fpl); in cache_fplookup_degenerate()
4752 return (cache_fpl_handled_error(fpl, EISDIR)); in cache_fplookup_degenerate()
4758 return (cache_fplookup_final_withparent(fpl)); in cache_fplookup_degenerate()
4762 cache_fpl_smr_exit(fpl); in cache_fplookup_degenerate()
4764 return (cache_fpl_aborted(fpl)); in cache_fplookup_degenerate()
4773 return (cache_fpl_aborted(fpl)); in cache_fplookup_degenerate()
4778 return (cache_fpl_handled(fpl)); in cache_fplookup_degenerate()
4782 cache_fplookup_emptypath(struct cache_fpl *fpl) in cache_fplookup_emptypath() argument
4790 fpl->tvp = fpl->dvp; in cache_fplookup_emptypath()
4791 fpl->tvp_seqc = fpl->dvp_seqc; in cache_fplookup_emptypath()
4793 ndp = fpl->ndp; in cache_fplookup_emptypath()
4794 cnp = fpl->cnp; in cache_fplookup_emptypath()
4795 tvp = fpl->tvp; in cache_fplookup_emptypath()
4800 cache_fpl_smr_exit(fpl); in cache_fplookup_emptypath()
4801 return (cache_fpl_handled_error(fpl, ENOENT)); in cache_fplookup_emptypath()
4807 cache_fpl_smr_exit(fpl); in cache_fplookup_emptypath()
4809 return (cache_fpl_aborted(fpl)); in cache_fplookup_emptypath()
4818 return (cache_fpl_aborted(fpl)); in cache_fplookup_emptypath()
4825 return (cache_fpl_handled(fpl)); in cache_fplookup_emptypath()
4829 cache_fplookup_noentry(struct cache_fpl *fpl) in cache_fplookup_noentry() argument
4838 ndp = fpl->ndp; in cache_fplookup_noentry()
4839 cnp = fpl->cnp; in cache_fplookup_noentry()
4840 dvp = fpl->dvp; in cache_fplookup_noentry()
4841 dvp_seqc = fpl->dvp_seqc; in cache_fplookup_noentry()
4853 cache_fpl_smr_exit(fpl); in cache_fplookup_noentry()
4854 return (cache_fpl_handled_error(fpl, ENAMETOOLONG)); in cache_fplookup_noentry()
4858 return (cache_fplookup_skip_slashes(fpl)); in cache_fplookup_noentry()
4862 return (cache_fplookup_emptypath(fpl)); in cache_fplookup_noentry()
4866 if (fpl->tvp == NULL) { in cache_fplookup_noentry()
4867 return (cache_fplookup_degenerate(fpl)); in cache_fplookup_noentry()
4869 return (cache_fplookup_trailingslash(fpl)); in cache_fplookup_noentry()
4873 fpl->tvp = NULL; in cache_fplookup_noentry()
4874 return (cache_fplookup_modifying(fpl)); in cache_fplookup_noentry()
4885 return (cache_fpl_partial(fpl)); in cache_fplookup_noentry()
4893 if (cache_fpl_istrailingslash(fpl)) { in cache_fplookup_noentry()
4894 return (cache_fpl_partial(fpl)); in cache_fplookup_noentry()
4902 cache_fpl_smr_exit(fpl); in cache_fplookup_noentry()
4904 return (cache_fpl_aborted(fpl)); in cache_fplookup_noentry()
4910 return (cache_fpl_aborted(fpl)); in cache_fplookup_noentry()
4916 return (cache_fpl_aborted(fpl)); in cache_fplookup_noentry()
4936 return (cache_fpl_handled_error(fpl, error)); in cache_fplookup_noentry()
4939 return (cache_fpl_aborted(fpl)); in cache_fplookup_noentry()
4942 fpl->tvp = tvp; in cache_fplookup_noentry()
4943 if (!fpl->savename) { in cache_fplookup_noentry()
4954 return (cache_fpl_handled(fpl)); in cache_fplookup_noentry()
4961 return (cache_fpl_aborted(fpl)); in cache_fplookup_noentry()
4965 if (__predict_false(cache_fplookup_is_mp(fpl))) { in cache_fplookup_noentry()
4968 return (cache_fpl_aborted(fpl)); in cache_fplookup_noentry()
4980 return (cache_fpl_handled(fpl)); in cache_fplookup_noentry()
4984 cache_fplookup_dot(struct cache_fpl *fpl) in cache_fplookup_dot() argument
4988 MPASS(!seqc_in_modify(fpl->dvp_seqc)); in cache_fplookup_dot()
4994 fpl->tvp = fpl->dvp; in cache_fplookup_dot()
4995 fpl->tvp_seqc = fpl->dvp_seqc; in cache_fplookup_dot()
4998 SDT_PROBE3(vfs, namecache, lookup, hit, fpl->dvp, ".", fpl->dvp); in cache_fplookup_dot()
5001 if (cache_fplookup_is_mp(fpl)) { in cache_fplookup_dot()
5002 error = cache_fplookup_cross_mount(fpl); in cache_fplookup_dot()
5008 cache_fplookup_dotdot(struct cache_fpl *fpl) in cache_fplookup_dotdot() argument
5017 ndp = fpl->ndp; in cache_fplookup_dotdot()
5018 cnp = fpl->cnp; in cache_fplookup_dotdot()
5019 dvp = fpl->dvp; in cache_fplookup_dotdot()
5035 fpl->tvp = dvp; in cache_fplookup_dotdot()
5036 fpl->tvp_seqc = vn_seqc_read_any(dvp); in cache_fplookup_dotdot()
5037 if (seqc_in_modify(fpl->tvp_seqc)) { in cache_fplookup_dotdot()
5038 return (cache_fpl_aborted(fpl)); in cache_fplookup_dotdot()
5048 return (cache_fpl_partial(fpl)); in cache_fplookup_dotdot()
5053 return (cache_fpl_aborted(fpl)); in cache_fplookup_dotdot()
5059 return (cache_fpl_aborted(fpl)); in cache_fplookup_dotdot()
5060 fpl->tvp = ncp->nc_vp; in cache_fplookup_dotdot()
5062 fpl->tvp = ncp->nc_dvp; in cache_fplookup_dotdot()
5065 fpl->tvp_seqc = vn_seqc_read_any(fpl->tvp); in cache_fplookup_dotdot()
5066 if (seqc_in_modify(fpl->tvp_seqc)) { in cache_fplookup_dotdot()
5067 return (cache_fpl_partial(fpl)); in cache_fplookup_dotdot()
5074 return (cache_fpl_aborted(fpl)); in cache_fplookup_dotdot()
5078 return (cache_fpl_aborted(fpl)); in cache_fplookup_dotdot()
5086 cache_fplookup_neg(struct cache_fpl *fpl, struct namecache *ncp, uint32_t hash) in cache_fplookup_neg() argument
5098 if (__predict_false(fpl->cnp->cn_nameiop != LOOKUP)) { in cache_fplookup_neg()
5099 fpl->tvp = NULL; in cache_fplookup_neg()
5100 return (cache_fplookup_modifying(fpl)); in cache_fplookup_neg()
5105 return (cache_fpl_partial(fpl)); in cache_fplookup_neg()
5108 return (cache_fplookup_negative_promote(fpl, ncp, hash)); in cache_fplookup_neg()
5111 cache_fpl_smr_exit(fpl); in cache_fplookup_neg()
5112 return (cache_fpl_handled_error(fpl, ENOENT)); in cache_fplookup_neg()
5122 cache_symlink_resolve(struct cache_fpl *fpl, const char *string, size_t len) in cache_symlink_resolve() argument
5128 ndp = fpl->ndp; in cache_symlink_resolve()
5129 cnp = fpl->cnp; in cache_symlink_resolve()
5136 if (cache_fpl_istrailingslash(fpl)) { in cache_symlink_resolve()
5141 ndp->ni_pathlen = fpl->nulchar - cnp->cn_nameptr - cnp->cn_namelen + 1; in cache_symlink_resolve()
5143 if (ndp->ni_pathlen != fpl->debug.ni_pathlen) { in cache_symlink_resolve()
5145 __func__, ndp->ni_pathlen, fpl->debug.ni_pathlen, fpl->nulchar, in cache_symlink_resolve()
5162 if (cache_fpl_istrailingslash(fpl)) { in cache_symlink_resolve()
5173 cache_fpl_pathlen_add(fpl, adjust); in cache_symlink_resolve()
5175 fpl->nulchar = &cnp->cn_nameptr[ndp->ni_pathlen - 1]; in cache_symlink_resolve()
5176 fpl->tvp = NULL; in cache_symlink_resolve()
5181 cache_fplookup_symlink(struct cache_fpl *fpl) in cache_fplookup_symlink() argument
5189 ndp = fpl->ndp; in cache_fplookup_symlink()
5190 cnp = fpl->cnp; in cache_fplookup_symlink()
5191 dvp = fpl->dvp; in cache_fplookup_symlink()
5192 tvp = fpl->tvp; in cache_fplookup_symlink()
5196 return (cache_fplookup_final(fpl)); in cache_fplookup_symlink()
5202 return (cache_fpl_aborted(fpl)); in cache_fplookup_symlink()
5210 cache_fpl_smr_exit(fpl); in cache_fplookup_symlink()
5211 return (cache_fpl_handled_error(fpl, EACCES)); in cache_fplookup_symlink()
5214 error = VOP_FPLOOKUP_SYMLINK(tvp, fpl); in cache_fplookup_symlink()
5218 return (cache_fpl_partial(fpl)); in cache_fplookup_symlink()
5222 cache_fpl_smr_exit(fpl); in cache_fplookup_symlink()
5223 return (cache_fpl_handled_error(fpl, error)); in cache_fplookup_symlink()
5225 return (cache_fpl_aborted(fpl)); in cache_fplookup_symlink()
5230 fpl->dvp = cache_fpl_handle_root(fpl); in cache_fplookup_symlink()
5231 fpl->dvp_seqc = vn_seqc_read_any(fpl->dvp); in cache_fplookup_symlink()
5232 if (seqc_in_modify(fpl->dvp_seqc)) { in cache_fplookup_symlink()
5233 return (cache_fpl_aborted(fpl)); in cache_fplookup_symlink()
5240 mp = atomic_load_ptr(&fpl->dvp->v_mount); in cache_fplookup_symlink()
5242 return (cache_fpl_aborted(fpl)); in cache_fplookup_symlink()
5245 cache_fpl_checkpoint(fpl); in cache_fplookup_symlink()
5246 return (cache_fpl_partial(fpl)); in cache_fplookup_symlink()
5253 cache_fplookup_next(struct cache_fpl *fpl) in cache_fplookup_next() argument
5262 cnp = fpl->cnp; in cache_fplookup_next()
5263 dvp = fpl->dvp; in cache_fplookup_next()
5264 hash = fpl->hash; in cache_fplookup_next()
5268 return (cache_fplookup_dot(fpl)); in cache_fplookup_next()
5271 return (cache_fplookup_dotdot(fpl)); in cache_fplookup_next()
5284 return (cache_fplookup_noentry(fpl)); in cache_fplookup_next()
5290 return (cache_fplookup_neg(fpl, ncp, hash)); in cache_fplookup_next()
5294 return (cache_fpl_partial(fpl)); in cache_fplookup_next()
5297 fpl->tvp = tvp; in cache_fplookup_next()
5298 fpl->tvp_seqc = vn_seqc_read_any(tvp); in cache_fplookup_next()
5299 if (seqc_in_modify(fpl->tvp_seqc)) { in cache_fplookup_next()
5300 return (cache_fpl_partial(fpl)); in cache_fplookup_next()
5307 if (cache_fplookup_is_mp(fpl)) { in cache_fplookup_next()
5308 error = cache_fplookup_cross_mount(fpl); in cache_fplookup_next()
5342 cache_fplookup_climb_mount(struct cache_fpl *fpl) in cache_fplookup_climb_mount() argument
5349 vp = fpl->tvp; in cache_fplookup_climb_mount()
5350 vp_seqc = fpl->tvp_seqc; in cache_fplookup_climb_mount()
5363 return (cache_fpl_partial(fpl)); in cache_fplookup_climb_mount()
5369 return (cache_fpl_partial(fpl)); in cache_fplookup_climb_mount()
5373 return (cache_fpl_partial(fpl)); in cache_fplookup_climb_mount()
5378 return (cache_fpl_partial(fpl)); in cache_fplookup_climb_mount()
5383 return (cache_fpl_partial(fpl)); in cache_fplookup_climb_mount()
5393 fpl->tvp = vp; in cache_fplookup_climb_mount()
5394 fpl->tvp_seqc = vp_seqc; in cache_fplookup_climb_mount()
5399 cache_fplookup_cross_mount(struct cache_fpl *fpl) in cache_fplookup_cross_mount() argument
5406 vp = fpl->tvp; in cache_fplookup_cross_mount()
5407 vp_seqc = fpl->tvp_seqc; in cache_fplookup_cross_mount()
5416 return (cache_fpl_partial(fpl)); in cache_fplookup_cross_mount()
5420 return (cache_fpl_partial(fpl)); in cache_fplookup_cross_mount()
5424 return (cache_fpl_partial(fpl)); in cache_fplookup_cross_mount()
5429 return (cache_fpl_partial(fpl)); in cache_fplookup_cross_mount()
5434 return (cache_fpl_partial(fpl)); in cache_fplookup_cross_mount()
5443 return (cache_fplookup_climb_mount(fpl)); in cache_fplookup_cross_mount()
5446 fpl->tvp = vp; in cache_fplookup_cross_mount()
5447 fpl->tvp_seqc = vp_seqc; in cache_fplookup_cross_mount()
5455 cache_fplookup_is_mp(struct cache_fpl *fpl) in cache_fplookup_is_mp() argument
5459 vp = fpl->tvp; in cache_fplookup_is_mp()
5477 cache_fpl_pathlen_add(struct cache_fpl *fpl, size_t n) in cache_fpl_pathlen_add() argument
5480 fpl->debug.ni_pathlen += n; in cache_fpl_pathlen_add()
5481 KASSERT(fpl->debug.ni_pathlen <= PATH_MAX, in cache_fpl_pathlen_add()
5482 ("%s: pathlen overflow to %zd\n", __func__, fpl->debug.ni_pathlen)); in cache_fpl_pathlen_add()
5486 cache_fpl_pathlen_sub(struct cache_fpl *fpl, size_t n) in cache_fpl_pathlen_sub() argument
5489 fpl->debug.ni_pathlen -= n; in cache_fpl_pathlen_sub()
5490 KASSERT(fpl->debug.ni_pathlen <= PATH_MAX, in cache_fpl_pathlen_sub()
5491 ("%s: pathlen underflow to %zd\n", __func__, fpl->debug.ni_pathlen)); in cache_fpl_pathlen_sub()
5495 cache_fpl_pathlen_inc(struct cache_fpl *fpl) in cache_fpl_pathlen_inc() argument
5498 cache_fpl_pathlen_add(fpl, 1); in cache_fpl_pathlen_inc()
5502 cache_fpl_pathlen_dec(struct cache_fpl *fpl) in cache_fpl_pathlen_dec() argument
5505 cache_fpl_pathlen_sub(fpl, 1); in cache_fpl_pathlen_dec()
5509 cache_fpl_pathlen_add(struct cache_fpl *fpl, size_t n) in cache_fpl_pathlen_add() argument
5514 cache_fpl_pathlen_sub(struct cache_fpl *fpl, size_t n) in cache_fpl_pathlen_sub() argument
5519 cache_fpl_pathlen_inc(struct cache_fpl *fpl) in cache_fpl_pathlen_inc() argument
5524 cache_fpl_pathlen_dec(struct cache_fpl *fpl) in cache_fpl_pathlen_dec() argument
5530 cache_fplookup_parse(struct cache_fpl *fpl) in cache_fplookup_parse() argument
5538 ndp = fpl->ndp; in cache_fplookup_parse()
5539 cnp = fpl->cnp; in cache_fplookup_parse()
5540 dvp = fpl->dvp; in cache_fplookup_parse()
5551 MPASS(&cnp->cn_nameptr[fpl->debug.ni_pathlen - 1] >= cnp->cn_pnbuf); in cache_fplookup_parse()
5552 KASSERT(&cnp->cn_nameptr[fpl->debug.ni_pathlen - 1] == fpl->nulchar, in cache_fplookup_parse()
5554 __func__, fpl->debug.ni_pathlen, &cnp->cn_nameptr[fpl->debug.ni_pathlen - 1], in cache_fplookup_parse()
5555 fpl->nulchar, cnp->cn_pnbuf)); in cache_fplookup_parse()
5556 KASSERT(*fpl->nulchar == '\0', in cache_fplookup_parse()
5557 ("%s: expected nul at %p; string [%s]\n", __func__, fpl->nulchar, in cache_fplookup_parse()
5560 *fpl->nulchar = '/'; in cache_fplookup_parse()
5568 *fpl->nulchar = '\0'; in cache_fplookup_parse()
5569 fpl->hash = cache_get_hash_iter_finish(hash); in cache_fplookup_parse()
5572 cache_fpl_pathlen_sub(fpl, cnp->cn_namelen); in cache_fplookup_parse()
5580 if (fpl->hash != cache_get_hash(cnp->cn_nameptr, cnp->cn_namelen, dvp)) { in cache_fplookup_parse()
5597 cache_fplookup_parse_advance(struct cache_fpl *fpl) in cache_fplookup_parse_advance() argument
5602 ndp = fpl->ndp; in cache_fplookup_parse_advance()
5603 cnp = fpl->cnp; in cache_fplookup_parse_advance()
5610 cache_fpl_pathlen_dec(fpl); in cache_fplookup_parse_advance()
5623 cache_fplookup_skip_slashes(struct cache_fpl *fpl) in cache_fplookup_skip_slashes() argument
5628 ndp = fpl->ndp; in cache_fplookup_skip_slashes()
5629 cnp = fpl->cnp; in cache_fplookup_skip_slashes()
5634 cache_fpl_pathlen_dec(fpl); in cache_fplookup_skip_slashes()
5642 cache_fpl_pathlen_inc(fpl); in cache_fplookup_skip_slashes()
5652 fpl->tvp = fpl->dvp; in cache_fplookup_skip_slashes()
5653 fpl->tvp_seqc = fpl->dvp_seqc; in cache_fplookup_skip_slashes()
5676 cache_fplookup_trailingslash(struct cache_fpl *fpl) in cache_fplookup_trailingslash() argument
5689 ndp = fpl->ndp; in cache_fplookup_trailingslash()
5690 cnp = fpl->cnp; in cache_fplookup_trailingslash()
5691 tvp = fpl->tvp; in cache_fplookup_trailingslash()
5692 tvp_seqc = fpl->tvp_seqc; in cache_fplookup_trailingslash()
5694 MPASS(fpl->dvp == fpl->tvp); in cache_fplookup_trailingslash()
5695 KASSERT(cache_fpl_istrailingslash(fpl), in cache_fplookup_trailingslash()
5696 ("%s: expected trailing slash at %p; string [%s]\n", __func__, fpl->nulchar - 1, in cache_fplookup_trailingslash()
5707 return (cache_fpl_aborted(fpl)); in cache_fplookup_trailingslash()
5712 return (cache_fpl_aborted(fpl)); in cache_fplookup_trailingslash()
5714 cache_fpl_smr_exit(fpl); in cache_fplookup_trailingslash()
5715 return (cache_fpl_handled_error(fpl, ENOTDIR)); in cache_fplookup_trailingslash()
5752 cache_fpl_pathlen_add(fpl, cn_nameptr_orig - cnp->cn_nameptr); in cache_fplookup_trailingslash()
5753 cache_fpl_checkpoint(fpl); in cache_fplookup_trailingslash()
5756 ni_pathlen = fpl->nulchar - cnp->cn_nameptr + 1; in cache_fplookup_trailingslash()
5757 if (ni_pathlen != fpl->debug.ni_pathlen) { in cache_fplookup_trailingslash()
5759 __func__, ni_pathlen, fpl->debug.ni_pathlen, fpl->nulchar, in cache_fplookup_trailingslash()
5774 tvp = fpl->tvp; in cache_fplookup_trailingslash()
5777 return (cache_fpl_aborted(fpl)); in cache_fplookup_trailingslash()
5781 return (cache_fpl_aborted(fpl)); in cache_fplookup_trailingslash()
5783 fpl->dvp = ncp->nc_dvp; in cache_fplookup_trailingslash()
5784 fpl->dvp_seqc = vn_seqc_read_any(fpl->dvp); in cache_fplookup_trailingslash()
5785 if (seqc_in_modify(fpl->dvp_seqc)) { in cache_fplookup_trailingslash()
5786 return (cache_fpl_aborted(fpl)); in cache_fplookup_trailingslash()
5795 cache_fplookup_failed_vexec(struct cache_fpl *fpl, int error) in cache_fplookup_failed_vexec() argument
5801 cnp = fpl->cnp; in cache_fplookup_failed_vexec()
5802 dvp = fpl->dvp; in cache_fplookup_failed_vexec()
5803 dvp_seqc = fpl->dvp_seqc; in cache_fplookup_failed_vexec()
5809 return (cache_fplookup_emptypath(fpl)); in cache_fplookup_failed_vexec()
5819 if (cache_fpl_istrailingslash(fpl)) { in cache_fplookup_failed_vexec()
5820 return (cache_fpl_aborted(fpl)); in cache_fplookup_failed_vexec()
5826 if (cnp->cn_nameptr[0] == '\0' && fpl->tvp == NULL) { in cache_fplookup_failed_vexec()
5827 return (cache_fplookup_degenerate(fpl)); in cache_fplookup_failed_vexec()
5834 cache_fpl_smr_exit(fpl); in cache_fplookup_failed_vexec()
5835 return (cache_fpl_handled_error(fpl, ENAMETOOLONG)); in cache_fplookup_failed_vexec()
5871 if (fpl->fsearch) { in cache_fplookup_failed_vexec()
5872 return (cache_fpl_aborted(fpl)); in cache_fplookup_failed_vexec()
5878 error = cache_fpl_aborted(fpl); in cache_fplookup_failed_vexec()
5880 cache_fpl_partial(fpl); in cache_fplookup_failed_vexec()
5885 error = cache_fpl_aborted(fpl); in cache_fplookup_failed_vexec()
5887 cache_fpl_smr_exit(fpl); in cache_fplookup_failed_vexec()
5888 cache_fpl_handled_error(fpl, error); in cache_fplookup_failed_vexec()
5896 cache_fplookup_impl(struct vnode *dvp, struct cache_fpl *fpl) in cache_fplookup_impl() argument
5903 ndp = fpl->ndp; in cache_fplookup_impl()
5904 cnp = fpl->cnp; in cache_fplookup_impl()
5906 cache_fpl_checkpoint(fpl); in cache_fplookup_impl()
5913 fpl->dvp = dvp; in cache_fplookup_impl()
5914 fpl->dvp_seqc = vn_seqc_read_notmodify(fpl->dvp); in cache_fplookup_impl()
5918 return (cache_fpl_aborted(fpl)); in cache_fplookup_impl()
5921 MPASS(fpl->tvp == NULL); in cache_fplookup_impl()
5924 cache_fplookup_parse(fpl); in cache_fplookup_impl()
5926 error = VOP_FPLOOKUP_VEXEC(fpl->dvp, cnp->cn_cred); in cache_fplookup_impl()
5928 error = cache_fplookup_failed_vexec(fpl, error); in cache_fplookup_impl()
5932 error = cache_fplookup_next(fpl); in cache_fplookup_impl()
5933 if (__predict_false(cache_fpl_terminated(fpl))) { in cache_fplookup_impl()
5937 VNPASS(!seqc_in_modify(fpl->tvp_seqc), fpl->tvp); in cache_fplookup_impl()
5939 if (fpl->tvp->v_type == VLNK) { in cache_fplookup_impl()
5940 error = cache_fplookup_symlink(fpl); in cache_fplookup_impl()
5941 if (cache_fpl_terminated(fpl)) { in cache_fplookup_impl()
5946 error = cache_fplookup_final(fpl); in cache_fplookup_impl()
5950 if (!vn_seqc_consistent(fpl->dvp, fpl->dvp_seqc)) { in cache_fplookup_impl()
5951 error = cache_fpl_aborted(fpl); in cache_fplookup_impl()
5955 fpl->dvp = fpl->tvp; in cache_fplookup_impl()
5956 fpl->dvp_seqc = fpl->tvp_seqc; in cache_fplookup_impl()
5957 cache_fplookup_parse_advance(fpl); in cache_fplookup_impl()
5960 cache_fpl_checkpoint(fpl); in cache_fplookup_impl()
6046 struct cache_fpl fpl; in cache_fplookup() local
6052 fpl.status = CACHE_FPL_STATUS_UNSET; in cache_fplookup()
6053 fpl.in_smr = false; in cache_fplookup()
6054 fpl.ndp = ndp; in cache_fplookup()
6055 fpl.cnp = cnp = &ndp->ni_cnd; in cache_fplookup()
6066 if (__predict_false(!cache_can_fplookup(&fpl))) { in cache_fplookup()
6067 *status = fpl.status; in cache_fplookup()
6068 SDT_PROBE3(vfs, fplookup, lookup, done, ndp, fpl.line, fpl.status); in cache_fplookup()
6072 cache_fpl_checkpoint_outer(&fpl); in cache_fplookup()
6074 cache_fpl_smr_enter_initial(&fpl); in cache_fplookup()
6076 fpl.debug.ni_pathlen = ndp->ni_pathlen; in cache_fplookup()
6078 fpl.nulchar = &cnp->cn_nameptr[ndp->ni_pathlen - 1]; in cache_fplookup()
6079 fpl.fsearch = false; in cache_fplookup()
6080 fpl.savename = (cnp->cn_flags & SAVENAME) != 0; in cache_fplookup()
6081 fpl.tvp = NULL; /* for degenerate path handling */ in cache_fplookup()
6082 fpl.pwd = pwdp; in cache_fplookup()
6084 *(fpl.pwd) = pwd; in cache_fplookup()
6089 dvp = cache_fpl_handle_root(&fpl); in cache_fplookup()
6096 error = cache_fplookup_dirfd(&fpl, &dvp); in cache_fplookup()
6104 error = cache_fplookup_impl(dvp, &fpl); in cache_fplookup()
6106 cache_fpl_smr_assert_not_entered(&fpl); in cache_fplookup()
6107 cache_fpl_assert_status(&fpl); in cache_fplookup()
6108 *status = fpl.status; in cache_fplookup()
6110 SDT_PROBE3(vfs, fplookup, lookup, done, ndp, fpl.line, fpl.status); in cache_fplookup()
6111 if (fpl.status == CACHE_FPL_STATUS_HANDLED) in cache_fplookup()
6116 if (__predict_true(fpl.status == CACHE_FPL_STATUS_HANDLED)) { in cache_fplookup()
6119 MPASS(fpl.dvp == NULL); in cache_fplookup()
6120 MPASS(fpl.tvp == NULL); in cache_fplookup()
6121 MPASS(fpl.savename == false); in cache_fplookup()
6123 ndp->ni_dvp = fpl.dvp; in cache_fplookup()
6124 ndp->ni_vp = fpl.tvp; in cache_fplookup()
6125 if (fpl.savename) { in cache_fplookup()