100 	if (nvme_is_ana_error(status) && ns->ctrl->ana_log_buf) {  in nvme_failover_req()
102 		queue_work(nvme_wq, &ns->ctrl->ana_work);  in nvme_failover_req()
135 		atomic_inc(&ns->ctrl->nr_active);  in nvme_mpath_start_request()
153 		atomic_dec_if_positive(&ns->ctrl->nr_active);  in nvme_mpath_end_request()
162 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)  in nvme_kick_requeue_lists()  argument
167 	srcu_idx = srcu_read_lock(&ctrl->srcu);  in nvme_kick_requeue_lists()
168 	list_for_each_entry_srcu(ns, &ctrl->namespaces, list,  in nvme_kick_requeue_lists()
169 				 srcu_read_lock_held(&ctrl->srcu)) {  in nvme_kick_requeue_lists()
173 		if (nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE)  in nvme_kick_requeue_lists()
176 	srcu_read_unlock(&ctrl->srcu, srcu_idx);  in nvme_kick_requeue_lists()
207 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)  in nvme_mpath_clear_ctrl_paths()  argument
212 	srcu_idx = srcu_read_lock(&ctrl->srcu);  in nvme_mpath_clear_ctrl_paths()
213 	list_for_each_entry_srcu(ns, &ctrl->namespaces, list,  in nvme_mpath_clear_ctrl_paths()
214 				 srcu_read_lock_held(&ctrl->srcu)) {  in nvme_mpath_clear_ctrl_paths()
218 	srcu_read_unlock(&ctrl->srcu, srcu_idx);  in nvme_mpath_clear_ctrl_paths()
243 	enum nvme_ctrl_state state = nvme_ctrl_state(ns->ctrl);  in nvme_path_is_disabled()
268 		if (ns->ctrl->numa_node != NUMA_NO_NODE &&  in __nvme_find_path()
270 			distance = node_distance(node, ns->ctrl->numa_node);  in __nvme_find_path()
368 		depth = atomic_read(&ns->ctrl->nr_active);  in nvme_queue_depth_path()
396 	return nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE &&  in nvme_path_is_optimized()
434 		if (test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ns->ctrl->flags))  in nvme_available_path()
436 		switch (nvme_ctrl_state(ns->ctrl)) {  in nvme_available_path()
620 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)  in nvme_mpath_alloc_disk()  argument
635 	if (!(ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||  in nvme_mpath_alloc_disk()
636 	    !nvme_is_unique_nsid(ctrl, head) || !multipath)  in nvme_mpath_alloc_disk()
646 	head->disk = blk_alloc_disk(&lim, ctrl->numa_node);  in nvme_mpath_alloc_disk()
662 			ctrl->subsys->instance, head->instance);  in nvme_mpath_alloc_disk()
707 static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data,  in nvme_parse_ana_log()  argument
708 		int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *,  in nvme_parse_ana_log()  argument
711 	void *base = ctrl->ana_log_buf;  in nvme_parse_ana_log()
715 	lockdep_assert_held(&ctrl->ana_lock);  in nvme_parse_ana_log()
717 	for (i = 0; i < le16_to_cpu(ctrl->ana_log_buf->ngrps); i++) {  in nvme_parse_ana_log()
722 		if (WARN_ON_ONCE(offset > ctrl->ana_log_size - sizeof(*desc)))  in nvme_parse_ana_log()
730 		if (WARN_ON_ONCE(le32_to_cpu(desc->grpid) > ctrl->anagrpmax))  in nvme_parse_ana_log()
738 		if (WARN_ON_ONCE(offset > ctrl->ana_log_size - nsid_buf_size))  in nvme_parse_ana_log()
741 		error = cb(ctrl, desc, data);  in nvme_parse_ana_log()
772 	    nvme_ctrl_state(ns->ctrl) == NVME_CTRL_LIVE)  in nvme_update_ns_ana_state()
795 static int nvme_update_ana_state(struct nvme_ctrl *ctrl,  in nvme_update_ana_state()  argument
803 	dev_dbg(ctrl->device, "ANA group %d: %s.\n",  in nvme_update_ana_state()
813 	srcu_idx = srcu_read_lock(&ctrl->srcu);  in nvme_update_ana_state()
814 	list_for_each_entry_srcu(ns, &ctrl->namespaces, list,  in nvme_update_ana_state()
815 				 srcu_read_lock_held(&ctrl->srcu)) {  in nvme_update_ana_state()
828 	srcu_read_unlock(&ctrl->srcu, srcu_idx);  in nvme_update_ana_state()
832 static int nvme_read_ana_log(struct nvme_ctrl *ctrl)  in nvme_read_ana_log()  argument
837 	mutex_lock(&ctrl->ana_lock);  in nvme_read_ana_log()
838 	error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA, 0, NVME_CSI_NVM,  in nvme_read_ana_log()
839 			ctrl->ana_log_buf, ctrl->ana_log_size, 0);  in nvme_read_ana_log()
841 		dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error);  in nvme_read_ana_log()
845 	error = nvme_parse_ana_log(ctrl, &nr_change_groups,  in nvme_read_ana_log()
862 		mod_timer(&ctrl->anatt_timer, ctrl->anatt * HZ * 2 + jiffies);  in nvme_read_ana_log()
864 		timer_delete_sync(&ctrl->anatt_timer);  in nvme_read_ana_log()
866 	mutex_unlock(&ctrl->ana_lock);  in nvme_read_ana_log()
872 	struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, ana_work);  in nvme_ana_work()  local
874 	if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)  in nvme_ana_work()
877 	nvme_read_ana_log(ctrl);  in nvme_ana_work()
880 void nvme_mpath_update(struct nvme_ctrl *ctrl)  in nvme_mpath_update()  argument
884 	if (!ctrl->ana_log_buf)  in nvme_mpath_update()
887 	mutex_lock(&ctrl->ana_lock);  in nvme_mpath_update()
888 	nvme_parse_ana_log(ctrl, &nr_change_groups, nvme_update_ana_state);  in nvme_mpath_update()
889 	mutex_unlock(&ctrl->ana_lock);  in nvme_mpath_update()
894 	struct nvme_ctrl *ctrl = from_timer(ctrl, t, anatt_timer);  in nvme_anatt_timeout()  local
896 	dev_info(ctrl->device, "ANATT timeout, resetting controller.\n");  in nvme_anatt_timeout()
897 	nvme_reset_ctrl(ctrl);  in nvme_anatt_timeout()
900 void nvme_mpath_stop(struct nvme_ctrl *ctrl)  in nvme_mpath_stop()  argument
902 	if (!nvme_ctrl_use_ana(ctrl))  in nvme_mpath_stop()
904 	timer_delete_sync(&ctrl->anatt_timer);  in nvme_mpath_stop()
905 	cancel_work_sync(&ctrl->ana_work);  in nvme_mpath_stop()
925 	struct nvme_ctrl *ctrl;  in nvme_subsys_iopolicy_update()  local
935 	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)  in nvme_subsys_iopolicy_update()
936 		nvme_mpath_clear_ctrl_paths(ctrl);  in nvme_subsys_iopolicy_update()
988 	return sysfs_emit(buf, "%d\n", atomic_read(&ns->ctrl->nr_active));  in queue_depth_show()
1019 static int nvme_lookup_ana_group_desc(struct nvme_ctrl *ctrl,  in nvme_lookup_ana_group_desc()  argument
1111 	if (nvme_ctrl_use_ana(ns->ctrl)) {  in nvme_mpath_add_disk()
1117 		mutex_lock(&ns->ctrl->ana_lock);  in nvme_mpath_add_disk()
1119 		nvme_parse_ana_log(ns->ctrl, &desc, nvme_lookup_ana_group_desc);  in nvme_mpath_add_disk()
1120 		mutex_unlock(&ns->ctrl->ana_lock);  in nvme_mpath_add_disk()
1127 			queue_work(nvme_wq, &ns->ctrl->ana_work);  in nvme_mpath_add_disk()
1167 void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl)  in nvme_mpath_init_ctrl()  argument
1169 	mutex_init(&ctrl->ana_lock);  in nvme_mpath_init_ctrl()
1170 	timer_setup(&ctrl->anatt_timer, nvme_anatt_timeout, 0);  in nvme_mpath_init_ctrl()
1171 	INIT_WORK(&ctrl->ana_work, nvme_ana_work);  in nvme_mpath_init_ctrl()
1174 int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)  in nvme_mpath_init_identify()  argument
1176 	size_t max_transfer_size = ctrl->max_hw_sectors << SECTOR_SHIFT;  in nvme_mpath_init_identify()
1181 	if (!multipath || !ctrl->subsys ||  in nvme_mpath_init_identify()
1182 	    !(ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA))  in nvme_mpath_init_identify()
1186 	atomic_set(&ctrl->nr_active, 0);  in nvme_mpath_init_identify()
1188 	if (!ctrl->max_namespaces ||  in nvme_mpath_init_identify()
1189 	    ctrl->max_namespaces > le32_to_cpu(id->nn)) {  in nvme_mpath_init_identify()
1190 		dev_err(ctrl->device,  in nvme_mpath_init_identify()
1191 			"Invalid MNAN value %u\n", ctrl->max_namespaces);  in nvme_mpath_init_identify()
1195 	ctrl->anacap = id->anacap;  in nvme_mpath_init_identify()
1196 	ctrl->anatt = id->anatt;  in nvme_mpath_init_identify()
1197 	ctrl->nanagrpid = le32_to_cpu(id->nanagrpid);  in nvme_mpath_init_identify()
1198 	ctrl->anagrpmax = le32_to_cpu(id->anagrpmax);  in nvme_mpath_init_identify()
1201 		ctrl->nanagrpid * sizeof(struct nvme_ana_group_desc) +  in nvme_mpath_init_identify()
1202 		ctrl->max_namespaces * sizeof(__le32);  in nvme_mpath_init_identify()
1204 		dev_err(ctrl->device,  in nvme_mpath_init_identify()
1207 		dev_err(ctrl->device, "disabling ANA support.\n");  in nvme_mpath_init_identify()
1210 	if (ana_log_size > ctrl->ana_log_size) {  in nvme_mpath_init_identify()
1211 		nvme_mpath_stop(ctrl);  in nvme_mpath_init_identify()
1212 		nvme_mpath_uninit(ctrl);  in nvme_mpath_init_identify()
1213 		ctrl->ana_log_buf = kvmalloc(ana_log_size, GFP_KERNEL);  in nvme_mpath_init_identify()
1214 		if (!ctrl->ana_log_buf)  in nvme_mpath_init_identify()
1217 	ctrl->ana_log_size = ana_log_size;  in nvme_mpath_init_identify()
1218 	error = nvme_read_ana_log(ctrl);  in nvme_mpath_init_identify()
1224 	nvme_mpath_uninit(ctrl);  in nvme_mpath_init_identify()
1228 void nvme_mpath_uninit(struct nvme_ctrl *ctrl)  in nvme_mpath_uninit()  argument
1230 	kvfree(ctrl->ana_log_buf);  in nvme_mpath_uninit()
1231 	ctrl->ana_log_buf = NULL;  in nvme_mpath_uninit()
1232 	ctrl->ana_log_size = 0;  in nvme_mpath_uninit()