197 	struct nvme_ctrl	*ctrl;  member
416 static inline enum nvme_ctrl_state nvme_ctrl_state(struct nvme_ctrl *ctrl)  in nvme_ctrl_state()  argument
418 	return READ_ONCE(ctrl->state);  in nvme_ctrl_state()
526 	struct nvme_ctrl *ctrl;  member
565 	int (*reg_read32)(struct nvme_ctrl *ctrl, u32 off, u32 *val);
566 	int (*reg_write32)(struct nvme_ctrl *ctrl, u32 off, u32 val);
567 	int (*reg_read64)(struct nvme_ctrl *ctrl, u32 off, u64 *val);
568 	void (*free_ctrl)(struct nvme_ctrl *ctrl);
569 	void (*submit_async_event)(struct nvme_ctrl *ctrl);
570 	int (*subsystem_reset)(struct nvme_ctrl *ctrl);
571 	void (*delete_ctrl)(struct nvme_ctrl *ctrl);
572 	void (*stop_ctrl)(struct nvme_ctrl *ctrl);
573 	int (*get_address)(struct nvme_ctrl *ctrl, char *buf, int size);
574 	void (*print_device_info)(struct nvme_ctrl *ctrl);
575 	bool (*supports_pci_p2pdma)(struct nvme_ctrl *ctrl);
607 		dev_err(nvme_req(rq)->ctrl->device,  in nvme_find_rq()
631 static inline void nvme_print_device_info(struct nvme_ctrl *ctrl)  in nvme_print_device_info()  argument
633 	struct nvme_subsystem *subsys = ctrl->subsys;  in nvme_print_device_info()
635 	if (ctrl->ops->print_device_info) {  in nvme_print_device_info()
636 		ctrl->ops->print_device_info(ctrl);  in nvme_print_device_info()
640 	dev_err(ctrl->device,  in nvme_print_device_info()
664 bool nvme_wait_reset(struct nvme_ctrl *ctrl);
665 int nvme_try_sched_reset(struct nvme_ctrl *ctrl);
667 static inline int nvme_reset_subsystem(struct nvme_ctrl *ctrl)  in nvme_reset_subsystem()  argument
669 	if (!ctrl->subsystem || !ctrl->ops->subsystem_reset)  in nvme_reset_subsystem()
671 	return ctrl->ops->subsystem_reset(ctrl);  in nvme_reset_subsystem()
726 	struct nvme_ctrl *ctrl = rq->ctrl;  in nvme_try_complete_req()  local
728 	if (!(ctrl->quirks & NVME_QUIRK_SKIP_CID_GEN))  in nvme_try_complete_req()
740 static inline void nvme_get_ctrl(struct nvme_ctrl *ctrl)  in nvme_get_ctrl()  argument
742 	get_device(ctrl->device);  in nvme_get_ctrl()
745 static inline void nvme_put_ctrl(struct nvme_ctrl *ctrl)  in nvme_put_ctrl()  argument
747 	put_device(ctrl->device);  in nvme_put_ctrl()
759 static inline bool nvme_state_terminal(struct nvme_ctrl *ctrl)  in nvme_state_terminal()  argument
761 	switch (nvme_ctrl_state(ctrl)) {  in nvme_state_terminal()
772 		WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state);  in nvme_state_terminal()
795 void nvme_cancel_tagset(struct nvme_ctrl *ctrl);
796 void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl);
797 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
799 int nvme_disable_ctrl(struct nvme_ctrl *ctrl, bool shutdown);
800 int nvme_enable_ctrl(struct nvme_ctrl *ctrl);
801 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
803 int nvme_add_ctrl(struct nvme_ctrl *ctrl);
804 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl);
805 void nvme_start_ctrl(struct nvme_ctrl *ctrl);
806 void nvme_stop_ctrl(struct nvme_ctrl *ctrl);
807 int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl, bool was_suspended);
808 int nvme_alloc_admin_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
810 void nvme_remove_admin_tag_set(struct nvme_ctrl *ctrl);
811 int nvme_alloc_io_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
814 void nvme_remove_io_tag_set(struct nvme_ctrl *ctrl);
816 void nvme_remove_namespaces(struct nvme_ctrl *ctrl);
818 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
821 void nvme_quiesce_io_queues(struct nvme_ctrl *ctrl);
822 void nvme_unquiesce_io_queues(struct nvme_ctrl *ctrl);
823 void nvme_quiesce_admin_queue(struct nvme_ctrl *ctrl);
824 void nvme_unquiesce_admin_queue(struct nvme_ctrl *ctrl);
825 void nvme_mark_namespaces_dead(struct nvme_ctrl *ctrl);
826 void nvme_sync_queues(struct nvme_ctrl *ctrl);
827 void nvme_sync_io_queues(struct nvme_ctrl *ctrl);
828 void nvme_unfreeze(struct nvme_ctrl *ctrl);
829 void nvme_wait_freeze(struct nvme_ctrl *ctrl);
830 int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout);
831 void nvme_start_freeze(struct nvme_ctrl *ctrl);
842 blk_status_t nvme_fail_nonready_command(struct nvme_ctrl *ctrl,
844 bool __nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq,
847 static inline bool nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq,  in nvme_check_ready()  argument
850 	enum nvme_ctrl_state state = nvme_ctrl_state(ctrl);  in nvme_check_ready()
854 	if (ctrl->ops->flags & NVME_F_FABRICS && state == NVME_CTRL_DELETING)  in nvme_check_ready()
856 	return __nvme_check_ready(ctrl, rq, queue_live, state);  in nvme_check_ready()
868 static inline bool nvme_is_unique_nsid(struct nvme_ctrl *ctrl,  in nvme_is_unique_nsid()  argument
872 		(ctrl->oacs & NVME_CTRL_OACS_NS_MNGT_SUPP) ||  in nvme_is_unique_nsid()
873 		(ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA) ||  in nvme_is_unique_nsid()
874 		(ctrl->ctratt & NVME_CTRL_CTRATT_NVM_SETS);  in nvme_is_unique_nsid()
904 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count);
905 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl);
906 int nvme_reset_ctrl(struct nvme_ctrl *ctrl);
907 int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl);
908 int nvme_delete_ctrl(struct nvme_ctrl *ctrl);
909 void nvme_queue_scan(struct nvme_ctrl *ctrl);
910 int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
932 int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid,
946 void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl);
949 static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl)  in nvme_ctrl_use_ana()  argument
951 	return ctrl->ana_log_buf != NULL;  in nvme_ctrl_use_ana()
959 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl);
960 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl,struct nvme_ns_head *head);
965 int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id);
966 void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl);
967 void nvme_mpath_update(struct nvme_ctrl *ctrl);
968 void nvme_mpath_uninit(struct nvme_ctrl *ctrl);
969 void nvme_mpath_stop(struct nvme_ctrl *ctrl);
972 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl);
998 static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl)  in nvme_ctrl_use_ana()  argument
1005 static inline void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)  in nvme_kick_requeue_lists()  argument
1008 static inline int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl,  in nvme_mpath_alloc_disk()  argument
1032 static inline void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)  in nvme_mpath_clear_ctrl_paths()  argument
1041 static inline void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl)  in nvme_mpath_init_ctrl()  argument
1044 static inline int nvme_mpath_init_identify(struct nvme_ctrl *ctrl,  in nvme_mpath_init_identify()  argument
1047 	if (ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA)  in nvme_mpath_init_identify()
1048 		dev_warn(ctrl->device,  in nvme_mpath_init_identify()
1052 static inline void nvme_mpath_update(struct nvme_ctrl *ctrl)  in nvme_mpath_update()  argument
1055 static inline void nvme_mpath_uninit(struct nvme_ctrl *ctrl)  in nvme_mpath_uninit()  argument
1058 static inline void nvme_mpath_stop(struct nvme_ctrl *ctrl)  in nvme_mpath_stop()  argument
1122 int nvme_hwmon_init(struct nvme_ctrl *ctrl);
1123 void nvme_hwmon_exit(struct nvme_ctrl *ctrl);
1125 static inline int nvme_hwmon_init(struct nvme_ctrl *ctrl)  in nvme_hwmon_init()  argument
1130 static inline void nvme_hwmon_exit(struct nvme_ctrl *ctrl)  in nvme_hwmon_exit()  argument
1142 static inline bool nvme_ctrl_sgl_supported(struct nvme_ctrl *ctrl)  in nvme_ctrl_sgl_supported()  argument
1144 	return ctrl->sgls & (NVME_CTRL_SGLS_BYTE_ALIGNED |  in nvme_ctrl_sgl_supported()
1148 static inline bool nvme_ctrl_meta_sgl_supported(struct nvme_ctrl *ctrl)  in nvme_ctrl_meta_sgl_supported()  argument
1150 	if (ctrl->ops->flags & NVME_F_FABRICS)  in nvme_ctrl_meta_sgl_supported()
1152 	return ctrl->sgls & NVME_CTRL_SGLS_MSDS;  in nvme_ctrl_meta_sgl_supported()
1158 int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl);
1159 void nvme_auth_stop(struct nvme_ctrl *ctrl);
1160 int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid);
1161 int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid);
1162 void nvme_auth_free(struct nvme_ctrl *ctrl);
1163 void nvme_auth_revoke_tls_key(struct nvme_ctrl *ctrl);
1165 static inline int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl)  in nvme_auth_init_ctrl()  argument
1176 static inline void nvme_auth_stop(struct nvme_ctrl *ctrl) {};  in nvme_auth_stop()  argument
1177 static inline int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid)  in nvme_auth_negotiate()  argument
1181 static inline int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid)  in nvme_auth_wait()  argument
1185 static inline void nvme_auth_free(struct nvme_ctrl *ctrl) {};  in nvme_auth_free()  argument
1186 static inline void nvme_auth_revoke_tls_key(struct nvme_ctrl *ctrl) {};  in nvme_auth_revoke_tls_key()  argument
1189 u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1191 u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode);
1193 void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects,
1196 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid);
1200 static inline bool nvme_multi_css(struct nvme_ctrl *ctrl)  in nvme_multi_css()  argument
1202 	return (ctrl->ctrl_config & NVME_CC_CSS_MASK) == NVME_CC_CSS_CSI;  in nvme_multi_css()