Lines Matching refs:ctrlr
49 static void nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
53 nvme_ctrlr_construct_admin_qpair(struct nvme_controller *ctrlr) in nvme_ctrlr_construct_admin_qpair() argument
59 qpair = &ctrlr->adminq; in nvme_ctrlr_construct_admin_qpair()
69 nvme_printf(ctrlr, "invalid hw.nvme.admin_entries=%d " in nvme_ctrlr_construct_admin_qpair()
83 ctrlr); in nvme_ctrlr_construct_admin_qpair()
88 nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr) in nvme_ctrlr_construct_io_qpairs() argument
103 cap_lo = nvme_mmio_read_4(ctrlr, cap_lo); in nvme_ctrlr_construct_io_qpairs()
125 ctrlr->max_hw_pend_io = num_trackers * ctrlr->num_io_queues * 3 / 4; in nvme_ctrlr_construct_io_qpairs()
132 ctrlr->num_cpus_per_ioq = howmany(mp_ncpus, ctrlr->num_io_queues); in nvme_ctrlr_construct_io_qpairs()
134 ctrlr->ioq = malloc(ctrlr->num_io_queues * sizeof(struct nvme_qpair), in nvme_ctrlr_construct_io_qpairs()
137 for (i = 0; i < ctrlr->num_io_queues; i++) { in nvme_ctrlr_construct_io_qpairs()
138 qpair = &ctrlr->ioq[i]; in nvme_ctrlr_construct_io_qpairs()
149 ctrlr->msix_enabled ? i+1 : 0, /* vector */ in nvme_ctrlr_construct_io_qpairs()
152 ctrlr); in nvme_ctrlr_construct_io_qpairs()
160 if (ctrlr->num_io_queues > 1) in nvme_ctrlr_construct_io_qpairs()
161 bus_bind_intr(ctrlr->dev, qpair->res, in nvme_ctrlr_construct_io_qpairs()
162 i * ctrlr->num_cpus_per_ioq); in nvme_ctrlr_construct_io_qpairs()
169 nvme_ctrlr_fail(struct nvme_controller *ctrlr) in nvme_ctrlr_fail() argument
173 ctrlr->is_failed = TRUE; in nvme_ctrlr_fail()
174 nvme_admin_qpair_disable(&ctrlr->adminq); in nvme_ctrlr_fail()
175 nvme_qpair_fail(&ctrlr->adminq); in nvme_ctrlr_fail()
176 if (ctrlr->ioq != NULL) { in nvme_ctrlr_fail()
177 for (i = 0; i < ctrlr->num_io_queues; i++) { in nvme_ctrlr_fail()
178 nvme_io_qpair_disable(&ctrlr->ioq[i]); in nvme_ctrlr_fail()
179 nvme_qpair_fail(&ctrlr->ioq[i]); in nvme_ctrlr_fail()
182 nvme_notify_fail_consumers(ctrlr); in nvme_ctrlr_fail()
186 nvme_ctrlr_post_failed_request(struct nvme_controller *ctrlr, in nvme_ctrlr_post_failed_request() argument
190 mtx_lock(&ctrlr->lock); in nvme_ctrlr_post_failed_request()
191 STAILQ_INSERT_TAIL(&ctrlr->fail_req, req, stailq); in nvme_ctrlr_post_failed_request()
192 mtx_unlock(&ctrlr->lock); in nvme_ctrlr_post_failed_request()
193 taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->fail_req_task); in nvme_ctrlr_post_failed_request()
199 struct nvme_controller *ctrlr = arg; in nvme_ctrlr_fail_req_task() local
202 mtx_lock(&ctrlr->lock); in nvme_ctrlr_fail_req_task()
203 while ((req = STAILQ_FIRST(&ctrlr->fail_req)) != NULL) { in nvme_ctrlr_fail_req_task()
204 STAILQ_REMOVE_HEAD(&ctrlr->fail_req, stailq); in nvme_ctrlr_fail_req_task()
205 mtx_unlock(&ctrlr->lock); in nvme_ctrlr_fail_req_task()
208 mtx_lock(&ctrlr->lock); in nvme_ctrlr_fail_req_task()
210 mtx_unlock(&ctrlr->lock); in nvme_ctrlr_fail_req_task()
214 nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr, int desired_val) in nvme_ctrlr_wait_for_ready() argument
221 csts = nvme_mmio_read_4(ctrlr, csts); in nvme_ctrlr_wait_for_ready()
227 if (ms_waited++ > ctrlr->ready_timeout_in_ms) { in nvme_ctrlr_wait_for_ready()
228 nvme_printf(ctrlr, "controller ready did not become %d " in nvme_ctrlr_wait_for_ready()
229 "within %d ms\n", desired_val, ctrlr->ready_timeout_in_ms); in nvme_ctrlr_wait_for_ready()
239 nvme_ctrlr_disable(struct nvme_controller *ctrlr) in nvme_ctrlr_disable() argument
246 cc = nvme_mmio_read_4(ctrlr, cc); in nvme_ctrlr_disable()
247 csts = nvme_mmio_read_4(ctrlr, csts); in nvme_ctrlr_disable()
261 err = nvme_ctrlr_wait_for_ready(ctrlr, 1); in nvme_ctrlr_disable()
270 return (nvme_ctrlr_wait_for_ready(ctrlr, 0)); in nvme_ctrlr_disable()
274 nvme_mmio_write_4(ctrlr, cc, cc); in nvme_ctrlr_disable()
280 if (ctrlr->quirks & QUIRK_DELAY_B4_CHK_RDY) in nvme_ctrlr_disable()
282 return (nvme_ctrlr_wait_for_ready(ctrlr, 0)); in nvme_ctrlr_disable()
286 nvme_ctrlr_enable(struct nvme_controller *ctrlr) in nvme_ctrlr_enable() argument
295 cc = nvme_mmio_read_4(ctrlr, cc); in nvme_ctrlr_enable()
296 csts = nvme_mmio_read_4(ctrlr, csts); in nvme_ctrlr_enable()
308 return (nvme_ctrlr_wait_for_ready(ctrlr, 1)); in nvme_ctrlr_enable()
311 err = nvme_ctrlr_wait_for_ready(ctrlr, 0); in nvme_ctrlr_enable()
316 nvme_mmio_write_8(ctrlr, asq, ctrlr->adminq.cmd_bus_addr); in nvme_ctrlr_enable()
318 nvme_mmio_write_8(ctrlr, acq, ctrlr->adminq.cpl_bus_addr); in nvme_ctrlr_enable()
322 qsize = ctrlr->adminq.num_entries - 1; in nvme_ctrlr_enable()
327 nvme_mmio_write_4(ctrlr, aqa, aqa); in nvme_ctrlr_enable()
342 nvme_mmio_write_4(ctrlr, cc, cc); in nvme_ctrlr_enable()
344 return (nvme_ctrlr_wait_for_ready(ctrlr, 1)); in nvme_ctrlr_enable()
348 nvme_ctrlr_disable_qpairs(struct nvme_controller *ctrlr) in nvme_ctrlr_disable_qpairs() argument
352 nvme_admin_qpair_disable(&ctrlr->adminq); in nvme_ctrlr_disable_qpairs()
358 if (ctrlr->is_initialized) { in nvme_ctrlr_disable_qpairs()
359 for (i = 0; i < ctrlr->num_io_queues; i++) in nvme_ctrlr_disable_qpairs()
360 nvme_io_qpair_disable(&ctrlr->ioq[i]); in nvme_ctrlr_disable_qpairs()
365 nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr) in nvme_ctrlr_hw_reset() argument
369 nvme_ctrlr_disable_qpairs(ctrlr); in nvme_ctrlr_hw_reset()
373 err = nvme_ctrlr_disable(ctrlr); in nvme_ctrlr_hw_reset()
376 return (nvme_ctrlr_enable(ctrlr)); in nvme_ctrlr_hw_reset()
380 nvme_ctrlr_reset(struct nvme_controller *ctrlr) in nvme_ctrlr_reset() argument
384 cmpset = atomic_cmpset_32(&ctrlr->is_resetting, 0, 1); in nvme_ctrlr_reset()
386 if (cmpset == 0 || ctrlr->is_failed) in nvme_ctrlr_reset()
394 taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->reset_task); in nvme_ctrlr_reset()
398 nvme_ctrlr_identify(struct nvme_controller *ctrlr) in nvme_ctrlr_identify() argument
403 nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata, in nvme_ctrlr_identify()
407 nvme_printf(ctrlr, "nvme_identify_controller failed!\n"); in nvme_ctrlr_identify()
412 nvme_controller_data_swapbytes(&ctrlr->cdata); in nvme_ctrlr_identify()
418 if (ctrlr->cdata.mdts > 0) in nvme_ctrlr_identify()
419 ctrlr->max_xfer_size = min(ctrlr->max_xfer_size, in nvme_ctrlr_identify()
420 ctrlr->min_page_size * (1 << (ctrlr->cdata.mdts))); in nvme_ctrlr_identify()
426 nvme_ctrlr_set_num_qpairs(struct nvme_controller *ctrlr) in nvme_ctrlr_set_num_qpairs() argument
432 nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->num_io_queues, in nvme_ctrlr_set_num_qpairs()
436 nvme_printf(ctrlr, "nvme_ctrlr_set_num_qpairs failed!\n"); in nvme_ctrlr_set_num_qpairs()
453 ctrlr->num_io_queues = min(ctrlr->num_io_queues, sq_allocated); in nvme_ctrlr_set_num_qpairs()
454 ctrlr->num_io_queues = min(ctrlr->num_io_queues, cq_allocated); in nvme_ctrlr_set_num_qpairs()
460 nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr) in nvme_ctrlr_create_qpairs() argument
466 for (i = 0; i < ctrlr->num_io_queues; i++) { in nvme_ctrlr_create_qpairs()
467 qpair = &ctrlr->ioq[i]; in nvme_ctrlr_create_qpairs()
470 nvme_ctrlr_cmd_create_io_cq(ctrlr, qpair, qpair->vector, in nvme_ctrlr_create_qpairs()
474 nvme_printf(ctrlr, "nvme_create_io_cq failed!\n"); in nvme_ctrlr_create_qpairs()
479 nvme_ctrlr_cmd_create_io_sq(qpair->ctrlr, qpair, in nvme_ctrlr_create_qpairs()
483 nvme_printf(ctrlr, "nvme_create_io_sq failed!\n"); in nvme_ctrlr_create_qpairs()
492 nvme_ctrlr_delete_qpairs(struct nvme_controller *ctrlr) in nvme_ctrlr_delete_qpairs() argument
497 for (int i = 0; i < ctrlr->num_io_queues; i++) { in nvme_ctrlr_delete_qpairs()
498 qpair = &ctrlr->ioq[i]; in nvme_ctrlr_delete_qpairs()
501 nvme_ctrlr_cmd_delete_io_sq(ctrlr, qpair, in nvme_ctrlr_delete_qpairs()
505 nvme_printf(ctrlr, "nvme_destroy_io_sq failed!\n"); in nvme_ctrlr_delete_qpairs()
510 nvme_ctrlr_cmd_delete_io_cq(ctrlr, qpair, in nvme_ctrlr_delete_qpairs()
514 nvme_printf(ctrlr, "nvme_destroy_io_cq failed!\n"); in nvme_ctrlr_delete_qpairs()
523 nvme_ctrlr_construct_namespaces(struct nvme_controller *ctrlr) in nvme_ctrlr_construct_namespaces() argument
528 for (i = 0; i < min(ctrlr->cdata.nn, NVME_MAX_NAMESPACES); i++) { in nvme_ctrlr_construct_namespaces()
529 ns = &ctrlr->ns[i]; in nvme_ctrlr_construct_namespaces()
530 nvme_ns_construct(ns, i+1, ctrlr); in nvme_ctrlr_construct_namespaces()
555 nvme_ctrlr_get_log_page_size(struct nvme_controller *ctrlr, uint8_t page_id) in nvme_ctrlr_get_log_page_size() argument
563 (ctrlr->cdata.elpe + 1), NVME_MAX_AER_LOG_SIZE); in nvme_ctrlr_get_log_page_size()
592 nvme_ctrlr_log_critical_warnings(struct nvme_controller *ctrlr, in nvme_ctrlr_log_critical_warnings() argument
597 nvme_printf(ctrlr, "available spare space below threshold\n"); in nvme_ctrlr_log_critical_warnings()
600 nvme_printf(ctrlr, "temperature above threshold\n"); in nvme_ctrlr_log_critical_warnings()
603 nvme_printf(ctrlr, "device reliability degraded\n"); in nvme_ctrlr_log_critical_warnings()
606 nvme_printf(ctrlr, "media placed in read only mode\n"); in nvme_ctrlr_log_critical_warnings()
609 nvme_printf(ctrlr, "volatile memory backup device failed\n"); in nvme_ctrlr_log_critical_warnings()
612 nvme_printf(ctrlr, in nvme_ctrlr_log_critical_warnings()
631 nvme_notify_async_consumers(aer->ctrlr, &aer->cpl, in nvme_ctrlr_async_event_log_page_cb()
638 for (i = 0; i < (aer->ctrlr->cdata.elpe + 1); i++) in nvme_ctrlr_async_event_log_page_cb()
676 nvme_ctrlr_log_critical_warnings(aer->ctrlr, in nvme_ctrlr_async_event_log_page_cb()
685 aer->ctrlr->async_event_config &= in nvme_ctrlr_async_event_log_page_cb()
687 nvme_ctrlr_cmd_set_async_event_config(aer->ctrlr, in nvme_ctrlr_async_event_log_page_cb()
688 aer->ctrlr->async_event_config, NULL, NULL); in nvme_ctrlr_async_event_log_page_cb()
695 nvme_notify_ns(aer->ctrlr, nsl->ns[i]); in nvme_ctrlr_async_event_log_page_cb()
704 nvme_notify_async_consumers(aer->ctrlr, &aer->cpl, in nvme_ctrlr_async_event_log_page_cb()
712 nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer); in nvme_ctrlr_async_event_log_page_cb()
733 nvme_printf(aer->ctrlr, "async event occurred (type 0x%x, info 0x%02x," in nvme_ctrlr_async_event_cb()
738 aer->log_page_size = nvme_ctrlr_get_log_page_size(aer->ctrlr, in nvme_ctrlr_async_event_cb()
741 nvme_ctrlr_cmd_get_log_page(aer->ctrlr, aer->log_page_id, in nvme_ctrlr_async_event_cb()
747 nvme_notify_async_consumers(aer->ctrlr, cpl, aer->log_page_id, in nvme_ctrlr_async_event_cb()
754 nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer); in nvme_ctrlr_async_event_cb()
759 nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr, in nvme_ctrlr_construct_and_submit_aer() argument
764 aer->ctrlr = ctrlr; in nvme_ctrlr_construct_and_submit_aer()
774 nvme_ctrlr_submit_admin_request(ctrlr, req); in nvme_ctrlr_construct_and_submit_aer()
778 nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr) in nvme_ctrlr_configure_aer() argument
784 ctrlr->async_event_config = NVME_CRIT_WARN_ST_AVAILABLE_SPARE | in nvme_ctrlr_configure_aer()
788 if (ctrlr->cdata.ver >= NVME_REV(1, 2)) in nvme_ctrlr_configure_aer()
789 ctrlr->async_event_config |= 0x300; in nvme_ctrlr_configure_aer()
792 nvme_ctrlr_cmd_get_feature(ctrlr, NVME_FEAT_TEMPERATURE_THRESHOLD, in nvme_ctrlr_configure_aer()
798 nvme_printf(ctrlr, "temperature threshold not supported\n"); in nvme_ctrlr_configure_aer()
800 ctrlr->async_event_config |= NVME_CRIT_WARN_ST_TEMPERATURE; in nvme_ctrlr_configure_aer()
802 nvme_ctrlr_cmd_set_async_event_config(ctrlr, in nvme_ctrlr_configure_aer()
803 ctrlr->async_event_config, NULL, NULL); in nvme_ctrlr_configure_aer()
806 ctrlr->num_aers = min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl+1)); in nvme_ctrlr_configure_aer()
808 for (i = 0; i < ctrlr->num_aers; i++) { in nvme_ctrlr_configure_aer()
809 aer = &ctrlr->aer[i]; in nvme_ctrlr_configure_aer()
810 nvme_ctrlr_construct_and_submit_aer(ctrlr, aer); in nvme_ctrlr_configure_aer()
815 nvme_ctrlr_configure_int_coalescing(struct nvme_controller *ctrlr) in nvme_ctrlr_configure_int_coalescing() argument
818 ctrlr->int_coal_time = 0; in nvme_ctrlr_configure_int_coalescing()
820 &ctrlr->int_coal_time); in nvme_ctrlr_configure_int_coalescing()
822 ctrlr->int_coal_threshold = 0; in nvme_ctrlr_configure_int_coalescing()
824 &ctrlr->int_coal_threshold); in nvme_ctrlr_configure_int_coalescing()
826 nvme_ctrlr_cmd_set_interrupt_coalescing(ctrlr, ctrlr->int_coal_time, in nvme_ctrlr_configure_int_coalescing()
827 ctrlr->int_coal_threshold, NULL, NULL); in nvme_ctrlr_configure_int_coalescing()
833 struct nvme_controller *ctrlr = ctrlr_arg; in nvme_ctrlr_start() local
845 nvme_qpair_reset(&ctrlr->adminq); in nvme_ctrlr_start()
847 for (i = 0; i < ctrlr->num_io_queues; i++) in nvme_ctrlr_start()
848 nvme_qpair_reset(&ctrlr->ioq[i]); in nvme_ctrlr_start()
850 nvme_admin_qpair_enable(&ctrlr->adminq); in nvme_ctrlr_start()
852 if (nvme_ctrlr_identify(ctrlr) != 0) { in nvme_ctrlr_start()
853 nvme_ctrlr_fail(ctrlr); in nvme_ctrlr_start()
866 old_num_io_queues = ctrlr->num_io_queues; in nvme_ctrlr_start()
867 if (nvme_ctrlr_set_num_qpairs(ctrlr) != 0) { in nvme_ctrlr_start()
868 nvme_ctrlr_fail(ctrlr); in nvme_ctrlr_start()
872 if (old_num_io_queues != ctrlr->num_io_queues) { in nvme_ctrlr_start()
874 old_num_io_queues, ctrlr->num_io_queues); in nvme_ctrlr_start()
878 if (nvme_ctrlr_create_qpairs(ctrlr) != 0) { in nvme_ctrlr_start()
879 nvme_ctrlr_fail(ctrlr); in nvme_ctrlr_start()
883 if (nvme_ctrlr_construct_namespaces(ctrlr) != 0) { in nvme_ctrlr_start()
884 nvme_ctrlr_fail(ctrlr); in nvme_ctrlr_start()
888 nvme_ctrlr_configure_aer(ctrlr); in nvme_ctrlr_start()
889 nvme_ctrlr_configure_int_coalescing(ctrlr); in nvme_ctrlr_start()
891 for (i = 0; i < ctrlr->num_io_queues; i++) in nvme_ctrlr_start()
892 nvme_io_qpair_enable(&ctrlr->ioq[i]); in nvme_ctrlr_start()
898 struct nvme_controller *ctrlr = arg; in nvme_ctrlr_start_config_hook() local
900 nvme_qpair_reset(&ctrlr->adminq); in nvme_ctrlr_start_config_hook()
901 nvme_admin_qpair_enable(&ctrlr->adminq); in nvme_ctrlr_start_config_hook()
903 if (nvme_ctrlr_set_num_qpairs(ctrlr) == 0 && in nvme_ctrlr_start_config_hook()
904 nvme_ctrlr_construct_io_qpairs(ctrlr) == 0) in nvme_ctrlr_start_config_hook()
905 nvme_ctrlr_start(ctrlr, false); in nvme_ctrlr_start_config_hook()
907 nvme_ctrlr_fail(ctrlr); in nvme_ctrlr_start_config_hook()
909 nvme_sysctl_initialize_ctrlr(ctrlr); in nvme_ctrlr_start_config_hook()
910 config_intrhook_disestablish(&ctrlr->config_hook); in nvme_ctrlr_start_config_hook()
912 ctrlr->is_initialized = 1; in nvme_ctrlr_start_config_hook()
913 nvme_notify_new_controller(ctrlr); in nvme_ctrlr_start_config_hook()
919 struct nvme_controller *ctrlr = arg; in nvme_ctrlr_reset_task() local
922 nvme_printf(ctrlr, "resetting controller\n"); in nvme_ctrlr_reset_task()
923 status = nvme_ctrlr_hw_reset(ctrlr); in nvme_ctrlr_reset_task()
934 nvme_ctrlr_start(ctrlr, true); in nvme_ctrlr_reset_task()
936 nvme_ctrlr_fail(ctrlr); in nvme_ctrlr_reset_task()
938 atomic_cmpset_32(&ctrlr->is_resetting, 1, 0); in nvme_ctrlr_reset_task()
945 nvme_ctrlr_poll(struct nvme_controller *ctrlr) in nvme_ctrlr_poll() argument
949 nvme_qpair_process_completions(&ctrlr->adminq); in nvme_ctrlr_poll()
951 for (i = 0; i < ctrlr->num_io_queues; i++) in nvme_ctrlr_poll()
952 if (ctrlr->ioq && ctrlr->ioq[i].cpl) in nvme_ctrlr_poll()
953 nvme_qpair_process_completions(&ctrlr->ioq[i]); in nvme_ctrlr_poll()
964 struct nvme_controller *ctrlr = arg; in nvme_ctrlr_intx_handler() local
966 nvme_mmio_write_4(ctrlr, intms, 1); in nvme_ctrlr_intx_handler()
967 nvme_ctrlr_poll(ctrlr); in nvme_ctrlr_intx_handler()
968 nvme_mmio_write_4(ctrlr, intmc, 1); in nvme_ctrlr_intx_handler()
992 nvme_ctrlr_passthrough_cmd(struct nvme_controller *ctrlr, in nvme_ctrlr_passthrough_cmd() argument
1014 if (pt->len > ctrlr->max_xfer_size) { in nvme_ctrlr_passthrough_cmd()
1015 nvme_printf(ctrlr, "pt->len (%d) " in nvme_ctrlr_passthrough_cmd()
1017 ctrlr->max_xfer_size); in nvme_ctrlr_passthrough_cmd()
1060 nvme_ctrlr_submit_admin_request(ctrlr, req); in nvme_ctrlr_passthrough_cmd()
1062 nvme_ctrlr_submit_io_request(ctrlr, req); in nvme_ctrlr_passthrough_cmd()
1082 struct nvme_controller *ctrlr; in nvme_ctrlr_ioctl() local
1085 ctrlr = cdev->si_drv1; in nvme_ctrlr_ioctl()
1089 nvme_ctrlr_reset(ctrlr); in nvme_ctrlr_ioctl()
1093 return (nvme_ctrlr_passthrough_cmd(ctrlr, pt, le32toh(pt->cmd.nsid), in nvme_ctrlr_ioctl()
1098 strncpy(gnsid->cdev, device_get_nameunit(ctrlr->dev), in nvme_ctrlr_ioctl()
1117 nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev) in nvme_ctrlr_construct() argument
1127 ctrlr->dev = dev; in nvme_ctrlr_construct()
1129 mtx_init(&ctrlr->lock, "nvme ctrlr lock", NULL, MTX_DEF); in nvme_ctrlr_construct()
1135 cap_hi = nvme_mmio_read_4(ctrlr, cap_hi); in nvme_ctrlr_construct()
1141 ctrlr->min_page_size = 1 << (12 + mpsmin); in nvme_ctrlr_construct()
1144 cap_lo = nvme_mmio_read_4(ctrlr, cap_lo); in nvme_ctrlr_construct()
1146 ctrlr->ready_timeout_in_ms = to * 500; in nvme_ctrlr_construct()
1152 ctrlr->timeout_period = timeout_period; in nvme_ctrlr_construct()
1157 ctrlr->enable_aborts = 0; in nvme_ctrlr_construct()
1158 TUNABLE_INT_FETCH("hw.nvme.enable_aborts", &ctrlr->enable_aborts); in nvme_ctrlr_construct()
1160 ctrlr->max_xfer_size = NVME_MAX_XFER_SIZE; in nvme_ctrlr_construct()
1161 if (nvme_ctrlr_construct_admin_qpair(ctrlr) != 0) in nvme_ctrlr_construct()
1164 ctrlr->taskqueue = taskqueue_create("nvme_taskq", M_WAITOK, in nvme_ctrlr_construct()
1165 taskqueue_thread_enqueue, &ctrlr->taskqueue); in nvme_ctrlr_construct()
1166 taskqueue_start_threads(&ctrlr->taskqueue, 1, PI_DISK, "nvme taskq"); in nvme_ctrlr_construct()
1168 ctrlr->is_resetting = 0; in nvme_ctrlr_construct()
1169 ctrlr->is_initialized = 0; in nvme_ctrlr_construct()
1170 ctrlr->notification_sent = 0; in nvme_ctrlr_construct()
1171 TASK_INIT(&ctrlr->reset_task, 0, nvme_ctrlr_reset_task, ctrlr); in nvme_ctrlr_construct()
1172 TASK_INIT(&ctrlr->fail_req_task, 0, nvme_ctrlr_fail_req_task, ctrlr); in nvme_ctrlr_construct()
1173 STAILQ_INIT(&ctrlr->fail_req); in nvme_ctrlr_construct()
1174 ctrlr->is_failed = FALSE; in nvme_ctrlr_construct()
1182 md_args.mda_si_drv1 = (void *)ctrlr; in nvme_ctrlr_construct()
1183 status = make_dev_s(&md_args, &ctrlr->cdev, "nvme%d", in nvme_ctrlr_construct()
1192 nvme_ctrlr_destruct(struct nvme_controller *ctrlr, device_t dev) in nvme_ctrlr_destruct() argument
1196 if (ctrlr->resource == NULL) in nvme_ctrlr_destruct()
1203 gone = (nvme_mmio_read_4(ctrlr, csts) == 0xffffffff); in nvme_ctrlr_destruct()
1205 nvme_ctrlr_fail(ctrlr); in nvme_ctrlr_destruct()
1207 nvme_notify_fail_consumers(ctrlr); in nvme_ctrlr_destruct()
1210 nvme_ns_destruct(&ctrlr->ns[i]); in nvme_ctrlr_destruct()
1212 if (ctrlr->cdev) in nvme_ctrlr_destruct()
1213 destroy_dev(ctrlr->cdev); in nvme_ctrlr_destruct()
1215 if (ctrlr->is_initialized) { in nvme_ctrlr_destruct()
1217 nvme_ctrlr_delete_qpairs(ctrlr); in nvme_ctrlr_destruct()
1218 for (i = 0; i < ctrlr->num_io_queues; i++) in nvme_ctrlr_destruct()
1219 nvme_io_qpair_destroy(&ctrlr->ioq[i]); in nvme_ctrlr_destruct()
1220 free(ctrlr->ioq, M_NVME); in nvme_ctrlr_destruct()
1221 nvme_admin_qpair_destroy(&ctrlr->adminq); in nvme_ctrlr_destruct()
1232 nvme_ctrlr_shutdown(ctrlr); in nvme_ctrlr_destruct()
1235 nvme_ctrlr_disable(ctrlr); in nvme_ctrlr_destruct()
1237 if (ctrlr->taskqueue) in nvme_ctrlr_destruct()
1238 taskqueue_free(ctrlr->taskqueue); in nvme_ctrlr_destruct()
1240 if (ctrlr->tag) in nvme_ctrlr_destruct()
1241 bus_teardown_intr(ctrlr->dev, ctrlr->res, ctrlr->tag); in nvme_ctrlr_destruct()
1243 if (ctrlr->res) in nvme_ctrlr_destruct()
1244 bus_release_resource(ctrlr->dev, SYS_RES_IRQ, in nvme_ctrlr_destruct()
1245 rman_get_rid(ctrlr->res), ctrlr->res); in nvme_ctrlr_destruct()
1247 if (ctrlr->bar4_resource != NULL) { in nvme_ctrlr_destruct()
1249 ctrlr->bar4_resource_id, ctrlr->bar4_resource); in nvme_ctrlr_destruct()
1253 ctrlr->resource_id, ctrlr->resource); in nvme_ctrlr_destruct()
1256 mtx_destroy(&ctrlr->lock); in nvme_ctrlr_destruct()
1260 nvme_ctrlr_shutdown(struct nvme_controller *ctrlr) in nvme_ctrlr_shutdown() argument
1266 cc = nvme_mmio_read_4(ctrlr, cc); in nvme_ctrlr_shutdown()
1269 nvme_mmio_write_4(ctrlr, cc, cc); in nvme_ctrlr_shutdown()
1272 csts = nvme_mmio_read_4(ctrlr, csts); in nvme_ctrlr_shutdown()
1278 nvme_printf(ctrlr, "did not complete shutdown within" in nvme_ctrlr_shutdown()
1287 nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr, in nvme_ctrlr_submit_admin_request() argument
1291 nvme_qpair_submit_request(&ctrlr->adminq, req); in nvme_ctrlr_submit_admin_request()
1295 nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr, in nvme_ctrlr_submit_io_request() argument
1300 qpair = &ctrlr->ioq[curcpu / ctrlr->num_cpus_per_ioq]; in nvme_ctrlr_submit_io_request()
1305 nvme_ctrlr_get_device(struct nvme_controller *ctrlr) in nvme_ctrlr_get_device() argument
1308 return (ctrlr->dev); in nvme_ctrlr_get_device()
1312 nvme_ctrlr_get_data(struct nvme_controller *ctrlr) in nvme_ctrlr_get_data() argument
1315 return (&ctrlr->cdata); in nvme_ctrlr_get_data()
1319 nvme_ctrlr_suspend(struct nvme_controller *ctrlr) in nvme_ctrlr_suspend() argument
1326 if (ctrlr->is_failed) in nvme_ctrlr_suspend()
1336 while (atomic_cmpset_32(&ctrlr->is_resetting, 0, 1) == 0 && to-- > 0) in nvme_ctrlr_suspend()
1339 nvme_printf(ctrlr, in nvme_ctrlr_suspend()
1354 nvme_ctrlr_delete_qpairs(ctrlr); in nvme_ctrlr_suspend()
1355 nvme_ctrlr_disable_qpairs(ctrlr); in nvme_ctrlr_suspend()
1357 nvme_ctrlr_shutdown(ctrlr); in nvme_ctrlr_suspend()
1363 nvme_ctrlr_resume(struct nvme_controller *ctrlr) in nvme_ctrlr_resume() argument
1369 if (ctrlr->is_failed) in nvme_ctrlr_resume()
1376 if (nvme_ctrlr_hw_reset(ctrlr) != 0) in nvme_ctrlr_resume()
1378 if (nvme_ctrlr_hw_reset(ctrlr) != 0) in nvme_ctrlr_resume()
1386 nvme_ctrlr_start(ctrlr, true); in nvme_ctrlr_resume()
1387 atomic_cmpset_32(&ctrlr->is_resetting, 1, 0); in nvme_ctrlr_resume()
1396 nvme_printf(ctrlr, "Failed to reset on resume, failing.\n"); in nvme_ctrlr_resume()
1397 nvme_ctrlr_fail(ctrlr); in nvme_ctrlr_resume()
1398 atomic_cmpset_32(&ctrlr->is_resetting, 1, 0); in nvme_ctrlr_resume()