53 static void nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
57 nvme_ctrlr_barrier(struct nvme_controller *ctrlr, int flags)  in nvme_ctrlr_barrier()  argument
59 	bus_barrier(ctrlr->resource, 0, rman_get_size(ctrlr->resource), flags);  in nvme_ctrlr_barrier()
63 nvme_ctrlr_devctl_log(struct nvme_controller *ctrlr, const char *type, const char *msg, ...)  in nvme_ctrlr_devctl_log()  argument
71 	sbuf_printf(&sb, "%s: ", device_get_nameunit(ctrlr->dev));  in nvme_ctrlr_devctl_log()
80 	sbuf_printf(&sb, "name=\"%s\" reason=\"", device_get_nameunit(ctrlr->dev));  in nvme_ctrlr_devctl_log()
92 nvme_ctrlr_construct_admin_qpair(struct nvme_controller *ctrlr)  in nvme_ctrlr_construct_admin_qpair()  argument
98 	qpair = &ctrlr->adminq;  in nvme_ctrlr_construct_admin_qpair()
100 	qpair->cpu = CPU_FFS(&cpuset_domain[ctrlr->domain]) - 1;  in nvme_ctrlr_construct_admin_qpair()
101 	qpair->domain = ctrlr->domain;  in nvme_ctrlr_construct_admin_qpair()
111 		nvme_printf(ctrlr, "invalid hw.nvme.admin_entries=%d "  in nvme_ctrlr_construct_admin_qpair()
121 	     ctrlr);  in nvme_ctrlr_construct_admin_qpair()
125 #define QP(ctrlr, c)	((c) * (ctrlr)->num_io_queues / mp_ncpus)  argument
128 nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr)  in nvme_ctrlr_construct_io_qpairs()  argument
145 	    (rman_get_size(ctrlr->resource) - nvme_mmio_offsetof(doorbell[0])) /  in nvme_ctrlr_construct_io_qpairs()
146 	    (1 << (ctrlr->dstrd + 1));  in nvme_ctrlr_construct_io_qpairs()
149 	cap_lo = nvme_mmio_read_4(ctrlr, cap_lo);  in nvme_ctrlr_construct_io_qpairs()
172 	ctrlr->max_hw_pend_io = num_trackers * ctrlr->num_io_queues * 3 / 4;  in nvme_ctrlr_construct_io_qpairs()
174 	ctrlr->ioq = malloc(ctrlr->num_io_queues * sizeof(struct nvme_qpair),  in nvme_ctrlr_construct_io_qpairs()
177 	for (i = c = n = 0; i < ctrlr->num_io_queues; i++, c += n) {  in nvme_ctrlr_construct_io_qpairs()
178 		qpair = &ctrlr->ioq[i];  in nvme_ctrlr_construct_io_qpairs()
185 		if (ctrlr->num_io_queues > 1) {  in nvme_ctrlr_construct_io_qpairs()
187 			for (n = 1; QP(ctrlr, c + n) == i; n++)  in nvme_ctrlr_construct_io_qpairs()
190 			qpair->cpu = c + (device_get_unit(ctrlr->dev)+n/2) % n;  in nvme_ctrlr_construct_io_qpairs()
193 			qpair->cpu = CPU_FFS(&cpuset_domain[ctrlr->domain]) - 1;  in nvme_ctrlr_construct_io_qpairs()
194 			qpair->domain = ctrlr->domain;  in nvme_ctrlr_construct_io_qpairs()
202 		    ctrlr);  in nvme_ctrlr_construct_io_qpairs()
210 		if (ctrlr->num_io_queues > 1)  in nvme_ctrlr_construct_io_qpairs()
211 			bus_bind_intr(ctrlr->dev, qpair->res, qpair->cpu);  in nvme_ctrlr_construct_io_qpairs()
218 nvme_ctrlr_fail(struct nvme_controller *ctrlr)  in nvme_ctrlr_fail()  argument
222 	ctrlr->is_failed = true;  in nvme_ctrlr_fail()
223 	nvme_admin_qpair_disable(&ctrlr->adminq);  in nvme_ctrlr_fail()
224 	nvme_qpair_fail(&ctrlr->adminq);  in nvme_ctrlr_fail()
225 	if (ctrlr->ioq != NULL) {  in nvme_ctrlr_fail()
226 		for (i = 0; i < ctrlr->num_io_queues; i++) {  in nvme_ctrlr_fail()
227 			nvme_io_qpair_disable(&ctrlr->ioq[i]);  in nvme_ctrlr_fail()
228 			nvme_qpair_fail(&ctrlr->ioq[i]);  in nvme_ctrlr_fail()
231 	nvme_notify_fail_consumers(ctrlr);  in nvme_ctrlr_fail()
235 nvme_ctrlr_post_failed_request(struct nvme_controller *ctrlr,  in nvme_ctrlr_post_failed_request()  argument
239 	mtx_lock(&ctrlr->lock);  in nvme_ctrlr_post_failed_request()
240 	STAILQ_INSERT_TAIL(&ctrlr->fail_req, req, stailq);  in nvme_ctrlr_post_failed_request()
241 	mtx_unlock(&ctrlr->lock);  in nvme_ctrlr_post_failed_request()
242 	if (!ctrlr->is_dying)  in nvme_ctrlr_post_failed_request()
243 		taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->fail_req_task);  in nvme_ctrlr_post_failed_request()
249 	struct nvme_controller	*ctrlr = arg;  in nvme_ctrlr_fail_req_task()  local
252 	mtx_lock(&ctrlr->lock);  in nvme_ctrlr_fail_req_task()
253 	while ((req = STAILQ_FIRST(&ctrlr->fail_req)) != NULL) {  in nvme_ctrlr_fail_req_task()
254 		STAILQ_REMOVE_HEAD(&ctrlr->fail_req, stailq);  in nvme_ctrlr_fail_req_task()
255 		mtx_unlock(&ctrlr->lock);  in nvme_ctrlr_fail_req_task()
258 		mtx_lock(&ctrlr->lock);  in nvme_ctrlr_fail_req_task()
260 	mtx_unlock(&ctrlr->lock);  in nvme_ctrlr_fail_req_task()
270 nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr, int desired_val)  in nvme_ctrlr_wait_for_ready()  argument
272 	int timeout = ticks + MSEC_2_TICKS(ctrlr->ready_timeout_in_ms);  in nvme_ctrlr_wait_for_ready()
277 		csts = nvme_mmio_read_4(ctrlr, csts);  in nvme_ctrlr_wait_for_ready()
284 			nvme_printf(ctrlr, "controller ready did not become %d "  in nvme_ctrlr_wait_for_ready()
285 			    "within %d ms\n", desired_val, ctrlr->ready_timeout_in_ms);  in nvme_ctrlr_wait_for_ready()
297 nvme_ctrlr_disable(struct nvme_controller *ctrlr)  in nvme_ctrlr_disable()  argument
304 	cc = nvme_mmio_read_4(ctrlr, cc);  in nvme_ctrlr_disable()
305 	csts = nvme_mmio_read_4(ctrlr, csts);  in nvme_ctrlr_disable()
320 		return (nvme_ctrlr_wait_for_ready(ctrlr, 0));  in nvme_ctrlr_disable()
324 		err = nvme_ctrlr_wait_for_ready(ctrlr, 1);  in nvme_ctrlr_disable()
330 	nvme_mmio_write_4(ctrlr, cc, cc);  in nvme_ctrlr_disable()
336 	if (ctrlr->quirks & QUIRK_DELAY_B4_CHK_RDY)  in nvme_ctrlr_disable()
338 	return (nvme_ctrlr_wait_for_ready(ctrlr, 0));  in nvme_ctrlr_disable()
342 nvme_ctrlr_enable(struct nvme_controller *ctrlr)  in nvme_ctrlr_enable()  argument
351 	cc = nvme_mmio_read_4(ctrlr, cc);  in nvme_ctrlr_enable()
352 	csts = nvme_mmio_read_4(ctrlr, csts);  in nvme_ctrlr_enable()
363 		return (nvme_ctrlr_wait_for_ready(ctrlr, 1));  in nvme_ctrlr_enable()
367 	err = nvme_ctrlr_wait_for_ready(ctrlr, 0);  in nvme_ctrlr_enable()
371 	nvme_mmio_write_8(ctrlr, asq, ctrlr->adminq.cmd_bus_addr);  in nvme_ctrlr_enable()
372 	nvme_mmio_write_8(ctrlr, acq, ctrlr->adminq.cpl_bus_addr);  in nvme_ctrlr_enable()
375 	qsize = ctrlr->adminq.num_entries - 1;  in nvme_ctrlr_enable()
380 	nvme_mmio_write_4(ctrlr, aqa, aqa);  in nvme_ctrlr_enable()
394 	nvme_ctrlr_barrier(ctrlr, BUS_SPACE_BARRIER_WRITE);  in nvme_ctrlr_enable()
395 	nvme_mmio_write_4(ctrlr, cc, cc);  in nvme_ctrlr_enable()
397 	return (nvme_ctrlr_wait_for_ready(ctrlr, 1));  in nvme_ctrlr_enable()
401 nvme_ctrlr_disable_qpairs(struct nvme_controller *ctrlr)  in nvme_ctrlr_disable_qpairs()  argument
405 	nvme_admin_qpair_disable(&ctrlr->adminq);  in nvme_ctrlr_disable_qpairs()
411 	if (ctrlr->is_initialized) {  in nvme_ctrlr_disable_qpairs()
412 		for (i = 0; i < ctrlr->num_io_queues; i++)  in nvme_ctrlr_disable_qpairs()
413 			nvme_io_qpair_disable(&ctrlr->ioq[i]);  in nvme_ctrlr_disable_qpairs()
418 nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr)  in nvme_ctrlr_hw_reset()  argument
424 	nvme_ctrlr_disable_qpairs(ctrlr);  in nvme_ctrlr_hw_reset()
426 	err = nvme_ctrlr_disable(ctrlr);  in nvme_ctrlr_hw_reset()
430 	err = nvme_ctrlr_enable(ctrlr);  in nvme_ctrlr_hw_reset()
436 nvme_ctrlr_reset(struct nvme_controller *ctrlr)  in nvme_ctrlr_reset()  argument
440 	cmpset = atomic_cmpset_32(&ctrlr->is_resetting, 0, 1);  in nvme_ctrlr_reset()
442 	if (cmpset == 0 || ctrlr->is_failed)  in nvme_ctrlr_reset()
450 	if (!ctrlr->is_dying)  in nvme_ctrlr_reset()
451 		taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->reset_task);  in nvme_ctrlr_reset()
455 nvme_ctrlr_identify(struct nvme_controller *ctrlr)  in nvme_ctrlr_identify()  argument
460 	nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata,  in nvme_ctrlr_identify()
464 		nvme_printf(ctrlr, "nvme_identify_controller failed!\n");  in nvme_ctrlr_identify()
469 	nvme_controller_data_swapbytes(&ctrlr->cdata);  in nvme_ctrlr_identify()
475 	if (ctrlr->cdata.mdts > 0)  in nvme_ctrlr_identify()
476 		ctrlr->max_xfer_size = min(ctrlr->max_xfer_size,  in nvme_ctrlr_identify()
477 		    ctrlr->min_page_size * (1 << (ctrlr->cdata.mdts)));  in nvme_ctrlr_identify()
483 nvme_ctrlr_set_num_qpairs(struct nvme_controller *ctrlr)  in nvme_ctrlr_set_num_qpairs()  argument
489 	nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->num_io_queues,  in nvme_ctrlr_set_num_qpairs()
493 		nvme_printf(ctrlr, "nvme_ctrlr_set_num_qpairs failed!\n");  in nvme_ctrlr_set_num_qpairs()
510 	ctrlr->num_io_queues = min(ctrlr->num_io_queues, sq_allocated);  in nvme_ctrlr_set_num_qpairs()
511 	ctrlr->num_io_queues = min(ctrlr->num_io_queues, cq_allocated);  in nvme_ctrlr_set_num_qpairs()
512 	if (ctrlr->num_io_queues > vm_ndomains)  in nvme_ctrlr_set_num_qpairs()
513 		ctrlr->num_io_queues -= ctrlr->num_io_queues % vm_ndomains;  in nvme_ctrlr_set_num_qpairs()
519 nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr)  in nvme_ctrlr_create_qpairs()  argument
525 	for (i = 0; i < ctrlr->num_io_queues; i++) {  in nvme_ctrlr_create_qpairs()
526 		qpair = &ctrlr->ioq[i];  in nvme_ctrlr_create_qpairs()
529 		nvme_ctrlr_cmd_create_io_cq(ctrlr, qpair,  in nvme_ctrlr_create_qpairs()
533 			nvme_printf(ctrlr, "nvme_create_io_cq failed!\n");  in nvme_ctrlr_create_qpairs()
538 		nvme_ctrlr_cmd_create_io_sq(ctrlr, qpair,  in nvme_ctrlr_create_qpairs()
542 			nvme_printf(ctrlr, "nvme_create_io_sq failed!\n");  in nvme_ctrlr_create_qpairs()
551 nvme_ctrlr_delete_qpairs(struct nvme_controller *ctrlr)  in nvme_ctrlr_delete_qpairs()  argument
556 	for (int i = 0; i < ctrlr->num_io_queues; i++) {  in nvme_ctrlr_delete_qpairs()
557 		qpair = &ctrlr->ioq[i];  in nvme_ctrlr_delete_qpairs()
560 		nvme_ctrlr_cmd_delete_io_sq(ctrlr, qpair,  in nvme_ctrlr_delete_qpairs()
564 			nvme_printf(ctrlr, "nvme_destroy_io_sq failed!\n");  in nvme_ctrlr_delete_qpairs()
569 		nvme_ctrlr_cmd_delete_io_cq(ctrlr, qpair,  in nvme_ctrlr_delete_qpairs()
573 			nvme_printf(ctrlr, "nvme_destroy_io_cq failed!\n");  in nvme_ctrlr_delete_qpairs()
582 nvme_ctrlr_construct_namespaces(struct nvme_controller *ctrlr)  in nvme_ctrlr_construct_namespaces()  argument
587 	for (i = 0; i < min(ctrlr->cdata.nn, NVME_MAX_NAMESPACES); i++) {  in nvme_ctrlr_construct_namespaces()
588 		ns = &ctrlr->ns[i];  in nvme_ctrlr_construct_namespaces()
589 		nvme_ns_construct(ns, i+1, ctrlr);  in nvme_ctrlr_construct_namespaces()
614 nvme_ctrlr_get_log_page_size(struct nvme_controller *ctrlr, uint8_t page_id)  in nvme_ctrlr_get_log_page_size()  argument
622 		    (ctrlr->cdata.elpe + 1), NVME_MAX_AER_LOG_SIZE);  in nvme_ctrlr_get_log_page_size()
651 nvme_ctrlr_log_critical_warnings(struct nvme_controller *ctrlr,  in nvme_ctrlr_log_critical_warnings()  argument
656 		nvme_ctrlr_devctl_log(ctrlr, "critical",  in nvme_ctrlr_log_critical_warnings()
660 		nvme_ctrlr_devctl_log(ctrlr, "critical",  in nvme_ctrlr_log_critical_warnings()
664 		nvme_ctrlr_devctl_log(ctrlr, "critical",  in nvme_ctrlr_log_critical_warnings()
668 		nvme_ctrlr_devctl_log(ctrlr, "critical",  in nvme_ctrlr_log_critical_warnings()
672 		nvme_ctrlr_devctl_log(ctrlr, "critical",  in nvme_ctrlr_log_critical_warnings()
676 		nvme_ctrlr_devctl_log(ctrlr, "critical",  in nvme_ctrlr_log_critical_warnings()
695 		nvme_notify_async_consumers(aer->ctrlr, &aer->cpl,  in nvme_ctrlr_async_event_log_page_cb()
702 			for (i = 0; i < (aer->ctrlr->cdata.elpe + 1); i++)  in nvme_ctrlr_async_event_log_page_cb()
740 			nvme_ctrlr_log_critical_warnings(aer->ctrlr,  in nvme_ctrlr_async_event_log_page_cb()
749 			aer->ctrlr->async_event_config &=  in nvme_ctrlr_async_event_log_page_cb()
751 			nvme_ctrlr_cmd_set_async_event_config(aer->ctrlr,  in nvme_ctrlr_async_event_log_page_cb()
752 			    aer->ctrlr->async_event_config, NULL, NULL);  in nvme_ctrlr_async_event_log_page_cb()
759 				nvme_notify_ns(aer->ctrlr, nsl->ns[i]);  in nvme_ctrlr_async_event_log_page_cb()
767 		nvme_notify_async_consumers(aer->ctrlr, &aer->cpl,  in nvme_ctrlr_async_event_log_page_cb()
775 	nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer);  in nvme_ctrlr_async_event_log_page_cb()
796 	nvme_printf(aer->ctrlr, "async event occurred (type 0x%x, info 0x%02x,"  in nvme_ctrlr_async_event_cb()
801 		aer->log_page_size = nvme_ctrlr_get_log_page_size(aer->ctrlr,  in nvme_ctrlr_async_event_cb()
804 		nvme_ctrlr_cmd_get_log_page(aer->ctrlr, aer->log_page_id,  in nvme_ctrlr_async_event_cb()
810 		nvme_notify_async_consumers(aer->ctrlr, cpl, aer->log_page_id,  in nvme_ctrlr_async_event_cb()
817 		nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer);  in nvme_ctrlr_async_event_cb()
822 nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,  in nvme_ctrlr_construct_and_submit_aer()  argument
827 	aer->ctrlr = ctrlr;  in nvme_ctrlr_construct_and_submit_aer()
837 	nvme_ctrlr_submit_admin_request(ctrlr, req);  in nvme_ctrlr_construct_and_submit_aer()
841 nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr)  in nvme_ctrlr_configure_aer()  argument
847 	ctrlr->async_event_config = NVME_CRIT_WARN_ST_AVAILABLE_SPARE |  in nvme_ctrlr_configure_aer()
851 	if (ctrlr->cdata.ver >= NVME_REV(1, 2))  in nvme_ctrlr_configure_aer()
852 		ctrlr->async_event_config |= NVME_ASYNC_EVENT_NS_ATTRIBUTE |  in nvme_ctrlr_configure_aer()
856 	nvme_ctrlr_cmd_get_feature(ctrlr, NVME_FEAT_TEMPERATURE_THRESHOLD,  in nvme_ctrlr_configure_aer()
862 		nvme_printf(ctrlr, "temperature threshold not supported\n");  in nvme_ctrlr_configure_aer()
864 		ctrlr->async_event_config |= NVME_CRIT_WARN_ST_TEMPERATURE;  in nvme_ctrlr_configure_aer()
866 	nvme_ctrlr_cmd_set_async_event_config(ctrlr,  in nvme_ctrlr_configure_aer()
867 	    ctrlr->async_event_config, NULL, NULL);  in nvme_ctrlr_configure_aer()
870 	ctrlr->num_aers = min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl+1));  in nvme_ctrlr_configure_aer()
872 	for (i = 0; i < ctrlr->num_aers; i++) {  in nvme_ctrlr_configure_aer()
873 		aer = &ctrlr->aer[i];  in nvme_ctrlr_configure_aer()
874 		nvme_ctrlr_construct_and_submit_aer(ctrlr, aer);  in nvme_ctrlr_configure_aer()
879 nvme_ctrlr_configure_int_coalescing(struct nvme_controller *ctrlr)  in nvme_ctrlr_configure_int_coalescing()  argument
882 	ctrlr->int_coal_time = 0;  in nvme_ctrlr_configure_int_coalescing()
884 	    &ctrlr->int_coal_time);  in nvme_ctrlr_configure_int_coalescing()
886 	ctrlr->int_coal_threshold = 0;  in nvme_ctrlr_configure_int_coalescing()
888 	    &ctrlr->int_coal_threshold);  in nvme_ctrlr_configure_int_coalescing()
890 	nvme_ctrlr_cmd_set_interrupt_coalescing(ctrlr, ctrlr->int_coal_time,  in nvme_ctrlr_configure_int_coalescing()
891 	    ctrlr->int_coal_threshold, NULL, NULL);  in nvme_ctrlr_configure_int_coalescing()
895 nvme_ctrlr_hmb_free(struct nvme_controller *ctrlr)  in nvme_ctrlr_hmb_free()  argument
900 	if (ctrlr->hmb_desc_paddr) {  in nvme_ctrlr_hmb_free()
901 		bus_dmamap_unload(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_map);  in nvme_ctrlr_hmb_free()
902 		bus_dmamem_free(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_vaddr,  in nvme_ctrlr_hmb_free()
903 		    ctrlr->hmb_desc_map);  in nvme_ctrlr_hmb_free()
904 		ctrlr->hmb_desc_paddr = 0;  in nvme_ctrlr_hmb_free()
906 	if (ctrlr->hmb_desc_tag) {  in nvme_ctrlr_hmb_free()
907 		bus_dma_tag_destroy(ctrlr->hmb_desc_tag);  in nvme_ctrlr_hmb_free()
908 		ctrlr->hmb_desc_tag = NULL;  in nvme_ctrlr_hmb_free()
910 	for (i = 0; i < ctrlr->hmb_nchunks; i++) {  in nvme_ctrlr_hmb_free()
911 		hmbc = &ctrlr->hmb_chunks[i];  in nvme_ctrlr_hmb_free()
912 		bus_dmamap_unload(ctrlr->hmb_tag, hmbc->hmbc_map);  in nvme_ctrlr_hmb_free()
913 		bus_dmamem_free(ctrlr->hmb_tag, hmbc->hmbc_vaddr,  in nvme_ctrlr_hmb_free()
916 	ctrlr->hmb_nchunks = 0;  in nvme_ctrlr_hmb_free()
917 	if (ctrlr->hmb_tag) {  in nvme_ctrlr_hmb_free()
918 		bus_dma_tag_destroy(ctrlr->hmb_tag);  in nvme_ctrlr_hmb_free()
919 		ctrlr->hmb_tag = NULL;  in nvme_ctrlr_hmb_free()
921 	if (ctrlr->hmb_chunks) {  in nvme_ctrlr_hmb_free()
922 		free(ctrlr->hmb_chunks, M_NVME);  in nvme_ctrlr_hmb_free()
923 		ctrlr->hmb_chunks = NULL;  in nvme_ctrlr_hmb_free()
928 nvme_ctrlr_hmb_alloc(struct nvme_controller *ctrlr)  in nvme_ctrlr_hmb_alloc()  argument
939 	min = (long long unsigned)ctrlr->cdata.hmmin * 4096;  in nvme_ctrlr_hmb_alloc()
942 	pref = MIN((long long unsigned)ctrlr->cdata.hmpre * 4096, max);  in nvme_ctrlr_hmb_alloc()
943 	minc = MAX(ctrlr->cdata.hmminds * 4096, PAGE_SIZE);  in nvme_ctrlr_hmb_alloc()
944 	if (min > 0 && ctrlr->cdata.hmmaxd > 0)  in nvme_ctrlr_hmb_alloc()
945 		minc = MAX(minc, min / ctrlr->cdata.hmmaxd);  in nvme_ctrlr_hmb_alloc()
946 	ctrlr->hmb_chunk = pref;  in nvme_ctrlr_hmb_alloc()
949 	ctrlr->hmb_chunk = roundup2(ctrlr->hmb_chunk, PAGE_SIZE);  in nvme_ctrlr_hmb_alloc()
950 	ctrlr->hmb_nchunks = howmany(pref, ctrlr->hmb_chunk);  in nvme_ctrlr_hmb_alloc()
951 	if (ctrlr->cdata.hmmaxd > 0 && ctrlr->hmb_nchunks > ctrlr->cdata.hmmaxd)  in nvme_ctrlr_hmb_alloc()
952 		ctrlr->hmb_nchunks = ctrlr->cdata.hmmaxd;  in nvme_ctrlr_hmb_alloc()
953 	ctrlr->hmb_chunks = malloc(sizeof(struct nvme_hmb_chunk) *  in nvme_ctrlr_hmb_alloc()
954 	    ctrlr->hmb_nchunks, M_NVME, M_WAITOK);  in nvme_ctrlr_hmb_alloc()
955 	err = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev),  in nvme_ctrlr_hmb_alloc()
957 	    ctrlr->hmb_chunk, 1, ctrlr->hmb_chunk, 0, NULL, NULL, &ctrlr->hmb_tag);  in nvme_ctrlr_hmb_alloc()
959 		nvme_printf(ctrlr, "HMB tag create failed %d\n", err);  in nvme_ctrlr_hmb_alloc()
960 		nvme_ctrlr_hmb_free(ctrlr);  in nvme_ctrlr_hmb_alloc()
964 	for (i = 0; i < ctrlr->hmb_nchunks; i++) {  in nvme_ctrlr_hmb_alloc()
965 		hmbc = &ctrlr->hmb_chunks[i];  in nvme_ctrlr_hmb_alloc()
966 		if (bus_dmamem_alloc(ctrlr->hmb_tag,  in nvme_ctrlr_hmb_alloc()
969 			nvme_printf(ctrlr, "failed to alloc HMB\n");  in nvme_ctrlr_hmb_alloc()
972 		if (bus_dmamap_load(ctrlr->hmb_tag, hmbc->hmbc_map,  in nvme_ctrlr_hmb_alloc()
973 		    hmbc->hmbc_vaddr, ctrlr->hmb_chunk, nvme_single_map,  in nvme_ctrlr_hmb_alloc()
975 			bus_dmamem_free(ctrlr->hmb_tag, hmbc->hmbc_vaddr,  in nvme_ctrlr_hmb_alloc()
977 			nvme_printf(ctrlr, "failed to load HMB\n");  in nvme_ctrlr_hmb_alloc()
980 		bus_dmamap_sync(ctrlr->hmb_tag, hmbc->hmbc_map,  in nvme_ctrlr_hmb_alloc()
984 	if (i < ctrlr->hmb_nchunks && i * ctrlr->hmb_chunk < min &&  in nvme_ctrlr_hmb_alloc()
985 	    ctrlr->hmb_chunk / 2 >= minc) {  in nvme_ctrlr_hmb_alloc()
986 		ctrlr->hmb_nchunks = i;  in nvme_ctrlr_hmb_alloc()
987 		nvme_ctrlr_hmb_free(ctrlr);  in nvme_ctrlr_hmb_alloc()
988 		ctrlr->hmb_chunk /= 2;  in nvme_ctrlr_hmb_alloc()
991 	ctrlr->hmb_nchunks = i;  in nvme_ctrlr_hmb_alloc()
992 	if (ctrlr->hmb_nchunks * ctrlr->hmb_chunk < min) {  in nvme_ctrlr_hmb_alloc()
993 		nvme_ctrlr_hmb_free(ctrlr);  in nvme_ctrlr_hmb_alloc()
997 	size = sizeof(struct nvme_hmb_desc) * ctrlr->hmb_nchunks;  in nvme_ctrlr_hmb_alloc()
998 	err = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev),  in nvme_ctrlr_hmb_alloc()
1000 	    size, 1, size, 0, NULL, NULL, &ctrlr->hmb_desc_tag);  in nvme_ctrlr_hmb_alloc()
1002 		nvme_printf(ctrlr, "HMB desc tag create failed %d\n", err);  in nvme_ctrlr_hmb_alloc()
1003 		nvme_ctrlr_hmb_free(ctrlr);  in nvme_ctrlr_hmb_alloc()
1006 	if (bus_dmamem_alloc(ctrlr->hmb_desc_tag,  in nvme_ctrlr_hmb_alloc()
1007 	    (void **)&ctrlr->hmb_desc_vaddr, BUS_DMA_WAITOK,  in nvme_ctrlr_hmb_alloc()
1008 	    &ctrlr->hmb_desc_map)) {  in nvme_ctrlr_hmb_alloc()
1009 		nvme_printf(ctrlr, "failed to alloc HMB desc\n");  in nvme_ctrlr_hmb_alloc()
1010 		nvme_ctrlr_hmb_free(ctrlr);  in nvme_ctrlr_hmb_alloc()
1013 	if (bus_dmamap_load(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_map,  in nvme_ctrlr_hmb_alloc()
1014 	    ctrlr->hmb_desc_vaddr, size, nvme_single_map,  in nvme_ctrlr_hmb_alloc()
1015 	    &ctrlr->hmb_desc_paddr, BUS_DMA_NOWAIT) != 0) {  in nvme_ctrlr_hmb_alloc()
1016 		bus_dmamem_free(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_vaddr,  in nvme_ctrlr_hmb_alloc()
1017 		    ctrlr->hmb_desc_map);  in nvme_ctrlr_hmb_alloc()
1018 		nvme_printf(ctrlr, "failed to load HMB desc\n");  in nvme_ctrlr_hmb_alloc()
1019 		nvme_ctrlr_hmb_free(ctrlr);  in nvme_ctrlr_hmb_alloc()
1023 	for (i = 0; i < ctrlr->hmb_nchunks; i++) {  in nvme_ctrlr_hmb_alloc()
1024 		ctrlr->hmb_desc_vaddr[i].addr =  in nvme_ctrlr_hmb_alloc()
1025 		    htole64(ctrlr->hmb_chunks[i].hmbc_paddr);  in nvme_ctrlr_hmb_alloc()
1026 		ctrlr->hmb_desc_vaddr[i].size = htole32(ctrlr->hmb_chunk / 4096);  in nvme_ctrlr_hmb_alloc()
1028 	bus_dmamap_sync(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_map,  in nvme_ctrlr_hmb_alloc()
1031 	nvme_printf(ctrlr, "Allocated %lluMB host memory buffer\n",  in nvme_ctrlr_hmb_alloc()
1032 	    (long long unsigned)ctrlr->hmb_nchunks * ctrlr->hmb_chunk  in nvme_ctrlr_hmb_alloc()
1037 nvme_ctrlr_hmb_enable(struct nvme_controller *ctrlr, bool enable, bool memret)  in nvme_ctrlr_hmb_enable()  argument
1048 	nvme_ctrlr_cmd_set_feature(ctrlr, NVME_FEAT_HOST_MEMORY_BUFFER, cdw11,  in nvme_ctrlr_hmb_enable()
1049 	    ctrlr->hmb_nchunks * ctrlr->hmb_chunk / 4096, ctrlr->hmb_desc_paddr,  in nvme_ctrlr_hmb_enable()
1050 	    ctrlr->hmb_desc_paddr >> 32, ctrlr->hmb_nchunks, NULL, 0,  in nvme_ctrlr_hmb_enable()
1054 		nvme_printf(ctrlr, "nvme_ctrlr_hmb_enable failed!\n");  in nvme_ctrlr_hmb_enable()
1060 	struct nvme_controller *ctrlr = ctrlr_arg;  in nvme_ctrlr_start()  local
1074 		nvme_qpair_reset(&ctrlr->adminq);  in nvme_ctrlr_start()
1075 		nvme_admin_qpair_enable(&ctrlr->adminq);  in nvme_ctrlr_start()
1078 	if (ctrlr->ioq != NULL) {  in nvme_ctrlr_start()
1079 		for (i = 0; i < ctrlr->num_io_queues; i++)  in nvme_ctrlr_start()
1080 			nvme_qpair_reset(&ctrlr->ioq[i]);  in nvme_ctrlr_start()
1087 	if (resetting && !ctrlr->is_initialized)  in nvme_ctrlr_start()
1090 	if (resetting && nvme_ctrlr_identify(ctrlr) != 0) {  in nvme_ctrlr_start()
1091 		nvme_ctrlr_fail(ctrlr);  in nvme_ctrlr_start()
1104 		old_num_io_queues = ctrlr->num_io_queues;  in nvme_ctrlr_start()
1105 		if (nvme_ctrlr_set_num_qpairs(ctrlr) != 0) {  in nvme_ctrlr_start()
1106 			nvme_ctrlr_fail(ctrlr);  in nvme_ctrlr_start()
1110 		if (old_num_io_queues != ctrlr->num_io_queues) {  in nvme_ctrlr_start()
1112 			      old_num_io_queues, ctrlr->num_io_queues);  in nvme_ctrlr_start()
1116 	if (ctrlr->cdata.hmpre > 0 && ctrlr->hmb_nchunks == 0) {  in nvme_ctrlr_start()
1117 		nvme_ctrlr_hmb_alloc(ctrlr);  in nvme_ctrlr_start()
1118 		if (ctrlr->hmb_nchunks > 0)  in nvme_ctrlr_start()
1119 			nvme_ctrlr_hmb_enable(ctrlr, true, false);  in nvme_ctrlr_start()
1120 	} else if (ctrlr->hmb_nchunks > 0)  in nvme_ctrlr_start()
1121 		nvme_ctrlr_hmb_enable(ctrlr, true, true);  in nvme_ctrlr_start()
1123 	if (nvme_ctrlr_create_qpairs(ctrlr) != 0) {  in nvme_ctrlr_start()
1124 		nvme_ctrlr_fail(ctrlr);  in nvme_ctrlr_start()
1128 	if (nvme_ctrlr_construct_namespaces(ctrlr) != 0) {  in nvme_ctrlr_start()
1129 		nvme_ctrlr_fail(ctrlr);  in nvme_ctrlr_start()
1133 	nvme_ctrlr_configure_aer(ctrlr);  in nvme_ctrlr_start()
1134 	nvme_ctrlr_configure_int_coalescing(ctrlr);  in nvme_ctrlr_start()
1136 	for (i = 0; i < ctrlr->num_io_queues; i++)  in nvme_ctrlr_start()
1137 		nvme_io_qpair_enable(&ctrlr->ioq[i]);  in nvme_ctrlr_start()
1144 	struct nvme_controller *ctrlr = arg;  in nvme_ctrlr_start_config_hook()  local
1148 	if (nvme_ctrlr_hw_reset(ctrlr) != 0) {  in nvme_ctrlr_start_config_hook()
1150 		nvme_ctrlr_fail(ctrlr);  in nvme_ctrlr_start_config_hook()
1151 		config_intrhook_disestablish(&ctrlr->config_hook);  in nvme_ctrlr_start_config_hook()
1163 	if (nvme_ctrlr_hw_reset(ctrlr) != 0)  in nvme_ctrlr_start_config_hook()
1167 	nvme_qpair_reset(&ctrlr->adminq);  in nvme_ctrlr_start_config_hook()
1168 	nvme_admin_qpair_enable(&ctrlr->adminq);  in nvme_ctrlr_start_config_hook()
1170 	if (nvme_ctrlr_identify(ctrlr) == 0 &&  in nvme_ctrlr_start_config_hook()
1171 	    nvme_ctrlr_set_num_qpairs(ctrlr) == 0 &&  in nvme_ctrlr_start_config_hook()
1172 	    nvme_ctrlr_construct_io_qpairs(ctrlr) == 0)  in nvme_ctrlr_start_config_hook()
1173 		nvme_ctrlr_start(ctrlr, false);  in nvme_ctrlr_start_config_hook()
1177 	nvme_sysctl_initialize_ctrlr(ctrlr);  in nvme_ctrlr_start_config_hook()
1178 	config_intrhook_disestablish(&ctrlr->config_hook);  in nvme_ctrlr_start_config_hook()
1180 	ctrlr->is_initialized = 1;  in nvme_ctrlr_start_config_hook()
1181 	nvme_notify_new_controller(ctrlr);  in nvme_ctrlr_start_config_hook()
1188 	struct nvme_controller	*ctrlr = arg;  in nvme_ctrlr_reset_task()  local
1191 	nvme_ctrlr_devctl_log(ctrlr, "RESET", "resetting controller");  in nvme_ctrlr_reset_task()
1192 	status = nvme_ctrlr_hw_reset(ctrlr);  in nvme_ctrlr_reset_task()
1203 		nvme_ctrlr_start(ctrlr, true);  in nvme_ctrlr_reset_task()
1205 		nvme_ctrlr_fail(ctrlr);  in nvme_ctrlr_reset_task()
1207 	atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);  in nvme_ctrlr_reset_task()
1214 nvme_ctrlr_poll(struct nvme_controller *ctrlr)  in nvme_ctrlr_poll()  argument
1218 	nvme_qpair_process_completions(&ctrlr->adminq);  in nvme_ctrlr_poll()
1220 	for (i = 0; i < ctrlr->num_io_queues; i++)  in nvme_ctrlr_poll()
1221 		if (ctrlr->ioq && ctrlr->ioq[i].cpl)  in nvme_ctrlr_poll()
1222 			nvme_qpair_process_completions(&ctrlr->ioq[i]);  in nvme_ctrlr_poll()
1233 	struct nvme_controller *ctrlr = arg;  in nvme_ctrlr_shared_handler()  local
1235 	nvme_mmio_write_4(ctrlr, intms, 1);  in nvme_ctrlr_shared_handler()
1236 	nvme_ctrlr_poll(ctrlr);  in nvme_ctrlr_shared_handler()
1237 	nvme_mmio_write_4(ctrlr, intmc, 1);  in nvme_ctrlr_shared_handler()
1261 nvme_ctrlr_passthrough_cmd(struct nvme_controller *ctrlr,  in nvme_ctrlr_passthrough_cmd()  argument
1271 		if (pt->len > ctrlr->max_xfer_size) {  in nvme_ctrlr_passthrough_cmd()
1272 			nvme_printf(ctrlr, "pt->len (%d) "  in nvme_ctrlr_passthrough_cmd()
1274 			    ctrlr->max_xfer_size);  in nvme_ctrlr_passthrough_cmd()
1315 		nvme_ctrlr_submit_admin_request(ctrlr, req);  in nvme_ctrlr_passthrough_cmd()
1317 		nvme_ctrlr_submit_io_request(ctrlr, req);  in nvme_ctrlr_passthrough_cmd()
1337 	struct nvme_controller			*ctrlr;  in nvme_ctrlr_ioctl()  local
1340 	ctrlr = cdev->si_drv1;  in nvme_ctrlr_ioctl()
1344 		nvme_ctrlr_reset(ctrlr);  in nvme_ctrlr_ioctl()
1348 		return (nvme_ctrlr_passthrough_cmd(ctrlr, pt, le32toh(pt->cmd.nsid),  in nvme_ctrlr_ioctl()
1353 		strncpy(gnsid->cdev, device_get_nameunit(ctrlr->dev),  in nvme_ctrlr_ioctl()
1360 		*(uint64_t *)arg = ctrlr->max_xfer_size;  in nvme_ctrlr_ioctl()
1376 nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev)  in nvme_ctrlr_construct()  argument
1385 	ctrlr->dev = dev;  in nvme_ctrlr_construct()
1387 	mtx_init(&ctrlr->lock, "nvme ctrlr lock", NULL, MTX_DEF);  in nvme_ctrlr_construct()
1388 	if (bus_get_domain(dev, &ctrlr->domain) != 0)  in nvme_ctrlr_construct()
1389 		ctrlr->domain = 0;  in nvme_ctrlr_construct()
1391 	cap_lo = nvme_mmio_read_4(ctrlr, cap_lo);  in nvme_ctrlr_construct()
1401 	cap_hi = nvme_mmio_read_4(ctrlr, cap_hi);  in nvme_ctrlr_construct()
1415 		vs = nvme_mmio_read_4(ctrlr, vs);  in nvme_ctrlr_construct()
1420 		pmrcap = nvme_mmio_read_4(ctrlr, pmrcap);  in nvme_ctrlr_construct()
1432 	ctrlr->dstrd = NVME_CAP_HI_DSTRD(cap_hi) + 2;  in nvme_ctrlr_construct()
1435 	ctrlr->min_page_size = 1 << (12 + mpsmin);  in nvme_ctrlr_construct()
1439 	ctrlr->ready_timeout_in_ms = to * 500;  in nvme_ctrlr_construct()
1445 	ctrlr->timeout_period = timeout_period;  in nvme_ctrlr_construct()
1450 	ctrlr->enable_aborts = 0;  in nvme_ctrlr_construct()
1451 	TUNABLE_INT_FETCH("hw.nvme.enable_aborts", &ctrlr->enable_aborts);  in nvme_ctrlr_construct()
1453 	ctrlr->max_xfer_size = NVME_MAX_XFER_SIZE;  in nvme_ctrlr_construct()
1454 	if (nvme_ctrlr_construct_admin_qpair(ctrlr) != 0)  in nvme_ctrlr_construct()
1468 	ctrlr->taskqueue = taskqueue_create("nvme_taskq", M_WAITOK,  in nvme_ctrlr_construct()
1469 	    taskqueue_thread_enqueue, &ctrlr->taskqueue);  in nvme_ctrlr_construct()
1470 	taskqueue_start_threads(&ctrlr->taskqueue, 2, PI_DISK, "nvme taskq");  in nvme_ctrlr_construct()
1472 	ctrlr->is_resetting = 0;  in nvme_ctrlr_construct()
1473 	ctrlr->is_initialized = 0;  in nvme_ctrlr_construct()
1474 	ctrlr->notification_sent = 0;  in nvme_ctrlr_construct()
1475 	TASK_INIT(&ctrlr->reset_task, 0, nvme_ctrlr_reset_task, ctrlr);  in nvme_ctrlr_construct()
1476 	TASK_INIT(&ctrlr->fail_req_task, 0, nvme_ctrlr_fail_req_task, ctrlr);  in nvme_ctrlr_construct()
1477 	STAILQ_INIT(&ctrlr->fail_req);  in nvme_ctrlr_construct()
1478 	ctrlr->is_failed = false;  in nvme_ctrlr_construct()
1486 	md_args.mda_si_drv1 = (void *)ctrlr;  in nvme_ctrlr_construct()
1487 	status = make_dev_s(&md_args, &ctrlr->cdev, "nvme%d",  in nvme_ctrlr_construct()
1496 nvme_ctrlr_destruct(struct nvme_controller *ctrlr, device_t dev)  in nvme_ctrlr_destruct()  argument
1500 	ctrlr->is_dying = true;  in nvme_ctrlr_destruct()
1502 	if (ctrlr->resource == NULL)  in nvme_ctrlr_destruct()
1504 	if (!mtx_initialized(&ctrlr->adminq.lock))  in nvme_ctrlr_destruct()
1511 	gone = (nvme_mmio_read_4(ctrlr, csts) == NVME_GONE);  in nvme_ctrlr_destruct()
1513 		nvme_ctrlr_fail(ctrlr);  in nvme_ctrlr_destruct()
1515 		nvme_notify_fail_consumers(ctrlr);  in nvme_ctrlr_destruct()
1518 		nvme_ns_destruct(&ctrlr->ns[i]);  in nvme_ctrlr_destruct()
1520 	if (ctrlr->cdev)  in nvme_ctrlr_destruct()
1521 		destroy_dev(ctrlr->cdev);  in nvme_ctrlr_destruct()
1523 	if (ctrlr->is_initialized) {  in nvme_ctrlr_destruct()
1525 			if (ctrlr->hmb_nchunks > 0)  in nvme_ctrlr_destruct()
1526 				nvme_ctrlr_hmb_enable(ctrlr, false, false);  in nvme_ctrlr_destruct()
1527 			nvme_ctrlr_delete_qpairs(ctrlr);  in nvme_ctrlr_destruct()
1529 		nvme_ctrlr_hmb_free(ctrlr);  in nvme_ctrlr_destruct()
1531 	if (ctrlr->ioq != NULL) {  in nvme_ctrlr_destruct()
1532 		for (i = 0; i < ctrlr->num_io_queues; i++)  in nvme_ctrlr_destruct()
1533 			nvme_io_qpair_destroy(&ctrlr->ioq[i]);  in nvme_ctrlr_destruct()
1534 		free(ctrlr->ioq, M_NVME);  in nvme_ctrlr_destruct()
1536 	nvme_admin_qpair_destroy(&ctrlr->adminq);  in nvme_ctrlr_destruct()
1546 		nvme_ctrlr_shutdown(ctrlr);  in nvme_ctrlr_destruct()
1549 		nvme_ctrlr_disable(ctrlr);  in nvme_ctrlr_destruct()
1552 	if (ctrlr->taskqueue)  in nvme_ctrlr_destruct()
1553 		taskqueue_free(ctrlr->taskqueue);  in nvme_ctrlr_destruct()
1555 	if (ctrlr->tag)  in nvme_ctrlr_destruct()
1556 		bus_teardown_intr(ctrlr->dev, ctrlr->res, ctrlr->tag);  in nvme_ctrlr_destruct()
1558 	if (ctrlr->res)  in nvme_ctrlr_destruct()
1559 		bus_release_resource(ctrlr->dev, SYS_RES_IRQ,  in nvme_ctrlr_destruct()
1560 		    rman_get_rid(ctrlr->res), ctrlr->res);  in nvme_ctrlr_destruct()
1562 	if (ctrlr->bar4_resource != NULL) {  in nvme_ctrlr_destruct()
1564 		    ctrlr->bar4_resource_id, ctrlr->bar4_resource);  in nvme_ctrlr_destruct()
1568 	    ctrlr->resource_id, ctrlr->resource);  in nvme_ctrlr_destruct()
1571 	mtx_destroy(&ctrlr->lock);  in nvme_ctrlr_destruct()
1575 nvme_ctrlr_shutdown(struct nvme_controller *ctrlr)  in nvme_ctrlr_shutdown()  argument
1581 	cc = nvme_mmio_read_4(ctrlr, cc);  in nvme_ctrlr_shutdown()
1584 	nvme_mmio_write_4(ctrlr, cc, cc);  in nvme_ctrlr_shutdown()
1586 	timeout = ticks + (ctrlr->cdata.rtd3e == 0 ? 5 * hz :  in nvme_ctrlr_shutdown()
1587 	    ((uint64_t)ctrlr->cdata.rtd3e * hz + 999999) / 1000000);  in nvme_ctrlr_shutdown()
1589 		csts = nvme_mmio_read_4(ctrlr, csts);  in nvme_ctrlr_shutdown()
1595 			nvme_printf(ctrlr, "shutdown timeout\n");  in nvme_ctrlr_shutdown()
1603 nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr,  in nvme_ctrlr_submit_admin_request()  argument
1607 	nvme_qpair_submit_request(&ctrlr->adminq, req);  in nvme_ctrlr_submit_admin_request()
1611 nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr,  in nvme_ctrlr_submit_io_request()  argument
1616 	qpair = &ctrlr->ioq[QP(ctrlr, curcpu)];  in nvme_ctrlr_submit_io_request()
1621 nvme_ctrlr_get_device(struct nvme_controller *ctrlr)  in nvme_ctrlr_get_device()  argument
1624 	return (ctrlr->dev);  in nvme_ctrlr_get_device()
1628 nvme_ctrlr_get_data(struct nvme_controller *ctrlr)  in nvme_ctrlr_get_data()  argument
1631 	return (&ctrlr->cdata);  in nvme_ctrlr_get_data()
1635 nvme_ctrlr_suspend(struct nvme_controller *ctrlr)  in nvme_ctrlr_suspend()  argument
1642 	if (ctrlr->is_failed)  in nvme_ctrlr_suspend()
1652 	while (atomic_cmpset_32(&ctrlr->is_resetting, 0, 1) == 0 && to-- > 0)  in nvme_ctrlr_suspend()
1655 		nvme_printf(ctrlr,  in nvme_ctrlr_suspend()
1660 	if (ctrlr->hmb_nchunks > 0)  in nvme_ctrlr_suspend()
1661 		nvme_ctrlr_hmb_enable(ctrlr, false, false);  in nvme_ctrlr_suspend()
1671 	nvme_ctrlr_delete_qpairs(ctrlr);  in nvme_ctrlr_suspend()
1672 	nvme_ctrlr_disable_qpairs(ctrlr);  in nvme_ctrlr_suspend()
1673 	nvme_ctrlr_shutdown(ctrlr);  in nvme_ctrlr_suspend()
1679 nvme_ctrlr_resume(struct nvme_controller *ctrlr)  in nvme_ctrlr_resume()  argument
1685 	if (ctrlr->is_failed)  in nvme_ctrlr_resume()
1688 	if (nvme_ctrlr_hw_reset(ctrlr) != 0)  in nvme_ctrlr_resume()
1696 	if (nvme_ctrlr_hw_reset(ctrlr) != 0)  in nvme_ctrlr_resume()
1705 	nvme_ctrlr_start(ctrlr, true);  in nvme_ctrlr_resume()
1706 	(void)atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);  in nvme_ctrlr_resume()
1715 	nvme_printf(ctrlr, "Failed to reset on resume, failing.\n");  in nvme_ctrlr_resume()
1716 	nvme_ctrlr_fail(ctrlr);  in nvme_ctrlr_resume()
1717 	(void)atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);  in nvme_ctrlr_resume()