51 static void nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
55 nvme_ctrlr_barrier(struct nvme_controller *ctrlr, int flags)  in nvme_ctrlr_barrier()  argument
57 	bus_barrier(ctrlr->resource, 0, rman_get_size(ctrlr->resource), flags);  in nvme_ctrlr_barrier()
61 nvme_ctrlr_devctl_log(struct nvme_controller *ctrlr, const char *type, const char *msg, ...)  in nvme_ctrlr_devctl_log()  argument
69 	sbuf_printf(&sb, "%s: ", device_get_nameunit(ctrlr->dev));  in nvme_ctrlr_devctl_log()
78 	sbuf_printf(&sb, "name=\"%s\" reason=\"", device_get_nameunit(ctrlr->dev));  in nvme_ctrlr_devctl_log()
90 nvme_ctrlr_construct_admin_qpair(struct nvme_controller *ctrlr)  in nvme_ctrlr_construct_admin_qpair()  argument
96 	qpair = &ctrlr->adminq;  in nvme_ctrlr_construct_admin_qpair()
98 	qpair->cpu = CPU_FFS(&cpuset_domain[ctrlr->domain]) - 1;  in nvme_ctrlr_construct_admin_qpair()
99 	qpair->domain = ctrlr->domain;  in nvme_ctrlr_construct_admin_qpair()
109 		nvme_printf(ctrlr, "invalid hw.nvme.admin_entries=%d "  in nvme_ctrlr_construct_admin_qpair()
119 	     ctrlr);  in nvme_ctrlr_construct_admin_qpair()
123 #define QP(ctrlr, c)	((c) * (ctrlr)->num_io_queues / mp_ncpus)  argument
126 nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr)  in nvme_ctrlr_construct_io_qpairs()  argument
143 	    (rman_get_size(ctrlr->resource) - nvme_mmio_offsetof(doorbell[0])) /  in nvme_ctrlr_construct_io_qpairs()
144 	    (1 << (ctrlr->dstrd + 1));  in nvme_ctrlr_construct_io_qpairs()
147 	cap_lo = nvme_mmio_read_4(ctrlr, cap_lo);  in nvme_ctrlr_construct_io_qpairs()
170 	ctrlr->max_hw_pend_io = num_trackers * ctrlr->num_io_queues * 3 / 4;  in nvme_ctrlr_construct_io_qpairs()
172 	ctrlr->ioq = malloc(ctrlr->num_io_queues * sizeof(struct nvme_qpair),  in nvme_ctrlr_construct_io_qpairs()
175 	for (i = c = n = 0; i < ctrlr->num_io_queues; i++, c += n) {  in nvme_ctrlr_construct_io_qpairs()
176 		qpair = &ctrlr->ioq[i];  in nvme_ctrlr_construct_io_qpairs()
183 		if (ctrlr->num_io_queues > 1) {  in nvme_ctrlr_construct_io_qpairs()
185 			for (n = 1; QP(ctrlr, c + n) == i; n++)  in nvme_ctrlr_construct_io_qpairs()
188 			qpair->cpu = c + (device_get_unit(ctrlr->dev)+n/2) % n;  in nvme_ctrlr_construct_io_qpairs()
191 			qpair->cpu = CPU_FFS(&cpuset_domain[ctrlr->domain]) - 1;  in nvme_ctrlr_construct_io_qpairs()
192 			qpair->domain = ctrlr->domain;  in nvme_ctrlr_construct_io_qpairs()
200 		    ctrlr);  in nvme_ctrlr_construct_io_qpairs()
208 		if (ctrlr->num_io_queues > 1)  in nvme_ctrlr_construct_io_qpairs()
209 			bus_bind_intr(ctrlr->dev, qpair->res, qpair->cpu);  in nvme_ctrlr_construct_io_qpairs()
216 nvme_ctrlr_fail(struct nvme_controller *ctrlr)  in nvme_ctrlr_fail()  argument
225 	ctrlr->is_failed = true;  in nvme_ctrlr_fail()
226 	nvme_qpair_fail(&ctrlr->adminq);  in nvme_ctrlr_fail()
227 	if (ctrlr->ioq != NULL) {  in nvme_ctrlr_fail()
228 		for (i = 0; i < ctrlr->num_io_queues; i++) {  in nvme_ctrlr_fail()
229 			nvme_qpair_fail(&ctrlr->ioq[i]);  in nvme_ctrlr_fail()
232 	nvme_notify_fail_consumers(ctrlr);  in nvme_ctrlr_fail()
236 nvme_ctrlr_post_failed_request(struct nvme_controller *ctrlr,  in nvme_ctrlr_post_failed_request()  argument
240 	mtx_lock(&ctrlr->lock);  in nvme_ctrlr_post_failed_request()
241 	STAILQ_INSERT_TAIL(&ctrlr->fail_req, req, stailq);  in nvme_ctrlr_post_failed_request()
242 	mtx_unlock(&ctrlr->lock);  in nvme_ctrlr_post_failed_request()
243 	if (!ctrlr->is_dying)  in nvme_ctrlr_post_failed_request()
244 		taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->fail_req_task);  in nvme_ctrlr_post_failed_request()
250 	struct nvme_controller	*ctrlr = arg;  in nvme_ctrlr_fail_req_task()  local
253 	mtx_lock(&ctrlr->lock);  in nvme_ctrlr_fail_req_task()
254 	while ((req = STAILQ_FIRST(&ctrlr->fail_req)) != NULL) {  in nvme_ctrlr_fail_req_task()
255 		STAILQ_REMOVE_HEAD(&ctrlr->fail_req, stailq);  in nvme_ctrlr_fail_req_task()
256 		mtx_unlock(&ctrlr->lock);  in nvme_ctrlr_fail_req_task()
259 		mtx_lock(&ctrlr->lock);  in nvme_ctrlr_fail_req_task()
261 	mtx_unlock(&ctrlr->lock);  in nvme_ctrlr_fail_req_task()
271 nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr, int desired_val)  in nvme_ctrlr_wait_for_ready()  argument
273 	int timeout = ticks + MSEC_2_TICKS(ctrlr->ready_timeout_in_ms);  in nvme_ctrlr_wait_for_ready()
278 		csts = nvme_mmio_read_4(ctrlr, csts);  in nvme_ctrlr_wait_for_ready()
284 			nvme_printf(ctrlr, "controller ready did not become %d "  in nvme_ctrlr_wait_for_ready()
285 			    "within %d ms\n", desired_val, ctrlr->ready_timeout_in_ms);  in nvme_ctrlr_wait_for_ready()
297 nvme_ctrlr_disable(struct nvme_controller *ctrlr)  in nvme_ctrlr_disable()  argument
304 	cc = nvme_mmio_read_4(ctrlr, cc);  in nvme_ctrlr_disable()
305 	csts = nvme_mmio_read_4(ctrlr, csts);  in nvme_ctrlr_disable()
320 		return (nvme_ctrlr_wait_for_ready(ctrlr, 0));  in nvme_ctrlr_disable()
324 		err = nvme_ctrlr_wait_for_ready(ctrlr, 1);  in nvme_ctrlr_disable()
330 	nvme_mmio_write_4(ctrlr, cc, cc);  in nvme_ctrlr_disable()
336 	if (ctrlr->quirks & QUIRK_DELAY_B4_CHK_RDY)  in nvme_ctrlr_disable()
338 	return (nvme_ctrlr_wait_for_ready(ctrlr, 0));  in nvme_ctrlr_disable()
342 nvme_ctrlr_enable(struct nvme_controller *ctrlr)  in nvme_ctrlr_enable()  argument
351 	cc = nvme_mmio_read_4(ctrlr, cc);  in nvme_ctrlr_enable()
352 	csts = nvme_mmio_read_4(ctrlr, csts);  in nvme_ctrlr_enable()
363 		return (nvme_ctrlr_wait_for_ready(ctrlr, 1));  in nvme_ctrlr_enable()
367 	err = nvme_ctrlr_wait_for_ready(ctrlr, 0);  in nvme_ctrlr_enable()
371 	nvme_mmio_write_8(ctrlr, asq, ctrlr->adminq.cmd_bus_addr);  in nvme_ctrlr_enable()
372 	nvme_mmio_write_8(ctrlr, acq, ctrlr->adminq.cpl_bus_addr);  in nvme_ctrlr_enable()
375 	qsize = ctrlr->adminq.num_entries - 1;  in nvme_ctrlr_enable()
380 	nvme_mmio_write_4(ctrlr, aqa, aqa);  in nvme_ctrlr_enable()
396 	cc |= NVMEF(NVME_CC_REG_MPS, ctrlr->mps);  in nvme_ctrlr_enable()
398 	nvme_ctrlr_barrier(ctrlr, BUS_SPACE_BARRIER_WRITE);  in nvme_ctrlr_enable()
399 	nvme_mmio_write_4(ctrlr, cc, cc);  in nvme_ctrlr_enable()
401 	return (nvme_ctrlr_wait_for_ready(ctrlr, 1));  in nvme_ctrlr_enable()
405 nvme_ctrlr_disable_qpairs(struct nvme_controller *ctrlr)  in nvme_ctrlr_disable_qpairs()  argument
409 	nvme_admin_qpair_disable(&ctrlr->adminq);  in nvme_ctrlr_disable_qpairs()
415 	if (ctrlr->is_initialized) {  in nvme_ctrlr_disable_qpairs()
416 		for (i = 0; i < ctrlr->num_io_queues; i++)  in nvme_ctrlr_disable_qpairs()
417 			nvme_io_qpair_disable(&ctrlr->ioq[i]);  in nvme_ctrlr_disable_qpairs()
422 nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr)  in nvme_ctrlr_hw_reset()  argument
428 	nvme_ctrlr_disable_qpairs(ctrlr);  in nvme_ctrlr_hw_reset()
430 	err = nvme_ctrlr_disable(ctrlr);  in nvme_ctrlr_hw_reset()
434 	err = nvme_ctrlr_enable(ctrlr);  in nvme_ctrlr_hw_reset()
442 nvme_ctrlr_reset(struct nvme_controller *ctrlr)  in nvme_ctrlr_reset()  argument
446 	cmpset = atomic_cmpset_32(&ctrlr->is_resetting, 0, 1);  in nvme_ctrlr_reset()
448 	if (cmpset == 0 || ctrlr->is_failed)  in nvme_ctrlr_reset()
456 	if (!ctrlr->is_dying)  in nvme_ctrlr_reset()
457 		taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->reset_task);  in nvme_ctrlr_reset()
461 nvme_ctrlr_identify(struct nvme_controller *ctrlr)  in nvme_ctrlr_identify()  argument
466 	nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata,  in nvme_ctrlr_identify()
470 		nvme_printf(ctrlr, "nvme_identify_controller failed!\n");  in nvme_ctrlr_identify()
475 	nvme_controller_data_swapbytes(&ctrlr->cdata);  in nvme_ctrlr_identify()
481 	if (ctrlr->cdata.mdts > 0)  in nvme_ctrlr_identify()
482 		ctrlr->max_xfer_size = min(ctrlr->max_xfer_size,  in nvme_ctrlr_identify()
483 		    1 << (ctrlr->cdata.mdts + NVME_MPS_SHIFT +  in nvme_ctrlr_identify()
484 			NVME_CAP_HI_MPSMIN(ctrlr->cap_hi)));  in nvme_ctrlr_identify()
490 nvme_ctrlr_set_num_qpairs(struct nvme_controller *ctrlr)  in nvme_ctrlr_set_num_qpairs()  argument
496 	nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->num_io_queues,  in nvme_ctrlr_set_num_qpairs()
500 		nvme_printf(ctrlr, "nvme_ctrlr_set_num_qpairs failed!\n");  in nvme_ctrlr_set_num_qpairs()
517 	ctrlr->num_io_queues = min(ctrlr->num_io_queues, sq_allocated);  in nvme_ctrlr_set_num_qpairs()
518 	ctrlr->num_io_queues = min(ctrlr->num_io_queues, cq_allocated);  in nvme_ctrlr_set_num_qpairs()
519 	if (ctrlr->num_io_queues > vm_ndomains)  in nvme_ctrlr_set_num_qpairs()
520 		ctrlr->num_io_queues -= ctrlr->num_io_queues % vm_ndomains;  in nvme_ctrlr_set_num_qpairs()
526 nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr)  in nvme_ctrlr_create_qpairs()  argument
532 	for (i = 0; i < ctrlr->num_io_queues; i++) {  in nvme_ctrlr_create_qpairs()
533 		qpair = &ctrlr->ioq[i];  in nvme_ctrlr_create_qpairs()
536 		nvme_ctrlr_cmd_create_io_cq(ctrlr, qpair,  in nvme_ctrlr_create_qpairs()
540 			nvme_printf(ctrlr, "nvme_create_io_cq failed!\n");  in nvme_ctrlr_create_qpairs()
545 		nvme_ctrlr_cmd_create_io_sq(ctrlr, qpair,  in nvme_ctrlr_create_qpairs()
549 			nvme_printf(ctrlr, "nvme_create_io_sq failed!\n");  in nvme_ctrlr_create_qpairs()
558 nvme_ctrlr_delete_qpairs(struct nvme_controller *ctrlr)  in nvme_ctrlr_delete_qpairs()  argument
563 	for (int i = 0; i < ctrlr->num_io_queues; i++) {  in nvme_ctrlr_delete_qpairs()
564 		qpair = &ctrlr->ioq[i];  in nvme_ctrlr_delete_qpairs()
567 		nvme_ctrlr_cmd_delete_io_sq(ctrlr, qpair,  in nvme_ctrlr_delete_qpairs()
571 			nvme_printf(ctrlr, "nvme_destroy_io_sq failed!\n");  in nvme_ctrlr_delete_qpairs()
576 		nvme_ctrlr_cmd_delete_io_cq(ctrlr, qpair,  in nvme_ctrlr_delete_qpairs()
580 			nvme_printf(ctrlr, "nvme_destroy_io_cq failed!\n");  in nvme_ctrlr_delete_qpairs()
589 nvme_ctrlr_construct_namespaces(struct nvme_controller *ctrlr)  in nvme_ctrlr_construct_namespaces()  argument
594 	for (i = 0; i < min(ctrlr->cdata.nn, NVME_MAX_NAMESPACES); i++) {  in nvme_ctrlr_construct_namespaces()
595 		ns = &ctrlr->ns[i];  in nvme_ctrlr_construct_namespaces()
596 		nvme_ns_construct(ns, i+1, ctrlr);  in nvme_ctrlr_construct_namespaces()
621 nvme_ctrlr_get_log_page_size(struct nvme_controller *ctrlr, uint8_t page_id)  in nvme_ctrlr_get_log_page_size()  argument
629 		    (ctrlr->cdata.elpe + 1), NVME_MAX_AER_LOG_SIZE);  in nvme_ctrlr_get_log_page_size()
658 nvme_ctrlr_log_critical_warnings(struct nvme_controller *ctrlr,  in nvme_ctrlr_log_critical_warnings()  argument
663 		nvme_ctrlr_devctl_log(ctrlr, "critical",  in nvme_ctrlr_log_critical_warnings()
667 		nvme_ctrlr_devctl_log(ctrlr, "critical",  in nvme_ctrlr_log_critical_warnings()
671 		nvme_ctrlr_devctl_log(ctrlr, "critical",  in nvme_ctrlr_log_critical_warnings()
675 		nvme_ctrlr_devctl_log(ctrlr, "critical",  in nvme_ctrlr_log_critical_warnings()
679 		nvme_ctrlr_devctl_log(ctrlr, "critical",  in nvme_ctrlr_log_critical_warnings()
683 		nvme_ctrlr_devctl_log(ctrlr, "critical",  in nvme_ctrlr_log_critical_warnings()
702 		nvme_notify_async_consumers(aer->ctrlr, &aer->cpl,  in nvme_ctrlr_async_event_log_page_cb()
709 			for (i = 0; i < (aer->ctrlr->cdata.elpe + 1); i++)  in nvme_ctrlr_async_event_log_page_cb()
739 			nvme_ctrlr_log_critical_warnings(aer->ctrlr,  in nvme_ctrlr_async_event_log_page_cb()
748 			aer->ctrlr->async_event_config &=  in nvme_ctrlr_async_event_log_page_cb()
750 			nvme_ctrlr_cmd_set_async_event_config(aer->ctrlr,  in nvme_ctrlr_async_event_log_page_cb()
751 			    aer->ctrlr->async_event_config, NULL, NULL);  in nvme_ctrlr_async_event_log_page_cb()
758 				nvme_notify_ns(aer->ctrlr, nsl->ns[i]);  in nvme_ctrlr_async_event_log_page_cb()
766 		nvme_notify_async_consumers(aer->ctrlr, &aer->cpl,  in nvme_ctrlr_async_event_log_page_cb()
774 	nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer);  in nvme_ctrlr_async_event_log_page_cb()
795 	nvme_printf(aer->ctrlr, "async event occurred (type 0x%x, info 0x%02x,"  in nvme_ctrlr_async_event_cb()
801 		aer->log_page_size = nvme_ctrlr_get_log_page_size(aer->ctrlr,  in nvme_ctrlr_async_event_cb()
804 		nvme_ctrlr_cmd_get_log_page(aer->ctrlr, aer->log_page_id,  in nvme_ctrlr_async_event_cb()
810 		nvme_notify_async_consumers(aer->ctrlr, cpl, aer->log_page_id,  in nvme_ctrlr_async_event_cb()
817 		nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer);  in nvme_ctrlr_async_event_cb()
822 nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,  in nvme_ctrlr_construct_and_submit_aer()  argument
827 	aer->ctrlr = ctrlr;  in nvme_ctrlr_construct_and_submit_aer()
837 	nvme_ctrlr_submit_admin_request(ctrlr, req);  in nvme_ctrlr_construct_and_submit_aer()
841 nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr)  in nvme_ctrlr_configure_aer()  argument
847 	ctrlr->async_event_config = NVME_CRIT_WARN_ST_AVAILABLE_SPARE |  in nvme_ctrlr_configure_aer()
851 	if (ctrlr->cdata.ver >= NVME_REV(1, 2))  in nvme_ctrlr_configure_aer()
852 		ctrlr->async_event_config |= NVME_ASYNC_EVENT_NS_ATTRIBUTE |  in nvme_ctrlr_configure_aer()
856 	nvme_ctrlr_cmd_get_feature(ctrlr, NVME_FEAT_TEMPERATURE_THRESHOLD,  in nvme_ctrlr_configure_aer()
862 		nvme_printf(ctrlr, "temperature threshold not supported\n");  in nvme_ctrlr_configure_aer()
864 		ctrlr->async_event_config |= NVME_CRIT_WARN_ST_TEMPERATURE;  in nvme_ctrlr_configure_aer()
866 	nvme_ctrlr_cmd_set_async_event_config(ctrlr,  in nvme_ctrlr_configure_aer()
867 	    ctrlr->async_event_config, NULL, NULL);  in nvme_ctrlr_configure_aer()
870 	ctrlr->num_aers = min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl+1));  in nvme_ctrlr_configure_aer()
872 	for (i = 0; i < ctrlr->num_aers; i++) {  in nvme_ctrlr_configure_aer()
873 		aer = &ctrlr->aer[i];  in nvme_ctrlr_configure_aer()
874 		nvme_ctrlr_construct_and_submit_aer(ctrlr, aer);  in nvme_ctrlr_configure_aer()
879 nvme_ctrlr_configure_int_coalescing(struct nvme_controller *ctrlr)  in nvme_ctrlr_configure_int_coalescing()  argument
882 	ctrlr->int_coal_time = 0;  in nvme_ctrlr_configure_int_coalescing()
884 	    &ctrlr->int_coal_time);  in nvme_ctrlr_configure_int_coalescing()
886 	ctrlr->int_coal_threshold = 0;  in nvme_ctrlr_configure_int_coalescing()
888 	    &ctrlr->int_coal_threshold);  in nvme_ctrlr_configure_int_coalescing()
890 	nvme_ctrlr_cmd_set_interrupt_coalescing(ctrlr, ctrlr->int_coal_time,  in nvme_ctrlr_configure_int_coalescing()
891 	    ctrlr->int_coal_threshold, NULL, NULL);  in nvme_ctrlr_configure_int_coalescing()
895 nvme_ctrlr_hmb_free(struct nvme_controller *ctrlr)  in nvme_ctrlr_hmb_free()  argument
900 	if (ctrlr->hmb_desc_paddr) {  in nvme_ctrlr_hmb_free()
901 		bus_dmamap_unload(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_map);  in nvme_ctrlr_hmb_free()
902 		bus_dmamem_free(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_vaddr,  in nvme_ctrlr_hmb_free()
903 		    ctrlr->hmb_desc_map);  in nvme_ctrlr_hmb_free()
904 		ctrlr->hmb_desc_paddr = 0;  in nvme_ctrlr_hmb_free()
906 	if (ctrlr->hmb_desc_tag) {  in nvme_ctrlr_hmb_free()
907 		bus_dma_tag_destroy(ctrlr->hmb_desc_tag);  in nvme_ctrlr_hmb_free()
908 		ctrlr->hmb_desc_tag = NULL;  in nvme_ctrlr_hmb_free()
910 	for (i = 0; i < ctrlr->hmb_nchunks; i++) {  in nvme_ctrlr_hmb_free()
911 		hmbc = &ctrlr->hmb_chunks[i];  in nvme_ctrlr_hmb_free()
912 		bus_dmamap_unload(ctrlr->hmb_tag, hmbc->hmbc_map);  in nvme_ctrlr_hmb_free()
913 		bus_dmamem_free(ctrlr->hmb_tag, hmbc->hmbc_vaddr,  in nvme_ctrlr_hmb_free()
916 	ctrlr->hmb_nchunks = 0;  in nvme_ctrlr_hmb_free()
917 	if (ctrlr->hmb_tag) {  in nvme_ctrlr_hmb_free()
918 		bus_dma_tag_destroy(ctrlr->hmb_tag);  in nvme_ctrlr_hmb_free()
919 		ctrlr->hmb_tag = NULL;  in nvme_ctrlr_hmb_free()
921 	if (ctrlr->hmb_chunks) {  in nvme_ctrlr_hmb_free()
922 		free(ctrlr->hmb_chunks, M_NVME);  in nvme_ctrlr_hmb_free()
923 		ctrlr->hmb_chunks = NULL;  in nvme_ctrlr_hmb_free()
928 nvme_ctrlr_hmb_alloc(struct nvme_controller *ctrlr)  in nvme_ctrlr_hmb_alloc()  argument
943 	min = (long long unsigned)ctrlr->cdata.hmmin * NVME_HMB_UNITS;  in nvme_ctrlr_hmb_alloc()
946 	pref = MIN((long long unsigned)ctrlr->cdata.hmpre * NVME_HMB_UNITS, max);  in nvme_ctrlr_hmb_alloc()
947 	minc = MAX(ctrlr->cdata.hmminds * NVME_HMB_UNITS, ctrlr->page_size);  in nvme_ctrlr_hmb_alloc()
948 	if (min > 0 && ctrlr->cdata.hmmaxd > 0)  in nvme_ctrlr_hmb_alloc()
949 		minc = MAX(minc, min / ctrlr->cdata.hmmaxd);  in nvme_ctrlr_hmb_alloc()
950 	ctrlr->hmb_chunk = pref;  in nvme_ctrlr_hmb_alloc()
957 	ctrlr->hmb_chunk = roundup2(ctrlr->hmb_chunk, ctrlr->page_size);  in nvme_ctrlr_hmb_alloc()
958 	ctrlr->hmb_nchunks = howmany(pref, ctrlr->hmb_chunk);  in nvme_ctrlr_hmb_alloc()
959 	if (ctrlr->cdata.hmmaxd > 0 && ctrlr->hmb_nchunks > ctrlr->cdata.hmmaxd)  in nvme_ctrlr_hmb_alloc()
960 		ctrlr->hmb_nchunks = ctrlr->cdata.hmmaxd;  in nvme_ctrlr_hmb_alloc()
961 	ctrlr->hmb_chunks = malloc(sizeof(struct nvme_hmb_chunk) *  in nvme_ctrlr_hmb_alloc()
962 	    ctrlr->hmb_nchunks, M_NVME, M_WAITOK);  in nvme_ctrlr_hmb_alloc()
963 	err = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev),  in nvme_ctrlr_hmb_alloc()
964 	    ctrlr->page_size, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,  in nvme_ctrlr_hmb_alloc()
965 	    ctrlr->hmb_chunk, 1, ctrlr->hmb_chunk, 0, NULL, NULL, &ctrlr->hmb_tag);  in nvme_ctrlr_hmb_alloc()
967 		nvme_printf(ctrlr, "HMB tag create failed %d\n", err);  in nvme_ctrlr_hmb_alloc()
968 		nvme_ctrlr_hmb_free(ctrlr);  in nvme_ctrlr_hmb_alloc()
972 	for (i = 0; i < ctrlr->hmb_nchunks; i++) {  in nvme_ctrlr_hmb_alloc()
973 		hmbc = &ctrlr->hmb_chunks[i];  in nvme_ctrlr_hmb_alloc()
974 		if (bus_dmamem_alloc(ctrlr->hmb_tag,  in nvme_ctrlr_hmb_alloc()
977 			nvme_printf(ctrlr, "failed to alloc HMB\n");  in nvme_ctrlr_hmb_alloc()
980 		if (bus_dmamap_load(ctrlr->hmb_tag, hmbc->hmbc_map,  in nvme_ctrlr_hmb_alloc()
981 		    hmbc->hmbc_vaddr, ctrlr->hmb_chunk, nvme_single_map,  in nvme_ctrlr_hmb_alloc()
983 			bus_dmamem_free(ctrlr->hmb_tag, hmbc->hmbc_vaddr,  in nvme_ctrlr_hmb_alloc()
985 			nvme_printf(ctrlr, "failed to load HMB\n");  in nvme_ctrlr_hmb_alloc()
988 		bus_dmamap_sync(ctrlr->hmb_tag, hmbc->hmbc_map,  in nvme_ctrlr_hmb_alloc()
992 	if (i < ctrlr->hmb_nchunks && i * ctrlr->hmb_chunk < min &&  in nvme_ctrlr_hmb_alloc()
993 	    ctrlr->hmb_chunk / 2 >= minc) {  in nvme_ctrlr_hmb_alloc()
994 		ctrlr->hmb_nchunks = i;  in nvme_ctrlr_hmb_alloc()
995 		nvme_ctrlr_hmb_free(ctrlr);  in nvme_ctrlr_hmb_alloc()
996 		ctrlr->hmb_chunk /= 2;  in nvme_ctrlr_hmb_alloc()
999 	ctrlr->hmb_nchunks = i;  in nvme_ctrlr_hmb_alloc()
1000 	if (ctrlr->hmb_nchunks * ctrlr->hmb_chunk < min) {  in nvme_ctrlr_hmb_alloc()
1001 		nvme_ctrlr_hmb_free(ctrlr);  in nvme_ctrlr_hmb_alloc()
1005 	size = sizeof(struct nvme_hmb_desc) * ctrlr->hmb_nchunks;  in nvme_ctrlr_hmb_alloc()
1006 	err = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev),  in nvme_ctrlr_hmb_alloc()
1008 	    size, 1, size, 0, NULL, NULL, &ctrlr->hmb_desc_tag);  in nvme_ctrlr_hmb_alloc()
1010 		nvme_printf(ctrlr, "HMB desc tag create failed %d\n", err);  in nvme_ctrlr_hmb_alloc()
1011 		nvme_ctrlr_hmb_free(ctrlr);  in nvme_ctrlr_hmb_alloc()
1014 	if (bus_dmamem_alloc(ctrlr->hmb_desc_tag,  in nvme_ctrlr_hmb_alloc()
1015 	    (void **)&ctrlr->hmb_desc_vaddr, BUS_DMA_WAITOK,  in nvme_ctrlr_hmb_alloc()
1016 	    &ctrlr->hmb_desc_map)) {  in nvme_ctrlr_hmb_alloc()
1017 		nvme_printf(ctrlr, "failed to alloc HMB desc\n");  in nvme_ctrlr_hmb_alloc()
1018 		nvme_ctrlr_hmb_free(ctrlr);  in nvme_ctrlr_hmb_alloc()
1021 	if (bus_dmamap_load(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_map,  in nvme_ctrlr_hmb_alloc()
1022 	    ctrlr->hmb_desc_vaddr, size, nvme_single_map,  in nvme_ctrlr_hmb_alloc()
1023 	    &ctrlr->hmb_desc_paddr, BUS_DMA_NOWAIT) != 0) {  in nvme_ctrlr_hmb_alloc()
1024 		bus_dmamem_free(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_vaddr,  in nvme_ctrlr_hmb_alloc()
1025 		    ctrlr->hmb_desc_map);  in nvme_ctrlr_hmb_alloc()
1026 		nvme_printf(ctrlr, "failed to load HMB desc\n");  in nvme_ctrlr_hmb_alloc()
1027 		nvme_ctrlr_hmb_free(ctrlr);  in nvme_ctrlr_hmb_alloc()
1031 	for (i = 0; i < ctrlr->hmb_nchunks; i++) {  in nvme_ctrlr_hmb_alloc()
1032 		memset(&ctrlr->hmb_desc_vaddr[i], 0,  in nvme_ctrlr_hmb_alloc()
1034 		ctrlr->hmb_desc_vaddr[i].addr =  in nvme_ctrlr_hmb_alloc()
1035 		    htole64(ctrlr->hmb_chunks[i].hmbc_paddr);  in nvme_ctrlr_hmb_alloc()
1036 		ctrlr->hmb_desc_vaddr[i].size = htole32(ctrlr->hmb_chunk / ctrlr->page_size);  in nvme_ctrlr_hmb_alloc()
1038 	bus_dmamap_sync(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_map,  in nvme_ctrlr_hmb_alloc()
1041 	nvme_printf(ctrlr, "Allocated %lluMB host memory buffer\n",  in nvme_ctrlr_hmb_alloc()
1042 	    (long long unsigned)ctrlr->hmb_nchunks * ctrlr->hmb_chunk  in nvme_ctrlr_hmb_alloc()
1047 nvme_ctrlr_hmb_enable(struct nvme_controller *ctrlr, bool enable, bool memret)  in nvme_ctrlr_hmb_enable()  argument
1058 	nvme_ctrlr_cmd_set_feature(ctrlr, NVME_FEAT_HOST_MEMORY_BUFFER, cdw11,  in nvme_ctrlr_hmb_enable()
1059 	    ctrlr->hmb_nchunks * ctrlr->hmb_chunk / ctrlr->page_size,  in nvme_ctrlr_hmb_enable()
1060 	    ctrlr->hmb_desc_paddr, ctrlr->hmb_desc_paddr >> 32,  in nvme_ctrlr_hmb_enable()
1061 	    ctrlr->hmb_nchunks, NULL, 0,  in nvme_ctrlr_hmb_enable()
1065 		nvme_printf(ctrlr, "nvme_ctrlr_hmb_enable failed!\n");  in nvme_ctrlr_hmb_enable()
1071 	struct nvme_controller *ctrlr = ctrlr_arg;  in nvme_ctrlr_start()  local
1085 		nvme_qpair_reset(&ctrlr->adminq);  in nvme_ctrlr_start()
1086 		nvme_admin_qpair_enable(&ctrlr->adminq);  in nvme_ctrlr_start()
1089 	if (ctrlr->ioq != NULL) {  in nvme_ctrlr_start()
1090 		for (i = 0; i < ctrlr->num_io_queues; i++)  in nvme_ctrlr_start()
1091 			nvme_qpair_reset(&ctrlr->ioq[i]);  in nvme_ctrlr_start()
1098 	if (resetting && !ctrlr->is_initialized)  in nvme_ctrlr_start()
1101 	if (resetting && nvme_ctrlr_identify(ctrlr) != 0) {  in nvme_ctrlr_start()
1102 		nvme_ctrlr_fail(ctrlr);  in nvme_ctrlr_start()
1115 		old_num_io_queues = ctrlr->num_io_queues;  in nvme_ctrlr_start()
1116 		if (nvme_ctrlr_set_num_qpairs(ctrlr) != 0) {  in nvme_ctrlr_start()
1117 			nvme_ctrlr_fail(ctrlr);  in nvme_ctrlr_start()
1121 		if (old_num_io_queues != ctrlr->num_io_queues) {  in nvme_ctrlr_start()
1123 			      old_num_io_queues, ctrlr->num_io_queues);  in nvme_ctrlr_start()
1127 	if (ctrlr->cdata.hmpre > 0 && ctrlr->hmb_nchunks == 0) {  in nvme_ctrlr_start()
1128 		nvme_ctrlr_hmb_alloc(ctrlr);  in nvme_ctrlr_start()
1129 		if (ctrlr->hmb_nchunks > 0)  in nvme_ctrlr_start()
1130 			nvme_ctrlr_hmb_enable(ctrlr, true, false);  in nvme_ctrlr_start()
1131 	} else if (ctrlr->hmb_nchunks > 0)  in nvme_ctrlr_start()
1132 		nvme_ctrlr_hmb_enable(ctrlr, true, true);  in nvme_ctrlr_start()
1134 	if (nvme_ctrlr_create_qpairs(ctrlr) != 0) {  in nvme_ctrlr_start()
1135 		nvme_ctrlr_fail(ctrlr);  in nvme_ctrlr_start()
1139 	if (nvme_ctrlr_construct_namespaces(ctrlr) != 0) {  in nvme_ctrlr_start()
1140 		nvme_ctrlr_fail(ctrlr);  in nvme_ctrlr_start()
1144 	nvme_ctrlr_configure_aer(ctrlr);  in nvme_ctrlr_start()
1145 	nvme_ctrlr_configure_int_coalescing(ctrlr);  in nvme_ctrlr_start()
1147 	for (i = 0; i < ctrlr->num_io_queues; i++)  in nvme_ctrlr_start()
1148 		nvme_io_qpair_enable(&ctrlr->ioq[i]);  in nvme_ctrlr_start()
1155 	struct nvme_controller *ctrlr = arg;  in nvme_ctrlr_start_config_hook()  local
1164 	if (nvme_ctrlr_hw_reset(ctrlr) != 0) {  in nvme_ctrlr_start_config_hook()
1166 		nvme_ctrlr_fail(ctrlr);  in nvme_ctrlr_start_config_hook()
1167 		config_intrhook_disestablish(&ctrlr->config_hook);  in nvme_ctrlr_start_config_hook()
1179 	if (nvme_ctrlr_hw_reset(ctrlr) != 0)  in nvme_ctrlr_start_config_hook()
1183 	nvme_qpair_reset(&ctrlr->adminq);  in nvme_ctrlr_start_config_hook()
1184 	nvme_admin_qpair_enable(&ctrlr->adminq);  in nvme_ctrlr_start_config_hook()
1186 	if (nvme_ctrlr_identify(ctrlr) == 0 &&  in nvme_ctrlr_start_config_hook()
1187 	    nvme_ctrlr_set_num_qpairs(ctrlr) == 0 &&  in nvme_ctrlr_start_config_hook()
1188 	    nvme_ctrlr_construct_io_qpairs(ctrlr) == 0)  in nvme_ctrlr_start_config_hook()
1189 		nvme_ctrlr_start(ctrlr, false);  in nvme_ctrlr_start_config_hook()
1193 	nvme_sysctl_initialize_ctrlr(ctrlr);  in nvme_ctrlr_start_config_hook()
1194 	config_intrhook_disestablish(&ctrlr->config_hook);  in nvme_ctrlr_start_config_hook()
1196 	ctrlr->is_initialized = 1;  in nvme_ctrlr_start_config_hook()
1197 	nvme_notify_new_controller(ctrlr);  in nvme_ctrlr_start_config_hook()
1204 	struct nvme_controller	*ctrlr = arg;  in nvme_ctrlr_reset_task()  local
1207 	nvme_ctrlr_devctl_log(ctrlr, "RESET", "resetting controller");  in nvme_ctrlr_reset_task()
1208 	status = nvme_ctrlr_hw_reset(ctrlr);  in nvme_ctrlr_reset_task()
1210 		nvme_ctrlr_start(ctrlr, true);  in nvme_ctrlr_reset_task()
1212 		nvme_ctrlr_fail(ctrlr);  in nvme_ctrlr_reset_task()
1214 	atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);  in nvme_ctrlr_reset_task()
1221 nvme_ctrlr_poll(struct nvme_controller *ctrlr)  in nvme_ctrlr_poll()  argument
1225 	nvme_qpair_process_completions(&ctrlr->adminq);  in nvme_ctrlr_poll()
1227 	for (i = 0; i < ctrlr->num_io_queues; i++)  in nvme_ctrlr_poll()
1228 		if (ctrlr->ioq && ctrlr->ioq[i].cpl)  in nvme_ctrlr_poll()
1229 			nvme_qpair_process_completions(&ctrlr->ioq[i]);  in nvme_ctrlr_poll()
1240 	struct nvme_controller *ctrlr = arg;  in nvme_ctrlr_shared_handler()  local
1242 	nvme_mmio_write_4(ctrlr, intms, 1);  in nvme_ctrlr_shared_handler()
1243 	nvme_ctrlr_poll(ctrlr);  in nvme_ctrlr_shared_handler()
1244 	nvme_mmio_write_4(ctrlr, intmc, 1);  in nvme_ctrlr_shared_handler()
1268 nvme_ctrlr_passthrough_cmd(struct nvme_controller *ctrlr,  in nvme_ctrlr_passthrough_cmd()  argument
1278 		if (pt->len > ctrlr->max_xfer_size) {  in nvme_ctrlr_passthrough_cmd()
1279 			nvme_printf(ctrlr, "pt->len (%d) "  in nvme_ctrlr_passthrough_cmd()
1281 			    ctrlr->max_xfer_size);  in nvme_ctrlr_passthrough_cmd()
1322 		nvme_ctrlr_submit_admin_request(ctrlr, req);  in nvme_ctrlr_passthrough_cmd()
1324 		nvme_ctrlr_submit_io_request(ctrlr, req);  in nvme_ctrlr_passthrough_cmd()
1345 	struct nvme_controller			*ctrlr;  in nvme_ctrlr_ioctl()  local
1348 	ctrlr = cdev->si_drv1;  in nvme_ctrlr_ioctl()
1352 		nvme_ctrlr_reset(ctrlr);  in nvme_ctrlr_ioctl()
1356 		return (nvme_ctrlr_passthrough_cmd(ctrlr, pt, le32toh(pt->cmd.nsid),  in nvme_ctrlr_ioctl()
1361 		strncpy(gnsid->cdev, device_get_nameunit(ctrlr->dev),  in nvme_ctrlr_ioctl()
1368 		*(uint64_t *)arg = ctrlr->max_xfer_size;  in nvme_ctrlr_ioctl()
1384 nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev)  in nvme_ctrlr_construct()  argument
1392 	ctrlr->dev = dev;  in nvme_ctrlr_construct()
1394 	mtx_init(&ctrlr->lock, "nvme ctrlr lock", NULL, MTX_DEF);  in nvme_ctrlr_construct()
1395 	if (bus_get_domain(dev, &ctrlr->domain) != 0)  in nvme_ctrlr_construct()
1396 		ctrlr->domain = 0;  in nvme_ctrlr_construct()
1398 	ctrlr->cap_lo = cap_lo = nvme_mmio_read_4(ctrlr, cap_lo);  in nvme_ctrlr_construct()
1408 	ctrlr->cap_hi = cap_hi = nvme_mmio_read_4(ctrlr, cap_hi);  in nvme_ctrlr_construct()
1426 		vs = nvme_mmio_read_4(ctrlr, vs);  in nvme_ctrlr_construct()
1431 		pmrcap = nvme_mmio_read_4(ctrlr, pmrcap);  in nvme_ctrlr_construct()
1443 	ctrlr->dstrd = NVME_CAP_HI_DSTRD(cap_hi) + 2;  in nvme_ctrlr_construct()
1445 	ctrlr->mps = NVME_CAP_HI_MPSMIN(cap_hi);  in nvme_ctrlr_construct()
1446 	ctrlr->page_size = 1 << (NVME_MPS_SHIFT + ctrlr->mps);  in nvme_ctrlr_construct()
1450 	ctrlr->ready_timeout_in_ms = to * 500;  in nvme_ctrlr_construct()
1456 	ctrlr->admin_timeout_period = timeout_period;  in nvme_ctrlr_construct()
1462 	ctrlr->timeout_period = timeout_period;  in nvme_ctrlr_construct()
1467 	ctrlr->enable_aborts = 0;  in nvme_ctrlr_construct()
1468 	TUNABLE_INT_FETCH("hw.nvme.enable_aborts", &ctrlr->enable_aborts);  in nvme_ctrlr_construct()
1471 	ctrlr->max_xfer_size = MIN(maxphys, (ctrlr->page_size / 8 * ctrlr->page_size));  in nvme_ctrlr_construct()
1472 	if (nvme_ctrlr_construct_admin_qpair(ctrlr) != 0)  in nvme_ctrlr_construct()
1486 	ctrlr->taskqueue = taskqueue_create("nvme_taskq", M_WAITOK,  in nvme_ctrlr_construct()
1487 	    taskqueue_thread_enqueue, &ctrlr->taskqueue);  in nvme_ctrlr_construct()
1488 	taskqueue_start_threads(&ctrlr->taskqueue, 2, PI_DISK, "nvme taskq");  in nvme_ctrlr_construct()
1490 	ctrlr->is_resetting = 0;  in nvme_ctrlr_construct()
1491 	ctrlr->is_initialized = 0;  in nvme_ctrlr_construct()
1492 	ctrlr->notification_sent = 0;  in nvme_ctrlr_construct()
1493 	TASK_INIT(&ctrlr->reset_task, 0, nvme_ctrlr_reset_task, ctrlr);  in nvme_ctrlr_construct()
1494 	TASK_INIT(&ctrlr->fail_req_task, 0, nvme_ctrlr_fail_req_task, ctrlr);  in nvme_ctrlr_construct()
1495 	STAILQ_INIT(&ctrlr->fail_req);  in nvme_ctrlr_construct()
1496 	ctrlr->is_failed = false;  in nvme_ctrlr_construct()
1504 	md_args.mda_si_drv1 = (void *)ctrlr;  in nvme_ctrlr_construct()
1505 	status = make_dev_s(&md_args, &ctrlr->cdev, "nvme%d",  in nvme_ctrlr_construct()
1514 nvme_ctrlr_destruct(struct nvme_controller *ctrlr, device_t dev)  in nvme_ctrlr_destruct()  argument
1518 	ctrlr->is_dying = true;  in nvme_ctrlr_destruct()
1520 	if (ctrlr->resource == NULL)  in nvme_ctrlr_destruct()
1522 	if (!mtx_initialized(&ctrlr->adminq.lock))  in nvme_ctrlr_destruct()
1529 	gone = (nvme_mmio_read_4(ctrlr, csts) == NVME_GONE);  in nvme_ctrlr_destruct()
1531 		nvme_ctrlr_fail(ctrlr);  in nvme_ctrlr_destruct()
1533 		nvme_notify_fail_consumers(ctrlr);  in nvme_ctrlr_destruct()
1536 		nvme_ns_destruct(&ctrlr->ns[i]);  in nvme_ctrlr_destruct()
1538 	if (ctrlr->cdev)  in nvme_ctrlr_destruct()
1539 		destroy_dev(ctrlr->cdev);  in nvme_ctrlr_destruct()
1541 	if (ctrlr->is_initialized) {  in nvme_ctrlr_destruct()
1543 			if (ctrlr->hmb_nchunks > 0)  in nvme_ctrlr_destruct()
1544 				nvme_ctrlr_hmb_enable(ctrlr, false, false);  in nvme_ctrlr_destruct()
1545 			nvme_ctrlr_delete_qpairs(ctrlr);  in nvme_ctrlr_destruct()
1547 		nvme_ctrlr_hmb_free(ctrlr);  in nvme_ctrlr_destruct()
1549 	if (ctrlr->ioq != NULL) {  in nvme_ctrlr_destruct()
1550 		for (i = 0; i < ctrlr->num_io_queues; i++)  in nvme_ctrlr_destruct()
1551 			nvme_io_qpair_destroy(&ctrlr->ioq[i]);  in nvme_ctrlr_destruct()
1552 		free(ctrlr->ioq, M_NVME);  in nvme_ctrlr_destruct()
1554 	nvme_admin_qpair_destroy(&ctrlr->adminq);  in nvme_ctrlr_destruct()
1564 		nvme_ctrlr_shutdown(ctrlr);  in nvme_ctrlr_destruct()
1567 		nvme_ctrlr_disable(ctrlr);  in nvme_ctrlr_destruct()
1570 	if (ctrlr->taskqueue)  in nvme_ctrlr_destruct()
1571 		taskqueue_free(ctrlr->taskqueue);  in nvme_ctrlr_destruct()
1573 	if (ctrlr->tag)  in nvme_ctrlr_destruct()
1574 		bus_teardown_intr(ctrlr->dev, ctrlr->res, ctrlr->tag);  in nvme_ctrlr_destruct()
1576 	if (ctrlr->res)  in nvme_ctrlr_destruct()
1577 		bus_release_resource(ctrlr->dev, SYS_RES_IRQ,  in nvme_ctrlr_destruct()
1578 		    rman_get_rid(ctrlr->res), ctrlr->res);  in nvme_ctrlr_destruct()
1580 	if (ctrlr->bar4_resource != NULL) {  in nvme_ctrlr_destruct()
1582 		    ctrlr->bar4_resource_id, ctrlr->bar4_resource);  in nvme_ctrlr_destruct()
1586 	    ctrlr->resource_id, ctrlr->resource);  in nvme_ctrlr_destruct()
1589 	mtx_destroy(&ctrlr->lock);  in nvme_ctrlr_destruct()
1593 nvme_ctrlr_shutdown(struct nvme_controller *ctrlr)  in nvme_ctrlr_shutdown()  argument
1599 	cc = nvme_mmio_read_4(ctrlr, cc);  in nvme_ctrlr_shutdown()
1602 	nvme_mmio_write_4(ctrlr, cc, cc);  in nvme_ctrlr_shutdown()
1604 	timeout = ticks + (ctrlr->cdata.rtd3e == 0 ? 5 * hz :  in nvme_ctrlr_shutdown()
1605 	    ((uint64_t)ctrlr->cdata.rtd3e * hz + 999999) / 1000000);  in nvme_ctrlr_shutdown()
1607 		csts = nvme_mmio_read_4(ctrlr, csts);  in nvme_ctrlr_shutdown()
1613 			nvme_printf(ctrlr, "shutdown timeout\n");  in nvme_ctrlr_shutdown()
1621 nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr,  in nvme_ctrlr_submit_admin_request()  argument
1625 	nvme_qpair_submit_request(&ctrlr->adminq, req);  in nvme_ctrlr_submit_admin_request()
1629 nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr,  in nvme_ctrlr_submit_io_request()  argument
1634 	qpair = &ctrlr->ioq[QP(ctrlr, curcpu)];  in nvme_ctrlr_submit_io_request()
1639 nvme_ctrlr_get_device(struct nvme_controller *ctrlr)  in nvme_ctrlr_get_device()  argument
1642 	return (ctrlr->dev);  in nvme_ctrlr_get_device()
1646 nvme_ctrlr_get_data(struct nvme_controller *ctrlr)  in nvme_ctrlr_get_data()  argument
1649 	return (&ctrlr->cdata);  in nvme_ctrlr_get_data()
1653 nvme_ctrlr_suspend(struct nvme_controller *ctrlr)  in nvme_ctrlr_suspend()  argument
1660 	if (ctrlr->is_failed)  in nvme_ctrlr_suspend()
1670 	while (atomic_cmpset_32(&ctrlr->is_resetting, 0, 1) == 0 && to-- > 0)  in nvme_ctrlr_suspend()
1673 		nvme_printf(ctrlr,  in nvme_ctrlr_suspend()
1678 	if (ctrlr->hmb_nchunks > 0)  in nvme_ctrlr_suspend()
1679 		nvme_ctrlr_hmb_enable(ctrlr, false, false);  in nvme_ctrlr_suspend()
1689 	nvme_ctrlr_delete_qpairs(ctrlr);  in nvme_ctrlr_suspend()
1690 	nvme_ctrlr_disable_qpairs(ctrlr);  in nvme_ctrlr_suspend()
1691 	nvme_ctrlr_shutdown(ctrlr);  in nvme_ctrlr_suspend()
1697 nvme_ctrlr_resume(struct nvme_controller *ctrlr)  in nvme_ctrlr_resume()  argument
1703 	if (ctrlr->is_failed)  in nvme_ctrlr_resume()
1706 	if (nvme_ctrlr_hw_reset(ctrlr) != 0)  in nvme_ctrlr_resume()
1714 	nvme_ctrlr_start(ctrlr, true);  in nvme_ctrlr_resume()
1715 	(void)atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);  in nvme_ctrlr_resume()
1724 	nvme_printf(ctrlr, "Failed to reset on resume, failing.\n");  in nvme_ctrlr_resume()
1725 	nvme_ctrlr_fail(ctrlr);  in nvme_ctrlr_resume()
1726 	(void)atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);  in nvme_ctrlr_resume()