80 static void mpi3mr_repost_reply_buf(struct mpi3mr_softc *sc,
82 static int mpi3mr_complete_admin_cmd(struct mpi3mr_softc *sc);
83 static void mpi3mr_port_enable_complete(struct mpi3mr_softc *sc,
85 static void mpi3mr_flush_io(struct mpi3mr_softc *sc);
86 static int mpi3mr_issue_reset(struct mpi3mr_softc *sc, U16 reset_type,
88 static void mpi3mr_dev_rmhs_send_tm(struct mpi3mr_softc *sc, U16 handle,
90 static void mpi3mr_dev_rmhs_complete_iou(struct mpi3mr_softc *sc,
92 static void mpi3mr_dev_rmhs_complete_tm(struct mpi3mr_softc *sc,
94 static void mpi3mr_send_evt_ack(struct mpi3mr_softc *sc, U8 event,
96 static void mpi3mr_print_fault_info(struct mpi3mr_softc *sc);
97 static inline void mpi3mr_set_diagsave(struct mpi3mr_softc *sc);
146 	    U32 timeout, struct mpi3mr_softc *sc)  in wait_for_completion_timeout_tm()  argument
151 		msleep(&sc->tm_chan, &sc->mpi3mr_mtx, PRIBIO,  in wait_for_completion_timeout_tm()
164 poll_for_command_completion(struct mpi3mr_softc *sc,  in poll_for_command_completion()  argument
169                mpi3mr_complete_admin_cmd(sc);  in poll_for_command_completion()
190 mpi3mr_trigger_snapdump(struct mpi3mr_softc *sc, U32 reason_code)  in mpi3mr_trigger_snapdump()  argument
194 	mpi3mr_dprint(sc, MPI3MR_INFO, "snapdump triggered: reason code: %s\n",  in mpi3mr_trigger_snapdump()
197 	mpi3mr_set_diagsave(sc);  in mpi3mr_trigger_snapdump()
198 	mpi3mr_issue_reset(sc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,  in mpi3mr_trigger_snapdump()
202 		host_diagnostic = mpi3mr_regread(sc, MPI3_SYSIF_HOST_DIAG_OFFSET);  in mpi3mr_trigger_snapdump()
225 static void mpi3mr_check_rh_fault_ioc(struct mpi3mr_softc *sc, U32 reason_code)  in mpi3mr_check_rh_fault_ioc()  argument
229 	if (sc->unrecoverable) {  in mpi3mr_check_rh_fault_ioc()
230 		mpi3mr_dprint(sc, MPI3MR_ERROR, "controller is unrecoverable\n");  in mpi3mr_check_rh_fault_ioc()
234 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_check_rh_fault_ioc()
237 		mpi3mr_print_fault_info(sc);  in mpi3mr_check_rh_fault_ioc()
241 	mpi3mr_trigger_snapdump(sc, reason_code);  in mpi3mr_check_rh_fault_ioc()
246 static void * mpi3mr_get_reply_virt_addr(struct mpi3mr_softc *sc,  in mpi3mr_get_reply_virt_addr()  argument
251 	if ((phys_addr < sc->reply_buf_dma_min_address) ||  in mpi3mr_get_reply_virt_addr()
252 	    (phys_addr > sc->reply_buf_dma_max_address))  in mpi3mr_get_reply_virt_addr()
255 	return sc->reply_buf + (phys_addr - sc->reply_buf_phys);  in mpi3mr_get_reply_virt_addr()
258 static void * mpi3mr_get_sensebuf_virt_addr(struct mpi3mr_softc *sc,  in mpi3mr_get_sensebuf_virt_addr()  argument
263 	return sc->sense_buf + (phys_addr - sc->sense_buf_phys);  in mpi3mr_get_sensebuf_virt_addr()
266 static void mpi3mr_repost_reply_buf(struct mpi3mr_softc *sc,  in mpi3mr_repost_reply_buf()  argument
271 	mtx_lock_spin(&sc->reply_free_q_lock);  in mpi3mr_repost_reply_buf()
272 	old_idx  =  sc->reply_free_q_host_index;  in mpi3mr_repost_reply_buf()
273 	sc->reply_free_q_host_index = ((sc->reply_free_q_host_index ==  in mpi3mr_repost_reply_buf()
274 	    (sc->reply_free_q_sz - 1)) ? 0 :  in mpi3mr_repost_reply_buf()
275 	    (sc->reply_free_q_host_index + 1));  in mpi3mr_repost_reply_buf()
276 	sc->reply_free_q[old_idx] = reply_dma;  in mpi3mr_repost_reply_buf()
277 	mpi3mr_regwrite(sc, MPI3_SYSIF_REPLY_FREE_HOST_INDEX_OFFSET,  in mpi3mr_repost_reply_buf()
278 		sc->reply_free_q_host_index);  in mpi3mr_repost_reply_buf()
279 	mtx_unlock_spin(&sc->reply_free_q_lock);  in mpi3mr_repost_reply_buf()
282 static void mpi3mr_repost_sense_buf(struct mpi3mr_softc *sc,  in mpi3mr_repost_sense_buf()  argument
287 	mtx_lock_spin(&sc->sense_buf_q_lock);  in mpi3mr_repost_sense_buf()
288 	old_idx  =  sc->sense_buf_q_host_index;  in mpi3mr_repost_sense_buf()
289 	sc->sense_buf_q_host_index = ((sc->sense_buf_q_host_index ==  in mpi3mr_repost_sense_buf()
290 	    (sc->sense_buf_q_sz - 1)) ? 0 :  in mpi3mr_repost_sense_buf()
291 	    (sc->sense_buf_q_host_index + 1));  in mpi3mr_repost_sense_buf()
292 	sc->sense_buf_q[old_idx] = sense_buf_phys;  in mpi3mr_repost_sense_buf()
293 	mpi3mr_regwrite(sc, MPI3_SYSIF_SENSE_BUF_FREE_HOST_INDEX_OFFSET,  in mpi3mr_repost_sense_buf()
294 		sc->sense_buf_q_host_index);  in mpi3mr_repost_sense_buf()
295 	mtx_unlock_spin(&sc->sense_buf_q_lock);  in mpi3mr_repost_sense_buf()
299 void mpi3mr_set_io_divert_for_all_vd_in_tg(struct mpi3mr_softc *sc,  in mpi3mr_set_io_divert_for_all_vd_in_tg()  argument
304 	mtx_lock_spin(&sc->target_lock);  in mpi3mr_set_io_divert_for_all_vd_in_tg()
305 	TAILQ_FOREACH(target, &sc->cam_sc->tgt_list, tgt_next) {  in mpi3mr_set_io_divert_for_all_vd_in_tg()
309 	mtx_unlock_spin(&sc->target_lock);  in mpi3mr_set_io_divert_for_all_vd_in_tg()
324 int mpi3mr_submit_admin_cmd(struct mpi3mr_softc *sc, void *admin_req,  in mpi3mr_submit_admin_cmd()  argument
331 	mtx_lock_spin(&sc->admin_req_lock);  in mpi3mr_submit_admin_cmd()
332 	areq_pi = sc->admin_req_pi;  in mpi3mr_submit_admin_cmd()
333 	areq_ci = sc->admin_req_ci;  in mpi3mr_submit_admin_cmd()
334 	max_entries = sc->num_admin_reqs;  in mpi3mr_submit_admin_cmd()
336 	if (sc->unrecoverable)  in mpi3mr_submit_admin_cmd()
342 		    sc->name);  in mpi3mr_submit_admin_cmd()
346 	areq_entry = (U8 *)sc->admin_req + (areq_pi *  in mpi3mr_submit_admin_cmd()
353 	sc->admin_req_pi = areq_pi;  in mpi3mr_submit_admin_cmd()
355 	mpi3mr_regwrite(sc, MPI3_SYSIF_ADMIN_REQ_Q_PI_OFFSET, sc->admin_req_pi);  in mpi3mr_submit_admin_cmd()
358 	mtx_unlock_spin(&sc->admin_req_lock);  in mpi3mr_submit_admin_cmd()
394 int mpi3mr_submit_io(struct mpi3mr_softc *sc,  in mpi3mr_submit_io()  argument
400 	U16 req_sz = sc->facts.op_req_sz;  in mpi3mr_submit_io()
408 		irq_ctx = &sc->irq_ctx[op_req_q->reply_qid - 1];  in mpi3mr_submit_io()
409 		mpi3mr_complete_io_cmd(sc, irq_ctx);  in mpi3mr_submit_io()
413 				sc->name);  in mpi3mr_submit_io()
426 	mpi3mr_atomic_inc(&sc->op_reply_q[op_req_q->reply_qid - 1].pend_ios);  in mpi3mr_submit_io()
428 	mpi3mr_regwrite(sc, MPI3_SYSIF_OPER_REQ_Q_N_PI_OFFSET(op_req_q->qid), op_req_q->pi);  in mpi3mr_submit_io()
429 	if (sc->mpi3mr_debug & MPI3MR_TRACE) {  in mpi3mr_submit_io()
430 		device_printf(sc->mpi3mr_dev, "IO submission: QID:%d PI:0x%x\n", op_req_q->qid, op_req_q->pi);  in mpi3mr_submit_io()
459 void mpi3mr_enable_interrupts(struct mpi3mr_softc *sc)  in mpi3mr_enable_interrupts()  argument
461 	sc->intr_enabled = 1;  in mpi3mr_enable_interrupts()
464 void mpi3mr_disable_interrupts(struct mpi3mr_softc *sc)  in mpi3mr_disable_interrupts()  argument
466 	sc->intr_enabled = 0;  in mpi3mr_disable_interrupts()
478 static int mpi3mr_delete_op_reply_queue(struct mpi3mr_softc *sc, U16 qid)  in mpi3mr_delete_op_reply_queue()  argument
485 	op_reply_q = &sc->op_reply_q[qid - 1];  in mpi3mr_delete_op_reply_queue()
491 		    sc->name);  in mpi3mr_delete_op_reply_queue()
497 	mtx_lock(&sc->init_cmds.completion.lock);  in mpi3mr_delete_op_reply_queue()
498 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_delete_op_reply_queue()
501 		    sc->name);  in mpi3mr_delete_op_reply_queue()
502 		mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_delete_op_reply_queue()
506 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_delete_op_reply_queue()
509 		    sc->name);  in mpi3mr_delete_op_reply_queue()
512 	sc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_delete_op_reply_queue()
513 	sc->init_cmds.is_waiting = 1;  in mpi3mr_delete_op_reply_queue()
514 	sc->init_cmds.callback = NULL;  in mpi3mr_delete_op_reply_queue()
519 	init_completion(&sc->init_cmds.completion);  in mpi3mr_delete_op_reply_queue()
520 	retval = mpi3mr_submit_admin_cmd(sc, &delq_req, sizeof(delq_req));  in mpi3mr_delete_op_reply_queue()
523 		    sc->name);  in mpi3mr_delete_op_reply_queue()
526 	wait_for_completion_timeout(&sc->init_cmds.completion,  in mpi3mr_delete_op_reply_queue()
528 	if (!(sc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_delete_op_reply_queue()
530 		    sc->name);  in mpi3mr_delete_op_reply_queue()
531 		mpi3mr_check_rh_fault_ioc(sc,  in mpi3mr_delete_op_reply_queue()
533 		sc->unrecoverable = 1;  in mpi3mr_delete_op_reply_queue()
538 	if ((sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_delete_op_reply_queue()
541 		    " Loginfo(0x%08x) \n" , sc->name,  in mpi3mr_delete_op_reply_queue()
542 		    (sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_delete_op_reply_queue()
543 		    sc->init_cmds.ioc_loginfo);  in mpi3mr_delete_op_reply_queue()
547 	sc->irq_ctx[qid - 1].op_reply_q = NULL;  in mpi3mr_delete_op_reply_queue()
549 	if (sc->op_reply_q[qid - 1].q_base_phys != 0)  in mpi3mr_delete_op_reply_queue()
550 		bus_dmamap_unload(sc->op_reply_q[qid - 1].q_base_tag, sc->op_reply_q[qid - 1].q_base_dmamap);  in mpi3mr_delete_op_reply_queue()
551 	if (sc->op_reply_q[qid - 1].q_base != NULL)  in mpi3mr_delete_op_reply_queue()
552 …bus_dmamem_free(sc->op_reply_q[qid - 1].q_base_tag, sc->op_reply_q[qid - 1].q_base, sc->op_reply_q…  in mpi3mr_delete_op_reply_queue()
553 	if (sc->op_reply_q[qid - 1].q_base_tag != NULL)  in mpi3mr_delete_op_reply_queue()
554 		bus_dma_tag_destroy(sc->op_reply_q[qid - 1].q_base_tag);  in mpi3mr_delete_op_reply_queue()
556 	sc->op_reply_q[qid - 1].q_base = NULL;  in mpi3mr_delete_op_reply_queue()
557 	sc->op_reply_q[qid - 1].qid = 0;  in mpi3mr_delete_op_reply_queue()
559 	sc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_delete_op_reply_queue()
560 	mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_delete_op_reply_queue()
575 static int mpi3mr_create_op_reply_queue(struct mpi3mr_softc *sc, U16 qid)  in mpi3mr_create_op_reply_queue()  argument
582 	op_reply_q = &sc->op_reply_q[qid - 1];  in mpi3mr_create_op_reply_queue()
588 		    sc->name, op_reply_q->qid);  in mpi3mr_create_op_reply_queue()
593 	if (pci_get_revid(sc->mpi3mr_dev) == SAS4116_CHIP_REV_A0)  in mpi3mr_create_op_reply_queue()
598 	op_reply_q->qsz = op_reply_q->num_replies * sc->op_reply_sz;  in mpi3mr_create_op_reply_queue()
605 		if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_create_op_reply_queue()
607 					sc->dma_loaddr,		/* lowaddr */  in mpi3mr_create_op_reply_queue()
616 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate Operational reply DMA tag\n");  in mpi3mr_create_op_reply_queue()
622 			mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: Cannot allocate replies memory\n", __func__);  in mpi3mr_create_op_reply_queue()
628 …mpi3mr_dprint(sc, MPI3MR_XINFO, "Operational Reply queue ID: %d phys addr= %#016jx virt_addr: %pa …  in mpi3mr_create_op_reply_queue()
635 			    sc->name, qid);  in mpi3mr_create_op_reply_queue()
642 	mtx_lock(&sc->init_cmds.completion.lock);  in mpi3mr_create_op_reply_queue()
643 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_create_op_reply_queue()
646 		    sc->name);  in mpi3mr_create_op_reply_queue()
647 		mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_create_op_reply_queue()
651 	sc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_create_op_reply_queue()
652 	sc->init_cmds.is_waiting = 1;  in mpi3mr_create_op_reply_queue()
653 	sc->init_cmds.callback = NULL;  in mpi3mr_create_op_reply_queue()
658 	create_req.MSIxIndex = sc->irq_ctx[qid - 1].msix_index;  in mpi3mr_create_op_reply_queue()
662 	init_completion(&sc->init_cmds.completion);  in mpi3mr_create_op_reply_queue()
663 	retval = mpi3mr_submit_admin_cmd(sc, &create_req,  in mpi3mr_create_op_reply_queue()
667 		    sc->name);  in mpi3mr_create_op_reply_queue()
671 	wait_for_completion_timeout(&sc->init_cmds.completion,  in mpi3mr_create_op_reply_queue()
673 	if (!(sc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_create_op_reply_queue()
675 		    sc->name);  in mpi3mr_create_op_reply_queue()
676 		mpi3mr_check_rh_fault_ioc(sc,  in mpi3mr_create_op_reply_queue()
678 		sc->unrecoverable = 1;  in mpi3mr_create_op_reply_queue()
683 	if ((sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_create_op_reply_queue()
686 		    " Loginfo(0x%08x) \n" , sc->name,  in mpi3mr_create_op_reply_queue()
687 		    (sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_create_op_reply_queue()
688 		    sc->init_cmds.ioc_loginfo);  in mpi3mr_create_op_reply_queue()
693 	sc->irq_ctx[qid - 1].op_reply_q = op_reply_q;  in mpi3mr_create_op_reply_queue()
696 	sc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_create_op_reply_queue()
697 	mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_create_op_reply_queue()
724 static int mpi3mr_create_op_req_queue(struct mpi3mr_softc *sc, U16 req_qid, U8 reply_qid)  in mpi3mr_create_op_req_queue()  argument
731 	op_req_q = &sc->op_req_q[req_qid - 1];  in mpi3mr_create_op_req_queue()
737 		    sc->name, op_req_q->qid);  in mpi3mr_create_op_req_queue()
744 	op_req_q->qsz = op_req_q->num_reqs * sc->facts.op_req_sz;  in mpi3mr_create_op_req_queue()
751 		if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_create_op_req_queue()
753 					sc->dma_loaddr,		/* lowaddr */  in mpi3mr_create_op_req_queue()
762 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate request DMA tag\n");  in mpi3mr_create_op_req_queue()
768 			mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: Cannot allocate replies memory\n", __func__);  in mpi3mr_create_op_req_queue()
777 …mpi3mr_dprint(sc, MPI3MR_XINFO, "Operational Request QID: %d phys addr= %#016jx virt addr= %pa siz…  in mpi3mr_create_op_req_queue()
783 			    sc->name, req_qid);  in mpi3mr_create_op_req_queue()
790 	mtx_lock(&sc->init_cmds.completion.lock);  in mpi3mr_create_op_req_queue()
791 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_create_op_req_queue()
794 		    sc->name);  in mpi3mr_create_op_req_queue()
795 		mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_create_op_req_queue()
799 	sc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_create_op_req_queue()
800 	sc->init_cmds.is_waiting = 1;  in mpi3mr_create_op_req_queue()
801 	sc->init_cmds.callback = NULL;  in mpi3mr_create_op_req_queue()
810 	init_completion(&sc->init_cmds.completion);  in mpi3mr_create_op_req_queue()
811 	retval = mpi3mr_submit_admin_cmd(sc, &create_req,  in mpi3mr_create_op_req_queue()
815 		    sc->name);  in mpi3mr_create_op_req_queue()
819 	wait_for_completion_timeout(&sc->init_cmds.completion,  in mpi3mr_create_op_req_queue()
822 	if (!(sc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_create_op_req_queue()
824 		    sc->name);  in mpi3mr_create_op_req_queue()
825 		mpi3mr_check_rh_fault_ioc(sc,  in mpi3mr_create_op_req_queue()
827 		sc->unrecoverable = 1;  in mpi3mr_create_op_req_queue()
832 	if ((sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_create_op_req_queue()
835 		    " Loginfo(0x%08x) \n" , sc->name,  in mpi3mr_create_op_req_queue()
836 		    (sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_create_op_req_queue()
837 		    sc->init_cmds.ioc_loginfo);  in mpi3mr_create_op_req_queue()
844 	sc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_create_op_req_queue()
845 	mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_create_op_req_queue()
867 static int mpi3mr_create_op_queues(struct mpi3mr_softc *sc)  in mpi3mr_create_op_queues()  argument
872 	num_queues = min(sc->facts.max_op_reply_q,  in mpi3mr_create_op_queues()
873 	    sc->facts.max_op_req_q);  in mpi3mr_create_op_queues()
874 	num_queues = min(num_queues, sc->msix_count);  in mpi3mr_create_op_queues()
880 	if (sc->num_queues)  in mpi3mr_create_op_queues()
881 		num_queues = sc->num_queues;  in mpi3mr_create_op_queues()
883 	mpi3mr_dprint(sc, MPI3MR_XINFO, "Trying to create %d Operational Q pairs\n",  in mpi3mr_create_op_queues()
886 	if (!sc->op_req_q) {  in mpi3mr_create_op_queues()
887 		sc->op_req_q = malloc(sizeof(struct mpi3mr_op_req_queue) *  in mpi3mr_create_op_queues()
890 		if (!sc->op_req_q) {  in mpi3mr_create_op_queues()
891 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to alloc memory for Request queue info\n");  in mpi3mr_create_op_queues()
897 	if (!sc->op_reply_q) {  in mpi3mr_create_op_queues()
898 		sc->op_reply_q = malloc(sizeof(struct mpi3mr_op_reply_queue) * num_queues,  in mpi3mr_create_op_queues()
901 		if (!sc->op_reply_q) {  in mpi3mr_create_op_queues()
902 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to alloc memory for Reply queue info\n");  in mpi3mr_create_op_queues()
908 	sc->num_hosttag_op_req_q = (sc->max_host_ios + 1) / num_queues;  in mpi3mr_create_op_queues()
913 		if (mpi3mr_create_op_reply_queue(sc, qid)) {  in mpi3mr_create_op_queues()
914 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to create Reply queue %d\n",  in mpi3mr_create_op_queues()
918 		if (mpi3mr_create_op_req_queue(sc, qid,  in mpi3mr_create_op_queues()
919 		    sc->op_reply_q[qid - 1].qid)) {  in mpi3mr_create_op_queues()
920 			mpi3mr_delete_op_reply_queue(sc, qid);  in mpi3mr_create_op_queues()
921 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to create Request queue %d\n",  in mpi3mr_create_op_queues()
934 	if (!sc->num_queues) {  in mpi3mr_create_op_queues()
935 		sc->num_queues = i;  in mpi3mr_create_op_queues()
938 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Number of queues (%d) post reset are not same as"   in mpi3mr_create_op_queues()
944 	mpi3mr_dprint(sc, MPI3MR_INFO, "Successfully created %d Operational Queue pairs\n",  in mpi3mr_create_op_queues()
945 	    sc->num_queues);  in mpi3mr_create_op_queues()
946 	mpi3mr_dprint(sc, MPI3MR_INFO, "Request Queue QD: %d Reply queue QD: %d\n",  in mpi3mr_create_op_queues()
947 	    sc->op_req_q[0].num_reqs, sc->op_reply_q[0].num_replies);  in mpi3mr_create_op_queues()
951 	if (sc->op_req_q) {  in mpi3mr_create_op_queues()
952 		free(sc->op_req_q, M_MPI3MR);  in mpi3mr_create_op_queues()
953 		sc->op_req_q = NULL;  in mpi3mr_create_op_queues()
955 	if (sc->op_reply_q) {  in mpi3mr_create_op_queues()
956 		free(sc->op_reply_q, M_MPI3MR);  in mpi3mr_create_op_queues()
957 		sc->op_reply_q = NULL;  in mpi3mr_create_op_queues()
969 static int mpi3mr_setup_admin_qpair(struct mpi3mr_softc *sc)  in mpi3mr_setup_admin_qpair()  argument
974 	sc->admin_req_q_sz = MPI3MR_AREQQ_SIZE;  in mpi3mr_setup_admin_qpair()
975 	sc->num_admin_reqs = sc->admin_req_q_sz / MPI3MR_AREQ_FRAME_SZ;  in mpi3mr_setup_admin_qpair()
976 	sc->admin_req_ci = sc->admin_req_pi = 0;  in mpi3mr_setup_admin_qpair()
978 	sc->admin_reply_q_sz = MPI3MR_AREPQ_SIZE;  in mpi3mr_setup_admin_qpair()
979 	sc->num_admin_replies = sc->admin_reply_q_sz/ MPI3MR_AREP_FRAME_SZ;  in mpi3mr_setup_admin_qpair()
980 	sc->admin_reply_ci = 0;  in mpi3mr_setup_admin_qpair()
981 	sc->admin_reply_ephase = 1;  in mpi3mr_setup_admin_qpair()
983 	if (!sc->admin_req) {  in mpi3mr_setup_admin_qpair()
989 		if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_setup_admin_qpair()
994 					sc->admin_req_q_sz,	/* maxsize */  in mpi3mr_setup_admin_qpair()
996 					sc->admin_req_q_sz,	/* maxsegsize */  in mpi3mr_setup_admin_qpair()
999 					&sc->admin_req_tag)) {  in mpi3mr_setup_admin_qpair()
1000 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate request DMA tag\n");  in mpi3mr_setup_admin_qpair()
1004 		if (bus_dmamem_alloc(sc->admin_req_tag, (void **)&sc->admin_req,  in mpi3mr_setup_admin_qpair()
1005 		    BUS_DMA_NOWAIT, &sc->admin_req_dmamap)) {  in mpi3mr_setup_admin_qpair()
1006 			mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: Cannot allocate replies memory\n", __func__);  in mpi3mr_setup_admin_qpair()
1009 		bzero(sc->admin_req, sc->admin_req_q_sz);  in mpi3mr_setup_admin_qpair()
1010 		bus_dmamap_load(sc->admin_req_tag, sc->admin_req_dmamap, sc->admin_req, sc->admin_req_q_sz,  in mpi3mr_setup_admin_qpair()
1011 		    mpi3mr_memaddr_cb, &sc->admin_req_phys, BUS_DMA_NOWAIT);  in mpi3mr_setup_admin_qpair()
1012 		mpi3mr_dprint(sc, MPI3MR_XINFO, "Admin Req queue phys addr= %#016jx size= %d\n",  in mpi3mr_setup_admin_qpair()
1013 		    (uintmax_t)sc->admin_req_phys, sc->admin_req_q_sz);  in mpi3mr_setup_admin_qpair()
1015 		if (!sc->admin_req)  in mpi3mr_setup_admin_qpair()
1019 			    sc->name);  in mpi3mr_setup_admin_qpair()
1024 	if (!sc->admin_reply) {  in mpi3mr_setup_admin_qpair()
1025 		mtx_init(&sc->admin_reply_lock, "Admin Reply Queue Lock", NULL, MTX_SPIN);  in mpi3mr_setup_admin_qpair()
1027 		if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_setup_admin_qpair()
1032 					sc->admin_reply_q_sz,	/* maxsize */  in mpi3mr_setup_admin_qpair()
1034 					sc->admin_reply_q_sz,	/* maxsegsize */  in mpi3mr_setup_admin_qpair()
1037 					&sc->admin_reply_tag)) {  in mpi3mr_setup_admin_qpair()
1038 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate reply DMA tag\n");  in mpi3mr_setup_admin_qpair()
1042 		if (bus_dmamem_alloc(sc->admin_reply_tag, (void **)&sc->admin_reply,  in mpi3mr_setup_admin_qpair()
1043 		    BUS_DMA_NOWAIT, &sc->admin_reply_dmamap)) {  in mpi3mr_setup_admin_qpair()
1044 			mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: Cannot allocate replies memory\n", __func__);  in mpi3mr_setup_admin_qpair()
1047 		bzero(sc->admin_reply, sc->admin_reply_q_sz);  in mpi3mr_setup_admin_qpair()
1048 …bus_dmamap_load(sc->admin_reply_tag, sc->admin_reply_dmamap, sc->admin_reply, sc->admin_reply_q_sz,  in mpi3mr_setup_admin_qpair()
1049 		    mpi3mr_memaddr_cb, &sc->admin_reply_phys, BUS_DMA_NOWAIT);  in mpi3mr_setup_admin_qpair()
1050 		mpi3mr_dprint(sc, MPI3MR_XINFO, "Admin Reply queue phys addr= %#016jx size= %d\n",  in mpi3mr_setup_admin_qpair()
1051 		    (uintmax_t)sc->admin_reply_phys, sc->admin_req_q_sz);  in mpi3mr_setup_admin_qpair()
1054 		if (!sc->admin_reply)  in mpi3mr_setup_admin_qpair()
1058 			    sc->name);  in mpi3mr_setup_admin_qpair()
1063 	num_adm_entries = (sc->num_admin_replies << 16) |  in mpi3mr_setup_admin_qpair()
1064 				(sc->num_admin_reqs);  in mpi3mr_setup_admin_qpair()
1065 	mpi3mr_regwrite(sc, MPI3_SYSIF_ADMIN_Q_NUM_ENTRIES_OFFSET, num_adm_entries);  in mpi3mr_setup_admin_qpair()
1066 	mpi3mr_regwrite64(sc, MPI3_SYSIF_ADMIN_REQ_Q_ADDR_LOW_OFFSET, sc->admin_req_phys);  in mpi3mr_setup_admin_qpair()
1067 	mpi3mr_regwrite64(sc, MPI3_SYSIF_ADMIN_REPLY_Q_ADDR_LOW_OFFSET, sc->admin_reply_phys);  in mpi3mr_setup_admin_qpair()
1068 	mpi3mr_regwrite(sc, MPI3_SYSIF_ADMIN_REQ_Q_PI_OFFSET, sc->admin_req_pi);  in mpi3mr_setup_admin_qpair()
1069 	mpi3mr_regwrite(sc, MPI3_SYSIF_ADMIN_REPLY_Q_CI_OFFSET, sc->admin_reply_ci);  in mpi3mr_setup_admin_qpair()
1075 	if (sc->admin_reply_phys)  in mpi3mr_setup_admin_qpair()
1076 		bus_dmamap_unload(sc->admin_reply_tag, sc->admin_reply_dmamap);  in mpi3mr_setup_admin_qpair()
1078 	if (sc->admin_reply != NULL)  in mpi3mr_setup_admin_qpair()
1079 		bus_dmamem_free(sc->admin_reply_tag, sc->admin_reply,  in mpi3mr_setup_admin_qpair()
1080 		    sc->admin_reply_dmamap);  in mpi3mr_setup_admin_qpair()
1082 	if (sc->admin_reply_tag != NULL)  in mpi3mr_setup_admin_qpair()
1083 		bus_dma_tag_destroy(sc->admin_reply_tag);  in mpi3mr_setup_admin_qpair()
1086 	if (sc->admin_req_phys)  in mpi3mr_setup_admin_qpair()
1087 		bus_dmamap_unload(sc->admin_req_tag, sc->admin_req_dmamap);  in mpi3mr_setup_admin_qpair()
1089 	if (sc->admin_req != NULL)  in mpi3mr_setup_admin_qpair()
1090 		bus_dmamem_free(sc->admin_req_tag, sc->admin_req,  in mpi3mr_setup_admin_qpair()
1091 		    sc->admin_req_dmamap);  in mpi3mr_setup_admin_qpair()
1093 	if (sc->admin_req_tag != NULL)  in mpi3mr_setup_admin_qpair()
1094 		bus_dma_tag_destroy(sc->admin_req_tag);  in mpi3mr_setup_admin_qpair()
1108 static void mpi3mr_print_fault_info(struct mpi3mr_softc *sc)  in mpi3mr_print_fault_info()  argument
1112 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_print_fault_info()
1115 		code = mpi3mr_regread(sc, MPI3_SYSIF_FAULT_OFFSET) &  in mpi3mr_print_fault_info()
1117 		code1 = mpi3mr_regread(sc, MPI3_SYSIF_FAULT_INFO0_OFFSET);  in mpi3mr_print_fault_info()
1118 		code2 = mpi3mr_regread(sc, MPI3_SYSIF_FAULT_INFO1_OFFSET);  in mpi3mr_print_fault_info()
1119 		code3 = mpi3mr_regread(sc, MPI3_SYSIF_FAULT_INFO2_OFFSET);  in mpi3mr_print_fault_info()
1121 		    sc->name, code, code1, code2, code3);  in mpi3mr_print_fault_info()
1125 enum mpi3mr_iocstate mpi3mr_get_iocstate(struct mpi3mr_softc *sc)  in mpi3mr_get_iocstate()  argument
1130 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_get_iocstate()
1131 	ioc_control = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_get_iocstate()
1133 	if(sc->unrecoverable)  in mpi3mr_get_iocstate()
1151 static inline void mpi3mr_clear_resethistory(struct mpi3mr_softc *sc)  in mpi3mr_clear_resethistory()  argument
1155 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_clear_resethistory()
1157 		mpi3mr_regwrite(sc, MPI3_SYSIF_IOC_STATUS_OFFSET, ioc_status);  in mpi3mr_clear_resethistory()
1171 static int mpi3mr_mur_ioc(struct mpi3mr_softc *sc, U32 reset_reason)  in mpi3mr_mur_ioc()  argument
1176         mpi3mr_dprint(sc, MPI3MR_INFO, "Issuing Message Unit Reset(MUR)\n");  in mpi3mr_mur_ioc()
1177         if (sc->unrecoverable) {  in mpi3mr_mur_ioc()
1178                 mpi3mr_dprint(sc, MPI3MR_ERROR, "IOC is unrecoverable MUR not issued\n");  in mpi3mr_mur_ioc()
1181         mpi3mr_clear_resethistory(sc);  in mpi3mr_mur_ioc()
1182 	mpi3mr_regwrite(sc, MPI3_SYSIF_SCRATCHPAD0_OFFSET, reset_reason);  in mpi3mr_mur_ioc()
1183 	ioc_config = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_mur_ioc()
1185 	mpi3mr_regwrite(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET, ioc_config);  in mpi3mr_mur_ioc()
1189 		ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_mur_ioc()
1191                         mpi3mr_clear_resethistory(sc);  in mpi3mr_mur_ioc()
1193 				mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_mur_ioc()
1204 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_mur_ioc()
1205 	ioc_config = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_mur_ioc()
1207         mpi3mr_dprint(sc, MPI3MR_INFO, "IOC Status/Config after %s MUR is (0x%x)/(0x%x)\n",  in mpi3mr_mur_ioc()
1221 static int mpi3mr_bring_ioc_ready(struct mpi3mr_softc *sc)  in mpi3mr_bring_ioc_ready()  argument
1226 	ioc_config = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_bring_ioc_ready()
1228 	mpi3mr_regwrite(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET, ioc_config);  in mpi3mr_bring_ioc_ready()
1230         timeout = sc->ready_timeout * 10;  in mpi3mr_bring_ioc_ready()
1232                 current_state = mpi3mr_get_iocstate(sc);  in mpi3mr_bring_ioc_ready()
1404 static inline bool mpi3mr_diagfault_success(struct mpi3mr_softc *sc,  in mpi3mr_diagfault_success()  argument
1411 	fault = mpi3mr_regread(sc, MPI3_SYSIF_FAULT_OFFSET) & MPI3_SYSIF_FAULT_CODE_MASK;  in mpi3mr_diagfault_success()
1427 static int mpi3mr_issue_iocfacts(struct mpi3mr_softc *sc,  in mpi3mr_issue_iocfacts()  argument
1449         if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_issue_iocfacts()
1460 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate request DMA tag\n");  in mpi3mr_issue_iocfacts()
1466 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Func: %s line: %d Data  DMA mem alloc failed\n",  in mpi3mr_issue_iocfacts()
1474 	mpi3mr_dprint(sc, MPI3MR_XINFO, "Func: %s line: %d IOCfacts data phys addr= %#016jx size= %d\n",  in mpi3mr_issue_iocfacts()
1481 		    sc->name);  in mpi3mr_issue_iocfacts()
1485 	mtx_lock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_iocfacts()
1488 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_iocfacts()
1491 		    sc->name);  in mpi3mr_issue_iocfacts()
1492 		mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_iocfacts()
1496 	sc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_iocfacts()
1497 	sc->init_cmds.is_waiting = 1;  in mpi3mr_issue_iocfacts()
1498 	sc->init_cmds.callback = NULL;  in mpi3mr_issue_iocfacts()
1505 	init_completion(&sc->init_cmds.completion);  in mpi3mr_issue_iocfacts()
1507 	retval = mpi3mr_submit_admin_cmd(sc, &iocfacts_req,  in mpi3mr_issue_iocfacts()
1512 		    sc->name);  in mpi3mr_issue_iocfacts()
1516 	wait_for_completion_timeout(&sc->init_cmds.completion,  in mpi3mr_issue_iocfacts()
1518 	if (!(sc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_iocfacts()
1520 		    sc->name);  in mpi3mr_issue_iocfacts()
1521 		mpi3mr_check_rh_fault_ioc(sc,  in mpi3mr_issue_iocfacts()
1523 		sc->unrecoverable = 1;  in mpi3mr_issue_iocfacts()
1528 	if ((sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_issue_iocfacts()
1531 		    " Loginfo(0x%08x) \n" , sc->name,  in mpi3mr_issue_iocfacts()
1532 		    (sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_issue_iocfacts()
1533 		    sc->init_cmds.ioc_loginfo);  in mpi3mr_issue_iocfacts()
1540 	sc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_iocfacts()
1541 	mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_iocfacts()
1563 static int mpi3mr_process_factsdata(struct mpi3mr_softc *sc,  in mpi3mr_process_factsdata()  argument
1572 		mpi3mr_dprint(sc, MPI3MR_INFO, "IOCFacts data length mismatch "  in mpi3mr_process_factsdata()
1578 	ioc_config = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_process_factsdata()
1583 		 mpi3mr_dprint(sc, MPI3MR_INFO, "IOCFacts data reqFrameSize mismatch "  in mpi3mr_process_factsdata()
1588 	memset(&sc->facts, 0, sizeof(sc->facts));  in mpi3mr_process_factsdata()
1591 	sc->facts.op_req_sz = req_sz;  in mpi3mr_process_factsdata()
1592 	sc->op_reply_sz = 1 << ((ioc_config &  in mpi3mr_process_factsdata()
1596 	sc->facts.ioc_num = facts_data->IOCNumber;  in mpi3mr_process_factsdata()
1597         sc->facts.who_init = facts_data->WhoInit;  in mpi3mr_process_factsdata()
1598         sc->facts.max_msix_vectors = facts_data->MaxMSIxVectors;  in mpi3mr_process_factsdata()
1599 	sc->facts.personality = (facts_flags &  in mpi3mr_process_factsdata()
1601 	sc->facts.dma_mask = (facts_flags &  in mpi3mr_process_factsdata()
1604         sc->facts.protocol_flags = facts_data->ProtocolFlags;  in mpi3mr_process_factsdata()
1605         sc->facts.mpi_version = (facts_data->MPIVersion.Word);  in mpi3mr_process_factsdata()
1606         sc->facts.max_reqs = (facts_data->MaxOutstandingRequests);  in mpi3mr_process_factsdata()
1607         sc->facts.product_id = (facts_data->ProductID);  in mpi3mr_process_factsdata()
1608 	sc->facts.reply_sz = (facts_data->ReplyFrameSize) * 4;  in mpi3mr_process_factsdata()
1609         sc->facts.exceptions = (facts_data->IOCExceptions);  in mpi3mr_process_factsdata()
1610         sc->facts.max_perids = (facts_data->MaxPersistentID);  in mpi3mr_process_factsdata()
1611         sc->facts.max_vds = (facts_data->MaxVDs);  in mpi3mr_process_factsdata()
1612         sc->facts.max_hpds = (facts_data->MaxHostPDs);  in mpi3mr_process_factsdata()
1613         sc->facts.max_advhpds = (facts_data->MaxAdvHostPDs);  in mpi3mr_process_factsdata()
1614         sc->facts.max_raidpds = (facts_data->MaxRAIDPDs);  in mpi3mr_process_factsdata()
1615         sc->facts.max_nvme = (facts_data->MaxNVMe);  in mpi3mr_process_factsdata()
1616         sc->facts.max_pcieswitches =  in mpi3mr_process_factsdata()
1618         sc->facts.max_sasexpanders =  in mpi3mr_process_factsdata()
1620         sc->facts.max_sasinitiators =  in mpi3mr_process_factsdata()
1622         sc->facts.max_enclosures = (facts_data->MaxEnclosures);  in mpi3mr_process_factsdata()
1623         sc->facts.min_devhandle = (facts_data->MinDevHandle);  in mpi3mr_process_factsdata()
1624         sc->facts.max_devhandle = (facts_data->MaxDevHandle);  in mpi3mr_process_factsdata()
1625 	sc->facts.max_op_req_q =  in mpi3mr_process_factsdata()
1627 	sc->facts.max_op_reply_q =  in mpi3mr_process_factsdata()
1629         sc->facts.ioc_capabilities =  in mpi3mr_process_factsdata()
1631         sc->facts.fw_ver.build_num =  in mpi3mr_process_factsdata()
1633         sc->facts.fw_ver.cust_id =  in mpi3mr_process_factsdata()
1635         sc->facts.fw_ver.ph_minor = facts_data->FWVersion.PhaseMinor;  in mpi3mr_process_factsdata()
1636         sc->facts.fw_ver.ph_major = facts_data->FWVersion.PhaseMajor;  in mpi3mr_process_factsdata()
1637         sc->facts.fw_ver.gen_minor = facts_data->FWVersion.GenMinor;  in mpi3mr_process_factsdata()
1638         sc->facts.fw_ver.gen_major = facts_data->FWVersion.GenMajor;  in mpi3mr_process_factsdata()
1639         sc->max_msix_vectors = min(sc->max_msix_vectors,  in mpi3mr_process_factsdata()
1640             sc->facts.max_msix_vectors);  in mpi3mr_process_factsdata()
1641         sc->facts.sge_mod_mask = facts_data->SGEModifierMask;  in mpi3mr_process_factsdata()
1642         sc->facts.sge_mod_value = facts_data->SGEModifierValue;  in mpi3mr_process_factsdata()
1643         sc->facts.sge_mod_shift = facts_data->SGEModifierShift;  in mpi3mr_process_factsdata()
1644         sc->facts.shutdown_timeout =  in mpi3mr_process_factsdata()
1646 	sc->facts.max_dev_per_tg = facts_data->MaxDevicesPerThrottleGroup;  in mpi3mr_process_factsdata()
1647 	sc->facts.io_throttle_data_length =  in mpi3mr_process_factsdata()
1649 	sc->facts.max_io_throttle_group =  in mpi3mr_process_factsdata()
1651 	sc->facts.io_throttle_low = facts_data->IOThrottleLow;  in mpi3mr_process_factsdata()
1652 	sc->facts.io_throttle_high = facts_data->IOThrottleHigh;  in mpi3mr_process_factsdata()
1655 	if (sc->facts.io_throttle_data_length)  in mpi3mr_process_factsdata()
1656 		sc->io_throttle_data_length =  in mpi3mr_process_factsdata()
1657 		    (sc->facts.io_throttle_data_length * 2 * 4);  in mpi3mr_process_factsdata()
1660 		sc->io_throttle_data_length = MPI3MR_MAX_SECTORS + 2;  in mpi3mr_process_factsdata()
1662 	sc->io_throttle_high = (sc->facts.io_throttle_high * 2 * 1024);  in mpi3mr_process_factsdata()
1663 	sc->io_throttle_low = (sc->facts.io_throttle_low * 2 * 1024);  in mpi3mr_process_factsdata()
1665 	fwver = &sc->facts.fw_ver;  in mpi3mr_process_factsdata()
1666 	snprintf(sc->fw_version, sizeof(sc->fw_version),  in mpi3mr_process_factsdata()
1671 	mpi3mr_dprint(sc, MPI3MR_INFO, "ioc_num(%d), maxopQ(%d), maxopRepQ(%d), maxdh(%d),"  in mpi3mr_process_factsdata()
1673 	    sc->facts.ioc_num, sc->facts.max_op_req_q,  in mpi3mr_process_factsdata()
1674 	    sc->facts.max_op_reply_q, sc->facts.max_devhandle,  in mpi3mr_process_factsdata()
1675             sc->facts.max_reqs, sc->facts.min_devhandle,  in mpi3mr_process_factsdata()
1676             sc->facts.max_pds, sc->facts.max_msix_vectors,  in mpi3mr_process_factsdata()
1677             sc->facts.max_perids);  in mpi3mr_process_factsdata()
1678         mpi3mr_dprint(sc, MPI3MR_INFO, "SGEModMask 0x%x SGEModVal 0x%x SGEModShift 0x%x\n",  in mpi3mr_process_factsdata()
1679             sc->facts.sge_mod_mask, sc->facts.sge_mod_value,  in mpi3mr_process_factsdata()
1680             sc->facts.sge_mod_shift);  in mpi3mr_process_factsdata()
1681 	mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_process_factsdata()
1683 	    sc->facts.max_dev_per_tg, sc->facts.max_io_throttle_group,  in mpi3mr_process_factsdata()
1684 	    sc->facts.io_throttle_data_length * 4,  in mpi3mr_process_factsdata()
1685 	    sc->facts.io_throttle_high, sc->facts.io_throttle_low);  in mpi3mr_process_factsdata()
1687 	sc->max_host_ios = sc->facts.max_reqs -  in mpi3mr_process_factsdata()
1696 	if (sc->facts.dma_mask == 0 ||  in mpi3mr_process_factsdata()
1697 	    (sc->facts.dma_mask >= sizeof(bus_addr_t) * 8))  in mpi3mr_process_factsdata()
1698 		sc->dma_loaddr = BUS_SPACE_MAXADDR;  in mpi3mr_process_factsdata()
1700 		sc->dma_loaddr = ~((1ull << sc->facts.dma_mask) - 1);  in mpi3mr_process_factsdata()
1701 	mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_process_factsdata()
1703 	    sc->facts.dma_mask, sc->dma_loaddr);  in mpi3mr_process_factsdata()
1708 static inline void mpi3mr_setup_reply_free_queues(struct mpi3mr_softc *sc)  in mpi3mr_setup_reply_free_queues()  argument
1714 	for (i = 0, phys_addr = sc->reply_buf_phys;  in mpi3mr_setup_reply_free_queues()
1715 	    i < sc->num_reply_bufs; i++, phys_addr += sc->reply_sz)  in mpi3mr_setup_reply_free_queues()
1716 		sc->reply_free_q[i] = phys_addr;  in mpi3mr_setup_reply_free_queues()
1717 	sc->reply_free_q[i] = (0);  in mpi3mr_setup_reply_free_queues()
1720 	for (i = 0, phys_addr = sc->sense_buf_phys;  in mpi3mr_setup_reply_free_queues()
1721 	    i < sc->num_sense_bufs; i++, phys_addr += MPI3MR_SENSEBUF_SZ)  in mpi3mr_setup_reply_free_queues()
1722 		sc->sense_buf_q[i] = phys_addr;  in mpi3mr_setup_reply_free_queues()
1723 	sc->sense_buf_q[i] = (0);  in mpi3mr_setup_reply_free_queues()
1727 static int mpi3mr_reply_dma_alloc(struct mpi3mr_softc *sc)  in mpi3mr_reply_dma_alloc()  argument
1731 	sc->num_reply_bufs = sc->facts.max_reqs + MPI3MR_NUM_EVTREPLIES;  in mpi3mr_reply_dma_alloc()
1732 	sc->reply_free_q_sz = sc->num_reply_bufs + 1;  in mpi3mr_reply_dma_alloc()
1733 	sc->num_sense_bufs = sc->facts.max_reqs / MPI3MR_SENSEBUF_FACTOR;  in mpi3mr_reply_dma_alloc()
1734 	sc->sense_buf_q_sz = sc->num_sense_bufs + 1;  in mpi3mr_reply_dma_alloc()
1736 	sz = sc->num_reply_bufs * sc->reply_sz;  in mpi3mr_reply_dma_alloc()
1738 	if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,  /* parent */  in mpi3mr_reply_dma_alloc()
1740 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_reply_dma_alloc()
1748                                 &sc->reply_buf_tag)) {  in mpi3mr_reply_dma_alloc()
1749 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate request DMA tag\n");  in mpi3mr_reply_dma_alloc()
1753 	if (bus_dmamem_alloc(sc->reply_buf_tag, (void **)&sc->reply_buf,  in mpi3mr_reply_dma_alloc()
1754 	    BUS_DMA_NOWAIT, &sc->reply_buf_dmamap)) {  in mpi3mr_reply_dma_alloc()
1755 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Func: %s line: %d  DMA mem alloc failed\n",  in mpi3mr_reply_dma_alloc()
1760 	bzero(sc->reply_buf, sz);  in mpi3mr_reply_dma_alloc()
1761         bus_dmamap_load(sc->reply_buf_tag, sc->reply_buf_dmamap, sc->reply_buf, sz,  in mpi3mr_reply_dma_alloc()
1762 	    mpi3mr_memaddr_cb, &sc->reply_buf_phys, BUS_DMA_NOWAIT);  in mpi3mr_reply_dma_alloc()
1764 	sc->reply_buf_dma_min_address = sc->reply_buf_phys;  in mpi3mr_reply_dma_alloc()
1765 	sc->reply_buf_dma_max_address = sc->reply_buf_phys + sz;  in mpi3mr_reply_dma_alloc()
1766 	mpi3mr_dprint(sc, MPI3MR_XINFO, "reply buf (0x%p): depth(%d), frame_size(%d), "  in mpi3mr_reply_dma_alloc()
1768 	    sc->reply_buf, sc->num_reply_bufs, sc->reply_sz,  in mpi3mr_reply_dma_alloc()
1769 	    (sz / 1024), (unsigned long long)sc->reply_buf_phys);  in mpi3mr_reply_dma_alloc()
1772 	sz = sc->reply_free_q_sz * 8;  in mpi3mr_reply_dma_alloc()
1774         if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_reply_dma_alloc()
1776 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_reply_dma_alloc()
1784                                 &sc->reply_free_q_tag)) {  in mpi3mr_reply_dma_alloc()
1785 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate reply free queue DMA tag\n");  in mpi3mr_reply_dma_alloc()
1789         if (bus_dmamem_alloc(sc->reply_free_q_tag, (void **)&sc->reply_free_q,  in mpi3mr_reply_dma_alloc()
1790 	    BUS_DMA_NOWAIT, &sc->reply_free_q_dmamap)) {  in mpi3mr_reply_dma_alloc()
1791 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Func: %s line: %d  DMA mem alloc failed\n",  in mpi3mr_reply_dma_alloc()
1796 	bzero(sc->reply_free_q, sz);  in mpi3mr_reply_dma_alloc()
1797         bus_dmamap_load(sc->reply_free_q_tag, sc->reply_free_q_dmamap, sc->reply_free_q, sz,  in mpi3mr_reply_dma_alloc()
1798 	    mpi3mr_memaddr_cb, &sc->reply_free_q_phys, BUS_DMA_NOWAIT);  in mpi3mr_reply_dma_alloc()
1800 	mpi3mr_dprint(sc, MPI3MR_XINFO, "reply_free_q (0x%p): depth(%d), frame_size(%d), "  in mpi3mr_reply_dma_alloc()
1802 	    sc->reply_free_q, sc->reply_free_q_sz, 8, (sz / 1024),  in mpi3mr_reply_dma_alloc()
1803 	    (unsigned long long)sc->reply_free_q_phys);  in mpi3mr_reply_dma_alloc()
1806 	sz = sc->num_sense_bufs * MPI3MR_SENSEBUF_SZ;  in mpi3mr_reply_dma_alloc()
1808         if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_reply_dma_alloc()
1810 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_reply_dma_alloc()
1818                                 &sc->sense_buf_tag)) {  in mpi3mr_reply_dma_alloc()
1819 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate Sense buffer DMA tag\n");  in mpi3mr_reply_dma_alloc()
1823 	if (bus_dmamem_alloc(sc->sense_buf_tag, (void **)&sc->sense_buf,  in mpi3mr_reply_dma_alloc()
1824 	    BUS_DMA_NOWAIT, &sc->sense_buf_dmamap)) {  in mpi3mr_reply_dma_alloc()
1825 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Func: %s line: %d  DMA mem alloc failed\n",  in mpi3mr_reply_dma_alloc()
1830 	bzero(sc->sense_buf, sz);  in mpi3mr_reply_dma_alloc()
1831         bus_dmamap_load(sc->sense_buf_tag, sc->sense_buf_dmamap, sc->sense_buf, sz,  in mpi3mr_reply_dma_alloc()
1832 	    mpi3mr_memaddr_cb, &sc->sense_buf_phys, BUS_DMA_NOWAIT);  in mpi3mr_reply_dma_alloc()
1834 	mpi3mr_dprint(sc, MPI3MR_XINFO, "sense_buf (0x%p): depth(%d), frame_size(%d), "  in mpi3mr_reply_dma_alloc()
1836 	    sc->sense_buf, sc->num_sense_bufs, MPI3MR_SENSEBUF_SZ,  in mpi3mr_reply_dma_alloc()
1837 	    (sz / 1024), (unsigned long long)sc->sense_buf_phys);  in mpi3mr_reply_dma_alloc()
1840 	sz = sc->sense_buf_q_sz * 8;  in mpi3mr_reply_dma_alloc()
1842         if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_reply_dma_alloc()
1844 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_reply_dma_alloc()
1852                                 &sc->sense_buf_q_tag)) {  in mpi3mr_reply_dma_alloc()
1853 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate Sense buffer Queue DMA tag\n");  in mpi3mr_reply_dma_alloc()
1857 	if (bus_dmamem_alloc(sc->sense_buf_q_tag, (void **)&sc->sense_buf_q,  in mpi3mr_reply_dma_alloc()
1858 	    BUS_DMA_NOWAIT, &sc->sense_buf_q_dmamap)) {  in mpi3mr_reply_dma_alloc()
1859 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Func: %s line: %d  DMA mem alloc failed\n",  in mpi3mr_reply_dma_alloc()
1864 	bzero(sc->sense_buf_q, sz);  in mpi3mr_reply_dma_alloc()
1865         bus_dmamap_load(sc->sense_buf_q_tag, sc->sense_buf_q_dmamap, sc->sense_buf_q, sz,  in mpi3mr_reply_dma_alloc()
1866 	    mpi3mr_memaddr_cb, &sc->sense_buf_q_phys, BUS_DMA_NOWAIT);  in mpi3mr_reply_dma_alloc()
1868 	mpi3mr_dprint(sc, MPI3MR_XINFO, "sense_buf_q (0x%p): depth(%d), frame_size(%d), "  in mpi3mr_reply_dma_alloc()
1870 	    sc->sense_buf_q, sc->sense_buf_q_sz, 8, (sz / 1024),  in mpi3mr_reply_dma_alloc()
1871 	    (unsigned long long)sc->sense_buf_q_phys);  in mpi3mr_reply_dma_alloc()
1876 static int mpi3mr_reply_alloc(struct mpi3mr_softc *sc)  in mpi3mr_reply_alloc()  argument
1881 	if (sc->init_cmds.reply)  in mpi3mr_reply_alloc()
1884 	sc->init_cmds.reply = malloc(sc->reply_sz,  in mpi3mr_reply_alloc()
1887 	if (!sc->init_cmds.reply) {  in mpi3mr_reply_alloc()
1889 		    sc->name);  in mpi3mr_reply_alloc()
1893 	sc->ioctl_cmds.reply = malloc(sc->reply_sz, M_MPI3MR, M_NOWAIT | M_ZERO);  in mpi3mr_reply_alloc()
1894 	if (!sc->ioctl_cmds.reply) {  in mpi3mr_reply_alloc()
1896 		    sc->name);  in mpi3mr_reply_alloc()
1900 	sc->host_tm_cmds.reply = malloc(sc->reply_sz, M_MPI3MR, M_NOWAIT | M_ZERO);  in mpi3mr_reply_alloc()
1901 	if (!sc->host_tm_cmds.reply) {  in mpi3mr_reply_alloc()
1903 		    sc->name);  in mpi3mr_reply_alloc()
1907 		sc->dev_rmhs_cmds[i].reply = malloc(sc->reply_sz,  in mpi3mr_reply_alloc()
1909 		if (!sc->dev_rmhs_cmds[i].reply) {  in mpi3mr_reply_alloc()
1912 			    sc->name, i);  in mpi3mr_reply_alloc()
1918 		sc->evtack_cmds[i].reply = malloc(sc->reply_sz,  in mpi3mr_reply_alloc()
1920 		if (!sc->evtack_cmds[i].reply)  in mpi3mr_reply_alloc()
1924 	sc->dev_handle_bitmap_sz = MPI3MR_DIV_ROUND_UP(sc->facts.max_devhandle, 8);  in mpi3mr_reply_alloc()
1926 	sc->removepend_bitmap = malloc(sc->dev_handle_bitmap_sz,  in mpi3mr_reply_alloc()
1928 	if (!sc->removepend_bitmap) {  in mpi3mr_reply_alloc()
1930 		    sc->name);  in mpi3mr_reply_alloc()
1934 	sc->devrem_bitmap_sz = MPI3MR_DIV_ROUND_UP(MPI3MR_NUM_DEVRMCMD, 8);  in mpi3mr_reply_alloc()
1935 	sc->devrem_bitmap = malloc(sc->devrem_bitmap_sz,  in mpi3mr_reply_alloc()
1937 	if (!sc->devrem_bitmap) {  in mpi3mr_reply_alloc()
1939 		    sc->name);  in mpi3mr_reply_alloc()
1943 	sc->evtack_cmds_bitmap_sz = MPI3MR_DIV_ROUND_UP(MPI3MR_NUM_EVTACKCMD, 8);  in mpi3mr_reply_alloc()
1945 	sc->evtack_cmds_bitmap = malloc(sc->evtack_cmds_bitmap_sz,  in mpi3mr_reply_alloc()
1947 	if (!sc->evtack_cmds_bitmap)  in mpi3mr_reply_alloc()
1950 	if (mpi3mr_reply_dma_alloc(sc)) {  in mpi3mr_reply_alloc()
1952 		    sc->name, __func__, __LINE__);  in mpi3mr_reply_alloc()
1957 	mpi3mr_setup_reply_free_queues(sc);  in mpi3mr_reply_alloc()
1960 	mpi3mr_cleanup_interrupts(sc);  in mpi3mr_reply_alloc()
1961 	mpi3mr_free_mem(sc);  in mpi3mr_reply_alloc()
1967 mpi3mr_print_fw_pkg_ver(struct mpi3mr_softc *sc)  in mpi3mr_print_fw_pkg_ver()  argument
1979 	if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,  /* parent */  in mpi3mr_print_fw_pkg_ver()
1981 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_print_fw_pkg_ver()
1990 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate fw package version request DMA tag\n");  in mpi3mr_print_fw_pkg_ver()
1995 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Func: %s line: %d fw package version DMA mem alloc failed\n",  in mpi3mr_print_fw_pkg_ver()
2005 …mpi3mr_dprint(sc, MPI3MR_XINFO, "Func: %s line: %d fw package version phys addr= %#016jx size= %d\…  in mpi3mr_print_fw_pkg_ver()
2009 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Memory alloc for fw package version failed\n");  in mpi3mr_print_fw_pkg_ver()
2014 	mtx_lock(&sc->init_cmds.completion.lock);  in mpi3mr_print_fw_pkg_ver()
2015 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_print_fw_pkg_ver()
2016 		mpi3mr_dprint(sc, MPI3MR_INFO,"Issue CI Header Upload: command is in use\n");  in mpi3mr_print_fw_pkg_ver()
2017 		mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_print_fw_pkg_ver()
2020 	sc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_print_fw_pkg_ver()
2021 	sc->init_cmds.is_waiting = 1;  in mpi3mr_print_fw_pkg_ver()
2022 	sc->init_cmds.callback = NULL;  in mpi3mr_print_fw_pkg_ver()
2032 	init_completion(&sc->init_cmds.completion);  in mpi3mr_print_fw_pkg_ver()
2033 	if ((retval = mpi3mr_submit_admin_cmd(sc, &ci_upload, sizeof(ci_upload)))) {  in mpi3mr_print_fw_pkg_ver()
2034 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Issue CI Header Upload: Admin Post failed\n");  in mpi3mr_print_fw_pkg_ver()
2037 	wait_for_completion_timeout(&sc->init_cmds.completion,  in mpi3mr_print_fw_pkg_ver()
2039 	if (!(sc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_print_fw_pkg_ver()
2040 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Issue CI Header Upload: command timed out\n");  in mpi3mr_print_fw_pkg_ver()
2041 		sc->init_cmds.is_waiting = 0;  in mpi3mr_print_fw_pkg_ver()
2042 		if (!(sc->init_cmds.state & MPI3MR_CMD_RESET))  in mpi3mr_print_fw_pkg_ver()
2043 			mpi3mr_check_rh_fault_ioc(sc,  in mpi3mr_print_fw_pkg_ver()
2047 	if ((GET_IOC_STATUS(sc->init_cmds.ioc_status)) != MPI3_IOCSTATUS_SUCCESS) {  in mpi3mr_print_fw_pkg_ver()
2048 		mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_print_fw_pkg_ver()
2050 			      GET_IOC_STATUS(sc->init_cmds.ioc_status), sc->init_cmds.ioc_loginfo);  in mpi3mr_print_fw_pkg_ver()
2055 	mpi3mr_dprint(sc, MPI3MR_XINFO,  in mpi3mr_print_fw_pkg_ver()
2061 	mpi3mr_dprint(sc, MPI3MR_INFO, "FW Package Version: %02d.%02d.%02d.%02d\n",  in mpi3mr_print_fw_pkg_ver()
2067 	sc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_print_fw_pkg_ver()
2068 	mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_print_fw_pkg_ver()
2089 static int mpi3mr_issue_iocinit(struct mpi3mr_softc *sc)  in mpi3mr_issue_iocinit()  argument
2101 	if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,  /* parent */  in mpi3mr_issue_iocinit()
2103 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_issue_iocinit()
2112 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate request DMA tag\n");  in mpi3mr_issue_iocinit()
2118 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Func: %s line: %d Data  DMA mem alloc failed\n",  in mpi3mr_issue_iocinit()
2126 …mpi3mr_dprint(sc, MPI3MR_XINFO, "Func: %s line: %d IOCfacts drvr_info phys addr= %#016jx size= %d\…  in mpi3mr_issue_iocinit()
2133 		    sc->name);  in mpi3mr_issue_iocinit()
2144 	memcpy((U8 *)&sc->driver_info, (U8 *)drvr_info, sizeof(sc->driver_info));  in mpi3mr_issue_iocinit()
2147 	mtx_lock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_iocinit()
2148 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_iocinit()
2151 		    sc->name);  in mpi3mr_issue_iocinit()
2152 		mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_iocinit()
2155 	sc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_iocinit()
2156 	sc->init_cmds.is_waiting = 1;  in mpi3mr_issue_iocinit()
2157 	sc->init_cmds.callback = NULL;  in mpi3mr_issue_iocinit()
2165         iocinit_req.ReplyFreeQueueDepth = sc->reply_free_q_sz;  in mpi3mr_issue_iocinit()
2167                 sc->reply_free_q_phys;  in mpi3mr_issue_iocinit()
2170                 sc->sense_buf_q_sz;  in mpi3mr_issue_iocinit()
2172                 sc->sense_buf_q_phys;  in mpi3mr_issue_iocinit()
2179 	init_completion(&sc->init_cmds.completion);  in mpi3mr_issue_iocinit()
2180 	retval = mpi3mr_submit_admin_cmd(sc, &iocinit_req,  in mpi3mr_issue_iocinit()
2185 		    sc->name);  in mpi3mr_issue_iocinit()
2189 	wait_for_completion_timeout(&sc->init_cmds.completion,  in mpi3mr_issue_iocinit()
2191 	if (!(sc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_iocinit()
2193 		    sc->name);  in mpi3mr_issue_iocinit()
2194 		mpi3mr_check_rh_fault_ioc(sc,  in mpi3mr_issue_iocinit()
2196 		sc->unrecoverable = 1;  in mpi3mr_issue_iocinit()
2201 	if ((sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_issue_iocinit()
2204 		    " Loginfo(0x%08x) \n" , sc->name,  in mpi3mr_issue_iocinit()
2205 		    (sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_issue_iocinit()
2206 		    sc->init_cmds.ioc_loginfo);  in mpi3mr_issue_iocinit()
2212 	sc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_iocinit()
2213 	mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_iocinit()
2226 mpi3mr_display_ioc_info(struct mpi3mr_softc *sc)  in mpi3mr_display_ioc_info()  argument
2231         switch (sc->facts.personality) {  in mpi3mr_display_ioc_info()
2243 	mpi3mr_dprint(sc, MPI3MR_INFO, "Current Personality: %s\n", personality);  in mpi3mr_display_ioc_info()
2245 	mpi3mr_dprint(sc, MPI3MR_INFO, "%s\n", sc->fw_version);  in mpi3mr_display_ioc_info()
2247         mpi3mr_dprint(sc, MPI3MR_INFO, "Protocol=(");  in mpi3mr_display_ioc_info()
2249         if (sc->facts.protocol_flags &  in mpi3mr_display_ioc_info()
2255         if (sc->facts.protocol_flags &  in mpi3mr_display_ioc_info()
2261         if (sc->facts.protocol_flags &  in mpi3mr_display_ioc_info()
2270         if (sc->facts.ioc_capabilities &  in mpi3mr_display_ioc_info()
2289 static void mpi3mr_unmask_events(struct mpi3mr_softc *sc, U16 event)  in mpi3mr_unmask_events()  argument
2299 		sc->event_masks[0] &= ~desired_event;  in mpi3mr_unmask_events()
2301 		sc->event_masks[1] &= ~desired_event;  in mpi3mr_unmask_events()
2303 		sc->event_masks[2] &= ~desired_event;  in mpi3mr_unmask_events()
2305 		sc->event_masks[3] &= ~desired_event;  in mpi3mr_unmask_events()
2308 static void mpi3mr_set_events_mask(struct mpi3mr_softc *sc)  in mpi3mr_set_events_mask()  argument
2312 		sc->event_masks[i] = -1;  in mpi3mr_set_events_mask()
2314         mpi3mr_unmask_events(sc, MPI3_EVENT_DEVICE_ADDED);  in mpi3mr_set_events_mask()
2315         mpi3mr_unmask_events(sc, MPI3_EVENT_DEVICE_INFO_CHANGED);  in mpi3mr_set_events_mask()
2316         mpi3mr_unmask_events(sc, MPI3_EVENT_DEVICE_STATUS_CHANGE);  in mpi3mr_set_events_mask()
2318         mpi3mr_unmask_events(sc, MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE);  in mpi3mr_set_events_mask()
2320         mpi3mr_unmask_events(sc, MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST);  in mpi3mr_set_events_mask()
2321         mpi3mr_unmask_events(sc, MPI3_EVENT_SAS_DISCOVERY);  in mpi3mr_set_events_mask()
2322         mpi3mr_unmask_events(sc, MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR);  in mpi3mr_set_events_mask()
2323         mpi3mr_unmask_events(sc, MPI3_EVENT_SAS_BROADCAST_PRIMITIVE);  in mpi3mr_set_events_mask()
2325         mpi3mr_unmask_events(sc, MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);  in mpi3mr_set_events_mask()
2326         mpi3mr_unmask_events(sc, MPI3_EVENT_PCIE_ENUMERATION);  in mpi3mr_set_events_mask()
2328         mpi3mr_unmask_events(sc, MPI3_EVENT_PREPARE_FOR_RESET);  in mpi3mr_set_events_mask()
2329         mpi3mr_unmask_events(sc, MPI3_EVENT_CABLE_MGMT);  in mpi3mr_set_events_mask()
2330         mpi3mr_unmask_events(sc, MPI3_EVENT_ENERGY_PACK_CHANGE);  in mpi3mr_set_events_mask()
2342 int mpi3mr_issue_event_notification(struct mpi3mr_softc *sc)  in mpi3mr_issue_event_notification()  argument
2349 	mtx_lock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_event_notification()
2350 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_event_notification()
2353 		    sc->name);  in mpi3mr_issue_event_notification()
2354 		mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_event_notification()
2357 	sc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_event_notification()
2358 	sc->init_cmds.is_waiting = 1;  in mpi3mr_issue_event_notification()
2359 	sc->init_cmds.callback = NULL;  in mpi3mr_issue_event_notification()
2364 		    (sc->event_masks[i]);  in mpi3mr_issue_event_notification()
2365 	init_completion(&sc->init_cmds.completion);  in mpi3mr_issue_event_notification()
2366 	retval = mpi3mr_submit_admin_cmd(sc, &evtnotify_req,  in mpi3mr_issue_event_notification()
2370 		    sc->name);  in mpi3mr_issue_event_notification()
2374 	poll_for_command_completion(sc,  in mpi3mr_issue_event_notification()
2375 				    &sc->init_cmds,  in mpi3mr_issue_event_notification()
2377 	if (!(sc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_event_notification()
2379 		    sc->name);  in mpi3mr_issue_event_notification()
2380 		mpi3mr_check_rh_fault_ioc(sc,  in mpi3mr_issue_event_notification()
2386 	if ((sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_issue_event_notification()
2389 		    " Loginfo(0x%08x) \n" , sc->name,  in mpi3mr_issue_event_notification()
2390 		    (sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_issue_event_notification()
2391 		    sc->init_cmds.ioc_loginfo);  in mpi3mr_issue_event_notification()
2397 	sc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_event_notification()
2398 	mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_event_notification()
2405 mpi3mr_register_events(struct mpi3mr_softc *sc)  in mpi3mr_register_events()  argument
2409 	mpi3mr_set_events_mask(sc);  in mpi3mr_register_events()
2411 	error = mpi3mr_issue_event_notification(sc);  in mpi3mr_register_events()
2415 		    sc->name, error);  in mpi3mr_register_events()
2432 int mpi3mr_process_event_ack(struct mpi3mr_softc *sc, U8 event,  in mpi3mr_process_event_ack()  argument
2439 	mtx_lock(&sc->init_cmds.completion.lock);  in mpi3mr_process_event_ack()
2440 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_process_event_ack()
2443 		    sc->name);  in mpi3mr_process_event_ack()
2444 		mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_process_event_ack()
2447 	sc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_process_event_ack()
2448 	sc->init_cmds.is_waiting = 1;  in mpi3mr_process_event_ack()
2449 	sc->init_cmds.callback = NULL;  in mpi3mr_process_event_ack()
2455 	init_completion(&sc->init_cmds.completion);  in mpi3mr_process_event_ack()
2456 	retval = mpi3mr_submit_admin_cmd(sc, &evtack_req,  in mpi3mr_process_event_ack()
2460 		    sc->name);  in mpi3mr_process_event_ack()
2464 	wait_for_completion_timeout(&sc->init_cmds.completion,  in mpi3mr_process_event_ack()
2466 	if (!(sc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_process_event_ack()
2468 		    sc->name);  in mpi3mr_process_event_ack()
2473 	if ((sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_process_event_ack()
2476 		    " Loginfo(0x%08x) \n" , sc->name,  in mpi3mr_process_event_ack()
2477 		    (sc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_process_event_ack()
2478 		    sc->init_cmds.ioc_loginfo);  in mpi3mr_process_event_ack()
2484 	sc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_process_event_ack()
2485 	mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_process_event_ack()
2492 static int mpi3mr_alloc_chain_bufs(struct mpi3mr_softc *sc)  in mpi3mr_alloc_chain_bufs()  argument
2498 	num_chains = sc->max_host_ios;  in mpi3mr_alloc_chain_bufs()
2500 	sc->chain_buf_count = num_chains;  in mpi3mr_alloc_chain_bufs()
2503 	sc->chain_sgl_list = malloc(sz, M_MPI3MR, M_NOWAIT | M_ZERO);  in mpi3mr_alloc_chain_bufs()
2505 	if (!sc->chain_sgl_list) {  in mpi3mr_alloc_chain_bufs()
2507 		    sc->name);  in mpi3mr_alloc_chain_bufs()
2514         if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,  /* parent */  in mpi3mr_alloc_chain_bufs()
2516 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_alloc_chain_bufs()
2524                                 &sc->chain_sgl_list_tag)) {  in mpi3mr_alloc_chain_bufs()
2525 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate Chain buffer DMA tag\n");  in mpi3mr_alloc_chain_bufs()
2530 		if (bus_dmamem_alloc(sc->chain_sgl_list_tag, (void **)&sc->chain_sgl_list[i].buf,  in mpi3mr_alloc_chain_bufs()
2531 		    BUS_DMA_NOWAIT, &sc->chain_sgl_list[i].buf_dmamap)) {  in mpi3mr_alloc_chain_bufs()
2532 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Func: %s line: %d  DMA mem alloc failed\n",  in mpi3mr_alloc_chain_bufs()
2537 		bzero(sc->chain_sgl_list[i].buf, sz);  in mpi3mr_alloc_chain_bufs()
2538 …bus_dmamap_load(sc->chain_sgl_list_tag, sc->chain_sgl_list[i].buf_dmamap, sc->chain_sgl_list[i].bu…  in mpi3mr_alloc_chain_bufs()
2539 		    mpi3mr_memaddr_cb, &sc->chain_sgl_list[i].buf_phys, BUS_DMA_NOWAIT);  in mpi3mr_alloc_chain_bufs()
2540 		mpi3mr_dprint(sc, MPI3MR_XINFO, "Func: %s line: %d phys addr= %#016jx size= %d\n",  in mpi3mr_alloc_chain_bufs()
2541 		    __func__, __LINE__, (uintmax_t)sc->chain_sgl_list[i].buf_phys, sz);  in mpi3mr_alloc_chain_bufs()
2544 	sc->chain_bitmap_sz = MPI3MR_DIV_ROUND_UP(num_chains, 8);  in mpi3mr_alloc_chain_bufs()
2546 	sc->chain_bitmap = malloc(sc->chain_bitmap_sz, M_MPI3MR, M_NOWAIT | M_ZERO);  in mpi3mr_alloc_chain_bufs()
2547 	if (!sc->chain_bitmap) {  in mpi3mr_alloc_chain_bufs()
2548 		mpi3mr_dprint(sc, MPI3MR_INFO, "Cannot alloc memory for chain bitmap\n");  in mpi3mr_alloc_chain_bufs()
2556 		if (sc->chain_sgl_list[i].buf_phys != 0)  in mpi3mr_alloc_chain_bufs()
2557 			bus_dmamap_unload(sc->chain_sgl_list_tag, sc->chain_sgl_list[i].buf_dmamap);  in mpi3mr_alloc_chain_bufs()
2558 		if (sc->chain_sgl_list[i].buf != NULL)  in mpi3mr_alloc_chain_bufs()
2559 …bus_dmamem_free(sc->chain_sgl_list_tag, sc->chain_sgl_list[i].buf, sc->chain_sgl_list[i].buf_dmama…  in mpi3mr_alloc_chain_bufs()
2561 	if (sc->chain_sgl_list_tag != NULL)  in mpi3mr_alloc_chain_bufs()
2562 		bus_dma_tag_destroy(sc->chain_sgl_list_tag);  in mpi3mr_alloc_chain_bufs()
2566 static int mpi3mr_pel_alloc(struct mpi3mr_softc *sc)  in mpi3mr_pel_alloc()  argument
2570 	if (!sc->pel_cmds.reply) {  in mpi3mr_pel_alloc()
2571 		sc->pel_cmds.reply = malloc(sc->reply_sz, M_MPI3MR, M_NOWAIT | M_ZERO);  in mpi3mr_pel_alloc()
2572 		if (!sc->pel_cmds.reply) {  in mpi3mr_pel_alloc()
2574 			    sc->name);  in mpi3mr_pel_alloc()
2579 	if (!sc->pel_abort_cmd.reply) {  in mpi3mr_pel_alloc()
2580 		sc->pel_abort_cmd.reply = malloc(sc->reply_sz, M_MPI3MR, M_NOWAIT | M_ZERO);  in mpi3mr_pel_alloc()
2581 		if (!sc->pel_abort_cmd.reply) {  in mpi3mr_pel_alloc()
2583 			    sc->name);  in mpi3mr_pel_alloc()
2588 	if (!sc->pel_seq_number) {  in mpi3mr_pel_alloc()
2589 		sc->pel_seq_number_sz = sizeof(Mpi3PELSeq_t);  in mpi3mr_pel_alloc()
2590 		if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,   /* parent */  in mpi3mr_pel_alloc()
2592 				 sc->dma_loaddr,	         /* lowaddr */  in mpi3mr_pel_alloc()
2595 				 sc->pel_seq_number_sz,		 /* maxsize */  in mpi3mr_pel_alloc()
2597 				 sc->pel_seq_number_sz,          /* maxsegsize */  in mpi3mr_pel_alloc()
2600 				 &sc->pel_seq_num_dmatag)) {  in mpi3mr_pel_alloc()
2601 			 mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot create PEL seq number dma memory tag\n");  in mpi3mr_pel_alloc()
2606 		if (bus_dmamem_alloc(sc->pel_seq_num_dmatag, (void **)&sc->pel_seq_number,  in mpi3mr_pel_alloc()
2607 		    BUS_DMA_NOWAIT, &sc->pel_seq_num_dmamap)) {  in mpi3mr_pel_alloc()
2608 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate PEL seq number kernel buffer dma memory\n");  in mpi3mr_pel_alloc()
2613 		bzero(sc->pel_seq_number, sc->pel_seq_number_sz);  in mpi3mr_pel_alloc()
2615 		bus_dmamap_load(sc->pel_seq_num_dmatag, sc->pel_seq_num_dmamap, sc->pel_seq_number,  in mpi3mr_pel_alloc()
2616 		    sc->pel_seq_number_sz, mpi3mr_memaddr_cb, &sc->pel_seq_number_dma, BUS_DMA_NOWAIT);  in mpi3mr_pel_alloc()
2618 		if (!sc->pel_seq_number) {  in mpi3mr_pel_alloc()
2619 			printf(IOCNAME "%s:%d Cannot load PEL seq number dma memory for size: %d\n", sc->name,  in mpi3mr_pel_alloc()
2620 				__func__, __LINE__, sc->pel_seq_number_sz);  in mpi3mr_pel_alloc()
2638 mpi3mr_validate_fw_update(struct mpi3mr_softc *sc)  in mpi3mr_validate_fw_update()  argument
2643 	if (sc->facts.reply_sz > sc->reply_sz) {  in mpi3mr_validate_fw_update()
2644 		mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_validate_fw_update()
2646 		    sc->reply_sz, sc->reply_sz);  in mpi3mr_validate_fw_update()
2650 	if (sc->num_io_throttle_group != sc->facts.max_io_throttle_group) {  in mpi3mr_validate_fw_update()
2651 		mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_validate_fw_update()
2653 		    sc->num_io_throttle_group,  in mpi3mr_validate_fw_update()
2654 		    sc->facts.max_io_throttle_group);  in mpi3mr_validate_fw_update()
2658 	if (sc->facts.max_op_reply_q < sc->num_queues) {  in mpi3mr_validate_fw_update()
2659 		mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_validate_fw_update()
2661 		    sc->num_queues,  in mpi3mr_validate_fw_update()
2662 		    sc->facts.max_op_reply_q);  in mpi3mr_validate_fw_update()
2666 	if (sc->facts.max_op_req_q < sc->num_queues) {  in mpi3mr_validate_fw_update()
2667 		mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_validate_fw_update()
2669 		    sc->num_queues, sc->facts.max_op_req_q);  in mpi3mr_validate_fw_update()
2673 	dev_handle_bitmap_sz = MPI3MR_DIV_ROUND_UP(sc->facts.max_devhandle, 8);  in mpi3mr_validate_fw_update()
2675 	if (dev_handle_bitmap_sz > sc->dev_handle_bitmap_sz) {  in mpi3mr_validate_fw_update()
2676 		removepend_bitmap = realloc(sc->removepend_bitmap,  in mpi3mr_validate_fw_update()
2680 			mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_validate_fw_update()
2682 			    sc->dev_handle_bitmap_sz, dev_handle_bitmap_sz);  in mpi3mr_validate_fw_update()
2686 		memset(removepend_bitmap + sc->dev_handle_bitmap_sz, 0,  in mpi3mr_validate_fw_update()
2687 		    dev_handle_bitmap_sz - sc->dev_handle_bitmap_sz);  in mpi3mr_validate_fw_update()
2688 		sc->removepend_bitmap = removepend_bitmap;  in mpi3mr_validate_fw_update()
2689 		mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_validate_fw_update()
2691 		    sc->dev_handle_bitmap_sz, dev_handle_bitmap_sz);  in mpi3mr_validate_fw_update()
2692 		sc->dev_handle_bitmap_sz = dev_handle_bitmap_sz;  in mpi3mr_validate_fw_update()
2707 int mpi3mr_initialize_ioc(struct mpi3mr_softc *sc, U8 init_type)  in mpi3mr_initialize_ioc()  argument
2717 	sc->cpu_count = mp_ncpus;  in mpi3mr_initialize_ioc()
2719 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_initialize_ioc()
2720 	ioc_control = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_initialize_ioc()
2721 	ioc_info = mpi3mr_regread64(sc, MPI3_SYSIF_IOC_INFO_LOW_OFFSET);  in mpi3mr_initialize_ioc()
2723 	mpi3mr_dprint(sc, MPI3MR_INFO, "SOD ioc_status: 0x%x ioc_control: 0x%x "  in mpi3mr_initialize_ioc()
2727 	sc->ready_timeout =  in mpi3mr_initialize_ioc()
2731 	ioc_state = mpi3mr_get_iocstate(sc);  in mpi3mr_initialize_ioc()
2733 	mpi3mr_dprint(sc, MPI3MR_INFO, "IOC state: %s   IOC ready timeout: %d\n",  in mpi3mr_initialize_ioc()
2734 	    mpi3mr_iocstate_name(ioc_state), sc->ready_timeout);  in mpi3mr_initialize_ioc()
2738 		timeout = sc->ready_timeout * 10;  in mpi3mr_initialize_ioc()
2743 		ioc_state = mpi3mr_get_iocstate(sc);  in mpi3mr_initialize_ioc()
2744 		mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_initialize_ioc()
2750                 retval = mpi3mr_mur_ioc(sc, MPI3MR_RESET_FROM_BRINGUP);  in mpi3mr_initialize_ioc()
2752                         mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to MU reset IOC, error 0x%x\n",  in mpi3mr_initialize_ioc()
2755                 ioc_state = mpi3mr_get_iocstate(sc);  in mpi3mr_initialize_ioc()
2759                 mpi3mr_print_fault_info(sc);  in mpi3mr_initialize_ioc()
2760 		 mpi3mr_dprint(sc, MPI3MR_ERROR, "issuing soft reset to bring to reset state\n");  in mpi3mr_initialize_ioc()
2761                  retval = mpi3mr_issue_reset(sc,  in mpi3mr_initialize_ioc()
2765                         mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_initialize_ioc()
2772 	ioc_state = mpi3mr_get_iocstate(sc);  in mpi3mr_initialize_ioc()
2775 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot bring IOC to reset state\n");  in mpi3mr_initialize_ioc()
2779 	retval = mpi3mr_setup_admin_qpair(sc);  in mpi3mr_initialize_ioc()
2781 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to setup Admin queues, error 0x%x\n",  in mpi3mr_initialize_ioc()
2786 	retval = mpi3mr_bring_ioc_ready(sc);  in mpi3mr_initialize_ioc()
2788 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to bring IOC ready, error 0x%x\n",  in mpi3mr_initialize_ioc()
2794 		retval = mpi3mr_alloc_interrupts(sc, 1);  in mpi3mr_initialize_ioc()
2796 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to allocate interrupts, error 0x%x\n",  in mpi3mr_initialize_ioc()
2801 		retval = mpi3mr_setup_irqs(sc);  in mpi3mr_initialize_ioc()
2803 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to setup ISR, error 0x%x\n",  in mpi3mr_initialize_ioc()
2809 	mpi3mr_enable_interrupts(sc);  in mpi3mr_initialize_ioc()
2812 		mtx_init(&sc->mpi3mr_mtx, "SIM lock", NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2813 		mtx_init(&sc->io_lock, "IO lock", NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2814 		mtx_init(&sc->admin_req_lock, "Admin Request Queue lock", NULL, MTX_SPIN);  in mpi3mr_initialize_ioc()
2815 		mtx_init(&sc->reply_free_q_lock, "Reply free Queue lock", NULL, MTX_SPIN);  in mpi3mr_initialize_ioc()
2816 		mtx_init(&sc->sense_buf_q_lock, "Sense buffer Queue lock", NULL, MTX_SPIN);  in mpi3mr_initialize_ioc()
2817 		mtx_init(&sc->chain_buf_lock, "Chain buffer lock", NULL, MTX_SPIN);  in mpi3mr_initialize_ioc()
2818 		mtx_init(&sc->cmd_pool_lock, "Command pool lock", NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2819 		mtx_init(&sc->fwevt_lock, "Firmware Event lock", NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2820 		mtx_init(&sc->target_lock, "Target lock", NULL, MTX_SPIN);  in mpi3mr_initialize_ioc()
2821 		mtx_init(&sc->reset_mutex, "Reset lock", NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2823 		mtx_init(&sc->init_cmds.completion.lock, "Init commands lock", NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2824 		sc->init_cmds.reply = NULL;  in mpi3mr_initialize_ioc()
2825 		sc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_initialize_ioc()
2826 		sc->init_cmds.dev_handle = MPI3MR_INVALID_DEV_HANDLE;  in mpi3mr_initialize_ioc()
2827 		sc->init_cmds.host_tag = MPI3MR_HOSTTAG_INITCMDS;  in mpi3mr_initialize_ioc()
2829 		mtx_init(&sc->ioctl_cmds.completion.lock, "IOCTL commands lock", NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2830 		sc->ioctl_cmds.reply = NULL;  in mpi3mr_initialize_ioc()
2831 		sc->ioctl_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_initialize_ioc()
2832 		sc->ioctl_cmds.dev_handle = MPI3MR_INVALID_DEV_HANDLE;  in mpi3mr_initialize_ioc()
2833 		sc->ioctl_cmds.host_tag = MPI3MR_HOSTTAG_IOCTLCMDS;  in mpi3mr_initialize_ioc()
2835 		mtx_init(&sc->pel_abort_cmd.completion.lock, "PEL Abort command lock", NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2836 		sc->pel_abort_cmd.reply = NULL;  in mpi3mr_initialize_ioc()
2837 		sc->pel_abort_cmd.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_initialize_ioc()
2838 		sc->pel_abort_cmd.dev_handle = MPI3MR_INVALID_DEV_HANDLE;  in mpi3mr_initialize_ioc()
2839 		sc->pel_abort_cmd.host_tag = MPI3MR_HOSTTAG_PELABORT;  in mpi3mr_initialize_ioc()
2841 		mtx_init(&sc->host_tm_cmds.completion.lock, "TM commands lock", NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2842 		sc->host_tm_cmds.reply = NULL;  in mpi3mr_initialize_ioc()
2843 		sc->host_tm_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_initialize_ioc()
2844 		sc->host_tm_cmds.dev_handle = MPI3MR_INVALID_DEV_HANDLE;  in mpi3mr_initialize_ioc()
2845 		sc->host_tm_cmds.host_tag = MPI3MR_HOSTTAG_TMS;  in mpi3mr_initialize_ioc()
2847 		TAILQ_INIT(&sc->cmd_list_head);  in mpi3mr_initialize_ioc()
2848 		TAILQ_INIT(&sc->event_list);  in mpi3mr_initialize_ioc()
2849 		TAILQ_INIT(&sc->delayed_rmhs_list);  in mpi3mr_initialize_ioc()
2850 		TAILQ_INIT(&sc->delayed_evtack_cmds_list);  in mpi3mr_initialize_ioc()
2854 			mtx_init(&sc->dev_rmhs_cmds[i].completion.lock, str, NULL, MTX_DEF);  in mpi3mr_initialize_ioc()
2855 			sc->dev_rmhs_cmds[i].reply = NULL;  in mpi3mr_initialize_ioc()
2856 			sc->dev_rmhs_cmds[i].state = MPI3MR_CMD_NOTUSED;  in mpi3mr_initialize_ioc()
2857 			sc->dev_rmhs_cmds[i].dev_handle = MPI3MR_INVALID_DEV_HANDLE;  in mpi3mr_initialize_ioc()
2858 			sc->dev_rmhs_cmds[i].host_tag = MPI3MR_HOSTTAG_DEVRMCMD_MIN  in mpi3mr_initialize_ioc()
2863 	retval = mpi3mr_issue_iocfacts(sc, &facts_data);  in mpi3mr_initialize_ioc()
2865 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to Issue IOC Facts, retval: 0x%x\n",  in mpi3mr_initialize_ioc()
2870 	retval = mpi3mr_process_factsdata(sc, &facts_data);  in mpi3mr_initialize_ioc()
2872 		mpi3mr_dprint(sc, MPI3MR_ERROR, "IOC Facts data processing failedi, retval: 0x%x\n",  in mpi3mr_initialize_ioc()
2877 	sc->num_io_throttle_group = sc->facts.max_io_throttle_group;  in mpi3mr_initialize_ioc()
2878 	mpi3mr_atomic_set(&sc->pend_large_data_sz, 0);  in mpi3mr_initialize_ioc()
2881 		retval = mpi3mr_validate_fw_update(sc);  in mpi3mr_initialize_ioc()
2885 		sc->reply_sz = sc->facts.reply_sz;  in mpi3mr_initialize_ioc()
2888 	mpi3mr_display_ioc_info(sc);  in mpi3mr_initialize_ioc()
2890 	retval = mpi3mr_reply_alloc(sc);  in mpi3mr_initialize_ioc()
2892 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to allocated reply and sense buffers, retval: 0x%x\n",  in mpi3mr_initialize_ioc()
2898 		retval = mpi3mr_alloc_chain_bufs(sc);  in mpi3mr_initialize_ioc()
2900 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to allocated chain buffers, retval: 0x%x\n",  in mpi3mr_initialize_ioc()
2906 	retval = mpi3mr_issue_iocinit(sc);  in mpi3mr_initialize_ioc()
2908 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to Issue IOC Init, retval: 0x%x\n",  in mpi3mr_initialize_ioc()
2913 	mpi3mr_print_fw_pkg_ver(sc);  in mpi3mr_initialize_ioc()
2915 	sc->reply_free_q_host_index = sc->num_reply_bufs;  in mpi3mr_initialize_ioc()
2916 	mpi3mr_regwrite(sc, MPI3_SYSIF_REPLY_FREE_HOST_INDEX_OFFSET,  in mpi3mr_initialize_ioc()
2917 		sc->reply_free_q_host_index);  in mpi3mr_initialize_ioc()
2919 	sc->sense_buf_q_host_index = sc->num_sense_bufs;  in mpi3mr_initialize_ioc()
2921 	mpi3mr_regwrite(sc, MPI3_SYSIF_SENSE_BUF_FREE_HOST_INDEX_OFFSET,  in mpi3mr_initialize_ioc()
2922 		sc->sense_buf_q_host_index);  in mpi3mr_initialize_ioc()
2925 		retval = mpi3mr_alloc_interrupts(sc, 0);  in mpi3mr_initialize_ioc()
2927 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to allocate interrupts, retval: 0x%x\n",  in mpi3mr_initialize_ioc()
2932 		retval = mpi3mr_setup_irqs(sc);  in mpi3mr_initialize_ioc()
2935 			    sc->name, retval);  in mpi3mr_initialize_ioc()
2939 		mpi3mr_enable_interrupts(sc);  in mpi3mr_initialize_ioc()
2942 		mpi3mr_enable_interrupts(sc);  in mpi3mr_initialize_ioc()
2944 	retval = mpi3mr_create_op_queues(sc);  in mpi3mr_initialize_ioc()
2947 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to create operational queues, error: %d\n",  in mpi3mr_initialize_ioc()
2952 	if (!sc->throttle_groups && sc->num_io_throttle_group) {  in mpi3mr_initialize_ioc()
2953 		mpi3mr_dprint(sc, MPI3MR_ERROR, "allocating memory for throttle groups\n");  in mpi3mr_initialize_ioc()
2955 		sc->throttle_groups = (struct mpi3mr_throttle_group_info *)  in mpi3mr_initialize_ioc()
2956 					  malloc(sc->num_io_throttle_group *  in mpi3mr_initialize_ioc()
2958 		if (!sc->throttle_groups)  in mpi3mr_initialize_ioc()
2963 		mpi3mr_dprint(sc, MPI3MR_INFO, "Re-register events\n");  in mpi3mr_initialize_ioc()
2964 		retval = mpi3mr_register_events(sc);  in mpi3mr_initialize_ioc()
2966 			mpi3mr_dprint(sc, MPI3MR_INFO, "Failed to re-register events, retval: 0x%x\n",  in mpi3mr_initialize_ioc()
2971 		mpi3mr_dprint(sc, MPI3MR_INFO, "Issuing Port Enable\n");  in mpi3mr_initialize_ioc()
2972 		retval = mpi3mr_issue_port_enable(sc, 0);  in mpi3mr_initialize_ioc()
2974 			mpi3mr_dprint(sc, MPI3MR_INFO, "Failed to issue port enable, retval: 0x%x\n",  in mpi3mr_initialize_ioc()
2979 	retval = mpi3mr_pel_alloc(sc);  in mpi3mr_initialize_ioc()
2981 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to allocate memory for PEL, retval: 0x%x\n",  in mpi3mr_initialize_ioc()
2993 static void mpi3mr_port_enable_complete(struct mpi3mr_softc *sc,  in mpi3mr_port_enable_complete()  argument
2998 	printf(IOCNAME "Completing Port Enable Request\n", sc->name);  in mpi3mr_port_enable_complete()
2999 	sc->mpi3mr_flags |= MPI3MR_FLAGS_PORT_ENABLE_DONE;  in mpi3mr_port_enable_complete()
3000 	mpi3mr_startup_decrement(sc->cam_sc);  in mpi3mr_port_enable_complete()
3003 int mpi3mr_issue_port_enable(struct mpi3mr_softc *sc, U8 async)  in mpi3mr_issue_port_enable()  argument
3009 	mtx_lock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_port_enable()
3010 	if (sc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_port_enable()
3012 		printf(IOCNAME "Issue PortEnable: Init command is in use\n", sc->name);  in mpi3mr_issue_port_enable()
3013 		mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_port_enable()
3017 	sc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_port_enable()
3020 		sc->init_cmds.is_waiting = 0;  in mpi3mr_issue_port_enable()
3021 		sc->init_cmds.callback = mpi3mr_port_enable_complete;  in mpi3mr_issue_port_enable()
3023 		sc->init_cmds.is_waiting = 1;  in mpi3mr_issue_port_enable()
3024 		sc->init_cmds.callback = NULL;  in mpi3mr_issue_port_enable()
3025 		init_completion(&sc->init_cmds.completion);  in mpi3mr_issue_port_enable()
3030 	printf(IOCNAME "Sending Port Enable Request\n", sc->name);  in mpi3mr_issue_port_enable()
3031 	retval = mpi3mr_submit_admin_cmd(sc, &pe_req, sizeof(pe_req));  in mpi3mr_issue_port_enable()
3034 		    sc->name);  in mpi3mr_issue_port_enable()
3039 		wait_for_completion_timeout(&sc->init_cmds.completion,  in mpi3mr_issue_port_enable()
3041 		if (!(sc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_port_enable()
3043 			    sc->name);  in mpi3mr_issue_port_enable()
3045 			mpi3mr_check_rh_fault_ioc(sc, MPI3MR_RESET_FROM_PE_TIMEOUT);  in mpi3mr_issue_port_enable()
3048 		mpi3mr_port_enable_complete(sc, &sc->init_cmds);  in mpi3mr_issue_port_enable()
3051 	mtx_unlock(&sc->init_cmds.completion.lock);  in mpi3mr_issue_port_enable()
3060 	struct mpi3mr_softc *sc;  in mpi3mr_watchdog_thread()  local
3064 	sc = (struct mpi3mr_softc *)arg;  in mpi3mr_watchdog_thread()
3066 	mpi3mr_dprint(sc, MPI3MR_XINFO, "%s\n", __func__);  in mpi3mr_watchdog_thread()
3068 	sc->watchdog_thread_active = 1;  in mpi3mr_watchdog_thread()
3069 	mtx_lock(&sc->reset_mutex);  in mpi3mr_watchdog_thread()
3071 		if (sc->mpi3mr_flags & MPI3MR_FLAGS_SHUTDOWN ||   in mpi3mr_watchdog_thread()
3072 		    (sc->unrecoverable == 1)) {  in mpi3mr_watchdog_thread()
3073 			mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_watchdog_thread()
3075 			   sc->mpi3mr_flags & MPI3MR_FLAGS_SHUTDOWN ? "Shutdown" :  in mpi3mr_watchdog_thread()
3079 		mtx_unlock(&sc->reset_mutex);  in mpi3mr_watchdog_thread()
3081 		if ((sc->prepare_for_reset) &&  in mpi3mr_watchdog_thread()
3082 		    ((sc->prepare_for_reset_timeout_counter++) >=  in mpi3mr_watchdog_thread()
3084 			mpi3mr_soft_reset_handler(sc,  in mpi3mr_watchdog_thread()
3089 		ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_watchdog_thread()
3092 			mpi3mr_soft_reset_handler(sc, MPI3MR_RESET_FROM_FIRMWARE, 0);  in mpi3mr_watchdog_thread()
3096 		ioc_state = mpi3mr_get_iocstate(sc);  in mpi3mr_watchdog_thread()
3098 			fault = mpi3mr_regread(sc, MPI3_SYSIF_FAULT_OFFSET) &  in mpi3mr_watchdog_thread()
3101 			host_diagnostic = mpi3mr_regread(sc, MPI3_SYSIF_HOST_DIAG_OFFSET);  in mpi3mr_watchdog_thread()
3103 				if (!sc->diagsave_timeout) {  in mpi3mr_watchdog_thread()
3104 					mpi3mr_print_fault_info(sc);  in mpi3mr_watchdog_thread()
3105 					mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_watchdog_thread()
3108 				if ((sc->diagsave_timeout++) <= MPI3_SYSIF_DIAG_SAVE_TIMEOUT)  in mpi3mr_watchdog_thread()
3111 			mpi3mr_print_fault_info(sc);  in mpi3mr_watchdog_thread()
3112 			sc->diagsave_timeout = 0;  in mpi3mr_watchdog_thread()
3116 				mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_watchdog_thread()
3119 				sc->unrecoverable = 1;  in mpi3mr_watchdog_thread()
3124 			    || (sc->reset_in_progress))  in mpi3mr_watchdog_thread()
3127 				mpi3mr_soft_reset_handler(sc,  in mpi3mr_watchdog_thread()
3130 				mpi3mr_soft_reset_handler(sc,  in mpi3mr_watchdog_thread()
3135 		if (sc->reset.type == MPI3MR_TRIGGER_SOFT_RESET) {  in mpi3mr_watchdog_thread()
3136 			mpi3mr_print_fault_info(sc);  in mpi3mr_watchdog_thread()
3137 			mpi3mr_soft_reset_handler(sc, sc->reset.reason, 1);  in mpi3mr_watchdog_thread()
3140 		mtx_lock(&sc->reset_mutex);  in mpi3mr_watchdog_thread()
3145 		if ((sc->mpi3mr_flags & MPI3MR_FLAGS_SHUTDOWN) == 0 &&  in mpi3mr_watchdog_thread()
3146 		    !sc->unrecoverable) {  in mpi3mr_watchdog_thread()
3147 			msleep(&sc->watchdog_chan, &sc->reset_mutex, PRIBIO,  in mpi3mr_watchdog_thread()
3151 	mtx_unlock(&sc->reset_mutex);  in mpi3mr_watchdog_thread()
3152 	sc->watchdog_thread_active = 0;  in mpi3mr_watchdog_thread()
3156 static void mpi3mr_display_event_data(struct mpi3mr_softc *sc,  in mpi3mr_display_event_data()  argument
3184 		mpi3mr_dprint(sc, MPI3MR_EVENT, "Device Added: Dev=0x%04x Form=0x%x Perst id: 0x%x\n",  in mpi3mr_display_event_data()
3192 		mpi3mr_dprint(sc, MPI3MR_EVENT, "Device Info Changed: Dev=0x%04x Form=0x%x\n",  in mpi3mr_display_event_data()
3200 		mpi3mr_dprint(sc, MPI3MR_EVENT, "Device Status Change: Dev=0x%04x RC=0x%x\n",  in mpi3mr_display_event_data()
3208 		mpi3mr_dprint(sc, MPI3MR_EVENT, "SAS Discovery: (%s)",  in mpi3mr_display_event_data()
3212 		    (sc->mpi3mr_debug & MPI3MR_EVENT)) {  in mpi3mr_display_event_data()
3218 		if (sc->mpi3mr_debug & MPI3MR_EVENT)  in mpi3mr_display_event_data()
3256 		mpi3mr_dprint(sc, MPI3MR_EVENT, "PCIE Enumeration: (%s)",  in mpi3mr_display_event_data()
3261 			mpi3mr_dprint(sc, MPI3MR_EVENT, "enumeration_status(0x%08x)",  in mpi3mr_display_event_data()
3263 		if (sc->mpi3mr_debug & MPI3MR_EVENT)  in mpi3mr_display_event_data()
3275 	mpi3mr_dprint(sc, MPI3MR_EVENT, "%s\n", desc);  in mpi3mr_display_event_data()
3284 	mtx_lock_spin(&cam_sc->sc->target_lock);  in mpi3mr_find_target_by_per_id()
3290 	mtx_unlock_spin(&cam_sc->sc->target_lock);  in mpi3mr_find_target_by_per_id()
3300 	mtx_lock_spin(&cam_sc->sc->target_lock);  in mpi3mr_find_target_by_dev_handle()
3306 	mtx_unlock_spin(&cam_sc->sc->target_lock);  in mpi3mr_find_target_by_dev_handle()
3310 void mpi3mr_update_device(struct mpi3mr_softc *sc,  in mpi3mr_update_device()  argument
3404 		if (vdinf->IOThrottleGroup < sc->num_io_throttle_group) {  in mpi3mr_update_device()
3405 			tg = sc->throttle_groups + vdinf->IOThrottleGroup;  in mpi3mr_update_device()
3425 int mpi3mr_create_device(struct mpi3mr_softc *sc,  in mpi3mr_create_device()  argument
3434 	mtx_lock_spin(&sc->target_lock);  in mpi3mr_create_device()
3435 	TAILQ_FOREACH(target, &sc->cam_sc->tgt_list, tgt_next) {  in mpi3mr_create_device()
3441 	mtx_unlock_spin(&sc->target_lock);  in mpi3mr_create_device()
3444 			mpi3mr_update_device(sc, target, dev_pg0, true);  in mpi3mr_create_device()
3455 			mpi3mr_update_device(sc, target, dev_pg0, true);  in mpi3mr_create_device()
3456 			mtx_lock_spin(&sc->target_lock);  in mpi3mr_create_device()
3457 			TAILQ_INSERT_TAIL(&sc->cam_sc->tgt_list, target, tgt_next);  in mpi3mr_create_device()
3459 			mtx_unlock_spin(&sc->target_lock);  in mpi3mr_create_device()
3476 static void mpi3mr_dev_rmhs_complete_iou(struct mpi3mr_softc *sc,  in mpi3mr_dev_rmhs_complete_iou()  argument
3482 	mpi3mr_dprint(sc, MPI3MR_EVENT,  in mpi3mr_dev_rmhs_complete_iou()
3489 			mpi3mr_dprint(sc, MPI3MR_EVENT,  in mpi3mr_dev_rmhs_complete_iou()
3493 			mpi3mr_dev_rmhs_send_tm(sc, drv_cmd->dev_handle,  in mpi3mr_dev_rmhs_complete_iou()
3497 		mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_dev_rmhs_complete_iou()
3501 		mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_dev_rmhs_complete_iou()
3504 		mpi3mr_clear_bit(drv_cmd->dev_handle, sc->removepend_bitmap);  in mpi3mr_dev_rmhs_complete_iou()
3507 	if (!TAILQ_EMPTY(&sc->delayed_rmhs_list)) {  in mpi3mr_dev_rmhs_complete_iou()
3508 		delayed_dev_rmhs = TAILQ_FIRST(&sc->delayed_rmhs_list);  in mpi3mr_dev_rmhs_complete_iou()
3512 		mpi3mr_dprint(sc, MPI3MR_EVENT,  in mpi3mr_dev_rmhs_complete_iou()
3515 		mpi3mr_dev_rmhs_send_tm(sc, drv_cmd->dev_handle, drv_cmd,  in mpi3mr_dev_rmhs_complete_iou()
3517 		TAILQ_REMOVE(&sc->delayed_rmhs_list, delayed_dev_rmhs, list);  in mpi3mr_dev_rmhs_complete_iou()
3525 	mpi3mr_clear_bit(cmd_idx, sc->devrem_bitmap);  in mpi3mr_dev_rmhs_complete_iou()
3539 static void mpi3mr_dev_rmhs_complete_tm(struct mpi3mr_softc *sc,  in mpi3mr_dev_rmhs_complete_tm()  argument
3553 		    sc->name, drv_cmd->dev_handle, drv_cmd->ioc_status,  in mpi3mr_dev_rmhs_complete_tm()
3558 	    sc->name, drv_cmd->dev_handle, cmd_idx);  in mpi3mr_dev_rmhs_complete_tm()
3570 	retval = mpi3mr_submit_admin_cmd(sc, &iou_ctrl, sizeof(iou_ctrl));  in mpi3mr_dev_rmhs_complete_tm()
3573 		    sc->name);  in mpi3mr_dev_rmhs_complete_tm()
3583 	mpi3mr_clear_bit(cmd_idx, sc->devrem_bitmap);  in mpi3mr_dev_rmhs_complete_tm()
3599 static void mpi3mr_dev_rmhs_send_tm(struct mpi3mr_softc *sc, U16 handle,  in mpi3mr_dev_rmhs_send_tm()  argument
3610 	mtx_lock_spin(&sc->target_lock);  in mpi3mr_dev_rmhs_send_tm()
3611 	TAILQ_FOREACH(tgtdev, &sc->cam_sc->tgt_list, tgt_next) {  in mpi3mr_dev_rmhs_send_tm()
3618 	mtx_unlock_spin(&sc->target_lock);  in mpi3mr_dev_rmhs_send_tm()
3623 		cmd_idx = mpi3mr_find_first_zero_bit(sc->devrem_bitmap,  in mpi3mr_dev_rmhs_send_tm()
3626 			if (!mpi3mr_test_and_set_bit(cmd_idx, sc->devrem_bitmap))  in mpi3mr_dev_rmhs_send_tm()
3640 		TAILQ_INSERT_TAIL(&(sc->delayed_rmhs_list), delayed_dev_rmhs, list);  in mpi3mr_dev_rmhs_send_tm()
3641 		mpi3mr_dprint(sc, MPI3MR_EVENT, "%s :DevRmHs: tr:handle(0x%04x) is postponed\n",  in mpi3mr_dev_rmhs_send_tm()
3647 	drv_cmd = &sc->dev_rmhs_cmds[cmd_idx];  in mpi3mr_dev_rmhs_send_tm()
3651 	mpi3mr_dprint(sc, MPI3MR_EVENT,  in mpi3mr_dev_rmhs_send_tm()
3657 		mpi3mr_dprint(sc, MPI3MR_EVENT, "%s :Issue TM: Command is in use\n", __func__);  in mpi3mr_dev_rmhs_send_tm()
3671 	mpi3mr_set_bit(handle, sc->removepend_bitmap);  in mpi3mr_dev_rmhs_send_tm()
3672 	retval = mpi3mr_submit_admin_cmd(sc, &tm_req, sizeof(tm_req));  in mpi3mr_dev_rmhs_send_tm()
3674 		mpi3mr_dprint(sc, MPI3MR_ERROR, "%s :Issue DevRmHsTM: Admin Post failed\n",  in mpi3mr_dev_rmhs_send_tm()
3685 	mpi3mr_clear_bit(cmd_idx, sc->devrem_bitmap);  in mpi3mr_dev_rmhs_send_tm()
3699 static void mpi3mr_complete_evt_ack(struct mpi3mr_softc *sc,  in mpi3mr_complete_evt_ack()  argument
3706 		mpi3mr_dprint(sc, MPI3MR_EVENT,  in mpi3mr_complete_evt_ack()
3712 	if (!TAILQ_EMPTY(&sc->delayed_evtack_cmds_list)) {  in mpi3mr_complete_evt_ack()
3713 		delayed_evtack = TAILQ_FIRST(&sc->delayed_evtack_cmds_list);  in mpi3mr_complete_evt_ack()
3714 		mpi3mr_dprint(sc, MPI3MR_EVENT,  in mpi3mr_complete_evt_ack()
3717 		mpi3mr_send_evt_ack(sc, delayed_evtack->event, drv_cmd,  in mpi3mr_complete_evt_ack()
3719 		TAILQ_REMOVE(&sc->delayed_evtack_cmds_list, delayed_evtack, list);  in mpi3mr_complete_evt_ack()
3725 	mpi3mr_clear_bit(cmd_idx, sc->evtack_cmds_bitmap);  in mpi3mr_complete_evt_ack()
3742 static void mpi3mr_send_evt_ack(struct mpi3mr_softc *sc, U8 event,  in mpi3mr_send_evt_ack()  argument
3755 		cmd_idx = mpi3mr_find_first_zero_bit(sc->evtack_cmds_bitmap,  in mpi3mr_send_evt_ack()
3759 			    sc->evtack_cmds_bitmap))  in mpi3mr_send_evt_ack()
3772 		TAILQ_INSERT_TAIL(&(sc->delayed_evtack_cmds_list), delayed_evtack, list);  in mpi3mr_send_evt_ack()
3773 		mpi3mr_dprint(sc, MPI3MR_EVENT, "%s : Event ack for event:%d is postponed\n",  in mpi3mr_send_evt_ack()
3777 	drv_cmd = &sc->evtack_cmds[cmd_idx];  in mpi3mr_send_evt_ack()
3784 		mpi3mr_dprint(sc, MPI3MR_EVENT, "%s: Command is in use\n", __func__);  in mpi3mr_send_evt_ack()
3794 	retval = mpi3mr_submit_admin_cmd(sc, &evtack_req,  in mpi3mr_send_evt_ack()
3798 		mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: Admin Post failed\n", __func__);  in mpi3mr_send_evt_ack()
3806 	mpi3mr_clear_bit(cmd_idx, sc->evtack_cmds_bitmap);  in mpi3mr_send_evt_ack()
3821 static void mpi3mr_pcietopochg_evt_th(struct mpi3mr_softc *sc,  in mpi3mr_pcietopochg_evt_th()  argument
3836 		tgtdev = mpi3mr_find_target_by_dev_handle(sc->cam_sc, handle);  in mpi3mr_pcietopochg_evt_th()
3844 			mpi3mr_dev_rmhs_send_tm(sc, handle, NULL,  in mpi3mr_pcietopochg_evt_th()
3880 static void mpi3mr_sastopochg_evt_th(struct mpi3mr_softc *sc,  in mpi3mr_sastopochg_evt_th()  argument
3896 		tgtdev = mpi3mr_find_target_by_dev_handle(sc->cam_sc, handle);  in mpi3mr_sastopochg_evt_th()
3904 			mpi3mr_dev_rmhs_send_tm(sc, handle, NULL,  in mpi3mr_sastopochg_evt_th()
3939 static void mpi3mr_devstatuschg_evt_th(struct mpi3mr_softc *sc,  in mpi3mr_devstatuschg_evt_th()  argument
3974 	tgtdev = mpi3mr_find_target_by_dev_handle(sc->cam_sc, dev_handle);  in mpi3mr_devstatuschg_evt_th()
3977 		mpi3mr_dprint(sc, MPI3MR_ERROR, "%s :target with dev_handle:0x%x not found\n",  in mpi3mr_devstatuschg_evt_th()
4002 		mpi3mr_dev_rmhs_send_tm(sc, dev_handle, NULL,  in mpi3mr_devstatuschg_evt_th()
4006 		mpi3mr_dev_rmhs_send_tm(sc, dev_handle, NULL,  in mpi3mr_devstatuschg_evt_th()
4019 static void mpi3mr_preparereset_evt_th(struct mpi3mr_softc *sc,  in mpi3mr_preparereset_evt_th()  argument
4026 		mpi3mr_dprint(sc, MPI3MR_EVENT, "%s :Recieved PrepForReset Event with RC=START\n",  in mpi3mr_preparereset_evt_th()
4028 		if (sc->prepare_for_reset)  in mpi3mr_preparereset_evt_th()
4030 		sc->prepare_for_reset = 1;  in mpi3mr_preparereset_evt_th()
4031 		sc->prepare_for_reset_timeout_counter = 0;  in mpi3mr_preparereset_evt_th()
4033 		mpi3mr_dprint(sc, MPI3MR_EVENT, "%s :Recieved PrepForReset Event with RC=ABORT\n",  in mpi3mr_preparereset_evt_th()
4035 		sc->prepare_for_reset = 0;  in mpi3mr_preparereset_evt_th()
4036 		sc->prepare_for_reset_timeout_counter = 0;  in mpi3mr_preparereset_evt_th()
4040 		mpi3mr_send_evt_ack(sc, event_reply->Event, NULL,  in mpi3mr_preparereset_evt_th()
4053 static void mpi3mr_energypackchg_evt_th(struct mpi3mr_softc *sc,  in mpi3mr_energypackchg_evt_th()  argument
4061 		mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_energypackchg_evt_th()
4067 	mpi3mr_dprint(sc, MPI3MR_EVENT,  in mpi3mr_energypackchg_evt_th()
4069 	    __func__, sc->facts.shutdown_timeout, shutdown_timeout);  in mpi3mr_energypackchg_evt_th()
4070 	sc->facts.shutdown_timeout = shutdown_timeout;  in mpi3mr_energypackchg_evt_th()
4082 static void mpi3mr_cablemgmt_evt_th(struct mpi3mr_softc *sc,  in mpi3mr_cablemgmt_evt_th()  argument
4091 		mpi3mr_dprint(sc, MPI3MR_INFO, "An active cable with ReceptacleID %d cannot be powered.\n"  in mpi3mr_cablemgmt_evt_th()
4100 		mpi3mr_dprint(sc, MPI3MR_INFO, "A cable with ReceptacleID %d is not running at optimal speed\n",  in mpi3mr_cablemgmt_evt_th()
4118 static void mpi3mr_process_events(struct mpi3mr_softc *sc,  in mpi3mr_process_events()  argument
4126 	if (sc->mpi3mr_flags & MPI3MR_FLAGS_SHUTDOWN)  in mpi3mr_process_events()
4140 		if (mpi3mr_create_device(sc, dev_pg0))  in mpi3mr_process_events()
4141 			mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_process_events()
4152 		mpi3mr_devstatuschg_evt_th(sc, event_reply);  in mpi3mr_process_events()
4158 		mpi3mr_sastopochg_evt_th(sc, event_reply);  in mpi3mr_process_events()
4164 		mpi3mr_pcietopochg_evt_th(sc, event_reply);  in mpi3mr_process_events()
4169 		mpi3mr_preparereset_evt_th(sc, event_reply);  in mpi3mr_process_events()
4181 		mpi3mr_energypackchg_evt_th(sc, event_reply);  in mpi3mr_process_events()
4186 		mpi3mr_cablemgmt_evt_th(sc, event_reply);  in mpi3mr_process_events()
4197 		mpi3mr_dprint(sc, MPI3MR_INFO, "%s :Event 0x%02x is not handled by driver\n",  in mpi3mr_process_events()
4225 		    sc->track_mapping_events)  in mpi3mr_process_events()
4226 			sc->pending_map_events++;  in mpi3mr_process_events()
4233 		    !(sc->mpi3mr_flags & MPI3MR_FLAGS_PORT_ENABLE_DONE))  in mpi3mr_process_events()
4234 			mpi3mr_startup_increment(sc->cam_sc);  in mpi3mr_process_events()
4241 		mtx_lock(&sc->fwevt_lock);  in mpi3mr_process_events()
4242 		TAILQ_INSERT_TAIL(&sc->cam_sc->ev_queue, fw_event, ev_link);  in mpi3mr_process_events()
4243 		taskqueue_enqueue(sc->cam_sc->ev_tq, &sc->cam_sc->ev_task);  in mpi3mr_process_events()
4244 		mtx_unlock(&sc->fwevt_lock);  in mpi3mr_process_events()
4251 static void mpi3mr_handle_events(struct mpi3mr_softc *sc, uintptr_t data,  in mpi3mr_handle_events()  argument
4257 	sc->change_count = event_reply->IOCChangeCount;  in mpi3mr_handle_events()
4258 	mpi3mr_display_event_data(sc, event_reply);  in mpi3mr_handle_events()
4260 	mpi3mr_process_events(sc, data, event_reply);  in mpi3mr_handle_events()
4263 static void mpi3mr_process_admin_reply_desc(struct mpi3mr_softc *sc,  in mpi3mr_process_admin_reply_desc()  argument
4293 		def_reply = mpi3mr_get_reply_virt_addr(sc, *reply_dma);  in mpi3mr_process_admin_reply_desc()
4304 			sense_buf = mpi3mr_get_sensebuf_virt_addr(sc,  in mpi3mr_process_admin_reply_desc()
4317 		cmdptr = &sc->init_cmds;  in mpi3mr_process_admin_reply_desc()
4320 		cmdptr = &sc->ioctl_cmds;  in mpi3mr_process_admin_reply_desc()
4323 		cmdptr = &sc->host_tm_cmds;  in mpi3mr_process_admin_reply_desc()
4324 		wakeup((void *)&sc->tm_chan);  in mpi3mr_process_admin_reply_desc()
4327 		cmdptr = &sc->pel_abort_cmd;  in mpi3mr_process_admin_reply_desc()
4330 		cmdptr = &sc->pel_cmds;  in mpi3mr_process_admin_reply_desc()
4335 			mpi3mr_handle_events(sc, *reply_dma ,def_reply);  in mpi3mr_process_admin_reply_desc()
4343 		cmdptr = &sc->dev_rmhs_cmds[idx];  in mpi3mr_process_admin_reply_desc()
4349 		cmdptr = &sc->evtack_cmds[idx];  in mpi3mr_process_admin_reply_desc()
4361 				    sc->reply_sz);  in mpi3mr_process_admin_reply_desc()
4372 				cmdptr->callback(sc, cmdptr);  in mpi3mr_process_admin_reply_desc()
4377 		mpi3mr_repost_sense_buf(sc,  in mpi3mr_process_admin_reply_desc()
4388 static int mpi3mr_complete_admin_cmd(struct mpi3mr_softc *sc)  in mpi3mr_complete_admin_cmd()  argument
4390 	U32 exp_phase = sc->admin_reply_ephase;  in mpi3mr_complete_admin_cmd()
4391 	U32 adm_reply_ci = sc->admin_reply_ci;  in mpi3mr_complete_admin_cmd()
4396 	mtx_lock_spin(&sc->admin_reply_lock);  in mpi3mr_complete_admin_cmd()
4397 	if (sc->admin_in_use == false) {  in mpi3mr_complete_admin_cmd()
4398 		sc->admin_in_use = true;  in mpi3mr_complete_admin_cmd()
4399 		mtx_unlock_spin(&sc->admin_reply_lock);  in mpi3mr_complete_admin_cmd()
4401 		mtx_unlock_spin(&sc->admin_reply_lock);  in mpi3mr_complete_admin_cmd()
4405 	reply_desc = (Mpi3DefaultReplyDescriptor_t *)sc->admin_reply +  in mpi3mr_complete_admin_cmd()
4410 		mtx_lock_spin(&sc->admin_reply_lock);  in mpi3mr_complete_admin_cmd()
4411 		sc->admin_in_use = false;  in mpi3mr_complete_admin_cmd()
4412 		mtx_unlock_spin(&sc->admin_reply_lock);  in mpi3mr_complete_admin_cmd()
4417 		sc->admin_req_ci = reply_desc->RequestQueueCI;  in mpi3mr_complete_admin_cmd()
4418 		mpi3mr_process_admin_reply_desc(sc, reply_desc, &reply_dma);  in mpi3mr_complete_admin_cmd()
4420 			mpi3mr_repost_reply_buf(sc, reply_dma);  in mpi3mr_complete_admin_cmd()
4422 		if (++adm_reply_ci == sc->num_admin_replies) {  in mpi3mr_complete_admin_cmd()
4427 			(Mpi3DefaultReplyDescriptor_t *)sc->admin_reply +  in mpi3mr_complete_admin_cmd()
4434 	mpi3mr_regwrite(sc, MPI3_SYSIF_ADMIN_REPLY_Q_CI_OFFSET, adm_reply_ci);  in mpi3mr_complete_admin_cmd()
4435 	sc->admin_reply_ci = adm_reply_ci;  in mpi3mr_complete_admin_cmd()
4436 	sc->admin_reply_ephase = exp_phase;  in mpi3mr_complete_admin_cmd()
4437 	mtx_lock_spin(&sc->admin_reply_lock);  in mpi3mr_complete_admin_cmd()
4438 	sc->admin_in_use = false;  in mpi3mr_complete_admin_cmd()
4439 	mtx_unlock_spin(&sc->admin_reply_lock);  in mpi3mr_complete_admin_cmd()
4444 mpi3mr_cmd_done(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cmd)  in mpi3mr_cmd_done()  argument
4446 	mpi3mr_unmap_request(sc, cmd);  in mpi3mr_cmd_done()
4448 	mtx_lock(&sc->mpi3mr_mtx);  in mpi3mr_cmd_done()
4454 	if (sc->unrecoverable)  in mpi3mr_cmd_done()
4459 	mtx_unlock(&sc->mpi3mr_mtx);  in mpi3mr_cmd_done()
4463 void mpi3mr_process_op_reply_desc(struct mpi3mr_softc *sc,  in mpi3mr_process_op_reply_desc()  argument
4504 		scsi_reply = mpi3mr_get_reply_virt_addr(sc,  in mpi3mr_process_op_reply_desc()
4507 			mpi3mr_dprint(sc, MPI3MR_ERROR, "scsi_reply is NULL, "  in mpi3mr_process_op_reply_desc()
4521 		sense_buf = mpi3mr_get_sensebuf_virt_addr(sc,  in mpi3mr_process_op_reply_desc()
4528 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Ran out of sense buffers\n");  in mpi3mr_process_op_reply_desc()
4539 	cm = sc->cmd_list[host_tag];  in mpi3mr_process_op_reply_desc()
4544 	cam_sc = sc->cam_sc;  in mpi3mr_process_op_reply_desc()
4552 	if (sc->iot_enable) {  in mpi3mr_process_op_reply_desc()
4561 		if ((data_len_blks >= sc->io_throttle_data_length) &&  in mpi3mr_process_op_reply_desc()
4563 			mpi3mr_atomic_sub(&sc->pend_large_data_sz, data_len_blks);  in mpi3mr_process_op_reply_desc()
4565 			    &sc->pend_large_data_sz);  in mpi3mr_process_op_reply_desc()
4571 					mpi3mr_dprint(sc, MPI3MR_IOT,  in mpi3mr_process_op_reply_desc()
4578 						    sc->io_throttle_low,  in mpi3mr_process_op_reply_desc()
4583 				    sc->io_throttle_low) &&  in mpi3mr_process_op_reply_desc()
4586 					mpi3mr_dprint(sc, MPI3MR_IOT,  in mpi3mr_process_op_reply_desc()
4590 					    sc, tg, 0);  in mpi3mr_process_op_reply_desc()
4594 					mpi3mr_dprint(sc, MPI3MR_IOT,  in mpi3mr_process_op_reply_desc()
4599 					    sc->io_throttle_low);  in mpi3mr_process_op_reply_desc()
4603 				if (ioc_pend_data_len <= sc->io_throttle_low) {  in mpi3mr_process_op_reply_desc()
4605 					mpi3mr_dprint(sc, MPI3MR_IOT,  in mpi3mr_process_op_reply_desc()
4612 			ioc_pend_data_len = mpi3mr_atomic_read(&sc->pend_large_data_sz);  in mpi3mr_process_op_reply_desc()
4615 					mpi3mr_dprint(sc, MPI3MR_IOT,  in mpi3mr_process_op_reply_desc()
4620 					    sc->io_throttle_low);  in mpi3mr_process_op_reply_desc()
4624 				if ( ioc_pend_data_len <= sc->io_throttle_low) {  in mpi3mr_process_op_reply_desc()
4625 					mpi3mr_dprint(sc, MPI3MR_IOT,  in mpi3mr_process_op_reply_desc()
4631 			} else if (ioc_pend_data_len <= sc->io_throttle_low) {  in mpi3mr_process_op_reply_desc()
4634 					mpi3mr_dprint(sc, MPI3MR_IOT,  in mpi3mr_process_op_reply_desc()
4641 						    sc->io_throttle_low,  in mpi3mr_process_op_reply_desc()
4647 					mpi3mr_dprint(sc, MPI3MR_IOT,  in mpi3mr_process_op_reply_desc()
4651 					    sc, tg, 0);  in mpi3mr_process_op_reply_desc()
4709 		mpi3mr_dprint(sc, MPI3MR_TRACE,  in mpi3mr_process_op_reply_desc()
4714 		mpi3mr_dprint(sc, MPI3MR_TRACE,  in mpi3mr_process_op_reply_desc()
4730 			mpi3mr_dprint(sc, MPI3MR_XINFO, "func: %s line: %d recovered error\n",  __func__, __LINE__);  in mpi3mr_process_op_reply_desc()
4814 	mpi3mr_cmd_done(sc, cm);  in mpi3mr_process_op_reply_desc()
4815 	mpi3mr_dprint(sc, MPI3MR_TRACE, "Completion IO path :"  in mpi3mr_process_op_reply_desc()
4819 	mpi3mr_atomic_dec(&sc->fw_outstanding);  in mpi3mr_process_op_reply_desc()
4823 		mpi3mr_repost_sense_buf(sc,  in mpi3mr_process_op_reply_desc()
4835 int mpi3mr_complete_io_cmd(struct mpi3mr_softc *sc,  in mpi3mr_complete_io_cmd()  argument
4856 	mpi3mr_dprint(sc, MPI3MR_TRACE, "[QID:%d]:reply_desc: (%pa) reply_ci: %x"  in mpi3mr_complete_io_cmd()
4872 		sc->op_req_q[req_qid - 1].ci =  in mpi3mr_complete_io_cmd()
4875 		mpi3mr_process_op_reply_desc(sc, reply_desc, &reply_dma);  in mpi3mr_complete_io_cmd()
4878 			mpi3mr_repost_reply_buf(sc, reply_dma);  in mpi3mr_complete_io_cmd()
4892 	mpi3mr_regwrite(sc, MPI3_SYSIF_OPER_REPLY_Q_N_CI_OFFSET(op_reply_q->qid), reply_ci);  in mpi3mr_complete_io_cmd()
4911 	struct mpi3mr_softc *sc = irq_ctx->sc;  in mpi3mr_isr()  local
4919 	if (!sc->intr_enabled)  in mpi3mr_isr()
4923 		mpi3mr_complete_admin_cmd(sc);  in mpi3mr_isr()
4926 		mpi3mr_complete_io_cmd(sc, irq_ctx);  in mpi3mr_isr()
4939 mpi3mr_alloc_requests(struct mpi3mr_softc *sc)  in mpi3mr_alloc_requests()  argument
4945 	ret = bus_dma_tag_create( sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_alloc_requests()
4947 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_alloc_requests()
4955 				&sc->io_lock,	/* lockarg */  in mpi3mr_alloc_requests()
4956 				&sc->buffer_dmat);  in mpi3mr_alloc_requests()
4958 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate buffer DMA tag ret: %d\n", ret);  in mpi3mr_alloc_requests()
4967 	sc->cmd_list = malloc(sizeof(struct mpi3mr_cmd *) * sc->max_host_ios,  in mpi3mr_alloc_requests()
4970 	if (!sc->cmd_list) {  in mpi3mr_alloc_requests()
4971 		device_printf(sc->mpi3mr_dev, "Cannot alloc memory for mpt_cmd_list.\n");  in mpi3mr_alloc_requests()
4975 	for (i = 0; i < sc->max_host_ios; i++) {  in mpi3mr_alloc_requests()
4976 		sc->cmd_list[i] = malloc(sizeof(struct mpi3mr_cmd),  in mpi3mr_alloc_requests()
4978 		if (!sc->cmd_list[i]) {  in mpi3mr_alloc_requests()
4980 				free(sc->cmd_list[j], M_MPI3MR);  in mpi3mr_alloc_requests()
4981 			free(sc->cmd_list, M_MPI3MR);  in mpi3mr_alloc_requests()
4982 			sc->cmd_list = NULL;  in mpi3mr_alloc_requests()
4987 	for (i = 1; i < sc->max_host_ios; i++) {  in mpi3mr_alloc_requests()
4988 		cmd = sc->cmd_list[i];  in mpi3mr_alloc_requests()
4990 		cmd->sc = sc;  in mpi3mr_alloc_requests()
4992 		callout_init_mtx(&cmd->callout, &sc->mpi3mr_mtx, 0);  in mpi3mr_alloc_requests()
4994 		TAILQ_INSERT_TAIL(&(sc->cmd_list_head), cmd, next);  in mpi3mr_alloc_requests()
4995 		if (bus_dmamap_create(sc->buffer_dmat, 0, &cmd->dmamap))  in mpi3mr_alloc_requests()
5009 mpi3mr_get_command(struct mpi3mr_softc *sc)  in mpi3mr_get_command()  argument
5013 	mtx_lock(&sc->cmd_pool_lock);  in mpi3mr_get_command()
5014 	if (!TAILQ_EMPTY(&sc->cmd_list_head)) {  in mpi3mr_get_command()
5015 		cmd = TAILQ_FIRST(&sc->cmd_list_head);  in mpi3mr_get_command()
5016 		TAILQ_REMOVE(&sc->cmd_list_head, cmd, next);  in mpi3mr_get_command()
5021 	mpi3mr_dprint(sc, MPI3MR_TRACE, "Get command SMID: 0x%x\n", cmd->hosttag);  in mpi3mr_get_command()
5031 	mtx_unlock(&sc->cmd_pool_lock);  in mpi3mr_get_command()
5044 	struct mpi3mr_softc *sc = cmd->sc;  in mpi3mr_release_command()  local
5046 	mtx_lock(&sc->cmd_pool_lock);  in mpi3mr_release_command()
5047 	TAILQ_INSERT_HEAD(&(sc->cmd_list_head), cmd, next);  in mpi3mr_release_command()
5050 	mpi3mr_dprint(sc, MPI3MR_TRACE, "Release command SMID: 0x%x\n", cmd->hosttag);  in mpi3mr_release_command()
5051 	mtx_unlock(&sc->cmd_pool_lock);  in mpi3mr_release_command()
5064 static void mpi3mr_free_ioctl_dma_memory(struct mpi3mr_softc *sc)  in mpi3mr_free_ioctl_dma_memory()  argument
5070 		mem_desc = &sc->ioctl_sge[i];  in mpi3mr_free_ioctl_dma_memory()
5080 	mem_desc = &sc->ioctl_chain_sge;  in mpi3mr_free_ioctl_dma_memory()
5089 	mem_desc = &sc->ioctl_resp_sge;  in mpi3mr_free_ioctl_dma_memory()
5098 	sc->ioctl_sges_allocated = false;  in mpi3mr_free_ioctl_dma_memory()
5110 void mpi3mr_alloc_ioctl_dma_memory(struct mpi3mr_softc *sc)  in mpi3mr_alloc_ioctl_dma_memory()  argument
5116 		mem_desc = &sc->ioctl_sge[i];  in mpi3mr_alloc_ioctl_dma_memory()
5119 		if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_alloc_ioctl_dma_memory()
5121 					sc->dma_loaddr,		/* lowaddr */  in mpi3mr_alloc_ioctl_dma_memory()
5130 			mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate request DMA tag\n");  in mpi3mr_alloc_ioctl_dma_memory()
5136 			mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: Cannot allocate replies memory\n", __func__);  in mpi3mr_alloc_ioctl_dma_memory()
5147 	mem_desc = &sc->ioctl_chain_sge;  in mpi3mr_alloc_ioctl_dma_memory()
5149 	if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_alloc_ioctl_dma_memory()
5151 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_alloc_ioctl_dma_memory()
5160 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate request DMA tag\n");  in mpi3mr_alloc_ioctl_dma_memory()
5166 		mpi3mr_dprint(sc, MPI3MR_ERROR, "%s: Cannot allocate replies memory\n", __func__);  in mpi3mr_alloc_ioctl_dma_memory()
5176 	mem_desc = &sc->ioctl_resp_sge;  in mpi3mr_alloc_ioctl_dma_memory()
5178 	if (bus_dma_tag_create(sc->mpi3mr_parent_dmat,    /* parent */  in mpi3mr_alloc_ioctl_dma_memory()
5180 				sc->dma_loaddr,		/* lowaddr */  in mpi3mr_alloc_ioctl_dma_memory()
5189 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate request DMA tag\n");  in mpi3mr_alloc_ioctl_dma_memory()
5195 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Cannot allocate replies memory\n");  in mpi3mr_alloc_ioctl_dma_memory()
5205 	sc->ioctl_sges_allocated = true;  in mpi3mr_alloc_ioctl_dma_memory()
5211 	mpi3mr_free_ioctl_dma_memory(sc);  in mpi3mr_alloc_ioctl_dma_memory()
5215 mpi3mr_destory_mtx(struct mpi3mr_softc *sc)  in mpi3mr_destory_mtx()  argument
5221 	if (sc->admin_reply) {  in mpi3mr_destory_mtx()
5222 		if (mtx_initialized(&sc->admin_reply_lock))  in mpi3mr_destory_mtx()
5223 			mtx_destroy(&sc->admin_reply_lock);  in mpi3mr_destory_mtx()
5226 	if (sc->op_reply_q) {  in mpi3mr_destory_mtx()
5227 		for(i = 0; i < sc->num_queues; i++) {  in mpi3mr_destory_mtx()
5228 			op_reply_q = sc->op_reply_q + i;  in mpi3mr_destory_mtx()
5234 	if (sc->op_req_q) {  in mpi3mr_destory_mtx()
5235 		for(i = 0; i < sc->num_queues; i++) {  in mpi3mr_destory_mtx()
5236 			op_req_q = sc->op_req_q + i;  in mpi3mr_destory_mtx()
5242 	if (mtx_initialized(&sc->init_cmds.completion.lock))  in mpi3mr_destory_mtx()
5243 		mtx_destroy(&sc->init_cmds.completion.lock);  in mpi3mr_destory_mtx()
5245 	if (mtx_initialized(&sc->ioctl_cmds.completion.lock))  in mpi3mr_destory_mtx()
5246 		mtx_destroy(&sc->ioctl_cmds.completion.lock);  in mpi3mr_destory_mtx()
5248 	if (mtx_initialized(&sc->host_tm_cmds.completion.lock))  in mpi3mr_destory_mtx()
5249 		mtx_destroy(&sc->host_tm_cmds.completion.lock);  in mpi3mr_destory_mtx()
5252 		if (mtx_initialized(&sc->dev_rmhs_cmds[i].completion.lock))  in mpi3mr_destory_mtx()
5253 			mtx_destroy(&sc->dev_rmhs_cmds[i].completion.lock);  in mpi3mr_destory_mtx()
5256 	if (mtx_initialized(&sc->reset_mutex))  in mpi3mr_destory_mtx()
5257 		mtx_destroy(&sc->reset_mutex);  in mpi3mr_destory_mtx()
5259 	if (mtx_initialized(&sc->target_lock))  in mpi3mr_destory_mtx()
5260 		mtx_destroy(&sc->target_lock);  in mpi3mr_destory_mtx()
5262 	if (mtx_initialized(&sc->fwevt_lock))  in mpi3mr_destory_mtx()
5263 		mtx_destroy(&sc->fwevt_lock);  in mpi3mr_destory_mtx()
5265 	if (mtx_initialized(&sc->cmd_pool_lock))  in mpi3mr_destory_mtx()
5266 		mtx_destroy(&sc->cmd_pool_lock);  in mpi3mr_destory_mtx()
5268 	if (mtx_initialized(&sc->reply_free_q_lock))  in mpi3mr_destory_mtx()
5269 		mtx_destroy(&sc->reply_free_q_lock);  in mpi3mr_destory_mtx()
5271 	if (mtx_initialized(&sc->sense_buf_q_lock))  in mpi3mr_destory_mtx()
5272 		mtx_destroy(&sc->sense_buf_q_lock);  in mpi3mr_destory_mtx()
5274 	if (mtx_initialized(&sc->chain_buf_lock))  in mpi3mr_destory_mtx()
5275 		mtx_destroy(&sc->chain_buf_lock);  in mpi3mr_destory_mtx()
5277 	if (mtx_initialized(&sc->admin_req_lock))  in mpi3mr_destory_mtx()
5278 		mtx_destroy(&sc->admin_req_lock);  in mpi3mr_destory_mtx()
5280 	if (mtx_initialized(&sc->mpi3mr_mtx))  in mpi3mr_destory_mtx()
5281 		mtx_destroy(&sc->mpi3mr_mtx);  in mpi3mr_destory_mtx()
5291 mpi3mr_free_mem(struct mpi3mr_softc *sc)  in mpi3mr_free_mem()  argument
5298 	if (sc->cmd_list) {  in mpi3mr_free_mem()
5299 		for (i = 0; i < sc->max_host_ios; i++) {  in mpi3mr_free_mem()
5300 			free(sc->cmd_list[i], M_MPI3MR);  in mpi3mr_free_mem()
5302 		free(sc->cmd_list, M_MPI3MR);  in mpi3mr_free_mem()
5303 		sc->cmd_list = NULL;  in mpi3mr_free_mem()
5306 	if (sc->pel_seq_number && sc->pel_seq_number_dma) {  in mpi3mr_free_mem()
5307 		bus_dmamap_unload(sc->pel_seq_num_dmatag, sc->pel_seq_num_dmamap);  in mpi3mr_free_mem()
5308 		bus_dmamem_free(sc->pel_seq_num_dmatag, sc->pel_seq_number, sc->pel_seq_num_dmamap);  in mpi3mr_free_mem()
5309 		sc->pel_seq_number = NULL;  in mpi3mr_free_mem()
5310 		if (sc->pel_seq_num_dmatag != NULL)  in mpi3mr_free_mem()
5311 			bus_dma_tag_destroy(sc->pel_seq_num_dmatag);  in mpi3mr_free_mem()
5314 	if (sc->throttle_groups) {  in mpi3mr_free_mem()
5315 		free(sc->throttle_groups, M_MPI3MR);  in mpi3mr_free_mem()
5316 		sc->throttle_groups = NULL;  in mpi3mr_free_mem()
5320 	if (sc->op_req_q) {  in mpi3mr_free_mem()
5321 		for (i = 0; i < sc->num_queues; i++) {  in mpi3mr_free_mem()
5322 			op_req_q = sc->op_req_q + i;  in mpi3mr_free_mem()
5331 		free(sc->op_req_q, M_MPI3MR);  in mpi3mr_free_mem()
5332 		sc->op_req_q = NULL;  in mpi3mr_free_mem()
5335 	if (sc->op_reply_q) {  in mpi3mr_free_mem()
5336 		for (i = 0; i < sc->num_queues; i++) {  in mpi3mr_free_mem()
5337 			op_reply_q = sc->op_reply_q + i;  in mpi3mr_free_mem()
5346 		free(sc->op_reply_q, M_MPI3MR);  in mpi3mr_free_mem()
5347 		sc->op_reply_q = NULL;  in mpi3mr_free_mem()
5351 	if (sc->chain_sgl_list) {  in mpi3mr_free_mem()
5352 		for (i = 0; i < sc->chain_buf_count; i++) {  in mpi3mr_free_mem()
5353 			if (sc->chain_sgl_list[i].buf && sc->chain_sgl_list[i].buf_phys) {  in mpi3mr_free_mem()
5354 				bus_dmamap_unload(sc->chain_sgl_list_tag, sc->chain_sgl_list[i].buf_dmamap);  in mpi3mr_free_mem()
5355 				bus_dmamem_free(sc->chain_sgl_list_tag, sc->chain_sgl_list[i].buf,  in mpi3mr_free_mem()
5356 						sc->chain_sgl_list[i].buf_dmamap);  in mpi3mr_free_mem()
5357 				sc->chain_sgl_list[i].buf = NULL;  in mpi3mr_free_mem()
5360 		if (sc->chain_sgl_list_tag != NULL)  in mpi3mr_free_mem()
5361 			bus_dma_tag_destroy(sc->chain_sgl_list_tag);  in mpi3mr_free_mem()
5362 		free(sc->chain_sgl_list, M_MPI3MR);  in mpi3mr_free_mem()
5363 		sc->chain_sgl_list = NULL;  in mpi3mr_free_mem()
5366 	if (sc->chain_bitmap) {  in mpi3mr_free_mem()
5367 		free(sc->chain_bitmap, M_MPI3MR);  in mpi3mr_free_mem()
5368 		sc->chain_bitmap = NULL;  in mpi3mr_free_mem()
5371 	for (i = 0; i < sc->msix_count; i++) {  in mpi3mr_free_mem()
5372 		irq_ctx = sc->irq_ctx + i;  in mpi3mr_free_mem()
5378 	if (sc->reply_buf && sc->reply_buf_phys) {   in mpi3mr_free_mem()
5379 		bus_dmamap_unload(sc->reply_buf_tag, sc->reply_buf_dmamap);  in mpi3mr_free_mem()
5380 		bus_dmamem_free(sc->reply_buf_tag, sc->reply_buf,  in mpi3mr_free_mem()
5381 				sc->reply_buf_dmamap);  in mpi3mr_free_mem()
5382 		sc->reply_buf = NULL;  in mpi3mr_free_mem()
5383 		if (sc->reply_buf_tag != NULL)  in mpi3mr_free_mem()
5384 			bus_dma_tag_destroy(sc->reply_buf_tag);  in mpi3mr_free_mem()
5388 	if (sc->reply_free_q && sc->reply_free_q_phys) {  in mpi3mr_free_mem()
5389 		bus_dmamap_unload(sc->reply_free_q_tag, sc->reply_free_q_dmamap);  in mpi3mr_free_mem()
5390 		bus_dmamem_free(sc->reply_free_q_tag, sc->reply_free_q,  in mpi3mr_free_mem()
5391 				sc->reply_free_q_dmamap);  in mpi3mr_free_mem()
5392 		sc->reply_free_q = NULL;  in mpi3mr_free_mem()
5393 		if (sc->reply_free_q_tag != NULL)  in mpi3mr_free_mem()
5394 			bus_dma_tag_destroy(sc->reply_free_q_tag);  in mpi3mr_free_mem()
5398 	if (sc->sense_buf && sc->sense_buf_phys) {  in mpi3mr_free_mem()
5399 		bus_dmamap_unload(sc->sense_buf_tag, sc->sense_buf_dmamap);  in mpi3mr_free_mem()
5400 		bus_dmamem_free(sc->sense_buf_tag, sc->sense_buf,  in mpi3mr_free_mem()
5401 				sc->sense_buf_dmamap);  in mpi3mr_free_mem()
5402 		sc->sense_buf = NULL;  in mpi3mr_free_mem()
5403 		if (sc->sense_buf_tag != NULL)  in mpi3mr_free_mem()
5404 			bus_dma_tag_destroy(sc->sense_buf_tag);  in mpi3mr_free_mem()
5408 	if (sc->sense_buf_q && sc->sense_buf_q_phys) {  in mpi3mr_free_mem()
5409 		bus_dmamap_unload(sc->sense_buf_q_tag, sc->sense_buf_q_dmamap);  in mpi3mr_free_mem()
5410 		bus_dmamem_free(sc->sense_buf_q_tag, sc->sense_buf_q,  in mpi3mr_free_mem()
5411 				sc->sense_buf_q_dmamap);  in mpi3mr_free_mem()
5412 		sc->sense_buf_q = NULL;  in mpi3mr_free_mem()
5413 		if (sc->sense_buf_q_tag != NULL)  in mpi3mr_free_mem()
5414 			bus_dma_tag_destroy(sc->sense_buf_q_tag);  in mpi3mr_free_mem()
5418 	if (sc->init_cmds.reply) {  in mpi3mr_free_mem()
5419 		free(sc->init_cmds.reply, M_MPI3MR);  in mpi3mr_free_mem()
5420 		sc->init_cmds.reply = NULL;  in mpi3mr_free_mem()
5423 	if (sc->ioctl_cmds.reply) {  in mpi3mr_free_mem()
5424 		free(sc->ioctl_cmds.reply, M_MPI3MR);  in mpi3mr_free_mem()
5425 		sc->ioctl_cmds.reply = NULL;  in mpi3mr_free_mem()
5428 	if (sc->pel_cmds.reply) {  in mpi3mr_free_mem()
5429 		free(sc->pel_cmds.reply, M_MPI3MR);  in mpi3mr_free_mem()
5430 		sc->pel_cmds.reply = NULL;  in mpi3mr_free_mem()
5433 	if (sc->pel_abort_cmd.reply) {  in mpi3mr_free_mem()
5434 		free(sc->pel_abort_cmd.reply, M_MPI3MR);  in mpi3mr_free_mem()
5435 		sc->pel_abort_cmd.reply = NULL;  in mpi3mr_free_mem()
5438 	if (sc->host_tm_cmds.reply) {  in mpi3mr_free_mem()
5439 		free(sc->host_tm_cmds.reply, M_MPI3MR);  in mpi3mr_free_mem()
5440 		sc->host_tm_cmds.reply = NULL;  in mpi3mr_free_mem()
5443 	if (sc->log_data_buffer) {  in mpi3mr_free_mem()
5444 		free(sc->log_data_buffer, M_MPI3MR);  in mpi3mr_free_mem()
5445 		sc->log_data_buffer = NULL;  in mpi3mr_free_mem()
5449 		if (sc->dev_rmhs_cmds[i].reply) {  in mpi3mr_free_mem()
5450 			free(sc->dev_rmhs_cmds[i].reply, M_MPI3MR);  in mpi3mr_free_mem()
5451 			sc->dev_rmhs_cmds[i].reply = NULL;  in mpi3mr_free_mem()
5456 		if (sc->evtack_cmds[i].reply) {  in mpi3mr_free_mem()
5457 			free(sc->evtack_cmds[i].reply, M_MPI3MR);  in mpi3mr_free_mem()
5458 			sc->evtack_cmds[i].reply = NULL;  in mpi3mr_free_mem()
5462 	if (sc->removepend_bitmap) {  in mpi3mr_free_mem()
5463 		free(sc->removepend_bitmap, M_MPI3MR);  in mpi3mr_free_mem()
5464 		sc->removepend_bitmap = NULL;  in mpi3mr_free_mem()
5467 	if (sc->devrem_bitmap) {  in mpi3mr_free_mem()
5468 		free(sc->devrem_bitmap, M_MPI3MR);  in mpi3mr_free_mem()
5469 		sc->devrem_bitmap = NULL;  in mpi3mr_free_mem()
5472 	if (sc->evtack_cmds_bitmap) {  in mpi3mr_free_mem()
5473 		free(sc->evtack_cmds_bitmap, M_MPI3MR);  in mpi3mr_free_mem()
5474 		sc->evtack_cmds_bitmap = NULL;  in mpi3mr_free_mem()
5478 	if (sc->admin_reply && sc->admin_reply_phys) {  in mpi3mr_free_mem()
5479 		bus_dmamap_unload(sc->admin_reply_tag, sc->admin_reply_dmamap);  in mpi3mr_free_mem()
5480 		bus_dmamem_free(sc->admin_reply_tag, sc->admin_reply,  in mpi3mr_free_mem()
5481 				sc->admin_reply_dmamap);  in mpi3mr_free_mem()
5482 		sc->admin_reply = NULL;  in mpi3mr_free_mem()
5483 		if (sc->admin_reply_tag != NULL)  in mpi3mr_free_mem()
5484 			bus_dma_tag_destroy(sc->admin_reply_tag);  in mpi3mr_free_mem()
5488 	if (sc->admin_req && sc->admin_req_phys) {  in mpi3mr_free_mem()
5489 		bus_dmamap_unload(sc->admin_req_tag, sc->admin_req_dmamap);  in mpi3mr_free_mem()
5490 		bus_dmamem_free(sc->admin_req_tag, sc->admin_req,  in mpi3mr_free_mem()
5491 				sc->admin_req_dmamap);  in mpi3mr_free_mem()
5492 		sc->admin_req = NULL;  in mpi3mr_free_mem()
5493 		if (sc->admin_req_tag != NULL)  in mpi3mr_free_mem()
5494 			bus_dma_tag_destroy(sc->admin_req_tag);  in mpi3mr_free_mem()
5496 	mpi3mr_free_ioctl_dma_memory(sc);  in mpi3mr_free_mem()
5510 static inline void mpi3mr_drv_cmd_comp_reset(struct mpi3mr_softc *sc,  in mpi3mr_drv_cmd_comp_reset()  argument
5520 			cmdptr->callback(sc, cmdptr);  in mpi3mr_drv_cmd_comp_reset()
5532 static void mpi3mr_flush_drv_cmds(struct mpi3mr_softc *sc)  in mpi3mr_flush_drv_cmds()  argument
5537 	cmdptr = &sc->init_cmds;  in mpi3mr_flush_drv_cmds()
5538 	mpi3mr_drv_cmd_comp_reset(sc, cmdptr);  in mpi3mr_flush_drv_cmds()
5540 	cmdptr = &sc->ioctl_cmds;  in mpi3mr_flush_drv_cmds()
5541 	mpi3mr_drv_cmd_comp_reset(sc, cmdptr);  in mpi3mr_flush_drv_cmds()
5543 	cmdptr = &sc->host_tm_cmds;  in mpi3mr_flush_drv_cmds()
5544 	mpi3mr_drv_cmd_comp_reset(sc, cmdptr);  in mpi3mr_flush_drv_cmds()
5547 		cmdptr = &sc->dev_rmhs_cmds[i];  in mpi3mr_flush_drv_cmds()
5548 		mpi3mr_drv_cmd_comp_reset(sc, cmdptr);  in mpi3mr_flush_drv_cmds()
5552 		cmdptr = &sc->evtack_cmds[i];  in mpi3mr_flush_drv_cmds()
5553 		mpi3mr_drv_cmd_comp_reset(sc, cmdptr);  in mpi3mr_flush_drv_cmds()
5556 	cmdptr = &sc->pel_cmds;  in mpi3mr_flush_drv_cmds()
5557 	mpi3mr_drv_cmd_comp_reset(sc, cmdptr);  in mpi3mr_flush_drv_cmds()
5559 	cmdptr = &sc->pel_abort_cmd;  in mpi3mr_flush_drv_cmds()
5560 	mpi3mr_drv_cmd_comp_reset(sc, cmdptr);  in mpi3mr_flush_drv_cmds()
5574 static void mpi3mr_memset_buffers(struct mpi3mr_softc *sc)  in mpi3mr_memset_buffers()  argument
5579 	memset(sc->admin_req, 0, sc->admin_req_q_sz);  in mpi3mr_memset_buffers()
5580 	memset(sc->admin_reply, 0, sc->admin_reply_q_sz);  in mpi3mr_memset_buffers()
5582 	memset(sc->init_cmds.reply, 0, sc->reply_sz);  in mpi3mr_memset_buffers()
5583 	memset(sc->ioctl_cmds.reply, 0, sc->reply_sz);  in mpi3mr_memset_buffers()
5584 	memset(sc->host_tm_cmds.reply, 0, sc->reply_sz);  in mpi3mr_memset_buffers()
5585 	memset(sc->pel_cmds.reply, 0, sc->reply_sz);  in mpi3mr_memset_buffers()
5586 	memset(sc->pel_abort_cmd.reply, 0, sc->reply_sz);  in mpi3mr_memset_buffers()
5588 		memset(sc->dev_rmhs_cmds[i].reply, 0, sc->reply_sz);  in mpi3mr_memset_buffers()
5590 		memset(sc->evtack_cmds[i].reply, 0, sc->reply_sz);  in mpi3mr_memset_buffers()
5591 	memset(sc->removepend_bitmap, 0, sc->dev_handle_bitmap_sz);  in mpi3mr_memset_buffers()
5592 	memset(sc->devrem_bitmap, 0, sc->devrem_bitmap_sz);  in mpi3mr_memset_buffers()
5593 	memset(sc->evtack_cmds_bitmap, 0, sc->evtack_cmds_bitmap_sz);  in mpi3mr_memset_buffers()
5595 	for (i = 0; i < sc->num_queues; i++) {  in mpi3mr_memset_buffers()
5596 		sc->op_reply_q[i].qid = 0;  in mpi3mr_memset_buffers()
5597 		sc->op_reply_q[i].ci = 0;  in mpi3mr_memset_buffers()
5598 		sc->op_reply_q[i].num_replies = 0;  in mpi3mr_memset_buffers()
5599 		sc->op_reply_q[i].ephase = 0;  in mpi3mr_memset_buffers()
5600 		mpi3mr_atomic_set(&sc->op_reply_q[i].pend_ios, 0);  in mpi3mr_memset_buffers()
5601 		memset(sc->op_reply_q[i].q_base, 0, sc->op_reply_q[i].qsz);  in mpi3mr_memset_buffers()
5603 		sc->op_req_q[i].ci = 0;  in mpi3mr_memset_buffers()
5604 		sc->op_req_q[i].pi = 0;  in mpi3mr_memset_buffers()
5605 		sc->op_req_q[i].num_reqs = 0;  in mpi3mr_memset_buffers()
5606 		sc->op_req_q[i].qid = 0;  in mpi3mr_memset_buffers()
5607 		sc->op_req_q[i].reply_qid = 0;  in mpi3mr_memset_buffers()
5608 		memset(sc->op_req_q[i].q_base, 0, sc->op_req_q[i].qsz);  in mpi3mr_memset_buffers()
5611 	mpi3mr_atomic_set(&sc->pend_large_data_sz, 0);  in mpi3mr_memset_buffers()
5612 	if (sc->throttle_groups) {  in mpi3mr_memset_buffers()
5613 		tg = sc->throttle_groups;  in mpi3mr_memset_buffers()
5614 		for (i = 0; i < sc->num_io_throttle_group; i++, tg++) {  in mpi3mr_memset_buffers()
5635 static void mpi3mr_invalidate_devhandles(struct mpi3mr_softc *sc)  in mpi3mr_invalidate_devhandles()  argument
5639 	mtx_lock_spin(&sc->target_lock);  in mpi3mr_invalidate_devhandles()
5640 	TAILQ_FOREACH(target, &sc->cam_sc->tgt_list, tgt_next) {  in mpi3mr_invalidate_devhandles()
5648 	mtx_unlock_spin(&sc->target_lock);  in mpi3mr_invalidate_devhandles()
5662 static void mpi3mr_rfresh_tgtdevs(struct mpi3mr_softc *sc)  in mpi3mr_rfresh_tgtdevs()  argument
5667 	TAILQ_FOREACH_SAFE(target, &sc->cam_sc->tgt_list, tgt_next, target_temp) {  in mpi3mr_rfresh_tgtdevs()
5670 				mpi3mr_remove_device_from_os(sc, target->dev_handle);  in mpi3mr_rfresh_tgtdevs()
5671 			mpi3mr_remove_device_from_list(sc, target, true);  in mpi3mr_rfresh_tgtdevs()
5675 	TAILQ_FOREACH(target, &sc->cam_sc->tgt_list, tgt_next) {  in mpi3mr_rfresh_tgtdevs()
5678 			mpi3mr_add_device(sc, target->per_id);  in mpi3mr_rfresh_tgtdevs()
5684 static void mpi3mr_flush_io(struct mpi3mr_softc *sc)  in mpi3mr_flush_io()  argument
5690 	for (i = 0; i < sc->max_host_ios; i++) {  in mpi3mr_flush_io()
5691 		cmd = sc->cmd_list[i];  in mpi3mr_flush_io()
5697 				mpi3mr_cmd_done(sc, cmd);  in mpi3mr_flush_io()
5714 static inline void mpi3mr_clear_reset_history(struct mpi3mr_softc *sc)  in mpi3mr_clear_reset_history()  argument
5718 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_clear_reset_history()
5720 		mpi3mr_regwrite(sc, MPI3_SYSIF_IOC_STATUS_OFFSET, ioc_status);  in mpi3mr_clear_reset_history()
5732 static inline void mpi3mr_set_diagsave(struct mpi3mr_softc *sc)  in mpi3mr_set_diagsave()  argument
5737 	    mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_set_diagsave()
5739 	mpi3mr_regwrite(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET, ioc_config);  in mpi3mr_set_diagsave()
5755 static int mpi3mr_issue_reset(struct mpi3mr_softc *sc, U16 reset_type,  in mpi3mr_issue_reset()  argument
5766 	if (sc->unrecoverable)  in mpi3mr_issue_reset()
5774 	mpi3mr_dprint(sc, MPI3MR_INFO, "%s reset due to %s(0x%x)\n",  in mpi3mr_issue_reset()
5778 	mpi3mr_clear_reset_history(sc);  in mpi3mr_issue_reset()
5780 		mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_issue_reset()
5784 			mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_issue_reset()
5788 			sc->unrecoverable = 1;  in mpi3mr_issue_reset()
5792 		mpi3mr_regwrite(sc, MPI3_SYSIF_WRITE_SEQUENCE_OFFSET,  in mpi3mr_issue_reset()
5794 		mpi3mr_regwrite(sc, MPI3_SYSIF_WRITE_SEQUENCE_OFFSET,  in mpi3mr_issue_reset()
5796 		mpi3mr_regwrite(sc, MPI3_SYSIF_WRITE_SEQUENCE_OFFSET,  in mpi3mr_issue_reset()
5798 		mpi3mr_regwrite(sc, MPI3_SYSIF_WRITE_SEQUENCE_OFFSET,  in mpi3mr_issue_reset()
5800 		mpi3mr_regwrite(sc, MPI3_SYSIF_WRITE_SEQUENCE_OFFSET,  in mpi3mr_issue_reset()
5802 		mpi3mr_regwrite(sc, MPI3_SYSIF_WRITE_SEQUENCE_OFFSET,  in mpi3mr_issue_reset()
5804 		mpi3mr_regwrite(sc, MPI3_SYSIF_WRITE_SEQUENCE_OFFSET,  in mpi3mr_issue_reset()
5808 		host_diagnostic = mpi3mr_regread(sc, MPI3_SYSIF_HOST_DIAG_OFFSET);  in mpi3mr_issue_reset()
5809 		mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_issue_reset()
5814 	mpi3mr_regwrite(sc, MPI3_SYSIF_SCRATCHPAD0_OFFSET, reset_reason);  in mpi3mr_issue_reset()
5815 	mpi3mr_regwrite(sc, MPI3_SYSIF_HOST_DIAG_OFFSET, host_diagnostic | reset_type);  in mpi3mr_issue_reset()
5819 			ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_issue_reset()
5823 				    mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_issue_reset()
5826 					mpi3mr_clear_reset_history(sc);  in mpi3mr_issue_reset()
5835 			ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_issue_reset()
5836 			if (mpi3mr_diagfault_success(sc, ioc_status)) {  in mpi3mr_issue_reset()
5844 	mpi3mr_regwrite(sc, MPI3_SYSIF_WRITE_SEQUENCE_OFFSET,   in mpi3mr_issue_reset()
5847 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_issue_reset()
5848 	ioc_config = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_issue_reset()
5850 	mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_issue_reset()
5856 		sc->unrecoverable = 1;  in mpi3mr_issue_reset()
5861 inline void mpi3mr_cleanup_event_taskq(struct mpi3mr_softc *sc)  in mpi3mr_cleanup_event_taskq()  argument
5870 	taskqueue_block(sc->cam_sc->ev_tq);  in mpi3mr_cleanup_event_taskq()
5871 	while (taskqueue_cancel(sc->cam_sc->ev_tq, &sc->cam_sc->ev_task, NULL) != 0) {  in mpi3mr_cleanup_event_taskq()
5872 		taskqueue_drain(sc->cam_sc->ev_tq, &sc->cam_sc->ev_task);  in mpi3mr_cleanup_event_taskq()
5892 int mpi3mr_soft_reset_handler(struct mpi3mr_softc *sc,  in mpi3mr_soft_reset_handler()  argument
5898 	mpi3mr_dprint(sc, MPI3MR_INFO, "soft reset invoked: reason code: %s\n",  in mpi3mr_soft_reset_handler()
5902 	     (sc->reset.ioctl_reset_snapdump != true))  in mpi3mr_soft_reset_handler()
5905 	mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_soft_reset_handler()
5907 	while (sc->diagsave_timeout)  in mpi3mr_soft_reset_handler()
5910 	ioc_state = mpi3mr_get_iocstate(sc);  in mpi3mr_soft_reset_handler()
5912 		mpi3mr_dprint(sc, MPI3MR_ERROR, "controller is in unrecoverable state, exit\n");  in mpi3mr_soft_reset_handler()
5913 		sc->reset.type = MPI3MR_NO_RESET;  in mpi3mr_soft_reset_handler()
5914 		sc->reset.reason = MPI3MR_DEFAULT_RESET_REASON;  in mpi3mr_soft_reset_handler()
5915 		sc->reset.status = -1;   in mpi3mr_soft_reset_handler()
5916 		sc->reset.ioctl_reset_snapdump = false;	  in mpi3mr_soft_reset_handler()
5920 	if (sc->reset_in_progress) {  in mpi3mr_soft_reset_handler()
5921 		mpi3mr_dprint(sc, MPI3MR_INFO, "reset is already in progress, exit\n");  in mpi3mr_soft_reset_handler()
5926 	xpt_freeze_simq(sc->cam_sc->sim, 1);  in mpi3mr_soft_reset_handler()
5928 	mpi3mr_cleanup_event_taskq(sc);  in mpi3mr_soft_reset_handler()
5930 	sc->reset_in_progress = 1;  in mpi3mr_soft_reset_handler()
5931 	sc->block_ioctls = 1;  in mpi3mr_soft_reset_handler()
5933 	while (mpi3mr_atomic_read(&sc->pend_ioctls) && (i < PEND_IOCTLS_COMP_WAIT_TIME)) {  in mpi3mr_soft_reset_handler()
5934 		ioc_state = mpi3mr_get_iocstate(sc);  in mpi3mr_soft_reset_handler()
5939 			mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_soft_reset_handler()
5949 		mpi3mr_dprint(sc, MPI3MR_INFO, "Turn off events prior to reset\n");  in mpi3mr_soft_reset_handler()
5952 			sc->event_masks[i] = -1;  in mpi3mr_soft_reset_handler()
5953 		mpi3mr_issue_event_notification(sc);  in mpi3mr_soft_reset_handler()
5956 	mpi3mr_disable_interrupts(sc);  in mpi3mr_soft_reset_handler()
5959 		mpi3mr_trigger_snapdump(sc, reset_reason);  in mpi3mr_soft_reset_handler()
5961 	retval = mpi3mr_issue_reset(sc,  in mpi3mr_soft_reset_handler()
5964 		mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to issue soft reset to the ioc\n");  in mpi3mr_soft_reset_handler()
5968 	mpi3mr_flush_drv_cmds(sc);  in mpi3mr_soft_reset_handler()
5969 	mpi3mr_flush_io(sc);  in mpi3mr_soft_reset_handler()
5970 	mpi3mr_invalidate_devhandles(sc);  in mpi3mr_soft_reset_handler()
5971 	mpi3mr_memset_buffers(sc);  in mpi3mr_soft_reset_handler()
5973 	if (sc->prepare_for_reset) {  in mpi3mr_soft_reset_handler()
5974 		sc->prepare_for_reset = 0;  in mpi3mr_soft_reset_handler()
5975 		sc->prepare_for_reset_timeout_counter = 0;  in mpi3mr_soft_reset_handler()
5978 	retval = mpi3mr_initialize_ioc(sc, MPI3MR_INIT_TYPE_RESET);  in mpi3mr_soft_reset_handler()
5980 		mpi3mr_dprint(sc, MPI3MR_ERROR, "reinit after soft reset failed: reason %d\n",  in mpi3mr_soft_reset_handler()
5988 		sc->diagsave_timeout = 0;  in mpi3mr_soft_reset_handler()
5989 		sc->reset_in_progress = 0;  in mpi3mr_soft_reset_handler()
5990 		mpi3mr_rfresh_tgtdevs(sc);  in mpi3mr_soft_reset_handler()
5991 		sc->ts_update_counter = 0;  in mpi3mr_soft_reset_handler()
5992 		sc->block_ioctls = 0;  in mpi3mr_soft_reset_handler()
5993 		sc->pel_abort_requested = 0;  in mpi3mr_soft_reset_handler()
5994 		if (sc->pel_wait_pend) {  in mpi3mr_soft_reset_handler()
5995 			sc->pel_cmds.retry_count = 0;  in mpi3mr_soft_reset_handler()
5996 			mpi3mr_issue_pel_wait(sc, &sc->pel_cmds);  in mpi3mr_soft_reset_handler()
5997 			mpi3mr_app_send_aen(sc);  in mpi3mr_soft_reset_handler()
6000 		mpi3mr_issue_reset(sc,  in mpi3mr_soft_reset_handler()
6002 		sc->unrecoverable = 1;  in mpi3mr_soft_reset_handler()
6003 		sc->reset_in_progress = 0;  in mpi3mr_soft_reset_handler()
6006 	mpi3mr_dprint(sc, MPI3MR_INFO, "Soft Reset: %s\n", ((retval == 0) ? "SUCCESS" : "FAILED"));  in mpi3mr_soft_reset_handler()
6008 	taskqueue_unblock(sc->cam_sc->ev_tq);  in mpi3mr_soft_reset_handler()
6009 	xpt_release_simq(sc->cam_sc->sim, 1);  in mpi3mr_soft_reset_handler()
6011 	sc->reset.type = MPI3MR_NO_RESET;  in mpi3mr_soft_reset_handler()
6012 	sc->reset.reason = MPI3MR_DEFAULT_RESET_REASON;  in mpi3mr_soft_reset_handler()
6013 	sc->reset.status = retval;   in mpi3mr_soft_reset_handler()
6014 	sc->reset.ioctl_reset_snapdump = false;	  in mpi3mr_soft_reset_handler()
6028 static void mpi3mr_issue_ioc_shutdown(struct mpi3mr_softc *sc)  in mpi3mr_issue_ioc_shutdown()  argument
6034 	mpi3mr_dprint(sc, MPI3MR_INFO, "sending shutdown notification\n");  in mpi3mr_issue_ioc_shutdown()
6035 	if (sc->unrecoverable) {  in mpi3mr_issue_ioc_shutdown()
6036 		mpi3mr_dprint(sc, MPI3MR_ERROR,   in mpi3mr_issue_ioc_shutdown()
6040 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_issue_ioc_shutdown()
6043 		mpi3mr_dprint(sc, MPI3MR_ERROR, "shutdown already in progress\n");  in mpi3mr_issue_ioc_shutdown()
6047 	ioc_config = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_issue_ioc_shutdown()
6051 	mpi3mr_regwrite(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET, ioc_config);  in mpi3mr_issue_ioc_shutdown()
6053 	if (sc->facts.shutdown_timeout)  in mpi3mr_issue_ioc_shutdown()
6054 		timeout = sc->facts.shutdown_timeout * 10;  in mpi3mr_issue_ioc_shutdown()
6057 		ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_issue_ioc_shutdown()
6064 		if (sc->unrecoverable)  in mpi3mr_issue_ioc_shutdown()
6068 			mpi3mr_print_fault_info(sc);  in mpi3mr_issue_ioc_shutdown()
6073 			if (mpi3mr_issue_reset(sc,  in mpi3mr_issue_ioc_shutdown()
6078 			ioc_config = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_issue_ioc_shutdown()
6082 			mpi3mr_regwrite(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET, ioc_config);  in mpi3mr_issue_ioc_shutdown()
6084 			if (sc->facts.shutdown_timeout)  in mpi3mr_issue_ioc_shutdown()
6085 				timeout = sc->facts.shutdown_timeout * 10;  in mpi3mr_issue_ioc_shutdown()
6094 	ioc_status = mpi3mr_regread(sc, MPI3_SYSIF_IOC_STATUS_OFFSET);  in mpi3mr_issue_ioc_shutdown()
6095 	ioc_config = mpi3mr_regread(sc, MPI3_SYSIF_IOC_CONFIG_OFFSET);  in mpi3mr_issue_ioc_shutdown()
6100 			mpi3mr_dprint(sc, MPI3MR_ERROR,  in mpi3mr_issue_ioc_shutdown()
6104 	mpi3mr_dprint(sc, MPI3MR_INFO,  in mpi3mr_issue_ioc_shutdown()
6119 void mpi3mr_cleanup_ioc(struct mpi3mr_softc *sc)  in mpi3mr_cleanup_ioc()  argument
6123 	mpi3mr_dprint(sc, MPI3MR_INFO, "cleaning up the controller\n");  in mpi3mr_cleanup_ioc()
6124 	mpi3mr_disable_interrupts(sc);  in mpi3mr_cleanup_ioc()
6126 	ioc_state = mpi3mr_get_iocstate(sc);  in mpi3mr_cleanup_ioc()
6128 	if ((!sc->unrecoverable) && (!sc->reset_in_progress) &&  in mpi3mr_cleanup_ioc()
6130 		if (mpi3mr_mur_ioc(sc,  in mpi3mr_cleanup_ioc()
6132 			mpi3mr_issue_reset(sc,  in mpi3mr_cleanup_ioc()
6135 		mpi3mr_issue_ioc_shutdown(sc);  in mpi3mr_cleanup_ioc()
6138 	mpi3mr_dprint(sc, MPI3MR_INFO, "controller cleanup completed\n");  in mpi3mr_cleanup_ioc()