52 					       void (* func)(struct twe_request *tr));
63 static int	twe_wait_request(struct twe_request *tr);
64 static int	twe_immediate_request(struct twe_request *tr, int usetmp);
65 static void	twe_completeio(struct twe_request *tr);
91 static void	twe_handle_aen(struct twe_request *tr);
100 static int	twe_get_request(struct twe_softc *sc, struct twe_request **tr);
101 static void	twe_release_request(struct twe_request *tr);
107 static int	twe_report_request(struct twe_request *tr);
122     struct twe_request	*tr;  in twe_setup()  local
143 	if ((tr = twe_allocate_request(sc, i)) == NULL)  in twe_setup()
148 	cmd = TWE_FIND_COMMAND(tr);  in twe_setup()
151 	sc->twe_lookup[i] = tr;  in twe_setup()
157 	twe_release_request(tr);  in twe_setup()
403     struct twe_request	*tr;  in twe_startio()  local
418 	tr = twe_dequeue_ready(sc);  in twe_startio()
421 	if (tr == NULL) {  in twe_startio()
424 	    if (twe_get_request(sc, &tr))  in twe_startio()
429 		twe_release_request(tr);  in twe_startio()
434 	    tr->tr_complete = twe_completeio;  in twe_startio()
435 	    tr->tr_private = bp;  in twe_startio()
436 	    tr->tr_data = bp->bio_data;  in twe_startio()
437 	    tr->tr_length = bp->bio_bcount;  in twe_startio()
438 	    cmd = TWE_FIND_COMMAND(tr);  in twe_startio()
440 		tr->tr_flags |= TWE_CMD_DATAIN;  in twe_startio()
443 		tr->tr_flags |= TWE_CMD_DATAOUT;  in twe_startio()
450 	    cmd->io.block_count = (tr->tr_length + TWE_BLOCK_SIZE - 1) / TWE_BLOCK_SIZE;  in twe_startio()
455 	if (tr == NULL)  in twe_startio()
459 	error = twe_map_request(tr);  in twe_startio()
464 	    tr->tr_status = TWE_CMD_ERROR;  in twe_startio()
465 	    if (tr->tr_private != NULL) {  in twe_startio()
466 		bp = (struct bio *)(tr->tr_private);  in twe_startio()
469 		tr->tr_private = NULL;  in twe_startio()
471 	        twe_release_request(tr);  in twe_startio()
472 	    } else if (tr->tr_flags & TWE_CMD_SLEEPER)  in twe_startio()
473 		wakeup_one(tr); /* wakeup the sleeping owner */  in twe_startio()
484     struct twe_request	*tr;  in twe_dump_blocks()  local
488     if (twe_get_request(sc, &tr))  in twe_dump_blocks()
491     tr->tr_data = data;  in twe_dump_blocks()
492     tr->tr_status = TWE_CMD_SETUP;  in twe_dump_blocks()
493     tr->tr_length = nblks * TWE_BLOCK_SIZE;  in twe_dump_blocks()
494     tr->tr_flags = TWE_CMD_DATAOUT;  in twe_dump_blocks()
496     cmd = TWE_FIND_COMMAND(tr);  in twe_dump_blocks()
503     error = twe_immediate_request(tr, 0);  in twe_dump_blocks()
505 	if (twe_report_request(tr))  in twe_dump_blocks()
507     twe_release_request(tr);  in twe_dump_blocks()
525     struct twe_request		*tr;  in twe_ioctl()  local
551 	while (twe_get_request(sc, &tr))  in twe_ioctl()
558 	cmd = TWE_FIND_COMMAND(tr);  in twe_ioctl()
563 	tr->tr_length = tr_length;  in twe_ioctl()
564 	tr->tr_data = data;  in twe_ioctl()
565 	if (tr->tr_length > 0) {  in twe_ioctl()
566 	    tr->tr_flags |= TWE_CMD_DATAIN | TWE_CMD_DATAOUT;  in twe_ioctl()
570 	error = twe_wait_request(tr);  in twe_ioctl()
579 	if (tr->tr_length > 0)  in twe_ioctl()
580 	    error = copyout(tr->tr_data, tu->tu_data, tu->tu_size);  in twe_ioctl()
584 	if (tr->tr_data != NULL)  in twe_ioctl()
585 	    free(tr->tr_data, M_DEVBUF);  in twe_ioctl()
587 	twe_release_request(tr);  in twe_ioctl()
766 	      void (* func)(struct twe_request *tr))  in twe_get_param()  argument
768     struct twe_request	*tr;  in twe_get_param()  local
776     tr = NULL;  in twe_get_param()
780     if (twe_get_request(sc, &tr))  in twe_get_param()
786     tr->tr_data = param;  in twe_get_param()
787     tr->tr_length = TWE_SECTOR_SIZE;  in twe_get_param()
788     tr->tr_flags = TWE_CMD_DATAIN | TWE_CMD_DATAOUT;  in twe_get_param()
791     cmd = TWE_FIND_COMMAND(tr);  in twe_get_param()
805 	error = twe_immediate_request(tr, 1 /* usetmp */);  in twe_get_param()
807 	    if (twe_report_request(tr))  in twe_get_param()
812 	twe_release_request(tr);  in twe_get_param()
815 	tr->tr_complete = func;  in twe_get_param()
816 	error = twe_map_request(tr);  in twe_get_param()
824     if (tr != NULL)  in twe_get_param()
825 	twe_release_request(tr);  in twe_get_param()
862     struct twe_request	*tr;  in twe_set_param()  local
870     tr = NULL;  in twe_set_param()
875     if (twe_get_request(sc, &tr))  in twe_set_param()
881     tr->tr_data = param;  in twe_set_param()
882     tr->tr_length = TWE_SECTOR_SIZE;  in twe_set_param()
883     tr->tr_flags = TWE_CMD_DATAIN | TWE_CMD_DATAOUT;  in twe_set_param()
886     cmd = TWE_FIND_COMMAND(tr);  in twe_set_param()
899     error = twe_immediate_request(tr, 1 /* usetmp */);  in twe_set_param()
901 	if (twe_report_request(tr))  in twe_set_param()
906     if (tr != NULL)  in twe_set_param()
907 	twe_release_request(tr);  in twe_set_param()
921     struct twe_request	*tr;  in twe_init_connection()  local
930     if (twe_get_request(sc, &tr))  in twe_init_connection()
934     cmd = TWE_FIND_COMMAND(tr);  in twe_init_connection()
942     error = twe_immediate_request(tr, 0 /* usetmp */);  in twe_init_connection()
943     twe_release_request(tr);  in twe_init_connection()
956 twe_wait_request(struct twe_request *tr)  in twe_wait_request()  argument
961     TWE_IO_ASSERT_LOCKED(tr->tr_sc);  in twe_wait_request()
962     tr->tr_flags |= TWE_CMD_SLEEPER;  in twe_wait_request()
963     tr->tr_status = TWE_CMD_BUSY;  in twe_wait_request()
964     twe_enqueue_ready(tr);  in twe_wait_request()
965     twe_startio(tr->tr_sc);  in twe_wait_request()
966     while (tr->tr_status == TWE_CMD_BUSY)  in twe_wait_request()
967 	mtx_sleep(tr, &tr->tr_sc->twe_io_lock, PRIBIO, "twewait", 0);  in twe_wait_request()
969     return(tr->tr_status != TWE_CMD_COMPLETE);  in twe_wait_request()
978 twe_immediate_request(struct twe_request *tr, int usetmp)  in twe_immediate_request()  argument
986     sc = tr->tr_sc;  in twe_immediate_request()
988     if (usetmp && (tr->tr_data != NULL)) {  in twe_immediate_request()
989 	tr->tr_flags |= TWE_CMD_IMMEDIATE;  in twe_immediate_request()
990 	if (tr->tr_length > DFLTPHYS)  in twe_immediate_request()
992 	bcopy(tr->tr_data, sc->twe_immediate, tr->tr_length);  in twe_immediate_request()
994     tr->tr_status = TWE_CMD_BUSY;  in twe_immediate_request()
995     if ((error = twe_map_request(tr)) != 0)  in twe_immediate_request()
1000     while ((count++ < 5000) && (tr->tr_status == TWE_CMD_BUSY)){  in twe_immediate_request()
1004     if (usetmp && (tr->tr_data != NULL))  in twe_immediate_request()
1005 	bcopy(sc->twe_immediate, tr->tr_data, tr->tr_length);  in twe_immediate_request()
1007     return(tr->tr_status != TWE_CMD_COMPLETE);  in twe_immediate_request()
1014 twe_completeio(struct twe_request *tr)  in twe_completeio()  argument
1016     TWE_Command		*cmd = TWE_FIND_COMMAND(tr);  in twe_completeio()
1017     struct twe_softc	*sc = tr->tr_sc;  in twe_completeio()
1018     struct bio		*bp = tr->tr_private;  in twe_completeio()
1022     if (tr->tr_status == TWE_CMD_COMPLETE) {  in twe_completeio()
1025 	    if (twe_report_request(tr)) {  in twe_completeio()
1033     tr->tr_private = NULL;  in twe_completeio()
1035     twe_release_request(tr);  in twe_completeio()
1045     struct twe_request	*tr;  in twe_reset()  local
1081     while ((tr = twe_dequeue_busy(sc)) != NULL) {  in twe_reset()
1082 	twe_enqueue_ready(tr);  in twe_reset()
1108 twe_start(struct twe_request *tr)  in twe_start()  argument
1110     struct twe_softc	*sc = tr->tr_sc;  in twe_start()
1121     tr->tr_status = TWE_CMD_BUSY;  in twe_start()
1122     cmd = TWE_FIND_COMMAND(tr);  in twe_start()
1137 	    twe_enqueue_busy(tr);  in twe_start()
1139 	    TWE_COMMAND_QUEUE(sc, TWE_FIND_COMMANDPHYS(tr));  in twe_start()
1143 	    if (tr->tr_complete != NULL) {  in twe_start()
1144 		debug(3, "queued request %d with callback %p", cmd->generic.request_id, tr->tr_complete);  in twe_start()
1145 	    } else if (tr->tr_flags & TWE_CMD_SLEEPER) {  in twe_start()
1146 		debug(3, "queued request %d with wait channel %p", cmd->generic.request_id, tr);  in twe_start()
1175     struct twe_request	*tr;  in twe_done()  local
1190 	    tr = sc->twe_lookup[rq.u.response_id];	/* find command */  in twe_done()
1191 	    cmd = TWE_FIND_COMMAND(tr);  in twe_done()
1192 	    if (tr->tr_status != TWE_CMD_BUSY)  in twe_done()
1194 	    tr->tr_status = TWE_CMD_COMPLETE;  in twe_done()
1198 	    twe_remove_busy(tr);  in twe_done()
1199 	    twe_enqueue_complete(tr);  in twe_done()
1223     struct twe_request	*tr;  in twe_complete()  local
1230     while ((tr = twe_dequeue_complete(sc)) != NULL) {  in twe_complete()
1232 	twe_unmap_request(tr);  in twe_complete()
1235 	if (tr->tr_complete != NULL) {		/* completion callback */  in twe_complete()
1236 	    debug(2, "call completion handler %p", tr->tr_complete);  in twe_complete()
1237 	    tr->tr_complete(tr);  in twe_complete()
1239 	} else if (tr->tr_flags & TWE_CMD_SLEEPER) {	/* caller is asleep waiting */  in twe_complete()
1240 	    debug(2, "wake up command owner on %p", tr);  in twe_complete()
1241 	    wakeup_one(tr);  in twe_complete()
1416 twe_handle_aen(struct twe_request *tr)  in twe_handle_aen()  argument
1418     struct twe_softc	*sc = tr->tr_sc;  in twe_handle_aen()
1426     param = (TWE_Param *)tr->tr_data;  in twe_handle_aen()
1429     free(tr->tr_data, M_DEVBUF);  in twe_handle_aen()
1430     twe_release_request(tr);  in twe_handle_aen()
1583 twe_get_request(struct twe_softc *sc, struct twe_request **tr)  in twe_get_request()  argument
1592     *tr = twe_dequeue_free(sc);  in twe_get_request()
1595     if (*tr != NULL) {  in twe_get_request()
1596 	cmd = TWE_FIND_COMMAND(*tr);  in twe_get_request()
1597 	(*tr)->tr_data = NULL;  in twe_get_request()
1598 	(*tr)->tr_private = NULL;  in twe_get_request()
1599 	(*tr)->tr_status = TWE_CMD_SETUP;		/* command is in setup phase */  in twe_get_request()
1600 	(*tr)->tr_flags = 0;  in twe_get_request()
1601 	(*tr)->tr_complete = NULL;  in twe_get_request()
1605     return(*tr == NULL);  in twe_get_request()
1613 twe_release_request(struct twe_request *tr)  in twe_release_request()  argument
1618 	TWE_IO_ASSERT_LOCKED(tr->tr_sc);  in twe_release_request()
1619     if (tr->tr_private != NULL)  in twe_release_request()
1620 	twe_panic(tr->tr_sc, "tr_private != NULL");  in twe_release_request()
1621     twe_enqueue_free(tr);  in twe_release_request()
1822 twe_report_request(struct twe_request *tr)  in twe_report_request()  argument
1824     struct twe_softc	*sc = tr->tr_sc;  in twe_report_request()
1825     TWE_Command		*cmd = TWE_FIND_COMMAND(tr);  in twe_report_request()
1926 twe_print_request(struct twe_request *tr)
1928     struct twe_softc	*sc = tr->tr_sc;
1929     TWE_Command	*cmd = TWE_FIND_COMMAND(tr);
1963 	       tr, TWE_FIND_COMMANDPHYS(tr), tr->tr_data, tr->tr_dataphys, tr->tr_length);
1965 	       tr->tr_status, tr->tr_flags, tr->tr_complete, tr->tr_private);