52 					       void (* func)(struct twe_request *tr));
63 static int	twe_wait_request(struct twe_request *tr);
64 static int	twe_immediate_request(struct twe_request *tr, int usetmp);
65 static void	twe_completeio(struct twe_request *tr);
91 static void	twe_handle_aen(struct twe_request *tr);
100 static int	twe_get_request(struct twe_softc *sc, struct twe_request **tr);
101 static void	twe_release_request(struct twe_request *tr);
107 static int	twe_report_request(struct twe_request *tr);
122     struct twe_request	*tr;  in twe_setup()  local
143 	if ((tr = twe_allocate_request(sc, i)) == NULL)  in twe_setup()
148 	cmd = TWE_FIND_COMMAND(tr);  in twe_setup()
151 	sc->twe_lookup[i] = tr;  in twe_setup()
157 	twe_release_request(tr);  in twe_setup()
402     struct twe_request	*tr;  in twe_startio()  local
416 	tr = twe_dequeue_ready(sc);  in twe_startio()
419 	if (tr == NULL) {  in twe_startio()
422 	    if (twe_get_request(sc, &tr))  in twe_startio()
427 		twe_release_request(tr);  in twe_startio()
432 	    tr->tr_complete = twe_completeio;  in twe_startio()
433 	    tr->tr_private = bp;  in twe_startio()
434 	    tr->tr_data = bp->bio_data;  in twe_startio()
435 	    tr->tr_length = bp->bio_bcount;  in twe_startio()
436 	    cmd = TWE_FIND_COMMAND(tr);  in twe_startio()
438 		tr->tr_flags |= TWE_CMD_DATAIN;  in twe_startio()
441 		tr->tr_flags |= TWE_CMD_DATAOUT;  in twe_startio()
444 		twe_release_request(tr);  in twe_startio()
452 	    cmd->io.block_count = (tr->tr_length + TWE_BLOCK_SIZE - 1) / TWE_BLOCK_SIZE;  in twe_startio()
457 	if (tr == NULL)  in twe_startio()
461 	error = twe_map_request(tr);  in twe_startio()
466 	    tr->tr_status = TWE_CMD_ERROR;  in twe_startio()
467 	    if (tr->tr_private != NULL) {  in twe_startio()
468 		bp = (struct bio *)(tr->tr_private);  in twe_startio()
471 		tr->tr_private = NULL;  in twe_startio()
473 	        twe_release_request(tr);  in twe_startio()
474 	    } else if (tr->tr_flags & TWE_CMD_SLEEPER)  in twe_startio()
475 		wakeup_one(tr); /* wakeup the sleeping owner */  in twe_startio()
486     struct twe_request	*tr;  in twe_dump_blocks()  local
490     if (twe_get_request(sc, &tr))  in twe_dump_blocks()
493     tr->tr_data = data;  in twe_dump_blocks()
494     tr->tr_status = TWE_CMD_SETUP;  in twe_dump_blocks()
495     tr->tr_length = nblks * TWE_BLOCK_SIZE;  in twe_dump_blocks()
496     tr->tr_flags = TWE_CMD_DATAOUT;  in twe_dump_blocks()
498     cmd = TWE_FIND_COMMAND(tr);  in twe_dump_blocks()
505     error = twe_immediate_request(tr, 0);  in twe_dump_blocks()
507 	if (twe_report_request(tr))  in twe_dump_blocks()
509     twe_release_request(tr);  in twe_dump_blocks()
527     struct twe_request		*tr;  in twe_ioctl()  local
553 	while (twe_get_request(sc, &tr))  in twe_ioctl()
560 	cmd = TWE_FIND_COMMAND(tr);  in twe_ioctl()
565 	tr->tr_length = tr_length;  in twe_ioctl()
566 	tr->tr_data = data;  in twe_ioctl()
567 	if (tr->tr_length > 0) {  in twe_ioctl()
568 	    tr->tr_flags |= TWE_CMD_DATAIN | TWE_CMD_DATAOUT;  in twe_ioctl()
572 	error = twe_wait_request(tr);  in twe_ioctl()
581 	if (tr->tr_length > 0)  in twe_ioctl()
582 	    error = copyout(tr->tr_data, tu->tu_data, tu->tu_size);  in twe_ioctl()
586 	if (tr->tr_data != NULL)  in twe_ioctl()
587 	    free(tr->tr_data, M_DEVBUF);  in twe_ioctl()
589 	twe_release_request(tr);  in twe_ioctl()
768 	      void (* func)(struct twe_request *tr))  in twe_get_param()  argument
770     struct twe_request	*tr;  in twe_get_param()  local
778     tr = NULL;  in twe_get_param()
782     if (twe_get_request(sc, &tr))  in twe_get_param()
788     tr->tr_data = param;  in twe_get_param()
789     tr->tr_length = TWE_SECTOR_SIZE;  in twe_get_param()
790     tr->tr_flags = TWE_CMD_DATAIN | TWE_CMD_DATAOUT;  in twe_get_param()
793     cmd = TWE_FIND_COMMAND(tr);  in twe_get_param()
807 	error = twe_immediate_request(tr, 1 /* usetmp */);  in twe_get_param()
809 	    if (twe_report_request(tr))  in twe_get_param()
814 	twe_release_request(tr);  in twe_get_param()
817 	tr->tr_complete = func;  in twe_get_param()
818 	error = twe_map_request(tr);  in twe_get_param()
826     if (tr != NULL)  in twe_get_param()
827 	twe_release_request(tr);  in twe_get_param()
864     struct twe_request	*tr;  in twe_set_param()  local
872     tr = NULL;  in twe_set_param()
877     if (twe_get_request(sc, &tr))  in twe_set_param()
883     tr->tr_data = param;  in twe_set_param()
884     tr->tr_length = TWE_SECTOR_SIZE;  in twe_set_param()
885     tr->tr_flags = TWE_CMD_DATAIN | TWE_CMD_DATAOUT;  in twe_set_param()
888     cmd = TWE_FIND_COMMAND(tr);  in twe_set_param()
901     error = twe_immediate_request(tr, 1 /* usetmp */);  in twe_set_param()
903 	if (twe_report_request(tr))  in twe_set_param()
908     if (tr != NULL)  in twe_set_param()
909 	twe_release_request(tr);  in twe_set_param()
923     struct twe_request	*tr;  in twe_init_connection()  local
932     if (twe_get_request(sc, &tr))  in twe_init_connection()
936     cmd = TWE_FIND_COMMAND(tr);  in twe_init_connection()
944     error = twe_immediate_request(tr, 0 /* usetmp */);  in twe_init_connection()
945     twe_release_request(tr);  in twe_init_connection()
958 twe_wait_request(struct twe_request *tr)  in twe_wait_request()  argument
963     TWE_IO_ASSERT_LOCKED(tr->tr_sc);  in twe_wait_request()
964     tr->tr_flags |= TWE_CMD_SLEEPER;  in twe_wait_request()
965     tr->tr_status = TWE_CMD_BUSY;  in twe_wait_request()
966     twe_enqueue_ready(tr);  in twe_wait_request()
967     twe_startio(tr->tr_sc);  in twe_wait_request()
968     while (tr->tr_status == TWE_CMD_BUSY)  in twe_wait_request()
969 	mtx_sleep(tr, &tr->tr_sc->twe_io_lock, PRIBIO, "twewait", 0);  in twe_wait_request()
971     return(tr->tr_status != TWE_CMD_COMPLETE);  in twe_wait_request()
980 twe_immediate_request(struct twe_request *tr, int usetmp)  in twe_immediate_request()  argument
988     sc = tr->tr_sc;  in twe_immediate_request()
990     if (usetmp && (tr->tr_data != NULL)) {  in twe_immediate_request()
991 	tr->tr_flags |= TWE_CMD_IMMEDIATE;  in twe_immediate_request()
992 	if (tr->tr_length > DFLTPHYS)  in twe_immediate_request()
994 	bcopy(tr->tr_data, sc->twe_immediate, tr->tr_length);  in twe_immediate_request()
996     tr->tr_status = TWE_CMD_BUSY;  in twe_immediate_request()
997     if ((error = twe_map_request(tr)) != 0)  in twe_immediate_request()
1002     while ((count++ < 5000) && (tr->tr_status == TWE_CMD_BUSY)){  in twe_immediate_request()
1006     if (usetmp && (tr->tr_data != NULL))  in twe_immediate_request()
1007 	bcopy(sc->twe_immediate, tr->tr_data, tr->tr_length);  in twe_immediate_request()
1009     return(tr->tr_status != TWE_CMD_COMPLETE);  in twe_immediate_request()
1016 twe_completeio(struct twe_request *tr)  in twe_completeio()  argument
1018     TWE_Command		*cmd = TWE_FIND_COMMAND(tr);  in twe_completeio()
1019     struct twe_softc	*sc = tr->tr_sc;  in twe_completeio()
1020     struct bio		*bp = tr->tr_private;  in twe_completeio()
1024     if (tr->tr_status == TWE_CMD_COMPLETE) {  in twe_completeio()
1026 	    if (twe_report_request(tr)) {  in twe_completeio()
1034     tr->tr_private = NULL;  in twe_completeio()
1036     twe_release_request(tr);  in twe_completeio()
1046     struct twe_request	*tr;  in twe_reset()  local
1081     while ((tr = twe_dequeue_busy(sc)) != NULL) {  in twe_reset()
1082 	twe_enqueue_ready(tr);  in twe_reset()
1108 twe_start(struct twe_request *tr)  in twe_start()  argument
1110     struct twe_softc	*sc = tr->tr_sc;  in twe_start()
1121     tr->tr_status = TWE_CMD_BUSY;  in twe_start()
1122     cmd = TWE_FIND_COMMAND(tr);  in twe_start()
1136 	    twe_enqueue_busy(tr);  in twe_start()
1138 	    TWE_COMMAND_QUEUE(sc, TWE_FIND_COMMANDPHYS(tr));  in twe_start()
1142 	    if (tr->tr_complete != NULL) {  in twe_start()
1143 		debug(3, "queued request %d with callback %p", cmd->generic.request_id, tr->tr_complete);  in twe_start()
1144 	    } else if (tr->tr_flags & TWE_CMD_SLEEPER) {  in twe_start()
1145 		debug(3, "queued request %d with wait channel %p", cmd->generic.request_id, tr);  in twe_start()
1174     struct twe_request	*tr;  in twe_done()  local
1189 	    tr = sc->twe_lookup[rq.u.response_id];	/* find command */  in twe_done()
1190 	    cmd = TWE_FIND_COMMAND(tr);  in twe_done()
1191 	    if (tr->tr_status != TWE_CMD_BUSY)  in twe_done()
1193 	    tr->tr_status = TWE_CMD_COMPLETE;  in twe_done()
1197 	    twe_remove_busy(tr);  in twe_done()
1198 	    twe_enqueue_complete(tr);  in twe_done()
1222     struct twe_request	*tr;  in twe_complete()  local
1229     while ((tr = twe_dequeue_complete(sc)) != NULL) {  in twe_complete()
1231 	twe_unmap_request(tr);  in twe_complete()
1234 	if (tr->tr_complete != NULL) {		/* completion callback */  in twe_complete()
1235 	    debug(2, "call completion handler %p", tr->tr_complete);  in twe_complete()
1236 	    tr->tr_complete(tr);  in twe_complete()
1238 	} else if (tr->tr_flags & TWE_CMD_SLEEPER) {	/* caller is asleep waiting */  in twe_complete()
1239 	    debug(2, "wake up command owner on %p", tr);  in twe_complete()
1240 	    wakeup_one(tr);  in twe_complete()
1415 twe_handle_aen(struct twe_request *tr)  in twe_handle_aen()  argument
1417     struct twe_softc	*sc = tr->tr_sc;  in twe_handle_aen()
1425     param = (TWE_Param *)tr->tr_data;  in twe_handle_aen()
1428     free(tr->tr_data, M_DEVBUF);  in twe_handle_aen()
1429     twe_release_request(tr);  in twe_handle_aen()
1581 twe_get_request(struct twe_softc *sc, struct twe_request **tr)  in twe_get_request()  argument
1590     *tr = twe_dequeue_free(sc);  in twe_get_request()
1593     if (*tr != NULL) {  in twe_get_request()
1594 	cmd = TWE_FIND_COMMAND(*tr);  in twe_get_request()
1595 	(*tr)->tr_data = NULL;  in twe_get_request()
1596 	(*tr)->tr_private = NULL;  in twe_get_request()
1597 	(*tr)->tr_status = TWE_CMD_SETUP;		/* command is in setup phase */  in twe_get_request()
1598 	(*tr)->tr_flags = 0;  in twe_get_request()
1599 	(*tr)->tr_complete = NULL;  in twe_get_request()
1603     return(*tr == NULL);  in twe_get_request()
1611 twe_release_request(struct twe_request *tr)  in twe_release_request()  argument
1616 	TWE_IO_ASSERT_LOCKED(tr->tr_sc);  in twe_release_request()
1617     if (tr->tr_private != NULL)  in twe_release_request()
1618 	twe_panic(tr->tr_sc, "tr_private != NULL");  in twe_release_request()
1619     twe_enqueue_free(tr);  in twe_release_request()
1819 twe_report_request(struct twe_request *tr)  in twe_report_request()  argument
1821     struct twe_softc	*sc = tr->tr_sc;  in twe_report_request()
1822     TWE_Command		*cmd = TWE_FIND_COMMAND(tr);  in twe_report_request()
1923 twe_print_request(struct twe_request *tr)
1925     struct twe_softc	*sc = tr->tr_sc;
1926     TWE_Command	*cmd = TWE_FIND_COMMAND(tr);
1960 	       tr, TWE_FIND_COMMANDPHYS(tr), tr->tr_data, tr->tr_dataphys, tr->tr_length);
1962 	       tr->tr_status, tr->tr_flags, tr->tr_complete, tr->tr_private);