468 	    sizeof (zio_t), offsetof(struct zio, io_queue_node));  in vdev_queue_init()
471 	    offsetof(struct zio, io_offset_node));  in vdev_queue_init()
474 	    offsetof(struct zio, io_offset_node));  in vdev_queue_init()
477 	    offsetof(struct zio, io_offset_node));  in vdev_queue_init()
495 		    sizeof (zio_t), offsetof(struct zio, io_queue_node));  in vdev_queue_init()
517 vdev_queue_io_add(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_io_add()  argument
519 	spa_t *spa = zio->io_spa;  in vdev_queue_io_add()
522 	ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);  in vdev_queue_io_add()
523 	avl_add(vdev_queue_class_tree(vq, zio->io_priority), zio);  in vdev_queue_io_add()
524 	avl_add(vdev_queue_type_tree(vq, zio->io_type), zio);  in vdev_queue_io_add()
534 vdev_queue_io_remove(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_io_remove()  argument
536 	spa_t *spa = zio->io_spa;  in vdev_queue_io_remove()
539 	ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);  in vdev_queue_io_remove()
540 	avl_remove(vdev_queue_class_tree(vq, zio->io_priority), zio);  in vdev_queue_io_remove()
541 	avl_remove(vdev_queue_type_tree(vq, zio->io_type), zio);  in vdev_queue_io_remove()
565 vdev_queue_pending_add(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_pending_add()  argument
567 	spa_t *spa = zio->io_spa;  in vdev_queue_pending_add()
571 	ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);  in vdev_queue_pending_add()
572 	vq->vq_class[zio->io_priority].vqc_active++;  in vdev_queue_pending_add()
573 	if (vdev_queue_is_interactive(zio->io_priority)) {  in vdev_queue_pending_add()
579 	avl_add(&vq->vq_active_tree, zio);  in vdev_queue_pending_add()
589 vdev_queue_pending_remove(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_pending_remove()  argument
591 	spa_t *spa = zio->io_spa;  in vdev_queue_pending_remove()
595 	ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);  in vdev_queue_pending_remove()
596 	vq->vq_class[zio->io_priority].vqc_active--;  in vdev_queue_pending_remove()
597 	if (vdev_queue_is_interactive(zio->io_priority)) {  in vdev_queue_pending_remove()
604 	avl_remove(&vq->vq_active_tree, zio);  in vdev_queue_pending_remove()
611 		if (zio->io_type == ZIO_TYPE_READ) {  in vdev_queue_pending_remove()
613 			ksio->nread += zio->io_size;  in vdev_queue_pending_remove()
614 		} else if (zio->io_type == ZIO_TYPE_WRITE) {  in vdev_queue_pending_remove()
616 			ksio->nwritten += zio->io_size;  in vdev_queue_pending_remove()
646 vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_aggregate()  argument
655 	avl_tree_t *t = vdev_queue_type_tree(vq, zio->io_type);  in vdev_queue_aggregate()
656 	enum zio_flag flags = zio->io_flags & ZIO_FLAG_AGG_INHERIT;  in vdev_queue_aggregate()
667 	if (zio->io_flags & ZIO_FLAG_DONT_AGGREGATE || limit == 0)  in vdev_queue_aggregate()
674 	if (zio->io_type == ZIO_TYPE_TRIM && !zfs_vdev_aggregate_trim)  in vdev_queue_aggregate()
684 	first = last = zio;  in vdev_queue_aggregate()
686 	if (zio->io_type == ZIO_TYPE_READ)  in vdev_queue_aggregate()
712 	    dio->io_type == zio->io_type) {  in vdev_queue_aggregate()
739 	    dio->io_type == zio->io_type) {  in vdev_queue_aggregate()
755 	if (zio->io_type == ZIO_TYPE_WRITE && mandatory != NULL) {  in vdev_queue_aggregate()
797 	    abd, size, first->io_type, zio->io_priority,  in vdev_queue_aggregate()
866 	zio_t *zio, *aio;  in vdev_queue_io_to_issue()  local
891 	zio = avl_nearest(tree, idx, AVL_AFTER);  in vdev_queue_io_to_issue()
892 	if (zio == NULL)  in vdev_queue_io_to_issue()
893 		zio = avl_first(tree);  in vdev_queue_io_to_issue()
894 	ASSERT3U(zio->io_priority, ==, p);  in vdev_queue_io_to_issue()
896 	aio = vdev_queue_aggregate(vq, zio);  in vdev_queue_io_to_issue()
898 		zio = aio;  in vdev_queue_io_to_issue()
900 		vdev_queue_io_remove(vq, zio);  in vdev_queue_io_to_issue()
908 	if (zio->io_flags & ZIO_FLAG_NODATA) {  in vdev_queue_io_to_issue()
910 		zio_vdev_io_bypass(zio);  in vdev_queue_io_to_issue()
911 		zio_execute(zio);  in vdev_queue_io_to_issue()
916 	vdev_queue_pending_add(vq, zio);  in vdev_queue_io_to_issue()
917 	vq->vq_last_offset = zio->io_offset + zio->io_size;  in vdev_queue_io_to_issue()
919 	return (zio);  in vdev_queue_io_to_issue()
923 vdev_queue_io(zio_t *zio)  in vdev_queue_io()  argument
925 	vdev_queue_t *vq = &zio->io_vd->vdev_queue;  in vdev_queue_io()
928 	if (zio->io_flags & ZIO_FLAG_DONT_QUEUE)  in vdev_queue_io()
929 		return (zio);  in vdev_queue_io()
935 	if (zio->io_type == ZIO_TYPE_READ) {  in vdev_queue_io()
936 		ASSERT(zio->io_priority != ZIO_PRIORITY_TRIM);  in vdev_queue_io()
938 		if (zio->io_priority != ZIO_PRIORITY_SYNC_READ &&  in vdev_queue_io()
939 		    zio->io_priority != ZIO_PRIORITY_ASYNC_READ &&  in vdev_queue_io()
940 		    zio->io_priority != ZIO_PRIORITY_SCRUB &&  in vdev_queue_io()
941 		    zio->io_priority != ZIO_PRIORITY_REMOVAL &&  in vdev_queue_io()
942 		    zio->io_priority != ZIO_PRIORITY_INITIALIZING &&  in vdev_queue_io()
943 		    zio->io_priority != ZIO_PRIORITY_REBUILD) {  in vdev_queue_io()
944 			zio->io_priority = ZIO_PRIORITY_ASYNC_READ;  in vdev_queue_io()
946 	} else if (zio->io_type == ZIO_TYPE_WRITE) {  in vdev_queue_io()
947 		ASSERT(zio->io_priority != ZIO_PRIORITY_TRIM);  in vdev_queue_io()
949 		if (zio->io_priority != ZIO_PRIORITY_SYNC_WRITE &&  in vdev_queue_io()
950 		    zio->io_priority != ZIO_PRIORITY_ASYNC_WRITE &&  in vdev_queue_io()
951 		    zio->io_priority != ZIO_PRIORITY_REMOVAL &&  in vdev_queue_io()
952 		    zio->io_priority != ZIO_PRIORITY_INITIALIZING &&  in vdev_queue_io()
953 		    zio->io_priority != ZIO_PRIORITY_REBUILD) {  in vdev_queue_io()
954 			zio->io_priority = ZIO_PRIORITY_ASYNC_WRITE;  in vdev_queue_io()
957 		ASSERT(zio->io_type == ZIO_TYPE_TRIM);  in vdev_queue_io()
958 		ASSERT(zio->io_priority == ZIO_PRIORITY_TRIM);  in vdev_queue_io()
961 	zio->io_flags |= ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_QUEUE;  in vdev_queue_io()
964 	zio->io_timestamp = gethrtime();  in vdev_queue_io()
965 	vdev_queue_io_add(vq, zio);  in vdev_queue_io()
981 vdev_queue_io_done(zio_t *zio)  in vdev_queue_io_done()  argument
983 	vdev_queue_t *vq = &zio->io_vd->vdev_queue;  in vdev_queue_io_done()
988 	vdev_queue_pending_remove(vq, zio);  in vdev_queue_io_done()
990 	zio->io_delta = gethrtime() - zio->io_timestamp;  in vdev_queue_io_done()
992 	vq->vq_io_delta_ts = vq->vq_io_complete_ts - zio->io_timestamp;  in vdev_queue_io_done()
1009 vdev_queue_change_io_priority(zio_t *zio, zio_priority_t priority)  in vdev_queue_change_io_priority()  argument
1011 	vdev_queue_t *vq = &zio->io_vd->vdev_queue;  in vdev_queue_change_io_priority()
1020 	if (zio->io_priority == ZIO_PRIORITY_NOW)  in vdev_queue_change_io_priority()
1023 	ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);  in vdev_queue_change_io_priority()
1026 	if (zio->io_type == ZIO_TYPE_READ) {  in vdev_queue_change_io_priority()
1032 		ASSERT(zio->io_type == ZIO_TYPE_WRITE);  in vdev_queue_change_io_priority()
1047 	tree = vdev_queue_class_tree(vq, zio->io_priority);  in vdev_queue_change_io_priority()
1048 	if (avl_find(tree, zio, NULL) == zio) {  in vdev_queue_change_io_priority()
1049 		avl_remove(vdev_queue_class_tree(vq, zio->io_priority), zio);  in vdev_queue_change_io_priority()
1050 		zio->io_priority = priority;  in vdev_queue_change_io_priority()
1051 		avl_add(vdev_queue_class_tree(vq, zio->io_priority), zio);  in vdev_queue_change_io_priority()
1052 	} else if (avl_find(&vq->vq_active_tree, zio, NULL) != zio) {  in vdev_queue_change_io_priority()
1053 		zio->io_priority = priority;  in vdev_queue_change_io_priority()