466 	    sizeof (zio_t), offsetof(struct zio, io_queue_node));  in vdev_queue_init()
469 	    offsetof(struct zio, io_offset_node));  in vdev_queue_init()
472 	    offsetof(struct zio, io_offset_node));  in vdev_queue_init()
475 	    offsetof(struct zio, io_offset_node));  in vdev_queue_init()
493 		    sizeof (zio_t), offsetof(struct zio, io_queue_node));  in vdev_queue_init()
515 vdev_queue_io_add(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_io_add()  argument
517 	ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);  in vdev_queue_io_add()
518 	avl_add(vdev_queue_class_tree(vq, zio->io_priority), zio);  in vdev_queue_io_add()
519 	avl_add(vdev_queue_type_tree(vq, zio->io_type), zio);  in vdev_queue_io_add()
523 vdev_queue_io_remove(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_io_remove()  argument
525 	ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);  in vdev_queue_io_remove()
526 	avl_remove(vdev_queue_class_tree(vq, zio->io_priority), zio);  in vdev_queue_io_remove()
527 	avl_remove(vdev_queue_type_tree(vq, zio->io_type), zio);  in vdev_queue_io_remove()
545 vdev_queue_pending_add(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_pending_add()  argument
548 	ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);  in vdev_queue_pending_add()
549 	vq->vq_class[zio->io_priority].vqc_active++;  in vdev_queue_pending_add()
550 	if (vdev_queue_is_interactive(zio->io_priority)) {  in vdev_queue_pending_add()
556 	avl_add(&vq->vq_active_tree, zio);  in vdev_queue_pending_add()
560 vdev_queue_pending_remove(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_pending_remove()  argument
563 	ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);  in vdev_queue_pending_remove()
564 	vq->vq_class[zio->io_priority].vqc_active--;  in vdev_queue_pending_remove()
565 	if (vdev_queue_is_interactive(zio->io_priority)) {  in vdev_queue_pending_remove()
572 	avl_remove(&vq->vq_active_tree, zio);  in vdev_queue_pending_remove()
599 vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio)  in vdev_queue_aggregate()  argument
607 	avl_tree_t *t = vdev_queue_type_tree(vq, zio->io_type);  in vdev_queue_aggregate()
608 	enum zio_flag flags = zio->io_flags & ZIO_FLAG_AGG_INHERIT;  in vdev_queue_aggregate()
619 	if (zio->io_flags & ZIO_FLAG_DONT_AGGREGATE || limit == 0)  in vdev_queue_aggregate()
626 	if (zio->io_type == ZIO_TYPE_TRIM && !zfs_vdev_aggregate_trim)  in vdev_queue_aggregate()
636 	first = last = zio;  in vdev_queue_aggregate()
638 	if (zio->io_type == ZIO_TYPE_READ)  in vdev_queue_aggregate()
664 	    dio->io_type == zio->io_type) {  in vdev_queue_aggregate()
691 	    dio->io_type == zio->io_type) {  in vdev_queue_aggregate()
707 	if (zio->io_type == ZIO_TYPE_WRITE && mandatory != NULL) {  in vdev_queue_aggregate()
749 	    abd, size, first->io_type, zio->io_priority,  in vdev_queue_aggregate()
811 	zio_t *zio, *aio;  in vdev_queue_io_to_issue()  local
836 	zio = avl_nearest(tree, idx, AVL_AFTER);  in vdev_queue_io_to_issue()
837 	if (zio == NULL)  in vdev_queue_io_to_issue()
838 		zio = avl_first(tree);  in vdev_queue_io_to_issue()
839 	ASSERT3U(zio->io_priority, ==, p);  in vdev_queue_io_to_issue()
841 	aio = vdev_queue_aggregate(vq, zio);  in vdev_queue_io_to_issue()
843 		zio = aio;  in vdev_queue_io_to_issue()
845 		vdev_queue_io_remove(vq, zio);  in vdev_queue_io_to_issue()
853 		if (zio->io_flags & ZIO_FLAG_NODATA) {  in vdev_queue_io_to_issue()
855 			zio_vdev_io_bypass(zio);  in vdev_queue_io_to_issue()
856 			zio_execute(zio);  in vdev_queue_io_to_issue()
862 	vdev_queue_pending_add(vq, zio);  in vdev_queue_io_to_issue()
863 	vq->vq_last_offset = zio->io_offset + zio->io_size;  in vdev_queue_io_to_issue()
865 	return (zio);  in vdev_queue_io_to_issue()
869 vdev_queue_io(zio_t *zio)  in vdev_queue_io()  argument
871 	vdev_queue_t *vq = &zio->io_vd->vdev_queue;  in vdev_queue_io()
875 	if (zio->io_flags & ZIO_FLAG_DONT_QUEUE)  in vdev_queue_io()
876 		return (zio);  in vdev_queue_io()
882 	if (zio->io_type == ZIO_TYPE_READ) {  in vdev_queue_io()
883 		ASSERT(zio->io_priority != ZIO_PRIORITY_TRIM);  in vdev_queue_io()
885 		if (zio->io_priority != ZIO_PRIORITY_SYNC_READ &&  in vdev_queue_io()
886 		    zio->io_priority != ZIO_PRIORITY_ASYNC_READ &&  in vdev_queue_io()
887 		    zio->io_priority != ZIO_PRIORITY_SCRUB &&  in vdev_queue_io()
888 		    zio->io_priority != ZIO_PRIORITY_REMOVAL &&  in vdev_queue_io()
889 		    zio->io_priority != ZIO_PRIORITY_INITIALIZING &&  in vdev_queue_io()
890 		    zio->io_priority != ZIO_PRIORITY_REBUILD) {  in vdev_queue_io()
891 			zio->io_priority = ZIO_PRIORITY_ASYNC_READ;  in vdev_queue_io()
893 	} else if (zio->io_type == ZIO_TYPE_WRITE) {  in vdev_queue_io()
894 		ASSERT(zio->io_priority != ZIO_PRIORITY_TRIM);  in vdev_queue_io()
896 		if (zio->io_priority != ZIO_PRIORITY_SYNC_WRITE &&  in vdev_queue_io()
897 		    zio->io_priority != ZIO_PRIORITY_ASYNC_WRITE &&  in vdev_queue_io()
898 		    zio->io_priority != ZIO_PRIORITY_REMOVAL &&  in vdev_queue_io()
899 		    zio->io_priority != ZIO_PRIORITY_INITIALIZING &&  in vdev_queue_io()
900 		    zio->io_priority != ZIO_PRIORITY_REBUILD) {  in vdev_queue_io()
901 			zio->io_priority = ZIO_PRIORITY_ASYNC_WRITE;  in vdev_queue_io()
904 		ASSERT(zio->io_type == ZIO_TYPE_TRIM);  in vdev_queue_io()
905 		ASSERT(zio->io_priority == ZIO_PRIORITY_TRIM);  in vdev_queue_io()
908 	zio->io_flags |= ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_QUEUE;  in vdev_queue_io()
909 	zio->io_timestamp = gethrtime();  in vdev_queue_io()
912 	vdev_queue_io_add(vq, zio);  in vdev_queue_io()
933 vdev_queue_io_done(zio_t *zio)  in vdev_queue_io_done()  argument
935 	vdev_queue_t *vq = &zio->io_vd->vdev_queue;  in vdev_queue_io_done()
941 	vq->vq_io_delta_ts = zio->io_delta = now - zio->io_timestamp;  in vdev_queue_io_done()
944 	vdev_queue_pending_remove(vq, zio);  in vdev_queue_io_done()
966 vdev_queue_change_io_priority(zio_t *zio, zio_priority_t priority)  in vdev_queue_change_io_priority()  argument
968 	vdev_queue_t *vq = &zio->io_vd->vdev_queue;  in vdev_queue_change_io_priority()
977 	if (zio->io_priority == ZIO_PRIORITY_NOW)  in vdev_queue_change_io_priority()
980 	ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);  in vdev_queue_change_io_priority()
983 	if (zio->io_type == ZIO_TYPE_READ) {  in vdev_queue_change_io_priority()
989 		ASSERT(zio->io_type == ZIO_TYPE_WRITE);  in vdev_queue_change_io_priority()
1004 	tree = vdev_queue_class_tree(vq, zio->io_priority);  in vdev_queue_change_io_priority()
1005 	if (avl_find(tree, zio, NULL) == zio) {  in vdev_queue_change_io_priority()
1006 		avl_remove(vdev_queue_class_tree(vq, zio->io_priority), zio);  in vdev_queue_change_io_priority()
1007 		zio->io_priority = priority;  in vdev_queue_change_io_priority()
1008 		avl_add(vdev_queue_class_tree(vq, zio->io_priority), zio);  in vdev_queue_change_io_priority()
1009 	} else if (avl_find(&vq->vq_active_tree, zio, NULL) != zio) {  in vdev_queue_change_io_priority()
1010 		zio->io_priority = priority;  in vdev_queue_change_io_priority()