Lines Matching refs:zio
272 vdev_queue_class_add(vdev_queue_t *vq, zio_t *zio) in vdev_queue_class_add() argument
274 zio_priority_t p = zio->io_priority; in vdev_queue_class_add()
277 list_insert_tail(&vq->vq_class[p].vqc_list, zio); in vdev_queue_class_add()
281 avl_add(&vq->vq_class[p].vqc_tree, zio); in vdev_queue_class_add()
285 vdev_queue_class_remove(vdev_queue_t *vq, zio_t *zio) in vdev_queue_class_remove() argument
287 zio_priority_t p = zio->io_priority; in vdev_queue_class_remove()
291 list_remove(list, zio); in vdev_queue_class_remove()
296 avl_remove(tree, zio); in vdev_queue_class_remove()
487 offsetof(struct zio, io_queue_node.l)); in vdev_queue_init()
491 offsetof(struct zio, io_queue_node.a)); in vdev_queue_init()
496 offsetof(struct zio, io_offset_node)); in vdev_queue_init()
499 offsetof(struct zio, io_offset_node)); in vdev_queue_init()
502 list_create(&vq->vq_active_list, sizeof (struct zio), in vdev_queue_init()
503 offsetof(struct zio, io_queue_node.l)); in vdev_queue_init()
526 vdev_queue_io_add(vdev_queue_t *vq, zio_t *zio) in vdev_queue_io_add() argument
528 zio->io_queue_state = ZIO_QS_QUEUED; in vdev_queue_io_add()
529 vdev_queue_class_add(vq, zio); in vdev_queue_io_add()
530 if (zio->io_type == ZIO_TYPE_READ) in vdev_queue_io_add()
531 avl_add(&vq->vq_read_offset_tree, zio); in vdev_queue_io_add()
532 else if (zio->io_type == ZIO_TYPE_WRITE) in vdev_queue_io_add()
533 avl_add(&vq->vq_write_offset_tree, zio); in vdev_queue_io_add()
537 vdev_queue_io_remove(vdev_queue_t *vq, zio_t *zio) in vdev_queue_io_remove() argument
539 vdev_queue_class_remove(vq, zio); in vdev_queue_io_remove()
540 if (zio->io_type == ZIO_TYPE_READ) in vdev_queue_io_remove()
541 avl_remove(&vq->vq_read_offset_tree, zio); in vdev_queue_io_remove()
542 else if (zio->io_type == ZIO_TYPE_WRITE) in vdev_queue_io_remove()
543 avl_remove(&vq->vq_write_offset_tree, zio); in vdev_queue_io_remove()
544 zio->io_queue_state = ZIO_QS_NONE; in vdev_queue_io_remove()
562 vdev_queue_pending_add(vdev_queue_t *vq, zio_t *zio) in vdev_queue_pending_add() argument
565 ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE); in vdev_queue_pending_add()
566 vq->vq_cactive[zio->io_priority]++; in vdev_queue_pending_add()
568 if (vdev_queue_is_interactive(zio->io_priority)) { in vdev_queue_pending_add()
574 zio->io_queue_state = ZIO_QS_ACTIVE; in vdev_queue_pending_add()
575 list_insert_tail(&vq->vq_active_list, zio); in vdev_queue_pending_add()
579 vdev_queue_pending_remove(vdev_queue_t *vq, zio_t *zio) in vdev_queue_pending_remove() argument
582 ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE); in vdev_queue_pending_remove()
583 vq->vq_cactive[zio->io_priority]--; in vdev_queue_pending_remove()
585 if (vdev_queue_is_interactive(zio->io_priority)) { in vdev_queue_pending_remove()
592 list_remove(&vq->vq_active_list, zio); in vdev_queue_pending_remove()
593 zio->io_queue_state = ZIO_QS_NONE; in vdev_queue_pending_remove()
620 vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio) in vdev_queue_aggregate() argument
635 if (zio->io_type == ZIO_TYPE_TRIM) in vdev_queue_aggregate()
638 if (zio->io_flags & ZIO_FLAG_DONT_AGGREGATE) in vdev_queue_aggregate()
656 first = last = zio; in vdev_queue_aggregate()
658 if (zio->io_type == ZIO_TYPE_READ) { in vdev_queue_aggregate()
662 ASSERT3U(zio->io_type, ==, ZIO_TYPE_WRITE); in vdev_queue_aggregate()
685 zio_flag_t flags = zio->io_flags & ZIO_FLAG_AGG_INHERIT; in vdev_queue_aggregate()
690 dio->io_type == zio->io_type) { in vdev_queue_aggregate()
717 dio->io_type == zio->io_type) { in vdev_queue_aggregate()
733 if (zio->io_type == ZIO_TYPE_WRITE && mandatory != NULL) { in vdev_queue_aggregate()
776 abd, size, first->io_type, zio->io_priority, in vdev_queue_aggregate()
838 zio_t *zio, *aio; in vdev_queue_io_to_issue() local
854 zio = list_head(&vq->vq_class[p].vqc_list); in vdev_queue_io_to_issue()
863 zio = aio = avl_first(tree); in vdev_queue_io_to_issue()
864 if (zio->io_offset < vq->vq_last_offset) { in vdev_queue_io_to_issue()
865 vq->vq_io_search.io_timestamp = zio->io_timestamp; in vdev_queue_io_to_issue()
867 zio = avl_find(tree, &vq->vq_io_search, &idx); in vdev_queue_io_to_issue()
868 if (zio == NULL) { in vdev_queue_io_to_issue()
869 zio = avl_nearest(tree, idx, AVL_AFTER); in vdev_queue_io_to_issue()
870 if (zio == NULL || in vdev_queue_io_to_issue()
871 (zio->io_timestamp >> VDQ_T_SHIFT) != in vdev_queue_io_to_issue()
873 zio = aio; in vdev_queue_io_to_issue()
877 ASSERT3U(zio->io_priority, ==, p); in vdev_queue_io_to_issue()
879 aio = vdev_queue_aggregate(vq, zio); in vdev_queue_io_to_issue()
881 zio = aio; in vdev_queue_io_to_issue()
883 vdev_queue_io_remove(vq, zio); in vdev_queue_io_to_issue()
891 if (zio->io_flags & ZIO_FLAG_NODATA) { in vdev_queue_io_to_issue()
893 zio_vdev_io_bypass(zio); in vdev_queue_io_to_issue()
894 zio_execute(zio); in vdev_queue_io_to_issue()
900 vdev_queue_pending_add(vq, zio); in vdev_queue_io_to_issue()
901 vq->vq_last_offset = zio->io_offset + zio->io_size; in vdev_queue_io_to_issue()
903 return (zio); in vdev_queue_io_to_issue()
907 vdev_queue_io(zio_t *zio) in vdev_queue_io() argument
909 vdev_queue_t *vq = &zio->io_vd->vdev_queue; in vdev_queue_io()
913 if (zio->io_flags & ZIO_FLAG_DONT_QUEUE) in vdev_queue_io()
914 return (zio); in vdev_queue_io()
920 if (zio->io_type == ZIO_TYPE_READ) { in vdev_queue_io()
921 ASSERT(zio->io_priority != ZIO_PRIORITY_TRIM); in vdev_queue_io()
923 if (zio->io_priority != ZIO_PRIORITY_SYNC_READ && in vdev_queue_io()
924 zio->io_priority != ZIO_PRIORITY_ASYNC_READ && in vdev_queue_io()
925 zio->io_priority != ZIO_PRIORITY_SCRUB && in vdev_queue_io()
926 zio->io_priority != ZIO_PRIORITY_REMOVAL && in vdev_queue_io()
927 zio->io_priority != ZIO_PRIORITY_INITIALIZING && in vdev_queue_io()
928 zio->io_priority != ZIO_PRIORITY_REBUILD) { in vdev_queue_io()
929 zio->io_priority = ZIO_PRIORITY_ASYNC_READ; in vdev_queue_io()
931 } else if (zio->io_type == ZIO_TYPE_WRITE) { in vdev_queue_io()
932 ASSERT(zio->io_priority != ZIO_PRIORITY_TRIM); in vdev_queue_io()
934 if (zio->io_priority != ZIO_PRIORITY_SYNC_WRITE && in vdev_queue_io()
935 zio->io_priority != ZIO_PRIORITY_ASYNC_WRITE && in vdev_queue_io()
936 zio->io_priority != ZIO_PRIORITY_REMOVAL && in vdev_queue_io()
937 zio->io_priority != ZIO_PRIORITY_INITIALIZING && in vdev_queue_io()
938 zio->io_priority != ZIO_PRIORITY_REBUILD) { in vdev_queue_io()
939 zio->io_priority = ZIO_PRIORITY_ASYNC_WRITE; in vdev_queue_io()
942 ASSERT(zio->io_type == ZIO_TYPE_TRIM); in vdev_queue_io()
943 ASSERT(zio->io_priority == ZIO_PRIORITY_TRIM); in vdev_queue_io()
946 zio->io_flags |= ZIO_FLAG_DONT_QUEUE; in vdev_queue_io()
947 zio->io_timestamp = gethrtime(); in vdev_queue_io()
950 vdev_queue_io_add(vq, zio); in vdev_queue_io()
971 vdev_queue_io_done(zio_t *zio) in vdev_queue_io_done() argument
973 vdev_queue_t *vq = &zio->io_vd->vdev_queue; in vdev_queue_io_done()
979 vq->vq_io_delta_ts = zio->io_delta = now - zio->io_timestamp; in vdev_queue_io_done()
982 vdev_queue_pending_remove(vq, zio); in vdev_queue_io_done()
1004 vdev_queue_change_io_priority(zio_t *zio, zio_priority_t priority) in vdev_queue_change_io_priority() argument
1006 vdev_queue_t *vq = &zio->io_vd->vdev_queue; in vdev_queue_change_io_priority()
1014 if (zio->io_priority == ZIO_PRIORITY_NOW) in vdev_queue_change_io_priority()
1017 ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE); in vdev_queue_change_io_priority()
1020 if (zio->io_type == ZIO_TYPE_READ) { in vdev_queue_change_io_priority()
1026 ASSERT(zio->io_type == ZIO_TYPE_WRITE); in vdev_queue_change_io_priority()
1041 if (zio->io_queue_state == ZIO_QS_QUEUED) { in vdev_queue_change_io_priority()
1042 vdev_queue_class_remove(vq, zio); in vdev_queue_change_io_priority()
1043 zio->io_priority = priority; in vdev_queue_change_io_priority()
1044 vdev_queue_class_add(vq, zio); in vdev_queue_change_io_priority()
1045 } else if (zio->io_queue_state == ZIO_QS_NONE) { in vdev_queue_change_io_priority()
1046 zio->io_priority = priority; in vdev_queue_change_io_priority()