482 vdev_submit_bio_impl(struct bio *bio)  in vdev_submit_bio_impl()  argument
485 	(void) submit_bio(bio);  in vdev_submit_bio_impl()
487 	(void) submit_bio(bio_data_dir(bio), bio);  in vdev_submit_bio_impl()
555 vdev_bio_associate_blkg(struct bio *bio)  in vdev_bio_associate_blkg()  argument
558 	struct request_queue *q = bio->bi_bdev->bd_disk->queue;  in vdev_bio_associate_blkg()
560 	struct request_queue *q = bio->bi_disk->queue;  in vdev_bio_associate_blkg()
564 	ASSERT3P(bio->bi_blkg, ==, NULL);  in vdev_bio_associate_blkg()
567 		bio->bi_blkg = q->root_blkg;  in vdev_bio_associate_blkg()
573 vdev_bio_set_dev(struct bio *bio, struct block_device *bdev)  in vdev_bio_set_dev()  argument
578 	struct request_queue *q = bio->bi_disk->queue;  in vdev_bio_set_dev()
580 	bio_clear_flag(bio, BIO_REMAPPED);  in vdev_bio_set_dev()
581 	if (bio->bi_bdev != bdev)  in vdev_bio_set_dev()
582 		bio_clear_flag(bio, BIO_THROTTLED);  in vdev_bio_set_dev()
583 	bio->bi_bdev = bdev;  in vdev_bio_set_dev()
586 	ASSERT3P(bio->bi_blkg, ==, NULL);  in vdev_bio_set_dev()
589 		bio->bi_blkg = q->root_blkg;  in vdev_bio_set_dev()
599 bio_set_dev(struct bio *bio, struct block_device *bdev)  in bio_set_dev()  argument
601 	bio->bi_bdev = bdev;  in bio_set_dev()
607 vdev_submit_bio(struct bio *bio)  in vdev_submit_bio()  argument
611 	vdev_submit_bio_impl(bio);  in vdev_submit_bio()
615 static inline struct bio *
619 	struct bio *bio;  in vdev_bio_alloc()  local
622 	bio = bio_alloc(bdev, nr_vecs, 0, gfp_mask);  in vdev_bio_alloc()
624 	bio = bio_alloc(gfp_mask, nr_vecs);  in vdev_bio_alloc()
625 	if (likely(bio != NULL))  in vdev_bio_alloc()
626 		bio_set_dev(bio, bdev);  in vdev_bio_alloc()
629 	return (bio);  in vdev_bio_alloc()
687 	struct bio	*vbio_bio;	/* pointer to the current bio */
709 BIO_END_IO_PROTO(vbio_completion, bio, error);
714 	struct bio *bio = vbio->vbio_bio;  in vbio_add_page()  local
718 		if (bio == NULL) {  in vbio_add_page()
720 			bio = vdev_bio_alloc(vbio->vbio_bdev, GFP_NOIO,  in vbio_add_page()
722 			VERIFY(bio);  in vbio_add_page()
724 			BIO_BI_SECTOR(bio) = vbio->vbio_offset >> 9;  in vbio_add_page()
725 			bio_set_op_attrs(bio,  in vbio_add_page()
730 				bio_chain(vbio->vbio_bio, bio);  in vbio_add_page()
733 			vbio->vbio_bio = bio;  in vbio_add_page()
744 		ssize = MIN(size, (vbio->vbio_max_bytes - BIO_BI_SIZE(bio)) &  in vbio_add_page()
747 		    bio_add_page(bio, page, ssize, offset) == ssize) {  in vbio_add_page()
755 		vbio->vbio_offset += BIO_BI_SIZE(bio);  in vbio_add_page()
758 		bio = NULL;  in vbio_add_page()
805 BIO_END_IO_PROTO(vbio_completion, bio, error)  in BIO_END_IO_PROTO()  argument
807 	vbio_t *vbio = bio->bi_private;  in BIO_END_IO_PROTO()
814 	zio->io_error = BIO_END_IO_ERROR(bio);  in BIO_END_IO_PROTO()
819 	else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))  in BIO_END_IO_PROTO()
828 	bio_put(bio);  in BIO_END_IO_PROTO()
1005 	struct bio		*dr_bio[];	/* Attached bio's */
1012 	    sizeof (struct bio *) * bio_count, KM_SLEEP);  in vdev_classic_dio_alloc()
1033 	    sizeof (struct bio *) * dr->dr_bio_count);  in vdev_classic_dio_free()
1068 BIO_END_IO_PROTO(vdev_classic_physio_completion, bio, error)  in BIO_END_IO_PROTO()  argument
1070 	dio_request_t *dr = bio->bi_private;  in BIO_END_IO_PROTO()
1074 		dr->dr_error = BIO_END_IO_ERROR(bio);  in BIO_END_IO_PROTO()
1078 		else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))  in BIO_END_IO_PROTO()
1219 BIO_END_IO_PROTO(vdev_disk_io_flush_completion, bio, error)  in BIO_END_IO_PROTO()  argument
1221 	zio_t *zio = bio->bi_private;  in BIO_END_IO_PROTO()
1223 	zio->io_error = BIO_END_IO_ERROR(bio);  in BIO_END_IO_PROTO()
1231 	bio_put(bio);  in BIO_END_IO_PROTO()
1242 	struct bio *bio;  in vdev_disk_io_flush()  local
1248 	bio = vdev_bio_alloc(bdev, GFP_NOIO, 0);  in vdev_disk_io_flush()
1249 	if (unlikely(bio == NULL))  in vdev_disk_io_flush()
1252 	bio->bi_end_io = vdev_disk_io_flush_completion;  in vdev_disk_io_flush()
1253 	bio->bi_private = zio;  in vdev_disk_io_flush()
1254 	bio_set_flush(bio);  in vdev_disk_io_flush()
1255 	vdev_submit_bio(bio);  in vdev_disk_io_flush()
1261 BIO_END_IO_PROTO(vdev_disk_discard_end_io, bio, error)  in BIO_END_IO_PROTO()  argument
1263 	zio_t *zio = bio->bi_private;  in BIO_END_IO_PROTO()
1265 	zio->io_error = BIO_END_IO_ERROR(bio);  in BIO_END_IO_PROTO()
1269 	bio_put(bio);  in BIO_END_IO_PROTO()
1281     sector_t nsect, struct bio **biop)  in vdev_bdev_issue_secure_erase()
1304     sector_t nsect, struct bio **biop)  in vdev_bdev_issue_discard()
1337 	struct bio *bio;  in vdev_disk_io_trim()  local
1344 		error = vdev_bdev_issue_secure_erase(bdh, sector, nsects, &bio);  in vdev_disk_io_trim()
1346 		error = vdev_bdev_issue_discard(bdh, sector, nsects, &bio);  in vdev_disk_io_trim()
1351 	if (bio == NULL) {  in vdev_disk_io_trim()
1362 		bio->bi_private = zio;  in vdev_disk_io_trim()
1363 		bio->bi_end_io = vdev_disk_discard_end_io;  in vdev_disk_io_trim()
1364 		vdev_submit_bio(bio);  in vdev_disk_io_trim()