74 static void copy_to_high_bio_irq(struct bio *to, struct bio *from)  in copy_to_high_bio_irq()
100 static void bounce_end_io(struct bio *bio)  in bounce_end_io()  argument
102 	struct bio *bio_orig = bio->bi_private;  in bounce_end_io()
110 	bio_for_each_segment_all(bvec, bio, iter_all) {  in bounce_end_io()
119 	bio_orig->bi_status = bio->bi_status;  in bounce_end_io()
121 	bio_put(bio);  in bounce_end_io()
124 static void bounce_end_io_write(struct bio *bio)  in bounce_end_io_write()  argument
126 	bounce_end_io(bio);  in bounce_end_io_write()
129 static void bounce_end_io_read(struct bio *bio)  in bounce_end_io_read()  argument
131 	struct bio *bio_orig = bio->bi_private;  in bounce_end_io_read()
133 	if (!bio->bi_status)  in bounce_end_io_read()
134 		copy_to_high_bio_irq(bio_orig, bio);  in bounce_end_io_read()
136 	bounce_end_io(bio);  in bounce_end_io_read()
139 static struct bio *bounce_clone_bio(struct bio *bio_src)  in bounce_clone_bio()
143 	struct bio *bio;  in bounce_clone_bio()  local
165 	bio = bio_alloc_bioset(bio_src->bi_bdev, bio_segments(bio_src),  in bounce_clone_bio()
168 		bio_set_flag(bio, BIO_REMAPPED);  in bounce_clone_bio()
169 	bio->bi_ioprio		= bio_src->bi_ioprio;  in bounce_clone_bio()
170 	bio->bi_write_hint	= bio_src->bi_write_hint;  in bounce_clone_bio()
171 	bio->bi_iter.bi_sector	= bio_src->bi_iter.bi_sector;  in bounce_clone_bio()
172 	bio->bi_iter.bi_size	= bio_src->bi_iter.bi_size;  in bounce_clone_bio()
174 	switch (bio_op(bio)) {  in bounce_clone_bio()
181 			bio->bi_io_vec[bio->bi_vcnt++] = bv;  in bounce_clone_bio()
185 	if (bio_crypt_clone(bio, bio_src, GFP_NOIO) < 0)  in bounce_clone_bio()
189 	    bio_integrity_clone(bio, bio_src, GFP_NOIO) < 0)  in bounce_clone_bio()
192 	bio_clone_blkg_association(bio, bio_src);  in bounce_clone_bio()
194 	return bio;  in bounce_clone_bio()
197 	bio_put(bio);  in bounce_clone_bio()
201 struct bio *__blk_queue_bounce(struct bio *bio_orig, struct request_queue *q)  in __blk_queue_bounce()
203 	struct bio *bio;  in __blk_queue_bounce()  local
228 		bio = bio_split(bio_orig, sectors, GFP_NOIO, &bounce_bio_split);  in __blk_queue_bounce()
229 		bio_chain(bio, bio_orig);  in __blk_queue_bounce()
231 		bio_orig = bio;  in __blk_queue_bounce()
233 	bio = bounce_clone_bio(bio_orig);  in __blk_queue_bounce()
240 	for (i = 0, to = bio->bi_io_vec; i < bio->bi_vcnt; to++, i++) {  in __blk_queue_bounce()
258 	bio->bi_flags |= (1 << BIO_BOUNCED);  in __blk_queue_bounce()
261 		bio->bi_end_io = bounce_end_io_read;  in __blk_queue_bounce()
263 		bio->bi_end_io = bounce_end_io_write;  in __blk_queue_bounce()
265 	bio->bi_private = bio_orig;  in __blk_queue_bounce()
266 	return bio;  in __blk_queue_bounce()