54 	struct bio bio;  in __blkdev_direct_IO_simple()  local
68 		bio_init(&bio, bdev, vecs, nr_pages, REQ_OP_READ);  in __blkdev_direct_IO_simple()
72 		bio_init(&bio, bdev, vecs, nr_pages, dio_bio_write_op(iocb));  in __blkdev_direct_IO_simple()
74 	bio.bi_iter.bi_sector = pos >> SECTOR_SHIFT;  in __blkdev_direct_IO_simple()
75 	bio.bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint;  in __blkdev_direct_IO_simple()
76 	bio.bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO_simple()
78 		bio.bi_opf |= REQ_ATOMIC;  in __blkdev_direct_IO_simple()
80 	ret = bio_iov_iter_get_pages(&bio, iter);  in __blkdev_direct_IO_simple()
83 	ret = bio.bi_iter.bi_size;  in __blkdev_direct_IO_simple()
89 		bio.bi_opf |= REQ_NOWAIT;  in __blkdev_direct_IO_simple()
91 	submit_bio_wait(&bio);  in __blkdev_direct_IO_simple()
93 	bio_release_pages(&bio, should_dirty);  in __blkdev_direct_IO_simple()
94 	if (unlikely(bio.bi_status))  in __blkdev_direct_IO_simple()
95 		ret = blk_status_to_errno(bio.bi_status);  in __blkdev_direct_IO_simple()
101 	bio_uninit(&bio);  in __blkdev_direct_IO_simple()
119 	struct bio		bio ____cacheline_aligned_in_smp;
124 static void blkdev_bio_end_io(struct bio *bio)  in blkdev_bio_end_io()  argument
126 	struct blkdev_dio *dio = bio->bi_private;  in blkdev_bio_end_io()
130 	if (bio->bi_status && !dio->bio.bi_status)  in blkdev_bio_end_io()
131 		dio->bio.bi_status = bio->bi_status;  in blkdev_bio_end_io()
134 		bio_integrity_unmap_user(bio);  in blkdev_bio_end_io()
143 			if (likely(!dio->bio.bi_status)) {  in blkdev_bio_end_io()
147 				ret = blk_status_to_errno(dio->bio.bi_status);  in blkdev_bio_end_io()
151 			bio_put(&dio->bio);  in blkdev_bio_end_io()
161 		bio_check_pages_dirty(bio);  in blkdev_bio_end_io()
163 		bio_release_pages(bio, false);  in blkdev_bio_end_io()
164 		bio_put(bio);  in blkdev_bio_end_io()
173 	struct bio *bio;  in __blkdev_direct_IO()  local
181 	bio = bio_alloc_bioset(bdev, nr_pages, opf, GFP_KERNEL,  in __blkdev_direct_IO()
183 	dio = container_of(bio, struct blkdev_dio, bio);  in __blkdev_direct_IO()
189 	bio_get(bio);  in __blkdev_direct_IO()
207 		bio->bi_iter.bi_sector = pos >> SECTOR_SHIFT;  in __blkdev_direct_IO()
208 		bio->bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint;  in __blkdev_direct_IO()
209 		bio->bi_private = dio;  in __blkdev_direct_IO()
210 		bio->bi_end_io = blkdev_bio_end_io;  in __blkdev_direct_IO()
211 		bio->bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO()
213 		ret = bio_iov_iter_get_pages(bio, iter);  in __blkdev_direct_IO()
215 			bio->bi_status = BLK_STS_IOERR;  in __blkdev_direct_IO()
216 			bio_endio(bio);  in __blkdev_direct_IO()
232 			bio->bi_opf |= REQ_NOWAIT;  in __blkdev_direct_IO()
235 			ret = bio_integrity_map_iter(bio, iocb->private);  in __blkdev_direct_IO()
242 				bio_set_pages_dirty(bio);  in __blkdev_direct_IO()
244 			task_io_account_write(bio->bi_iter.bi_size);  in __blkdev_direct_IO()
246 		dio->size += bio->bi_iter.bi_size;  in __blkdev_direct_IO()
247 		pos += bio->bi_iter.bi_size;  in __blkdev_direct_IO()
251 			submit_bio(bio);  in __blkdev_direct_IO()
255 		submit_bio(bio);  in __blkdev_direct_IO()
256 		bio = bio_alloc(bdev, nr_pages, opf, GFP_KERNEL);  in __blkdev_direct_IO()
273 		ret = blk_status_to_errno(dio->bio.bi_status);  in __blkdev_direct_IO()
277 	bio_put(&dio->bio);  in __blkdev_direct_IO()
280 	bio_release_pages(bio, false);  in __blkdev_direct_IO()
281 	bio_clear_flag(bio, BIO_REFFED);  in __blkdev_direct_IO()
282 	bio_put(bio);  in __blkdev_direct_IO()
287 static void blkdev_bio_end_io_async(struct bio *bio)  in blkdev_bio_end_io_async()  argument
289 	struct blkdev_dio *dio = container_of(bio, struct blkdev_dio, bio);  in blkdev_bio_end_io_async()
295 	if (likely(!bio->bi_status)) {  in blkdev_bio_end_io_async()
299 		ret = blk_status_to_errno(bio->bi_status);  in blkdev_bio_end_io_async()
303 		bio_integrity_unmap_user(bio);  in blkdev_bio_end_io_async()
308 		bio_check_pages_dirty(bio);  in blkdev_bio_end_io_async()
310 		bio_release_pages(bio, false);  in blkdev_bio_end_io_async()
311 		bio_put(bio);  in blkdev_bio_end_io_async()
323 	struct bio *bio;  in __blkdev_direct_IO_async()  local
329 	bio = bio_alloc_bioset(bdev, nr_pages, opf, GFP_KERNEL,  in __blkdev_direct_IO_async()
331 	dio = container_of(bio, struct blkdev_dio, bio);  in __blkdev_direct_IO_async()
334 	bio->bi_iter.bi_sector = pos >> SECTOR_SHIFT;  in __blkdev_direct_IO_async()
335 	bio->bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint;  in __blkdev_direct_IO_async()
336 	bio->bi_end_io = blkdev_bio_end_io_async;  in __blkdev_direct_IO_async()
337 	bio->bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO_async()
346 		bio_iov_bvec_set(bio, iter);  in __blkdev_direct_IO_async()
348 		ret = bio_iov_iter_get_pages(bio, iter);  in __blkdev_direct_IO_async()
352 	dio->size = bio->bi_iter.bi_size;  in __blkdev_direct_IO_async()
357 			bio_set_pages_dirty(bio);  in __blkdev_direct_IO_async()
360 		task_io_account_write(bio->bi_iter.bi_size);  in __blkdev_direct_IO_async()
364 		ret = bio_integrity_map_iter(bio, iocb->private);  in __blkdev_direct_IO_async()
371 		bio->bi_opf |= REQ_ATOMIC;  in __blkdev_direct_IO_async()
374 		bio->bi_opf |= REQ_NOWAIT;  in __blkdev_direct_IO_async()
377 		bio->bi_opf |= REQ_POLLED;  in __blkdev_direct_IO_async()
378 		submit_bio(bio);  in __blkdev_direct_IO_async()
379 		WRITE_ONCE(iocb->private, bio);  in __blkdev_direct_IO_async()
381 		submit_bio(bio);  in __blkdev_direct_IO_async()
386 	bio_put(bio);  in __blkdev_direct_IO_async()
923 				offsetof(struct blkdev_dio, bio),  in blkdev_init()