126 	struct blkdev_dio *dio = bio->bi_private;  in blkdev_bio_end_io()  local
127 	bool should_dirty = dio->flags & DIO_SHOULD_DIRTY;  in blkdev_bio_end_io()
128 	bool is_sync = dio->flags & DIO_IS_SYNC;  in blkdev_bio_end_io()
130 	if (bio->bi_status && !dio->bio.bi_status)  in blkdev_bio_end_io()
131 		dio->bio.bi_status = bio->bi_status;  in blkdev_bio_end_io()
133 	if (!is_sync && (dio->iocb->ki_flags & IOCB_HAS_METADATA))  in blkdev_bio_end_io()
136 	if (atomic_dec_and_test(&dio->ref)) {  in blkdev_bio_end_io()
138 			struct kiocb *iocb = dio->iocb;  in blkdev_bio_end_io()
143 			if (likely(!dio->bio.bi_status)) {  in blkdev_bio_end_io()
144 				ret = dio->size;  in blkdev_bio_end_io()
147 				ret = blk_status_to_errno(dio->bio.bi_status);  in blkdev_bio_end_io()
150 			dio->iocb->ki_complete(iocb, ret);  in blkdev_bio_end_io()
151 			bio_put(&dio->bio);  in blkdev_bio_end_io()
153 			struct task_struct *waiter = dio->waiter;  in blkdev_bio_end_io()
155 			WRITE_ONCE(dio->waiter, NULL);  in blkdev_bio_end_io()
172 	struct blkdev_dio *dio;  in __blkdev_direct_IO()  local
183 	dio = container_of(bio, struct blkdev_dio, bio);  in __blkdev_direct_IO()
184 	atomic_set(&dio->ref, 1);  in __blkdev_direct_IO()
193 		dio->flags = DIO_IS_SYNC;  in __blkdev_direct_IO()
194 		dio->waiter = current;  in __blkdev_direct_IO()
196 		dio->flags = 0;  in __blkdev_direct_IO()
197 		dio->iocb = iocb;  in __blkdev_direct_IO()
200 	dio->size = 0;  in __blkdev_direct_IO()
202 		dio->flags |= DIO_SHOULD_DIRTY;  in __blkdev_direct_IO()
209 		bio->bi_private = dio;  in __blkdev_direct_IO()
241 			if (dio->flags & DIO_SHOULD_DIRTY)  in __blkdev_direct_IO()
246 		dio->size += bio->bi_iter.bi_size;  in __blkdev_direct_IO()
254 		atomic_inc(&dio->ref);  in __blkdev_direct_IO()
266 		if (!READ_ONCE(dio->waiter))  in __blkdev_direct_IO()
273 		ret = blk_status_to_errno(dio->bio.bi_status);  in __blkdev_direct_IO()
275 		ret = dio->size;  in __blkdev_direct_IO()
277 	bio_put(&dio->bio);  in __blkdev_direct_IO()
289 	struct blkdev_dio *dio = container_of(bio, struct blkdev_dio, bio);  in blkdev_bio_end_io_async()  local
290 	struct kiocb *iocb = dio->iocb;  in blkdev_bio_end_io_async()
296 		ret = dio->size;  in blkdev_bio_end_io_async()
307 	if (dio->flags & DIO_SHOULD_DIRTY) {  in blkdev_bio_end_io_async()
322 	struct blkdev_dio *dio;  in __blkdev_direct_IO_async()  local
331 	dio = container_of(bio, struct blkdev_dio, bio);  in __blkdev_direct_IO_async()
332 	dio->flags = 0;  in __blkdev_direct_IO_async()
333 	dio->iocb = iocb;  in __blkdev_direct_IO_async()
352 	dio->size = bio->bi_iter.bi_size;  in __blkdev_direct_IO_async()
356 			dio->flags |= DIO_SHOULD_DIRTY;  in __blkdev_direct_IO_async()