28 static blk_opf_t dio_bio_write_op(struct kiocb *iocb)  in dio_bio_write_op()  argument
33 	if (iocb_is_dsync(iocb))  in dio_bio_write_op()
38 static bool blkdev_dio_invalid(struct block_device *bdev, struct kiocb *iocb,  in blkdev_dio_invalid()  argument
41 	return iocb->ki_pos & (bdev_logical_block_size(bdev) - 1) ||  in blkdev_dio_invalid()
47 static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb,  in __blkdev_direct_IO_simple()  argument
52 	loff_t pos = iocb->ki_pos;  in __blkdev_direct_IO_simple()
57 	WARN_ON_ONCE(iocb->ki_flags & IOCB_HAS_METADATA);  in __blkdev_direct_IO_simple()
72 		bio_init(&bio, bdev, vecs, nr_pages, dio_bio_write_op(iocb));  in __blkdev_direct_IO_simple()
75 	bio.bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint;  in __blkdev_direct_IO_simple()
76 	bio.bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO_simple()
77 	if (iocb->ki_flags & IOCB_ATOMIC)  in __blkdev_direct_IO_simple()
88 	if (iocb->ki_flags & IOCB_NOWAIT)  in __blkdev_direct_IO_simple()
113 		struct kiocb		*iocb;  member
133 	if (!is_sync && (dio->iocb->ki_flags & IOCB_HAS_METADATA))  in blkdev_bio_end_io()
138 			struct kiocb *iocb = dio->iocb;  in blkdev_bio_end_io()  local
141 			WRITE_ONCE(iocb->private, NULL);  in blkdev_bio_end_io()
145 				iocb->ki_pos += ret;  in blkdev_bio_end_io()
150 			dio->iocb->ki_complete(iocb, ret);  in blkdev_bio_end_io()
168 static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,  in __blkdev_direct_IO()  argument
175 	blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb);  in __blkdev_direct_IO()
176 	loff_t pos = iocb->ki_pos;  in __blkdev_direct_IO()
179 	if (iocb->ki_flags & IOCB_ALLOC_CACHE)  in __blkdev_direct_IO()
191 	is_sync = is_sync_kiocb(iocb);  in __blkdev_direct_IO()
197 		dio->iocb = iocb;  in __blkdev_direct_IO()
208 		bio->bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint;  in __blkdev_direct_IO()
211 		bio->bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO()
219 		if (iocb->ki_flags & IOCB_NOWAIT) {  in __blkdev_direct_IO()
234 		if (!is_sync && (iocb->ki_flags & IOCB_HAS_METADATA)) {  in __blkdev_direct_IO()
235 			ret = bio_integrity_map_iter(bio, iocb->private);  in __blkdev_direct_IO()
290 	struct kiocb *iocb = dio->iocb;  in blkdev_bio_end_io_async()  local
293 	WRITE_ONCE(iocb->private, NULL);  in blkdev_bio_end_io_async()
297 		iocb->ki_pos += ret;  in blkdev_bio_end_io_async()
302 	if (iocb->ki_flags & IOCB_HAS_METADATA)  in blkdev_bio_end_io_async()
305 	iocb->ki_complete(iocb, ret);  in blkdev_bio_end_io_async()
315 static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb,  in __blkdev_direct_IO_async()  argument
321 	blk_opf_t opf = is_read ? REQ_OP_READ : dio_bio_write_op(iocb);  in __blkdev_direct_IO_async()
324 	loff_t pos = iocb->ki_pos;  in __blkdev_direct_IO_async()
327 	if (iocb->ki_flags & IOCB_ALLOC_CACHE)  in __blkdev_direct_IO_async()
333 	dio->iocb = iocb;  in __blkdev_direct_IO_async()
335 	bio->bi_write_hint = file_inode(iocb->ki_filp)->i_write_hint;  in __blkdev_direct_IO_async()
337 	bio->bi_ioprio = iocb->ki_ioprio;  in __blkdev_direct_IO_async()
363 	if (iocb->ki_flags & IOCB_HAS_METADATA) {  in __blkdev_direct_IO_async()
364 		ret = bio_integrity_map_iter(bio, iocb->private);  in __blkdev_direct_IO_async()
365 		WRITE_ONCE(iocb->private, NULL);  in __blkdev_direct_IO_async()
370 	if (iocb->ki_flags & IOCB_ATOMIC)  in __blkdev_direct_IO_async()
373 	if (iocb->ki_flags & IOCB_NOWAIT)  in __blkdev_direct_IO_async()
376 	if (iocb->ki_flags & IOCB_HIPRI) {  in __blkdev_direct_IO_async()
379 		WRITE_ONCE(iocb->private, bio);  in __blkdev_direct_IO_async()
390 static ssize_t blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter)  in blkdev_direct_IO()  argument
392 	struct block_device *bdev = I_BDEV(iocb->ki_filp->f_mapping->host);  in blkdev_direct_IO()
398 	if (blkdev_dio_invalid(bdev, iocb, iter))  in blkdev_direct_IO()
403 		if (is_sync_kiocb(iocb))  in blkdev_direct_IO()
404 			return __blkdev_direct_IO_simple(iocb, iter, bdev,  in blkdev_direct_IO()
406 		return __blkdev_direct_IO_async(iocb, iter, bdev, nr_pages);  in blkdev_direct_IO()
407 	} else if (iocb->ki_flags & IOCB_ATOMIC) {  in blkdev_direct_IO()
410 	return __blkdev_direct_IO(iocb, iter, bdev, bio_max_segs(nr_pages));  in blkdev_direct_IO()
665 blkdev_direct_write(struct kiocb *iocb, struct iov_iter *from)  in blkdev_direct_write()  argument
670 	written = kiocb_invalidate_pages(iocb, count);  in blkdev_direct_write()
677 	written = blkdev_direct_IO(iocb, from);  in blkdev_direct_write()
679 		kiocb_invalidate_post_direct_write(iocb, count);  in blkdev_direct_write()
680 		iocb->ki_pos += written;  in blkdev_direct_write()
688 static ssize_t blkdev_buffered_write(struct kiocb *iocb, struct iov_iter *from)  in blkdev_buffered_write()  argument
690 	return iomap_file_buffered_write(iocb, from, &blkdev_iomap_ops, NULL);  in blkdev_buffered_write()
700 static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)  in blkdev_write_iter()  argument
702 	struct file *file = iocb->ki_filp;  in blkdev_write_iter()
705 	bool atomic = iocb->ki_flags & IOCB_ATOMIC;  in blkdev_write_iter()
719 	if (iocb->ki_pos >= size)  in blkdev_write_iter()
722 	if ((iocb->ki_flags & (IOCB_NOWAIT | IOCB_DIRECT)) == IOCB_NOWAIT)  in blkdev_write_iter()
726 		ret = generic_atomic_write_valid(iocb, from);  in blkdev_write_iter()
731 	size -= iocb->ki_pos;  in blkdev_write_iter()
743 	if (iocb->ki_flags & IOCB_DIRECT) {  in blkdev_write_iter()
744 		ret = blkdev_direct_write(iocb, from);  in blkdev_write_iter()
746 			ret = direct_write_fallback(iocb, from, ret,  in blkdev_write_iter()
747 					blkdev_buffered_write(iocb, from));  in blkdev_write_iter()
755 		ret = blkdev_buffered_write(iocb, from);  in blkdev_write_iter()
760 		ret = generic_write_sync(iocb, ret);  in blkdev_write_iter()
765 static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to)  in blkdev_read_iter()  argument
767 	struct inode *bd_inode = bdev_file_inode(iocb->ki_filp);  in blkdev_read_iter()
768 	struct block_device *bdev = I_BDEV(iocb->ki_filp->f_mapping->host);  in blkdev_read_iter()
770 	loff_t pos = iocb->ki_pos;  in blkdev_read_iter()
787 	if (iocb->ki_flags & IOCB_DIRECT) {  in blkdev_read_iter()
788 		ret = kiocb_write_and_wait(iocb, count);  in blkdev_read_iter()
791 		file_accessed(iocb->ki_filp);  in blkdev_read_iter()
793 		ret = blkdev_direct_IO(iocb, to);  in blkdev_read_iter()
795 			iocb->ki_pos += ret;  in blkdev_read_iter()
809 	ret = filemap_read(iocb, to, ret);  in blkdev_read_iter()