26 static void io_complete_rw(struct kiocb *kiocb, long res);
27 static void io_complete_rw_iopoll(struct kiocb *kiocb, long res);
31 	struct kiocb			kiocb;  member
217 static inline void io_meta_restore(struct io_async_rw *io, struct kiocb *kiocb)  in io_meta_restore()  argument
219 	if (kiocb->ki_flags & IOCB_HAS_METADATA) {  in io_meta_restore()
263 	rw->kiocb.ki_pos = READ_ONCE(sqe->off);  in __io_prep_rw()
273 		rw->kiocb.ki_ioprio = ioprio;  in __io_prep_rw()
275 		rw->kiocb.ki_ioprio = get_current_ioprio();  in __io_prep_rw()
277 	rw->kiocb.dio_complete = NULL;  in __io_prep_rw()
278 	rw->kiocb.ki_flags = 0;  in __io_prep_rw()
281 		rw->kiocb.ki_complete = io_complete_rw_iopoll;  in __io_prep_rw()
283 		rw->kiocb.ki_complete = io_complete_rw;  in __io_prep_rw()
468 	if (rw->kiocb.ki_pos != -1)  in io_kiocb_update_pos()
469 		return &rw->kiocb.ki_pos;  in io_kiocb_update_pos()
473 		rw->kiocb.ki_pos = req->file->f_pos;  in io_kiocb_update_pos()
474 		return &rw->kiocb.ki_pos;  in io_kiocb_update_pos()
477 	rw->kiocb.ki_pos = 0;  in io_kiocb_update_pos()
502 	io_meta_restore(io, &rw->kiocb);  in io_rw_should_reissue()
515 		kiocb_end_write(&rw->kiocb);  in io_req_end_write()
527 	if (rw->kiocb.ki_flags & IOCB_WRITE) {  in io_req_io_end()
564 	struct kiocb *kiocb = &rw->kiocb;  in io_req_rw_complete()  local
566 	if ((kiocb->ki_flags & IOCB_DIO_CALLER_COMP) && kiocb->dio_complete) {  in io_req_rw_complete()
567 		long res = kiocb->dio_complete(rw->kiocb.private);  in io_req_rw_complete()
581 static void io_complete_rw(struct kiocb *kiocb, long res)  in io_complete_rw()  argument
583 	struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);  in io_complete_rw()
586 	if (!kiocb->dio_complete || !(kiocb->ki_flags & IOCB_DIO_CALLER_COMP)) {  in io_complete_rw()
594 static void io_complete_rw_iopoll(struct kiocb *kiocb, long res)  in io_complete_rw_iopoll()  argument
596 	struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);  in io_complete_rw_iopoll()
599 	if (kiocb->ki_flags & IOCB_WRITE)  in io_complete_rw_iopoll()
638 		io_complete_rw_iopoll(&rw->kiocb, ret);  in io_rw_done()
640 		io_complete_rw(&rw->kiocb, ret);  in io_rw_done()
650 		req->file->f_pos = rw->kiocb.ki_pos;  in kiocb_done()
668 static inline loff_t *io_kiocb_ppos(struct kiocb *kiocb)  in io_kiocb_ppos()  argument
670 	return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos;  in io_kiocb_ppos()
680 	struct kiocb *kiocb = &rw->kiocb;  in loop_rw_iter()  local
681 	struct file *file = kiocb->ki_filp;  in loop_rw_iter()
690 	if (kiocb->ki_flags & IOCB_HIPRI)  in loop_rw_iter()
692 	if ((kiocb->ki_flags & IOCB_NOWAIT) &&  in loop_rw_iter()
693 	    !(kiocb->ki_filp->f_flags & O_NONBLOCK))  in loop_rw_iter()
698 	ppos = io_kiocb_ppos(kiocb);  in loop_rw_iter()
765 	rw->kiocb.ki_flags &= ~IOCB_WAITQ;  in io_async_buf_func()
788 	struct kiocb *kiocb = &rw->kiocb;  in io_rw_should_retry()  local
798 	if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI))  in io_rw_should_retry()
813 	kiocb->ki_flags |= IOCB_WAITQ;  in io_rw_should_retry()
814 	kiocb->ki_flags &= ~IOCB_NOWAIT;  in io_rw_should_retry()
815 	kiocb->ki_waitq = wait;  in io_rw_should_retry()
821 	struct file *file = rw->kiocb.ki_filp;  in io_iter_do_read()
824 		return file->f_op->read_iter(&rw->kiocb, iter);  in io_iter_do_read()
840 	struct kiocb *kiocb = &rw->kiocb;  in io_rw_init_file()  local
851 	kiocb->ki_flags = file->f_iocb_flags;  in io_rw_init_file()
852 	ret = kiocb_set_rw_flags(kiocb, rw->flags, rw_type);  in io_rw_init_file()
855 	kiocb->ki_flags |= IOCB_ALLOC_CACHE;  in io_rw_init_file()
862 	if (kiocb->ki_flags & IOCB_NOWAIT ||  in io_rw_init_file()
867 		if (!(kiocb->ki_flags & IOCB_DIRECT) || !file->f_op->iopoll)  in io_rw_init_file()
869 		kiocb->private = NULL;  in io_rw_init_file()
870 		kiocb->ki_flags |= IOCB_HIPRI;  in io_rw_init_file()
878 		if (kiocb->ki_flags & IOCB_HIPRI)  in io_rw_init_file()
891 		kiocb->ki_flags |= IOCB_HAS_METADATA;  in io_rw_init_file()
892 		kiocb->private = &io->meta;  in io_rw_init_file()
903 	struct kiocb *kiocb = &rw->kiocb;  in __io_read()  local
925 		kiocb->ki_flags |= IOCB_NOWAIT;  in __io_read()
928 		kiocb->ki_flags &= ~IOCB_NOWAIT;  in __io_read()
973 	io_meta_restore(io, kiocb);  in __io_read()
989 			kiocb->ki_flags &= ~IOCB_WAITQ;  in __io_read()
1004 		kiocb->ki_flags &= ~IOCB_WAITQ;  in __io_read()
1036 	rw->kiocb.ki_complete = NULL;  in io_read_mshot()
1089 static bool io_kiocb_start_write(struct io_kiocb *req, struct kiocb *kiocb)  in io_kiocb_start_write()  argument
1096 	if (!(kiocb->ki_flags & IOCB_NOWAIT)) {  in io_kiocb_start_write()
1097 		kiocb_start_write(kiocb);  in io_kiocb_start_write()
1101 	inode = file_inode(kiocb->ki_filp);  in io_kiocb_start_write()
1113 	struct kiocb *kiocb = &rw->kiocb;  in io_write()  local
1134 		if (!(kiocb->ki_flags & IOCB_DIRECT) &&  in io_write()
1139 		kiocb->ki_flags |= IOCB_NOWAIT;  in io_write()
1142 		kiocb->ki_flags &= ~IOCB_NOWAIT;  in io_write()
1151 	if (unlikely(!io_kiocb_start_write(req, kiocb)))  in io_write()
1153 	kiocb->ki_flags |= IOCB_WRITE;  in io_write()
1156 		ret2 = req->file->f_op->write_iter(kiocb, &io->iter);  in io_write()
1166 	if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))  in io_write()
1177 			trace_io_uring_short_write(req->ctx, kiocb->ki_pos - ret2,  in io_write()
1188 			if (kiocb->ki_flags & IOCB_WRITE)  in io_write()
1197 		io_meta_restore(io, kiocb);  in io_write()
1198 		if (kiocb->ki_flags & IOCB_WRITE)  in io_write()
1247 		return file->f_op->iopoll(&rw->kiocb, iob, poll_flags);  in io_uring_classic_poll()