52 static int io_iov_compat_buffer_select_prep(struct io_rw *rw)  in io_iov_compat_buffer_select_prep()  argument
54 	struct compat_iovec __user *uiov = u64_to_user_ptr(rw->addr);  in io_iov_compat_buffer_select_prep()
59 	rw->len = iov.iov_len;  in io_iov_compat_buffer_select_prep()
67 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_iov_buffer_select_prep()  local
69 	if (rw->len != 1)  in io_iov_buffer_select_prep()
73 		return io_iov_compat_buffer_select_prep(rw);  in io_iov_buffer_select_prep()
75 	uiov = u64_to_user_ptr(rw->addr);  in io_iov_buffer_select_prep()
78 	rw->len = iov.iov_len;  in io_iov_buffer_select_prep()
114 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in __io_import_rw_buffer()  local
115 	void __user *buf = u64_to_user_ptr(rw->addr);  in __io_import_rw_buffer()
116 	size_t sqe_len = rw->len;  in __io_import_rw_buffer()
125 		rw->addr = (unsigned long) buf;  in __io_import_rw_buffer()
126 		rw->len = sqe_len;  in __io_import_rw_buffer()
131 static inline int io_import_rw_buffer(int rw, struct io_kiocb *req,  in io_import_rw_buffer()  argument
137 	ret = __io_import_rw_buffer(rw, req, io, issue_flags);  in io_import_rw_buffer()
147 	struct io_async_rw *rw = req->async_data;  in io_rw_recycle()  local
152 	io_alloc_cache_vec_kasan(&rw->vec);  in io_rw_recycle()
153 	if (rw->vec.nr > IO_VEC_CACHE_SOFT_CAP)  in io_rw_recycle()
154 		io_vec_free(&rw->vec);  in io_rw_recycle()
156 	if (io_alloc_cache_put(&req->ctx->rw_cache, rw)) {  in io_rw_recycle()
200 	struct io_async_rw *rw;  in io_rw_alloc_async()  local
202 	rw = io_uring_alloc_async_data(&ctx->rw_cache, req);  in io_rw_alloc_async()
203 	if (!rw)  in io_rw_alloc_async()
205 	if (rw->vec.iovec)  in io_rw_alloc_async()
207 	rw->bytes_done = 0;  in io_rw_alloc_async()
225 static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir,  in io_prep_rw_pi()  argument
255 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in __io_prep_rw()  local
263 	rw->kiocb.ki_pos = READ_ONCE(sqe->off);  in __io_prep_rw()
273 		rw->kiocb.ki_ioprio = ioprio;  in __io_prep_rw()
275 		rw->kiocb.ki_ioprio = get_current_ioprio();  in __io_prep_rw()
277 	rw->kiocb.dio_complete = NULL;  in __io_prep_rw()
278 	rw->kiocb.ki_flags = 0;  in __io_prep_rw()
281 		rw->kiocb.ki_complete = io_complete_rw_iopoll;  in __io_prep_rw()
283 		rw->kiocb.ki_complete = io_complete_rw;  in __io_prep_rw()
285 	rw->addr = READ_ONCE(sqe->addr);  in __io_prep_rw()
286 	rw->len = READ_ONCE(sqe->len);  in __io_prep_rw()
287 	rw->flags = READ_ONCE(sqe->rw_flags);  in __io_prep_rw()
298 		return io_prep_rw_pi(req, rw, ddir, attr_ptr, attr_type_mask);  in __io_prep_rw()
364 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_init_rw_fixed()  local
371 	ret = io_import_reg_buf(req, &io->iter, rw->addr, rw->len, ddir,  in io_init_rw_fixed()
391 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_rw_import_reg_vec()  local
392 	unsigned uvec_segs = rw->len;  in io_rw_import_reg_vec()
406 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_rw_prep_reg_vec()  local
410 	uvec = u64_to_user_ptr(rw->addr);  in io_rw_prep_reg_vec()
411 	return io_prep_reg_iovec(req, &io->vec, uvec, rw->len);  in io_rw_prep_reg_vec()
440 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_read_mshot_prep()  local
451 	if (rw->addr || rw->len)  in io_read_mshot_prep()
466 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_kiocb_update_pos()  local
468 	if (rw->kiocb.ki_pos != -1)  in io_kiocb_update_pos()
469 		return &rw->kiocb.ki_pos;  in io_kiocb_update_pos()
473 		rw->kiocb.ki_pos = req->file->f_pos;  in io_kiocb_update_pos()
474 		return &rw->kiocb.ki_pos;  in io_kiocb_update_pos()
477 	rw->kiocb.ki_pos = 0;  in io_kiocb_update_pos()
484 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_rw_should_reissue()  local
502 	io_meta_restore(io, &rw->kiocb);  in io_rw_should_reissue()
513 		struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_req_end_write()  local
515 		kiocb_end_write(&rw->kiocb);  in io_req_end_write()
525 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_req_io_end()  local
527 	if (rw->kiocb.ki_flags & IOCB_WRITE) {  in io_req_io_end()
563 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_req_rw_complete()  local
564 	struct kiocb *kiocb = &rw->kiocb;  in io_req_rw_complete()
567 		long res = kiocb->dio_complete(rw->kiocb.private);  in io_req_rw_complete()
583 	struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);  in io_complete_rw()  local
584 	struct io_kiocb *req = cmd_to_io_kiocb(rw);  in io_complete_rw()
596 	struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);  in io_complete_rw_iopoll()  local
597 	struct io_kiocb *req = cmd_to_io_kiocb(rw);  in io_complete_rw_iopoll()
614 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_rw_done()  local
638 		io_complete_rw_iopoll(&rw->kiocb, ret);  in io_rw_done()
640 		io_complete_rw(&rw->kiocb, ret);  in io_rw_done()
646 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in kiocb_done()  local
650 		req->file->f_pos = rw->kiocb.ki_pos;  in kiocb_done()
677 static ssize_t loop_rw_iter(int ddir, struct io_rw *rw, struct iov_iter *iter)  in loop_rw_iter()  argument
679 	struct io_kiocb *req = cmd_to_io_kiocb(rw);  in loop_rw_iter()
680 	struct kiocb *kiocb = &rw->kiocb;  in loop_rw_iter()
712 			addr = u64_to_user_ptr(rw->addr);  in loop_rw_iter()
713 			len = rw->len;  in loop_rw_iter()
730 			rw->addr += nr;  in loop_rw_iter()
731 			rw->len -= nr;  in loop_rw_iter()
732 			if (!rw->len)  in loop_rw_iter()
757 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_async_buf_func()  local
765 	rw->kiocb.ki_flags &= ~IOCB_WAITQ;  in io_async_buf_func()
787 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_rw_should_retry()  local
788 	struct kiocb *kiocb = &rw->kiocb;  in io_rw_should_retry()
819 static inline int io_iter_do_read(struct io_rw *rw, struct iov_iter *iter)  in io_iter_do_read()  argument
821 	struct file *file = rw->kiocb.ki_filp;  in io_iter_do_read()
824 		return file->f_op->read_iter(&rw->kiocb, iter);  in io_iter_do_read()
826 		return loop_rw_iter(READ, rw, iter);  in io_iter_do_read()
839 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_rw_init_file()  local
840 	struct kiocb *kiocb = &rw->kiocb;  in io_rw_init_file()
852 	ret = kiocb_set_rw_flags(kiocb, rw->flags, rw_type);  in io_rw_init_file()
901 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in __io_read()  local
903 	struct kiocb *kiocb = &rw->kiocb;  in __io_read()
937 	ret = io_iter_do_read(rw, &io->iter);  in __io_read()
1000 		ret = io_iter_do_read(rw, &io->iter);  in __io_read()
1025 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_read_mshot()  local
1036 	rw->kiocb.ki_complete = NULL;  in io_read_mshot()
1049 			rw->len = 0;  in io_read_mshot()
1065 		rw->len = 0; /* similarly to above, reset len to 0 */  in io_read_mshot()
1111 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_write()  local
1113 	struct kiocb *kiocb = &rw->kiocb;  in io_write()
1158 		ret2 = loop_rw_iter(WRITE, rw, &io->iter);  in io_write()
1245 		struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);  in io_uring_classic_poll()  local
1247 		return file->f_op->iopoll(&rw->kiocb, iob, poll_flags);  in io_uring_classic_poll()
1380 	struct io_async_rw *rw = (struct io_async_rw *) entry;  in io_rw_cache_free()  local
1382 	io_vec_free(&rw->vec);  in io_rw_cache_free()
1383 	kfree(rw);  in io_rw_cache_free()