Lines Matching refs:req
37 static bool io_file_supports_nowait(struct io_kiocb *req, __poll_t mask) in io_file_supports_nowait() argument
40 if (req->flags & REQ_F_SUPPORT_NOWAIT) in io_file_supports_nowait()
43 if (io_file_can_poll(req)) { in io_file_supports_nowait()
46 return vfs_poll(req->file, &pt) & mask; in io_file_supports_nowait()
63 static int io_iov_buffer_select_prep(struct io_kiocb *req) in io_iov_buffer_select_prep() argument
67 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_iov_buffer_select_prep()
72 if (io_is_compat(req->ctx)) in io_iov_buffer_select_prep()
82 static int io_import_vec(int ddir, struct io_kiocb *req, in io_import_vec() argument
99 io_is_compat(req->ctx)); in io_import_vec()
103 req->flags |= REQ_F_NEED_CLEANUP; in io_import_vec()
109 static int __io_import_rw_buffer(int ddir, struct io_kiocb *req, in __io_import_rw_buffer() argument
113 const struct io_issue_def *def = &io_issue_defs[req->opcode]; in __io_import_rw_buffer()
114 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in __io_import_rw_buffer()
118 if (def->vectored && !(req->flags & REQ_F_BUFFER_SELECT)) in __io_import_rw_buffer()
119 return io_import_vec(ddir, req, io, buf, sqe_len); in __io_import_rw_buffer()
121 if (io_do_buffer_select(req)) { in __io_import_rw_buffer()
122 buf = io_buffer_select(req, &sqe_len, issue_flags); in __io_import_rw_buffer()
131 static inline int io_import_rw_buffer(int rw, struct io_kiocb *req, in io_import_rw_buffer() argument
137 ret = __io_import_rw_buffer(rw, req, io, issue_flags); in io_import_rw_buffer()
145 static void io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags) in io_rw_recycle() argument
147 struct io_async_rw *rw = req->async_data; in io_rw_recycle()
156 if (io_alloc_cache_put(&req->ctx->rw_cache, rw)) { in io_rw_recycle()
157 req->async_data = NULL; in io_rw_recycle()
158 req->flags &= ~REQ_F_ASYNC_DATA; in io_rw_recycle()
162 static void io_req_rw_cleanup(struct io_kiocb *req, unsigned int issue_flags) in io_req_rw_cleanup() argument
191 if (!(req->flags & (REQ_F_REISSUE | REQ_F_REFCOUNT))) { in io_req_rw_cleanup()
192 req->flags &= ~REQ_F_NEED_CLEANUP; in io_req_rw_cleanup()
193 io_rw_recycle(req, issue_flags); in io_req_rw_cleanup()
197 static int io_rw_alloc_async(struct io_kiocb *req) in io_rw_alloc_async() argument
199 struct io_ring_ctx *ctx = req->ctx; in io_rw_alloc_async()
202 rw = io_uring_alloc_async_data(&ctx->rw_cache, req); in io_rw_alloc_async()
206 req->flags |= REQ_F_NEED_CLEANUP; in io_rw_alloc_async()
225 static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir, in io_prep_rw_pi() argument
239 io = req->async_data; in io_prep_rw_pi()
247 req->flags |= REQ_F_HAS_METADATA; in io_prep_rw_pi()
252 static int __io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, in __io_prep_rw() argument
255 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in __io_prep_rw()
260 if (io_rw_alloc_async(req)) in __io_prep_rw()
265 req->buf_index = READ_ONCE(sqe->buf_index); in __io_prep_rw()
280 if (req->ctx->flags & IORING_SETUP_IOPOLL) in __io_prep_rw()
298 return io_prep_rw_pi(req, rw, ddir, attr_ptr, attr_type_mask); in __io_prep_rw()
303 static int io_rw_do_import(struct io_kiocb *req, int ddir) in io_rw_do_import() argument
305 if (io_do_buffer_select(req)) in io_rw_do_import()
308 return io_import_rw_buffer(ddir, req, req->async_data, 0); in io_rw_do_import()
311 static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, in io_prep_rw() argument
316 ret = __io_prep_rw(req, sqe, ddir); in io_prep_rw()
320 return io_rw_do_import(req, ddir); in io_prep_rw()
323 int io_prep_read(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_prep_read() argument
325 return io_prep_rw(req, sqe, ITER_DEST); in io_prep_read()
328 int io_prep_write(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_prep_write() argument
330 return io_prep_rw(req, sqe, ITER_SOURCE); in io_prep_write()
333 static int io_prep_rwv(struct io_kiocb *req, const struct io_uring_sqe *sqe, in io_prep_rwv() argument
338 ret = io_prep_rw(req, sqe, ddir); in io_prep_rwv()
341 if (!(req->flags & REQ_F_BUFFER_SELECT)) in io_prep_rwv()
348 return io_iov_buffer_select_prep(req); in io_prep_rwv()
351 int io_prep_readv(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_prep_readv() argument
353 return io_prep_rwv(req, sqe, ITER_DEST); in io_prep_readv()
356 int io_prep_writev(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_prep_writev() argument
358 return io_prep_rwv(req, sqe, ITER_SOURCE); in io_prep_writev()
361 static int io_init_rw_fixed(struct io_kiocb *req, unsigned int issue_flags, in io_init_rw_fixed() argument
364 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_init_rw_fixed()
365 struct io_async_rw *io = req->async_data; in io_init_rw_fixed()
371 ret = io_import_reg_buf(req, &io->iter, rw->addr, rw->len, ddir, in io_init_rw_fixed()
377 int io_prep_read_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_prep_read_fixed() argument
379 return __io_prep_rw(req, sqe, ITER_DEST); in io_prep_read_fixed()
382 int io_prep_write_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_prep_write_fixed() argument
384 return __io_prep_rw(req, sqe, ITER_SOURCE); in io_prep_write_fixed()
387 static int io_rw_import_reg_vec(struct io_kiocb *req, in io_rw_import_reg_vec() argument
391 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_rw_import_reg_vec()
395 ret = io_import_reg_vec(ddir, &io->iter, req, &io->vec, in io_rw_import_reg_vec()
400 req->flags &= ~REQ_F_IMPORT_BUFFER; in io_rw_import_reg_vec()
404 static int io_rw_prep_reg_vec(struct io_kiocb *req) in io_rw_prep_reg_vec() argument
406 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_rw_prep_reg_vec()
407 struct io_async_rw *io = req->async_data; in io_rw_prep_reg_vec()
411 return io_prep_reg_iovec(req, &io->vec, uvec, rw->len); in io_rw_prep_reg_vec()
414 int io_prep_readv_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_prep_readv_fixed() argument
418 ret = __io_prep_rw(req, sqe, ITER_DEST); in io_prep_readv_fixed()
421 return io_rw_prep_reg_vec(req); in io_prep_readv_fixed()
424 int io_prep_writev_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_prep_writev_fixed() argument
428 ret = __io_prep_rw(req, sqe, ITER_SOURCE); in io_prep_writev_fixed()
431 return io_rw_prep_reg_vec(req); in io_prep_writev_fixed()
438 int io_read_mshot_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_read_mshot_prep() argument
440 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_read_mshot_prep()
444 if (!(req->flags & REQ_F_BUFFER_SELECT)) in io_read_mshot_prep()
447 ret = __io_prep_rw(req, sqe, ITER_DEST); in io_read_mshot_prep()
454 req->flags |= REQ_F_APOLL_MULTISHOT; in io_read_mshot_prep()
458 void io_readv_writev_cleanup(struct io_kiocb *req) in io_readv_writev_cleanup() argument
460 lockdep_assert_held(&req->ctx->uring_lock); in io_readv_writev_cleanup()
461 io_rw_recycle(req, 0); in io_readv_writev_cleanup()
464 static inline loff_t *io_kiocb_update_pos(struct io_kiocb *req) in io_kiocb_update_pos() argument
466 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_kiocb_update_pos()
471 if (!(req->file->f_mode & FMODE_STREAM)) { in io_kiocb_update_pos()
472 req->flags |= REQ_F_CUR_POS; in io_kiocb_update_pos()
473 rw->kiocb.ki_pos = req->file->f_pos; in io_kiocb_update_pos()
481 static bool io_rw_should_reissue(struct io_kiocb *req) in io_rw_should_reissue() argument
484 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_rw_should_reissue()
485 umode_t mode = file_inode(req->file)->i_mode; in io_rw_should_reissue()
486 struct io_async_rw *io = req->async_data; in io_rw_should_reissue()
487 struct io_ring_ctx *ctx = req->ctx; in io_rw_should_reissue()
491 if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() && in io_rw_should_reissue()
510 static void io_req_end_write(struct io_kiocb *req) in io_req_end_write() argument
512 if (req->flags & REQ_F_ISREG) { in io_req_end_write()
513 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_req_end_write()
523 static void io_req_io_end(struct io_kiocb *req) in io_req_io_end() argument
525 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_req_io_end()
528 io_req_end_write(req); in io_req_io_end()
529 fsnotify_modify(req->file); in io_req_io_end()
531 fsnotify_access(req->file); in io_req_io_end()
535 static void __io_complete_rw_common(struct io_kiocb *req, long res) in __io_complete_rw_common() argument
537 if (res == req->cqe.res) in __io_complete_rw_common()
539 if (res == -EAGAIN && io_rw_should_reissue(req)) { in __io_complete_rw_common()
540 req->flags |= REQ_F_REISSUE | REQ_F_BL_NO_RECYCLE; in __io_complete_rw_common()
542 req_set_fail(req); in __io_complete_rw_common()
543 req->cqe.res = res; in __io_complete_rw_common()
547 static inline int io_fixup_rw_res(struct io_kiocb *req, long res) in io_fixup_rw_res() argument
549 struct io_async_rw *io = req->async_data; in io_fixup_rw_res()
552 if (req_has_async_data(req) && io->bytes_done > 0) { in io_fixup_rw_res()
561 void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw) in io_req_rw_complete() argument
563 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_req_rw_complete()
569 io_req_set_res(req, io_fixup_rw_res(req, res), 0); in io_req_rw_complete()
572 io_req_io_end(req); in io_req_rw_complete()
574 if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) in io_req_rw_complete()
575 req->cqe.flags |= io_put_kbuf(req, req->cqe.res, 0); in io_req_rw_complete()
577 io_req_rw_cleanup(req, 0); in io_req_rw_complete()
578 io_req_task_complete(req, tw); in io_req_rw_complete()
584 struct io_kiocb *req = cmd_to_io_kiocb(rw); in io_complete_rw() local
587 __io_complete_rw_common(req, res); in io_complete_rw()
588 io_req_set_res(req, io_fixup_rw_res(req, res), 0); in io_complete_rw()
590 req->io_task_work.func = io_req_rw_complete; in io_complete_rw()
591 __io_req_task_work_add(req, IOU_F_TWQ_LAZY_WAKE); in io_complete_rw()
597 struct io_kiocb *req = cmd_to_io_kiocb(rw); in io_complete_rw_iopoll() local
600 io_req_end_write(req); in io_complete_rw_iopoll()
601 if (unlikely(res != req->cqe.res)) { in io_complete_rw_iopoll()
602 if (res == -EAGAIN && io_rw_should_reissue(req)) in io_complete_rw_iopoll()
603 req->flags |= REQ_F_REISSUE | REQ_F_BL_NO_RECYCLE; in io_complete_rw_iopoll()
605 req->cqe.res = res; in io_complete_rw_iopoll()
609 smp_store_release(&req->iopoll_completed, 1); in io_complete_rw_iopoll()
612 static inline void io_rw_done(struct io_kiocb *req, ssize_t ret) in io_rw_done() argument
614 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_rw_done()
637 if (req->ctx->flags & IORING_SETUP_IOPOLL) in io_rw_done()
643 static int kiocb_done(struct io_kiocb *req, ssize_t ret, in kiocb_done() argument
646 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in kiocb_done()
647 unsigned final_ret = io_fixup_rw_res(req, ret); in kiocb_done()
649 if (ret >= 0 && req->flags & REQ_F_CUR_POS) in kiocb_done()
650 req->file->f_pos = rw->kiocb.ki_pos; in kiocb_done()
651 if (ret >= 0 && !(req->ctx->flags & IORING_SETUP_IOPOLL)) { in kiocb_done()
652 __io_complete_rw_common(req, ret); in kiocb_done()
657 io_req_io_end(req); in kiocb_done()
658 io_req_set_res(req, final_ret, io_put_kbuf(req, ret, issue_flags)); in kiocb_done()
659 io_req_rw_cleanup(req, issue_flags); in kiocb_done()
662 io_rw_done(req, ret); in kiocb_done()
679 struct io_kiocb *req = cmd_to_io_kiocb(rw); in loop_rw_iter() local
695 if ((req->flags & REQ_F_BUF_NODE) && req->buf_node->buf->is_kbuf) in loop_rw_iter()
756 struct io_kiocb *req = wait->private; in io_async_buf_func() local
757 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_async_buf_func()
767 io_req_task_queue(req); in io_async_buf_func()
783 static bool io_rw_should_retry(struct io_kiocb *req) in io_rw_should_retry() argument
785 struct io_async_rw *io = req->async_data; in io_rw_should_retry()
787 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_rw_should_retry()
794 if (req->flags & (REQ_F_NOWAIT | REQ_F_HAS_METADATA)) in io_rw_should_retry()
805 if (io_file_can_poll(req) || in io_rw_should_retry()
806 !(req->file->f_op->fop_flags & FOP_BUFFER_RASYNC)) in io_rw_should_retry()
810 wait->wait.private = req; in io_rw_should_retry()
831 static bool need_complete_io(struct io_kiocb *req) in need_complete_io() argument
833 return req->flags & REQ_F_ISREG || in need_complete_io()
834 S_ISBLK(file_inode(req->file)->i_mode); in need_complete_io()
837 static int io_rw_init_file(struct io_kiocb *req, fmode_t mode, int rw_type) in io_rw_init_file() argument
839 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_rw_init_file()
841 struct io_ring_ctx *ctx = req->ctx; in io_rw_init_file()
842 struct file *file = req->file; in io_rw_init_file()
848 if (!(req->flags & REQ_F_FIXED_FILE)) in io_rw_init_file()
849 req->flags |= io_file_get_flags(file); in io_rw_init_file()
863 ((file->f_flags & O_NONBLOCK && !(req->flags & REQ_F_SUPPORT_NOWAIT)))) in io_rw_init_file()
864 req->flags |= REQ_F_NOWAIT; in io_rw_init_file()
871 req->iopoll_completed = 0; in io_rw_init_file()
874 req->flags &= ~REQ_F_IOPOLL_STATE; in io_rw_init_file()
875 req->iopoll_start = ktime_get_ns(); in io_rw_init_file()
882 if (req->flags & REQ_F_HAS_METADATA) { in io_rw_init_file()
883 struct io_async_rw *io = req->async_data; in io_rw_init_file()
889 if (!(req->file->f_flags & O_DIRECT)) in io_rw_init_file()
898 static int __io_read(struct io_kiocb *req, unsigned int issue_flags) in __io_read() argument
901 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in __io_read()
902 struct io_async_rw *io = req->async_data; in __io_read()
907 if (req->flags & REQ_F_IMPORT_BUFFER) { in __io_read()
908 ret = io_rw_import_reg_vec(req, io, ITER_DEST, issue_flags); in __io_read()
911 } else if (io_do_buffer_select(req)) { in __io_read()
912 ret = io_import_rw_buffer(ITER_DEST, req, io, issue_flags); in __io_read()
916 ret = io_rw_init_file(req, FMODE_READ, READ); in __io_read()
919 req->cqe.res = iov_iter_count(&io->iter); in __io_read()
923 if (unlikely(!io_file_supports_nowait(req, EPOLLIN))) in __io_read()
931 ppos = io_kiocb_update_pos(req); in __io_read()
933 ret = rw_verify_area(READ, req->file, ppos, req->cqe.res); in __io_read()
949 if (io_file_can_poll(req)) in __io_read()
952 if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) in __io_read()
955 if (req->flags & REQ_F_NOWAIT) in __io_read()
960 } else if (ret == req->cqe.res || ret <= 0 || !force_nonblock || in __io_read()
961 (req->flags & REQ_F_NOWAIT) || !need_complete_io(req) || in __io_read()
988 if (!io_rw_should_retry(req)) { in __io_read()
993 req->cqe.res = iov_iter_count(&io->iter); in __io_read()
1012 int io_read(struct io_kiocb *req, unsigned int issue_flags) in io_read() argument
1016 ret = __io_read(req, issue_flags); in io_read()
1018 return kiocb_done(req, ret, issue_flags); in io_read()
1023 int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags) in io_read_mshot() argument
1025 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_read_mshot()
1032 if (!io_file_can_poll(req)) in io_read_mshot()
1037 ret = __io_read(req, issue_flags); in io_read_mshot()
1048 if (io_kbuf_recycle(req, issue_flags)) in io_read_mshot()
1052 io_kbuf_recycle(req, issue_flags); in io_read_mshot()
1054 req_set_fail(req); in io_read_mshot()
1055 } else if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { in io_read_mshot()
1056 cflags = io_put_kbuf(req, ret, issue_flags); in io_read_mshot()
1064 cflags = io_put_kbuf(req, ret, issue_flags); in io_read_mshot()
1067 if (io_req_post_cqe(req, ret, cflags | IORING_CQE_F_MORE)) { in io_read_mshot()
1074 io_poll_multishot_retry(req); in io_read_mshot()
1084 io_req_set_res(req, ret, cflags); in io_read_mshot()
1085 io_req_rw_cleanup(req, issue_flags); in io_read_mshot()
1089 static bool io_kiocb_start_write(struct io_kiocb *req, struct kiocb *kiocb) in io_kiocb_start_write() argument
1094 if (!(req->flags & REQ_F_ISREG)) in io_kiocb_start_write()
1108 int io_write(struct io_kiocb *req, unsigned int issue_flags) in io_write() argument
1111 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_write()
1112 struct io_async_rw *io = req->async_data; in io_write()
1117 if (req->flags & REQ_F_IMPORT_BUFFER) { in io_write()
1118 ret = io_rw_import_reg_vec(req, io, ITER_SOURCE, issue_flags); in io_write()
1123 ret = io_rw_init_file(req, FMODE_WRITE, WRITE); in io_write()
1126 req->cqe.res = iov_iter_count(&io->iter); in io_write()
1130 if (unlikely(!io_file_supports_nowait(req, EPOLLOUT))) in io_write()
1135 !(req->file->f_op->fop_flags & FOP_BUFFER_WASYNC) && in io_write()
1136 (req->flags & REQ_F_ISREG)) in io_write()
1145 ppos = io_kiocb_update_pos(req); in io_write()
1147 ret = rw_verify_area(WRITE, req->file, ppos, req->cqe.res); in io_write()
1151 if (unlikely(!io_kiocb_start_write(req, kiocb))) in io_write()
1155 if (likely(req->file->f_op->write_iter)) in io_write()
1156 ret2 = req->file->f_op->write_iter(kiocb, &io->iter); in io_write()
1157 else if (req->file->f_op->write) in io_write()
1169 if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT)) in io_write()
1173 if (ret2 == -EAGAIN && (req->ctx->flags & IORING_SETUP_IOPOLL)) in io_write()
1176 if (ret2 != req->cqe.res && ret2 >= 0 && need_complete_io(req)) { in io_write()
1177 trace_io_uring_short_write(req->ctx, kiocb->ki_pos - ret2, in io_write()
1178 req->cqe.res, ret2); in io_write()
1189 io_req_end_write(req); in io_write()
1193 return kiocb_done(req, ret2, issue_flags); in io_write()
1199 io_req_end_write(req); in io_write()
1204 int io_read_fixed(struct io_kiocb *req, unsigned int issue_flags) in io_read_fixed() argument
1208 ret = io_init_rw_fixed(req, issue_flags, ITER_DEST); in io_read_fixed()
1212 return io_read(req, issue_flags); in io_read_fixed()
1215 int io_write_fixed(struct io_kiocb *req, unsigned int issue_flags) in io_write_fixed() argument
1219 ret = io_init_rw_fixed(req, issue_flags, ITER_SOURCE); in io_write_fixed()
1223 return io_write(req, issue_flags); in io_write_fixed()
1226 void io_rw_fail(struct io_kiocb *req) in io_rw_fail() argument
1230 res = io_fixup_rw_res(req, req->cqe.res); in io_rw_fail()
1231 io_req_set_res(req, res, req->cqe.flags); in io_rw_fail()
1234 static int io_uring_classic_poll(struct io_kiocb *req, struct io_comp_batch *iob, in io_uring_classic_poll() argument
1237 struct file *file = req->file; in io_uring_classic_poll()
1239 if (req->opcode == IORING_OP_URING_CMD) { in io_uring_classic_poll()
1242 ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); in io_uring_classic_poll()
1245 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_uring_classic_poll()
1251 static u64 io_hybrid_iopoll_delay(struct io_ring_ctx *ctx, struct io_kiocb *req) in io_hybrid_iopoll_delay() argument
1258 if (req->flags & REQ_F_IOPOLL_STATE) in io_hybrid_iopoll_delay()
1268 req->flags |= REQ_F_IOPOLL_STATE; in io_hybrid_iopoll_delay()
1285 static int io_uring_hybrid_poll(struct io_kiocb *req, in io_uring_hybrid_poll() argument
1288 struct io_ring_ctx *ctx = req->ctx; in io_uring_hybrid_poll()
1292 sleep_time = io_hybrid_iopoll_delay(ctx, req); in io_uring_hybrid_poll()
1293 ret = io_uring_classic_poll(req, iob, poll_flags); in io_uring_hybrid_poll()
1294 runtime = ktime_get_ns() - req->iopoll_start - sleep_time; in io_uring_hybrid_poll()
1321 struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list); in io_do_iopoll() local
1329 if (READ_ONCE(req->iopoll_completed)) in io_do_iopoll()
1333 ret = io_uring_hybrid_poll(req, &iob, poll_flags); in io_do_iopoll()
1335 ret = io_uring_classic_poll(req, &iob, poll_flags); in io_do_iopoll()
1344 READ_ONCE(req->iopoll_completed)) in io_do_iopoll()
1355 struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list); in io_do_iopoll() local
1358 if (!smp_load_acquire(&req->iopoll_completed)) in io_do_iopoll()
1361 req->cqe.flags = io_put_kbuf(req, req->cqe.res, 0); in io_do_iopoll()
1362 if (req->opcode != IORING_OP_URING_CMD) in io_do_iopoll()
1363 io_req_rw_cleanup(req, 0); in io_do_iopoll()