Lines Matching refs:sr
187 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); in io_mshot_prep_retry() local
190 sr->done_io = 0; in io_mshot_prep_retry()
191 sr->retry = false; in io_mshot_prep_retry()
192 sr->len = 0; /* get from the provided buffer */ in io_mshot_prep_retry()
193 req->buf_index = sr->buf_group; in io_mshot_prep_retry()
228 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); in io_compat_msg_copy_hdr() local
232 if (copy_from_user(msg, sr->umsg_compat, sizeof(*msg))) in io_compat_msg_copy_hdr()
242 sr->len = 0; in io_compat_msg_copy_hdr()
250 sr->len = tmp_iov.iov_len; in io_compat_msg_copy_hdr()
278 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); in io_msg_copy_hdr() local
279 struct user_msghdr __user *umsg = sr->umsg; in io_msg_copy_hdr()
312 sr->len = 0; in io_msg_copy_hdr()
321 sr->len = tmp_iov.iov_len; in io_msg_copy_hdr()
336 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); in io_send_setup() local
342 sr->buf = u64_to_user_ptr(READ_ONCE(sqe->addr)); in io_send_setup()
362 if (sr->flags & IORING_RECVSEND_FIXED_BUF) in io_send_setup()
365 ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len, in io_send_setup()
375 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); in io_sendmsg_setup() local
380 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); in io_sendmsg_setup()
385 sr->msg_control = kmsg->msg.msg_control_user; in io_sendmsg_setup()
387 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { in io_sendmsg_setup()
401 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); in io_sendmsg_prep() local
403 sr->done_io = 0; in io_sendmsg_prep()
404 sr->retry = false; in io_sendmsg_prep()
405 sr->len = READ_ONCE(sqe->len); in io_sendmsg_prep()
406 sr->flags = READ_ONCE(sqe->ioprio); in io_sendmsg_prep()
407 if (sr->flags & ~SENDMSG_FLAGS) in io_sendmsg_prep()
409 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; in io_sendmsg_prep()
410 if (sr->msg_flags & MSG_DONTWAIT) in io_sendmsg_prep()
412 if (sr->flags & IORING_RECVSEND_BUNDLE) { in io_sendmsg_prep()
417 sr->msg_flags |= MSG_WAITALL; in io_sendmsg_prep()
418 sr->buf_group = req->buf_index; in io_sendmsg_prep()
424 sr->msg_flags |= MSG_CMSG_COMPAT; in io_sendmsg_prep()
484 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); in io_send_finish() local
488 if (!(sr->flags & IORING_RECVSEND_BUNDLE)) { in io_send_finish()
516 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); in io_sendmsg() local
528 (sr->flags & IORING_RECVSEND_POLL_FIRST)) in io_sendmsg()
531 flags = sr->msg_flags; in io_sendmsg()
537 kmsg->msg.msg_control_user = sr->msg_control; in io_sendmsg()
547 sr->done_io += ret; in io_sendmsg()
557 ret += sr->done_io; in io_sendmsg()
558 else if (sr->done_io) in io_sendmsg()
559 ret = sr->done_io; in io_sendmsg()
567 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); in io_send_select_buffer() local
572 .max_len = min_not_zero(sr->len, INT_MAX), in io_send_select_buffer()
582 if (!(sr->flags & IORING_RECVSEND_BUNDLE)) in io_send_select_buffer()
596 sr->len = arg.out_len; in io_send_select_buffer()
599 sr->buf = arg.iovs[0].iov_base; in io_send_select_buffer()
600 ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len, in io_send_select_buffer()
614 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); in io_send() local
626 (sr->flags & IORING_RECVSEND_POLL_FIRST)) in io_send()
629 flags = sr->msg_flags; in io_send()
645 if (flags & MSG_WAITALL || sr->flags & IORING_RECVSEND_BUNDLE) in io_send()
656 sr->len -= ret; in io_send()
657 sr->buf += ret; in io_send()
658 sr->done_io += ret; in io_send()
667 ret += sr->done_io; in io_send()
668 else if (sr->done_io) in io_send()
669 ret = sr->done_io; in io_send()
724 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); in io_recvmsg_prep_setup() local
743 ret = import_ubuf(ITER_DEST, sr->buf, sr->len, in io_recvmsg_prep_setup()
759 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); in io_recvmsg_prep() local
761 sr->done_io = 0; in io_recvmsg_prep()
762 sr->retry = false; in io_recvmsg_prep()
767 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); in io_recvmsg_prep()
768 sr->len = READ_ONCE(sqe->len); in io_recvmsg_prep()
769 sr->flags = READ_ONCE(sqe->ioprio); in io_recvmsg_prep()
770 if (sr->flags & ~RECVMSG_FLAGS) in io_recvmsg_prep()
772 sr->msg_flags = READ_ONCE(sqe->msg_flags); in io_recvmsg_prep()
773 if (sr->msg_flags & MSG_DONTWAIT) in io_recvmsg_prep()
775 if (sr->msg_flags & MSG_ERRQUEUE) in io_recvmsg_prep()
786 sr->buf_group = req->buf_index; in io_recvmsg_prep()
789 if (sr->flags & IORING_RECV_MULTISHOT) { in io_recvmsg_prep()
792 if (sr->msg_flags & MSG_WAITALL) in io_recvmsg_prep()
794 if (req->opcode == IORING_OP_RECV && sr->len) in io_recvmsg_prep()
798 if (sr->flags & IORING_RECVSEND_BUNDLE) { in io_recvmsg_prep()
804 sr->msg_flags |= MSG_CMSG_COMPAT; in io_recvmsg_prep()
806 sr->nr_multishot_loops = 0; in io_recvmsg_prep()
823 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); in io_recv_finish() local
829 if (sr->flags & IORING_RECVSEND_BUNDLE) { in io_recv_finish()
830 size_t this_ret = *ret - sr->done_io; in io_recv_finish()
834 if (sr->retry) in io_recv_finish()
843 if (!sr->retry && kmsg->msg.msg_inq > 0 && this_ret > 0 && in io_recv_finish()
846 sr->len = kmsg->msg.msg_inq; in io_recv_finish()
847 sr->done_io += this_ret; in io_recv_finish()
848 sr->retry = true; in io_recv_finish()
865 if (sr->nr_multishot_loops++ < MULTISHOT_MAX_RETRY) in io_recv_finish()
868 sr->nr_multishot_loops = 0; in io_recv_finish()
884 struct io_sr_msg *sr, void __user **buf, in io_recvmsg_prep_multishot() argument
902 sr->buf = *buf; /* stash for later copy */ in io_recvmsg_prep_multishot()
969 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); in io_recvmsg() local
982 (sr->flags & IORING_RECVSEND_POLL_FIRST)) in io_recvmsg()
985 flags = sr->msg_flags; in io_recvmsg()
992 size_t len = sr->len; in io_recvmsg()
999 ret = io_recvmsg_prep_multishot(kmsg, sr, &buf, &len); in io_recvmsg()
1012 ret = io_recvmsg_multishot(sock, sr, kmsg, flags, in io_recvmsg()
1019 ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, in io_recvmsg()
1031 sr->done_io += ret; in io_recvmsg()
1043 ret += sr->done_io; in io_recvmsg()
1044 else if (sr->done_io) in io_recvmsg()
1045 ret = sr->done_io; in io_recvmsg()
1058 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); in io_recv_buf_select() local
1067 sr->flags & IORING_RECVSEND_BUNDLE) { in io_recv_buf_select()
1081 arg.max_len = min_not_zero(sr->len, kmsg->msg.msg_inq); in io_recv_buf_select()
1089 sr->buf = arg.iovs[0].iov_base; in io_recv_buf_select()
1090 sr->len = arg.iovs[0].iov_len; in io_recv_buf_select()
1103 *len = sr->len; in io_recv_buf_select()
1107 sr->buf = buf; in io_recv_buf_select()
1108 sr->len = *len; in io_recv_buf_select()
1110 ret = import_ubuf(ITER_DEST, sr->buf, sr->len, in io_recv_buf_select()
1121 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); in io_recv() local
1127 size_t len = sr->len; in io_recv()
1131 (sr->flags & IORING_RECVSEND_POLL_FIRST)) in io_recv()
1138 flags = sr->msg_flags; in io_recv()
1149 sr->buf = NULL; in io_recv()
1167 sr->len -= ret; in io_recv()
1168 sr->buf += ret; in io_recv()
1169 sr->done_io += ret; in io_recv()
1183 ret += sr->done_io; in io_recv()
1184 else if (sr->done_io) in io_recv()
1185 ret = sr->done_io; in io_recv()
1401 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); in io_send_zc_import() local
1404 WARN_ON_ONCE(!(sr->flags & IORING_RECVSEND_FIXED_BUF)); in io_send_zc_import()
1406 sr->notif->buf_index = req->buf_index; in io_send_zc_import()
1407 return io_import_reg_buf(sr->notif, &kmsg->msg.msg_iter, in io_send_zc_import()
1408 (u64)(uintptr_t)sr->buf, sr->len, in io_send_zc_import()
1484 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); in io_sendmsg_zc() local
1508 (sr->flags & IORING_RECVSEND_POLL_FIRST)) in io_sendmsg_zc()
1511 flags = sr->msg_flags; in io_sendmsg_zc()
1517 kmsg->msg.msg_control_user = sr->msg_control; in io_sendmsg_zc()
1518 kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg; in io_sendmsg_zc()
1526 sr->done_io += ret; in io_sendmsg_zc()
1536 ret += sr->done_io; in io_sendmsg_zc()
1537 else if (sr->done_io) in io_sendmsg_zc()
1538 ret = sr->done_io; in io_sendmsg_zc()
1545 io_notif_flush(sr->notif); in io_sendmsg_zc()
1546 sr->notif = NULL; in io_sendmsg_zc()
1555 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); in io_sendrecv_fail() local
1557 if (sr->done_io) in io_sendrecv_fail()
1558 req->cqe.res = sr->done_io; in io_sendrecv_fail()