Lines Matching refs:vq
98 struct vhost_virtqueue *vq; member
109 struct vhost_virtqueue vq; member
227 static void vhost_net_enable_zcopy(int vq) in vhost_net_enable_zcopy() argument
229 vhost_net_zcopy_mask |= 0x1 << vq; in vhost_net_enable_zcopy()
233 vhost_net_ubuf_alloc(struct vhost_virtqueue *vq, bool zcopy) in vhost_net_ubuf_alloc() argument
244 ubufs->vq = vq; in vhost_net_ubuf_alloc()
358 struct vhost_virtqueue *vq) in vhost_zerocopy_signal_used() argument
361 container_of(vq, struct vhost_net_virtqueue, vq); in vhost_zerocopy_signal_used()
366 if (vq->heads[i].len == VHOST_DMA_FAILED_LEN) in vhost_zerocopy_signal_used()
368 if (VHOST_DMA_IS_DONE(vq->heads[i].len)) { in vhost_zerocopy_signal_used()
369 vq->heads[i].len = VHOST_DMA_CLEAR_LEN; in vhost_zerocopy_signal_used()
376 vhost_add_used_and_signal_n(vq->dev, vq, in vhost_zerocopy_signal_used()
377 &vq->heads[nvq->done_idx], add); in vhost_zerocopy_signal_used()
388 struct vhost_virtqueue *vq = ubufs->vq; in vhost_zerocopy_complete() local
394 vq->heads[ubuf->desc].len = success ? in vhost_zerocopy_complete()
406 vhost_poll_queue(&vq->poll); in vhost_zerocopy_complete()
427 struct vhost_virtqueue *vq) in vhost_net_disable_vq() argument
430 container_of(vq, struct vhost_net_virtqueue, vq); in vhost_net_disable_vq()
432 if (!vhost_vq_get_backend(vq)) in vhost_net_disable_vq()
438 struct vhost_virtqueue *vq) in vhost_net_enable_vq() argument
441 container_of(vq, struct vhost_net_virtqueue, vq); in vhost_net_enable_vq()
445 sock = vhost_vq_get_backend(vq); in vhost_net_enable_vq()
454 struct vhost_virtqueue *vq = &nvq->vq; in vhost_net_signal_used() local
455 struct vhost_dev *dev = vq->dev; in vhost_net_signal_used()
460 vhost_add_used_and_signal_n(dev, vq, vq->heads, nvq->done_idx); in vhost_net_signal_used()
483 vq_err(&nvq->vq, "Fail to batch sending packets\n"); in vhost_tx_batch()
513 struct vhost_virtqueue *vq) in vhost_net_busy_poll_try_queue() argument
515 if (!vhost_vq_avail_empty(&net->dev, vq)) { in vhost_net_busy_poll_try_queue()
516 vhost_poll_queue(&vq->poll); in vhost_net_busy_poll_try_queue()
517 } else if (unlikely(vhost_enable_notify(&net->dev, vq))) { in vhost_net_busy_poll_try_queue()
518 vhost_disable_notify(&net->dev, vq); in vhost_net_busy_poll_try_queue()
519 vhost_poll_queue(&vq->poll); in vhost_net_busy_poll_try_queue()
532 struct vhost_virtqueue *vq = poll_rx ? tvq : rvq; in vhost_net_busy_poll() local
538 if (!mutex_trylock(&vq->mutex)) in vhost_net_busy_poll()
541 vhost_disable_notify(&net->dev, vq); in vhost_net_busy_poll()
551 if (vhost_vq_has_work(vq)) { in vhost_net_busy_poll()
567 vhost_net_busy_poll_try_queue(net, vq); in vhost_net_busy_poll()
571 mutex_unlock(&vq->mutex); in vhost_net_busy_poll()
580 struct vhost_virtqueue *rvq = &rnvq->vq; in vhost_net_tx_get_vq_desc()
581 struct vhost_virtqueue *tvq = &tnvq->vq; in vhost_net_tx_get_vq_desc()
605 struct vhost_virtqueue *vq = &nvq->vq; in vhost_exceeds_maxpend() local
608 min_t(unsigned int, VHOST_MAX_PEND, vq->num >> 2); in vhost_exceeds_maxpend()
611 static size_t init_iov_iter(struct vhost_virtqueue *vq, struct iov_iter *iter, in init_iov_iter() argument
615 size_t len = iov_length(vq->iov, out); in init_iov_iter()
617 iov_iter_init(iter, ITER_SOURCE, vq->iov, out, len); in init_iov_iter()
629 struct vhost_virtqueue *vq = &nvq->vq; in get_tx_bufs() local
634 if (ret < 0 || ret == vq->num) in get_tx_bufs()
638 vq_err(vq, "Unexpected descriptor format for TX: out %d, int %d\n", in get_tx_bufs()
644 *len = init_iov_iter(vq, &msg->msg_iter, nvq->vhost_hlen, *out); in get_tx_bufs()
646 vq_err(vq, "Unexpected header len for TX: %zd expected %zd\n", in get_tx_bufs()
654 static bool tx_can_batch(struct vhost_virtqueue *vq, size_t total_len) in tx_can_batch() argument
657 !vhost_vq_avail_empty(vq->dev, vq); in tx_can_batch()
665 struct vhost_virtqueue *vq = &nvq->vq; in vhost_net_build_xdp() local
666 struct vhost_net *net = container_of(vq->dev, struct vhost_net, in vhost_net_build_xdp()
668 struct socket *sock = vhost_vq_get_backend(vq); in vhost_net_build_xdp()
708 vhost16_to_cpu(vq, gso->csum_start) + in vhost_net_build_xdp()
709 vhost16_to_cpu(vq, gso->csum_offset) + 2 > in vhost_net_build_xdp()
710 vhost16_to_cpu(vq, gso->hdr_len)) { in vhost_net_build_xdp()
711 gso->hdr_len = cpu_to_vhost16(vq, in vhost_net_build_xdp()
712 vhost16_to_cpu(vq, gso->csum_start) + in vhost_net_build_xdp()
713 vhost16_to_cpu(vq, gso->csum_offset) + 2); in vhost_net_build_xdp()
715 if (vhost16_to_cpu(vq, gso->hdr_len) > len) { in vhost_net_build_xdp()
744 struct vhost_virtqueue *vq = &nvq->vq; in handle_tx_copy() local
771 if (head == vq->num) { in handle_tx_copy()
773 vhost_poll_queue(&vq->poll); in handle_tx_copy()
775 vq))) { in handle_tx_copy()
776 vhost_disable_notify(&net->dev, vq); in handle_tx_copy()
793 vhost_discard_vq_desc(vq, 1); in handle_tx_copy()
794 vhost_net_enable_vq(net, vq); in handle_tx_copy()
805 if (tx_can_batch(vq, total_len)) in handle_tx_copy()
814 vhost_discard_vq_desc(vq, 1); in handle_tx_copy()
815 vhost_net_enable_vq(net, vq); in handle_tx_copy()
823 vq->heads[nvq->done_idx].id = cpu_to_vhost32(vq, head); in handle_tx_copy()
824 vq->heads[nvq->done_idx].len = 0; in handle_tx_copy()
826 } while (likely(!vhost_exceeds_weight(vq, ++sent_pkts, total_len))); in handle_tx_copy()
834 struct vhost_virtqueue *vq = &nvq->vq; in handle_tx_zerocopy() local
856 vhost_zerocopy_signal_used(net, vq); in handle_tx_zerocopy()
865 if (head == vq->num) { in handle_tx_zerocopy()
867 vhost_poll_queue(&vq->poll); in handle_tx_zerocopy()
868 } else if (unlikely(vhost_enable_notify(&net->dev, vq))) { in handle_tx_zerocopy()
869 vhost_disable_notify(&net->dev, vq); in handle_tx_zerocopy()
882 vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head); in handle_tx_zerocopy()
883 vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS; in handle_tx_zerocopy()
901 if (tx_can_batch(vq, total_len) && in handle_tx_zerocopy()
913 if (vq->heads[ubuf->desc].len == VHOST_DMA_IN_PROGRESS) in handle_tx_zerocopy()
919 vq->heads[ubuf->desc].len = VHOST_DMA_DONE_LEN; in handle_tx_zerocopy()
922 vhost_discard_vq_desc(vq, 1); in handle_tx_zerocopy()
923 vhost_net_enable_vq(net, vq); in handle_tx_zerocopy()
931 vhost_add_used_and_signal(&net->dev, vq, head, 0); in handle_tx_zerocopy()
933 vhost_zerocopy_signal_used(net, vq); in handle_tx_zerocopy()
935 } while (likely(!vhost_exceeds_weight(vq, ++sent_pkts, total_len))); in handle_tx_zerocopy()
943 struct vhost_virtqueue *vq = &nvq->vq; in handle_tx() local
946 mutex_lock_nested(&vq->mutex, VHOST_NET_VQ_TX); in handle_tx()
947 sock = vhost_vq_get_backend(vq); in handle_tx()
951 if (!vq_meta_prefetch(vq)) in handle_tx()
954 vhost_disable_notify(&net->dev, vq); in handle_tx()
955 vhost_net_disable_vq(net, vq); in handle_tx()
963 mutex_unlock(&vq->mutex); in handle_tx()
992 struct vhost_virtqueue *rvq = &rnvq->vq; in vhost_net_rx_peek_head_len()
993 struct vhost_virtqueue *tvq = &tnvq->vq; in vhost_net_rx_peek_head_len()
1018 static int get_rx_bufs(struct vhost_virtqueue *vq, in get_rx_bufs() argument
1041 r = vhost_get_vq_desc(vq, vq->iov + seg, in get_rx_bufs()
1042 ARRAY_SIZE(vq->iov) - seg, &out, in get_rx_bufs()
1048 if (d == vq->num) { in get_rx_bufs()
1053 vq_err(vq, "unexpected descriptor format for RX: " in get_rx_bufs()
1062 heads[headcount].id = cpu_to_vhost32(vq, d); in get_rx_bufs()
1063 len = iov_length(vq->iov + seg, in); in get_rx_bufs()
1064 heads[headcount].len = cpu_to_vhost32(vq, len); in get_rx_bufs()
1069 heads[headcount - 1].len = cpu_to_vhost32(vq, len + datalen); in get_rx_bufs()
1081 vhost_discard_vq_desc(vq, headcount); in get_rx_bufs()
1090 struct vhost_virtqueue *vq = &nvq->vq; in handle_rx() local
1116 mutex_lock_nested(&vq->mutex, VHOST_NET_VQ_RX); in handle_rx()
1117 sock = vhost_vq_get_backend(vq); in handle_rx()
1121 if (!vq_meta_prefetch(vq)) in handle_rx()
1124 vhost_disable_notify(&net->dev, vq); in handle_rx()
1125 vhost_net_disable_vq(net, vq); in handle_rx()
1130 vq_log = unlikely(vhost_has_feature(vq, VHOST_F_LOG_ALL)) ? in handle_rx()
1131 vq->log : NULL; in handle_rx()
1132 mergeable = vhost_has_feature(vq, VIRTIO_NET_F_MRG_RXBUF); in handle_rx()
1134 vhost_has_feature(vq, VIRTIO_F_VERSION_1); in handle_rx()
1143 headcount = get_rx_bufs(vq, vq->heads + nvq->done_idx, in handle_rx()
1152 vhost_poll_queue(&vq->poll); in handle_rx()
1153 } else if (unlikely(vhost_enable_notify(&net->dev, vq))) { in handle_rx()
1156 vhost_disable_notify(&net->dev, vq); in handle_rx()
1168 iov_iter_init(&msg.msg_iter, ITER_DEST, vq->iov, 1, 1); in handle_rx()
1175 iov_iter_init(&msg.msg_iter, ITER_DEST, vq->iov, in, vhost_len); in handle_rx()
1191 vhost_discard_vq_desc(vq, headcount); in handle_rx()
1198 vq_err(vq, "Unable to write vnet_hdr " in handle_rx()
1199 "at addr %p\n", vq->iov->iov_base); in handle_rx()
1210 num_buffers = cpu_to_vhost16(vq, headcount); in handle_rx()
1214 vq_err(vq, "Failed num_buffers write"); in handle_rx()
1215 vhost_discard_vq_desc(vq, headcount); in handle_rx()
1222 vhost_log_write(vq, vq_log, log, vhost_len, in handle_rx()
1223 vq->iov, in); in handle_rx()
1225 } while (likely(!vhost_exceeds_weight(vq, ++recv_pkts, total_len))); in handle_rx()
1228 vhost_poll_queue(&vq->poll); in handle_rx()
1230 vhost_net_enable_vq(net, vq); in handle_rx()
1233 mutex_unlock(&vq->mutex); in handle_rx()
1238 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, in handle_tx_kick() local
1240 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); in handle_tx_kick()
1247 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, in handle_rx_kick() local
1249 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); in handle_rx_kick()
1305 vqs[VHOST_NET_VQ_TX] = &n->vqs[VHOST_NET_VQ_TX].vq; in vhost_net_open()
1306 vqs[VHOST_NET_VQ_RX] = &n->vqs[VHOST_NET_VQ_RX].vq; in vhost_net_open()
1307 n->vqs[VHOST_NET_VQ_TX].vq.handle_kick = handle_tx_kick; in vhost_net_open()
1308 n->vqs[VHOST_NET_VQ_RX].vq.handle_kick = handle_rx_kick; in vhost_net_open()
1337 struct vhost_virtqueue *vq) in vhost_net_stop_vq() argument
1341 container_of(vq, struct vhost_net_virtqueue, vq); in vhost_net_stop_vq()
1343 mutex_lock(&vq->mutex); in vhost_net_stop_vq()
1344 sock = vhost_vq_get_backend(vq); in vhost_net_stop_vq()
1345 vhost_net_disable_vq(n, vq); in vhost_net_stop_vq()
1346 vhost_vq_set_backend(vq, NULL); in vhost_net_stop_vq()
1349 mutex_unlock(&vq->mutex); in vhost_net_stop_vq()
1356 *tx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_TX].vq); in vhost_net_stop()
1357 *rx_sock = vhost_net_stop_vq(n, &n->vqs[VHOST_NET_VQ_RX].vq); in vhost_net_stop()
1364 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); in vhost_net_flush()
1366 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); in vhost_net_flush()
1369 mutex_lock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); in vhost_net_flush()
1372 mutex_unlock(&n->vqs[VHOST_NET_VQ_TX].vq.mutex); in vhost_net_flush()
1477 struct vhost_virtqueue *vq; in vhost_net_set_backend() local
1491 vq = &n->vqs[index].vq; in vhost_net_set_backend()
1493 mutex_lock(&vq->mutex); in vhost_net_set_backend()
1499 if (!vhost_vq_access_ok(vq)) { in vhost_net_set_backend()
1510 oldsock = vhost_vq_get_backend(vq); in vhost_net_set_backend()
1512 ubufs = vhost_net_ubuf_alloc(vq, in vhost_net_set_backend()
1519 vhost_net_disable_vq(n, vq); in vhost_net_set_backend()
1520 vhost_vq_set_backend(vq, sock); in vhost_net_set_backend()
1522 r = vhost_vq_init_access(vq); in vhost_net_set_backend()
1525 r = vhost_net_enable_vq(n, vq); in vhost_net_set_backend()
1543 mutex_unlock(&vq->mutex); in vhost_net_set_backend()
1547 mutex_lock(&vq->mutex); in vhost_net_set_backend()
1548 vhost_zerocopy_signal_used(n, vq); in vhost_net_set_backend()
1549 mutex_unlock(&vq->mutex); in vhost_net_set_backend()
1561 vhost_vq_set_backend(vq, oldsock); in vhost_net_set_backend()
1562 vhost_net_enable_vq(n, vq); in vhost_net_set_backend()
1569 mutex_unlock(&vq->mutex); in vhost_net_set_backend()
1634 mutex_lock(&n->vqs[i].vq.mutex); in vhost_net_set_features()
1635 n->vqs[i].vq.acked_features = features; in vhost_net_set_features()
1638 mutex_unlock(&n->vqs[i].vq.mutex); in vhost_net_set_features()