175 static int vhost_net_buf_produce(struct vhost_net_virtqueue *nvq)  in vhost_net_buf_produce()  argument
177 	struct vhost_net_buf *rxq = &nvq->rxq;  in vhost_net_buf_produce()
180 	rxq->tail = ptr_ring_consume_batched(nvq->rx_ring, rxq->queue,  in vhost_net_buf_produce()
185 static void vhost_net_buf_unproduce(struct vhost_net_virtqueue *nvq)  in vhost_net_buf_unproduce()  argument
187 	struct vhost_net_buf *rxq = &nvq->rxq;  in vhost_net_buf_unproduce()
189 	if (nvq->rx_ring && !vhost_net_buf_is_empty(rxq)) {  in vhost_net_buf_unproduce()
190 		ptr_ring_unconsume(nvq->rx_ring, rxq->queue + rxq->head,  in vhost_net_buf_unproduce()
208 static int vhost_net_buf_peek(struct vhost_net_virtqueue *nvq)  in vhost_net_buf_peek()  argument
210 	struct vhost_net_buf *rxq = &nvq->rxq;  in vhost_net_buf_peek()
215 	if (!vhost_net_buf_produce(nvq))  in vhost_net_buf_peek()
360 	struct vhost_net_virtqueue *nvq =  in vhost_zerocopy_signal_used()  local
365 	for (i = nvq->done_idx; i != nvq->upend_idx; i = (i + 1) % UIO_MAXIOV) {  in vhost_zerocopy_signal_used()
375 		add = min(UIO_MAXIOV - nvq->done_idx, j);  in vhost_zerocopy_signal_used()
377 					    &vq->heads[nvq->done_idx], add);  in vhost_zerocopy_signal_used()
378 		nvq->done_idx = (nvq->done_idx + add) % UIO_MAXIOV;  in vhost_zerocopy_signal_used()
429 	struct vhost_net_virtqueue *nvq =  in vhost_net_disable_vq()  local
431 	struct vhost_poll *poll = n->poll + (nvq - n->vqs);  in vhost_net_disable_vq()
440 	struct vhost_net_virtqueue *nvq =  in vhost_net_enable_vq()  local
442 	struct vhost_poll *poll = n->poll + (nvq - n->vqs);  in vhost_net_enable_vq()
452 static void vhost_net_signal_used(struct vhost_net_virtqueue *nvq)  in vhost_net_signal_used()  argument
454 	struct vhost_virtqueue *vq = &nvq->vq;  in vhost_net_signal_used()
457 	if (!nvq->done_idx)  in vhost_net_signal_used()
460 	vhost_add_used_and_signal_n(dev, vq, vq->heads, nvq->done_idx);  in vhost_net_signal_used()
461 	nvq->done_idx = 0;  in vhost_net_signal_used()
465 			   struct vhost_net_virtqueue *nvq,  in vhost_tx_batch()  argument
471 		.num = nvq->batched_xdp,  in vhost_tx_batch()
472 		.ptr = nvq->xdp,  in vhost_tx_batch()
476 	if (nvq->batched_xdp == 0)  in vhost_tx_batch()
483 		vq_err(&nvq->vq, "Fail to batch sending packets\n");  in vhost_tx_batch()
489 		for (i = 0; i < nvq->batched_xdp; ++i)  in vhost_tx_batch()
490 			put_page(virt_to_head_page(nvq->xdp[i].data));  in vhost_tx_batch()
491 		nvq->batched_xdp = 0;  in vhost_tx_batch()
492 		nvq->done_idx = 0;  in vhost_tx_batch()
497 	vhost_net_signal_used(nvq);  in vhost_tx_batch()
498 	nvq->batched_xdp = 0;  in vhost_tx_batch()
604 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in vhost_exceeds_maxpend()  local
605 	struct vhost_virtqueue *vq = &nvq->vq;  in vhost_exceeds_maxpend()
607 	return (nvq->upend_idx + UIO_MAXIOV - nvq->done_idx) % UIO_MAXIOV >  in vhost_exceeds_maxpend()
624 		       struct vhost_net_virtqueue *nvq,  in get_tx_bufs()  argument
629 	struct vhost_virtqueue *vq = &nvq->vq;  in get_tx_bufs()
632 	ret = vhost_net_tx_get_vq_desc(net, nvq, out, in, msg, busyloop_intr);  in get_tx_bufs()
644 	*len = init_iov_iter(vq, &msg->msg_iter, nvq->vhost_hlen, *out);  in get_tx_bufs()
647 			*len, nvq->vhost_hlen);  in get_tx_bufs()
662 static int vhost_net_build_xdp(struct vhost_net_virtqueue *nvq,  in vhost_net_build_xdp()  argument
665 	struct vhost_virtqueue *vq = &nvq->vq;  in vhost_net_build_xdp()
670 	struct xdp_buff *xdp = &nvq->xdp[nvq->batched_xdp];  in vhost_net_build_xdp()
675 	int pad = SKB_DATA_ALIGN(VHOST_NET_RX_PAD + headroom + nvq->sock_hlen);  in vhost_net_build_xdp()
676 	int sock_hlen = nvq->sock_hlen;  in vhost_net_build_xdp()
681 	if (unlikely(len < nvq->sock_hlen))  in vhost_net_build_xdp()
732 	++nvq->batched_xdp;  in vhost_net_build_xdp()
743 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in handle_tx_copy()  local
744 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_tx_copy()
762 		if (nvq->done_idx == VHOST_NET_BATCH)  in handle_tx_copy()
763 			vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
765 		head = get_tx_bufs(net, nvq, &msg, &out, &in, &len,  in handle_tx_copy()
788 			err = vhost_net_build_xdp(nvq, &msg.msg_iter);  in handle_tx_copy()
792 				vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
802 			vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
823 		vq->heads[nvq->done_idx].id = cpu_to_vhost32(vq, head);  in handle_tx_copy()
824 		vq->heads[nvq->done_idx].len = 0;  in handle_tx_copy()
825 		++nvq->done_idx;  in handle_tx_copy()
828 	vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
833 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in handle_tx_zerocopy()  local
834 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_tx_zerocopy()
859 		head = get_tx_bufs(net, nvq, &msg, &out, &in, &len,  in handle_tx_zerocopy()
881 			ubuf = nvq->ubuf_info + nvq->upend_idx;  in handle_tx_zerocopy()
882 			vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head);  in handle_tx_zerocopy()
883 			vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;  in handle_tx_zerocopy()
884 			ubuf->ctx = nvq->ubufs;  in handle_tx_zerocopy()
885 			ubuf->desc = nvq->upend_idx;  in handle_tx_zerocopy()
893 			ubufs = nvq->ubufs;  in handle_tx_zerocopy()
895 			nvq->upend_idx = (nvq->upend_idx + 1) % UIO_MAXIOV;  in handle_tx_zerocopy()
916 					nvq->upend_idx = ((unsigned)nvq->upend_idx - 1)  in handle_tx_zerocopy()
942 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in handle_tx()  local
943 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_tx()
1089 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_RX];  in handle_rx()  local
1090 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_rx()
1127 	vhost_hlen = nvq->vhost_hlen;  in handle_rx()
1128 	sock_hlen = nvq->sock_hlen;  in handle_rx()
1143 		headcount = get_rx_bufs(vq, vq->heads + nvq->done_idx,  in handle_rx()
1164 		if (nvq->rx_ring)  in handle_rx()
1165 			msg.msg_control = vhost_net_buf_consume(&nvq->rxq);  in handle_rx()
1218 		nvq->done_idx += headcount;  in handle_rx()
1219 		if (nvq->done_idx > VHOST_NET_BATCH)  in handle_rx()
1220 			vhost_net_signal_used(nvq);  in handle_rx()
1232 	vhost_net_signal_used(nvq);  in handle_rx()
1340 	struct vhost_net_virtqueue *nvq =  in vhost_net_stop_vq()  local
1347 	vhost_net_buf_unproduce(nvq);  in vhost_net_stop_vq()
1348 	nvq->rx_ring = NULL;  in vhost_net_stop_vq()
1478 	struct vhost_net_virtqueue *nvq;  in vhost_net_set_backend()  local
1492 	nvq = &n->vqs[index];  in vhost_net_set_backend()
1521 		vhost_net_buf_unproduce(nvq);  in vhost_net_set_backend()
1530 				nvq->rx_ring = get_tap_ptr_ring(sock->file);  in vhost_net_set_backend()
1532 				nvq->rx_ring = NULL;  in vhost_net_set_backend()
1535 		oldubufs = nvq->ubufs;  in vhost_net_set_backend()
1536 		nvq->ubufs = ubufs;  in vhost_net_set_backend()