125 #define VIRTIO_MBUF_ADDR(mb, vq) \  argument
126 	((uint64_t)(*(uintptr_t *)((uintptr_t)(mb) + (vq)->mbuf_addr_offset)))
132 #define VIRTIO_MBUF_DATA_DMA_ADDR(mb, vq) \  argument
133 	(VIRTIO_MBUF_ADDR(mb, vq) + (mb)->data_off)
365 desc_is_used(struct vring_packed_desc *desc, struct virtqueue *vq)  in desc_is_used()  argument
369 	flags = virtqueue_fetch_flags_packed(desc, vq->hw->weak_barriers);  in desc_is_used()
373 	return avail == used && used == vq->vq_packed.used_wrap_counter;  in desc_is_used()
377 vring_desc_init_packed(struct virtqueue *vq, int n)  in vring_desc_init_packed()  argument
381 		vq->vq_packed.ring.desc[i].id = i;  in vring_desc_init_packed()
382 		vq->vq_descx[i].next = i + 1;  in vring_desc_init_packed()
384 	vq->vq_packed.ring.desc[i].id = i;  in vring_desc_init_packed()
385 	vq->vq_descx[i].next = VQ_RING_DESC_CHAIN_END;  in vring_desc_init_packed()
413 virtqueue_disable_intr_packed(struct virtqueue *vq)  in virtqueue_disable_intr_packed()  argument
415 	if (vq->vq_packed.event_flags_shadow != RING_EVENT_FLAGS_DISABLE) {  in virtqueue_disable_intr_packed()
416 		vq->vq_packed.event_flags_shadow = RING_EVENT_FLAGS_DISABLE;  in virtqueue_disable_intr_packed()
417 		vq->vq_packed.ring.driver->desc_event_flags =  in virtqueue_disable_intr_packed()
418 			vq->vq_packed.event_flags_shadow;  in virtqueue_disable_intr_packed()
426 virtqueue_disable_intr_split(struct virtqueue *vq)  in virtqueue_disable_intr_split()  argument
428 	vq->vq_split.ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;  in virtqueue_disable_intr_split()
435 virtqueue_disable_intr(struct virtqueue *vq)  in virtqueue_disable_intr()  argument
437 	if (virtio_with_packed_queue(vq->hw))  in virtqueue_disable_intr()
438 		virtqueue_disable_intr_packed(vq);  in virtqueue_disable_intr()
440 		virtqueue_disable_intr_split(vq);  in virtqueue_disable_intr()
447 virtqueue_enable_intr_packed(struct virtqueue *vq)  in virtqueue_enable_intr_packed()  argument
449 	if (vq->vq_packed.event_flags_shadow == RING_EVENT_FLAGS_DISABLE) {  in virtqueue_enable_intr_packed()
450 		vq->vq_packed.event_flags_shadow = RING_EVENT_FLAGS_ENABLE;  in virtqueue_enable_intr_packed()
451 		vq->vq_packed.ring.driver->desc_event_flags =  in virtqueue_enable_intr_packed()
452 			vq->vq_packed.event_flags_shadow;  in virtqueue_enable_intr_packed()
460 virtqueue_enable_intr_split(struct virtqueue *vq)  in virtqueue_enable_intr_split()  argument
462 	vq->vq_split.ring.avail->flags &= (~VRING_AVAIL_F_NO_INTERRUPT);  in virtqueue_enable_intr_split()
469 virtqueue_enable_intr(struct virtqueue *vq)  in virtqueue_enable_intr()  argument
471 	if (virtio_with_packed_queue(vq->hw))  in virtqueue_enable_intr()
472 		virtqueue_enable_intr_packed(vq);  in virtqueue_enable_intr()
474 		virtqueue_enable_intr_split(vq);  in virtqueue_enable_intr()
480 void virtqueue_dump(struct virtqueue *vq);
484 struct rte_mbuf *virtqueue_detach_unused(struct virtqueue *vq);
487 void virtqueue_rxvq_flush(struct virtqueue *vq);
489 int virtqueue_rxvq_reset_packed(struct virtqueue *vq);
491 int virtqueue_txvq_reset_packed(struct virtqueue *vq);
494 virtqueue_full(const struct virtqueue *vq)  in virtqueue_full()  argument
496 	return vq->vq_free_cnt == 0;  in virtqueue_full()
512 virtqueue_nused(const struct virtqueue *vq)  in virtqueue_nused()  argument
516 	if (vq->hw->weak_barriers) {  in virtqueue_nused()
525 		idx = vq->vq_split.ring.used->idx;  in virtqueue_nused()
528 		idx = __atomic_load_n(&(vq)->vq_split.ring.used->idx,  in virtqueue_nused()
532 		idx = vq->vq_split.ring.used->idx;  in virtqueue_nused()
535 	return idx - vq->vq_used_cons_idx;  in virtqueue_nused()
538 void vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx);
539 void vq_ring_free_chain_packed(struct virtqueue *vq, uint16_t used_idx);
540 void vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx,
544 vq_update_avail_idx(struct virtqueue *vq)  in vq_update_avail_idx()  argument
546 	if (vq->hw->weak_barriers) {  in vq_update_avail_idx()
555 		vq->vq_split.ring.avail->idx = vq->vq_avail_idx;  in vq_update_avail_idx()
557 		__atomic_store_n(&vq->vq_split.ring.avail->idx,  in vq_update_avail_idx()
558 				 vq->vq_avail_idx, __ATOMIC_RELEASE);  in vq_update_avail_idx()
562 		vq->vq_split.ring.avail->idx = vq->vq_avail_idx;  in vq_update_avail_idx()
567 vq_update_avail_ring(struct virtqueue *vq, uint16_t desc_idx)  in vq_update_avail_ring()  argument
577 	avail_idx = (uint16_t)(vq->vq_avail_idx & (vq->vq_nentries - 1));  in vq_update_avail_ring()
578 	if (unlikely(vq->vq_split.ring.avail->ring[avail_idx] != desc_idx))  in vq_update_avail_ring()
579 		vq->vq_split.ring.avail->ring[avail_idx] = desc_idx;  in vq_update_avail_ring()
580 	vq->vq_avail_idx++;  in vq_update_avail_ring()
584 virtqueue_kick_prepare(struct virtqueue *vq)  in virtqueue_kick_prepare()  argument
590 	virtio_mb(vq->hw->weak_barriers);  in virtqueue_kick_prepare()
591 	return !(vq->vq_split.ring.used->flags & VRING_USED_F_NO_NOTIFY);  in virtqueue_kick_prepare()
595 virtqueue_kick_prepare_packed(struct virtqueue *vq)  in virtqueue_kick_prepare_packed()  argument
602 	virtio_mb(vq->hw->weak_barriers);  in virtqueue_kick_prepare_packed()
603 	flags = vq->vq_packed.ring.device->desc_event_flags;  in virtqueue_kick_prepare_packed()
613 virtqueue_notify(struct virtqueue *vq)  in virtqueue_notify()  argument
615 	VIRTIO_OPS(vq->hw)->notify_queue(vq->hw, vq);  in virtqueue_notify()
619 #define VIRTQUEUE_DUMP(vq) do { \  argument
621 	used_idx = __atomic_load_n(&(vq)->vq_split.ring.used->idx, \
623 	nused = (uint16_t)(used_idx - (vq)->vq_used_cons_idx); \
624 	if (virtio_with_packed_queue((vq)->hw)) { \
628 		(vq)->vq_nentries, (vq)->vq_free_cnt, (vq)->vq_used_cons_idx, \
629 		(vq)->vq_avail_idx, (vq)->vq_packed.cached_flags, \
630 		(vq)->vq_packed.used_wrap_counter); \
637 	  (vq)->vq_nentries, (vq)->vq_free_cnt, nused, (vq)->vq_desc_head_idx, \
638 	  (vq)->vq_split.ring.avail->idx, (vq)->vq_used_cons_idx, \
639 	  __atomic_load_n(&(vq)->vq_split.ring.used->idx, __ATOMIC_RELAXED), \
640 	  (vq)->vq_split.ring.avail->flags, (vq)->vq_split.ring.used->flags); \
643 #define VIRTQUEUE_DUMP(vq) do { } while (0)  argument
716 	struct virtqueue *vq = virtnet_txq_to_vq(txvq);  in virtqueue_enqueue_xmit_packed()  local
719 	int16_t head_size = vq->hw->vtnet_hdr_size;  in virtqueue_enqueue_xmit_packed()
725 	id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx;  in virtqueue_enqueue_xmit_packed()
727 	dxp = &vq->vq_descx[id];  in virtqueue_enqueue_xmit_packed()
731 	head_idx = vq->vq_avail_idx;  in virtqueue_enqueue_xmit_packed()
734 	start_dp = vq->vq_packed.ring.desc;  in virtqueue_enqueue_xmit_packed()
736 	head_dp = &vq->vq_packed.ring.desc[idx];  in virtqueue_enqueue_xmit_packed()
738 	head_flags |= vq->vq_packed.cached_flags;  in virtqueue_enqueue_xmit_packed()
747 		if (!vq->hw->has_tx_offload)  in virtqueue_enqueue_xmit_packed()
765 		head_flags |= vq->vq_packed.cached_flags;  in virtqueue_enqueue_xmit_packed()
777 		start_dp[idx].len   = vq->hw->vtnet_hdr_size;  in virtqueue_enqueue_xmit_packed()
780 		if (idx >= vq->vq_nentries) {  in virtqueue_enqueue_xmit_packed()
781 			idx -= vq->vq_nentries;  in virtqueue_enqueue_xmit_packed()
782 			vq->vq_packed.cached_flags ^=  in virtqueue_enqueue_xmit_packed()
787 	if (vq->hw->has_tx_offload)  in virtqueue_enqueue_xmit_packed()
793 		start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq);  in virtqueue_enqueue_xmit_packed()
803 			flags |= vq->vq_packed.cached_flags;  in virtqueue_enqueue_xmit_packed()
808 		if (idx >= vq->vq_nentries) {  in virtqueue_enqueue_xmit_packed()
809 			idx -= vq->vq_nentries;  in virtqueue_enqueue_xmit_packed()
810 			vq->vq_packed.cached_flags ^=  in virtqueue_enqueue_xmit_packed()
819 		if (++idx >= vq->vq_nentries) {  in virtqueue_enqueue_xmit_packed()
820 			idx -= vq->vq_nentries;  in virtqueue_enqueue_xmit_packed()
821 			vq->vq_packed.cached_flags ^=  in virtqueue_enqueue_xmit_packed()
826 	vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);  in virtqueue_enqueue_xmit_packed()
827 	vq->vq_avail_idx = idx;  in virtqueue_enqueue_xmit_packed()
830 		vq->vq_desc_head_idx = dxp->next;  in virtqueue_enqueue_xmit_packed()
831 		if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)  in virtqueue_enqueue_xmit_packed()
832 			vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;  in virtqueue_enqueue_xmit_packed()
836 				     vq->hw->weak_barriers);  in virtqueue_enqueue_xmit_packed()
840 vq_ring_free_id_packed(struct virtqueue *vq, uint16_t id)  in vq_ring_free_id_packed()  argument
844 	dxp = &vq->vq_descx[id];  in vq_ring_free_id_packed()
845 	vq->vq_free_cnt += dxp->ndescs;  in vq_ring_free_id_packed()
847 	if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END)  in vq_ring_free_id_packed()
848 		vq->vq_desc_head_idx = id;  in vq_ring_free_id_packed()
850 		vq->vq_descx[vq->vq_desc_tail_idx].next = id;  in vq_ring_free_id_packed()
852 	vq->vq_desc_tail_idx = id;  in vq_ring_free_id_packed()
857 virtio_xmit_cleanup_inorder_packed(struct virtqueue *vq, uint16_t num)  in virtio_xmit_cleanup_inorder_packed()  argument
860 	uint16_t size = vq->vq_nentries;  in virtio_xmit_cleanup_inorder_packed()
861 	struct vring_packed_desc *desc = vq->vq_packed.ring.desc;  in virtio_xmit_cleanup_inorder_packed()
865 	used_idx = vq->vq_used_cons_idx;  in virtio_xmit_cleanup_inorder_packed()
869 	while (nb > 0 && desc_is_used(&desc[used_idx], vq)) {  in virtio_xmit_cleanup_inorder_packed()
873 			dxp = &vq->vq_descx[used_idx];  in virtio_xmit_cleanup_inorder_packed()
879 				vq->vq_packed.used_wrap_counter ^= 1;  in virtio_xmit_cleanup_inorder_packed()
887 	vq->vq_used_cons_idx = used_idx;  in virtio_xmit_cleanup_inorder_packed()
888 	vq->vq_free_cnt += free_cnt;  in virtio_xmit_cleanup_inorder_packed()
892 virtio_xmit_cleanup_normal_packed(struct virtqueue *vq, uint16_t num)  in virtio_xmit_cleanup_normal_packed()  argument
895 	uint16_t size = vq->vq_nentries;  in virtio_xmit_cleanup_normal_packed()
896 	struct vring_packed_desc *desc = vq->vq_packed.ring.desc;  in virtio_xmit_cleanup_normal_packed()
899 	used_idx = vq->vq_used_cons_idx;  in virtio_xmit_cleanup_normal_packed()
903 	while (num-- && desc_is_used(&desc[used_idx], vq)) {  in virtio_xmit_cleanup_normal_packed()
905 		dxp = &vq->vq_descx[id];  in virtio_xmit_cleanup_normal_packed()
906 		vq->vq_used_cons_idx += dxp->ndescs;  in virtio_xmit_cleanup_normal_packed()
907 		if (vq->vq_used_cons_idx >= size) {  in virtio_xmit_cleanup_normal_packed()
908 			vq->vq_used_cons_idx -= size;  in virtio_xmit_cleanup_normal_packed()
909 			vq->vq_packed.used_wrap_counter ^= 1;  in virtio_xmit_cleanup_normal_packed()
911 		vq_ring_free_id_packed(vq, id);  in virtio_xmit_cleanup_normal_packed()
916 		used_idx = vq->vq_used_cons_idx;  in virtio_xmit_cleanup_normal_packed()
922 virtio_xmit_cleanup_packed(struct virtqueue *vq, uint16_t num, int in_order)  in virtio_xmit_cleanup_packed()  argument
925 		virtio_xmit_cleanup_inorder_packed(vq, num);  in virtio_xmit_cleanup_packed()
927 		virtio_xmit_cleanup_normal_packed(vq, num);  in virtio_xmit_cleanup_packed()
931 virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num)  in virtio_xmit_cleanup()  argument
938 		used_idx = (uint16_t)(vq->vq_used_cons_idx &  in virtio_xmit_cleanup()
939 				(vq->vq_nentries - 1));  in virtio_xmit_cleanup()
940 		uep = &vq->vq_split.ring.used->ring[used_idx];  in virtio_xmit_cleanup()
943 		dxp = &vq->vq_descx[desc_idx];  in virtio_xmit_cleanup()
944 		vq->vq_used_cons_idx++;  in virtio_xmit_cleanup()
945 		vq_ring_free_chain(vq, desc_idx);  in virtio_xmit_cleanup()
956 virtio_xmit_cleanup_inorder(struct virtqueue *vq, uint16_t num)  in virtio_xmit_cleanup_inorder()  argument
958 	uint16_t i, idx = vq->vq_used_cons_idx;  in virtio_xmit_cleanup_inorder()
966 		dxp = &vq->vq_descx[idx++ & (vq->vq_nentries - 1)];  in virtio_xmit_cleanup_inorder()
974 	vq->vq_free_cnt += free_cnt;  in virtio_xmit_cleanup_inorder()
975 	vq->vq_used_cons_idx = idx;  in virtio_xmit_cleanup_inorder()