514 	uint16_t idx;  in virtqueue_nused()  local
525 		idx = vq->vq_split.ring.used->idx;  in virtqueue_nused()
528 		idx = __atomic_load_n(&(vq)->vq_split.ring.used->idx,  in virtqueue_nused()
532 		idx = vq->vq_split.ring.used->idx;  in virtqueue_nused()
535 	return idx - vq->vq_used_cons_idx;  in virtqueue_nused()
555 		vq->vq_split.ring.avail->idx = vq->vq_avail_idx;  in vq_update_avail_idx()
557 		__atomic_store_n(&vq->vq_split.ring.avail->idx,  in vq_update_avail_idx()
562 		vq->vq_split.ring.avail->idx = vq->vq_avail_idx;  in vq_update_avail_idx()
621 	used_idx = __atomic_load_n(&(vq)->vq_split.ring.used->idx, \
638 	  (vq)->vq_split.ring.avail->idx, (vq)->vq_used_cons_idx, \
639 	  __atomic_load_n(&(vq)->vq_split.ring.used->idx, __ATOMIC_RELAXED), \
718 	uint16_t idx, id, head_idx, head_flags;  in virtqueue_enqueue_xmit_packed()  local
732 	idx = head_idx;  in virtqueue_enqueue_xmit_packed()
736 	head_dp = &vq->vq_packed.ring.desc[idx];  in virtqueue_enqueue_xmit_packed()
756 		start_dp[idx].addr  = txvq->virtio_net_hdr_mem +  in virtqueue_enqueue_xmit_packed()
757 			RTE_PTR_DIFF(&txr[idx].tx_packed_indir, txr);  in virtqueue_enqueue_xmit_packed()
758 		start_dp[idx].len   = (seg_num + 1) *  in virtqueue_enqueue_xmit_packed()
762 			start_dp[idx].id = idx;  in virtqueue_enqueue_xmit_packed()
766 		hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr;  in virtqueue_enqueue_xmit_packed()
769 		start_dp = txr[idx].tx_packed_indir;  in virtqueue_enqueue_xmit_packed()
770 		idx = 1;  in virtqueue_enqueue_xmit_packed()
775 		start_dp[idx].addr  = txvq->virtio_net_hdr_mem +  in virtqueue_enqueue_xmit_packed()
776 			RTE_PTR_DIFF(&txr[idx].tx_hdr, txr);  in virtqueue_enqueue_xmit_packed()
777 		start_dp[idx].len   = vq->hw->vtnet_hdr_size;  in virtqueue_enqueue_xmit_packed()
778 		hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr;  in virtqueue_enqueue_xmit_packed()
779 		idx++;  in virtqueue_enqueue_xmit_packed()
780 		if (idx >= vq->vq_nentries) {  in virtqueue_enqueue_xmit_packed()
781 			idx -= vq->vq_nentries;  in virtqueue_enqueue_xmit_packed()
793 		start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq);  in virtqueue_enqueue_xmit_packed()
794 		start_dp[idx].len  = cookie->data_len;  in virtqueue_enqueue_xmit_packed()
796 			start_dp[idx].addr -= head_size;  in virtqueue_enqueue_xmit_packed()
797 			start_dp[idx].len += head_size;  in virtqueue_enqueue_xmit_packed()
801 		if (likely(idx != head_idx)) {  in virtqueue_enqueue_xmit_packed()
804 			start_dp[idx].flags = flags;  in virtqueue_enqueue_xmit_packed()
806 		prev = idx;  in virtqueue_enqueue_xmit_packed()
807 		idx++;  in virtqueue_enqueue_xmit_packed()
808 		if (idx >= vq->vq_nentries) {  in virtqueue_enqueue_xmit_packed()
809 			idx -= vq->vq_nentries;  in virtqueue_enqueue_xmit_packed()
818 		idx = head_idx;  in virtqueue_enqueue_xmit_packed()
819 		if (++idx >= vq->vq_nentries) {  in virtqueue_enqueue_xmit_packed()
820 			idx -= vq->vq_nentries;  in virtqueue_enqueue_xmit_packed()
827 	vq->vq_avail_idx = idx;  in virtqueue_enqueue_xmit_packed()
958 	uint16_t i, idx = vq->vq_used_cons_idx;  in virtio_xmit_cleanup_inorder()  local
966 		dxp = &vq->vq_descx[idx++ & (vq->vq_nentries - 1)];  in virtio_xmit_cleanup_inorder()
975 	vq->vq_used_cons_idx = idx;  in virtio_xmit_cleanup_inorder()