19 virtqueue_detach_unused(struct virtqueue *vq)  in virtqueue_detach_unused()  argument
26 	if (vq == NULL)  in virtqueue_detach_unused()
29 	hw = vq->hw;  in virtqueue_detach_unused()
30 	type = virtio_get_queue_type(hw, vq->vq_queue_index);  in virtqueue_detach_unused()
31 	start = vq->vq_avail_idx & (vq->vq_nentries - 1);  in virtqueue_detach_unused()
32 	end = (vq->vq_avail_idx + vq->vq_free_cnt) & (vq->vq_nentries - 1);  in virtqueue_detach_unused()
34 	for (idx = 0; idx < vq->vq_nentries; idx++) {  in virtqueue_detach_unused()
41 			cookie = vq->sw_ring[idx];  in virtqueue_detach_unused()
43 				vq->sw_ring[idx] = NULL;  in virtqueue_detach_unused()
47 			cookie = vq->vq_descx[idx].cookie;  in virtqueue_detach_unused()
49 				vq->vq_descx[idx].cookie = NULL;  in virtqueue_detach_unused()
60 virtqueue_rxvq_flush_packed(struct virtqueue *vq)  in virtqueue_rxvq_flush_packed()  argument
65 	struct vring_packed_desc *descs = vq->vq_packed.ring.desc;  in virtqueue_rxvq_flush_packed()
68 	i = vq->vq_used_cons_idx;  in virtqueue_rxvq_flush_packed()
69 	while (desc_is_used(&descs[i], vq) && cnt++ < vq->vq_nentries) {  in virtqueue_rxvq_flush_packed()
70 		dxp = &vq->vq_descx[descs[i].id];  in virtqueue_rxvq_flush_packed()
75 		vq->vq_free_cnt++;  in virtqueue_rxvq_flush_packed()
76 		vq->vq_used_cons_idx++;  in virtqueue_rxvq_flush_packed()
77 		if (vq->vq_used_cons_idx >= vq->vq_nentries) {  in virtqueue_rxvq_flush_packed()
78 			vq->vq_used_cons_idx -= vq->vq_nentries;  in virtqueue_rxvq_flush_packed()
79 			vq->vq_packed.used_wrap_counter ^= 1;  in virtqueue_rxvq_flush_packed()
81 		i = vq->vq_used_cons_idx;  in virtqueue_rxvq_flush_packed()
87 virtqueue_rxvq_flush_split(struct virtqueue *vq)  in virtqueue_rxvq_flush_split()  argument
89 	struct virtnet_rx *rxq = &vq->rxq;  in virtqueue_rxvq_flush_split()
90 	struct virtio_hw *hw = vq->hw;  in virtqueue_rxvq_flush_split()
96 	nb_used = virtqueue_nused(vq);  in virtqueue_rxvq_flush_split()
99 		used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1);  in virtqueue_rxvq_flush_split()
100 		uep = &vq->vq_split.ring.used->ring[used_idx];  in virtqueue_rxvq_flush_split()
103 			rte_pktmbuf_free(vq->sw_ring[desc_idx]);  in virtqueue_rxvq_flush_split()
104 			vq->vq_free_cnt++;  in virtqueue_rxvq_flush_split()
107 			dxp = &vq->vq_descx[desc_idx];  in virtqueue_rxvq_flush_split()
112 			vq_ring_free_inorder(vq, desc_idx, 1);  in virtqueue_rxvq_flush_split()
115 			dxp = &vq->vq_descx[desc_idx];  in virtqueue_rxvq_flush_split()
120 			vq_ring_free_chain(vq, desc_idx);  in virtqueue_rxvq_flush_split()
122 		vq->vq_used_cons_idx++;  in virtqueue_rxvq_flush_split()
126 		while (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) {  in virtqueue_rxvq_flush_split()
128 			if (virtqueue_kick_prepare(vq))  in virtqueue_rxvq_flush_split()
129 				virtqueue_notify(vq);  in virtqueue_rxvq_flush_split()
136 virtqueue_rxvq_flush(struct virtqueue *vq)  in virtqueue_rxvq_flush()  argument
138 	struct virtio_hw *hw = vq->hw;  in virtqueue_rxvq_flush()
141 		virtqueue_rxvq_flush_packed(vq);  in virtqueue_rxvq_flush()
143 		virtqueue_rxvq_flush_split(vq);  in virtqueue_rxvq_flush()
147 virtqueue_rxvq_reset_packed(struct virtqueue *vq)  in virtqueue_rxvq_reset_packed()  argument
149 	int size = vq->vq_nentries;  in virtqueue_rxvq_reset_packed()
154 	vq->vq_used_cons_idx = 0;  in virtqueue_rxvq_reset_packed()
155 	vq->vq_desc_head_idx = 0;  in virtqueue_rxvq_reset_packed()
156 	vq->vq_avail_idx = 0;  in virtqueue_rxvq_reset_packed()
157 	vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);  in virtqueue_rxvq_reset_packed()
158 	vq->vq_free_cnt = vq->vq_nentries;  in virtqueue_rxvq_reset_packed()
160 	vq->vq_packed.used_wrap_counter = 1;  in virtqueue_rxvq_reset_packed()
161 	vq->vq_packed.cached_flags = VRING_PACKED_DESC_F_AVAIL;  in virtqueue_rxvq_reset_packed()
162 	vq->vq_packed.event_flags_shadow = 0;  in virtqueue_rxvq_reset_packed()
163 	vq->vq_packed.cached_flags |= VRING_DESC_F_WRITE;  in virtqueue_rxvq_reset_packed()
165 	rxvq = &vq->rxq;  in virtqueue_rxvq_reset_packed()
168 	for (desc_idx = 0; desc_idx < vq->vq_nentries; desc_idx++) {  in virtqueue_rxvq_reset_packed()
169 		dxp = &vq->vq_descx[desc_idx];  in virtqueue_rxvq_reset_packed()
176 	vring_desc_init_packed(vq, size);  in virtqueue_rxvq_reset_packed()
178 	virtqueue_disable_intr(vq);  in virtqueue_rxvq_reset_packed()
183 virtqueue_txvq_reset_packed(struct virtqueue *vq)  in virtqueue_txvq_reset_packed()  argument
185 	int size = vq->vq_nentries;  in virtqueue_txvq_reset_packed()
192 	vq->vq_used_cons_idx = 0;  in virtqueue_txvq_reset_packed()
193 	vq->vq_desc_head_idx = 0;  in virtqueue_txvq_reset_packed()
194 	vq->vq_avail_idx = 0;  in virtqueue_txvq_reset_packed()
195 	vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);  in virtqueue_txvq_reset_packed()
196 	vq->vq_free_cnt = vq->vq_nentries;  in virtqueue_txvq_reset_packed()
198 	vq->vq_packed.used_wrap_counter = 1;  in virtqueue_txvq_reset_packed()
199 	vq->vq_packed.cached_flags = VRING_PACKED_DESC_F_AVAIL;  in virtqueue_txvq_reset_packed()
200 	vq->vq_packed.event_flags_shadow = 0;  in virtqueue_txvq_reset_packed()
202 	txvq = &vq->txq;  in virtqueue_txvq_reset_packed()
208 	for (desc_idx = 0; desc_idx < vq->vq_nentries; desc_idx++) {  in virtqueue_txvq_reset_packed()
209 		dxp = &vq->vq_descx[desc_idx];  in virtqueue_txvq_reset_packed()
215 		if (virtio_with_feature(vq->hw, VIRTIO_RING_F_INDIRECT_DESC)) {  in virtqueue_txvq_reset_packed()
223 			start_dp->len = vq->hw->vtnet_hdr_size;  in virtqueue_txvq_reset_packed()
227 	vring_desc_init_packed(vq, size);  in virtqueue_txvq_reset_packed()
229 	virtqueue_disable_intr(vq);  in virtqueue_txvq_reset_packed()