151 	struct virtqueue *vq = cvq->vq;  in virtio_send_command_packed()  local
153 	struct vring_packed_desc *desc = vq->vq_packed.ring.desc;  in virtio_send_command_packed()
166 	head = vq->vq_avail_idx;  in virtio_send_command_packed()
167 	flags = vq->vq_packed.cached_flags;  in virtio_send_command_packed()
170 	vq->vq_free_cnt--;  in virtio_send_command_packed()
172 	if (++vq->vq_avail_idx >= vq->vq_nentries) {  in virtio_send_command_packed()
173 		vq->vq_avail_idx -= vq->vq_nentries;  in virtio_send_command_packed()
174 		vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED;  in virtio_send_command_packed()
178 		desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem  in virtio_send_command_packed()
181 		desc[vq->vq_avail_idx].len = dlen[k];  in virtio_send_command_packed()
182 		desc[vq->vq_avail_idx].flags = VRING_DESC_F_NEXT |  in virtio_send_command_packed()
183 			vq->vq_packed.cached_flags;  in virtio_send_command_packed()
185 		vq->vq_free_cnt--;  in virtio_send_command_packed()
187 		if (++vq->vq_avail_idx >= vq->vq_nentries) {  in virtio_send_command_packed()
188 			vq->vq_avail_idx -= vq->vq_nentries;  in virtio_send_command_packed()
189 			vq->vq_packed.cached_flags ^=  in virtio_send_command_packed()
194 	desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem  in virtio_send_command_packed()
196 	desc[vq->vq_avail_idx].len = sizeof(ctrl->status);  in virtio_send_command_packed()
197 	desc[vq->vq_avail_idx].flags = VRING_DESC_F_WRITE |  in virtio_send_command_packed()
198 		vq->vq_packed.cached_flags;  in virtio_send_command_packed()
199 	vq->vq_free_cnt--;  in virtio_send_command_packed()
201 	if (++vq->vq_avail_idx >= vq->vq_nentries) {  in virtio_send_command_packed()
202 		vq->vq_avail_idx -= vq->vq_nentries;  in virtio_send_command_packed()
203 		vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED;  in virtio_send_command_packed()
206 	virtio_wmb(vq->hw->weak_barriers);  in virtio_send_command_packed()
209 	virtio_wmb(vq->hw->weak_barriers);  in virtio_send_command_packed()
210 	virtqueue_notify(vq);  in virtio_send_command_packed()
213 	while (!desc_is_used(&desc[head], vq))  in virtio_send_command_packed()
216 	virtio_rmb(vq->hw->weak_barriers);  in virtio_send_command_packed()
219 	vq->vq_free_cnt += nb_descs;  in virtio_send_command_packed()
220 	vq->vq_used_cons_idx += nb_descs;  in virtio_send_command_packed()
221 	if (vq->vq_used_cons_idx >= vq->vq_nentries) {  in virtio_send_command_packed()
222 		vq->vq_used_cons_idx -= vq->vq_nentries;  in virtio_send_command_packed()
223 		vq->vq_packed.used_wrap_counter ^= 1;  in virtio_send_command_packed()
231 			vq->vq_free_cnt,  in virtio_send_command_packed()
232 			vq->vq_avail_idx,  in virtio_send_command_packed()
233 			vq->vq_used_cons_idx,  in virtio_send_command_packed()
234 			vq->vq_packed.cached_flags,  in virtio_send_command_packed()
235 			vq->vq_packed.used_wrap_counter);  in virtio_send_command_packed()
247 	struct virtqueue *vq = cvq->vq;  in virtio_send_command_split()  local
251 	head = vq->vq_desc_head_idx;  in virtio_send_command_split()
259 	vq->vq_split.ring.desc[head].flags = VRING_DESC_F_NEXT;  in virtio_send_command_split()
260 	vq->vq_split.ring.desc[head].addr = cvq->virtio_net_hdr_mem;  in virtio_send_command_split()
261 	vq->vq_split.ring.desc[head].len = sizeof(struct virtio_net_ctrl_hdr);  in virtio_send_command_split()
262 	vq->vq_free_cnt--;  in virtio_send_command_split()
263 	i = vq->vq_split.ring.desc[head].next;  in virtio_send_command_split()
266 		vq->vq_split.ring.desc[i].flags = VRING_DESC_F_NEXT;  in virtio_send_command_split()
267 		vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem  in virtio_send_command_split()
270 		vq->vq_split.ring.desc[i].len = dlen[k];  in virtio_send_command_split()
272 		vq->vq_free_cnt--;  in virtio_send_command_split()
273 		i = vq->vq_split.ring.desc[i].next;  in virtio_send_command_split()
276 	vq->vq_split.ring.desc[i].flags = VRING_DESC_F_WRITE;  in virtio_send_command_split()
277 	vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem  in virtio_send_command_split()
279 	vq->vq_split.ring.desc[i].len = sizeof(ctrl->status);  in virtio_send_command_split()
280 	vq->vq_free_cnt--;  in virtio_send_command_split()
282 	vq->vq_desc_head_idx = vq->vq_split.ring.desc[i].next;  in virtio_send_command_split()
284 	vq_update_avail_ring(vq, head);  in virtio_send_command_split()
285 	vq_update_avail_idx(vq);  in virtio_send_command_split()
287 	PMD_INIT_LOG(DEBUG, "vq->vq_queue_index = %d", vq->vq_queue_index);  in virtio_send_command_split()
289 	virtqueue_notify(vq);  in virtio_send_command_split()
291 	while (virtqueue_nused(vq) == 0)  in virtio_send_command_split()
294 	while (virtqueue_nused(vq)) {  in virtio_send_command_split()
298 		used_idx = (uint32_t)(vq->vq_used_cons_idx  in virtio_send_command_split()
299 				& (vq->vq_nentries - 1));  in virtio_send_command_split()
300 		uep = &vq->vq_split.ring.used->ring[used_idx];  in virtio_send_command_split()
304 		while (vq->vq_split.ring.desc[desc_idx].flags &  in virtio_send_command_split()
306 			desc_idx = vq->vq_split.ring.desc[desc_idx].next;  in virtio_send_command_split()
307 			vq->vq_free_cnt++;  in virtio_send_command_split()
310 		vq->vq_split.ring.desc[desc_idx].next = vq->vq_desc_head_idx;  in virtio_send_command_split()
311 		vq->vq_desc_head_idx = idx;  in virtio_send_command_split()
313 		vq->vq_used_cons_idx++;  in virtio_send_command_split()
314 		vq->vq_free_cnt++;  in virtio_send_command_split()
318 			vq->vq_free_cnt, vq->vq_desc_head_idx);  in virtio_send_command_split()
330 	struct virtqueue *vq;  in virtio_send_command()  local
334 	if (!cvq || !cvq->vq) {  in virtio_send_command()
340 	vq = cvq->vq;  in virtio_send_command()
344 		vq->vq_desc_head_idx, status, vq->hw->cvq, vq);  in virtio_send_command()
346 	if (vq->vq_free_cnt < pkt_num + 2 || pkt_num < 1) {  in virtio_send_command()
354 	if (vtpci_packed_queue(vq->hw))  in virtio_send_command()
405 virtio_init_vring(struct virtqueue *vq)  in virtio_init_vring()  argument
407 	int size = vq->vq_nentries;  in virtio_init_vring()
408 	uint8_t *ring_mem = vq->vq_ring_virt_mem;  in virtio_init_vring()
412 	memset(ring_mem, 0, vq->vq_ring_size);  in virtio_init_vring()
414 	vq->vq_used_cons_idx = 0;  in virtio_init_vring()
415 	vq->vq_desc_head_idx = 0;  in virtio_init_vring()
416 	vq->vq_avail_idx = 0;  in virtio_init_vring()
417 	vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);  in virtio_init_vring()
418 	vq->vq_free_cnt = vq->vq_nentries;  in virtio_init_vring()
419 	memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries);  in virtio_init_vring()
420 	if (vtpci_packed_queue(vq->hw)) {  in virtio_init_vring()
421 		vring_init_packed(&vq->vq_packed.ring, ring_mem,  in virtio_init_vring()
423 		vring_desc_init_packed(vq, size);  in virtio_init_vring()
425 		struct vring *vr = &vq->vq_split.ring;  in virtio_init_vring()
433 	virtqueue_disable_intr(vq);  in virtio_init_vring()
447 	struct virtqueue *vq;  in virtio_init_queue()  local
476 	size = RTE_ALIGN_CEIL(sizeof(*vq) +  in virtio_init_queue()
490 	vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE,  in virtio_init_queue()
492 	if (vq == NULL) {  in virtio_init_queue()
496 	hw->vqs[vtpci_queue_idx] = vq;  in virtio_init_queue()
498 	vq->hw = hw;  in virtio_init_queue()
499 	vq->vq_queue_index = vtpci_queue_idx;  in virtio_init_queue()
500 	vq->vq_nentries = vq_size;  in virtio_init_queue()
502 		vq->vq_packed.used_wrap_counter = 1;  in virtio_init_queue()
503 		vq->vq_packed.cached_flags = VRING_PACKED_DESC_F_AVAIL;  in virtio_init_queue()
504 		vq->vq_packed.event_flags_shadow = 0;  in virtio_init_queue()
506 			vq->vq_packed.cached_flags |= VRING_DESC_F_WRITE;  in virtio_init_queue()
513 	vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN);  in virtio_init_queue()
515 		     size, vq->vq_ring_size);  in virtio_init_queue()
517 	mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,  in virtio_init_queue()
531 	vq->vq_ring_mem = mz->iova;  in virtio_init_queue()
532 	vq->vq_ring_virt_mem = mz->addr;  in virtio_init_queue()
538 	virtio_init_vring(vq);  in virtio_init_queue()
558 			       sizeof(vq->sw_ring[0]);  in virtio_init_queue()
568 		vq->sw_ring = sw_ring;  in virtio_init_queue()
569 		rxvq = &vq->rxq;  in virtio_init_queue()
570 		rxvq->vq = vq;  in virtio_init_queue()
574 		txvq = &vq->txq;  in virtio_init_queue()
575 		txvq->vq = vq;  in virtio_init_queue()
581 		cvq = &vq->cq;  in virtio_init_queue()
582 		cvq->vq = vq;  in virtio_init_queue()
596 		vq->offset = offsetof(struct rte_mbuf, buf_iova);  in virtio_init_queue()
598 		vq->vq_ring_mem = (uintptr_t)mz->addr;  in virtio_init_queue()
599 		vq->offset = offsetof(struct rte_mbuf, buf_addr);  in virtio_init_queue()
638 	if (VTPCI_OPS(hw)->setup_queue(hw, vq) < 0) {  in virtio_init_queue()
649 	rte_free(vq);  in virtio_init_queue()
658 	struct virtqueue *vq;  in virtio_free_queues()  local
666 		vq = hw->vqs[i];  in virtio_free_queues()
667 		if (!vq)  in virtio_free_queues()
672 			rte_free(vq->sw_ring);  in virtio_free_queues()
673 			rte_memzone_free(vq->rxq.mz);  in virtio_free_queues()
675 			rte_memzone_free(vq->txq.mz);  in virtio_free_queues()
676 			rte_memzone_free(vq->txq.virtio_net_hdr_mz);  in virtio_free_queues()
678 			rte_memzone_free(vq->cq.mz);  in virtio_free_queues()
679 			rte_memzone_free(vq->cq.virtio_net_hdr_mz);  in virtio_free_queues()
682 		rte_free(vq);  in virtio_free_queues()
895 	struct virtqueue *vq = rxvq->vq;  in virtio_dev_rx_queue_intr_enable()  local
897 	virtqueue_enable_intr(vq);  in virtio_dev_rx_queue_intr_enable()
906 	struct virtqueue *vq = rxvq->vq;  in virtio_dev_rx_queue_intr_disable()  local
908 	virtqueue_disable_intr(vq);  in virtio_dev_rx_queue_intr_disable()
2447 		virtqueue_rxvq_flush(rxvq->vq);  in virtio_dev_start()
2448 		virtqueue_notify(rxvq->vq);  in virtio_dev_start()
2453 		virtqueue_notify(txvq->vq);  in virtio_dev_start()
2460 		VIRTQUEUE_DUMP(rxvq->vq);  in virtio_dev_start()
2465 		VIRTQUEUE_DUMP(txvq->vq);  in virtio_dev_start()
2483 	struct virtqueue *vq;  in virtio_dev_free_mbufs()  local
2491 		vq = hw->vqs[i];  in virtio_dev_free_mbufs()
2492 		if (!vq)  in virtio_dev_free_mbufs()
2506 		VIRTQUEUE_DUMP(vq);  in virtio_dev_free_mbufs()
2508 		while ((buf = virtqueue_detach_unused(vq)) != NULL) {  in virtio_dev_free_mbufs()
2516 		VIRTQUEUE_DUMP(vq);  in virtio_dev_free_mbufs()