Lines Matching refs:vq
149 struct virtqueue *vq = virtnet_cq_to_vq(cvq); in virtio_send_command_packed() local
151 struct vring_packed_desc *desc = vq->vq_packed.ring.desc; in virtio_send_command_packed()
164 head = vq->vq_avail_idx; in virtio_send_command_packed()
165 flags = vq->vq_packed.cached_flags; in virtio_send_command_packed()
168 vq->vq_free_cnt--; in virtio_send_command_packed()
170 if (++vq->vq_avail_idx >= vq->vq_nentries) { in virtio_send_command_packed()
171 vq->vq_avail_idx -= vq->vq_nentries; in virtio_send_command_packed()
172 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED; in virtio_send_command_packed()
176 desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem in virtio_send_command_packed()
179 desc[vq->vq_avail_idx].len = dlen[k]; in virtio_send_command_packed()
180 desc[vq->vq_avail_idx].flags = VRING_DESC_F_NEXT | in virtio_send_command_packed()
181 vq->vq_packed.cached_flags; in virtio_send_command_packed()
183 vq->vq_free_cnt--; in virtio_send_command_packed()
185 if (++vq->vq_avail_idx >= vq->vq_nentries) { in virtio_send_command_packed()
186 vq->vq_avail_idx -= vq->vq_nentries; in virtio_send_command_packed()
187 vq->vq_packed.cached_flags ^= in virtio_send_command_packed()
192 desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem in virtio_send_command_packed()
194 desc[vq->vq_avail_idx].len = sizeof(ctrl->status); in virtio_send_command_packed()
195 desc[vq->vq_avail_idx].flags = VRING_DESC_F_WRITE | in virtio_send_command_packed()
196 vq->vq_packed.cached_flags; in virtio_send_command_packed()
197 vq->vq_free_cnt--; in virtio_send_command_packed()
199 if (++vq->vq_avail_idx >= vq->vq_nentries) { in virtio_send_command_packed()
200 vq->vq_avail_idx -= vq->vq_nentries; in virtio_send_command_packed()
201 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED; in virtio_send_command_packed()
205 vq->hw->weak_barriers); in virtio_send_command_packed()
207 virtio_wmb(vq->hw->weak_barriers); in virtio_send_command_packed()
208 virtqueue_notify(vq); in virtio_send_command_packed()
213 while (!desc_is_used(&desc[head], vq)) in virtio_send_command_packed()
217 vq->vq_free_cnt += nb_descs; in virtio_send_command_packed()
218 vq->vq_used_cons_idx += nb_descs; in virtio_send_command_packed()
219 if (vq->vq_used_cons_idx >= vq->vq_nentries) { in virtio_send_command_packed()
220 vq->vq_used_cons_idx -= vq->vq_nentries; in virtio_send_command_packed()
221 vq->vq_packed.used_wrap_counter ^= 1; in virtio_send_command_packed()
229 vq->vq_free_cnt, in virtio_send_command_packed()
230 vq->vq_avail_idx, in virtio_send_command_packed()
231 vq->vq_used_cons_idx, in virtio_send_command_packed()
232 vq->vq_packed.cached_flags, in virtio_send_command_packed()
233 vq->vq_packed.used_wrap_counter); in virtio_send_command_packed()
245 struct virtqueue *vq = virtnet_cq_to_vq(cvq); in virtio_send_command_split() local
249 head = vq->vq_desc_head_idx; in virtio_send_command_split()
257 vq->vq_split.ring.desc[head].flags = VRING_DESC_F_NEXT; in virtio_send_command_split()
258 vq->vq_split.ring.desc[head].addr = cvq->virtio_net_hdr_mem; in virtio_send_command_split()
259 vq->vq_split.ring.desc[head].len = sizeof(struct virtio_net_ctrl_hdr); in virtio_send_command_split()
260 vq->vq_free_cnt--; in virtio_send_command_split()
261 i = vq->vq_split.ring.desc[head].next; in virtio_send_command_split()
264 vq->vq_split.ring.desc[i].flags = VRING_DESC_F_NEXT; in virtio_send_command_split()
265 vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem in virtio_send_command_split()
268 vq->vq_split.ring.desc[i].len = dlen[k]; in virtio_send_command_split()
270 vq->vq_free_cnt--; in virtio_send_command_split()
271 i = vq->vq_split.ring.desc[i].next; in virtio_send_command_split()
274 vq->vq_split.ring.desc[i].flags = VRING_DESC_F_WRITE; in virtio_send_command_split()
275 vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem in virtio_send_command_split()
277 vq->vq_split.ring.desc[i].len = sizeof(ctrl->status); in virtio_send_command_split()
278 vq->vq_free_cnt--; in virtio_send_command_split()
280 vq->vq_desc_head_idx = vq->vq_split.ring.desc[i].next; in virtio_send_command_split()
282 vq_update_avail_ring(vq, head); in virtio_send_command_split()
283 vq_update_avail_idx(vq); in virtio_send_command_split()
285 PMD_INIT_LOG(DEBUG, "vq->vq_queue_index = %d", vq->vq_queue_index); in virtio_send_command_split()
287 virtqueue_notify(vq); in virtio_send_command_split()
289 while (virtqueue_nused(vq) == 0) in virtio_send_command_split()
292 while (virtqueue_nused(vq)) { in virtio_send_command_split()
296 used_idx = (uint32_t)(vq->vq_used_cons_idx in virtio_send_command_split()
297 & (vq->vq_nentries - 1)); in virtio_send_command_split()
298 uep = &vq->vq_split.ring.used->ring[used_idx]; in virtio_send_command_split()
302 while (vq->vq_split.ring.desc[desc_idx].flags & in virtio_send_command_split()
304 desc_idx = vq->vq_split.ring.desc[desc_idx].next; in virtio_send_command_split()
305 vq->vq_free_cnt++; in virtio_send_command_split()
308 vq->vq_split.ring.desc[desc_idx].next = vq->vq_desc_head_idx; in virtio_send_command_split()
309 vq->vq_desc_head_idx = idx; in virtio_send_command_split()
311 vq->vq_used_cons_idx++; in virtio_send_command_split()
312 vq->vq_free_cnt++; in virtio_send_command_split()
316 vq->vq_free_cnt, vq->vq_desc_head_idx); in virtio_send_command_split()
328 struct virtqueue *vq; in virtio_send_command() local
338 vq = virtnet_cq_to_vq(cvq); in virtio_send_command()
342 vq->vq_desc_head_idx, status, vq->hw->cvq, vq); in virtio_send_command()
344 if (vq->vq_free_cnt < pkt_num + 2 || pkt_num < 1) { in virtio_send_command()
352 if (virtio_with_packed_queue(vq->hw)) in virtio_send_command()
440 virtio_init_vring(struct virtqueue *vq) in virtio_init_vring() argument
442 int size = vq->vq_nentries; in virtio_init_vring()
443 uint8_t *ring_mem = vq->vq_ring_virt_mem; in virtio_init_vring()
447 memset(ring_mem, 0, vq->vq_ring_size); in virtio_init_vring()
449 vq->vq_used_cons_idx = 0; in virtio_init_vring()
450 vq->vq_desc_head_idx = 0; in virtio_init_vring()
451 vq->vq_avail_idx = 0; in virtio_init_vring()
452 vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1); in virtio_init_vring()
453 vq->vq_free_cnt = vq->vq_nentries; in virtio_init_vring()
454 memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries); in virtio_init_vring()
455 if (virtio_with_packed_queue(vq->hw)) { in virtio_init_vring()
456 vring_init_packed(&vq->vq_packed.ring, ring_mem, in virtio_init_vring()
458 vring_desc_init_packed(vq, size); in virtio_init_vring()
460 struct vring *vr = &vq->vq_split.ring; in virtio_init_vring()
468 virtqueue_disable_intr(vq); in virtio_init_vring()
482 struct virtqueue *vq; in virtio_init_queue() local
512 size = RTE_ALIGN_CEIL(sizeof(*vq) + in virtio_init_queue()
526 vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE, in virtio_init_queue()
528 if (vq == NULL) { in virtio_init_queue()
532 hw->vqs[queue_idx] = vq; in virtio_init_queue()
534 vq->hw = hw; in virtio_init_queue()
535 vq->vq_queue_index = queue_idx; in virtio_init_queue()
536 vq->vq_nentries = vq_size; in virtio_init_queue()
538 vq->vq_packed.used_wrap_counter = 1; in virtio_init_queue()
539 vq->vq_packed.cached_flags = VRING_PACKED_DESC_F_AVAIL; in virtio_init_queue()
540 vq->vq_packed.event_flags_shadow = 0; in virtio_init_queue()
542 vq->vq_packed.cached_flags |= VRING_DESC_F_WRITE; in virtio_init_queue()
549 vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_VRING_ALIGN); in virtio_init_queue()
551 size, vq->vq_ring_size); in virtio_init_queue()
553 mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size, in virtio_init_queue()
568 vq->vq_ring_mem = (uintptr_t)mz->addr; in virtio_init_queue()
570 vq->vq_ring_mem = mz->iova; in virtio_init_queue()
572 vq->vq_ring_virt_mem = mz->addr; in virtio_init_queue()
573 PMD_INIT_LOG(DEBUG, "vq->vq_ring_mem: 0x%" PRIx64, vq->vq_ring_mem); in virtio_init_queue()
574 PMD_INIT_LOG(DEBUG, "vq->vq_ring_virt_mem: %p", vq->vq_ring_virt_mem); in virtio_init_queue()
576 virtio_init_vring(vq); in virtio_init_queue()
596 sizeof(vq->sw_ring[0]); in virtio_init_queue()
614 vq->sw_ring = sw_ring; in virtio_init_queue()
615 rxvq = &vq->rxq; in virtio_init_queue()
620 txvq = &vq->txq; in virtio_init_queue()
629 cvq = &vq->cq; in virtio_init_queue()
642 vq->mbuf_addr_offset = offsetof(struct rte_mbuf, buf_addr); in virtio_init_queue()
644 vq->mbuf_addr_offset = offsetof(struct rte_mbuf, buf_iova); in virtio_init_queue()
678 if (VIRTIO_OPS(hw)->setup_queue(hw, vq) < 0) { in virtio_init_queue()
696 rte_free(vq); in virtio_init_queue()
706 struct virtqueue *vq; in virtio_free_queues() local
714 vq = hw->vqs[i]; in virtio_free_queues()
715 if (!vq) in virtio_free_queues()
720 rte_free(vq->rxq.fake_mbuf); in virtio_free_queues()
721 rte_free(vq->sw_ring); in virtio_free_queues()
722 rte_memzone_free(vq->rxq.mz); in virtio_free_queues()
724 rte_memzone_free(vq->txq.mz); in virtio_free_queues()
725 rte_memzone_free(vq->txq.virtio_net_hdr_mz); in virtio_free_queues()
727 rte_memzone_free(vq->cq.mz); in virtio_free_queues()
728 rte_memzone_free(vq->cq.virtio_net_hdr_mz); in virtio_free_queues()
731 rte_free(vq); in virtio_free_queues()
942 struct virtqueue *vq; in virtio_check_scatter_on_all_rx_queues() local
951 vq = hw->vqs[2 * qidx + VTNET_SQ_RQ_QUEUE_IDX]; in virtio_check_scatter_on_all_rx_queues()
952 if (vq == NULL) in virtio_check_scatter_on_all_rx_queues()
955 rxvq = &vq->rxq; in virtio_check_scatter_on_all_rx_queues()
1004 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); in virtio_dev_rx_queue_intr_enable() local
1006 virtqueue_enable_intr(vq); in virtio_dev_rx_queue_intr_enable()
1015 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); in virtio_dev_rx_queue_intr_disable() local
1017 virtqueue_disable_intr(vq); in virtio_dev_rx_queue_intr_disable()
1409 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); in virtio_get_monitor_addr() local
1412 if (vq == NULL) in virtio_get_monitor_addr()
1415 hw = vq->hw; in virtio_get_monitor_addr()
1418 desc = vq->vq_packed.ring.desc; in virtio_get_monitor_addr()
1419 pmc->addr = &desc[vq->vq_used_cons_idx].flags; in virtio_get_monitor_addr()
1420 if (vq->vq_packed.used_wrap_counter) in virtio_get_monitor_addr()
1427 pmc->size = sizeof(desc[vq->vq_used_cons_idx].flags); in virtio_get_monitor_addr()
1429 pmc->addr = &vq->vq_split.ring.used->idx; in virtio_get_monitor_addr()
1430 pmc->opaque[CLB_VAL_IDX] = vq->vq_used_cons_idx in virtio_get_monitor_addr()
1431 & (vq->vq_nentries - 1); in virtio_get_monitor_addr()
1432 pmc->opaque[CLB_MSK_IDX] = vq->vq_nentries - 1; in virtio_get_monitor_addr()
1434 pmc->size = sizeof(vq->vq_split.ring.used->idx); in virtio_get_monitor_addr()
2755 struct virtqueue *vq; in virtio_dev_start() local
2812 vq = virtnet_rxq_to_vq(dev->data->rx_queues[i]); in virtio_dev_start()
2814 virtqueue_rxvq_flush(vq); in virtio_dev_start()
2815 virtqueue_notify(vq); in virtio_dev_start()
2819 vq = virtnet_txq_to_vq(dev->data->tx_queues[i]); in virtio_dev_start()
2820 virtqueue_notify(vq); in virtio_dev_start()
2826 vq = virtnet_rxq_to_vq(dev->data->rx_queues[i]); in virtio_dev_start()
2827 VIRTQUEUE_DUMP(vq); in virtio_dev_start()
2831 vq = virtnet_txq_to_vq(dev->data->tx_queues[i]); in virtio_dev_start()
2832 VIRTQUEUE_DUMP(vq); in virtio_dev_start()
2850 struct virtqueue *vq; in virtio_dev_free_mbufs() local
2858 vq = hw->vqs[i]; in virtio_dev_free_mbufs()
2859 if (!vq) in virtio_dev_free_mbufs()
2873 VIRTQUEUE_DUMP(vq); in virtio_dev_free_mbufs()
2875 while ((buf = virtqueue_detach_unused(vq)) != NULL) { in virtio_dev_free_mbufs()
2883 VIRTQUEUE_DUMP(vq); in virtio_dev_free_mbufs()
2893 struct virtqueue *vq; in virtio_tx_completed_cleanup() local
2895 void (*xmit_cleanup)(struct virtqueue *vq, uint16_t nb_used); in virtio_tx_completed_cleanup()
2912 vq = hw->vqs[2 * qidx + VTNET_SQ_TQ_QUEUE_IDX]; in virtio_tx_completed_cleanup()
2913 if (vq != NULL) in virtio_tx_completed_cleanup()
2914 xmit_cleanup(vq, virtqueue_nused(vq)); in virtio_tx_completed_cleanup()