Lines Matching refs:vq
29 __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, in __vhost_iova_to_vva() argument
39 vva = vhost_user_iotlb_cache_find(vq, iova, &tmp_size, perm); in __vhost_iova_to_vva()
45 if (!vhost_user_iotlb_pending_miss(vq, iova, perm)) { in __vhost_iova_to_vva()
53 vhost_user_iotlb_rd_unlock(vq); in __vhost_iova_to_vva()
55 vhost_user_iotlb_pending_insert(dev, vq, iova, perm); in __vhost_iova_to_vva()
59 vhost_user_iotlb_pending_remove(vq, iova, 1, perm); in __vhost_iova_to_vva()
62 vhost_user_iotlb_rd_lock(vq); in __vhost_iova_to_vva()
115 __vhost_log_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq, in __vhost_log_write_iova() argument
121 hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW); in __vhost_log_write_iova()
135 __vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq) in __vhost_log_cache_sync() argument
144 if (unlikely(!vq->log_cache)) in __vhost_log_cache_sync()
151 for (i = 0; i < vq->log_cache_nb_elem; i++) { in __vhost_log_cache_sync()
152 struct log_cache_entry *elem = vq->log_cache + i; in __vhost_log_cache_sync()
168 vq->log_cache_nb_elem = 0; in __vhost_log_cache_sync()
172 vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq, in vhost_log_cache_page() argument
179 if (unlikely(!vq->log_cache)) { in vhost_log_cache_page()
187 for (i = 0; i < vq->log_cache_nb_elem; i++) { in vhost_log_cache_page()
188 struct log_cache_entry *elem = vq->log_cache + i; in vhost_log_cache_page()
207 vq->log_cache[i].offset = offset; in vhost_log_cache_page()
208 vq->log_cache[i].val = (1UL << bit_nr); in vhost_log_cache_page()
209 vq->log_cache_nb_elem++; in vhost_log_cache_page()
213 __vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq, in __vhost_log_cache_write() argument
226 vhost_log_cache_page(dev, vq, page); in __vhost_log_cache_write()
232 __vhost_log_cache_write_iova(struct virtio_net *dev, struct vhost_virtqueue *vq, in __vhost_log_cache_write_iova() argument
238 hva = __vhost_iova_to_vva(dev, vq, iova, &map_len, VHOST_ACCESS_RW); in __vhost_log_cache_write_iova()
248 __vhost_log_cache_write(dev, vq, gpa, len); in __vhost_log_cache_write_iova()
252 vhost_alloc_copy_ind_table(struct virtio_net *dev, struct vhost_virtqueue *vq, in vhost_alloc_copy_ind_table() argument
259 idesc = rte_malloc_socket(__func__, desc_len, 0, vq->numa_node); in vhost_alloc_copy_ind_table()
267 src = vhost_iova_to_vva(dev, vq, desc_addr, &len, in vhost_alloc_copy_ind_table()
285 cleanup_vq(struct vhost_virtqueue *vq, int destroy) in cleanup_vq() argument
287 if ((vq->callfd >= 0) && (destroy != 0)) in cleanup_vq()
288 close(vq->callfd); in cleanup_vq()
289 if (vq->kickfd >= 0) in cleanup_vq()
290 close(vq->kickfd); in cleanup_vq()
294 cleanup_vq_inflight(struct virtio_net *dev, struct vhost_virtqueue *vq) in cleanup_vq_inflight() argument
301 if (vq->inflight_packed) in cleanup_vq_inflight()
302 vq->inflight_packed = NULL; in cleanup_vq_inflight()
304 if (vq->inflight_split) in cleanup_vq_inflight()
305 vq->inflight_split = NULL; in cleanup_vq_inflight()
308 if (vq->resubmit_inflight) { in cleanup_vq_inflight()
309 if (vq->resubmit_inflight->resubmit_list) { in cleanup_vq_inflight()
310 rte_free(vq->resubmit_inflight->resubmit_list); in cleanup_vq_inflight()
311 vq->resubmit_inflight->resubmit_list = NULL; in cleanup_vq_inflight()
313 rte_free(vq->resubmit_inflight); in cleanup_vq_inflight()
314 vq->resubmit_inflight = NULL; in cleanup_vq_inflight()
336 vhost_free_async_mem(struct vhost_virtqueue *vq) in vhost_free_async_mem() argument
338 if (!vq->async) in vhost_free_async_mem()
341 rte_free(vq->async->pkts_info); in vhost_free_async_mem()
342 rte_free(vq->async->pkts_cmpl_flag); in vhost_free_async_mem()
344 rte_free(vq->async->buffers_packed); in vhost_free_async_mem()
345 vq->async->buffers_packed = NULL; in vhost_free_async_mem()
346 rte_free(vq->async->descs_split); in vhost_free_async_mem()
347 vq->async->descs_split = NULL; in vhost_free_async_mem()
349 rte_free(vq->async); in vhost_free_async_mem()
350 vq->async = NULL; in vhost_free_async_mem()
354 free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq) in free_vq() argument
357 rte_free(vq->shadow_used_packed); in free_vq()
359 rte_free(vq->shadow_used_split); in free_vq()
361 vhost_free_async_mem(vq); in free_vq()
362 rte_free(vq->batch_copy_elems); in free_vq()
363 rte_mempool_free(vq->iotlb_pool); in free_vq()
364 rte_free(vq->log_cache); in free_vq()
365 rte_free(vq); in free_vq()
383 log_translate(struct virtio_net *dev, struct vhost_virtqueue *vq) in log_translate() argument
385 if (likely(!(vq->ring_addrs.flags & (1 << VHOST_VRING_F_LOG)))) in log_translate()
388 vq->log_guest_addr = translate_log_addr(dev, vq, in log_translate()
389 vq->ring_addrs.log_guest_addr); in log_translate()
390 if (vq->log_guest_addr == 0) in log_translate()
404 translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq, in translate_log_addr() argument
412 hva = vhost_iova_to_vva(dev, vq, log_addr, in translate_log_addr()
434 vring_translate_split(struct virtio_net *dev, struct vhost_virtqueue *vq) in vring_translate_split() argument
438 req_size = sizeof(struct vring_desc) * vq->size; in vring_translate_split()
440 vq->desc = (struct vring_desc *)(uintptr_t)vhost_iova_to_vva(dev, vq, in vring_translate_split()
441 vq->ring_addrs.desc_user_addr, in vring_translate_split()
443 if (!vq->desc || size != req_size) in vring_translate_split()
447 req_size += sizeof(uint16_t) * vq->size; in vring_translate_split()
451 vq->avail = (struct vring_avail *)(uintptr_t)vhost_iova_to_vva(dev, vq, in vring_translate_split()
452 vq->ring_addrs.avail_user_addr, in vring_translate_split()
454 if (!vq->avail || size != req_size) in vring_translate_split()
458 req_size += sizeof(struct vring_used_elem) * vq->size; in vring_translate_split()
462 vq->used = (struct vring_used *)(uintptr_t)vhost_iova_to_vva(dev, vq, in vring_translate_split()
463 vq->ring_addrs.used_user_addr, in vring_translate_split()
465 if (!vq->used || size != req_size) in vring_translate_split()
473 vring_translate_packed(struct virtio_net *dev, struct vhost_virtqueue *vq) in vring_translate_packed() argument
477 req_size = sizeof(struct vring_packed_desc) * vq->size; in vring_translate_packed()
479 vq->desc_packed = (struct vring_packed_desc *)(uintptr_t) in vring_translate_packed()
480 vhost_iova_to_vva(dev, vq, vq->ring_addrs.desc_user_addr, in vring_translate_packed()
482 if (!vq->desc_packed || size != req_size) in vring_translate_packed()
487 vq->driver_event = (struct vring_packed_desc_event *)(uintptr_t) in vring_translate_packed()
488 vhost_iova_to_vva(dev, vq, vq->ring_addrs.avail_user_addr, in vring_translate_packed()
490 if (!vq->driver_event || size != req_size) in vring_translate_packed()
495 vq->device_event = (struct vring_packed_desc_event *)(uintptr_t) in vring_translate_packed()
496 vhost_iova_to_vva(dev, vq, vq->ring_addrs.used_user_addr, in vring_translate_packed()
498 if (!vq->device_event || size != req_size) in vring_translate_packed()
505 vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq) in vring_translate() argument
512 if (vring_translate_packed(dev, vq) < 0) in vring_translate()
515 if (vring_translate_split(dev, vq) < 0) in vring_translate()
519 if (log_translate(dev, vq) < 0) in vring_translate()
522 vq->access_ok = true; in vring_translate()
528 vring_invalidate(struct virtio_net *dev, struct vhost_virtqueue *vq) in vring_invalidate() argument
531 vhost_user_iotlb_wr_lock(vq); in vring_invalidate()
533 vq->access_ok = false; in vring_invalidate()
534 vq->desc = NULL; in vring_invalidate()
535 vq->avail = NULL; in vring_invalidate()
536 vq->used = NULL; in vring_invalidate()
537 vq->log_guest_addr = 0; in vring_invalidate()
540 vhost_user_iotlb_wr_unlock(vq); in vring_invalidate()
546 struct vhost_virtqueue *vq; in init_vring_queue() local
555 vq = dev->virtqueue[vring_idx]; in init_vring_queue()
556 if (!vq) { in init_vring_queue()
562 memset(vq, 0, sizeof(struct vhost_virtqueue)); in init_vring_queue()
564 vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD; in init_vring_queue()
565 vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD; in init_vring_queue()
566 vq->notif_enable = VIRTIO_UNINITIALIZED_NOTIF; in init_vring_queue()
569 if (get_mempolicy(&numa_node, NULL, 0, vq, MPOL_F_NODE | MPOL_F_ADDR)) { in init_vring_queue()
575 vq->numa_node = numa_node; in init_vring_queue()
583 struct vhost_virtqueue *vq; in reset_vring_queue() local
593 vq = dev->virtqueue[vring_idx]; in reset_vring_queue()
594 if (!vq) { in reset_vring_queue()
600 callfd = vq->callfd; in reset_vring_queue()
602 vq->callfd = callfd; in reset_vring_queue()
608 struct vhost_virtqueue *vq; in alloc_vring_queue() local
616 vq = rte_zmalloc(NULL, sizeof(struct vhost_virtqueue), 0); in alloc_vring_queue()
617 if (vq == NULL) { in alloc_vring_queue()
623 dev->virtqueue[i] = vq; in alloc_vring_queue()
625 rte_spinlock_init(&vq->access_lock); in alloc_vring_queue()
626 vq->avail_wrap_counter = 1; in alloc_vring_queue()
627 vq->used_wrap_counter = 1; in alloc_vring_queue()
628 vq->signalled_used_valid = false; in alloc_vring_queue()
935 struct vhost_virtqueue *vq; in rte_vhost_get_vhost_vring() local
944 vq = dev->virtqueue[vring_idx]; in rte_vhost_get_vhost_vring()
945 if (!vq) in rte_vhost_get_vhost_vring()
949 vring->desc_packed = vq->desc_packed; in rte_vhost_get_vhost_vring()
950 vring->driver_event = vq->driver_event; in rte_vhost_get_vhost_vring()
951 vring->device_event = vq->device_event; in rte_vhost_get_vhost_vring()
953 vring->desc = vq->desc; in rte_vhost_get_vhost_vring()
954 vring->avail = vq->avail; in rte_vhost_get_vhost_vring()
955 vring->used = vq->used; in rte_vhost_get_vhost_vring()
957 vring->log_guest_addr = vq->log_guest_addr; in rte_vhost_get_vhost_vring()
959 vring->callfd = vq->callfd; in rte_vhost_get_vhost_vring()
960 vring->kickfd = vq->kickfd; in rte_vhost_get_vhost_vring()
961 vring->size = vq->size; in rte_vhost_get_vhost_vring()
971 struct vhost_virtqueue *vq; in rte_vhost_get_vhost_ring_inflight() local
980 vq = dev->virtqueue[vring_idx]; in rte_vhost_get_vhost_ring_inflight()
981 if (unlikely(!vq)) in rte_vhost_get_vhost_ring_inflight()
985 if (unlikely(!vq->inflight_packed)) in rte_vhost_get_vhost_ring_inflight()
988 vring->inflight_packed = vq->inflight_packed; in rte_vhost_get_vhost_ring_inflight()
990 if (unlikely(!vq->inflight_split)) in rte_vhost_get_vhost_ring_inflight()
993 vring->inflight_split = vq->inflight_split; in rte_vhost_get_vhost_ring_inflight()
996 vring->resubmit_inflight = vq->resubmit_inflight; in rte_vhost_get_vhost_ring_inflight()
1005 struct vhost_virtqueue *vq; in rte_vhost_set_inflight_desc_split() local
1022 vq = dev->virtqueue[vring_idx]; in rte_vhost_set_inflight_desc_split()
1023 if (unlikely(!vq)) in rte_vhost_set_inflight_desc_split()
1026 if (unlikely(!vq->inflight_split)) in rte_vhost_set_inflight_desc_split()
1029 if (unlikely(idx >= vq->size)) in rte_vhost_set_inflight_desc_split()
1032 vq->inflight_split->desc[idx].counter = vq->global_counter++; in rte_vhost_set_inflight_desc_split()
1033 vq->inflight_split->desc[idx].inflight = 1; in rte_vhost_set_inflight_desc_split()
1044 struct vhost_virtqueue *vq; in rte_vhost_set_inflight_desc_packed() local
1062 vq = dev->virtqueue[vring_idx]; in rte_vhost_set_inflight_desc_packed()
1063 if (unlikely(!vq)) in rte_vhost_set_inflight_desc_packed()
1066 inflight_info = vq->inflight_packed; in rte_vhost_set_inflight_desc_packed()
1070 if (unlikely(head >= vq->size)) in rte_vhost_set_inflight_desc_packed()
1073 desc = vq->desc_packed; in rte_vhost_set_inflight_desc_packed()
1075 if (unlikely(old_free_head >= vq->size)) in rte_vhost_set_inflight_desc_packed()
1082 inflight_info->desc[old_free_head].counter = vq->global_counter++; in rte_vhost_set_inflight_desc_packed()
1086 while (head != ((last + 1) % vq->size)) { in rte_vhost_set_inflight_desc_packed()
1096 head = (head + 1) % vq->size; in rte_vhost_set_inflight_desc_packed()
1110 struct vhost_virtqueue *vq; in rte_vhost_clr_inflight_desc_split() local
1126 vq = dev->virtqueue[vring_idx]; in rte_vhost_clr_inflight_desc_split()
1127 if (unlikely(!vq)) in rte_vhost_clr_inflight_desc_split()
1130 if (unlikely(!vq->inflight_split)) in rte_vhost_clr_inflight_desc_split()
1133 if (unlikely(idx >= vq->size)) in rte_vhost_clr_inflight_desc_split()
1138 vq->inflight_split->desc[idx].inflight = 0; in rte_vhost_clr_inflight_desc_split()
1142 vq->inflight_split->used_idx = last_used_idx; in rte_vhost_clr_inflight_desc_split()
1152 struct vhost_virtqueue *vq; in rte_vhost_clr_inflight_desc_packed() local
1168 vq = dev->virtqueue[vring_idx]; in rte_vhost_clr_inflight_desc_packed()
1169 if (unlikely(!vq)) in rte_vhost_clr_inflight_desc_packed()
1172 inflight_info = vq->inflight_packed; in rte_vhost_clr_inflight_desc_packed()
1176 if (unlikely(head >= vq->size)) in rte_vhost_clr_inflight_desc_packed()
1197 struct vhost_virtqueue *vq; in rte_vhost_set_last_inflight_io_split() local
1213 vq = dev->virtqueue[vring_idx]; in rte_vhost_set_last_inflight_io_split()
1214 if (unlikely(!vq)) in rte_vhost_set_last_inflight_io_split()
1217 if (unlikely(!vq->inflight_split)) in rte_vhost_set_last_inflight_io_split()
1220 if (unlikely(idx >= vq->size)) in rte_vhost_set_last_inflight_io_split()
1223 vq->inflight_split->last_inflight_io = idx; in rte_vhost_set_last_inflight_io_split()
1233 struct vhost_virtqueue *vq; in rte_vhost_set_last_inflight_io_packed() local
1250 vq = dev->virtqueue[vring_idx]; in rte_vhost_set_last_inflight_io_packed()
1251 if (unlikely(!vq)) in rte_vhost_set_last_inflight_io_packed()
1254 inflight_info = vq->inflight_packed; in rte_vhost_set_last_inflight_io_packed()
1258 if (unlikely(head >= vq->size)) in rte_vhost_set_last_inflight_io_packed()
1262 if (unlikely(last >= vq->size)) in rte_vhost_set_last_inflight_io_packed()
1281 struct vhost_virtqueue *vq; in rte_vhost_vring_call() local
1290 vq = dev->virtqueue[vring_idx]; in rte_vhost_vring_call()
1291 if (!vq) in rte_vhost_vring_call()
1294 rte_spinlock_lock(&vq->access_lock); in rte_vhost_vring_call()
1297 vhost_vring_call_packed(dev, vq); in rte_vhost_vring_call()
1299 vhost_vring_call_split(dev, vq); in rte_vhost_vring_call()
1301 rte_spinlock_unlock(&vq->access_lock); in rte_vhost_vring_call()
1310 struct vhost_virtqueue *vq; in rte_vhost_avail_entries() local
1320 vq = dev->virtqueue[queue_id]; in rte_vhost_avail_entries()
1321 if (!vq) in rte_vhost_avail_entries()
1324 rte_spinlock_lock(&vq->access_lock); in rte_vhost_avail_entries()
1326 if (unlikely(!vq->enabled || vq->avail == NULL)) in rte_vhost_avail_entries()
1329 ret = *(volatile uint16_t *)&vq->avail->idx - vq->last_used_idx; in rte_vhost_avail_entries()
1332 rte_spinlock_unlock(&vq->access_lock); in rte_vhost_avail_entries()
1338 struct vhost_virtqueue *vq, int enable) in vhost_enable_notify_split() argument
1340 if (vq->used == NULL) in vhost_enable_notify_split()
1345 vq->used->flags &= ~VRING_USED_F_NO_NOTIFY; in vhost_enable_notify_split()
1347 vq->used->flags |= VRING_USED_F_NO_NOTIFY; in vhost_enable_notify_split()
1350 vhost_avail_event(vq) = vq->last_avail_idx; in vhost_enable_notify_split()
1357 struct vhost_virtqueue *vq, int enable) in vhost_enable_notify_packed() argument
1361 if (vq->device_event == NULL) in vhost_enable_notify_packed()
1365 vq->device_event->flags = VRING_EVENT_F_DISABLE; in vhost_enable_notify_packed()
1372 vq->device_event->off_wrap = vq->last_avail_idx | in vhost_enable_notify_packed()
1373 vq->avail_wrap_counter << 15; in vhost_enable_notify_packed()
1378 vq->device_event->flags = flags; in vhost_enable_notify_packed()
1384 struct vhost_virtqueue *vq, int enable) in vhost_enable_guest_notification() argument
1390 if (!vq->ready) in vhost_enable_guest_notification()
1394 return vhost_enable_notify_packed(dev, vq, enable); in vhost_enable_guest_notification()
1396 return vhost_enable_notify_split(dev, vq, enable); in vhost_enable_guest_notification()
1403 struct vhost_virtqueue *vq; in rte_vhost_enable_guest_notification() local
1412 vq = dev->virtqueue[queue_id]; in rte_vhost_enable_guest_notification()
1413 if (!vq) in rte_vhost_enable_guest_notification()
1416 rte_spinlock_lock(&vq->access_lock); in rte_vhost_enable_guest_notification()
1418 vq->notif_enable = enable; in rte_vhost_enable_guest_notification()
1419 ret = vhost_enable_guest_notification(dev, vq, enable); in rte_vhost_enable_guest_notification()
1421 rte_spinlock_unlock(&vq->access_lock); in rte_vhost_enable_guest_notification()
1442 struct vhost_virtqueue *vq; in rte_vhost_log_used_vring() local
1450 vq = dev->virtqueue[vring_idx]; in rte_vhost_log_used_vring()
1451 if (!vq) in rte_vhost_log_used_vring()
1454 vhost_log_used_vring(dev, vq, offset, len); in rte_vhost_log_used_vring()
1461 struct vhost_virtqueue *vq; in rte_vhost_rx_queue_count() local
1474 vq = dev->virtqueue[qid]; in rte_vhost_rx_queue_count()
1475 if (vq == NULL) in rte_vhost_rx_queue_count()
1478 rte_spinlock_lock(&vq->access_lock); in rte_vhost_rx_queue_count()
1480 if (unlikely(!vq->enabled || vq->avail == NULL)) in rte_vhost_rx_queue_count()
1483 ret = *((volatile uint16_t *)&vq->avail->idx) - vq->last_avail_idx; in rte_vhost_rx_queue_count()
1486 rte_spinlock_unlock(&vq->access_lock); in rte_vhost_rx_queue_count()
1520 struct vhost_virtqueue *vq; in rte_vhost_get_vring_base() local
1529 vq = dev->virtqueue[queue_id]; in rte_vhost_get_vring_base()
1530 if (!vq) in rte_vhost_get_vring_base()
1534 *last_avail_idx = (vq->avail_wrap_counter << 15) | in rte_vhost_get_vring_base()
1535 vq->last_avail_idx; in rte_vhost_get_vring_base()
1536 *last_used_idx = (vq->used_wrap_counter << 15) | in rte_vhost_get_vring_base()
1537 vq->last_used_idx; in rte_vhost_get_vring_base()
1539 *last_avail_idx = vq->last_avail_idx; in rte_vhost_get_vring_base()
1540 *last_used_idx = vq->last_used_idx; in rte_vhost_get_vring_base()
1550 struct vhost_virtqueue *vq; in rte_vhost_set_vring_base() local
1559 vq = dev->virtqueue[queue_id]; in rte_vhost_set_vring_base()
1560 if (!vq) in rte_vhost_set_vring_base()
1564 vq->last_avail_idx = last_avail_idx & 0x7fff; in rte_vhost_set_vring_base()
1565 vq->avail_wrap_counter = !!(last_avail_idx & (1 << 15)); in rte_vhost_set_vring_base()
1566 vq->last_used_idx = last_used_idx & 0x7fff; in rte_vhost_set_vring_base()
1567 vq->used_wrap_counter = !!(last_used_idx & (1 << 15)); in rte_vhost_set_vring_base()
1569 vq->last_avail_idx = last_avail_idx; in rte_vhost_set_vring_base()
1570 vq->last_used_idx = last_used_idx; in rte_vhost_set_vring_base()
1583 struct vhost_virtqueue *vq; in rte_vhost_get_vring_base_from_inflight() local
1592 vq = dev->virtqueue[queue_id]; in rte_vhost_get_vring_base_from_inflight()
1593 if (!vq) in rte_vhost_get_vring_base_from_inflight()
1599 inflight_info = vq->inflight_packed; in rte_vhost_get_vring_base_from_inflight()
1628 struct vhost_virtqueue *vq = dev->virtqueue[queue_id]; in async_channel_register() local
1630 int node = vq->numa_node; in async_channel_register()
1632 if (unlikely(vq->async)) { in async_channel_register()
1646 async->pkts_info = rte_malloc_socket(NULL, vq->size * sizeof(struct async_inflight_info), in async_channel_register()
1654 async->pkts_cmpl_flag = rte_zmalloc_socket(NULL, vq->size * sizeof(bool), in async_channel_register()
1664 vq->size * sizeof(struct vring_used_elem_packed), in async_channel_register()
1673 vq->size * sizeof(struct vring_used_elem), in async_channel_register()
1682 vq->async = async; in async_channel_register()
1696 struct vhost_virtqueue *vq; in rte_vhost_async_channel_register() local
1706 vq = dev->virtqueue[queue_id]; in rte_vhost_async_channel_register()
1708 if (unlikely(vq == NULL || !dev->async_copy)) in rte_vhost_async_channel_register()
1711 rte_spinlock_lock(&vq->access_lock); in rte_vhost_async_channel_register()
1713 rte_spinlock_unlock(&vq->access_lock); in rte_vhost_async_channel_register()
1721 struct vhost_virtqueue *vq; in rte_vhost_async_channel_register_thread_unsafe() local
1730 vq = dev->virtqueue[queue_id]; in rte_vhost_async_channel_register_thread_unsafe()
1732 if (unlikely(vq == NULL || !dev->async_copy)) in rte_vhost_async_channel_register_thread_unsafe()
1741 struct vhost_virtqueue *vq; in rte_vhost_async_channel_unregister() local
1751 vq = dev->virtqueue[queue_id]; in rte_vhost_async_channel_unregister()
1753 if (vq == NULL) in rte_vhost_async_channel_unregister()
1758 if (!vq->async) in rte_vhost_async_channel_unregister()
1761 if (!rte_spinlock_trylock(&vq->access_lock)) { in rte_vhost_async_channel_unregister()
1767 if (vq->async->pkts_inflight_n) { in rte_vhost_async_channel_unregister()
1775 vhost_free_async_mem(vq); in rte_vhost_async_channel_unregister()
1777 rte_spinlock_unlock(&vq->access_lock); in rte_vhost_async_channel_unregister()
1785 struct vhost_virtqueue *vq; in rte_vhost_async_channel_unregister_thread_unsafe() local
1794 vq = dev->virtqueue[queue_id]; in rte_vhost_async_channel_unregister_thread_unsafe()
1796 if (vq == NULL) in rte_vhost_async_channel_unregister_thread_unsafe()
1799 if (!vq->async) in rte_vhost_async_channel_unregister_thread_unsafe()
1802 if (vq->async->pkts_inflight_n) { in rte_vhost_async_channel_unregister_thread_unsafe()
1809 vhost_free_async_mem(vq); in rte_vhost_async_channel_unregister_thread_unsafe()
1879 struct vhost_virtqueue *vq; in rte_vhost_async_get_inflight() local
1889 vq = dev->virtqueue[queue_id]; in rte_vhost_async_get_inflight()
1891 if (vq == NULL) in rte_vhost_async_get_inflight()
1894 if (!vq->async) in rte_vhost_async_get_inflight()
1897 if (!rte_spinlock_trylock(&vq->access_lock)) { in rte_vhost_async_get_inflight()
1904 ret = vq->async->pkts_inflight_n; in rte_vhost_async_get_inflight()
1905 rte_spinlock_unlock(&vq->access_lock); in rte_vhost_async_get_inflight()
1913 struct vhost_virtqueue *vq; in rte_vhost_async_get_inflight_thread_unsafe() local
1923 vq = dev->virtqueue[queue_id]; in rte_vhost_async_get_inflight_thread_unsafe()
1925 if (vq == NULL) in rte_vhost_async_get_inflight_thread_unsafe()
1928 if (!vq->async) in rte_vhost_async_get_inflight_thread_unsafe()
1931 ret = vq->async->pkts_inflight_n; in rte_vhost_async_get_inflight_thread_unsafe()
1941 struct vhost_virtqueue *vq; in rte_vhost_get_monitor_addr() local
1948 vq = dev->virtqueue[queue_id]; in rte_vhost_get_monitor_addr()
1949 if (vq == NULL) in rte_vhost_get_monitor_addr()
1954 desc = vq->desc_packed; in rte_vhost_get_monitor_addr()
1955 pmc->addr = &desc[vq->last_avail_idx].flags; in rte_vhost_get_monitor_addr()
1956 if (vq->avail_wrap_counter) in rte_vhost_get_monitor_addr()
1961 pmc->size = sizeof(desc[vq->last_avail_idx].flags); in rte_vhost_get_monitor_addr()
1964 pmc->addr = &vq->avail->idx; in rte_vhost_get_monitor_addr()
1965 pmc->val = vq->last_avail_idx & (vq->size - 1); in rte_vhost_get_monitor_addr()
1966 pmc->mask = vq->size - 1; in rte_vhost_get_monitor_addr()
1967 pmc->size = sizeof(vq->avail->idx); in rte_vhost_get_monitor_addr()