Lines Matching refs:vq
247 struct vhost_virtqueue *vq = dev->virtqueue[index]; in vhost_user_notify_queue_state() local
250 if (enable && vq->notif_enable != VIRTIO_UNINITIALIZED_NOTIF) in vhost_user_notify_queue_state()
251 vhost_enable_guest_notification(dev, vq, vq->notif_enable); in vhost_user_notify_queue_state()
393 struct vhost_virtqueue *vq; in vhost_user_set_features() local
395 vq = dev->virtqueue[--dev->nr_vring]; in vhost_user_set_features()
396 if (!vq) in vhost_user_set_features()
400 cleanup_vq(vq, 1); in vhost_user_set_features()
401 cleanup_vq_inflight(dev, vq); in vhost_user_set_features()
402 free_vq(dev, vq); in vhost_user_set_features()
423 struct vhost_virtqueue *vq = dev->virtqueue[ctx->msg.payload.state.index]; in vhost_user_set_vring_num() local
431 vq->size = ctx->msg.payload.state.num; in vhost_user_set_vring_num()
443 if (vq->size & (vq->size - 1)) { in vhost_user_set_vring_num()
445 dev->ifname, vq->size); in vhost_user_set_vring_num()
451 rte_free(vq->shadow_used_packed); in vhost_user_set_vring_num()
452 vq->shadow_used_packed = rte_malloc_socket(NULL, in vhost_user_set_vring_num()
453 vq->size * in vhost_user_set_vring_num()
455 RTE_CACHE_LINE_SIZE, vq->numa_node); in vhost_user_set_vring_num()
456 if (!vq->shadow_used_packed) { in vhost_user_set_vring_num()
464 rte_free(vq->shadow_used_split); in vhost_user_set_vring_num()
466 vq->shadow_used_split = rte_malloc_socket(NULL, in vhost_user_set_vring_num()
467 vq->size * sizeof(struct vring_used_elem), in vhost_user_set_vring_num()
468 RTE_CACHE_LINE_SIZE, vq->numa_node); in vhost_user_set_vring_num()
470 if (!vq->shadow_used_split) { in vhost_user_set_vring_num()
478 rte_free(vq->batch_copy_elems); in vhost_user_set_vring_num()
479 vq->batch_copy_elems = rte_malloc_socket(NULL, in vhost_user_set_vring_num()
480 vq->size * sizeof(struct batch_copy_elem), in vhost_user_set_vring_num()
481 RTE_CACHE_LINE_SIZE, vq->numa_node); in vhost_user_set_vring_num()
482 if (!vq->batch_copy_elems) { in vhost_user_set_vring_num()
501 struct vhost_virtqueue *vq; in numa_realloc() local
509 vq = dev->virtqueue[index]; in numa_realloc()
515 if (vq->ready) in numa_realloc()
518 ret = get_mempolicy(&node, NULL, 0, vq->desc, MPOL_F_NODE | MPOL_F_ADDR); in numa_realloc()
525 if (node == vq->numa_node) in numa_realloc()
528 vq = rte_realloc_socket(vq, sizeof(*vq), 0, node); in numa_realloc()
529 if (!vq) { in numa_realloc()
535 if (vq != dev->virtqueue[index]) { in numa_realloc()
538 dev->virtqueue[index] = vq; in numa_realloc()
545 sup = rte_realloc_socket(vq->shadow_used_packed, vq->size * sizeof(*sup), in numa_realloc()
552 vq->shadow_used_packed = sup; in numa_realloc()
556 sus = rte_realloc_socket(vq->shadow_used_split, vq->size * sizeof(*sus), in numa_realloc()
563 vq->shadow_used_split = sus; in numa_realloc()
566 bce = rte_realloc_socket(vq->batch_copy_elems, vq->size * sizeof(*bce), in numa_realloc()
573 vq->batch_copy_elems = bce; in numa_realloc()
575 if (vq->log_cache) { in numa_realloc()
578 lc = rte_realloc_socket(vq->log_cache, sizeof(*lc) * VHOST_LOG_CACHE_NR, 0, node); in numa_realloc()
584 vq->log_cache = lc; in numa_realloc()
587 if (vq->resubmit_inflight) { in numa_realloc()
590 ri = rte_realloc_socket(vq->resubmit_inflight, sizeof(*ri), 0, node); in numa_realloc()
596 vq->resubmit_inflight = ri; in numa_realloc()
612 vq->numa_node = node; in numa_realloc()
704 ring_addr_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, in ring_addr_to_vva() argument
710 vhost_user_iotlb_rd_lock(vq); in ring_addr_to_vva()
711 vva = vhost_iova_to_vva(dev, vq, ra, in ring_addr_to_vva()
713 vhost_user_iotlb_rd_unlock(vq); in ring_addr_to_vva()
722 log_addr_to_gpa(struct virtio_net *dev, struct vhost_virtqueue *vq) in log_addr_to_gpa() argument
726 vhost_user_iotlb_rd_lock(vq); in log_addr_to_gpa()
727 log_gpa = translate_log_addr(dev, vq, vq->ring_addrs.log_guest_addr); in log_addr_to_gpa()
728 vhost_user_iotlb_rd_unlock(vq); in log_addr_to_gpa()
736 struct vhost_virtqueue *vq = dev->virtqueue[vq_index]; in translate_ring_addresses() local
737 struct vhost_vring_addr *addr = &vq->ring_addrs; in translate_ring_addresses()
741 vq->log_guest_addr = in translate_ring_addresses()
742 log_addr_to_gpa(dev, vq); in translate_ring_addresses()
743 if (vq->log_guest_addr == 0) { in translate_ring_addresses()
751 len = sizeof(struct vring_packed_desc) * vq->size; in translate_ring_addresses()
752 vq->desc_packed = (struct vring_packed_desc *)(uintptr_t) in translate_ring_addresses()
753 ring_addr_to_vva(dev, vq, addr->desc_user_addr, &len); in translate_ring_addresses()
754 if (vq->desc_packed == NULL || in translate_ring_addresses()
756 vq->size) { in translate_ring_addresses()
763 vq = dev->virtqueue[vq_index]; in translate_ring_addresses()
764 addr = &vq->ring_addrs; in translate_ring_addresses()
767 vq->driver_event = (struct vring_packed_desc_event *) in translate_ring_addresses()
769 vq, addr->avail_user_addr, &len); in translate_ring_addresses()
770 if (vq->driver_event == NULL || in translate_ring_addresses()
778 vq->device_event = (struct vring_packed_desc_event *) in translate_ring_addresses()
780 vq, addr->used_user_addr, &len); in translate_ring_addresses()
781 if (vq->device_event == NULL || in translate_ring_addresses()
788 vq->access_ok = true; in translate_ring_addresses()
793 if (vq->desc && vq->avail && vq->used) in translate_ring_addresses()
796 len = sizeof(struct vring_desc) * vq->size; in translate_ring_addresses()
797 vq->desc = (struct vring_desc *)(uintptr_t)ring_addr_to_vva(dev, in translate_ring_addresses()
798 vq, addr->desc_user_addr, &len); in translate_ring_addresses()
799 if (vq->desc == 0 || len != sizeof(struct vring_desc) * vq->size) { in translate_ring_addresses()
805 vq = dev->virtqueue[vq_index]; in translate_ring_addresses()
806 addr = &vq->ring_addrs; in translate_ring_addresses()
808 len = sizeof(struct vring_avail) + sizeof(uint16_t) * vq->size; in translate_ring_addresses()
812 vq->avail = (struct vring_avail *)(uintptr_t)ring_addr_to_vva(dev, in translate_ring_addresses()
813 vq, addr->avail_user_addr, &len); in translate_ring_addresses()
814 if (vq->avail == 0 || len != expected_len) { in translate_ring_addresses()
820 sizeof(struct vring_used_elem) * vq->size; in translate_ring_addresses()
824 vq->used = (struct vring_used *)(uintptr_t)ring_addr_to_vva(dev, in translate_ring_addresses()
825 vq, addr->used_user_addr, &len); in translate_ring_addresses()
826 if (vq->used == 0 || len != expected_len) { in translate_ring_addresses()
831 if (vq->last_used_idx != vq->used->idx) { in translate_ring_addresses()
834 vq->last_used_idx, vq->used->idx); in translate_ring_addresses()
835 vq->last_used_idx = vq->used->idx; in translate_ring_addresses()
836 vq->last_avail_idx = vq->used->idx; in translate_ring_addresses()
841 vq->access_ok = true; in translate_ring_addresses()
843 VHOST_LOG_CONFIG(DEBUG, "(%s) mapped address desc: %p\n", dev->ifname, vq->desc); in translate_ring_addresses()
844 VHOST_LOG_CONFIG(DEBUG, "(%s) mapped address avail: %p\n", dev->ifname, vq->avail); in translate_ring_addresses()
845 VHOST_LOG_CONFIG(DEBUG, "(%s) mapped address used: %p\n", dev->ifname, vq->used); in translate_ring_addresses()
847 dev->ifname, vq->log_guest_addr); in translate_ring_addresses()
862 struct vhost_virtqueue *vq; in vhost_user_set_vring_addr() local
870 vq = dev->virtqueue[ctx->msg.payload.addr.index]; in vhost_user_set_vring_addr()
872 access_ok = vq->access_ok; in vhost_user_set_vring_addr()
878 memcpy(&vq->ring_addrs, addr, sizeof(*addr)); in vhost_user_set_vring_addr()
880 vring_invalidate(dev, vq); in vhost_user_set_vring_addr()
882 if ((vq->enabled && (dev->features & in vhost_user_set_vring_addr()
904 struct vhost_virtqueue *vq = dev->virtqueue[ctx->msg.payload.state.index]; in vhost_user_set_vring_base() local
912 vq->last_avail_idx = val & 0x7fff; in vhost_user_set_vring_base()
913 vq->avail_wrap_counter = !!(val & (0x1 << 15)); in vhost_user_set_vring_base()
919 vq->last_used_idx = vq->last_avail_idx; in vhost_user_set_vring_base()
920 vq->used_wrap_counter = vq->avail_wrap_counter; in vhost_user_set_vring_base()
922 vq->last_used_idx = ctx->msg.payload.state.num; in vhost_user_set_vring_base()
923 vq->last_avail_idx = ctx->msg.payload.state.num; in vhost_user_set_vring_base()
928 dev->ifname, ctx->msg.payload.state.index, vq->last_used_idx, in vhost_user_set_vring_base()
929 vq->last_avail_idx); in vhost_user_set_vring_base()
1376 struct vhost_virtqueue *vq = dev->virtqueue[i]; in vhost_user_set_mem_table() local
1378 if (!vq) in vhost_user_set_mem_table()
1381 if (vq->desc || vq->avail || vq->used) { in vhost_user_set_mem_table()
1387 vring_invalidate(dev, vq); in vhost_user_set_mem_table()
1422 vq_is_ready(struct virtio_net *dev, struct vhost_virtqueue *vq) in vq_is_ready() argument
1426 if (!vq) in vq_is_ready()
1430 rings_ok = vq->desc_packed && vq->driver_event && in vq_is_ready()
1431 vq->device_event; in vq_is_ready()
1433 rings_ok = vq->desc && vq->avail && vq->used; in vq_is_ready()
1436 vq->kickfd != VIRTIO_UNINITIALIZED_EVENTFD && in vq_is_ready()
1437 vq->callfd != VIRTIO_UNINITIALIZED_EVENTFD && in vq_is_ready()
1438 vq->enabled; in vq_is_ready()
1446 struct vhost_virtqueue *vq; in virtio_is_ready() local
1463 vq = dev->virtqueue[i]; in virtio_is_ready()
1465 if (!vq_is_ready(dev, vq)) in virtio_is_ready()
1644 struct vhost_virtqueue *vq; in vhost_user_set_inflight_fd() local
1719 vq = dev->virtqueue[i]; in vhost_user_set_inflight_fd()
1720 if (!vq) in vhost_user_set_inflight_fd()
1724 vq->inflight_packed = addr; in vhost_user_set_inflight_fd()
1725 vq->inflight_packed->desc_num = queue_size; in vhost_user_set_inflight_fd()
1727 vq->inflight_split = addr; in vhost_user_set_inflight_fd()
1728 vq->inflight_split->desc_num = queue_size; in vhost_user_set_inflight_fd()
1743 struct vhost_virtqueue *vq; in vhost_user_set_vring_call() local
1758 vq = dev->virtqueue[file.index]; in vhost_user_set_vring_call()
1760 if (vq->ready) { in vhost_user_set_vring_call()
1761 vq->ready = false; in vhost_user_set_vring_call()
1765 if (vq->callfd >= 0) in vhost_user_set_vring_call()
1766 close(vq->callfd); in vhost_user_set_vring_call()
1768 vq->callfd = file.fd; in vhost_user_set_vring_call()
1805 struct vhost_virtqueue *vq) in vhost_check_queue_inflights_split() argument
1809 struct vring_used *used = vq->used; in vhost_check_queue_inflights_split()
1820 if ((!vq->inflight_split)) in vhost_check_queue_inflights_split()
1823 if (!vq->inflight_split->version) { in vhost_check_queue_inflights_split()
1824 vq->inflight_split->version = INFLIGHT_VERSION; in vhost_check_queue_inflights_split()
1828 if (vq->resubmit_inflight) in vhost_check_queue_inflights_split()
1831 inflight_split = vq->inflight_split; in vhost_check_queue_inflights_split()
1832 vq->global_counter = 0; in vhost_check_queue_inflights_split()
1846 vq->last_avail_idx += resubmit_num; in vhost_check_queue_inflights_split()
1850 0, vq->numa_node); in vhost_check_queue_inflights_split()
1860 0, vq->numa_node); in vhost_check_queue_inflights_split()
1870 for (i = 0; i < vq->inflight_split->desc_num; i++) { in vhost_check_queue_inflights_split()
1871 if (vq->inflight_split->desc[i].inflight == 1) { in vhost_check_queue_inflights_split()
1885 vq->global_counter = resubmit->resubmit_list[0].counter + 1; in vhost_check_queue_inflights_split()
1886 vq->resubmit_inflight = resubmit; in vhost_check_queue_inflights_split()
1894 struct vhost_virtqueue *vq) in vhost_check_queue_inflights_packed() argument
1908 if ((!vq->inflight_packed)) in vhost_check_queue_inflights_packed()
1911 if (!vq->inflight_packed->version) { in vhost_check_queue_inflights_packed()
1912 vq->inflight_packed->version = INFLIGHT_VERSION; in vhost_check_queue_inflights_packed()
1916 if (vq->resubmit_inflight) in vhost_check_queue_inflights_packed()
1919 inflight_packed = vq->inflight_packed; in vhost_check_queue_inflights_packed()
1920 vq->global_counter = 0; in vhost_check_queue_inflights_packed()
1948 0, vq->numa_node); in vhost_check_queue_inflights_packed()
1958 0, vq->numa_node); in vhost_check_queue_inflights_packed()
1969 if (vq->inflight_packed->desc[i].inflight == 1) { in vhost_check_queue_inflights_packed()
1983 vq->global_counter = resubmit->resubmit_list[0].counter + 1; in vhost_check_queue_inflights_packed()
1984 vq->resubmit_inflight = resubmit; in vhost_check_queue_inflights_packed()
1997 struct vhost_virtqueue *vq; in vhost_user_set_vring_kick() local
2023 vq = dev->virtqueue[file.index]; in vhost_user_set_vring_kick()
2031 vq->enabled = true; in vhost_user_set_vring_kick()
2034 if (vq->ready) { in vhost_user_set_vring_kick()
2035 vq->ready = false; in vhost_user_set_vring_kick()
2039 if (vq->kickfd >= 0) in vhost_user_set_vring_kick()
2040 close(vq->kickfd); in vhost_user_set_vring_kick()
2041 vq->kickfd = file.fd; in vhost_user_set_vring_kick()
2044 if (vhost_check_queue_inflights_packed(dev, vq)) { in vhost_user_set_vring_kick()
2050 if (vhost_check_queue_inflights_split(dev, vq)) { in vhost_user_set_vring_kick()
2069 struct vhost_virtqueue *vq = dev->virtqueue[ctx->msg.payload.state.index]; in vhost_user_get_vring_base() local
2084 val = vq->last_avail_idx & 0x7fff; in vhost_user_get_vring_base()
2085 val |= vq->avail_wrap_counter << 15; in vhost_user_get_vring_base()
2088 ctx->msg.payload.state.num = vq->last_avail_idx; in vhost_user_get_vring_base()
2099 if (vq->kickfd >= 0) in vhost_user_get_vring_base()
2100 close(vq->kickfd); in vhost_user_get_vring_base()
2102 vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD; in vhost_user_get_vring_base()
2104 if (vq->callfd >= 0) in vhost_user_get_vring_base()
2105 close(vq->callfd); in vhost_user_get_vring_base()
2107 vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD; in vhost_user_get_vring_base()
2109 vq->signalled_used_valid = false; in vhost_user_get_vring_base()
2112 rte_free(vq->shadow_used_packed); in vhost_user_get_vring_base()
2113 vq->shadow_used_packed = NULL; in vhost_user_get_vring_base()
2115 rte_free(vq->shadow_used_split); in vhost_user_get_vring_base()
2116 vq->shadow_used_split = NULL; in vhost_user_get_vring_base()
2119 rte_free(vq->batch_copy_elems); in vhost_user_get_vring_base()
2120 vq->batch_copy_elems = NULL; in vhost_user_get_vring_base()
2122 rte_free(vq->log_cache); in vhost_user_get_vring_base()
2123 vq->log_cache = NULL; in vhost_user_get_vring_base()
2128 vhost_user_iotlb_flush_all(vq); in vhost_user_get_vring_base()
2130 vring_invalidate(dev, vq); in vhost_user_get_vring_base()
2268 struct vhost_virtqueue *vq = dev->virtqueue[i]; in vhost_user_set_log_base() local
2270 rte_free(vq->log_cache); in vhost_user_set_log_base()
2271 vq->log_cache = NULL; in vhost_user_set_log_base()
2272 vq->log_cache_nb_elem = 0; in vhost_user_set_log_base()
2273 vq->log_cache = rte_malloc_socket("vq log cache", in vhost_user_set_log_base()
2275 0, vq->numa_node); in vhost_user_set_log_base()
2280 if (!vq->log_cache) in vhost_user_set_log_base()
2396 is_vring_iotlb_split(struct vhost_virtqueue *vq, struct vhost_iotlb_msg *imsg) in is_vring_iotlb_split() argument
2404 ra = &vq->ring_addrs; in is_vring_iotlb_split()
2405 len = sizeof(struct vring_desc) * vq->size; in is_vring_iotlb_split()
2409 len = sizeof(struct vring_avail) + sizeof(uint16_t) * vq->size; in is_vring_iotlb_split()
2414 sizeof(struct vring_used_elem) * vq->size; in is_vring_iotlb_split()
2429 is_vring_iotlb_packed(struct vhost_virtqueue *vq, struct vhost_iotlb_msg *imsg) in is_vring_iotlb_packed() argument
2437 ra = &vq->ring_addrs; in is_vring_iotlb_packed()
2438 len = sizeof(struct vring_packed_desc) * vq->size; in is_vring_iotlb_packed()
2461 struct vhost_virtqueue *vq, in is_vring_iotlb() argument
2465 return is_vring_iotlb_packed(vq, imsg); in is_vring_iotlb()
2467 return is_vring_iotlb_split(vq, imsg); in is_vring_iotlb()
2488 struct vhost_virtqueue *vq = dev->virtqueue[i]; in vhost_user_iotlb_msg() local
2490 if (!vq) in vhost_user_iotlb_msg()
2493 vhost_user_iotlb_cache_insert(dev, vq, imsg->iova, vva, in vhost_user_iotlb_msg()
2496 if (is_vring_iotlb(dev, vq, imsg)) { in vhost_user_iotlb_msg()
2497 rte_spinlock_lock(&vq->access_lock); in vhost_user_iotlb_msg()
2499 rte_spinlock_unlock(&vq->access_lock); in vhost_user_iotlb_msg()
2505 struct vhost_virtqueue *vq = dev->virtqueue[i]; in vhost_user_iotlb_msg() local
2507 if (!vq) in vhost_user_iotlb_msg()
2510 vhost_user_iotlb_cache_remove(vq, imsg->iova, in vhost_user_iotlb_msg()
2513 if (is_vring_iotlb(dev, vq, imsg)) { in vhost_user_iotlb_msg()
2514 rte_spinlock_lock(&vq->access_lock); in vhost_user_iotlb_msg()
2515 vring_invalidate(dev, vq); in vhost_user_iotlb_msg()
2516 rte_spinlock_unlock(&vq->access_lock); in vhost_user_iotlb_msg()
2827 struct vhost_virtqueue *vq = dev->virtqueue[i]; in vhost_user_lock_all_queue_pairs() local
2829 if (vq) { in vhost_user_lock_all_queue_pairs()
2830 rte_spinlock_lock(&vq->access_lock); in vhost_user_lock_all_queue_pairs()
2844 struct vhost_virtqueue *vq = dev->virtqueue[i]; in vhost_user_unlock_all_queue_pairs() local
2846 if (vq) { in vhost_user_unlock_all_queue_pairs()
2847 rte_spinlock_unlock(&vq->access_lock); in vhost_user_unlock_all_queue_pairs()
3038 struct vhost_virtqueue *vq = dev->virtqueue[i]; in vhost_user_msg_handler() local
3039 bool cur_ready = vq_is_ready(dev, vq); in vhost_user_msg_handler()
3041 if (cur_ready != (vq && vq->ready)) { in vhost_user_msg_handler()
3042 vq->ready = cur_ready; in vhost_user_msg_handler()