Lines Matching refs:vq
49 #define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num]) argument
50 #define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num]) argument
53 static void vhost_disable_cross_endian(struct vhost_virtqueue *vq) in vhost_disable_cross_endian() argument
55 vq->user_be = !virtio_legacy_is_little_endian(); in vhost_disable_cross_endian()
58 static void vhost_enable_cross_endian_big(struct vhost_virtqueue *vq) in vhost_enable_cross_endian_big() argument
60 vq->user_be = true; in vhost_enable_cross_endian_big()
63 static void vhost_enable_cross_endian_little(struct vhost_virtqueue *vq) in vhost_enable_cross_endian_little() argument
65 vq->user_be = false; in vhost_enable_cross_endian_little()
68 static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp) in vhost_set_vring_endian() argument
72 if (vq->private_data) in vhost_set_vring_endian()
83 vhost_enable_cross_endian_big(vq); in vhost_set_vring_endian()
85 vhost_enable_cross_endian_little(vq); in vhost_set_vring_endian()
90 static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx, in vhost_get_vring_endian() argument
95 .num = vq->user_be in vhost_get_vring_endian()
104 static void vhost_init_is_le(struct vhost_virtqueue *vq) in vhost_init_is_le() argument
111 vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) || !vq->user_be; in vhost_init_is_le()
114 static void vhost_disable_cross_endian(struct vhost_virtqueue *vq) in vhost_disable_cross_endian() argument
118 static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp) in vhost_set_vring_endian() argument
123 static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx, in vhost_get_vring_endian() argument
129 static void vhost_init_is_le(struct vhost_virtqueue *vq) in vhost_init_is_le() argument
131 vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) in vhost_init_is_le()
136 static void vhost_reset_is_le(struct vhost_virtqueue *vq) in vhost_reset_is_le() argument
138 vhost_init_is_le(vq); in vhost_reset_is_le()
191 struct vhost_virtqueue *vq) in vhost_poll_init() argument
198 poll->vq = vq; in vhost_poll_init()
249 bool vhost_vq_work_queue(struct vhost_virtqueue *vq, struct vhost_work *work) in vhost_vq_work_queue() argument
255 worker = rcu_dereference(vq->worker); in vhost_vq_work_queue()
310 bool vhost_vq_has_work(struct vhost_virtqueue *vq) in vhost_vq_has_work() argument
316 worker = rcu_dereference(vq->worker); in vhost_vq_has_work()
327 vhost_vq_work_queue(poll->vq, &poll->work); in vhost_poll_queue()
331 static void __vhost_vq_meta_reset(struct vhost_virtqueue *vq) in __vhost_vq_meta_reset() argument
336 vq->meta_iotlb[j] = NULL; in __vhost_vq_meta_reset()
353 bool vhost_vq_is_setup(struct vhost_virtqueue *vq) in vhost_vq_is_setup() argument
355 return vq->avail && vq->desc && vq->used && vhost_vq_access_ok(vq); in vhost_vq_is_setup()
360 struct vhost_virtqueue *vq) in vhost_vq_reset() argument
362 vq->num = 1; in vhost_vq_reset()
363 vq->desc = NULL; in vhost_vq_reset()
364 vq->avail = NULL; in vhost_vq_reset()
365 vq->used = NULL; in vhost_vq_reset()
366 vq->last_avail_idx = 0; in vhost_vq_reset()
367 vq->avail_idx = 0; in vhost_vq_reset()
368 vq->last_used_idx = 0; in vhost_vq_reset()
369 vq->signalled_used = 0; in vhost_vq_reset()
370 vq->signalled_used_valid = false; in vhost_vq_reset()
371 vq->used_flags = 0; in vhost_vq_reset()
372 vq->log_used = false; in vhost_vq_reset()
373 vq->log_addr = -1ull; in vhost_vq_reset()
374 vq->private_data = NULL; in vhost_vq_reset()
375 vq->acked_features = 0; in vhost_vq_reset()
376 vq->acked_backend_features = 0; in vhost_vq_reset()
377 vq->log_base = NULL; in vhost_vq_reset()
378 vq->error_ctx = NULL; in vhost_vq_reset()
379 vq->kick = NULL; in vhost_vq_reset()
380 vq->log_ctx = NULL; in vhost_vq_reset()
381 vhost_disable_cross_endian(vq); in vhost_vq_reset()
382 vhost_reset_is_le(vq); in vhost_vq_reset()
383 vq->busyloop_timeout = 0; in vhost_vq_reset()
384 vq->umem = NULL; in vhost_vq_reset()
385 vq->iotlb = NULL; in vhost_vq_reset()
386 rcu_assign_pointer(vq->worker, NULL); in vhost_vq_reset()
387 vhost_vring_call_reset(&vq->call_ctx); in vhost_vq_reset()
388 __vhost_vq_meta_reset(vq); in vhost_vq_reset()
420 struct vhost_virtqueue *vq; in vhost_worker_killed() local
427 vq = dev->vqs[i]; in vhost_worker_killed()
429 mutex_lock(&vq->mutex); in vhost_worker_killed()
431 rcu_dereference_check(vq->worker, in vhost_worker_killed()
432 lockdep_is_held(&vq->mutex))) { in vhost_worker_killed()
433 rcu_assign_pointer(vq->worker, NULL); in vhost_worker_killed()
436 mutex_unlock(&vq->mutex); in vhost_worker_killed()
450 static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq) in vhost_vq_free_iovecs() argument
452 kfree(vq->indirect); in vhost_vq_free_iovecs()
453 vq->indirect = NULL; in vhost_vq_free_iovecs()
454 kfree(vq->log); in vhost_vq_free_iovecs()
455 vq->log = NULL; in vhost_vq_free_iovecs()
456 kfree(vq->heads); in vhost_vq_free_iovecs()
457 vq->heads = NULL; in vhost_vq_free_iovecs()
463 struct vhost_virtqueue *vq; in vhost_dev_alloc_iovecs() local
467 vq = dev->vqs[i]; in vhost_dev_alloc_iovecs()
468 vq->indirect = kmalloc_array(UIO_MAXIOV, in vhost_dev_alloc_iovecs()
469 sizeof(*vq->indirect), in vhost_dev_alloc_iovecs()
471 vq->log = kmalloc_array(dev->iov_limit, sizeof(*vq->log), in vhost_dev_alloc_iovecs()
473 vq->heads = kmalloc_array(dev->iov_limit, sizeof(*vq->heads), in vhost_dev_alloc_iovecs()
475 if (!vq->indirect || !vq->log || !vq->heads) in vhost_dev_alloc_iovecs()
494 bool vhost_exceeds_weight(struct vhost_virtqueue *vq, in vhost_exceeds_weight() argument
497 struct vhost_dev *dev = vq->dev; in vhost_exceeds_weight()
501 vhost_poll_queue(&vq->poll); in vhost_exceeds_weight()
509 static size_t vhost_get_avail_size(struct vhost_virtqueue *vq, in vhost_get_avail_size() argument
513 vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; in vhost_get_avail_size()
515 return size_add(struct_size(vq->avail, ring, num), event); in vhost_get_avail_size()
518 static size_t vhost_get_used_size(struct vhost_virtqueue *vq, in vhost_get_used_size() argument
522 vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; in vhost_get_used_size()
524 return size_add(struct_size(vq->used, ring, num), event); in vhost_get_used_size()
527 static size_t vhost_get_desc_size(struct vhost_virtqueue *vq, in vhost_get_desc_size() argument
530 return sizeof(*vq->desc) * num; in vhost_get_desc_size()
540 struct vhost_virtqueue *vq; in vhost_dev_init() local
562 vq = dev->vqs[i]; in vhost_dev_init()
563 vq->log = NULL; in vhost_dev_init()
564 vq->indirect = NULL; in vhost_dev_init()
565 vq->heads = NULL; in vhost_dev_init()
566 vq->dev = dev; in vhost_dev_init()
567 mutex_init(&vq->mutex); in vhost_dev_init()
568 vhost_vq_reset(dev, vq); in vhost_dev_init()
569 if (vq->handle_kick) in vhost_dev_init()
570 vhost_poll_init(&vq->poll, vq->handle_kick, in vhost_dev_init()
571 EPOLLIN, dev, vq); in vhost_dev_init()
694 static void __vhost_vq_attach_worker(struct vhost_virtqueue *vq, in __vhost_vq_attach_worker() argument
705 mutex_lock(&vq->mutex); in __vhost_vq_attach_worker()
707 old_worker = rcu_dereference_check(vq->worker, in __vhost_vq_attach_worker()
708 lockdep_is_held(&vq->mutex)); in __vhost_vq_attach_worker()
709 rcu_assign_pointer(vq->worker, worker); in __vhost_vq_attach_worker()
713 mutex_unlock(&vq->mutex); in __vhost_vq_attach_worker()
717 mutex_unlock(&vq->mutex); in __vhost_vq_attach_worker()
737 mutex_lock(&vq->mutex); in __vhost_vq_attach_worker()
738 if (!vhost_vq_get_backend(vq) && !vq->kick) { in __vhost_vq_attach_worker()
739 mutex_unlock(&vq->mutex); in __vhost_vq_attach_worker()
752 mutex_unlock(&vq->mutex); in __vhost_vq_attach_worker()
763 static int vhost_vq_attach_worker(struct vhost_virtqueue *vq, in vhost_vq_attach_worker() argument
767 struct vhost_dev *dev = vq->dev; in vhost_vq_attach_worker()
777 __vhost_vq_attach_worker(vq, worker); in vhost_vq_attach_worker()
824 struct vhost_virtqueue **vq, u32 *id) in vhost_get_vq_from_user() argument
839 *vq = dev->vqs[idx]; in vhost_get_vq_from_user()
851 struct vhost_virtqueue *vq; in vhost_worker_ioctl() local
884 ret = vhost_get_vq_from_user(dev, argp, &vq, &idx); in vhost_worker_ioctl()
895 ret = vhost_vq_attach_worker(vq, &ring_worker); in vhost_worker_ioctl()
898 worker = rcu_dereference_check(vq->worker, in vhost_worker_ioctl()
1107 static inline void __user *vhost_vq_meta_fetch(struct vhost_virtqueue *vq, in vhost_vq_meta_fetch() argument
1111 const struct vhost_iotlb_map *map = vq->meta_iotlb[type]; in vhost_vq_meta_fetch()
1145 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
1148 static int vhost_copy_to_user(struct vhost_virtqueue *vq, void __user *to, in vhost_copy_to_user() argument
1153 if (!vq->iotlb) in vhost_copy_to_user()
1162 void __user *uaddr = vhost_vq_meta_fetch(vq, in vhost_copy_to_user()
1169 ret = translate_desc(vq, (u64)(uintptr_t)to, size, vq->iotlb_iov, in vhost_copy_to_user()
1170 ARRAY_SIZE(vq->iotlb_iov), in vhost_copy_to_user()
1174 iov_iter_init(&t, ITER_DEST, vq->iotlb_iov, ret, size); in vhost_copy_to_user()
1183 static int vhost_copy_from_user(struct vhost_virtqueue *vq, void *to, in vhost_copy_from_user() argument
1188 if (!vq->iotlb) in vhost_copy_from_user()
1196 void __user *uaddr = vhost_vq_meta_fetch(vq, in vhost_copy_from_user()
1204 ret = translate_desc(vq, (u64)(uintptr_t)from, size, vq->iotlb_iov, in vhost_copy_from_user()
1205 ARRAY_SIZE(vq->iotlb_iov), in vhost_copy_from_user()
1208 vq_err(vq, "IOTLB translation failure: uaddr " in vhost_copy_from_user()
1213 iov_iter_init(&f, ITER_SOURCE, vq->iotlb_iov, ret, size); in vhost_copy_from_user()
1223 static void __user *__vhost_get_user_slow(struct vhost_virtqueue *vq, in __vhost_get_user_slow() argument
1229 ret = translate_desc(vq, (u64)(uintptr_t)addr, size, vq->iotlb_iov, in __vhost_get_user_slow()
1230 ARRAY_SIZE(vq->iotlb_iov), in __vhost_get_user_slow()
1233 vq_err(vq, "IOTLB translation failure: uaddr " in __vhost_get_user_slow()
1239 if (ret != 1 || vq->iotlb_iov[0].iov_len != size) { in __vhost_get_user_slow()
1240 vq_err(vq, "Non atomic userspace memory access: uaddr " in __vhost_get_user_slow()
1246 return vq->iotlb_iov[0].iov_base; in __vhost_get_user_slow()
1254 static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq, in __vhost_get_user() argument
1258 void __user *uaddr = vhost_vq_meta_fetch(vq, in __vhost_get_user()
1263 return __vhost_get_user_slow(vq, addr, size, type); in __vhost_get_user()
1266 #define vhost_put_user(vq, x, ptr) \ argument
1269 if (!vq->iotlb) { \
1273 (__typeof__(ptr)) __vhost_get_user(vq, ptr, \
1283 static inline int vhost_put_avail_event(struct vhost_virtqueue *vq) in vhost_put_avail_event() argument
1285 return vhost_put_user(vq, cpu_to_vhost16(vq, vq->avail_idx), in vhost_put_avail_event()
1286 vhost_avail_event(vq)); in vhost_put_avail_event()
1289 static inline int vhost_put_used(struct vhost_virtqueue *vq, in vhost_put_used() argument
1293 return vhost_copy_to_user(vq, vq->used->ring + idx, head, in vhost_put_used()
1297 static inline int vhost_put_used_flags(struct vhost_virtqueue *vq) in vhost_put_used_flags() argument
1300 return vhost_put_user(vq, cpu_to_vhost16(vq, vq->used_flags), in vhost_put_used_flags()
1301 &vq->used->flags); in vhost_put_used_flags()
1304 static inline int vhost_put_used_idx(struct vhost_virtqueue *vq) in vhost_put_used_idx() argument
1307 return vhost_put_user(vq, cpu_to_vhost16(vq, vq->last_used_idx), in vhost_put_used_idx()
1308 &vq->used->idx); in vhost_put_used_idx()
1311 #define vhost_get_user(vq, x, ptr, type) \ argument
1314 if (!vq->iotlb) { \
1318 (__typeof__(ptr)) __vhost_get_user(vq, ptr, \
1329 #define vhost_get_avail(vq, x, ptr) \ argument
1330 vhost_get_user(vq, x, ptr, VHOST_ADDR_AVAIL)
1332 #define vhost_get_used(vq, x, ptr) \ argument
1333 vhost_get_user(vq, x, ptr, VHOST_ADDR_USED)
1349 static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq) in vhost_get_avail_idx() argument
1354 r = vhost_get_avail(vq, idx, &vq->avail->idx); in vhost_get_avail_idx()
1356 vq_err(vq, "Failed to access available index at %p (%d)\n", in vhost_get_avail_idx()
1357 &vq->avail->idx, r); in vhost_get_avail_idx()
1362 vq->avail_idx = vhost16_to_cpu(vq, idx); in vhost_get_avail_idx()
1363 if (unlikely((u16)(vq->avail_idx - vq->last_avail_idx) > vq->num)) { in vhost_get_avail_idx()
1364 vq_err(vq, "Invalid available index change from %u to %u", in vhost_get_avail_idx()
1365 vq->last_avail_idx, vq->avail_idx); in vhost_get_avail_idx()
1370 if (vq->avail_idx == vq->last_avail_idx) in vhost_get_avail_idx()
1381 static inline int vhost_get_avail_head(struct vhost_virtqueue *vq, in vhost_get_avail_head() argument
1384 return vhost_get_avail(vq, *head, in vhost_get_avail_head()
1385 &vq->avail->ring[idx & (vq->num - 1)]); in vhost_get_avail_head()
1388 static inline int vhost_get_avail_flags(struct vhost_virtqueue *vq, in vhost_get_avail_flags() argument
1391 return vhost_get_avail(vq, *flags, &vq->avail->flags); in vhost_get_avail_flags()
1394 static inline int vhost_get_used_event(struct vhost_virtqueue *vq, in vhost_get_used_event() argument
1397 return vhost_get_avail(vq, *event, vhost_used_event(vq)); in vhost_get_used_event()
1400 static inline int vhost_get_used_idx(struct vhost_virtqueue *vq, in vhost_get_used_idx() argument
1403 return vhost_get_used(vq, *idx, &vq->used->idx); in vhost_get_used_idx()
1406 static inline int vhost_get_desc(struct vhost_virtqueue *vq, in vhost_get_desc() argument
1409 return vhost_copy_from_user(vq, desc, vq->desc + idx, sizeof(*desc)); in vhost_get_desc()
1424 vhost_poll_queue(&node->vq->poll); in vhost_iotlb_notify_vq()
1646 static int vhost_iotlb_miss(struct vhost_virtqueue *vq, u64 iova, int access) in vhost_iotlb_miss() argument
1648 struct vhost_dev *dev = vq->dev; in vhost_iotlb_miss()
1651 bool v2 = vhost_backend_has_feature(vq, VHOST_BACKEND_F_IOTLB_MSG_V2); in vhost_iotlb_miss()
1653 node = vhost_new_msg(vq, v2 ? VHOST_IOTLB_MSG_V2 : VHOST_IOTLB_MSG); in vhost_iotlb_miss()
1673 static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num, in vq_access_ok() argument
1681 if (vq->iotlb) in vq_access_ok()
1684 return access_ok(desc, vhost_get_desc_size(vq, num)) && in vq_access_ok()
1685 access_ok(avail, vhost_get_avail_size(vq, num)) && in vq_access_ok()
1686 access_ok(used, vhost_get_used_size(vq, num)); in vq_access_ok()
1689 static void vhost_vq_meta_update(struct vhost_virtqueue *vq, in vhost_vq_meta_update() argument
1697 vq->meta_iotlb[type] = map; in vhost_vq_meta_update()
1700 static bool iotlb_access_ok(struct vhost_virtqueue *vq, in iotlb_access_ok() argument
1704 struct vhost_iotlb *umem = vq->iotlb; in iotlb_access_ok()
1707 if (vhost_vq_meta_fetch(vq, addr, len, type)) in iotlb_access_ok()
1713 vhost_iotlb_miss(vq, addr, access); in iotlb_access_ok()
1725 vhost_vq_meta_update(vq, map, type); in iotlb_access_ok()
1734 int vq_meta_prefetch(struct vhost_virtqueue *vq) in vq_meta_prefetch() argument
1736 unsigned int num = vq->num; in vq_meta_prefetch()
1738 if (!vq->iotlb) in vq_meta_prefetch()
1741 return iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->desc, in vq_meta_prefetch()
1742 vhost_get_desc_size(vq, num), VHOST_ADDR_DESC) && in vq_meta_prefetch()
1743 iotlb_access_ok(vq, VHOST_MAP_RO, (u64)(uintptr_t)vq->avail, in vq_meta_prefetch()
1744 vhost_get_avail_size(vq, num), in vq_meta_prefetch()
1746 iotlb_access_ok(vq, VHOST_MAP_WO, (u64)(uintptr_t)vq->used, in vq_meta_prefetch()
1747 vhost_get_used_size(vq, num), VHOST_ADDR_USED); in vq_meta_prefetch()
1759 static bool vq_log_used_access_ok(struct vhost_virtqueue *vq, in vq_log_used_access_ok() argument
1766 if (vq->iotlb) in vq_log_used_access_ok()
1770 vhost_get_used_size(vq, vq->num)); in vq_log_used_access_ok()
1775 static bool vq_log_access_ok(struct vhost_virtqueue *vq, in vq_log_access_ok() argument
1778 return vq_memory_access_ok(log_base, vq->umem, in vq_log_access_ok()
1779 vhost_has_feature(vq, VHOST_F_LOG_ALL)) && in vq_log_access_ok()
1780 vq_log_used_access_ok(vq, log_base, vq->log_used, vq->log_addr); in vq_log_access_ok()
1785 bool vhost_vq_access_ok(struct vhost_virtqueue *vq) in vhost_vq_access_ok() argument
1787 if (!vq_log_access_ok(vq, vq->log_base)) in vhost_vq_access_ok()
1790 return vq_access_ok(vq, vq->num, vq->desc, vq->avail, vq->used); in vhost_vq_access_ok()
1862 struct vhost_virtqueue *vq, in vhost_vring_set_num() argument
1869 if (vq->private_data) in vhost_vring_set_num()
1877 vq->num = s.num; in vhost_vring_set_num()
1883 struct vhost_virtqueue *vq, in vhost_vring_set_addr() argument
1901 BUILD_BUG_ON(__alignof__ *vq->avail > VRING_AVAIL_ALIGN_SIZE); in vhost_vring_set_addr()
1902 BUILD_BUG_ON(__alignof__ *vq->used > VRING_USED_ALIGN_SIZE); in vhost_vring_set_addr()
1911 if (vq->private_data) { in vhost_vring_set_addr()
1912 if (!vq_access_ok(vq, vq->num, in vhost_vring_set_addr()
1919 if (!vq_log_used_access_ok(vq, vq->log_base, in vhost_vring_set_addr()
1925 vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG)); in vhost_vring_set_addr()
1926 vq->desc = (void __user *)(unsigned long)a.desc_user_addr; in vhost_vring_set_addr()
1927 vq->avail = (void __user *)(unsigned long)a.avail_user_addr; in vhost_vring_set_addr()
1928 vq->log_addr = a.log_guest_addr; in vhost_vring_set_addr()
1929 vq->used = (void __user *)(unsigned long)a.used_user_addr; in vhost_vring_set_addr()
1935 struct vhost_virtqueue *vq, in vhost_vring_set_num_addr() argument
1941 mutex_lock(&vq->mutex); in vhost_vring_set_num_addr()
1945 r = vhost_vring_set_num(d, vq, argp); in vhost_vring_set_num_addr()
1948 r = vhost_vring_set_addr(d, vq, argp); in vhost_vring_set_num_addr()
1954 mutex_unlock(&vq->mutex); in vhost_vring_set_num_addr()
1963 struct vhost_virtqueue *vq; in vhost_vring_ioctl() local
1969 r = vhost_get_vq_from_user(d, argp, &vq, &idx); in vhost_vring_ioctl()
1975 return vhost_vring_set_num_addr(d, vq, ioctl, argp); in vhost_vring_ioctl()
1978 mutex_lock(&vq->mutex); in vhost_vring_ioctl()
1984 if (vq->private_data) { in vhost_vring_ioctl()
1992 if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) { in vhost_vring_ioctl()
1993 vq->last_avail_idx = s.num & 0xffff; in vhost_vring_ioctl()
1994 vq->last_used_idx = (s.num >> 16) & 0xffff; in vhost_vring_ioctl()
2000 vq->last_avail_idx = s.num; in vhost_vring_ioctl()
2003 vq->avail_idx = vq->last_avail_idx; in vhost_vring_ioctl()
2007 if (vhost_has_feature(vq, VIRTIO_F_RING_PACKED)) in vhost_vring_ioctl()
2008 s.num = (u32)vq->last_avail_idx | ((u32)vq->last_used_idx << 16); in vhost_vring_ioctl()
2010 s.num = vq->last_avail_idx; in vhost_vring_ioctl()
2024 if (eventfp != vq->kick) { in vhost_vring_ioctl()
2025 pollstop = (filep = vq->kick) != NULL; in vhost_vring_ioctl()
2026 pollstart = (vq->kick = eventfp) != NULL; in vhost_vring_ioctl()
2041 swap(ctx, vq->call_ctx.ctx); in vhost_vring_ioctl()
2053 swap(ctx, vq->error_ctx); in vhost_vring_ioctl()
2056 r = vhost_set_vring_endian(vq, argp); in vhost_vring_ioctl()
2059 r = vhost_get_vring_endian(vq, idx, argp); in vhost_vring_ioctl()
2066 vq->busyloop_timeout = s.num; in vhost_vring_ioctl()
2070 s.num = vq->busyloop_timeout; in vhost_vring_ioctl()
2078 if (pollstop && vq->handle_kick) in vhost_vring_ioctl()
2079 vhost_poll_stop(&vq->poll); in vhost_vring_ioctl()
2086 if (pollstart && vq->handle_kick) in vhost_vring_ioctl()
2087 r = vhost_poll_start(&vq->poll, vq->kick); in vhost_vring_ioctl()
2089 mutex_unlock(&vq->mutex); in vhost_vring_ioctl()
2091 if (pollstop && vq->handle_kick) in vhost_vring_ioctl()
2092 vhost_dev_flush(vq->poll.dev); in vhost_vring_ioctl()
2110 struct vhost_virtqueue *vq = d->vqs[i]; in vhost_init_device_iotlb() local
2112 mutex_lock(&vq->mutex); in vhost_init_device_iotlb()
2113 vq->iotlb = niotlb; in vhost_init_device_iotlb()
2114 __vhost_vq_meta_reset(vq); in vhost_init_device_iotlb()
2115 mutex_unlock(&vq->mutex); in vhost_init_device_iotlb()
2157 struct vhost_virtqueue *vq; in vhost_dev_ioctl() local
2159 vq = d->vqs[i]; in vhost_dev_ioctl()
2160 mutex_lock(&vq->mutex); in vhost_dev_ioctl()
2162 if (vq->private_data && !vq_log_access_ok(vq, base)) in vhost_dev_ioctl()
2165 vq->log_base = base; in vhost_dev_ioctl()
2166 mutex_unlock(&vq->mutex); in vhost_dev_ioctl()
2245 static int log_write_hva(struct vhost_virtqueue *vq, u64 hva, u64 len) in log_write_hva() argument
2247 struct vhost_iotlb *umem = vq->umem; in log_write_hva()
2265 r = log_write(vq->log_base, in log_write_hva()
2284 static int log_used(struct vhost_virtqueue *vq, u64 used_offset, u64 len) in log_used() argument
2286 struct iovec *iov = vq->log_iov; in log_used()
2289 if (!vq->iotlb) in log_used()
2290 return log_write(vq->log_base, vq->log_addr + used_offset, len); in log_used()
2292 ret = translate_desc(vq, (uintptr_t)vq->used + used_offset, in log_used()
2298 ret = log_write_hva(vq, (uintptr_t)iov[i].iov_base, in log_used()
2307 int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, in vhost_log_write() argument
2315 if (vq->iotlb) { in vhost_log_write()
2317 r = log_write_hva(vq, (uintptr_t)iov[i].iov_base, in vhost_log_write()
2327 r = log_write(vq->log_base, log[i].addr, l); in vhost_log_write()
2332 if (vq->log_ctx) in vhost_log_write()
2333 eventfd_signal(vq->log_ctx); in vhost_log_write()
2343 static int vhost_update_used_flags(struct vhost_virtqueue *vq) in vhost_update_used_flags() argument
2346 if (vhost_put_used_flags(vq)) in vhost_update_used_flags()
2348 if (unlikely(vq->log_used)) { in vhost_update_used_flags()
2352 used = &vq->used->flags; in vhost_update_used_flags()
2353 log_used(vq, (used - (void __user *)vq->used), in vhost_update_used_flags()
2354 sizeof vq->used->flags); in vhost_update_used_flags()
2355 if (vq->log_ctx) in vhost_update_used_flags()
2356 eventfd_signal(vq->log_ctx); in vhost_update_used_flags()
2361 static int vhost_update_avail_event(struct vhost_virtqueue *vq) in vhost_update_avail_event() argument
2363 if (vhost_put_avail_event(vq)) in vhost_update_avail_event()
2365 if (unlikely(vq->log_used)) { in vhost_update_avail_event()
2370 used = vhost_avail_event(vq); in vhost_update_avail_event()
2371 log_used(vq, (used - (void __user *)vq->used), in vhost_update_avail_event()
2372 sizeof *vhost_avail_event(vq)); in vhost_update_avail_event()
2373 if (vq->log_ctx) in vhost_update_avail_event()
2374 eventfd_signal(vq->log_ctx); in vhost_update_avail_event()
2379 int vhost_vq_init_access(struct vhost_virtqueue *vq) in vhost_vq_init_access() argument
2383 bool is_le = vq->is_le; in vhost_vq_init_access()
2385 if (!vq->private_data) in vhost_vq_init_access()
2388 vhost_init_is_le(vq); in vhost_vq_init_access()
2390 r = vhost_update_used_flags(vq); in vhost_vq_init_access()
2393 vq->signalled_used_valid = false; in vhost_vq_init_access()
2394 if (!vq->iotlb && in vhost_vq_init_access()
2395 !access_ok(&vq->used->idx, sizeof vq->used->idx)) { in vhost_vq_init_access()
2399 r = vhost_get_used_idx(vq, &last_used_idx); in vhost_vq_init_access()
2401 vq_err(vq, "Can't access used idx at %p\n", in vhost_vq_init_access()
2402 &vq->used->idx); in vhost_vq_init_access()
2405 vq->last_used_idx = vhost16_to_cpu(vq, last_used_idx); in vhost_vq_init_access()
2409 vq->is_le = is_le; in vhost_vq_init_access()
2414 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len, in translate_desc() argument
2418 struct vhost_dev *dev = vq->dev; in translate_desc()
2455 vhost_iotlb_miss(vq, addr, access); in translate_desc()
2462 static unsigned next_desc(struct vhost_virtqueue *vq, struct vring_desc *desc) in next_desc() argument
2467 if (!(desc->flags & cpu_to_vhost16(vq, VRING_DESC_F_NEXT))) in next_desc()
2471 next = vhost16_to_cpu(vq, READ_ONCE(desc->next)); in next_desc()
2475 static int get_indirect(struct vhost_virtqueue *vq, in get_indirect() argument
2483 u32 len = vhost32_to_cpu(vq, indirect->len); in get_indirect()
2489 vq_err(vq, "Invalid length in indirect descriptor: " in get_indirect()
2496 ret = translate_desc(vq, vhost64_to_cpu(vq, indirect->addr), len, vq->indirect, in get_indirect()
2500 vq_err(vq, "Translation failure %d in indirect.\n", ret); in get_indirect()
2503 iov_iter_init(&from, ITER_SOURCE, vq->indirect, ret, len); in get_indirect()
2508 vq_err(vq, "Indirect buffer length too big: %d\n", in get_indirect()
2516 vq_err(vq, "Loop detected: last one at %u " in get_indirect()
2522 vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n", in get_indirect()
2523 i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc); in get_indirect()
2526 if (unlikely(desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT))) { in get_indirect()
2527 vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n", in get_indirect()
2528 i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc); in get_indirect()
2532 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) in get_indirect()
2537 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr), in get_indirect()
2538 vhost32_to_cpu(vq, desc.len), iov + iov_count, in get_indirect()
2542 vq_err(vq, "Translation failure %d indirect idx %d\n", in get_indirect()
2550 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr); in get_indirect()
2551 log[*log_num].len = vhost32_to_cpu(vq, desc.len); in get_indirect()
2558 vq_err(vq, "Indirect descriptor " in get_indirect()
2564 } while ((i = next_desc(vq, &desc)) != -1); in get_indirect()
2576 int vhost_get_vq_desc(struct vhost_virtqueue *vq, in vhost_get_vq_desc() argument
2583 u16 last_avail_idx = vq->last_avail_idx; in vhost_get_vq_desc()
2587 if (vq->avail_idx == vq->last_avail_idx) { in vhost_get_vq_desc()
2588 ret = vhost_get_avail_idx(vq); in vhost_get_vq_desc()
2593 return vq->num; in vhost_get_vq_desc()
2598 if (unlikely(vhost_get_avail_head(vq, &ring_head, last_avail_idx))) { in vhost_get_vq_desc()
2599 vq_err(vq, "Failed to read head: idx %d address %p\n", in vhost_get_vq_desc()
2601 &vq->avail->ring[last_avail_idx % vq->num]); in vhost_get_vq_desc()
2605 head = vhost16_to_cpu(vq, ring_head); in vhost_get_vq_desc()
2608 if (unlikely(head >= vq->num)) { in vhost_get_vq_desc()
2609 vq_err(vq, "Guest says index %u > %u is available", in vhost_get_vq_desc()
2610 head, vq->num); in vhost_get_vq_desc()
2622 if (unlikely(i >= vq->num)) { in vhost_get_vq_desc()
2623 vq_err(vq, "Desc index is %u > %u, head = %u", in vhost_get_vq_desc()
2624 i, vq->num, head); in vhost_get_vq_desc()
2627 if (unlikely(++found > vq->num)) { in vhost_get_vq_desc()
2628 vq_err(vq, "Loop detected: last one at %u " in vhost_get_vq_desc()
2630 i, vq->num, head); in vhost_get_vq_desc()
2633 ret = vhost_get_desc(vq, &desc, i); in vhost_get_vq_desc()
2635 vq_err(vq, "Failed to get descriptor: idx %d addr %p\n", in vhost_get_vq_desc()
2636 i, vq->desc + i); in vhost_get_vq_desc()
2639 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT)) { in vhost_get_vq_desc()
2640 ret = get_indirect(vq, iov, iov_size, in vhost_get_vq_desc()
2645 vq_err(vq, "Failure detected " in vhost_get_vq_desc()
2652 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) in vhost_get_vq_desc()
2656 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr), in vhost_get_vq_desc()
2657 vhost32_to_cpu(vq, desc.len), iov + iov_count, in vhost_get_vq_desc()
2661 vq_err(vq, "Translation failure %d descriptor idx %d\n", in vhost_get_vq_desc()
2670 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr); in vhost_get_vq_desc()
2671 log[*log_num].len = vhost32_to_cpu(vq, desc.len); in vhost_get_vq_desc()
2678 vq_err(vq, "Descriptor has out after in: " in vhost_get_vq_desc()
2684 } while ((i = next_desc(vq, &desc)) != -1); in vhost_get_vq_desc()
2687 vq->last_avail_idx++; in vhost_get_vq_desc()
2691 BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY)); in vhost_get_vq_desc()
2697 void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n) in vhost_discard_vq_desc() argument
2699 vq->last_avail_idx -= n; in vhost_discard_vq_desc()
2705 int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len) in vhost_add_used() argument
2708 cpu_to_vhost32(vq, head), in vhost_add_used()
2709 cpu_to_vhost32(vq, len) in vhost_add_used()
2712 return vhost_add_used_n(vq, &heads, 1); in vhost_add_used()
2716 static int __vhost_add_used_n(struct vhost_virtqueue *vq, in __vhost_add_used_n() argument
2724 start = vq->last_used_idx & (vq->num - 1); in __vhost_add_used_n()
2725 used = vq->used->ring + start; in __vhost_add_used_n()
2726 if (vhost_put_used(vq, heads, start, count)) { in __vhost_add_used_n()
2727 vq_err(vq, "Failed to write used"); in __vhost_add_used_n()
2730 if (unlikely(vq->log_used)) { in __vhost_add_used_n()
2734 log_used(vq, ((void __user *)used - (void __user *)vq->used), in __vhost_add_used_n()
2737 old = vq->last_used_idx; in __vhost_add_used_n()
2738 new = (vq->last_used_idx += count); in __vhost_add_used_n()
2743 if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old))) in __vhost_add_used_n()
2744 vq->signalled_used_valid = false; in __vhost_add_used_n()
2750 int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads, in vhost_add_used_n() argument
2755 start = vq->last_used_idx & (vq->num - 1); in vhost_add_used_n()
2756 n = vq->num - start; in vhost_add_used_n()
2758 r = __vhost_add_used_n(vq, heads, n); in vhost_add_used_n()
2764 r = __vhost_add_used_n(vq, heads, count); in vhost_add_used_n()
2768 if (vhost_put_used_idx(vq)) { in vhost_add_used_n()
2769 vq_err(vq, "Failed to increment used idx"); in vhost_add_used_n()
2772 if (unlikely(vq->log_used)) { in vhost_add_used_n()
2776 log_used(vq, offsetof(struct vring_used, idx), in vhost_add_used_n()
2777 sizeof vq->used->idx); in vhost_add_used_n()
2778 if (vq->log_ctx) in vhost_add_used_n()
2779 eventfd_signal(vq->log_ctx); in vhost_add_used_n()
2785 static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) in vhost_notify() argument
2795 if (vhost_has_feature(vq, VIRTIO_F_NOTIFY_ON_EMPTY) && in vhost_notify()
2796 unlikely(vq->avail_idx == vq->last_avail_idx)) in vhost_notify()
2799 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { in vhost_notify()
2801 if (vhost_get_avail_flags(vq, &flags)) { in vhost_notify()
2802 vq_err(vq, "Failed to get flags"); in vhost_notify()
2805 return !(flags & cpu_to_vhost16(vq, VRING_AVAIL_F_NO_INTERRUPT)); in vhost_notify()
2807 old = vq->signalled_used; in vhost_notify()
2808 v = vq->signalled_used_valid; in vhost_notify()
2809 new = vq->signalled_used = vq->last_used_idx; in vhost_notify()
2810 vq->signalled_used_valid = true; in vhost_notify()
2815 if (vhost_get_used_event(vq, &event)) { in vhost_notify()
2816 vq_err(vq, "Failed to get used event idx"); in vhost_notify()
2819 return vring_need_event(vhost16_to_cpu(vq, event), new, old); in vhost_notify()
2823 void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq) in vhost_signal() argument
2826 if (vq->call_ctx.ctx && vhost_notify(dev, vq)) in vhost_signal()
2827 eventfd_signal(vq->call_ctx.ctx); in vhost_signal()
2833 struct vhost_virtqueue *vq, in vhost_add_used_and_signal() argument
2836 vhost_add_used(vq, head, len); in vhost_add_used_and_signal()
2837 vhost_signal(dev, vq); in vhost_add_used_and_signal()
2843 struct vhost_virtqueue *vq, in vhost_add_used_and_signal_n() argument
2846 vhost_add_used_n(vq, heads, count); in vhost_add_used_and_signal_n()
2847 vhost_signal(dev, vq); in vhost_add_used_and_signal_n()
2852 bool vhost_vq_avail_empty(struct vhost_dev *dev, struct vhost_virtqueue *vq) in vhost_vq_avail_empty() argument
2856 if (vq->avail_idx != vq->last_avail_idx) in vhost_vq_avail_empty()
2859 r = vhost_get_avail_idx(vq); in vhost_vq_avail_empty()
2867 bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) in vhost_enable_notify() argument
2871 if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY)) in vhost_enable_notify()
2873 vq->used_flags &= ~VRING_USED_F_NO_NOTIFY; in vhost_enable_notify()
2874 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { in vhost_enable_notify()
2875 r = vhost_update_used_flags(vq); in vhost_enable_notify()
2877 vq_err(vq, "Failed to enable notification at %p: %d\n", in vhost_enable_notify()
2878 &vq->used->flags, r); in vhost_enable_notify()
2882 r = vhost_update_avail_event(vq); in vhost_enable_notify()
2884 vq_err(vq, "Failed to update avail event index at %p: %d\n", in vhost_enable_notify()
2885 vhost_avail_event(vq), r); in vhost_enable_notify()
2893 r = vhost_get_avail_idx(vq); in vhost_enable_notify()
2903 void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) in vhost_disable_notify() argument
2907 if (vq->used_flags & VRING_USED_F_NO_NOTIFY) in vhost_disable_notify()
2909 vq->used_flags |= VRING_USED_F_NO_NOTIFY; in vhost_disable_notify()
2910 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { in vhost_disable_notify()
2911 r = vhost_update_used_flags(vq); in vhost_disable_notify()
2913 vq_err(vq, "Failed to disable notification at %p: %d\n", in vhost_disable_notify()
2914 &vq->used->flags, r); in vhost_disable_notify()
2920 struct vhost_msg_node *vhost_new_msg(struct vhost_virtqueue *vq, int type) in vhost_new_msg() argument
2927 node->vq = vq; in vhost_new_msg()
2963 struct vhost_virtqueue *vq; in vhost_set_backend_features() local
2968 vq = dev->vqs[i]; in vhost_set_backend_features()
2969 mutex_lock(&vq->mutex); in vhost_set_backend_features()
2970 vq->acked_backend_features = features; in vhost_set_backend_features()
2971 mutex_unlock(&vq->mutex); in vhost_set_backend_features()