1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
3 */
4
5 /**
6 * @file
7 *
8 * Device specific vhost lib
9 */
10
11 #include <sys/queue.h>
12
13 #include <rte_class.h>
14 #include <rte_malloc.h>
15 #include <rte_spinlock.h>
16 #include <rte_tailq.h>
17
18 #include "rte_vdpa.h"
19 #include "vdpa_driver.h"
20 #include "vhost.h"
21
22 /** Double linked list of vDPA devices. */
23 TAILQ_HEAD(vdpa_device_list, rte_vdpa_device);
24
25 static struct vdpa_device_list vdpa_device_list =
26 TAILQ_HEAD_INITIALIZER(vdpa_device_list);
27 static rte_spinlock_t vdpa_device_list_lock = RTE_SPINLOCK_INITIALIZER;
28
29
30 /* Unsafe, needs to be called with vdpa_device_list_lock held */
31 static struct rte_vdpa_device *
__vdpa_find_device_by_name(const char * name)32 __vdpa_find_device_by_name(const char *name)
33 {
34 struct rte_vdpa_device *dev, *ret = NULL;
35
36 if (name == NULL)
37 return NULL;
38
39 TAILQ_FOREACH(dev, &vdpa_device_list, next) {
40 if (!strncmp(dev->device->name, name, RTE_DEV_NAME_MAX_LEN)) {
41 ret = dev;
42 break;
43 }
44 }
45
46 return ret;
47 }
48
49 struct rte_vdpa_device *
rte_vdpa_find_device_by_name(const char * name)50 rte_vdpa_find_device_by_name(const char *name)
51 {
52 struct rte_vdpa_device *dev;
53
54 rte_spinlock_lock(&vdpa_device_list_lock);
55 dev = __vdpa_find_device_by_name(name);
56 rte_spinlock_unlock(&vdpa_device_list_lock);
57
58 return dev;
59 }
60
61 struct rte_device *
rte_vdpa_get_rte_device(struct rte_vdpa_device * vdpa_dev)62 rte_vdpa_get_rte_device(struct rte_vdpa_device *vdpa_dev)
63 {
64 if (vdpa_dev == NULL)
65 return NULL;
66
67 return vdpa_dev->device;
68 }
69
70 struct rte_vdpa_device *
rte_vdpa_register_device(struct rte_device * rte_dev,struct rte_vdpa_dev_ops * ops)71 rte_vdpa_register_device(struct rte_device *rte_dev,
72 struct rte_vdpa_dev_ops *ops)
73 {
74 struct rte_vdpa_device *dev;
75
76 if (ops == NULL)
77 return NULL;
78
79 /* Check mandatory ops are implemented */
80 if (!ops->get_queue_num || !ops->get_features ||
81 !ops->get_protocol_features || !ops->dev_conf ||
82 !ops->dev_close || !ops->set_vring_state ||
83 !ops->set_features) {
84 VHOST_LOG_CONFIG(ERR, "(%s) Some mandatory vDPA ops aren't implemented\n",
85 rte_dev->name);
86 return NULL;
87 }
88
89 rte_spinlock_lock(&vdpa_device_list_lock);
90 /* Check the device hasn't been register already */
91 dev = __vdpa_find_device_by_name(rte_dev->name);
92 if (dev) {
93 dev = NULL;
94 goto out_unlock;
95 }
96
97 dev = rte_zmalloc(NULL, sizeof(*dev), 0);
98 if (!dev)
99 goto out_unlock;
100
101 dev->device = rte_dev;
102 dev->ops = ops;
103 TAILQ_INSERT_TAIL(&vdpa_device_list, dev, next);
104 out_unlock:
105 rte_spinlock_unlock(&vdpa_device_list_lock);
106
107 return dev;
108 }
109
110 int
rte_vdpa_unregister_device(struct rte_vdpa_device * dev)111 rte_vdpa_unregister_device(struct rte_vdpa_device *dev)
112 {
113 struct rte_vdpa_device *cur_dev, *tmp_dev;
114 int ret = -1;
115
116 rte_spinlock_lock(&vdpa_device_list_lock);
117 RTE_TAILQ_FOREACH_SAFE(cur_dev, &vdpa_device_list, next, tmp_dev) {
118 if (dev != cur_dev)
119 continue;
120
121 TAILQ_REMOVE(&vdpa_device_list, dev, next);
122 rte_free(dev);
123 ret = 0;
124 break;
125 }
126 rte_spinlock_unlock(&vdpa_device_list_lock);
127
128 return ret;
129 }
130
131 int
rte_vdpa_relay_vring_used(int vid,uint16_t qid,void * vring_m)132 rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m)
133 {
134 struct virtio_net *dev = get_device(vid);
135 uint16_t idx, idx_m, desc_id;
136 struct vhost_virtqueue *vq;
137 struct vring_desc desc;
138 struct vring_desc *desc_ring;
139 struct vring_desc *idesc = NULL;
140 struct vring *s_vring;
141 uint64_t dlen;
142 uint32_t nr_descs;
143 int ret;
144
145 if (!dev || !vring_m)
146 return -1;
147
148 if (qid >= dev->nr_vring)
149 return -1;
150
151 if (vq_is_packed(dev))
152 return -1;
153
154 s_vring = (struct vring *)vring_m;
155 vq = dev->virtqueue[qid];
156 idx = vq->used->idx;
157 idx_m = s_vring->used->idx;
158 ret = (uint16_t)(idx_m - idx);
159
160 while (idx != idx_m) {
161 /* copy used entry, used ring logging is not covered here */
162 vq->used->ring[idx & (vq->size - 1)] =
163 s_vring->used->ring[idx & (vq->size - 1)];
164
165 desc_id = vq->used->ring[idx & (vq->size - 1)].id;
166 desc_ring = vq->desc;
167 nr_descs = vq->size;
168
169 if (unlikely(desc_id >= vq->size))
170 return -1;
171
172 if (vq->desc[desc_id].flags & VRING_DESC_F_INDIRECT) {
173 dlen = vq->desc[desc_id].len;
174 nr_descs = dlen / sizeof(struct vring_desc);
175 if (unlikely(nr_descs > vq->size))
176 return -1;
177
178 desc_ring = (struct vring_desc *)(uintptr_t)
179 vhost_iova_to_vva(dev, vq,
180 vq->desc[desc_id].addr, &dlen,
181 VHOST_ACCESS_RO);
182 if (unlikely(!desc_ring))
183 return -1;
184
185 if (unlikely(dlen < vq->desc[desc_id].len)) {
186 idesc = vhost_alloc_copy_ind_table(dev, vq,
187 vq->desc[desc_id].addr,
188 vq->desc[desc_id].len);
189 if (unlikely(!idesc))
190 return -1;
191
192 desc_ring = idesc;
193 }
194
195 desc_id = 0;
196 }
197
198 /* dirty page logging for DMA writeable buffer */
199 do {
200 if (unlikely(desc_id >= vq->size))
201 goto fail;
202 if (unlikely(nr_descs-- == 0))
203 goto fail;
204 desc = desc_ring[desc_id];
205 if (desc.flags & VRING_DESC_F_WRITE)
206 vhost_log_write_iova(dev, vq, desc.addr,
207 desc.len);
208 desc_id = desc.next;
209 } while (desc.flags & VRING_DESC_F_NEXT);
210
211 if (unlikely(idesc)) {
212 free_ind_table(idesc);
213 idesc = NULL;
214 }
215
216 idx++;
217 }
218
219 /* used idx is the synchronization point for the split vring */
220 __atomic_store_n(&vq->used->idx, idx_m, __ATOMIC_RELEASE);
221
222 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
223 vring_used_event(s_vring) = idx_m;
224
225 return ret;
226
227 fail:
228 if (unlikely(idesc))
229 free_ind_table(idesc);
230 return -1;
231 }
232
233 int
rte_vdpa_get_queue_num(struct rte_vdpa_device * dev,uint32_t * queue_num)234 rte_vdpa_get_queue_num(struct rte_vdpa_device *dev, uint32_t *queue_num)
235 {
236 if (dev == NULL || dev->ops == NULL || dev->ops->get_queue_num == NULL)
237 return -1;
238
239 return dev->ops->get_queue_num(dev, queue_num);
240 }
241
242 int
rte_vdpa_get_features(struct rte_vdpa_device * dev,uint64_t * features)243 rte_vdpa_get_features(struct rte_vdpa_device *dev, uint64_t *features)
244 {
245 if (dev == NULL || dev->ops == NULL || dev->ops->get_features == NULL)
246 return -1;
247
248 return dev->ops->get_features(dev, features);
249 }
250
251 int
rte_vdpa_get_protocol_features(struct rte_vdpa_device * dev,uint64_t * features)252 rte_vdpa_get_protocol_features(struct rte_vdpa_device *dev, uint64_t *features)
253 {
254 if (dev == NULL || dev->ops == NULL ||
255 dev->ops->get_protocol_features == NULL)
256 return -1;
257
258 return dev->ops->get_protocol_features(dev, features);
259 }
260
261 int
rte_vdpa_get_stats_names(struct rte_vdpa_device * dev,struct rte_vdpa_stat_name * stats_names,unsigned int size)262 rte_vdpa_get_stats_names(struct rte_vdpa_device *dev,
263 struct rte_vdpa_stat_name *stats_names,
264 unsigned int size)
265 {
266 if (!dev)
267 return -EINVAL;
268
269 RTE_FUNC_PTR_OR_ERR_RET(dev->ops->get_stats_names, -ENOTSUP);
270
271 return dev->ops->get_stats_names(dev, stats_names, size);
272 }
273
274 int
rte_vdpa_get_stats(struct rte_vdpa_device * dev,uint16_t qid,struct rte_vdpa_stat * stats,unsigned int n)275 rte_vdpa_get_stats(struct rte_vdpa_device *dev, uint16_t qid,
276 struct rte_vdpa_stat *stats, unsigned int n)
277 {
278 if (!dev || !stats || !n)
279 return -EINVAL;
280
281 RTE_FUNC_PTR_OR_ERR_RET(dev->ops->get_stats, -ENOTSUP);
282
283 return dev->ops->get_stats(dev, qid, stats, n);
284 }
285
286 int
rte_vdpa_reset_stats(struct rte_vdpa_device * dev,uint16_t qid)287 rte_vdpa_reset_stats(struct rte_vdpa_device *dev, uint16_t qid)
288 {
289 if (!dev)
290 return -EINVAL;
291
292 RTE_FUNC_PTR_OR_ERR_RET(dev->ops->reset_stats, -ENOTSUP);
293
294 return dev->ops->reset_stats(dev, qid);
295 }
296
297 static int
vdpa_dev_match(struct rte_vdpa_device * dev,const struct rte_device * rte_dev)298 vdpa_dev_match(struct rte_vdpa_device *dev,
299 const struct rte_device *rte_dev)
300 {
301 if (dev->device == rte_dev)
302 return 0;
303
304 return -1;
305 }
306
307 /* Generic rte_vdpa_dev comparison function. */
308 typedef int (*rte_vdpa_cmp_t)(struct rte_vdpa_device *,
309 const struct rte_device *rte_dev);
310
311 static struct rte_vdpa_device *
vdpa_find_device(const struct rte_vdpa_device * start,rte_vdpa_cmp_t cmp,struct rte_device * rte_dev)312 vdpa_find_device(const struct rte_vdpa_device *start, rte_vdpa_cmp_t cmp,
313 struct rte_device *rte_dev)
314 {
315 struct rte_vdpa_device *dev;
316
317 rte_spinlock_lock(&vdpa_device_list_lock);
318 if (start == NULL)
319 dev = TAILQ_FIRST(&vdpa_device_list);
320 else
321 dev = TAILQ_NEXT(start, next);
322
323 while (dev != NULL) {
324 if (cmp(dev, rte_dev) == 0)
325 break;
326
327 dev = TAILQ_NEXT(dev, next);
328 }
329 rte_spinlock_unlock(&vdpa_device_list_lock);
330
331 return dev;
332 }
333
334 static void *
vdpa_dev_iterate(const void * start,const char * str,const struct rte_dev_iterator * it)335 vdpa_dev_iterate(const void *start,
336 const char *str,
337 const struct rte_dev_iterator *it)
338 {
339 struct rte_vdpa_device *vdpa_dev = NULL;
340
341 RTE_SET_USED(str);
342
343 vdpa_dev = vdpa_find_device(start, vdpa_dev_match, it->device);
344
345 return vdpa_dev;
346 }
347
348 static struct rte_class rte_class_vdpa = {
349 .dev_iterate = vdpa_dev_iterate,
350 };
351
352 RTE_REGISTER_CLASS(vdpa, rte_class_vdpa);
353