1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016 RehiveTech. All rights reserved.
3 */
4
5 #include <string.h>
6 #include <inttypes.h>
7 #include <stdio.h>
8 #include <stdlib.h>
9 #include <stdint.h>
10 #include <stdbool.h>
11 #include <sys/queue.h>
12
13 #include <rte_eal.h>
14 #include <rte_dev.h>
15 #include <rte_bus.h>
16 #include <rte_common.h>
17 #include <rte_devargs.h>
18 #include <rte_memory.h>
19 #include <rte_tailq.h>
20 #include <rte_spinlock.h>
21 #include <rte_string_fns.h>
22 #include <rte_errno.h>
23
24 #include "rte_bus_vdev.h"
25 #include "vdev_logs.h"
26 #include "vdev_private.h"
27
28 #define VDEV_MP_KEY "bus_vdev_mp"
29
30 /* Forward declare to access virtual bus name */
31 static struct rte_bus rte_vdev_bus;
32
33 /** Double linked list of virtual device drivers. */
34 TAILQ_HEAD(vdev_device_list, rte_vdev_device);
35
36 static struct vdev_device_list vdev_device_list =
37 TAILQ_HEAD_INITIALIZER(vdev_device_list);
38 /* The lock needs to be recursive because a vdev can manage another vdev. */
39 static rte_spinlock_recursive_t vdev_device_list_lock =
40 RTE_SPINLOCK_RECURSIVE_INITIALIZER;
41
42 static struct vdev_driver_list vdev_driver_list =
43 TAILQ_HEAD_INITIALIZER(vdev_driver_list);
44
45 struct vdev_custom_scan {
46 TAILQ_ENTRY(vdev_custom_scan) next;
47 rte_vdev_scan_callback callback;
48 void *user_arg;
49 };
50 TAILQ_HEAD(vdev_custom_scans, vdev_custom_scan);
51 static struct vdev_custom_scans vdev_custom_scans =
52 TAILQ_HEAD_INITIALIZER(vdev_custom_scans);
53 static rte_spinlock_t vdev_custom_scan_lock = RTE_SPINLOCK_INITIALIZER;
54
55 /* register a driver */
56 void
rte_vdev_register(struct rte_vdev_driver * driver)57 rte_vdev_register(struct rte_vdev_driver *driver)
58 {
59 TAILQ_INSERT_TAIL(&vdev_driver_list, driver, next);
60 }
61
62 /* unregister a driver */
63 void
rte_vdev_unregister(struct rte_vdev_driver * driver)64 rte_vdev_unregister(struct rte_vdev_driver *driver)
65 {
66 TAILQ_REMOVE(&vdev_driver_list, driver, next);
67 }
68
69 int
rte_vdev_add_custom_scan(rte_vdev_scan_callback callback,void * user_arg)70 rte_vdev_add_custom_scan(rte_vdev_scan_callback callback, void *user_arg)
71 {
72 struct vdev_custom_scan *custom_scan;
73
74 rte_spinlock_lock(&vdev_custom_scan_lock);
75
76 /* check if already registered */
77 TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) {
78 if (custom_scan->callback == callback &&
79 custom_scan->user_arg == user_arg)
80 break;
81 }
82
83 if (custom_scan == NULL) {
84 custom_scan = malloc(sizeof(struct vdev_custom_scan));
85 if (custom_scan != NULL) {
86 custom_scan->callback = callback;
87 custom_scan->user_arg = user_arg;
88 TAILQ_INSERT_TAIL(&vdev_custom_scans, custom_scan, next);
89 }
90 }
91
92 rte_spinlock_unlock(&vdev_custom_scan_lock);
93
94 return (custom_scan == NULL) ? -1 : 0;
95 }
96
97 int
rte_vdev_remove_custom_scan(rte_vdev_scan_callback callback,void * user_arg)98 rte_vdev_remove_custom_scan(rte_vdev_scan_callback callback, void *user_arg)
99 {
100 struct vdev_custom_scan *custom_scan, *tmp_scan;
101
102 rte_spinlock_lock(&vdev_custom_scan_lock);
103 RTE_TAILQ_FOREACH_SAFE(custom_scan, &vdev_custom_scans, next,
104 tmp_scan) {
105 if (custom_scan->callback != callback ||
106 (custom_scan->user_arg != (void *)-1 &&
107 custom_scan->user_arg != user_arg))
108 continue;
109 TAILQ_REMOVE(&vdev_custom_scans, custom_scan, next);
110 free(custom_scan);
111 }
112 rte_spinlock_unlock(&vdev_custom_scan_lock);
113
114 return 0;
115 }
116
117 static int
vdev_parse(const char * name,void * addr)118 vdev_parse(const char *name, void *addr)
119 {
120 struct rte_vdev_driver **out = addr;
121 struct rte_vdev_driver *driver = NULL;
122
123 TAILQ_FOREACH(driver, &vdev_driver_list, next) {
124 if (strncmp(driver->driver.name, name,
125 strlen(driver->driver.name)) == 0)
126 break;
127 if (driver->driver.alias &&
128 strncmp(driver->driver.alias, name,
129 strlen(driver->driver.alias)) == 0)
130 break;
131 }
132 if (driver != NULL &&
133 addr != NULL)
134 *out = driver;
135 return driver == NULL;
136 }
137
138 static int
vdev_dma_map(struct rte_device * dev,void * addr,uint64_t iova,size_t len)139 vdev_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
140 {
141 struct rte_vdev_device *vdev = RTE_DEV_TO_VDEV(dev);
142 const struct rte_vdev_driver *driver;
143
144 if (!vdev) {
145 rte_errno = EINVAL;
146 return -1;
147 }
148
149 if (!vdev->device.driver) {
150 VDEV_LOG(DEBUG, "no driver attach to device %s", dev->name);
151 return 1;
152 }
153
154 driver = container_of(vdev->device.driver, const struct rte_vdev_driver,
155 driver);
156
157 if (driver->dma_map)
158 return driver->dma_map(vdev, addr, iova, len);
159
160 return 0;
161 }
162
163 static int
vdev_dma_unmap(struct rte_device * dev,void * addr,uint64_t iova,size_t len)164 vdev_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
165 {
166 struct rte_vdev_device *vdev = RTE_DEV_TO_VDEV(dev);
167 const struct rte_vdev_driver *driver;
168
169 if (!vdev) {
170 rte_errno = EINVAL;
171 return -1;
172 }
173
174 if (!vdev->device.driver) {
175 VDEV_LOG(DEBUG, "no driver attach to device %s", dev->name);
176 return 1;
177 }
178
179 driver = container_of(vdev->device.driver, const struct rte_vdev_driver,
180 driver);
181
182 if (driver->dma_unmap)
183 return driver->dma_unmap(vdev, addr, iova, len);
184
185 return 0;
186 }
187
188 static int
vdev_probe_all_drivers(struct rte_vdev_device * dev)189 vdev_probe_all_drivers(struct rte_vdev_device *dev)
190 {
191 const char *name;
192 struct rte_vdev_driver *driver;
193 enum rte_iova_mode iova_mode;
194 int ret;
195
196 if (rte_dev_is_probed(&dev->device))
197 return -EEXIST;
198
199 name = rte_vdev_device_name(dev);
200 VDEV_LOG(DEBUG, "Search driver to probe device %s", name);
201
202 if (vdev_parse(name, &driver))
203 return -1;
204
205 iova_mode = rte_eal_iova_mode();
206 if ((driver->drv_flags & RTE_VDEV_DRV_NEED_IOVA_AS_VA) && (iova_mode == RTE_IOVA_PA)) {
207 VDEV_LOG(ERR, "%s requires VA IOVA mode but current mode is PA, not initializing",
208 name);
209 return -1;
210 }
211
212 ret = driver->probe(dev);
213 if (ret == 0)
214 dev->device.driver = &driver->driver;
215 return ret;
216 }
217
218 /* The caller shall be responsible for thread-safe */
219 static struct rte_vdev_device *
find_vdev(const char * name)220 find_vdev(const char *name)
221 {
222 struct rte_vdev_device *dev;
223
224 if (!name)
225 return NULL;
226
227 TAILQ_FOREACH(dev, &vdev_device_list, next) {
228 const char *devname = rte_vdev_device_name(dev);
229
230 if (!strcmp(devname, name))
231 return dev;
232 }
233
234 return NULL;
235 }
236
237 static struct rte_devargs *
alloc_devargs(const char * name,const char * args)238 alloc_devargs(const char *name, const char *args)
239 {
240 struct rte_devargs *devargs;
241 int ret;
242
243 devargs = calloc(1, sizeof(*devargs));
244 if (!devargs)
245 return NULL;
246
247 devargs->bus = &rte_vdev_bus;
248 if (args)
249 devargs->data = strdup(args);
250 else
251 devargs->data = strdup("");
252 devargs->args = devargs->data;
253
254 ret = strlcpy(devargs->name, name, sizeof(devargs->name));
255 if (ret < 0 || ret >= (int)sizeof(devargs->name)) {
256 rte_devargs_reset(devargs);
257 free(devargs);
258 return NULL;
259 }
260
261 return devargs;
262 }
263
264 static int
insert_vdev(const char * name,const char * args,struct rte_vdev_device ** p_dev,bool init)265 insert_vdev(const char *name, const char *args,
266 struct rte_vdev_device **p_dev,
267 bool init)
268 {
269 struct rte_vdev_device *dev;
270 struct rte_devargs *devargs;
271 int ret;
272
273 if (name == NULL)
274 return -EINVAL;
275
276 devargs = alloc_devargs(name, args);
277 if (!devargs)
278 return -ENOMEM;
279
280 dev = calloc(1, sizeof(*dev));
281 if (!dev) {
282 ret = -ENOMEM;
283 goto fail;
284 }
285
286 dev->device.bus = &rte_vdev_bus;
287 dev->device.numa_node = SOCKET_ID_ANY;
288 dev->device.name = devargs->name;
289
290 if (find_vdev(name)) {
291 /*
292 * A vdev is expected to have only one port.
293 * So there is no reason to try probing again,
294 * even with new arguments.
295 */
296 ret = -EEXIST;
297 goto fail;
298 }
299
300 if (init)
301 rte_devargs_insert(&devargs);
302 dev->device.devargs = devargs;
303 TAILQ_INSERT_TAIL(&vdev_device_list, dev, next);
304
305 if (p_dev)
306 *p_dev = dev;
307
308 return 0;
309 fail:
310 rte_devargs_reset(devargs);
311 free(devargs);
312 free(dev);
313 return ret;
314 }
315
316 int
rte_vdev_init(const char * name,const char * args)317 rte_vdev_init(const char *name, const char *args)
318 {
319 struct rte_vdev_device *dev;
320 int ret;
321
322 rte_spinlock_recursive_lock(&vdev_device_list_lock);
323 ret = insert_vdev(name, args, &dev, true);
324 if (ret == 0) {
325 ret = vdev_probe_all_drivers(dev);
326 if (ret) {
327 if (ret > 0)
328 VDEV_LOG(ERR, "no driver found for %s", name);
329 /* If fails, remove it from vdev list */
330 TAILQ_REMOVE(&vdev_device_list, dev, next);
331 rte_devargs_remove(dev->device.devargs);
332 free(dev);
333 }
334 }
335 rte_spinlock_recursive_unlock(&vdev_device_list_lock);
336 return ret;
337 }
338
339 static int
vdev_remove_driver(struct rte_vdev_device * dev)340 vdev_remove_driver(struct rte_vdev_device *dev)
341 {
342 const char *name = rte_vdev_device_name(dev);
343 const struct rte_vdev_driver *driver;
344
345 if (!dev->device.driver) {
346 VDEV_LOG(DEBUG, "no driver attach to device %s", name);
347 return 1;
348 }
349
350 driver = container_of(dev->device.driver, const struct rte_vdev_driver,
351 driver);
352 return driver->remove(dev);
353 }
354
355 int
rte_vdev_uninit(const char * name)356 rte_vdev_uninit(const char *name)
357 {
358 struct rte_vdev_device *dev;
359 int ret;
360
361 if (name == NULL)
362 return -EINVAL;
363
364 rte_spinlock_recursive_lock(&vdev_device_list_lock);
365
366 dev = find_vdev(name);
367 if (!dev) {
368 ret = -ENOENT;
369 goto unlock;
370 }
371
372 ret = vdev_remove_driver(dev);
373 if (ret)
374 goto unlock;
375
376 TAILQ_REMOVE(&vdev_device_list, dev, next);
377 rte_devargs_remove(dev->device.devargs);
378 free(dev);
379
380 unlock:
381 rte_spinlock_recursive_unlock(&vdev_device_list_lock);
382 return ret;
383 }
384
385 struct vdev_param {
386 #define VDEV_SCAN_REQ 1
387 #define VDEV_SCAN_ONE 2
388 #define VDEV_SCAN_REP 3
389 int type;
390 int num;
391 char name[RTE_DEV_NAME_MAX_LEN];
392 };
393
394 static int vdev_plug(struct rte_device *dev);
395
396 /**
397 * This function works as the action for both primary and secondary process
398 * for static vdev discovery when a secondary process is booting.
399 *
400 * step 1, secondary process sends a sync request to ask for vdev in primary;
401 * step 2, primary process receives the request, and send vdevs one by one;
402 * step 3, primary process sends back reply, which indicates how many vdevs
403 * are sent.
404 */
405 static int
vdev_action(const struct rte_mp_msg * mp_msg,const void * peer)406 vdev_action(const struct rte_mp_msg *mp_msg, const void *peer)
407 {
408 struct rte_vdev_device *dev;
409 struct rte_mp_msg mp_resp;
410 struct vdev_param *ou = (struct vdev_param *)&mp_resp.param;
411 const struct vdev_param *in = (const struct vdev_param *)mp_msg->param;
412 const char *devname;
413 int num;
414 int ret;
415
416 strlcpy(mp_resp.name, VDEV_MP_KEY, sizeof(mp_resp.name));
417 mp_resp.len_param = sizeof(*ou);
418 mp_resp.num_fds = 0;
419
420 switch (in->type) {
421 case VDEV_SCAN_REQ:
422 ou->type = VDEV_SCAN_ONE;
423 ou->num = 1;
424 num = 0;
425
426 rte_spinlock_recursive_lock(&vdev_device_list_lock);
427 TAILQ_FOREACH(dev, &vdev_device_list, next) {
428 devname = rte_vdev_device_name(dev);
429 if (strlen(devname) == 0) {
430 VDEV_LOG(INFO, "vdev with no name is not sent");
431 continue;
432 }
433 VDEV_LOG(INFO, "send vdev, %s", devname);
434 strlcpy(ou->name, devname, RTE_DEV_NAME_MAX_LEN);
435 if (rte_mp_sendmsg(&mp_resp) < 0)
436 VDEV_LOG(ERR, "send vdev, %s, failed, %s",
437 devname, strerror(rte_errno));
438 num++;
439 }
440 rte_spinlock_recursive_unlock(&vdev_device_list_lock);
441
442 ou->type = VDEV_SCAN_REP;
443 ou->num = num;
444 if (rte_mp_reply(&mp_resp, peer) < 0)
445 VDEV_LOG(ERR, "Failed to reply a scan request");
446 break;
447 case VDEV_SCAN_ONE:
448 VDEV_LOG(INFO, "receive vdev, %s", in->name);
449 ret = insert_vdev(in->name, NULL, NULL, false);
450 if (ret == -EEXIST)
451 VDEV_LOG(DEBUG, "device already exist, %s", in->name);
452 else if (ret < 0)
453 VDEV_LOG(ERR, "failed to add vdev, %s", in->name);
454 break;
455 default:
456 VDEV_LOG(ERR, "vdev cannot recognize this message");
457 }
458
459 return 0;
460 }
461
462 static int
vdev_scan(void)463 vdev_scan(void)
464 {
465 struct rte_vdev_device *dev;
466 struct rte_devargs *devargs;
467 struct vdev_custom_scan *custom_scan;
468
469 if (rte_mp_action_register(VDEV_MP_KEY, vdev_action) < 0 &&
470 rte_errno != EEXIST) {
471 /* for primary, unsupported IPC is not an error */
472 if (rte_eal_process_type() == RTE_PROC_PRIMARY &&
473 rte_errno == ENOTSUP)
474 goto scan;
475 VDEV_LOG(ERR, "Failed to add vdev mp action");
476 return -1;
477 }
478
479 if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
480 struct rte_mp_msg mp_req, *mp_rep;
481 struct rte_mp_reply mp_reply;
482 struct timespec ts = {.tv_sec = 5, .tv_nsec = 0};
483 struct vdev_param *req = (struct vdev_param *)mp_req.param;
484 struct vdev_param *resp;
485
486 strlcpy(mp_req.name, VDEV_MP_KEY, sizeof(mp_req.name));
487 mp_req.len_param = sizeof(*req);
488 mp_req.num_fds = 0;
489 req->type = VDEV_SCAN_REQ;
490 if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0 &&
491 mp_reply.nb_received == 1) {
492 mp_rep = &mp_reply.msgs[0];
493 resp = (struct vdev_param *)mp_rep->param;
494 VDEV_LOG(INFO, "Received %d vdevs", resp->num);
495 free(mp_reply.msgs);
496 } else
497 VDEV_LOG(ERR, "Failed to request vdev from primary");
498
499 /* Fall through to allow private vdevs in secondary process */
500 }
501
502 scan:
503 /* call custom scan callbacks if any */
504 rte_spinlock_lock(&vdev_custom_scan_lock);
505 TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) {
506 if (custom_scan->callback != NULL)
507 /*
508 * the callback should update devargs list
509 * by calling rte_devargs_insert() with
510 * devargs.bus = rte_bus_find_by_name("vdev");
511 * devargs.type = RTE_DEVTYPE_VIRTUAL;
512 * devargs.policy = RTE_DEV_ALLOWED;
513 */
514 custom_scan->callback(custom_scan->user_arg);
515 }
516 rte_spinlock_unlock(&vdev_custom_scan_lock);
517
518 /* for virtual devices we scan the devargs_list populated via cmdline */
519 RTE_EAL_DEVARGS_FOREACH("vdev", devargs) {
520
521 dev = calloc(1, sizeof(*dev));
522 if (!dev)
523 return -1;
524
525 rte_spinlock_recursive_lock(&vdev_device_list_lock);
526
527 if (find_vdev(devargs->name)) {
528 rte_spinlock_recursive_unlock(&vdev_device_list_lock);
529 free(dev);
530 continue;
531 }
532
533 dev->device.bus = &rte_vdev_bus;
534 dev->device.devargs = devargs;
535 dev->device.numa_node = SOCKET_ID_ANY;
536 dev->device.name = devargs->name;
537
538 TAILQ_INSERT_TAIL(&vdev_device_list, dev, next);
539
540 rte_spinlock_recursive_unlock(&vdev_device_list_lock);
541 }
542
543 return 0;
544 }
545
546 static int
vdev_probe(void)547 vdev_probe(void)
548 {
549 struct rte_vdev_device *dev;
550 int r, ret = 0;
551
552 /* call the init function for each virtual device */
553 TAILQ_FOREACH(dev, &vdev_device_list, next) {
554 /* we don't use the vdev lock here, as it's only used in DPDK
555 * initialization; and we don't want to hold such a lock when
556 * we call each driver probe.
557 */
558
559 r = vdev_probe_all_drivers(dev);
560 if (r != 0) {
561 if (r == -EEXIST)
562 continue;
563 VDEV_LOG(ERR, "failed to initialize %s device",
564 rte_vdev_device_name(dev));
565 ret = -1;
566 }
567 }
568
569 return ret;
570 }
571
572 struct rte_device *
rte_vdev_find_device(const struct rte_device * start,rte_dev_cmp_t cmp,const void * data)573 rte_vdev_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
574 const void *data)
575 {
576 const struct rte_vdev_device *vstart;
577 struct rte_vdev_device *dev;
578
579 rte_spinlock_recursive_lock(&vdev_device_list_lock);
580 if (start != NULL) {
581 vstart = RTE_DEV_TO_VDEV_CONST(start);
582 dev = TAILQ_NEXT(vstart, next);
583 } else {
584 dev = TAILQ_FIRST(&vdev_device_list);
585 }
586 while (dev != NULL) {
587 if (cmp(&dev->device, data) == 0)
588 break;
589 dev = TAILQ_NEXT(dev, next);
590 }
591 rte_spinlock_recursive_unlock(&vdev_device_list_lock);
592
593 return dev ? &dev->device : NULL;
594 }
595
596 static int
vdev_plug(struct rte_device * dev)597 vdev_plug(struct rte_device *dev)
598 {
599 return vdev_probe_all_drivers(RTE_DEV_TO_VDEV(dev));
600 }
601
602 static int
vdev_unplug(struct rte_device * dev)603 vdev_unplug(struct rte_device *dev)
604 {
605 return rte_vdev_uninit(dev->name);
606 }
607
608 static enum rte_iova_mode
vdev_get_iommu_class(void)609 vdev_get_iommu_class(void)
610 {
611 const char *name;
612 struct rte_vdev_device *dev;
613 struct rte_vdev_driver *driver;
614
615 TAILQ_FOREACH(dev, &vdev_device_list, next) {
616 name = rte_vdev_device_name(dev);
617 if (vdev_parse(name, &driver))
618 continue;
619
620 if (driver->drv_flags & RTE_VDEV_DRV_NEED_IOVA_AS_VA)
621 return RTE_IOVA_VA;
622 }
623
624 return RTE_IOVA_DC;
625 }
626
627 static struct rte_bus rte_vdev_bus = {
628 .scan = vdev_scan,
629 .probe = vdev_probe,
630 .find_device = rte_vdev_find_device,
631 .plug = vdev_plug,
632 .unplug = vdev_unplug,
633 .parse = vdev_parse,
634 .dma_map = vdev_dma_map,
635 .dma_unmap = vdev_dma_unmap,
636 .get_iommu_class = vdev_get_iommu_class,
637 .dev_iterate = rte_vdev_dev_iterate,
638 };
639
640 RTE_REGISTER_BUS(vdev, rte_vdev_bus);
641 RTE_LOG_REGISTER_DEFAULT(vdev_logtype_bus, NOTICE);
642