1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation. 3 * Copyright(c) 2014 6WIND S.A. 4 */ 5 6 #include <stdio.h> 7 #include <string.h> 8 #include <inttypes.h> 9 #include <sys/queue.h> 10 11 #include <rte_compat.h> 12 #include <rte_bus.h> 13 #include <rte_class.h> 14 #include <rte_dev.h> 15 #include <rte_devargs.h> 16 #include <rte_debug.h> 17 #include <rte_errno.h> 18 #include <rte_kvargs.h> 19 #include <rte_log.h> 20 #include <rte_spinlock.h> 21 #include <rte_malloc.h> 22 #include <rte_string_fns.h> 23 24 #include "eal_private.h" 25 #include "hotplug_mp.h" 26 27 /** 28 * The device event callback description. 29 * 30 * It contains callback address to be registered by user application, 31 * the pointer to the parameters for callback, and the device name. 32 */ 33 struct dev_event_callback { 34 TAILQ_ENTRY(dev_event_callback) next; /**< Callbacks list */ 35 rte_dev_event_cb_fn cb_fn; /**< Callback address */ 36 void *cb_arg; /**< Callback parameter */ 37 char *dev_name; /**< Callback device name, NULL is for all device */ 38 uint32_t active; /**< Callback is executing */ 39 }; 40 41 /** @internal Structure to keep track of registered callbacks */ 42 TAILQ_HEAD(dev_event_cb_list, dev_event_callback); 43 44 /* The device event callback list for all registered callbacks. */ 45 static struct dev_event_cb_list dev_event_cbs; 46 47 /* spinlock for device callbacks */ 48 static rte_spinlock_t dev_event_lock = RTE_SPINLOCK_INITIALIZER; 49 50 struct dev_next_ctx { 51 struct rte_dev_iterator *it; 52 const char *bus_str; 53 const char *cls_str; 54 }; 55 56 #define CTX(it, bus_str, cls_str) \ 57 (&(const struct dev_next_ctx){ \ 58 .it = it, \ 59 .bus_str = bus_str, \ 60 .cls_str = cls_str, \ 61 }) 62 63 #define ITCTX(ptr) \ 64 (((struct dev_next_ctx *)(intptr_t)ptr)->it) 65 66 #define BUSCTX(ptr) \ 67 (((struct dev_next_ctx *)(intptr_t)ptr)->bus_str) 68 69 #define CLSCTX(ptr) \ 70 (((struct dev_next_ctx *)(intptr_t)ptr)->cls_str) 71 72 static int cmp_dev_name(const struct rte_device *dev, const void *_name) 73 { 74 const char *name = _name; 75 76 return strcmp(dev->name, name); 77 } 78 79 int __rte_experimental 80 rte_dev_is_probed(const struct rte_device *dev) 81 { 82 /* The field driver should be set only when the probe is successful. */ 83 return dev->driver != NULL; 84 } 85 86 /* helper function to build devargs, caller should free the memory */ 87 static int 88 build_devargs(const char *busname, const char *devname, 89 const char *drvargs, char **devargs) 90 { 91 int length; 92 93 length = snprintf(NULL, 0, "%s:%s,%s", busname, devname, drvargs); 94 if (length < 0) 95 return -EINVAL; 96 97 *devargs = malloc(length + 1); 98 if (*devargs == NULL) 99 return -ENOMEM; 100 101 length = snprintf(*devargs, length + 1, "%s:%s,%s", 102 busname, devname, drvargs); 103 if (length < 0) { 104 free(*devargs); 105 return -EINVAL; 106 } 107 108 return 0; 109 } 110 111 int 112 rte_eal_hotplug_add(const char *busname, const char *devname, 113 const char *drvargs) 114 { 115 116 char *devargs; 117 int ret; 118 119 ret = build_devargs(busname, devname, drvargs, &devargs); 120 if (ret != 0) 121 return ret; 122 123 ret = rte_dev_probe(devargs); 124 free(devargs); 125 126 return ret; 127 } 128 129 /* probe device at local process. */ 130 int 131 local_dev_probe(const char *devargs, struct rte_device **new_dev) 132 { 133 struct rte_device *dev; 134 struct rte_devargs *da; 135 int ret; 136 137 *new_dev = NULL; 138 da = calloc(1, sizeof(*da)); 139 if (da == NULL) 140 return -ENOMEM; 141 142 ret = rte_devargs_parse(da, devargs); 143 if (ret) 144 goto err_devarg; 145 146 if (da->bus->plug == NULL) { 147 RTE_LOG(ERR, EAL, "Function plug not supported by bus (%s)\n", 148 da->bus->name); 149 ret = -ENOTSUP; 150 goto err_devarg; 151 } 152 153 ret = rte_devargs_insert(&da); 154 if (ret) 155 goto err_devarg; 156 157 /* the rte_devargs will be referenced in the matching rte_device */ 158 ret = da->bus->scan(); 159 if (ret) 160 goto err_devarg; 161 162 dev = da->bus->find_device(NULL, cmp_dev_name, da->name); 163 if (dev == NULL) { 164 RTE_LOG(ERR, EAL, "Cannot find device (%s)\n", 165 da->name); 166 ret = -ENODEV; 167 goto err_devarg; 168 } 169 /* Since there is a matching device, it is now its responsibility 170 * to manage the devargs we've just inserted. From this point 171 * those devargs shouldn't be removed manually anymore. 172 */ 173 174 ret = dev->bus->plug(dev); 175 if (ret > 0) 176 ret = -ENOTSUP; 177 178 if (ret && !rte_dev_is_probed(dev)) { /* if hasn't ever succeeded */ 179 RTE_LOG(ERR, EAL, "Driver cannot attach the device (%s)\n", 180 dev->name); 181 return ret; 182 } 183 184 *new_dev = dev; 185 return ret; 186 187 err_devarg: 188 if (rte_devargs_remove(da) != 0) { 189 free(da->args); 190 free(da); 191 } 192 return ret; 193 } 194 195 int 196 rte_dev_probe(const char *devargs) 197 { 198 struct eal_dev_mp_req req; 199 struct rte_device *dev; 200 int ret; 201 202 memset(&req, 0, sizeof(req)); 203 req.t = EAL_DEV_REQ_TYPE_ATTACH; 204 strlcpy(req.devargs, devargs, EAL_DEV_MP_DEV_ARGS_MAX_LEN); 205 206 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 207 /** 208 * If in secondary process, just send IPC request to 209 * primary process. 210 */ 211 ret = eal_dev_hotplug_request_to_primary(&req); 212 if (ret != 0) { 213 RTE_LOG(ERR, EAL, 214 "Failed to send hotplug request to primary\n"); 215 return -ENOMSG; 216 } 217 if (req.result != 0) 218 RTE_LOG(ERR, EAL, 219 "Failed to hotplug add device\n"); 220 return req.result; 221 } 222 223 /* attach a shared device from primary start from here: */ 224 225 /* primary attach the new device itself. */ 226 ret = local_dev_probe(devargs, &dev); 227 228 if (ret != 0) { 229 RTE_LOG(ERR, EAL, 230 "Failed to attach device on primary process\n"); 231 232 /** 233 * it is possible that secondary process failed to attached a 234 * device that primary process have during initialization, 235 * so for -EEXIST case, we still need to sync with secondary 236 * process. 237 */ 238 if (ret != -EEXIST) 239 return ret; 240 } 241 242 /* primary send attach sync request to secondary. */ 243 ret = eal_dev_hotplug_request_to_secondary(&req); 244 245 /* if any communication error, we need to rollback. */ 246 if (ret != 0) { 247 RTE_LOG(ERR, EAL, 248 "Failed to send hotplug add request to secondary\n"); 249 ret = -ENOMSG; 250 goto rollback; 251 } 252 253 /** 254 * if any secondary failed to attach, we need to consider if rollback 255 * is necessary. 256 */ 257 if (req.result != 0) { 258 RTE_LOG(ERR, EAL, 259 "Failed to attach device on secondary process\n"); 260 ret = req.result; 261 262 /* for -EEXIST, we don't need to rollback. */ 263 if (ret == -EEXIST) 264 return ret; 265 goto rollback; 266 } 267 268 return 0; 269 270 rollback: 271 req.t = EAL_DEV_REQ_TYPE_ATTACH_ROLLBACK; 272 273 /* primary send rollback request to secondary. */ 274 if (eal_dev_hotplug_request_to_secondary(&req) != 0) 275 RTE_LOG(WARNING, EAL, 276 "Failed to rollback device attach on secondary." 277 "Devices in secondary may not sync with primary\n"); 278 279 /* primary rollback itself. */ 280 if (local_dev_remove(dev) != 0) 281 RTE_LOG(WARNING, EAL, 282 "Failed to rollback device attach on primary." 283 "Devices in secondary may not sync with primary\n"); 284 285 return ret; 286 } 287 288 int 289 rte_eal_hotplug_remove(const char *busname, const char *devname) 290 { 291 struct rte_device *dev; 292 struct rte_bus *bus; 293 294 bus = rte_bus_find_by_name(busname); 295 if (bus == NULL) { 296 RTE_LOG(ERR, EAL, "Cannot find bus (%s)\n", busname); 297 return -ENOENT; 298 } 299 300 dev = bus->find_device(NULL, cmp_dev_name, devname); 301 if (dev == NULL) { 302 RTE_LOG(ERR, EAL, "Cannot find plugged device (%s)\n", devname); 303 return -EINVAL; 304 } 305 306 return rte_dev_remove(dev); 307 } 308 309 /* remove device at local process. */ 310 int 311 local_dev_remove(struct rte_device *dev) 312 { 313 int ret; 314 315 if (dev->bus->unplug == NULL) { 316 RTE_LOG(ERR, EAL, "Function unplug not supported by bus (%s)\n", 317 dev->bus->name); 318 return -ENOTSUP; 319 } 320 321 ret = dev->bus->unplug(dev); 322 if (ret) { 323 RTE_LOG(ERR, EAL, "Driver cannot detach the device (%s)\n", 324 dev->name); 325 return (ret < 0) ? ret : -ENOENT; 326 } 327 328 return 0; 329 } 330 331 int 332 rte_dev_remove(struct rte_device *dev) 333 { 334 struct eal_dev_mp_req req; 335 char *devargs; 336 int ret; 337 338 if (!rte_dev_is_probed(dev)) { 339 RTE_LOG(ERR, EAL, "Device is not probed\n"); 340 return -ENOENT; 341 } 342 343 ret = build_devargs(dev->bus->name, dev->name, "", &devargs); 344 if (ret != 0) 345 return ret; 346 347 memset(&req, 0, sizeof(req)); 348 req.t = EAL_DEV_REQ_TYPE_DETACH; 349 strlcpy(req.devargs, devargs, EAL_DEV_MP_DEV_ARGS_MAX_LEN); 350 free(devargs); 351 352 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 353 /** 354 * If in secondary process, just send IPC request to 355 * primary process. 356 */ 357 ret = eal_dev_hotplug_request_to_primary(&req); 358 if (ret != 0) { 359 RTE_LOG(ERR, EAL, 360 "Failed to send hotplug request to primary\n"); 361 return -ENOMSG; 362 } 363 if (req.result != 0) 364 RTE_LOG(ERR, EAL, 365 "Failed to hotplug remove device\n"); 366 return req.result; 367 } 368 369 /* detach a device from primary start from here: */ 370 371 /* primary send detach sync request to secondary */ 372 ret = eal_dev_hotplug_request_to_secondary(&req); 373 374 /** 375 * if communication error, we need to rollback, because it is possible 376 * part of the secondary processes still detached it successfully. 377 */ 378 if (ret != 0) { 379 RTE_LOG(ERR, EAL, 380 "Failed to send device detach request to secondary\n"); 381 ret = -ENOMSG; 382 goto rollback; 383 } 384 385 /** 386 * if any secondary failed to detach, we need to consider if rollback 387 * is necessary. 388 */ 389 if (req.result != 0) { 390 RTE_LOG(ERR, EAL, 391 "Failed to detach device on secondary process\n"); 392 ret = req.result; 393 /** 394 * if -ENOENT, we don't need to rollback, since devices is 395 * already detached on secondary process. 396 */ 397 if (ret != -ENOENT) 398 goto rollback; 399 } 400 401 /* primary detach the device itself. */ 402 ret = local_dev_remove(dev); 403 404 /* if primary failed, still need to consider if rollback is necessary */ 405 if (ret != 0) { 406 RTE_LOG(ERR, EAL, 407 "Failed to detach device on primary process\n"); 408 /* if -ENOENT, we don't need to rollback */ 409 if (ret == -ENOENT) 410 return ret; 411 goto rollback; 412 } 413 414 return 0; 415 416 rollback: 417 req.t = EAL_DEV_REQ_TYPE_DETACH_ROLLBACK; 418 419 /* primary send rollback request to secondary. */ 420 if (eal_dev_hotplug_request_to_secondary(&req) != 0) 421 RTE_LOG(WARNING, EAL, 422 "Failed to rollback device detach on secondary." 423 "Devices in secondary may not sync with primary\n"); 424 425 return ret; 426 } 427 428 int __rte_experimental 429 rte_dev_event_callback_register(const char *device_name, 430 rte_dev_event_cb_fn cb_fn, 431 void *cb_arg) 432 { 433 struct dev_event_callback *event_cb; 434 int ret; 435 436 if (!cb_fn) 437 return -EINVAL; 438 439 rte_spinlock_lock(&dev_event_lock); 440 441 if (TAILQ_EMPTY(&dev_event_cbs)) 442 TAILQ_INIT(&dev_event_cbs); 443 444 TAILQ_FOREACH(event_cb, &dev_event_cbs, next) { 445 if (event_cb->cb_fn == cb_fn && event_cb->cb_arg == cb_arg) { 446 if (device_name == NULL && event_cb->dev_name == NULL) 447 break; 448 if (device_name == NULL || event_cb->dev_name == NULL) 449 continue; 450 if (!strcmp(event_cb->dev_name, device_name)) 451 break; 452 } 453 } 454 455 /* create a new callback. */ 456 if (event_cb == NULL) { 457 event_cb = malloc(sizeof(struct dev_event_callback)); 458 if (event_cb != NULL) { 459 event_cb->cb_fn = cb_fn; 460 event_cb->cb_arg = cb_arg; 461 event_cb->active = 0; 462 if (!device_name) { 463 event_cb->dev_name = NULL; 464 } else { 465 event_cb->dev_name = strdup(device_name); 466 if (event_cb->dev_name == NULL) { 467 ret = -ENOMEM; 468 goto error; 469 } 470 } 471 TAILQ_INSERT_TAIL(&dev_event_cbs, event_cb, next); 472 } else { 473 RTE_LOG(ERR, EAL, 474 "Failed to allocate memory for device " 475 "event callback."); 476 ret = -ENOMEM; 477 goto error; 478 } 479 } else { 480 RTE_LOG(ERR, EAL, 481 "The callback is already exist, no need " 482 "to register again.\n"); 483 ret = -EEXIST; 484 } 485 486 rte_spinlock_unlock(&dev_event_lock); 487 return 0; 488 error: 489 free(event_cb); 490 rte_spinlock_unlock(&dev_event_lock); 491 return ret; 492 } 493 494 int __rte_experimental 495 rte_dev_event_callback_unregister(const char *device_name, 496 rte_dev_event_cb_fn cb_fn, 497 void *cb_arg) 498 { 499 int ret = 0; 500 struct dev_event_callback *event_cb, *next; 501 502 if (!cb_fn) 503 return -EINVAL; 504 505 rte_spinlock_lock(&dev_event_lock); 506 /*walk through the callbacks and remove all that match. */ 507 for (event_cb = TAILQ_FIRST(&dev_event_cbs); event_cb != NULL; 508 event_cb = next) { 509 510 next = TAILQ_NEXT(event_cb, next); 511 512 if (device_name != NULL && event_cb->dev_name != NULL) { 513 if (!strcmp(event_cb->dev_name, device_name)) { 514 if (event_cb->cb_fn != cb_fn || 515 (cb_arg != (void *)-1 && 516 event_cb->cb_arg != cb_arg)) 517 continue; 518 } 519 } else if (device_name != NULL) { 520 continue; 521 } 522 523 /* 524 * if this callback is not executing right now, 525 * then remove it. 526 */ 527 if (event_cb->active == 0) { 528 TAILQ_REMOVE(&dev_event_cbs, event_cb, next); 529 free(event_cb); 530 ret++; 531 } else { 532 continue; 533 } 534 } 535 rte_spinlock_unlock(&dev_event_lock); 536 return ret; 537 } 538 539 void __rte_experimental 540 rte_dev_event_callback_process(const char *device_name, 541 enum rte_dev_event_type event) 542 { 543 struct dev_event_callback *cb_lst; 544 545 if (device_name == NULL) 546 return; 547 548 rte_spinlock_lock(&dev_event_lock); 549 550 TAILQ_FOREACH(cb_lst, &dev_event_cbs, next) { 551 if (cb_lst->dev_name) { 552 if (strcmp(cb_lst->dev_name, device_name)) 553 continue; 554 } 555 cb_lst->active = 1; 556 rte_spinlock_unlock(&dev_event_lock); 557 cb_lst->cb_fn(device_name, event, 558 cb_lst->cb_arg); 559 rte_spinlock_lock(&dev_event_lock); 560 cb_lst->active = 0; 561 } 562 rte_spinlock_unlock(&dev_event_lock); 563 } 564 565 __rte_experimental 566 int 567 rte_dev_iterator_init(struct rte_dev_iterator *it, 568 const char *dev_str) 569 { 570 struct rte_devargs devargs; 571 struct rte_class *cls = NULL; 572 struct rte_bus *bus = NULL; 573 574 /* Having both bus_str and cls_str NULL is illegal, 575 * marking this iterator as invalid unless 576 * everything goes well. 577 */ 578 it->bus_str = NULL; 579 it->cls_str = NULL; 580 581 devargs.data = dev_str; 582 if (rte_devargs_layers_parse(&devargs, dev_str)) 583 goto get_out; 584 585 bus = devargs.bus; 586 cls = devargs.cls; 587 /* The string should have at least 588 * one layer specified. 589 */ 590 if (bus == NULL && cls == NULL) { 591 RTE_LOG(ERR, EAL, 592 "Either bus or class must be specified.\n"); 593 rte_errno = EINVAL; 594 goto get_out; 595 } 596 if (bus != NULL && bus->dev_iterate == NULL) { 597 RTE_LOG(ERR, EAL, "Bus %s not supported\n", bus->name); 598 rte_errno = ENOTSUP; 599 goto get_out; 600 } 601 if (cls != NULL && cls->dev_iterate == NULL) { 602 RTE_LOG(ERR, EAL, "Class %s not supported\n", cls->name); 603 rte_errno = ENOTSUP; 604 goto get_out; 605 } 606 it->bus_str = devargs.bus_str; 607 it->cls_str = devargs.cls_str; 608 it->dev_str = dev_str; 609 it->bus = bus; 610 it->cls = cls; 611 it->device = NULL; 612 it->class_device = NULL; 613 get_out: 614 return -rte_errno; 615 } 616 617 static char * 618 dev_str_sane_copy(const char *str) 619 { 620 size_t end; 621 char *copy; 622 623 end = strcspn(str, ",/"); 624 if (str[end] == ',') { 625 copy = strdup(&str[end + 1]); 626 } else { 627 /* '/' or '\0' */ 628 copy = strdup(""); 629 } 630 if (copy == NULL) { 631 rte_errno = ENOMEM; 632 } else { 633 char *slash; 634 635 slash = strchr(copy, '/'); 636 if (slash != NULL) 637 slash[0] = '\0'; 638 } 639 return copy; 640 } 641 642 static int 643 class_next_dev_cmp(const struct rte_class *cls, 644 const void *ctx) 645 { 646 struct rte_dev_iterator *it; 647 const char *cls_str = NULL; 648 void *dev; 649 650 if (cls->dev_iterate == NULL) 651 return 1; 652 it = ITCTX(ctx); 653 cls_str = CLSCTX(ctx); 654 dev = it->class_device; 655 /* it->cls_str != NULL means a class 656 * was specified in the devstr. 657 */ 658 if (it->cls_str != NULL && cls != it->cls) 659 return 1; 660 /* If an error occurred previously, 661 * no need to test further. 662 */ 663 if (rte_errno != 0) 664 return -1; 665 dev = cls->dev_iterate(dev, cls_str, it); 666 it->class_device = dev; 667 return dev == NULL; 668 } 669 670 static int 671 bus_next_dev_cmp(const struct rte_bus *bus, 672 const void *ctx) 673 { 674 struct rte_device *dev = NULL; 675 struct rte_class *cls = NULL; 676 struct rte_dev_iterator *it; 677 const char *bus_str = NULL; 678 679 if (bus->dev_iterate == NULL) 680 return 1; 681 it = ITCTX(ctx); 682 bus_str = BUSCTX(ctx); 683 dev = it->device; 684 /* it->bus_str != NULL means a bus 685 * was specified in the devstr. 686 */ 687 if (it->bus_str != NULL && bus != it->bus) 688 return 1; 689 /* If an error occurred previously, 690 * no need to test further. 691 */ 692 if (rte_errno != 0) 693 return -1; 694 if (it->cls_str == NULL) { 695 dev = bus->dev_iterate(dev, bus_str, it); 696 goto end; 697 } 698 /* cls_str != NULL */ 699 if (dev == NULL) { 700 next_dev_on_bus: 701 dev = bus->dev_iterate(dev, bus_str, it); 702 it->device = dev; 703 } 704 if (dev == NULL) 705 return 1; 706 if (it->cls != NULL) 707 cls = TAILQ_PREV(it->cls, rte_class_list, next); 708 cls = rte_class_find(cls, class_next_dev_cmp, ctx); 709 if (cls != NULL) { 710 it->cls = cls; 711 goto end; 712 } 713 goto next_dev_on_bus; 714 end: 715 it->device = dev; 716 return dev == NULL; 717 } 718 __rte_experimental 719 struct rte_device * 720 rte_dev_iterator_next(struct rte_dev_iterator *it) 721 { 722 struct rte_bus *bus = NULL; 723 int old_errno = rte_errno; 724 char *bus_str = NULL; 725 char *cls_str = NULL; 726 727 rte_errno = 0; 728 if (it->bus_str == NULL && it->cls_str == NULL) { 729 /* Invalid iterator. */ 730 rte_errno = EINVAL; 731 return NULL; 732 } 733 if (it->bus != NULL) 734 bus = TAILQ_PREV(it->bus, rte_bus_list, next); 735 if (it->bus_str != NULL) { 736 bus_str = dev_str_sane_copy(it->bus_str); 737 if (bus_str == NULL) 738 goto out; 739 } 740 if (it->cls_str != NULL) { 741 cls_str = dev_str_sane_copy(it->cls_str); 742 if (cls_str == NULL) 743 goto out; 744 } 745 while ((bus = rte_bus_find(bus, bus_next_dev_cmp, 746 CTX(it, bus_str, cls_str)))) { 747 if (it->device != NULL) { 748 it->bus = bus; 749 goto out; 750 } 751 if (it->bus_str != NULL || 752 rte_errno != 0) 753 break; 754 } 755 if (rte_errno == 0) 756 rte_errno = old_errno; 757 out: 758 free(bus_str); 759 free(cls_str); 760 return it->device; 761 } 762