1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2012 6WIND S.A. 3 * Copyright 2012 Mellanox Technologies, Ltd 4 */ 5 6 /** 7 * @file 8 * mlx4 driver initialization. 9 */ 10 11 #include <assert.h> 12 #include <dlfcn.h> 13 #include <errno.h> 14 #include <inttypes.h> 15 #include <stddef.h> 16 #include <stdint.h> 17 #include <stdio.h> 18 #include <stdlib.h> 19 #include <string.h> 20 #include <unistd.h> 21 22 /* Verbs headers do not support -pedantic. */ 23 #ifdef PEDANTIC 24 #pragma GCC diagnostic ignored "-Wpedantic" 25 #endif 26 #include <infiniband/verbs.h> 27 #ifdef PEDANTIC 28 #pragma GCC diagnostic error "-Wpedantic" 29 #endif 30 31 #include <rte_common.h> 32 #include <rte_config.h> 33 #include <rte_dev.h> 34 #include <rte_errno.h> 35 #include <rte_ethdev_driver.h> 36 #include <rte_ethdev_pci.h> 37 #include <rte_ether.h> 38 #include <rte_flow.h> 39 #include <rte_interrupts.h> 40 #include <rte_kvargs.h> 41 #include <rte_malloc.h> 42 #include <rte_mbuf.h> 43 44 #include "mlx4.h" 45 #include "mlx4_glue.h" 46 #include "mlx4_flow.h" 47 #include "mlx4_rxtx.h" 48 #include "mlx4_utils.h" 49 50 /** Configuration structure for device arguments. */ 51 struct mlx4_conf { 52 struct { 53 uint32_t present; /**< Bit-field for existing ports. */ 54 uint32_t enabled; /**< Bit-field for user-enabled ports. */ 55 } ports; 56 }; 57 58 /* Available parameters list. */ 59 const char *pmd_mlx4_init_params[] = { 60 MLX4_PMD_PORT_KVARG, 61 NULL, 62 }; 63 64 /** 65 * DPDK callback for Ethernet device configuration. 66 * 67 * @param dev 68 * Pointer to Ethernet device structure. 69 * 70 * @return 71 * 0 on success, negative errno value otherwise and rte_errno is set. 72 */ 73 static int 74 mlx4_dev_configure(struct rte_eth_dev *dev) 75 { 76 struct priv *priv = dev->data->dev_private; 77 struct rte_flow_error error; 78 int ret; 79 80 /* Prepare internal flow rules. */ 81 ret = mlx4_flow_sync(priv, &error); 82 if (ret) { 83 ERROR("cannot set up internal flow rules (code %d, \"%s\")," 84 " flow error type %d, cause %p, message: %s", 85 -ret, strerror(-ret), error.type, error.cause, 86 error.message ? error.message : "(unspecified)"); 87 goto exit; 88 } 89 ret = mlx4_intr_install(priv); 90 if (ret) 91 ERROR("%p: interrupt handler installation failed", 92 (void *)dev); 93 exit: 94 return ret; 95 } 96 97 /** 98 * DPDK callback to start the device. 99 * 100 * Simulate device start by initializing common RSS resources and attaching 101 * all configured flows. 102 * 103 * @param dev 104 * Pointer to Ethernet device structure. 105 * 106 * @return 107 * 0 on success, negative errno value otherwise and rte_errno is set. 108 */ 109 static int 110 mlx4_dev_start(struct rte_eth_dev *dev) 111 { 112 struct priv *priv = dev->data->dev_private; 113 struct rte_flow_error error; 114 int ret; 115 116 if (priv->started) 117 return 0; 118 DEBUG("%p: attaching configured flows to all RX queues", (void *)dev); 119 priv->started = 1; 120 ret = mlx4_rss_init(priv); 121 if (ret) { 122 ERROR("%p: cannot initialize RSS resources: %s", 123 (void *)dev, strerror(-ret)); 124 goto err; 125 } 126 ret = mlx4_rxq_intr_enable(priv); 127 if (ret) { 128 ERROR("%p: interrupt handler installation failed", 129 (void *)dev); 130 goto err; 131 } 132 ret = mlx4_flow_sync(priv, &error); 133 if (ret) { 134 ERROR("%p: cannot attach flow rules (code %d, \"%s\")," 135 " flow error type %d, cause %p, message: %s", 136 (void *)dev, 137 -ret, strerror(-ret), error.type, error.cause, 138 error.message ? error.message : "(unspecified)"); 139 goto err; 140 } 141 rte_wmb(); 142 dev->tx_pkt_burst = mlx4_tx_burst; 143 dev->rx_pkt_burst = mlx4_rx_burst; 144 return 0; 145 err: 146 /* Rollback. */ 147 priv->started = 0; 148 return ret; 149 } 150 151 /** 152 * DPDK callback to stop the device. 153 * 154 * Simulate device stop by detaching all configured flows. 155 * 156 * @param dev 157 * Pointer to Ethernet device structure. 158 */ 159 static void 160 mlx4_dev_stop(struct rte_eth_dev *dev) 161 { 162 struct priv *priv = dev->data->dev_private; 163 164 if (!priv->started) 165 return; 166 DEBUG("%p: detaching flows from all RX queues", (void *)dev); 167 priv->started = 0; 168 dev->tx_pkt_burst = mlx4_tx_burst_removed; 169 dev->rx_pkt_burst = mlx4_rx_burst_removed; 170 rte_wmb(); 171 mlx4_flow_sync(priv, NULL); 172 mlx4_rxq_intr_disable(priv); 173 mlx4_rss_deinit(priv); 174 } 175 176 /** 177 * DPDK callback to close the device. 178 * 179 * Destroy all queues and objects, free memory. 180 * 181 * @param dev 182 * Pointer to Ethernet device structure. 183 */ 184 static void 185 mlx4_dev_close(struct rte_eth_dev *dev) 186 { 187 struct priv *priv = dev->data->dev_private; 188 unsigned int i; 189 190 DEBUG("%p: closing device \"%s\"", 191 (void *)dev, 192 ((priv->ctx != NULL) ? priv->ctx->device->name : "")); 193 dev->rx_pkt_burst = mlx4_rx_burst_removed; 194 dev->tx_pkt_burst = mlx4_tx_burst_removed; 195 rte_wmb(); 196 mlx4_flow_clean(priv); 197 for (i = 0; i != dev->data->nb_rx_queues; ++i) 198 mlx4_rx_queue_release(dev->data->rx_queues[i]); 199 for (i = 0; i != dev->data->nb_tx_queues; ++i) 200 mlx4_tx_queue_release(dev->data->tx_queues[i]); 201 if (priv->pd != NULL) { 202 assert(priv->ctx != NULL); 203 claim_zero(mlx4_glue->dealloc_pd(priv->pd)); 204 claim_zero(mlx4_glue->close_device(priv->ctx)); 205 } else 206 assert(priv->ctx == NULL); 207 mlx4_intr_uninstall(priv); 208 memset(priv, 0, sizeof(*priv)); 209 } 210 211 static const struct eth_dev_ops mlx4_dev_ops = { 212 .dev_configure = mlx4_dev_configure, 213 .dev_start = mlx4_dev_start, 214 .dev_stop = mlx4_dev_stop, 215 .dev_set_link_down = mlx4_dev_set_link_down, 216 .dev_set_link_up = mlx4_dev_set_link_up, 217 .dev_close = mlx4_dev_close, 218 .link_update = mlx4_link_update, 219 .promiscuous_enable = mlx4_promiscuous_enable, 220 .promiscuous_disable = mlx4_promiscuous_disable, 221 .allmulticast_enable = mlx4_allmulticast_enable, 222 .allmulticast_disable = mlx4_allmulticast_disable, 223 .mac_addr_remove = mlx4_mac_addr_remove, 224 .mac_addr_add = mlx4_mac_addr_add, 225 .mac_addr_set = mlx4_mac_addr_set, 226 .stats_get = mlx4_stats_get, 227 .stats_reset = mlx4_stats_reset, 228 .dev_infos_get = mlx4_dev_infos_get, 229 .dev_supported_ptypes_get = mlx4_dev_supported_ptypes_get, 230 .vlan_filter_set = mlx4_vlan_filter_set, 231 .rx_queue_setup = mlx4_rx_queue_setup, 232 .tx_queue_setup = mlx4_tx_queue_setup, 233 .rx_queue_release = mlx4_rx_queue_release, 234 .tx_queue_release = mlx4_tx_queue_release, 235 .flow_ctrl_get = mlx4_flow_ctrl_get, 236 .flow_ctrl_set = mlx4_flow_ctrl_set, 237 .mtu_set = mlx4_mtu_set, 238 .filter_ctrl = mlx4_filter_ctrl, 239 .rx_queue_intr_enable = mlx4_rx_intr_enable, 240 .rx_queue_intr_disable = mlx4_rx_intr_disable, 241 .is_removed = mlx4_is_removed, 242 }; 243 244 /** 245 * Get PCI information from struct ibv_device. 246 * 247 * @param device 248 * Pointer to Ethernet device structure. 249 * @param[out] pci_addr 250 * PCI bus address output buffer. 251 * 252 * @return 253 * 0 on success, negative errno value otherwise and rte_errno is set. 254 */ 255 static int 256 mlx4_ibv_device_to_pci_addr(const struct ibv_device *device, 257 struct rte_pci_addr *pci_addr) 258 { 259 FILE *file; 260 char line[32]; 261 MKSTR(path, "%s/device/uevent", device->ibdev_path); 262 263 file = fopen(path, "rb"); 264 if (file == NULL) { 265 rte_errno = errno; 266 return -rte_errno; 267 } 268 while (fgets(line, sizeof(line), file) == line) { 269 size_t len = strlen(line); 270 int ret; 271 272 /* Truncate long lines. */ 273 if (len == (sizeof(line) - 1)) 274 while (line[(len - 1)] != '\n') { 275 ret = fgetc(file); 276 if (ret == EOF) 277 break; 278 line[(len - 1)] = ret; 279 } 280 /* Extract information. */ 281 if (sscanf(line, 282 "PCI_SLOT_NAME=" 283 "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n", 284 &pci_addr->domain, 285 &pci_addr->bus, 286 &pci_addr->devid, 287 &pci_addr->function) == 4) { 288 ret = 0; 289 break; 290 } 291 } 292 fclose(file); 293 return 0; 294 } 295 296 /** 297 * Verify and store value for device argument. 298 * 299 * @param[in] key 300 * Key argument to verify. 301 * @param[in] val 302 * Value associated with key. 303 * @param[in, out] conf 304 * Shared configuration data. 305 * 306 * @return 307 * 0 on success, negative errno value otherwise and rte_errno is set. 308 */ 309 static int 310 mlx4_arg_parse(const char *key, const char *val, struct mlx4_conf *conf) 311 { 312 unsigned long tmp; 313 314 errno = 0; 315 tmp = strtoul(val, NULL, 0); 316 if (errno) { 317 rte_errno = errno; 318 WARN("%s: \"%s\" is not a valid integer", key, val); 319 return -rte_errno; 320 } 321 if (strcmp(MLX4_PMD_PORT_KVARG, key) == 0) { 322 uint32_t ports = rte_log2_u32(conf->ports.present + 1); 323 324 if (tmp >= ports) { 325 ERROR("port index %lu outside range [0,%" PRIu32 ")", 326 tmp, ports); 327 return -EINVAL; 328 } 329 if (!(conf->ports.present & (1 << tmp))) { 330 rte_errno = EINVAL; 331 ERROR("invalid port index %lu", tmp); 332 return -rte_errno; 333 } 334 conf->ports.enabled |= 1 << tmp; 335 } else { 336 rte_errno = EINVAL; 337 WARN("%s: unknown parameter", key); 338 return -rte_errno; 339 } 340 return 0; 341 } 342 343 /** 344 * Parse device parameters. 345 * 346 * @param devargs 347 * Device arguments structure. 348 * 349 * @return 350 * 0 on success, negative errno value otherwise and rte_errno is set. 351 */ 352 static int 353 mlx4_args(struct rte_devargs *devargs, struct mlx4_conf *conf) 354 { 355 struct rte_kvargs *kvlist; 356 unsigned int arg_count; 357 int ret = 0; 358 int i; 359 360 if (devargs == NULL) 361 return 0; 362 kvlist = rte_kvargs_parse(devargs->args, pmd_mlx4_init_params); 363 if (kvlist == NULL) { 364 rte_errno = EINVAL; 365 ERROR("failed to parse kvargs"); 366 return -rte_errno; 367 } 368 /* Process parameters. */ 369 for (i = 0; pmd_mlx4_init_params[i]; ++i) { 370 arg_count = rte_kvargs_count(kvlist, MLX4_PMD_PORT_KVARG); 371 while (arg_count-- > 0) { 372 ret = rte_kvargs_process(kvlist, 373 MLX4_PMD_PORT_KVARG, 374 (int (*)(const char *, 375 const char *, 376 void *)) 377 mlx4_arg_parse, 378 conf); 379 if (ret != 0) 380 goto free_kvlist; 381 } 382 } 383 free_kvlist: 384 rte_kvargs_free(kvlist); 385 return ret; 386 } 387 388 static struct rte_pci_driver mlx4_driver; 389 390 /** 391 * DPDK callback to register a PCI device. 392 * 393 * This function creates an Ethernet device for each port of a given 394 * PCI device. 395 * 396 * @param[in] pci_drv 397 * PCI driver structure (mlx4_driver). 398 * @param[in] pci_dev 399 * PCI device information. 400 * 401 * @return 402 * 0 on success, negative errno value otherwise and rte_errno is set. 403 */ 404 static int 405 mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) 406 { 407 struct ibv_device **list; 408 struct ibv_device *ibv_dev; 409 int err = 0; 410 struct ibv_context *attr_ctx = NULL; 411 struct ibv_device_attr device_attr; 412 struct ibv_device_attr_ex device_attr_ex; 413 struct mlx4_conf conf = { 414 .ports.present = 0, 415 }; 416 unsigned int vf; 417 int i; 418 419 (void)pci_drv; 420 assert(pci_drv == &mlx4_driver); 421 list = mlx4_glue->get_device_list(&i); 422 if (list == NULL) { 423 rte_errno = errno; 424 assert(rte_errno); 425 if (rte_errno == ENOSYS) 426 ERROR("cannot list devices, is ib_uverbs loaded?"); 427 return -rte_errno; 428 } 429 assert(i >= 0); 430 /* 431 * For each listed device, check related sysfs entry against 432 * the provided PCI ID. 433 */ 434 while (i != 0) { 435 struct rte_pci_addr pci_addr; 436 437 --i; 438 DEBUG("checking device \"%s\"", list[i]->name); 439 if (mlx4_ibv_device_to_pci_addr(list[i], &pci_addr)) 440 continue; 441 if ((pci_dev->addr.domain != pci_addr.domain) || 442 (pci_dev->addr.bus != pci_addr.bus) || 443 (pci_dev->addr.devid != pci_addr.devid) || 444 (pci_dev->addr.function != pci_addr.function)) 445 continue; 446 vf = (pci_dev->id.device_id == 447 PCI_DEVICE_ID_MELLANOX_CONNECTX3VF); 448 INFO("PCI information matches, using device \"%s\" (VF: %s)", 449 list[i]->name, (vf ? "true" : "false")); 450 attr_ctx = mlx4_glue->open_device(list[i]); 451 err = errno; 452 break; 453 } 454 if (attr_ctx == NULL) { 455 mlx4_glue->free_device_list(list); 456 switch (err) { 457 case 0: 458 rte_errno = ENODEV; 459 ERROR("cannot access device, is mlx4_ib loaded?"); 460 return -rte_errno; 461 case EINVAL: 462 rte_errno = EINVAL; 463 ERROR("cannot use device, are drivers up to date?"); 464 return -rte_errno; 465 } 466 assert(err > 0); 467 rte_errno = err; 468 return -rte_errno; 469 } 470 ibv_dev = list[i]; 471 DEBUG("device opened"); 472 if (mlx4_glue->query_device(attr_ctx, &device_attr)) { 473 rte_errno = ENODEV; 474 goto error; 475 } 476 INFO("%u port(s) detected", device_attr.phys_port_cnt); 477 conf.ports.present |= (UINT64_C(1) << device_attr.phys_port_cnt) - 1; 478 if (mlx4_args(pci_dev->device.devargs, &conf)) { 479 ERROR("failed to process device arguments"); 480 rte_errno = EINVAL; 481 goto error; 482 } 483 /* Use all ports when none are defined */ 484 if (!conf.ports.enabled) 485 conf.ports.enabled = conf.ports.present; 486 /* Retrieve extended device attributes. */ 487 if (mlx4_glue->query_device_ex(attr_ctx, NULL, &device_attr_ex)) { 488 rte_errno = ENODEV; 489 goto error; 490 } 491 assert(device_attr.max_sge >= MLX4_MAX_SGE); 492 for (i = 0; i < device_attr.phys_port_cnt; i++) { 493 uint32_t port = i + 1; /* ports are indexed from one */ 494 struct ibv_context *ctx = NULL; 495 struct ibv_port_attr port_attr; 496 struct ibv_pd *pd = NULL; 497 struct priv *priv = NULL; 498 struct rte_eth_dev *eth_dev = NULL; 499 struct ether_addr mac; 500 501 /* If port is not enabled, skip. */ 502 if (!(conf.ports.enabled & (1 << i))) 503 continue; 504 DEBUG("using port %u", port); 505 ctx = mlx4_glue->open_device(ibv_dev); 506 if (ctx == NULL) { 507 rte_errno = ENODEV; 508 goto port_error; 509 } 510 /* Check port status. */ 511 err = mlx4_glue->query_port(ctx, port, &port_attr); 512 if (err) { 513 rte_errno = err; 514 ERROR("port query failed: %s", strerror(rte_errno)); 515 goto port_error; 516 } 517 if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) { 518 rte_errno = ENOTSUP; 519 ERROR("port %d is not configured in Ethernet mode", 520 port); 521 goto port_error; 522 } 523 if (port_attr.state != IBV_PORT_ACTIVE) 524 DEBUG("port %d is not active: \"%s\" (%d)", 525 port, mlx4_glue->port_state_str(port_attr.state), 526 port_attr.state); 527 /* Make asynchronous FD non-blocking to handle interrupts. */ 528 if (mlx4_fd_set_non_blocking(ctx->async_fd) < 0) { 529 ERROR("cannot make asynchronous FD non-blocking: %s", 530 strerror(rte_errno)); 531 goto port_error; 532 } 533 /* Allocate protection domain. */ 534 pd = mlx4_glue->alloc_pd(ctx); 535 if (pd == NULL) { 536 rte_errno = ENOMEM; 537 ERROR("PD allocation failure"); 538 goto port_error; 539 } 540 /* from rte_ethdev.c */ 541 priv = rte_zmalloc("ethdev private structure", 542 sizeof(*priv), 543 RTE_CACHE_LINE_SIZE); 544 if (priv == NULL) { 545 rte_errno = ENOMEM; 546 ERROR("priv allocation failure"); 547 goto port_error; 548 } 549 priv->ctx = ctx; 550 priv->device_attr = device_attr; 551 priv->port = port; 552 priv->pd = pd; 553 priv->mtu = ETHER_MTU; 554 priv->vf = vf; 555 priv->hw_csum = !!(device_attr.device_cap_flags & 556 IBV_DEVICE_RAW_IP_CSUM); 557 DEBUG("checksum offloading is %ssupported", 558 (priv->hw_csum ? "" : "not ")); 559 /* Only ConnectX-3 Pro supports tunneling. */ 560 priv->hw_csum_l2tun = 561 priv->hw_csum && 562 (device_attr.vendor_part_id == 563 PCI_DEVICE_ID_MELLANOX_CONNECTX3PRO); 564 DEBUG("L2 tunnel checksum offloads are %ssupported", 565 (priv->hw_csum_l2tun ? "" : "not ")); 566 priv->hw_rss_sup = device_attr_ex.rss_caps.rx_hash_fields_mask; 567 if (!priv->hw_rss_sup) { 568 WARN("no RSS capabilities reported; disabling support" 569 " for UDP RSS and inner VXLAN RSS"); 570 /* Fake support for all possible RSS hash fields. */ 571 priv->hw_rss_sup = ~UINT64_C(0); 572 priv->hw_rss_sup = mlx4_conv_rss_hf(priv, -1); 573 /* Filter out known unsupported fields. */ 574 priv->hw_rss_sup &= 575 ~(uint64_t)(IBV_RX_HASH_SRC_PORT_UDP | 576 IBV_RX_HASH_DST_PORT_UDP | 577 IBV_RX_HASH_INNER); 578 } 579 DEBUG("supported RSS hash fields mask: %016" PRIx64, 580 priv->hw_rss_sup); 581 /* Configure the first MAC address by default. */ 582 if (mlx4_get_mac(priv, &mac.addr_bytes)) { 583 ERROR("cannot get MAC address, is mlx4_en loaded?" 584 " (rte_errno: %s)", strerror(rte_errno)); 585 goto port_error; 586 } 587 INFO("port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x", 588 priv->port, 589 mac.addr_bytes[0], mac.addr_bytes[1], 590 mac.addr_bytes[2], mac.addr_bytes[3], 591 mac.addr_bytes[4], mac.addr_bytes[5]); 592 /* Register MAC address. */ 593 priv->mac[0] = mac; 594 #ifndef NDEBUG 595 { 596 char ifname[IF_NAMESIZE]; 597 598 if (mlx4_get_ifname(priv, &ifname) == 0) 599 DEBUG("port %u ifname is \"%s\"", 600 priv->port, ifname); 601 else 602 DEBUG("port %u ifname is unknown", priv->port); 603 } 604 #endif 605 /* Get actual MTU if possible. */ 606 mlx4_mtu_get(priv, &priv->mtu); 607 DEBUG("port %u MTU is %u", priv->port, priv->mtu); 608 /* from rte_ethdev.c */ 609 { 610 char name[RTE_ETH_NAME_MAX_LEN]; 611 612 snprintf(name, sizeof(name), "%s port %u", 613 mlx4_glue->get_device_name(ibv_dev), port); 614 eth_dev = rte_eth_dev_allocate(name); 615 } 616 if (eth_dev == NULL) { 617 ERROR("can not allocate rte ethdev"); 618 rte_errno = ENOMEM; 619 goto port_error; 620 } 621 eth_dev->data->dev_private = priv; 622 eth_dev->data->mac_addrs = priv->mac; 623 eth_dev->device = &pci_dev->device; 624 rte_eth_copy_pci_info(eth_dev, pci_dev); 625 eth_dev->device->driver = &mlx4_driver.driver; 626 /* Initialize local interrupt handle for current port. */ 627 priv->intr_handle = (struct rte_intr_handle){ 628 .fd = -1, 629 .type = RTE_INTR_HANDLE_EXT, 630 }; 631 /* 632 * Override ethdev interrupt handle pointer with private 633 * handle instead of that of the parent PCI device used by 634 * default. This prevents it from being shared between all 635 * ports of the same PCI device since each of them is 636 * associated its own Verbs context. 637 * 638 * Rx interrupts in particular require this as the PMD has 639 * no control over the registration of queue interrupts 640 * besides setting up eth_dev->intr_handle, the rest is 641 * handled by rte_intr_rx_ctl(). 642 */ 643 eth_dev->intr_handle = &priv->intr_handle; 644 priv->dev = eth_dev; 645 eth_dev->dev_ops = &mlx4_dev_ops; 646 /* Bring Ethernet device up. */ 647 DEBUG("forcing Ethernet interface up"); 648 mlx4_dev_set_link_up(priv->dev); 649 /* Update link status once if waiting for LSC. */ 650 if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) 651 mlx4_link_update(eth_dev, 0); 652 continue; 653 port_error: 654 rte_free(priv); 655 if (pd) 656 claim_zero(mlx4_glue->dealloc_pd(pd)); 657 if (ctx) 658 claim_zero(mlx4_glue->close_device(ctx)); 659 if (eth_dev) 660 rte_eth_dev_release_port(eth_dev); 661 break; 662 } 663 if (i == device_attr.phys_port_cnt) 664 return 0; 665 /* 666 * XXX if something went wrong in the loop above, there is a resource 667 * leak (ctx, pd, priv, dpdk ethdev) but we can do nothing about it as 668 * long as the dpdk does not provide a way to deallocate a ethdev and a 669 * way to enumerate the registered ethdevs to free the previous ones. 670 */ 671 error: 672 if (attr_ctx) 673 claim_zero(mlx4_glue->close_device(attr_ctx)); 674 if (list) 675 mlx4_glue->free_device_list(list); 676 assert(rte_errno >= 0); 677 return -rte_errno; 678 } 679 680 static const struct rte_pci_id mlx4_pci_id_map[] = { 681 { 682 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 683 PCI_DEVICE_ID_MELLANOX_CONNECTX3) 684 }, 685 { 686 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 687 PCI_DEVICE_ID_MELLANOX_CONNECTX3PRO) 688 }, 689 { 690 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 691 PCI_DEVICE_ID_MELLANOX_CONNECTX3VF) 692 }, 693 { 694 .vendor_id = 0 695 } 696 }; 697 698 static struct rte_pci_driver mlx4_driver = { 699 .driver = { 700 .name = MLX4_DRIVER_NAME 701 }, 702 .id_table = mlx4_pci_id_map, 703 .probe = mlx4_pci_probe, 704 .drv_flags = RTE_PCI_DRV_INTR_LSC | 705 RTE_PCI_DRV_INTR_RMV, 706 }; 707 708 #ifdef RTE_LIBRTE_MLX4_DLOPEN_DEPS 709 710 /** 711 * Suffix RTE_EAL_PMD_PATH with "-glue". 712 * 713 * This function performs a sanity check on RTE_EAL_PMD_PATH before 714 * suffixing its last component. 715 * 716 * @param buf[out] 717 * Output buffer, should be large enough otherwise NULL is returned. 718 * @param size 719 * Size of @p out. 720 * 721 * @return 722 * Pointer to @p buf or @p NULL in case suffix cannot be appended. 723 */ 724 static char * 725 mlx4_glue_path(char *buf, size_t size) 726 { 727 static const char *const bad[] = { "/", ".", "..", NULL }; 728 const char *path = RTE_EAL_PMD_PATH; 729 size_t len = strlen(path); 730 size_t off; 731 int i; 732 733 while (len && path[len - 1] == '/') 734 --len; 735 for (off = len; off && path[off - 1] != '/'; --off) 736 ; 737 for (i = 0; bad[i]; ++i) 738 if (!strncmp(path + off, bad[i], (int)(len - off))) 739 goto error; 740 i = snprintf(buf, size, "%.*s-glue", (int)len, path); 741 if (i == -1 || (size_t)i >= size) 742 goto error; 743 return buf; 744 error: 745 ERROR("unable to append \"-glue\" to last component of" 746 " RTE_EAL_PMD_PATH (\"" RTE_EAL_PMD_PATH "\")," 747 " please re-configure DPDK"); 748 return NULL; 749 } 750 751 /** 752 * Initialization routine for run-time dependency on rdma-core. 753 */ 754 static int 755 mlx4_glue_init(void) 756 { 757 char glue_path[sizeof(RTE_EAL_PMD_PATH) - 1 + sizeof("-glue")]; 758 const char *path[] = { 759 /* 760 * A basic security check is necessary before trusting 761 * MLX4_GLUE_PATH, which may override RTE_EAL_PMD_PATH. 762 */ 763 (geteuid() == getuid() && getegid() == getgid() ? 764 getenv("MLX4_GLUE_PATH") : NULL), 765 /* 766 * When RTE_EAL_PMD_PATH is set, use its glue-suffixed 767 * variant, otherwise let dlopen() look up libraries on its 768 * own. 769 */ 770 (*RTE_EAL_PMD_PATH ? 771 mlx4_glue_path(glue_path, sizeof(glue_path)) : ""), 772 }; 773 unsigned int i = 0; 774 void *handle = NULL; 775 void **sym; 776 const char *dlmsg; 777 778 while (!handle && i != RTE_DIM(path)) { 779 const char *end; 780 size_t len; 781 int ret; 782 783 if (!path[i]) { 784 ++i; 785 continue; 786 } 787 end = strpbrk(path[i], ":;"); 788 if (!end) 789 end = path[i] + strlen(path[i]); 790 len = end - path[i]; 791 ret = 0; 792 do { 793 char name[ret + 1]; 794 795 ret = snprintf(name, sizeof(name), "%.*s%s" MLX4_GLUE, 796 (int)len, path[i], 797 (!len || *(end - 1) == '/') ? "" : "/"); 798 if (ret == -1) 799 break; 800 if (sizeof(name) != (size_t)ret + 1) 801 continue; 802 DEBUG("looking for rdma-core glue as \"%s\"", name); 803 handle = dlopen(name, RTLD_LAZY); 804 break; 805 } while (1); 806 path[i] = end + 1; 807 if (!*end) 808 ++i; 809 } 810 if (!handle) { 811 rte_errno = EINVAL; 812 dlmsg = dlerror(); 813 if (dlmsg) 814 WARN("cannot load glue library: %s", dlmsg); 815 goto glue_error; 816 } 817 sym = dlsym(handle, "mlx4_glue"); 818 if (!sym || !*sym) { 819 rte_errno = EINVAL; 820 dlmsg = dlerror(); 821 if (dlmsg) 822 ERROR("cannot resolve glue symbol: %s", dlmsg); 823 goto glue_error; 824 } 825 mlx4_glue = *sym; 826 return 0; 827 glue_error: 828 if (handle) 829 dlclose(handle); 830 WARN("cannot initialize PMD due to missing run-time" 831 " dependency on rdma-core libraries (libibverbs," 832 " libmlx4)"); 833 return -rte_errno; 834 } 835 836 #endif 837 838 /** 839 * Driver initialization routine. 840 */ 841 RTE_INIT(rte_mlx4_pmd_init); 842 static void 843 rte_mlx4_pmd_init(void) 844 { 845 /* 846 * MLX4_DEVICE_FATAL_CLEANUP tells ibv_destroy functions we 847 * want to get success errno value in case of calling them 848 * when the device was removed. 849 */ 850 setenv("MLX4_DEVICE_FATAL_CLEANUP", "1", 1); 851 /* 852 * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use 853 * huge pages. Calling ibv_fork_init() during init allows 854 * applications to use fork() safely for purposes other than 855 * using this PMD, which is not supported in forked processes. 856 */ 857 setenv("RDMAV_HUGEPAGES_SAFE", "1", 1); 858 #ifdef RTE_LIBRTE_MLX4_DLOPEN_DEPS 859 if (mlx4_glue_init()) 860 return; 861 assert(mlx4_glue); 862 #endif 863 #ifndef NDEBUG 864 /* Glue structure must not contain any NULL pointers. */ 865 { 866 unsigned int i; 867 868 for (i = 0; i != sizeof(*mlx4_glue) / sizeof(void *); ++i) 869 assert(((const void *const *)mlx4_glue)[i]); 870 } 871 #endif 872 if (strcmp(mlx4_glue->version, MLX4_GLUE_VERSION)) { 873 ERROR("rdma-core glue \"%s\" mismatch: \"%s\" is required", 874 mlx4_glue->version, MLX4_GLUE_VERSION); 875 return; 876 } 877 mlx4_glue->fork_init(); 878 rte_pci_register(&mlx4_driver); 879 } 880 881 RTE_PMD_EXPORT_NAME(net_mlx4, __COUNTER__); 882 RTE_PMD_REGISTER_PCI_TABLE(net_mlx4, mlx4_pci_id_map); 883 RTE_PMD_REGISTER_KMOD_DEP(net_mlx4, 884 "* ib_uverbs & mlx4_en & mlx4_core & mlx4_ib"); 885