1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2012 6WIND S.A. 3 * Copyright 2012 Mellanox Technologies, Ltd 4 */ 5 6 /** 7 * @file 8 * mlx4 driver initialization. 9 */ 10 11 #include <assert.h> 12 #include <dlfcn.h> 13 #include <errno.h> 14 #include <inttypes.h> 15 #include <stddef.h> 16 #include <stdint.h> 17 #include <stdio.h> 18 #include <stdlib.h> 19 #include <string.h> 20 #include <unistd.h> 21 22 /* Verbs headers do not support -pedantic. */ 23 #ifdef PEDANTIC 24 #pragma GCC diagnostic ignored "-Wpedantic" 25 #endif 26 #include <infiniband/verbs.h> 27 #ifdef PEDANTIC 28 #pragma GCC diagnostic error "-Wpedantic" 29 #endif 30 31 #include <rte_common.h> 32 #include <rte_config.h> 33 #include <rte_dev.h> 34 #include <rte_errno.h> 35 #include <rte_ethdev_driver.h> 36 #include <rte_ethdev_pci.h> 37 #include <rte_ether.h> 38 #include <rte_flow.h> 39 #include <rte_interrupts.h> 40 #include <rte_kvargs.h> 41 #include <rte_malloc.h> 42 #include <rte_mbuf.h> 43 44 #include "mlx4.h" 45 #include "mlx4_glue.h" 46 #include "mlx4_flow.h" 47 #include "mlx4_mr.h" 48 #include "mlx4_rxtx.h" 49 #include "mlx4_utils.h" 50 51 struct mlx4_dev_list mlx4_mem_event_cb_list = 52 LIST_HEAD_INITIALIZER(mlx4_mem_event_cb_list); 53 54 rte_rwlock_t mlx4_mem_event_rwlock = RTE_RWLOCK_INITIALIZER; 55 56 /** Configuration structure for device arguments. */ 57 struct mlx4_conf { 58 struct { 59 uint32_t present; /**< Bit-field for existing ports. */ 60 uint32_t enabled; /**< Bit-field for user-enabled ports. */ 61 } ports; 62 }; 63 64 /* Available parameters list. */ 65 const char *pmd_mlx4_init_params[] = { 66 MLX4_PMD_PORT_KVARG, 67 NULL, 68 }; 69 70 static void mlx4_dev_stop(struct rte_eth_dev *dev); 71 72 /** 73 * DPDK callback for Ethernet device configuration. 74 * 75 * @param dev 76 * Pointer to Ethernet device structure. 77 * 78 * @return 79 * 0 on success, negative errno value otherwise and rte_errno is set. 80 */ 81 static int 82 mlx4_dev_configure(struct rte_eth_dev *dev) 83 { 84 struct mlx4_priv *priv = dev->data->dev_private; 85 struct rte_flow_error error; 86 int ret; 87 88 /* Prepare internal flow rules. */ 89 ret = mlx4_flow_sync(priv, &error); 90 if (ret) { 91 ERROR("cannot set up internal flow rules (code %d, \"%s\")," 92 " flow error type %d, cause %p, message: %s", 93 -ret, strerror(-ret), error.type, error.cause, 94 error.message ? error.message : "(unspecified)"); 95 goto exit; 96 } 97 ret = mlx4_intr_install(priv); 98 if (ret) 99 ERROR("%p: interrupt handler installation failed", 100 (void *)dev); 101 exit: 102 return ret; 103 } 104 105 /** 106 * DPDK callback to start the device. 107 * 108 * Simulate device start by initializing common RSS resources and attaching 109 * all configured flows. 110 * 111 * @param dev 112 * Pointer to Ethernet device structure. 113 * 114 * @return 115 * 0 on success, negative errno value otherwise and rte_errno is set. 116 */ 117 static int 118 mlx4_dev_start(struct rte_eth_dev *dev) 119 { 120 struct mlx4_priv *priv = dev->data->dev_private; 121 struct rte_flow_error error; 122 int ret; 123 124 if (priv->started) 125 return 0; 126 DEBUG("%p: attaching configured flows to all RX queues", (void *)dev); 127 priv->started = 1; 128 ret = mlx4_rss_init(priv); 129 if (ret) { 130 ERROR("%p: cannot initialize RSS resources: %s", 131 (void *)dev, strerror(-ret)); 132 goto err; 133 } 134 #ifndef NDEBUG 135 mlx4_mr_dump_dev(dev); 136 #endif 137 ret = mlx4_rxq_intr_enable(priv); 138 if (ret) { 139 ERROR("%p: interrupt handler installation failed", 140 (void *)dev); 141 goto err; 142 } 143 ret = mlx4_flow_sync(priv, &error); 144 if (ret) { 145 ERROR("%p: cannot attach flow rules (code %d, \"%s\")," 146 " flow error type %d, cause %p, message: %s", 147 (void *)dev, 148 -ret, strerror(-ret), error.type, error.cause, 149 error.message ? error.message : "(unspecified)"); 150 goto err; 151 } 152 rte_wmb(); 153 dev->tx_pkt_burst = mlx4_tx_burst; 154 dev->rx_pkt_burst = mlx4_rx_burst; 155 return 0; 156 err: 157 mlx4_dev_stop(dev); 158 return ret; 159 } 160 161 /** 162 * DPDK callback to stop the device. 163 * 164 * Simulate device stop by detaching all configured flows. 165 * 166 * @param dev 167 * Pointer to Ethernet device structure. 168 */ 169 static void 170 mlx4_dev_stop(struct rte_eth_dev *dev) 171 { 172 struct mlx4_priv *priv = dev->data->dev_private; 173 174 if (!priv->started) 175 return; 176 DEBUG("%p: detaching flows from all RX queues", (void *)dev); 177 priv->started = 0; 178 dev->tx_pkt_burst = mlx4_tx_burst_removed; 179 dev->rx_pkt_burst = mlx4_rx_burst_removed; 180 rte_wmb(); 181 mlx4_flow_sync(priv, NULL); 182 mlx4_rxq_intr_disable(priv); 183 mlx4_rss_deinit(priv); 184 } 185 186 /** 187 * DPDK callback to close the device. 188 * 189 * Destroy all queues and objects, free memory. 190 * 191 * @param dev 192 * Pointer to Ethernet device structure. 193 */ 194 static void 195 mlx4_dev_close(struct rte_eth_dev *dev) 196 { 197 struct mlx4_priv *priv = dev->data->dev_private; 198 unsigned int i; 199 200 DEBUG("%p: closing device \"%s\"", 201 (void *)dev, 202 ((priv->ctx != NULL) ? priv->ctx->device->name : "")); 203 dev->rx_pkt_burst = mlx4_rx_burst_removed; 204 dev->tx_pkt_burst = mlx4_tx_burst_removed; 205 rte_wmb(); 206 mlx4_flow_clean(priv); 207 mlx4_rss_deinit(priv); 208 for (i = 0; i != dev->data->nb_rx_queues; ++i) 209 mlx4_rx_queue_release(dev->data->rx_queues[i]); 210 for (i = 0; i != dev->data->nb_tx_queues; ++i) 211 mlx4_tx_queue_release(dev->data->tx_queues[i]); 212 mlx4_mr_release(dev); 213 if (priv->pd != NULL) { 214 assert(priv->ctx != NULL); 215 claim_zero(mlx4_glue->dealloc_pd(priv->pd)); 216 claim_zero(mlx4_glue->close_device(priv->ctx)); 217 } else 218 assert(priv->ctx == NULL); 219 mlx4_intr_uninstall(priv); 220 memset(priv, 0, sizeof(*priv)); 221 } 222 223 static const struct eth_dev_ops mlx4_dev_ops = { 224 .dev_configure = mlx4_dev_configure, 225 .dev_start = mlx4_dev_start, 226 .dev_stop = mlx4_dev_stop, 227 .dev_set_link_down = mlx4_dev_set_link_down, 228 .dev_set_link_up = mlx4_dev_set_link_up, 229 .dev_close = mlx4_dev_close, 230 .link_update = mlx4_link_update, 231 .promiscuous_enable = mlx4_promiscuous_enable, 232 .promiscuous_disable = mlx4_promiscuous_disable, 233 .allmulticast_enable = mlx4_allmulticast_enable, 234 .allmulticast_disable = mlx4_allmulticast_disable, 235 .mac_addr_remove = mlx4_mac_addr_remove, 236 .mac_addr_add = mlx4_mac_addr_add, 237 .mac_addr_set = mlx4_mac_addr_set, 238 .stats_get = mlx4_stats_get, 239 .stats_reset = mlx4_stats_reset, 240 .fw_version_get = mlx4_fw_version_get, 241 .dev_infos_get = mlx4_dev_infos_get, 242 .dev_supported_ptypes_get = mlx4_dev_supported_ptypes_get, 243 .vlan_filter_set = mlx4_vlan_filter_set, 244 .rx_queue_setup = mlx4_rx_queue_setup, 245 .tx_queue_setup = mlx4_tx_queue_setup, 246 .rx_queue_release = mlx4_rx_queue_release, 247 .tx_queue_release = mlx4_tx_queue_release, 248 .flow_ctrl_get = mlx4_flow_ctrl_get, 249 .flow_ctrl_set = mlx4_flow_ctrl_set, 250 .mtu_set = mlx4_mtu_set, 251 .filter_ctrl = mlx4_filter_ctrl, 252 .rx_queue_intr_enable = mlx4_rx_intr_enable, 253 .rx_queue_intr_disable = mlx4_rx_intr_disable, 254 .is_removed = mlx4_is_removed, 255 }; 256 257 /** 258 * Get PCI information from struct ibv_device. 259 * 260 * @param device 261 * Pointer to Ethernet device structure. 262 * @param[out] pci_addr 263 * PCI bus address output buffer. 264 * 265 * @return 266 * 0 on success, negative errno value otherwise and rte_errno is set. 267 */ 268 static int 269 mlx4_ibv_device_to_pci_addr(const struct ibv_device *device, 270 struct rte_pci_addr *pci_addr) 271 { 272 FILE *file; 273 char line[32]; 274 MKSTR(path, "%s/device/uevent", device->ibdev_path); 275 276 file = fopen(path, "rb"); 277 if (file == NULL) { 278 rte_errno = errno; 279 return -rte_errno; 280 } 281 while (fgets(line, sizeof(line), file) == line) { 282 size_t len = strlen(line); 283 int ret; 284 285 /* Truncate long lines. */ 286 if (len == (sizeof(line) - 1)) 287 while (line[(len - 1)] != '\n') { 288 ret = fgetc(file); 289 if (ret == EOF) 290 break; 291 line[(len - 1)] = ret; 292 } 293 /* Extract information. */ 294 if (sscanf(line, 295 "PCI_SLOT_NAME=" 296 "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n", 297 &pci_addr->domain, 298 &pci_addr->bus, 299 &pci_addr->devid, 300 &pci_addr->function) == 4) { 301 ret = 0; 302 break; 303 } 304 } 305 fclose(file); 306 return 0; 307 } 308 309 /** 310 * Verify and store value for device argument. 311 * 312 * @param[in] key 313 * Key argument to verify. 314 * @param[in] val 315 * Value associated with key. 316 * @param[in, out] conf 317 * Shared configuration data. 318 * 319 * @return 320 * 0 on success, negative errno value otherwise and rte_errno is set. 321 */ 322 static int 323 mlx4_arg_parse(const char *key, const char *val, struct mlx4_conf *conf) 324 { 325 unsigned long tmp; 326 327 errno = 0; 328 tmp = strtoul(val, NULL, 0); 329 if (errno) { 330 rte_errno = errno; 331 WARN("%s: \"%s\" is not a valid integer", key, val); 332 return -rte_errno; 333 } 334 if (strcmp(MLX4_PMD_PORT_KVARG, key) == 0) { 335 uint32_t ports = rte_log2_u32(conf->ports.present + 1); 336 337 if (tmp >= ports) { 338 ERROR("port index %lu outside range [0,%" PRIu32 ")", 339 tmp, ports); 340 return -EINVAL; 341 } 342 if (!(conf->ports.present & (1 << tmp))) { 343 rte_errno = EINVAL; 344 ERROR("invalid port index %lu", tmp); 345 return -rte_errno; 346 } 347 conf->ports.enabled |= 1 << tmp; 348 } else { 349 rte_errno = EINVAL; 350 WARN("%s: unknown parameter", key); 351 return -rte_errno; 352 } 353 return 0; 354 } 355 356 /** 357 * Parse device parameters. 358 * 359 * @param devargs 360 * Device arguments structure. 361 * 362 * @return 363 * 0 on success, negative errno value otherwise and rte_errno is set. 364 */ 365 static int 366 mlx4_args(struct rte_devargs *devargs, struct mlx4_conf *conf) 367 { 368 struct rte_kvargs *kvlist; 369 unsigned int arg_count; 370 int ret = 0; 371 int i; 372 373 if (devargs == NULL) 374 return 0; 375 kvlist = rte_kvargs_parse(devargs->args, pmd_mlx4_init_params); 376 if (kvlist == NULL) { 377 rte_errno = EINVAL; 378 ERROR("failed to parse kvargs"); 379 return -rte_errno; 380 } 381 /* Process parameters. */ 382 for (i = 0; pmd_mlx4_init_params[i]; ++i) { 383 arg_count = rte_kvargs_count(kvlist, MLX4_PMD_PORT_KVARG); 384 while (arg_count-- > 0) { 385 ret = rte_kvargs_process(kvlist, 386 MLX4_PMD_PORT_KVARG, 387 (int (*)(const char *, 388 const char *, 389 void *)) 390 mlx4_arg_parse, 391 conf); 392 if (ret != 0) 393 goto free_kvlist; 394 } 395 } 396 free_kvlist: 397 rte_kvargs_free(kvlist); 398 return ret; 399 } 400 401 /** 402 * Interpret RSS capabilities reported by device. 403 * 404 * This function returns the set of usable Verbs RSS hash fields, kernel 405 * quirks taken into account. 406 * 407 * @param ctx 408 * Verbs context. 409 * @param pd 410 * Verbs protection domain. 411 * @param device_attr_ex 412 * Extended device attributes to interpret. 413 * 414 * @return 415 * Usable RSS hash fields mask in Verbs format. 416 */ 417 static uint64_t 418 mlx4_hw_rss_sup(struct ibv_context *ctx, struct ibv_pd *pd, 419 struct ibv_device_attr_ex *device_attr_ex) 420 { 421 uint64_t hw_rss_sup = device_attr_ex->rss_caps.rx_hash_fields_mask; 422 struct ibv_cq *cq = NULL; 423 struct ibv_wq *wq = NULL; 424 struct ibv_rwq_ind_table *ind = NULL; 425 struct ibv_qp *qp = NULL; 426 427 if (!hw_rss_sup) { 428 WARN("no RSS capabilities reported; disabling support for UDP" 429 " RSS and inner VXLAN RSS"); 430 return IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4 | 431 IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6 | 432 IBV_RX_HASH_SRC_PORT_TCP | IBV_RX_HASH_DST_PORT_TCP; 433 } 434 if (!(hw_rss_sup & IBV_RX_HASH_INNER)) 435 return hw_rss_sup; 436 /* 437 * Although reported as supported, missing code in some Linux 438 * versions (v4.15, v4.16) prevents the creation of hash QPs with 439 * inner capability. 440 * 441 * There is no choice but to attempt to instantiate a temporary RSS 442 * context in order to confirm its support. 443 */ 444 cq = mlx4_glue->create_cq(ctx, 1, NULL, NULL, 0); 445 wq = cq ? mlx4_glue->create_wq 446 (ctx, 447 &(struct ibv_wq_init_attr){ 448 .wq_type = IBV_WQT_RQ, 449 .max_wr = 1, 450 .max_sge = 1, 451 .pd = pd, 452 .cq = cq, 453 }) : NULL; 454 ind = wq ? mlx4_glue->create_rwq_ind_table 455 (ctx, 456 &(struct ibv_rwq_ind_table_init_attr){ 457 .log_ind_tbl_size = 0, 458 .ind_tbl = &wq, 459 .comp_mask = 0, 460 }) : NULL; 461 qp = ind ? mlx4_glue->create_qp_ex 462 (ctx, 463 &(struct ibv_qp_init_attr_ex){ 464 .comp_mask = 465 (IBV_QP_INIT_ATTR_PD | 466 IBV_QP_INIT_ATTR_RX_HASH | 467 IBV_QP_INIT_ATTR_IND_TABLE), 468 .qp_type = IBV_QPT_RAW_PACKET, 469 .pd = pd, 470 .rwq_ind_tbl = ind, 471 .rx_hash_conf = { 472 .rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ, 473 .rx_hash_key_len = MLX4_RSS_HASH_KEY_SIZE, 474 .rx_hash_key = mlx4_rss_hash_key_default, 475 .rx_hash_fields_mask = hw_rss_sup, 476 }, 477 }) : NULL; 478 if (!qp) { 479 WARN("disabling unusable inner RSS capability due to kernel" 480 " quirk"); 481 hw_rss_sup &= ~IBV_RX_HASH_INNER; 482 } else { 483 claim_zero(mlx4_glue->destroy_qp(qp)); 484 } 485 if (ind) 486 claim_zero(mlx4_glue->destroy_rwq_ind_table(ind)); 487 if (wq) 488 claim_zero(mlx4_glue->destroy_wq(wq)); 489 if (cq) 490 claim_zero(mlx4_glue->destroy_cq(cq)); 491 return hw_rss_sup; 492 } 493 494 static struct rte_pci_driver mlx4_driver; 495 496 /** 497 * DPDK callback to register a PCI device. 498 * 499 * This function creates an Ethernet device for each port of a given 500 * PCI device. 501 * 502 * @param[in] pci_drv 503 * PCI driver structure (mlx4_driver). 504 * @param[in] pci_dev 505 * PCI device information. 506 * 507 * @return 508 * 0 on success, negative errno value otherwise and rte_errno is set. 509 */ 510 static int 511 mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) 512 { 513 struct ibv_device **list; 514 struct ibv_device *ibv_dev; 515 int err = 0; 516 struct ibv_context *attr_ctx = NULL; 517 struct ibv_device_attr device_attr; 518 struct ibv_device_attr_ex device_attr_ex; 519 struct mlx4_conf conf = { 520 .ports.present = 0, 521 }; 522 unsigned int vf; 523 int i; 524 525 (void)pci_drv; 526 assert(pci_drv == &mlx4_driver); 527 list = mlx4_glue->get_device_list(&i); 528 if (list == NULL) { 529 rte_errno = errno; 530 assert(rte_errno); 531 if (rte_errno == ENOSYS) 532 ERROR("cannot list devices, is ib_uverbs loaded?"); 533 return -rte_errno; 534 } 535 assert(i >= 0); 536 /* 537 * For each listed device, check related sysfs entry against 538 * the provided PCI ID. 539 */ 540 while (i != 0) { 541 struct rte_pci_addr pci_addr; 542 543 --i; 544 DEBUG("checking device \"%s\"", list[i]->name); 545 if (mlx4_ibv_device_to_pci_addr(list[i], &pci_addr)) 546 continue; 547 if ((pci_dev->addr.domain != pci_addr.domain) || 548 (pci_dev->addr.bus != pci_addr.bus) || 549 (pci_dev->addr.devid != pci_addr.devid) || 550 (pci_dev->addr.function != pci_addr.function)) 551 continue; 552 vf = (pci_dev->id.device_id == 553 PCI_DEVICE_ID_MELLANOX_CONNECTX3VF); 554 INFO("PCI information matches, using device \"%s\" (VF: %s)", 555 list[i]->name, (vf ? "true" : "false")); 556 attr_ctx = mlx4_glue->open_device(list[i]); 557 err = errno; 558 break; 559 } 560 if (attr_ctx == NULL) { 561 mlx4_glue->free_device_list(list); 562 switch (err) { 563 case 0: 564 rte_errno = ENODEV; 565 ERROR("cannot access device, is mlx4_ib loaded?"); 566 return -rte_errno; 567 case EINVAL: 568 rte_errno = EINVAL; 569 ERROR("cannot use device, are drivers up to date?"); 570 return -rte_errno; 571 } 572 assert(err > 0); 573 rte_errno = err; 574 return -rte_errno; 575 } 576 ibv_dev = list[i]; 577 DEBUG("device opened"); 578 if (mlx4_glue->query_device(attr_ctx, &device_attr)) { 579 err = ENODEV; 580 goto error; 581 } 582 INFO("%u port(s) detected", device_attr.phys_port_cnt); 583 conf.ports.present |= (UINT64_C(1) << device_attr.phys_port_cnt) - 1; 584 if (mlx4_args(pci_dev->device.devargs, &conf)) { 585 ERROR("failed to process device arguments"); 586 err = EINVAL; 587 goto error; 588 } 589 /* Use all ports when none are defined */ 590 if (!conf.ports.enabled) 591 conf.ports.enabled = conf.ports.present; 592 /* Retrieve extended device attributes. */ 593 if (mlx4_glue->query_device_ex(attr_ctx, NULL, &device_attr_ex)) { 594 err = ENODEV; 595 goto error; 596 } 597 assert(device_attr.max_sge >= MLX4_MAX_SGE); 598 for (i = 0; i < device_attr.phys_port_cnt; i++) { 599 uint32_t port = i + 1; /* ports are indexed from one */ 600 struct ibv_context *ctx = NULL; 601 struct ibv_port_attr port_attr; 602 struct ibv_pd *pd = NULL; 603 struct mlx4_priv *priv = NULL; 604 struct rte_eth_dev *eth_dev = NULL; 605 struct ether_addr mac; 606 607 /* If port is not enabled, skip. */ 608 if (!(conf.ports.enabled & (1 << i))) 609 continue; 610 DEBUG("using port %u", port); 611 ctx = mlx4_glue->open_device(ibv_dev); 612 if (ctx == NULL) { 613 err = ENODEV; 614 goto port_error; 615 } 616 /* Check port status. */ 617 err = mlx4_glue->query_port(ctx, port, &port_attr); 618 if (err) { 619 err = ENODEV; 620 ERROR("port query failed: %s", strerror(err)); 621 goto port_error; 622 } 623 if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) { 624 err = ENOTSUP; 625 ERROR("port %d is not configured in Ethernet mode", 626 port); 627 goto port_error; 628 } 629 if (port_attr.state != IBV_PORT_ACTIVE) 630 DEBUG("port %d is not active: \"%s\" (%d)", 631 port, mlx4_glue->port_state_str(port_attr.state), 632 port_attr.state); 633 /* Make asynchronous FD non-blocking to handle interrupts. */ 634 err = mlx4_fd_set_non_blocking(ctx->async_fd); 635 if (err) { 636 ERROR("cannot make asynchronous FD non-blocking: %s", 637 strerror(err)); 638 goto port_error; 639 } 640 /* Allocate protection domain. */ 641 pd = mlx4_glue->alloc_pd(ctx); 642 if (pd == NULL) { 643 err = ENOMEM; 644 ERROR("PD allocation failure"); 645 goto port_error; 646 } 647 /* from rte_ethdev.c */ 648 priv = rte_zmalloc("ethdev private structure", 649 sizeof(*priv), 650 RTE_CACHE_LINE_SIZE); 651 if (priv == NULL) { 652 err = ENOMEM; 653 ERROR("priv allocation failure"); 654 goto port_error; 655 } 656 priv->ctx = ctx; 657 priv->device_attr = device_attr; 658 priv->port = port; 659 priv->pd = pd; 660 priv->mtu = ETHER_MTU; 661 priv->vf = vf; 662 priv->hw_csum = !!(device_attr.device_cap_flags & 663 IBV_DEVICE_RAW_IP_CSUM); 664 DEBUG("checksum offloading is %ssupported", 665 (priv->hw_csum ? "" : "not ")); 666 /* Only ConnectX-3 Pro supports tunneling. */ 667 priv->hw_csum_l2tun = 668 priv->hw_csum && 669 (device_attr.vendor_part_id == 670 PCI_DEVICE_ID_MELLANOX_CONNECTX3PRO); 671 DEBUG("L2 tunnel checksum offloads are %ssupported", 672 priv->hw_csum_l2tun ? "" : "not "); 673 priv->hw_rss_sup = mlx4_hw_rss_sup(priv->ctx, priv->pd, 674 &device_attr_ex); 675 DEBUG("supported RSS hash fields mask: %016" PRIx64, 676 priv->hw_rss_sup); 677 priv->hw_rss_max_qps = 678 device_attr_ex.rss_caps.max_rwq_indirection_table_size; 679 DEBUG("MAX RSS queues %d", priv->hw_rss_max_qps); 680 priv->hw_fcs_strip = !!(device_attr_ex.raw_packet_caps & 681 IBV_RAW_PACKET_CAP_SCATTER_FCS); 682 DEBUG("FCS stripping toggling is %ssupported", 683 priv->hw_fcs_strip ? "" : "not "); 684 priv->tso = 685 ((device_attr_ex.tso_caps.max_tso > 0) && 686 (device_attr_ex.tso_caps.supported_qpts & 687 (1 << IBV_QPT_RAW_PACKET))); 688 if (priv->tso) 689 priv->tso_max_payload_sz = 690 device_attr_ex.tso_caps.max_tso; 691 DEBUG("TSO is %ssupported", 692 priv->tso ? "" : "not "); 693 /* Configure the first MAC address by default. */ 694 err = mlx4_get_mac(priv, &mac.addr_bytes); 695 if (err) { 696 ERROR("cannot get MAC address, is mlx4_en loaded?" 697 " (error: %s)", strerror(err)); 698 goto port_error; 699 } 700 INFO("port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x", 701 priv->port, 702 mac.addr_bytes[0], mac.addr_bytes[1], 703 mac.addr_bytes[2], mac.addr_bytes[3], 704 mac.addr_bytes[4], mac.addr_bytes[5]); 705 /* Register MAC address. */ 706 priv->mac[0] = mac; 707 #ifndef NDEBUG 708 { 709 char ifname[IF_NAMESIZE]; 710 711 if (mlx4_get_ifname(priv, &ifname) == 0) 712 DEBUG("port %u ifname is \"%s\"", 713 priv->port, ifname); 714 else 715 DEBUG("port %u ifname is unknown", priv->port); 716 } 717 #endif 718 /* Get actual MTU if possible. */ 719 mlx4_mtu_get(priv, &priv->mtu); 720 DEBUG("port %u MTU is %u", priv->port, priv->mtu); 721 /* from rte_ethdev.c */ 722 { 723 char name[RTE_ETH_NAME_MAX_LEN]; 724 725 snprintf(name, sizeof(name), "%s port %u", 726 mlx4_glue->get_device_name(ibv_dev), port); 727 eth_dev = rte_eth_dev_allocate(name); 728 } 729 if (eth_dev == NULL) { 730 err = ENOMEM; 731 ERROR("can not allocate rte ethdev"); 732 goto port_error; 733 } 734 eth_dev->data->dev_private = priv; 735 eth_dev->data->mac_addrs = priv->mac; 736 eth_dev->device = &pci_dev->device; 737 rte_eth_copy_pci_info(eth_dev, pci_dev); 738 /* Initialize local interrupt handle for current port. */ 739 priv->intr_handle = (struct rte_intr_handle){ 740 .fd = -1, 741 .type = RTE_INTR_HANDLE_EXT, 742 }; 743 /* 744 * Override ethdev interrupt handle pointer with private 745 * handle instead of that of the parent PCI device used by 746 * default. This prevents it from being shared between all 747 * ports of the same PCI device since each of them is 748 * associated its own Verbs context. 749 * 750 * Rx interrupts in particular require this as the PMD has 751 * no control over the registration of queue interrupts 752 * besides setting up eth_dev->intr_handle, the rest is 753 * handled by rte_intr_rx_ctl(). 754 */ 755 eth_dev->intr_handle = &priv->intr_handle; 756 priv->dev = eth_dev; 757 eth_dev->dev_ops = &mlx4_dev_ops; 758 /* Bring Ethernet device up. */ 759 DEBUG("forcing Ethernet interface up"); 760 mlx4_dev_set_link_up(priv->dev); 761 /* Update link status once if waiting for LSC. */ 762 if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) 763 mlx4_link_update(eth_dev, 0); 764 /* 765 * Once the device is added to the list of memory event 766 * callback, its global MR cache table cannot be expanded 767 * on the fly because of deadlock. If it overflows, lookup 768 * should be done by searching MR list linearly, which is slow. 769 */ 770 err = mlx4_mr_btree_init(&priv->mr.cache, 771 MLX4_MR_BTREE_CACHE_N * 2, 772 eth_dev->device->numa_node); 773 if (err) { 774 /* rte_errno is already set. */ 775 goto port_error; 776 } 777 /* Add device to memory callback list. */ 778 rte_rwlock_write_lock(&mlx4_mem_event_rwlock); 779 LIST_INSERT_HEAD(&mlx4_mem_event_cb_list, priv, mem_event_cb); 780 rte_rwlock_write_unlock(&mlx4_mem_event_rwlock); 781 rte_eth_dev_probing_finish(eth_dev); 782 continue; 783 port_error: 784 rte_free(priv); 785 if (eth_dev != NULL) 786 eth_dev->data->dev_private = NULL; 787 if (pd) 788 claim_zero(mlx4_glue->dealloc_pd(pd)); 789 if (ctx) 790 claim_zero(mlx4_glue->close_device(ctx)); 791 if (eth_dev != NULL) { 792 /* mac_addrs must not be freed because part of dev_private */ 793 eth_dev->data->mac_addrs = NULL; 794 rte_eth_dev_release_port(eth_dev); 795 } 796 break; 797 } 798 /* 799 * XXX if something went wrong in the loop above, there is a resource 800 * leak (ctx, pd, priv, dpdk ethdev) but we can do nothing about it as 801 * long as the dpdk does not provide a way to deallocate a ethdev and a 802 * way to enumerate the registered ethdevs to free the previous ones. 803 */ 804 error: 805 if (attr_ctx) 806 claim_zero(mlx4_glue->close_device(attr_ctx)); 807 if (list) 808 mlx4_glue->free_device_list(list); 809 if (err) 810 rte_errno = err; 811 return -err; 812 } 813 814 static const struct rte_pci_id mlx4_pci_id_map[] = { 815 { 816 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 817 PCI_DEVICE_ID_MELLANOX_CONNECTX3) 818 }, 819 { 820 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 821 PCI_DEVICE_ID_MELLANOX_CONNECTX3PRO) 822 }, 823 { 824 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 825 PCI_DEVICE_ID_MELLANOX_CONNECTX3VF) 826 }, 827 { 828 .vendor_id = 0 829 } 830 }; 831 832 static struct rte_pci_driver mlx4_driver = { 833 .driver = { 834 .name = MLX4_DRIVER_NAME 835 }, 836 .id_table = mlx4_pci_id_map, 837 .probe = mlx4_pci_probe, 838 .drv_flags = RTE_PCI_DRV_INTR_LSC | 839 RTE_PCI_DRV_INTR_RMV, 840 }; 841 842 #ifdef RTE_IBVERBS_LINK_DLOPEN 843 844 /** 845 * Suffix RTE_EAL_PMD_PATH with "-glue". 846 * 847 * This function performs a sanity check on RTE_EAL_PMD_PATH before 848 * suffixing its last component. 849 * 850 * @param buf[out] 851 * Output buffer, should be large enough otherwise NULL is returned. 852 * @param size 853 * Size of @p out. 854 * 855 * @return 856 * Pointer to @p buf or @p NULL in case suffix cannot be appended. 857 */ 858 static char * 859 mlx4_glue_path(char *buf, size_t size) 860 { 861 static const char *const bad[] = { "/", ".", "..", NULL }; 862 const char *path = RTE_EAL_PMD_PATH; 863 size_t len = strlen(path); 864 size_t off; 865 int i; 866 867 while (len && path[len - 1] == '/') 868 --len; 869 for (off = len; off && path[off - 1] != '/'; --off) 870 ; 871 for (i = 0; bad[i]; ++i) 872 if (!strncmp(path + off, bad[i], (int)(len - off))) 873 goto error; 874 i = snprintf(buf, size, "%.*s-glue", (int)len, path); 875 if (i == -1 || (size_t)i >= size) 876 goto error; 877 return buf; 878 error: 879 ERROR("unable to append \"-glue\" to last component of" 880 " RTE_EAL_PMD_PATH (\"" RTE_EAL_PMD_PATH "\")," 881 " please re-configure DPDK"); 882 return NULL; 883 } 884 885 /** 886 * Initialization routine for run-time dependency on rdma-core. 887 */ 888 static int 889 mlx4_glue_init(void) 890 { 891 char glue_path[sizeof(RTE_EAL_PMD_PATH) - 1 + sizeof("-glue")]; 892 const char *path[] = { 893 /* 894 * A basic security check is necessary before trusting 895 * MLX4_GLUE_PATH, which may override RTE_EAL_PMD_PATH. 896 */ 897 (geteuid() == getuid() && getegid() == getgid() ? 898 getenv("MLX4_GLUE_PATH") : NULL), 899 /* 900 * When RTE_EAL_PMD_PATH is set, use its glue-suffixed 901 * variant, otherwise let dlopen() look up libraries on its 902 * own. 903 */ 904 (*RTE_EAL_PMD_PATH ? 905 mlx4_glue_path(glue_path, sizeof(glue_path)) : ""), 906 }; 907 unsigned int i = 0; 908 void *handle = NULL; 909 void **sym; 910 const char *dlmsg; 911 912 while (!handle && i != RTE_DIM(path)) { 913 const char *end; 914 size_t len; 915 int ret; 916 917 if (!path[i]) { 918 ++i; 919 continue; 920 } 921 end = strpbrk(path[i], ":;"); 922 if (!end) 923 end = path[i] + strlen(path[i]); 924 len = end - path[i]; 925 ret = 0; 926 do { 927 char name[ret + 1]; 928 929 ret = snprintf(name, sizeof(name), "%.*s%s" MLX4_GLUE, 930 (int)len, path[i], 931 (!len || *(end - 1) == '/') ? "" : "/"); 932 if (ret == -1) 933 break; 934 if (sizeof(name) != (size_t)ret + 1) 935 continue; 936 DEBUG("looking for rdma-core glue as \"%s\"", name); 937 handle = dlopen(name, RTLD_LAZY); 938 break; 939 } while (1); 940 path[i] = end + 1; 941 if (!*end) 942 ++i; 943 } 944 if (!handle) { 945 rte_errno = EINVAL; 946 dlmsg = dlerror(); 947 if (dlmsg) 948 WARN("cannot load glue library: %s", dlmsg); 949 goto glue_error; 950 } 951 sym = dlsym(handle, "mlx4_glue"); 952 if (!sym || !*sym) { 953 rte_errno = EINVAL; 954 dlmsg = dlerror(); 955 if (dlmsg) 956 ERROR("cannot resolve glue symbol: %s", dlmsg); 957 goto glue_error; 958 } 959 mlx4_glue = *sym; 960 return 0; 961 glue_error: 962 if (handle) 963 dlclose(handle); 964 WARN("cannot initialize PMD due to missing run-time" 965 " dependency on rdma-core libraries (libibverbs," 966 " libmlx4)"); 967 return -rte_errno; 968 } 969 970 #endif 971 972 /** 973 * Driver initialization routine. 974 */ 975 RTE_INIT(rte_mlx4_pmd_init) 976 { 977 /* 978 * MLX4_DEVICE_FATAL_CLEANUP tells ibv_destroy functions we 979 * want to get success errno value in case of calling them 980 * when the device was removed. 981 */ 982 setenv("MLX4_DEVICE_FATAL_CLEANUP", "1", 1); 983 /* 984 * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use 985 * huge pages. Calling ibv_fork_init() during init allows 986 * applications to use fork() safely for purposes other than 987 * using this PMD, which is not supported in forked processes. 988 */ 989 setenv("RDMAV_HUGEPAGES_SAFE", "1", 1); 990 #ifdef RTE_IBVERBS_LINK_DLOPEN 991 if (mlx4_glue_init()) 992 return; 993 assert(mlx4_glue); 994 #endif 995 #ifndef NDEBUG 996 /* Glue structure must not contain any NULL pointers. */ 997 { 998 unsigned int i; 999 1000 for (i = 0; i != sizeof(*mlx4_glue) / sizeof(void *); ++i) 1001 assert(((const void *const *)mlx4_glue)[i]); 1002 } 1003 #endif 1004 if (strcmp(mlx4_glue->version, MLX4_GLUE_VERSION)) { 1005 ERROR("rdma-core glue \"%s\" mismatch: \"%s\" is required", 1006 mlx4_glue->version, MLX4_GLUE_VERSION); 1007 return; 1008 } 1009 mlx4_glue->fork_init(); 1010 rte_pci_register(&mlx4_driver); 1011 rte_mem_event_callback_register("MLX4_MEM_EVENT_CB", 1012 mlx4_mr_mem_event_cb, NULL); 1013 } 1014 1015 RTE_PMD_EXPORT_NAME(net_mlx4, __COUNTER__); 1016 RTE_PMD_REGISTER_PCI_TABLE(net_mlx4, mlx4_pci_id_map); 1017 RTE_PMD_REGISTER_KMOD_DEP(net_mlx4, 1018 "* ib_uverbs & mlx4_en & mlx4_core & mlx4_ib"); 1019