1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2012 6WIND S.A. 3 * Copyright 2012 Mellanox Technologies, Ltd 4 */ 5 6 /** 7 * @file 8 * mlx4 driver initialization. 9 */ 10 11 #include <errno.h> 12 #include <inttypes.h> 13 #include <stddef.h> 14 #include <stdint.h> 15 #include <stdio.h> 16 #include <stdlib.h> 17 #include <string.h> 18 #include <sys/mman.h> 19 #include <unistd.h> 20 #ifdef RTE_IBVERBS_LINK_DLOPEN 21 #include <dlfcn.h> 22 #endif 23 24 /* Verbs headers do not support -pedantic. */ 25 #ifdef PEDANTIC 26 #pragma GCC diagnostic ignored "-Wpedantic" 27 #endif 28 #include <infiniband/verbs.h> 29 #ifdef PEDANTIC 30 #pragma GCC diagnostic error "-Wpedantic" 31 #endif 32 33 #include <rte_common.h> 34 #include <rte_dev.h> 35 #include <rte_errno.h> 36 #include <rte_ethdev_driver.h> 37 #include <rte_ethdev_pci.h> 38 #include <rte_ether.h> 39 #include <rte_flow.h> 40 #include <rte_interrupts.h> 41 #include <rte_kvargs.h> 42 #include <rte_malloc.h> 43 #include <rte_mbuf.h> 44 45 #include "mlx4.h" 46 #include "mlx4_glue.h" 47 #include "mlx4_flow.h" 48 #include "mlx4_mr.h" 49 #include "mlx4_rxtx.h" 50 #include "mlx4_utils.h" 51 52 #ifdef MLX4_GLUE 53 const struct mlx4_glue *mlx4_glue; 54 #endif 55 56 static const char *MZ_MLX4_PMD_SHARED_DATA = "mlx4_pmd_shared_data"; 57 58 /* Shared memory between primary and secondary processes. */ 59 struct mlx4_shared_data *mlx4_shared_data; 60 61 /* Spinlock for mlx4_shared_data allocation. */ 62 static rte_spinlock_t mlx4_shared_data_lock = RTE_SPINLOCK_INITIALIZER; 63 64 /* Process local data for secondary processes. */ 65 static struct mlx4_local_data mlx4_local_data; 66 67 /** Driver-specific log messages type. */ 68 int mlx4_logtype; 69 70 /** Configuration structure for device arguments. */ 71 struct mlx4_conf { 72 struct { 73 uint32_t present; /**< Bit-field for existing ports. */ 74 uint32_t enabled; /**< Bit-field for user-enabled ports. */ 75 } ports; 76 int mr_ext_memseg_en; 77 /** Whether memseg should be extended for MR creation. */ 78 }; 79 80 /* Available parameters list. */ 81 const char *pmd_mlx4_init_params[] = { 82 MLX4_PMD_PORT_KVARG, 83 MLX4_MR_EXT_MEMSEG_EN_KVARG, 84 NULL, 85 }; 86 87 static void mlx4_dev_stop(struct rte_eth_dev *dev); 88 89 /** 90 * Initialize shared data between primary and secondary process. 91 * 92 * A memzone is reserved by primary process and secondary processes attach to 93 * the memzone. 94 * 95 * @return 96 * 0 on success, a negative errno value otherwise and rte_errno is set. 97 */ 98 static int 99 mlx4_init_shared_data(void) 100 { 101 const struct rte_memzone *mz; 102 int ret = 0; 103 104 rte_spinlock_lock(&mlx4_shared_data_lock); 105 if (mlx4_shared_data == NULL) { 106 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 107 /* Allocate shared memory. */ 108 mz = rte_memzone_reserve(MZ_MLX4_PMD_SHARED_DATA, 109 sizeof(*mlx4_shared_data), 110 SOCKET_ID_ANY, 0); 111 if (mz == NULL) { 112 ERROR("Cannot allocate mlx4 shared data\n"); 113 ret = -rte_errno; 114 goto error; 115 } 116 mlx4_shared_data = mz->addr; 117 memset(mlx4_shared_data, 0, sizeof(*mlx4_shared_data)); 118 rte_spinlock_init(&mlx4_shared_data->lock); 119 } else { 120 /* Lookup allocated shared memory. */ 121 mz = rte_memzone_lookup(MZ_MLX4_PMD_SHARED_DATA); 122 if (mz == NULL) { 123 ERROR("Cannot attach mlx4 shared data\n"); 124 ret = -rte_errno; 125 goto error; 126 } 127 mlx4_shared_data = mz->addr; 128 memset(&mlx4_local_data, 0, sizeof(mlx4_local_data)); 129 } 130 } 131 error: 132 rte_spinlock_unlock(&mlx4_shared_data_lock); 133 return ret; 134 } 135 136 #ifdef HAVE_IBV_MLX4_BUF_ALLOCATORS 137 /** 138 * Verbs callback to allocate a memory. This function should allocate the space 139 * according to the size provided residing inside a huge page. 140 * Please note that all allocation must respect the alignment from libmlx4 141 * (i.e. currently sysconf(_SC_PAGESIZE)). 142 * 143 * @param[in] size 144 * The size in bytes of the memory to allocate. 145 * @param[in] data 146 * A pointer to the callback data. 147 * 148 * @return 149 * Allocated buffer, NULL otherwise and rte_errno is set. 150 */ 151 static void * 152 mlx4_alloc_verbs_buf(size_t size, void *data) 153 { 154 struct mlx4_priv *priv = data; 155 void *ret; 156 size_t alignment = sysconf(_SC_PAGESIZE); 157 unsigned int socket = SOCKET_ID_ANY; 158 159 if (priv->verbs_alloc_ctx.type == MLX4_VERBS_ALLOC_TYPE_TX_QUEUE) { 160 const struct txq *txq = priv->verbs_alloc_ctx.obj; 161 162 socket = txq->socket; 163 } else if (priv->verbs_alloc_ctx.type == 164 MLX4_VERBS_ALLOC_TYPE_RX_QUEUE) { 165 const struct rxq *rxq = priv->verbs_alloc_ctx.obj; 166 167 socket = rxq->socket; 168 } 169 MLX4_ASSERT(data != NULL); 170 ret = rte_malloc_socket(__func__, size, alignment, socket); 171 if (!ret && size) 172 rte_errno = ENOMEM; 173 return ret; 174 } 175 176 /** 177 * Verbs callback to free a memory. 178 * 179 * @param[in] ptr 180 * A pointer to the memory to free. 181 * @param[in] data 182 * A pointer to the callback data. 183 */ 184 static void 185 mlx4_free_verbs_buf(void *ptr, void *data __rte_unused) 186 { 187 MLX4_ASSERT(data != NULL); 188 rte_free(ptr); 189 } 190 #endif 191 192 /** 193 * Initialize process private data structure. 194 * 195 * @param dev 196 * Pointer to Ethernet device structure. 197 * 198 * @return 199 * 0 on success, a negative errno value otherwise and rte_errno is set. 200 */ 201 static int 202 mlx4_proc_priv_init(struct rte_eth_dev *dev) 203 { 204 struct mlx4_proc_priv *ppriv; 205 size_t ppriv_size; 206 207 /* 208 * UAR register table follows the process private structure. BlueFlame 209 * registers for Tx queues are stored in the table. 210 */ 211 ppriv_size = sizeof(struct mlx4_proc_priv) + 212 dev->data->nb_tx_queues * sizeof(void *); 213 ppriv = rte_malloc_socket("mlx4_proc_priv", ppriv_size, 214 RTE_CACHE_LINE_SIZE, dev->device->numa_node); 215 if (!ppriv) { 216 rte_errno = ENOMEM; 217 return -rte_errno; 218 } 219 ppriv->uar_table_sz = ppriv_size; 220 dev->process_private = ppriv; 221 return 0; 222 } 223 224 /** 225 * Un-initialize process private data structure. 226 * 227 * @param dev 228 * Pointer to Ethernet device structure. 229 */ 230 static void 231 mlx4_proc_priv_uninit(struct rte_eth_dev *dev) 232 { 233 if (!dev->process_private) 234 return; 235 rte_free(dev->process_private); 236 dev->process_private = NULL; 237 } 238 239 /** 240 * DPDK callback for Ethernet device configuration. 241 * 242 * @param dev 243 * Pointer to Ethernet device structure. 244 * 245 * @return 246 * 0 on success, negative errno value otherwise and rte_errno is set. 247 */ 248 static int 249 mlx4_dev_configure(struct rte_eth_dev *dev) 250 { 251 struct mlx4_priv *priv = dev->data->dev_private; 252 struct rte_flow_error error; 253 int ret; 254 255 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) 256 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; 257 258 /* Prepare internal flow rules. */ 259 ret = mlx4_flow_sync(priv, &error); 260 if (ret) { 261 ERROR("cannot set up internal flow rules (code %d, \"%s\")," 262 " flow error type %d, cause %p, message: %s", 263 -ret, strerror(-ret), error.type, error.cause, 264 error.message ? error.message : "(unspecified)"); 265 goto exit; 266 } 267 ret = mlx4_intr_install(priv); 268 if (ret) { 269 ERROR("%p: interrupt handler installation failed", 270 (void *)dev); 271 goto exit; 272 } 273 ret = mlx4_proc_priv_init(dev); 274 if (ret) { 275 ERROR("%p: process private data allocation failed", 276 (void *)dev); 277 goto exit; 278 } 279 exit: 280 return ret; 281 } 282 283 /** 284 * DPDK callback to start the device. 285 * 286 * Simulate device start by initializing common RSS resources and attaching 287 * all configured flows. 288 * 289 * @param dev 290 * Pointer to Ethernet device structure. 291 * 292 * @return 293 * 0 on success, negative errno value otherwise and rte_errno is set. 294 */ 295 static int 296 mlx4_dev_start(struct rte_eth_dev *dev) 297 { 298 struct mlx4_priv *priv = dev->data->dev_private; 299 struct rte_flow_error error; 300 int ret; 301 302 if (priv->started) 303 return 0; 304 DEBUG("%p: attaching configured flows to all RX queues", (void *)dev); 305 priv->started = 1; 306 ret = mlx4_rss_init(priv); 307 if (ret) { 308 ERROR("%p: cannot initialize RSS resources: %s", 309 (void *)dev, strerror(-ret)); 310 goto err; 311 } 312 #ifdef RTE_LIBRTE_MLX4_DEBUG 313 mlx4_mr_dump_dev(dev); 314 #endif 315 ret = mlx4_rxq_intr_enable(priv); 316 if (ret) { 317 ERROR("%p: interrupt handler installation failed", 318 (void *)dev); 319 goto err; 320 } 321 ret = mlx4_flow_sync(priv, &error); 322 if (ret) { 323 ERROR("%p: cannot attach flow rules (code %d, \"%s\")," 324 " flow error type %d, cause %p, message: %s", 325 (void *)dev, 326 -ret, strerror(-ret), error.type, error.cause, 327 error.message ? error.message : "(unspecified)"); 328 goto err; 329 } 330 rte_wmb(); 331 dev->tx_pkt_burst = mlx4_tx_burst; 332 dev->rx_pkt_burst = mlx4_rx_burst; 333 /* Enable datapath on secondary process. */ 334 mlx4_mp_req_start_rxtx(dev); 335 return 0; 336 err: 337 mlx4_dev_stop(dev); 338 return ret; 339 } 340 341 /** 342 * DPDK callback to stop the device. 343 * 344 * Simulate device stop by detaching all configured flows. 345 * 346 * @param dev 347 * Pointer to Ethernet device structure. 348 */ 349 static void 350 mlx4_dev_stop(struct rte_eth_dev *dev) 351 { 352 struct mlx4_priv *priv = dev->data->dev_private; 353 354 if (!priv->started) 355 return; 356 DEBUG("%p: detaching flows from all RX queues", (void *)dev); 357 priv->started = 0; 358 dev->tx_pkt_burst = mlx4_tx_burst_removed; 359 dev->rx_pkt_burst = mlx4_rx_burst_removed; 360 rte_wmb(); 361 /* Disable datapath on secondary process. */ 362 mlx4_mp_req_stop_rxtx(dev); 363 mlx4_flow_sync(priv, NULL); 364 mlx4_rxq_intr_disable(priv); 365 mlx4_rss_deinit(priv); 366 } 367 368 /** 369 * DPDK callback to close the device. 370 * 371 * Destroy all queues and objects, free memory. 372 * 373 * @param dev 374 * Pointer to Ethernet device structure. 375 */ 376 static void 377 mlx4_dev_close(struct rte_eth_dev *dev) 378 { 379 struct mlx4_priv *priv = dev->data->dev_private; 380 unsigned int i; 381 382 DEBUG("%p: closing device \"%s\"", 383 (void *)dev, 384 ((priv->ctx != NULL) ? priv->ctx->device->name : "")); 385 dev->rx_pkt_burst = mlx4_rx_burst_removed; 386 dev->tx_pkt_burst = mlx4_tx_burst_removed; 387 rte_wmb(); 388 /* Disable datapath on secondary process. */ 389 mlx4_mp_req_stop_rxtx(dev); 390 mlx4_flow_clean(priv); 391 mlx4_rss_deinit(priv); 392 for (i = 0; i != dev->data->nb_rx_queues; ++i) 393 mlx4_rx_queue_release(dev->data->rx_queues[i]); 394 for (i = 0; i != dev->data->nb_tx_queues; ++i) 395 mlx4_tx_queue_release(dev->data->tx_queues[i]); 396 mlx4_proc_priv_uninit(dev); 397 mlx4_mr_release(dev); 398 if (priv->pd != NULL) { 399 MLX4_ASSERT(priv->ctx != NULL); 400 claim_zero(mlx4_glue->dealloc_pd(priv->pd)); 401 claim_zero(mlx4_glue->close_device(priv->ctx)); 402 } else 403 MLX4_ASSERT(priv->ctx == NULL); 404 mlx4_intr_uninstall(priv); 405 memset(priv, 0, sizeof(*priv)); 406 } 407 408 static const struct eth_dev_ops mlx4_dev_ops = { 409 .dev_configure = mlx4_dev_configure, 410 .dev_start = mlx4_dev_start, 411 .dev_stop = mlx4_dev_stop, 412 .dev_set_link_down = mlx4_dev_set_link_down, 413 .dev_set_link_up = mlx4_dev_set_link_up, 414 .dev_close = mlx4_dev_close, 415 .link_update = mlx4_link_update, 416 .promiscuous_enable = mlx4_promiscuous_enable, 417 .promiscuous_disable = mlx4_promiscuous_disable, 418 .allmulticast_enable = mlx4_allmulticast_enable, 419 .allmulticast_disable = mlx4_allmulticast_disable, 420 .mac_addr_remove = mlx4_mac_addr_remove, 421 .mac_addr_add = mlx4_mac_addr_add, 422 .mac_addr_set = mlx4_mac_addr_set, 423 .set_mc_addr_list = mlx4_set_mc_addr_list, 424 .stats_get = mlx4_stats_get, 425 .stats_reset = mlx4_stats_reset, 426 .fw_version_get = mlx4_fw_version_get, 427 .dev_infos_get = mlx4_dev_infos_get, 428 .dev_supported_ptypes_get = mlx4_dev_supported_ptypes_get, 429 .vlan_filter_set = mlx4_vlan_filter_set, 430 .rx_queue_setup = mlx4_rx_queue_setup, 431 .tx_queue_setup = mlx4_tx_queue_setup, 432 .rx_queue_release = mlx4_rx_queue_release, 433 .tx_queue_release = mlx4_tx_queue_release, 434 .flow_ctrl_get = mlx4_flow_ctrl_get, 435 .flow_ctrl_set = mlx4_flow_ctrl_set, 436 .mtu_set = mlx4_mtu_set, 437 .filter_ctrl = mlx4_filter_ctrl, 438 .rx_queue_intr_enable = mlx4_rx_intr_enable, 439 .rx_queue_intr_disable = mlx4_rx_intr_disable, 440 .is_removed = mlx4_is_removed, 441 }; 442 443 /* Available operations from secondary process. */ 444 static const struct eth_dev_ops mlx4_dev_sec_ops = { 445 .stats_get = mlx4_stats_get, 446 .stats_reset = mlx4_stats_reset, 447 .fw_version_get = mlx4_fw_version_get, 448 .dev_infos_get = mlx4_dev_infos_get, 449 }; 450 451 /** 452 * Get PCI information from struct ibv_device. 453 * 454 * @param device 455 * Pointer to Ethernet device structure. 456 * @param[out] pci_addr 457 * PCI bus address output buffer. 458 * 459 * @return 460 * 0 on success, negative errno value otherwise and rte_errno is set. 461 */ 462 static int 463 mlx4_ibv_device_to_pci_addr(const struct ibv_device *device, 464 struct rte_pci_addr *pci_addr) 465 { 466 FILE *file; 467 char line[32]; 468 MKSTR(path, "%s/device/uevent", device->ibdev_path); 469 470 file = fopen(path, "rb"); 471 if (file == NULL) { 472 rte_errno = errno; 473 return -rte_errno; 474 } 475 while (fgets(line, sizeof(line), file) == line) { 476 size_t len = strlen(line); 477 int ret; 478 479 /* Truncate long lines. */ 480 if (len == (sizeof(line) - 1)) 481 while (line[(len - 1)] != '\n') { 482 ret = fgetc(file); 483 if (ret == EOF) 484 break; 485 line[(len - 1)] = ret; 486 } 487 /* Extract information. */ 488 if (sscanf(line, 489 "PCI_SLOT_NAME=" 490 "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n", 491 &pci_addr->domain, 492 &pci_addr->bus, 493 &pci_addr->devid, 494 &pci_addr->function) == 4) { 495 break; 496 } 497 } 498 fclose(file); 499 return 0; 500 } 501 502 /** 503 * Verify and store value for device argument. 504 * 505 * @param[in] key 506 * Key argument to verify. 507 * @param[in] val 508 * Value associated with key. 509 * @param[in, out] conf 510 * Shared configuration data. 511 * 512 * @return 513 * 0 on success, negative errno value otherwise and rte_errno is set. 514 */ 515 static int 516 mlx4_arg_parse(const char *key, const char *val, struct mlx4_conf *conf) 517 { 518 unsigned long tmp; 519 520 errno = 0; 521 tmp = strtoul(val, NULL, 0); 522 if (errno) { 523 rte_errno = errno; 524 WARN("%s: \"%s\" is not a valid integer", key, val); 525 return -rte_errno; 526 } 527 if (strcmp(MLX4_PMD_PORT_KVARG, key) == 0) { 528 uint32_t ports = rte_log2_u32(conf->ports.present + 1); 529 530 if (tmp >= ports) { 531 ERROR("port index %lu outside range [0,%" PRIu32 ")", 532 tmp, ports); 533 return -EINVAL; 534 } 535 if (!(conf->ports.present & (1 << tmp))) { 536 rte_errno = EINVAL; 537 ERROR("invalid port index %lu", tmp); 538 return -rte_errno; 539 } 540 conf->ports.enabled |= 1 << tmp; 541 } else if (strcmp(MLX4_MR_EXT_MEMSEG_EN_KVARG, key) == 0) { 542 conf->mr_ext_memseg_en = !!tmp; 543 } else { 544 rte_errno = EINVAL; 545 WARN("%s: unknown parameter", key); 546 return -rte_errno; 547 } 548 return 0; 549 } 550 551 /** 552 * Parse device parameters. 553 * 554 * @param devargs 555 * Device arguments structure. 556 * 557 * @return 558 * 0 on success, negative errno value otherwise and rte_errno is set. 559 */ 560 static int 561 mlx4_args(struct rte_devargs *devargs, struct mlx4_conf *conf) 562 { 563 struct rte_kvargs *kvlist; 564 unsigned int arg_count; 565 int ret = 0; 566 int i; 567 568 if (devargs == NULL) 569 return 0; 570 kvlist = rte_kvargs_parse(devargs->args, pmd_mlx4_init_params); 571 if (kvlist == NULL) { 572 rte_errno = EINVAL; 573 ERROR("failed to parse kvargs"); 574 return -rte_errno; 575 } 576 /* Process parameters. */ 577 for (i = 0; pmd_mlx4_init_params[i]; ++i) { 578 arg_count = rte_kvargs_count(kvlist, pmd_mlx4_init_params[i]); 579 while (arg_count-- > 0) { 580 ret = rte_kvargs_process(kvlist, 581 pmd_mlx4_init_params[i], 582 (int (*)(const char *, 583 const char *, 584 void *)) 585 mlx4_arg_parse, 586 conf); 587 if (ret != 0) 588 goto free_kvlist; 589 } 590 } 591 free_kvlist: 592 rte_kvargs_free(kvlist); 593 return ret; 594 } 595 596 /** 597 * Interpret RSS capabilities reported by device. 598 * 599 * This function returns the set of usable Verbs RSS hash fields, kernel 600 * quirks taken into account. 601 * 602 * @param ctx 603 * Verbs context. 604 * @param pd 605 * Verbs protection domain. 606 * @param device_attr_ex 607 * Extended device attributes to interpret. 608 * 609 * @return 610 * Usable RSS hash fields mask in Verbs format. 611 */ 612 static uint64_t 613 mlx4_hw_rss_sup(struct ibv_context *ctx, struct ibv_pd *pd, 614 struct ibv_device_attr_ex *device_attr_ex) 615 { 616 uint64_t hw_rss_sup = device_attr_ex->rss_caps.rx_hash_fields_mask; 617 struct ibv_cq *cq = NULL; 618 struct ibv_wq *wq = NULL; 619 struct ibv_rwq_ind_table *ind = NULL; 620 struct ibv_qp *qp = NULL; 621 622 if (!hw_rss_sup) { 623 WARN("no RSS capabilities reported; disabling support for UDP" 624 " RSS and inner VXLAN RSS"); 625 return IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4 | 626 IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6 | 627 IBV_RX_HASH_SRC_PORT_TCP | IBV_RX_HASH_DST_PORT_TCP; 628 } 629 if (!(hw_rss_sup & IBV_RX_HASH_INNER)) 630 return hw_rss_sup; 631 /* 632 * Although reported as supported, missing code in some Linux 633 * versions (v4.15, v4.16) prevents the creation of hash QPs with 634 * inner capability. 635 * 636 * There is no choice but to attempt to instantiate a temporary RSS 637 * context in order to confirm its support. 638 */ 639 cq = mlx4_glue->create_cq(ctx, 1, NULL, NULL, 0); 640 wq = cq ? mlx4_glue->create_wq 641 (ctx, 642 &(struct ibv_wq_init_attr){ 643 .wq_type = IBV_WQT_RQ, 644 .max_wr = 1, 645 .max_sge = 1, 646 .pd = pd, 647 .cq = cq, 648 }) : NULL; 649 ind = wq ? mlx4_glue->create_rwq_ind_table 650 (ctx, 651 &(struct ibv_rwq_ind_table_init_attr){ 652 .log_ind_tbl_size = 0, 653 .ind_tbl = &wq, 654 .comp_mask = 0, 655 }) : NULL; 656 qp = ind ? mlx4_glue->create_qp_ex 657 (ctx, 658 &(struct ibv_qp_init_attr_ex){ 659 .comp_mask = 660 (IBV_QP_INIT_ATTR_PD | 661 IBV_QP_INIT_ATTR_RX_HASH | 662 IBV_QP_INIT_ATTR_IND_TABLE), 663 .qp_type = IBV_QPT_RAW_PACKET, 664 .pd = pd, 665 .rwq_ind_tbl = ind, 666 .rx_hash_conf = { 667 .rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ, 668 .rx_hash_key_len = MLX4_RSS_HASH_KEY_SIZE, 669 .rx_hash_key = mlx4_rss_hash_key_default, 670 .rx_hash_fields_mask = hw_rss_sup, 671 }, 672 }) : NULL; 673 if (!qp) { 674 WARN("disabling unusable inner RSS capability due to kernel" 675 " quirk"); 676 hw_rss_sup &= ~IBV_RX_HASH_INNER; 677 } else { 678 claim_zero(mlx4_glue->destroy_qp(qp)); 679 } 680 if (ind) 681 claim_zero(mlx4_glue->destroy_rwq_ind_table(ind)); 682 if (wq) 683 claim_zero(mlx4_glue->destroy_wq(wq)); 684 if (cq) 685 claim_zero(mlx4_glue->destroy_cq(cq)); 686 return hw_rss_sup; 687 } 688 689 static struct rte_pci_driver mlx4_driver; 690 691 /** 692 * PMD global initialization. 693 * 694 * Independent from individual device, this function initializes global 695 * per-PMD data structures distinguishing primary and secondary processes. 696 * Hence, each initialization is called once per a process. 697 * 698 * @return 699 * 0 on success, a negative errno value otherwise and rte_errno is set. 700 */ 701 static int 702 mlx4_init_once(void) 703 { 704 struct mlx4_shared_data *sd; 705 struct mlx4_local_data *ld = &mlx4_local_data; 706 int ret = 0; 707 708 if (mlx4_init_shared_data()) 709 return -rte_errno; 710 sd = mlx4_shared_data; 711 MLX4_ASSERT(sd); 712 rte_spinlock_lock(&sd->lock); 713 switch (rte_eal_process_type()) { 714 case RTE_PROC_PRIMARY: 715 if (sd->init_done) 716 break; 717 LIST_INIT(&sd->mem_event_cb_list); 718 rte_rwlock_init(&sd->mem_event_rwlock); 719 rte_mem_event_callback_register("MLX4_MEM_EVENT_CB", 720 mlx4_mr_mem_event_cb, NULL); 721 ret = mlx4_mp_init_primary(); 722 if (ret) 723 goto out; 724 sd->init_done = 1; 725 break; 726 case RTE_PROC_SECONDARY: 727 if (ld->init_done) 728 break; 729 ret = mlx4_mp_init_secondary(); 730 if (ret) 731 goto out; 732 ++sd->secondary_cnt; 733 ld->init_done = 1; 734 break; 735 default: 736 break; 737 } 738 out: 739 rte_spinlock_unlock(&sd->lock); 740 return ret; 741 } 742 743 /** 744 * DPDK callback to register a PCI device. 745 * 746 * This function creates an Ethernet device for each port of a given 747 * PCI device. 748 * 749 * @param[in] pci_drv 750 * PCI driver structure (mlx4_driver). 751 * @param[in] pci_dev 752 * PCI device information. 753 * 754 * @return 755 * 0 on success, negative errno value otherwise and rte_errno is set. 756 */ 757 static int 758 mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) 759 { 760 struct ibv_device **list; 761 struct ibv_device *ibv_dev; 762 int err = 0; 763 struct ibv_context *attr_ctx = NULL; 764 struct ibv_device_attr device_attr; 765 struct ibv_device_attr_ex device_attr_ex; 766 struct mlx4_conf conf = { 767 .ports.present = 0, 768 .mr_ext_memseg_en = 1, 769 }; 770 unsigned int vf; 771 int i; 772 char ifname[IF_NAMESIZE]; 773 774 (void)pci_drv; 775 err = mlx4_init_once(); 776 if (err) { 777 ERROR("unable to init PMD global data: %s", 778 strerror(rte_errno)); 779 return -rte_errno; 780 } 781 MLX4_ASSERT(pci_drv == &mlx4_driver); 782 list = mlx4_glue->get_device_list(&i); 783 if (list == NULL) { 784 rte_errno = errno; 785 MLX4_ASSERT(rte_errno); 786 if (rte_errno == ENOSYS) 787 ERROR("cannot list devices, is ib_uverbs loaded?"); 788 return -rte_errno; 789 } 790 MLX4_ASSERT(i >= 0); 791 /* 792 * For each listed device, check related sysfs entry against 793 * the provided PCI ID. 794 */ 795 while (i != 0) { 796 struct rte_pci_addr pci_addr; 797 798 --i; 799 DEBUG("checking device \"%s\"", list[i]->name); 800 if (mlx4_ibv_device_to_pci_addr(list[i], &pci_addr)) 801 continue; 802 if ((pci_dev->addr.domain != pci_addr.domain) || 803 (pci_dev->addr.bus != pci_addr.bus) || 804 (pci_dev->addr.devid != pci_addr.devid) || 805 (pci_dev->addr.function != pci_addr.function)) 806 continue; 807 vf = (pci_dev->id.device_id == 808 PCI_DEVICE_ID_MELLANOX_CONNECTX3VF); 809 INFO("PCI information matches, using device \"%s\" (VF: %s)", 810 list[i]->name, (vf ? "true" : "false")); 811 attr_ctx = mlx4_glue->open_device(list[i]); 812 err = errno; 813 break; 814 } 815 if (attr_ctx == NULL) { 816 mlx4_glue->free_device_list(list); 817 switch (err) { 818 case 0: 819 rte_errno = ENODEV; 820 ERROR("cannot access device, is mlx4_ib loaded?"); 821 return -rte_errno; 822 case EINVAL: 823 rte_errno = EINVAL; 824 ERROR("cannot use device, are drivers up to date?"); 825 return -rte_errno; 826 } 827 MLX4_ASSERT(err > 0); 828 rte_errno = err; 829 return -rte_errno; 830 } 831 ibv_dev = list[i]; 832 DEBUG("device opened"); 833 if (mlx4_glue->query_device(attr_ctx, &device_attr)) { 834 err = ENODEV; 835 goto error; 836 } 837 INFO("%u port(s) detected", device_attr.phys_port_cnt); 838 conf.ports.present |= (UINT64_C(1) << device_attr.phys_port_cnt) - 1; 839 if (mlx4_args(pci_dev->device.devargs, &conf)) { 840 ERROR("failed to process device arguments"); 841 err = EINVAL; 842 goto error; 843 } 844 /* Use all ports when none are defined */ 845 if (!conf.ports.enabled) 846 conf.ports.enabled = conf.ports.present; 847 /* Retrieve extended device attributes. */ 848 if (mlx4_glue->query_device_ex(attr_ctx, NULL, &device_attr_ex)) { 849 err = ENODEV; 850 goto error; 851 } 852 MLX4_ASSERT(device_attr.max_sge >= MLX4_MAX_SGE); 853 for (i = 0; i < device_attr.phys_port_cnt; i++) { 854 uint32_t port = i + 1; /* ports are indexed from one */ 855 struct ibv_context *ctx = NULL; 856 struct ibv_port_attr port_attr; 857 struct ibv_pd *pd = NULL; 858 struct mlx4_priv *priv = NULL; 859 struct rte_eth_dev *eth_dev = NULL; 860 struct rte_ether_addr mac; 861 char name[RTE_ETH_NAME_MAX_LEN]; 862 863 /* If port is not enabled, skip. */ 864 if (!(conf.ports.enabled & (1 << i))) 865 continue; 866 DEBUG("using port %u", port); 867 ctx = mlx4_glue->open_device(ibv_dev); 868 if (ctx == NULL) { 869 err = ENODEV; 870 goto port_error; 871 } 872 snprintf(name, sizeof(name), "%s port %u", 873 mlx4_glue->get_device_name(ibv_dev), port); 874 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 875 eth_dev = rte_eth_dev_attach_secondary(name); 876 if (eth_dev == NULL) { 877 ERROR("can not attach rte ethdev"); 878 rte_errno = ENOMEM; 879 err = rte_errno; 880 goto error; 881 } 882 priv = eth_dev->data->dev_private; 883 if (!priv->verbs_alloc_ctx.enabled) { 884 ERROR("secondary process is not supported" 885 " due to lack of external allocator" 886 " from Verbs"); 887 rte_errno = ENOTSUP; 888 err = rte_errno; 889 goto error; 890 } 891 eth_dev->device = &pci_dev->device; 892 eth_dev->dev_ops = &mlx4_dev_sec_ops; 893 err = mlx4_proc_priv_init(eth_dev); 894 if (err) 895 goto error; 896 /* Receive command fd from primary process. */ 897 err = mlx4_mp_req_verbs_cmd_fd(eth_dev); 898 if (err < 0) { 899 err = rte_errno; 900 goto error; 901 } 902 /* Remap UAR for Tx queues. */ 903 err = mlx4_tx_uar_init_secondary(eth_dev, err); 904 if (err) { 905 err = rte_errno; 906 goto error; 907 } 908 /* 909 * Ethdev pointer is still required as input since 910 * the primary device is not accessible from the 911 * secondary process. 912 */ 913 eth_dev->tx_pkt_burst = mlx4_tx_burst; 914 eth_dev->rx_pkt_burst = mlx4_rx_burst; 915 claim_zero(mlx4_glue->close_device(ctx)); 916 rte_eth_copy_pci_info(eth_dev, pci_dev); 917 rte_eth_dev_probing_finish(eth_dev); 918 continue; 919 } 920 /* Check port status. */ 921 err = mlx4_glue->query_port(ctx, port, &port_attr); 922 if (err) { 923 err = ENODEV; 924 ERROR("port query failed: %s", strerror(err)); 925 goto port_error; 926 } 927 if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) { 928 err = ENOTSUP; 929 ERROR("port %d is not configured in Ethernet mode", 930 port); 931 goto port_error; 932 } 933 if (port_attr.state != IBV_PORT_ACTIVE) 934 DEBUG("port %d is not active: \"%s\" (%d)", 935 port, mlx4_glue->port_state_str(port_attr.state), 936 port_attr.state); 937 /* Make asynchronous FD non-blocking to handle interrupts. */ 938 err = mlx4_fd_set_non_blocking(ctx->async_fd); 939 if (err) { 940 ERROR("cannot make asynchronous FD non-blocking: %s", 941 strerror(err)); 942 goto port_error; 943 } 944 /* Allocate protection domain. */ 945 pd = mlx4_glue->alloc_pd(ctx); 946 if (pd == NULL) { 947 err = ENOMEM; 948 ERROR("PD allocation failure"); 949 goto port_error; 950 } 951 /* from rte_ethdev.c */ 952 priv = rte_zmalloc("ethdev private structure", 953 sizeof(*priv), 954 RTE_CACHE_LINE_SIZE); 955 if (priv == NULL) { 956 err = ENOMEM; 957 ERROR("priv allocation failure"); 958 goto port_error; 959 } 960 priv->ctx = ctx; 961 priv->device_attr = device_attr; 962 priv->port = port; 963 priv->pd = pd; 964 priv->mtu = RTE_ETHER_MTU; 965 priv->vf = vf; 966 priv->hw_csum = !!(device_attr.device_cap_flags & 967 IBV_DEVICE_RAW_IP_CSUM); 968 DEBUG("checksum offloading is %ssupported", 969 (priv->hw_csum ? "" : "not ")); 970 /* Only ConnectX-3 Pro supports tunneling. */ 971 priv->hw_csum_l2tun = 972 priv->hw_csum && 973 (device_attr.vendor_part_id == 974 PCI_DEVICE_ID_MELLANOX_CONNECTX3PRO); 975 DEBUG("L2 tunnel checksum offloads are %ssupported", 976 priv->hw_csum_l2tun ? "" : "not "); 977 priv->hw_rss_sup = mlx4_hw_rss_sup(priv->ctx, priv->pd, 978 &device_attr_ex); 979 DEBUG("supported RSS hash fields mask: %016" PRIx64, 980 priv->hw_rss_sup); 981 priv->hw_rss_max_qps = 982 device_attr_ex.rss_caps.max_rwq_indirection_table_size; 983 DEBUG("MAX RSS queues %d", priv->hw_rss_max_qps); 984 priv->hw_fcs_strip = !!(device_attr_ex.raw_packet_caps & 985 IBV_RAW_PACKET_CAP_SCATTER_FCS); 986 DEBUG("FCS stripping toggling is %ssupported", 987 priv->hw_fcs_strip ? "" : "not "); 988 priv->tso = 989 ((device_attr_ex.tso_caps.max_tso > 0) && 990 (device_attr_ex.tso_caps.supported_qpts & 991 (1 << IBV_QPT_RAW_PACKET))); 992 if (priv->tso) 993 priv->tso_max_payload_sz = 994 device_attr_ex.tso_caps.max_tso; 995 DEBUG("TSO is %ssupported", 996 priv->tso ? "" : "not "); 997 priv->mr_ext_memseg_en = conf.mr_ext_memseg_en; 998 /* Configure the first MAC address by default. */ 999 err = mlx4_get_mac(priv, &mac.addr_bytes); 1000 if (err) { 1001 ERROR("cannot get MAC address, is mlx4_en loaded?" 1002 " (error: %s)", strerror(err)); 1003 goto port_error; 1004 } 1005 INFO("port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x", 1006 priv->port, 1007 mac.addr_bytes[0], mac.addr_bytes[1], 1008 mac.addr_bytes[2], mac.addr_bytes[3], 1009 mac.addr_bytes[4], mac.addr_bytes[5]); 1010 /* Register MAC address. */ 1011 priv->mac[0] = mac; 1012 1013 if (mlx4_get_ifname(priv, &ifname) == 0) { 1014 DEBUG("port %u ifname is \"%s\"", 1015 priv->port, ifname); 1016 priv->if_index = if_nametoindex(ifname); 1017 } else { 1018 DEBUG("port %u ifname is unknown", priv->port); 1019 } 1020 1021 /* Get actual MTU if possible. */ 1022 mlx4_mtu_get(priv, &priv->mtu); 1023 DEBUG("port %u MTU is %u", priv->port, priv->mtu); 1024 eth_dev = rte_eth_dev_allocate(name); 1025 if (eth_dev == NULL) { 1026 err = ENOMEM; 1027 ERROR("can not allocate rte ethdev"); 1028 goto port_error; 1029 } 1030 eth_dev->data->dev_private = priv; 1031 eth_dev->data->mac_addrs = priv->mac; 1032 eth_dev->device = &pci_dev->device; 1033 rte_eth_copy_pci_info(eth_dev, pci_dev); 1034 /* Initialize local interrupt handle for current port. */ 1035 priv->intr_handle = (struct rte_intr_handle){ 1036 .fd = -1, 1037 .type = RTE_INTR_HANDLE_EXT, 1038 }; 1039 /* 1040 * Override ethdev interrupt handle pointer with private 1041 * handle instead of that of the parent PCI device used by 1042 * default. This prevents it from being shared between all 1043 * ports of the same PCI device since each of them is 1044 * associated its own Verbs context. 1045 * 1046 * Rx interrupts in particular require this as the PMD has 1047 * no control over the registration of queue interrupts 1048 * besides setting up eth_dev->intr_handle, the rest is 1049 * handled by rte_intr_rx_ctl(). 1050 */ 1051 eth_dev->intr_handle = &priv->intr_handle; 1052 priv->dev_data = eth_dev->data; 1053 eth_dev->dev_ops = &mlx4_dev_ops; 1054 #ifdef HAVE_IBV_MLX4_BUF_ALLOCATORS 1055 /* Hint libmlx4 to use PMD allocator for data plane resources */ 1056 err = mlx4_glue->dv_set_context_attr 1057 (ctx, MLX4DV_SET_CTX_ATTR_BUF_ALLOCATORS, 1058 (void *)((uintptr_t)&(struct mlx4dv_ctx_allocators){ 1059 .alloc = &mlx4_alloc_verbs_buf, 1060 .free = &mlx4_free_verbs_buf, 1061 .data = priv, 1062 })); 1063 if (err) 1064 WARN("Verbs external allocator is not supported"); 1065 else 1066 priv->verbs_alloc_ctx.enabled = 1; 1067 #endif 1068 /* Bring Ethernet device up. */ 1069 DEBUG("forcing Ethernet interface up"); 1070 mlx4_dev_set_link_up(eth_dev); 1071 /* Update link status once if waiting for LSC. */ 1072 if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) 1073 mlx4_link_update(eth_dev, 0); 1074 /* 1075 * Once the device is added to the list of memory event 1076 * callback, its global MR cache table cannot be expanded 1077 * on the fly because of deadlock. If it overflows, lookup 1078 * should be done by searching MR list linearly, which is slow. 1079 */ 1080 err = mlx4_mr_btree_init(&priv->mr.cache, 1081 MLX4_MR_BTREE_CACHE_N * 2, 1082 eth_dev->device->numa_node); 1083 if (err) { 1084 /* rte_errno is already set. */ 1085 goto port_error; 1086 } 1087 /* Add device to memory callback list. */ 1088 rte_rwlock_write_lock(&mlx4_shared_data->mem_event_rwlock); 1089 LIST_INSERT_HEAD(&mlx4_shared_data->mem_event_cb_list, 1090 priv, mem_event_cb); 1091 rte_rwlock_write_unlock(&mlx4_shared_data->mem_event_rwlock); 1092 rte_eth_dev_probing_finish(eth_dev); 1093 continue; 1094 port_error: 1095 rte_free(priv); 1096 if (eth_dev != NULL) 1097 eth_dev->data->dev_private = NULL; 1098 if (pd) 1099 claim_zero(mlx4_glue->dealloc_pd(pd)); 1100 if (ctx) 1101 claim_zero(mlx4_glue->close_device(ctx)); 1102 if (eth_dev != NULL) { 1103 /* mac_addrs must not be freed because part of dev_private */ 1104 eth_dev->data->mac_addrs = NULL; 1105 rte_eth_dev_release_port(eth_dev); 1106 } 1107 break; 1108 } 1109 /* 1110 * XXX if something went wrong in the loop above, there is a resource 1111 * leak (ctx, pd, priv, dpdk ethdev) but we can do nothing about it as 1112 * long as the dpdk does not provide a way to deallocate a ethdev and a 1113 * way to enumerate the registered ethdevs to free the previous ones. 1114 */ 1115 error: 1116 if (attr_ctx) 1117 claim_zero(mlx4_glue->close_device(attr_ctx)); 1118 if (list) 1119 mlx4_glue->free_device_list(list); 1120 if (err) 1121 rte_errno = err; 1122 return -err; 1123 } 1124 1125 static const struct rte_pci_id mlx4_pci_id_map[] = { 1126 { 1127 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 1128 PCI_DEVICE_ID_MELLANOX_CONNECTX3) 1129 }, 1130 { 1131 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 1132 PCI_DEVICE_ID_MELLANOX_CONNECTX3PRO) 1133 }, 1134 { 1135 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 1136 PCI_DEVICE_ID_MELLANOX_CONNECTX3VF) 1137 }, 1138 { 1139 .vendor_id = 0 1140 } 1141 }; 1142 1143 static struct rte_pci_driver mlx4_driver = { 1144 .driver = { 1145 .name = MLX4_DRIVER_NAME 1146 }, 1147 .id_table = mlx4_pci_id_map, 1148 .probe = mlx4_pci_probe, 1149 .drv_flags = RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV, 1150 }; 1151 1152 #ifdef RTE_IBVERBS_LINK_DLOPEN 1153 1154 /** 1155 * Suffix RTE_EAL_PMD_PATH with "-glue". 1156 * 1157 * This function performs a sanity check on RTE_EAL_PMD_PATH before 1158 * suffixing its last component. 1159 * 1160 * @param buf[out] 1161 * Output buffer, should be large enough otherwise NULL is returned. 1162 * @param size 1163 * Size of @p out. 1164 * 1165 * @return 1166 * Pointer to @p buf or @p NULL in case suffix cannot be appended. 1167 */ 1168 static char * 1169 mlx4_glue_path(char *buf, size_t size) 1170 { 1171 static const char *const bad[] = { "/", ".", "..", NULL }; 1172 const char *path = RTE_EAL_PMD_PATH; 1173 size_t len = strlen(path); 1174 size_t off; 1175 int i; 1176 1177 while (len && path[len - 1] == '/') 1178 --len; 1179 for (off = len; off && path[off - 1] != '/'; --off) 1180 ; 1181 for (i = 0; bad[i]; ++i) 1182 if (!strncmp(path + off, bad[i], (int)(len - off))) 1183 goto error; 1184 i = snprintf(buf, size, "%.*s-glue", (int)len, path); 1185 if (i == -1 || (size_t)i >= size) 1186 goto error; 1187 return buf; 1188 error: 1189 ERROR("unable to append \"-glue\" to last component of" 1190 " RTE_EAL_PMD_PATH (\"" RTE_EAL_PMD_PATH "\")," 1191 " please re-configure DPDK"); 1192 return NULL; 1193 } 1194 1195 /** 1196 * Initialization routine for run-time dependency on rdma-core. 1197 */ 1198 static int 1199 mlx4_glue_init(void) 1200 { 1201 char glue_path[sizeof(RTE_EAL_PMD_PATH) - 1 + sizeof("-glue")]; 1202 const char *path[] = { 1203 /* 1204 * A basic security check is necessary before trusting 1205 * MLX4_GLUE_PATH, which may override RTE_EAL_PMD_PATH. 1206 */ 1207 (geteuid() == getuid() && getegid() == getgid() ? 1208 getenv("MLX4_GLUE_PATH") : NULL), 1209 /* 1210 * When RTE_EAL_PMD_PATH is set, use its glue-suffixed 1211 * variant, otherwise let dlopen() look up libraries on its 1212 * own. 1213 */ 1214 (*RTE_EAL_PMD_PATH ? 1215 mlx4_glue_path(glue_path, sizeof(glue_path)) : ""), 1216 }; 1217 unsigned int i = 0; 1218 void *handle = NULL; 1219 void **sym; 1220 const char *dlmsg; 1221 1222 while (!handle && i != RTE_DIM(path)) { 1223 const char *end; 1224 size_t len; 1225 int ret; 1226 1227 if (!path[i]) { 1228 ++i; 1229 continue; 1230 } 1231 end = strpbrk(path[i], ":;"); 1232 if (!end) 1233 end = path[i] + strlen(path[i]); 1234 len = end - path[i]; 1235 ret = 0; 1236 do { 1237 char name[ret + 1]; 1238 1239 ret = snprintf(name, sizeof(name), "%.*s%s" MLX4_GLUE, 1240 (int)len, path[i], 1241 (!len || *(end - 1) == '/') ? "" : "/"); 1242 if (ret == -1) 1243 break; 1244 if (sizeof(name) != (size_t)ret + 1) 1245 continue; 1246 DEBUG("looking for rdma-core glue as \"%s\"", name); 1247 handle = dlopen(name, RTLD_LAZY); 1248 break; 1249 } while (1); 1250 path[i] = end + 1; 1251 if (!*end) 1252 ++i; 1253 } 1254 if (!handle) { 1255 rte_errno = EINVAL; 1256 dlmsg = dlerror(); 1257 if (dlmsg) 1258 WARN("cannot load glue library: %s", dlmsg); 1259 goto glue_error; 1260 } 1261 sym = dlsym(handle, "mlx4_glue"); 1262 if (!sym || !*sym) { 1263 rte_errno = EINVAL; 1264 dlmsg = dlerror(); 1265 if (dlmsg) 1266 ERROR("cannot resolve glue symbol: %s", dlmsg); 1267 goto glue_error; 1268 } 1269 mlx4_glue = *sym; 1270 return 0; 1271 glue_error: 1272 if (handle) 1273 dlclose(handle); 1274 WARN("cannot initialize PMD due to missing run-time" 1275 " dependency on rdma-core libraries (libibverbs," 1276 " libmlx4)"); 1277 return -rte_errno; 1278 } 1279 1280 #endif 1281 1282 /** 1283 * Driver initialization routine. 1284 */ 1285 RTE_INIT(rte_mlx4_pmd_init) 1286 { 1287 /* Initialize driver log type. */ 1288 mlx4_logtype = rte_log_register("pmd.net.mlx4"); 1289 if (mlx4_logtype >= 0) 1290 rte_log_set_level(mlx4_logtype, RTE_LOG_NOTICE); 1291 1292 /* 1293 * MLX4_DEVICE_FATAL_CLEANUP tells ibv_destroy functions we 1294 * want to get success errno value in case of calling them 1295 * when the device was removed. 1296 */ 1297 setenv("MLX4_DEVICE_FATAL_CLEANUP", "1", 1); 1298 /* 1299 * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use 1300 * huge pages. Calling ibv_fork_init() during init allows 1301 * applications to use fork() safely for purposes other than 1302 * using this PMD, which is not supported in forked processes. 1303 */ 1304 setenv("RDMAV_HUGEPAGES_SAFE", "1", 1); 1305 #ifdef RTE_IBVERBS_LINK_DLOPEN 1306 if (mlx4_glue_init()) 1307 return; 1308 MLX4_ASSERT(mlx4_glue); 1309 #endif 1310 #ifdef RTE_LIBRTE_MLX4_DEBUG 1311 /* Glue structure must not contain any NULL pointers. */ 1312 { 1313 unsigned int i; 1314 1315 for (i = 0; i != sizeof(*mlx4_glue) / sizeof(void *); ++i) 1316 MLX4_ASSERT(((const void *const *)mlx4_glue)[i]); 1317 } 1318 #endif 1319 if (strcmp(mlx4_glue->version, MLX4_GLUE_VERSION)) { 1320 ERROR("rdma-core glue \"%s\" mismatch: \"%s\" is required", 1321 mlx4_glue->version, MLX4_GLUE_VERSION); 1322 return; 1323 } 1324 mlx4_glue->fork_init(); 1325 rte_pci_register(&mlx4_driver); 1326 } 1327 1328 RTE_PMD_EXPORT_NAME(net_mlx4, __COUNTER__); 1329 RTE_PMD_REGISTER_PCI_TABLE(net_mlx4, mlx4_pci_id_map); 1330 RTE_PMD_REGISTER_KMOD_DEP(net_mlx4, 1331 "* ib_uverbs & mlx4_en & mlx4_core & mlx4_ib"); 1332