1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2012 6WIND S.A. 3 * Copyright 2012 Mellanox Technologies, Ltd 4 */ 5 6 /** 7 * @file 8 * mlx4 driver initialization. 9 */ 10 11 #include <errno.h> 12 #include <inttypes.h> 13 #include <stddef.h> 14 #include <stdint.h> 15 #include <stdio.h> 16 #include <stdlib.h> 17 #include <string.h> 18 #include <sys/mman.h> 19 #include <unistd.h> 20 #ifdef RTE_IBVERBS_LINK_DLOPEN 21 #include <dlfcn.h> 22 #endif 23 24 /* Verbs headers do not support -pedantic. */ 25 #ifdef PEDANTIC 26 #pragma GCC diagnostic ignored "-Wpedantic" 27 #endif 28 #include <infiniband/verbs.h> 29 #ifdef PEDANTIC 30 #pragma GCC diagnostic error "-Wpedantic" 31 #endif 32 33 #include <rte_common.h> 34 #include <rte_dev.h> 35 #include <rte_errno.h> 36 #include <rte_ethdev_driver.h> 37 #include <rte_ethdev_pci.h> 38 #include <rte_ether.h> 39 #include <rte_flow.h> 40 #include <rte_interrupts.h> 41 #include <rte_kvargs.h> 42 #include <rte_malloc.h> 43 #include <rte_mbuf.h> 44 45 #include "mlx4.h" 46 #include "mlx4_glue.h" 47 #include "mlx4_flow.h" 48 #include "mlx4_mr.h" 49 #include "mlx4_rxtx.h" 50 #include "mlx4_utils.h" 51 52 #ifdef MLX4_GLUE 53 const struct mlx4_glue *mlx4_glue; 54 #endif 55 56 static const char *MZ_MLX4_PMD_SHARED_DATA = "mlx4_pmd_shared_data"; 57 58 /* Shared memory between primary and secondary processes. */ 59 struct mlx4_shared_data *mlx4_shared_data; 60 61 /* Spinlock for mlx4_shared_data allocation. */ 62 static rte_spinlock_t mlx4_shared_data_lock = RTE_SPINLOCK_INITIALIZER; 63 64 /* Process local data for secondary processes. */ 65 static struct mlx4_local_data mlx4_local_data; 66 67 /** Configuration structure for device arguments. */ 68 struct mlx4_conf { 69 struct { 70 uint32_t present; /**< Bit-field for existing ports. */ 71 uint32_t enabled; /**< Bit-field for user-enabled ports. */ 72 } ports; 73 int mr_ext_memseg_en; 74 /** Whether memseg should be extended for MR creation. */ 75 }; 76 77 /* Available parameters list. */ 78 const char *pmd_mlx4_init_params[] = { 79 MLX4_PMD_PORT_KVARG, 80 MLX4_MR_EXT_MEMSEG_EN_KVARG, 81 NULL, 82 }; 83 84 static void mlx4_dev_stop(struct rte_eth_dev *dev); 85 86 /** 87 * Initialize shared data between primary and secondary process. 88 * 89 * A memzone is reserved by primary process and secondary processes attach to 90 * the memzone. 91 * 92 * @return 93 * 0 on success, a negative errno value otherwise and rte_errno is set. 94 */ 95 static int 96 mlx4_init_shared_data(void) 97 { 98 const struct rte_memzone *mz; 99 int ret = 0; 100 101 rte_spinlock_lock(&mlx4_shared_data_lock); 102 if (mlx4_shared_data == NULL) { 103 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 104 /* Allocate shared memory. */ 105 mz = rte_memzone_reserve(MZ_MLX4_PMD_SHARED_DATA, 106 sizeof(*mlx4_shared_data), 107 SOCKET_ID_ANY, 0); 108 if (mz == NULL) { 109 ERROR("Cannot allocate mlx4 shared data\n"); 110 ret = -rte_errno; 111 goto error; 112 } 113 mlx4_shared_data = mz->addr; 114 memset(mlx4_shared_data, 0, sizeof(*mlx4_shared_data)); 115 rte_spinlock_init(&mlx4_shared_data->lock); 116 } else { 117 /* Lookup allocated shared memory. */ 118 mz = rte_memzone_lookup(MZ_MLX4_PMD_SHARED_DATA); 119 if (mz == NULL) { 120 ERROR("Cannot attach mlx4 shared data\n"); 121 ret = -rte_errno; 122 goto error; 123 } 124 mlx4_shared_data = mz->addr; 125 memset(&mlx4_local_data, 0, sizeof(mlx4_local_data)); 126 } 127 } 128 error: 129 rte_spinlock_unlock(&mlx4_shared_data_lock); 130 return ret; 131 } 132 133 #ifdef HAVE_IBV_MLX4_BUF_ALLOCATORS 134 /** 135 * Verbs callback to allocate a memory. This function should allocate the space 136 * according to the size provided residing inside a huge page. 137 * Please note that all allocation must respect the alignment from libmlx4 138 * (i.e. currently sysconf(_SC_PAGESIZE)). 139 * 140 * @param[in] size 141 * The size in bytes of the memory to allocate. 142 * @param[in] data 143 * A pointer to the callback data. 144 * 145 * @return 146 * Allocated buffer, NULL otherwise and rte_errno is set. 147 */ 148 static void * 149 mlx4_alloc_verbs_buf(size_t size, void *data) 150 { 151 struct mlx4_priv *priv = data; 152 void *ret; 153 size_t alignment = sysconf(_SC_PAGESIZE); 154 unsigned int socket = SOCKET_ID_ANY; 155 156 if (priv->verbs_alloc_ctx.type == MLX4_VERBS_ALLOC_TYPE_TX_QUEUE) { 157 const struct txq *txq = priv->verbs_alloc_ctx.obj; 158 159 socket = txq->socket; 160 } else if (priv->verbs_alloc_ctx.type == 161 MLX4_VERBS_ALLOC_TYPE_RX_QUEUE) { 162 const struct rxq *rxq = priv->verbs_alloc_ctx.obj; 163 164 socket = rxq->socket; 165 } 166 MLX4_ASSERT(data != NULL); 167 ret = rte_malloc_socket(__func__, size, alignment, socket); 168 if (!ret && size) 169 rte_errno = ENOMEM; 170 return ret; 171 } 172 173 /** 174 * Verbs callback to free a memory. 175 * 176 * @param[in] ptr 177 * A pointer to the memory to free. 178 * @param[in] data 179 * A pointer to the callback data. 180 */ 181 static void 182 mlx4_free_verbs_buf(void *ptr, void *data __rte_unused) 183 { 184 MLX4_ASSERT(data != NULL); 185 rte_free(ptr); 186 } 187 #endif 188 189 /** 190 * Initialize process private data structure. 191 * 192 * @param dev 193 * Pointer to Ethernet device structure. 194 * 195 * @return 196 * 0 on success, a negative errno value otherwise and rte_errno is set. 197 */ 198 static int 199 mlx4_proc_priv_init(struct rte_eth_dev *dev) 200 { 201 struct mlx4_proc_priv *ppriv; 202 size_t ppriv_size; 203 204 /* 205 * UAR register table follows the process private structure. BlueFlame 206 * registers for Tx queues are stored in the table. 207 */ 208 ppriv_size = sizeof(struct mlx4_proc_priv) + 209 dev->data->nb_tx_queues * sizeof(void *); 210 ppriv = rte_malloc_socket("mlx4_proc_priv", ppriv_size, 211 RTE_CACHE_LINE_SIZE, dev->device->numa_node); 212 if (!ppriv) { 213 rte_errno = ENOMEM; 214 return -rte_errno; 215 } 216 ppriv->uar_table_sz = ppriv_size; 217 dev->process_private = ppriv; 218 return 0; 219 } 220 221 /** 222 * Un-initialize process private data structure. 223 * 224 * @param dev 225 * Pointer to Ethernet device structure. 226 */ 227 static void 228 mlx4_proc_priv_uninit(struct rte_eth_dev *dev) 229 { 230 if (!dev->process_private) 231 return; 232 rte_free(dev->process_private); 233 dev->process_private = NULL; 234 } 235 236 /** 237 * DPDK callback for Ethernet device configuration. 238 * 239 * @param dev 240 * Pointer to Ethernet device structure. 241 * 242 * @return 243 * 0 on success, negative errno value otherwise and rte_errno is set. 244 */ 245 static int 246 mlx4_dev_configure(struct rte_eth_dev *dev) 247 { 248 struct mlx4_priv *priv = dev->data->dev_private; 249 struct rte_flow_error error; 250 int ret; 251 252 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) 253 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; 254 255 /* Prepare internal flow rules. */ 256 ret = mlx4_flow_sync(priv, &error); 257 if (ret) { 258 ERROR("cannot set up internal flow rules (code %d, \"%s\")," 259 " flow error type %d, cause %p, message: %s", 260 -ret, strerror(-ret), error.type, error.cause, 261 error.message ? error.message : "(unspecified)"); 262 goto exit; 263 } 264 ret = mlx4_intr_install(priv); 265 if (ret) { 266 ERROR("%p: interrupt handler installation failed", 267 (void *)dev); 268 goto exit; 269 } 270 ret = mlx4_proc_priv_init(dev); 271 if (ret) { 272 ERROR("%p: process private data allocation failed", 273 (void *)dev); 274 goto exit; 275 } 276 exit: 277 return ret; 278 } 279 280 /** 281 * DPDK callback to start the device. 282 * 283 * Simulate device start by initializing common RSS resources and attaching 284 * all configured flows. 285 * 286 * @param dev 287 * Pointer to Ethernet device structure. 288 * 289 * @return 290 * 0 on success, negative errno value otherwise and rte_errno is set. 291 */ 292 static int 293 mlx4_dev_start(struct rte_eth_dev *dev) 294 { 295 struct mlx4_priv *priv = dev->data->dev_private; 296 struct rte_flow_error error; 297 int ret; 298 299 if (priv->started) 300 return 0; 301 DEBUG("%p: attaching configured flows to all RX queues", (void *)dev); 302 priv->started = 1; 303 ret = mlx4_rss_init(priv); 304 if (ret) { 305 ERROR("%p: cannot initialize RSS resources: %s", 306 (void *)dev, strerror(-ret)); 307 goto err; 308 } 309 #ifdef RTE_LIBRTE_MLX4_DEBUG 310 mlx4_mr_dump_dev(dev); 311 #endif 312 ret = mlx4_rxq_intr_enable(priv); 313 if (ret) { 314 ERROR("%p: interrupt handler installation failed", 315 (void *)dev); 316 goto err; 317 } 318 ret = mlx4_flow_sync(priv, &error); 319 if (ret) { 320 ERROR("%p: cannot attach flow rules (code %d, \"%s\")," 321 " flow error type %d, cause %p, message: %s", 322 (void *)dev, 323 -ret, strerror(-ret), error.type, error.cause, 324 error.message ? error.message : "(unspecified)"); 325 goto err; 326 } 327 rte_wmb(); 328 dev->tx_pkt_burst = mlx4_tx_burst; 329 dev->rx_pkt_burst = mlx4_rx_burst; 330 /* Enable datapath on secondary process. */ 331 mlx4_mp_req_start_rxtx(dev); 332 return 0; 333 err: 334 mlx4_dev_stop(dev); 335 return ret; 336 } 337 338 /** 339 * DPDK callback to stop the device. 340 * 341 * Simulate device stop by detaching all configured flows. 342 * 343 * @param dev 344 * Pointer to Ethernet device structure. 345 */ 346 static void 347 mlx4_dev_stop(struct rte_eth_dev *dev) 348 { 349 struct mlx4_priv *priv = dev->data->dev_private; 350 351 if (!priv->started) 352 return; 353 DEBUG("%p: detaching flows from all RX queues", (void *)dev); 354 priv->started = 0; 355 dev->tx_pkt_burst = mlx4_tx_burst_removed; 356 dev->rx_pkt_burst = mlx4_rx_burst_removed; 357 rte_wmb(); 358 /* Disable datapath on secondary process. */ 359 mlx4_mp_req_stop_rxtx(dev); 360 mlx4_flow_sync(priv, NULL); 361 mlx4_rxq_intr_disable(priv); 362 mlx4_rss_deinit(priv); 363 } 364 365 /** 366 * DPDK callback to close the device. 367 * 368 * Destroy all queues and objects, free memory. 369 * 370 * @param dev 371 * Pointer to Ethernet device structure. 372 */ 373 static void 374 mlx4_dev_close(struct rte_eth_dev *dev) 375 { 376 struct mlx4_priv *priv = dev->data->dev_private; 377 unsigned int i; 378 379 DEBUG("%p: closing device \"%s\"", 380 (void *)dev, 381 ((priv->ctx != NULL) ? priv->ctx->device->name : "")); 382 dev->rx_pkt_burst = mlx4_rx_burst_removed; 383 dev->tx_pkt_burst = mlx4_tx_burst_removed; 384 rte_wmb(); 385 /* Disable datapath on secondary process. */ 386 mlx4_mp_req_stop_rxtx(dev); 387 mlx4_flow_clean(priv); 388 mlx4_rss_deinit(priv); 389 for (i = 0; i != dev->data->nb_rx_queues; ++i) 390 mlx4_rx_queue_release(dev->data->rx_queues[i]); 391 for (i = 0; i != dev->data->nb_tx_queues; ++i) 392 mlx4_tx_queue_release(dev->data->tx_queues[i]); 393 mlx4_proc_priv_uninit(dev); 394 mlx4_mr_release(dev); 395 if (priv->pd != NULL) { 396 MLX4_ASSERT(priv->ctx != NULL); 397 claim_zero(mlx4_glue->dealloc_pd(priv->pd)); 398 claim_zero(mlx4_glue->close_device(priv->ctx)); 399 } else 400 MLX4_ASSERT(priv->ctx == NULL); 401 mlx4_intr_uninstall(priv); 402 memset(priv, 0, sizeof(*priv)); 403 } 404 405 static const struct eth_dev_ops mlx4_dev_ops = { 406 .dev_configure = mlx4_dev_configure, 407 .dev_start = mlx4_dev_start, 408 .dev_stop = mlx4_dev_stop, 409 .dev_set_link_down = mlx4_dev_set_link_down, 410 .dev_set_link_up = mlx4_dev_set_link_up, 411 .dev_close = mlx4_dev_close, 412 .link_update = mlx4_link_update, 413 .promiscuous_enable = mlx4_promiscuous_enable, 414 .promiscuous_disable = mlx4_promiscuous_disable, 415 .allmulticast_enable = mlx4_allmulticast_enable, 416 .allmulticast_disable = mlx4_allmulticast_disable, 417 .mac_addr_remove = mlx4_mac_addr_remove, 418 .mac_addr_add = mlx4_mac_addr_add, 419 .mac_addr_set = mlx4_mac_addr_set, 420 .set_mc_addr_list = mlx4_set_mc_addr_list, 421 .stats_get = mlx4_stats_get, 422 .stats_reset = mlx4_stats_reset, 423 .fw_version_get = mlx4_fw_version_get, 424 .dev_infos_get = mlx4_dev_infos_get, 425 .dev_supported_ptypes_get = mlx4_dev_supported_ptypes_get, 426 .vlan_filter_set = mlx4_vlan_filter_set, 427 .rx_queue_setup = mlx4_rx_queue_setup, 428 .tx_queue_setup = mlx4_tx_queue_setup, 429 .rx_queue_release = mlx4_rx_queue_release, 430 .tx_queue_release = mlx4_tx_queue_release, 431 .flow_ctrl_get = mlx4_flow_ctrl_get, 432 .flow_ctrl_set = mlx4_flow_ctrl_set, 433 .mtu_set = mlx4_mtu_set, 434 .filter_ctrl = mlx4_filter_ctrl, 435 .rx_queue_intr_enable = mlx4_rx_intr_enable, 436 .rx_queue_intr_disable = mlx4_rx_intr_disable, 437 .is_removed = mlx4_is_removed, 438 }; 439 440 /* Available operations from secondary process. */ 441 static const struct eth_dev_ops mlx4_dev_sec_ops = { 442 .stats_get = mlx4_stats_get, 443 .stats_reset = mlx4_stats_reset, 444 .fw_version_get = mlx4_fw_version_get, 445 .dev_infos_get = mlx4_dev_infos_get, 446 }; 447 448 /** 449 * Get PCI information from struct ibv_device. 450 * 451 * @param device 452 * Pointer to Ethernet device structure. 453 * @param[out] pci_addr 454 * PCI bus address output buffer. 455 * 456 * @return 457 * 0 on success, negative errno value otherwise and rte_errno is set. 458 */ 459 static int 460 mlx4_ibv_device_to_pci_addr(const struct ibv_device *device, 461 struct rte_pci_addr *pci_addr) 462 { 463 FILE *file; 464 char line[32]; 465 MKSTR(path, "%s/device/uevent", device->ibdev_path); 466 467 file = fopen(path, "rb"); 468 if (file == NULL) { 469 rte_errno = errno; 470 return -rte_errno; 471 } 472 while (fgets(line, sizeof(line), file) == line) { 473 size_t len = strlen(line); 474 int ret; 475 476 /* Truncate long lines. */ 477 if (len == (sizeof(line) - 1)) 478 while (line[(len - 1)] != '\n') { 479 ret = fgetc(file); 480 if (ret == EOF) 481 break; 482 line[(len - 1)] = ret; 483 } 484 /* Extract information. */ 485 if (sscanf(line, 486 "PCI_SLOT_NAME=" 487 "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n", 488 &pci_addr->domain, 489 &pci_addr->bus, 490 &pci_addr->devid, 491 &pci_addr->function) == 4) { 492 break; 493 } 494 } 495 fclose(file); 496 return 0; 497 } 498 499 /** 500 * Verify and store value for device argument. 501 * 502 * @param[in] key 503 * Key argument to verify. 504 * @param[in] val 505 * Value associated with key. 506 * @param[in, out] conf 507 * Shared configuration data. 508 * 509 * @return 510 * 0 on success, negative errno value otherwise and rte_errno is set. 511 */ 512 static int 513 mlx4_arg_parse(const char *key, const char *val, struct mlx4_conf *conf) 514 { 515 unsigned long tmp; 516 517 errno = 0; 518 tmp = strtoul(val, NULL, 0); 519 if (errno) { 520 rte_errno = errno; 521 WARN("%s: \"%s\" is not a valid integer", key, val); 522 return -rte_errno; 523 } 524 if (strcmp(MLX4_PMD_PORT_KVARG, key) == 0) { 525 uint32_t ports = rte_log2_u32(conf->ports.present + 1); 526 527 if (tmp >= ports) { 528 ERROR("port index %lu outside range [0,%" PRIu32 ")", 529 tmp, ports); 530 return -EINVAL; 531 } 532 if (!(conf->ports.present & (1 << tmp))) { 533 rte_errno = EINVAL; 534 ERROR("invalid port index %lu", tmp); 535 return -rte_errno; 536 } 537 conf->ports.enabled |= 1 << tmp; 538 } else if (strcmp(MLX4_MR_EXT_MEMSEG_EN_KVARG, key) == 0) { 539 conf->mr_ext_memseg_en = !!tmp; 540 } else { 541 rte_errno = EINVAL; 542 WARN("%s: unknown parameter", key); 543 return -rte_errno; 544 } 545 return 0; 546 } 547 548 /** 549 * Parse device parameters. 550 * 551 * @param devargs 552 * Device arguments structure. 553 * 554 * @return 555 * 0 on success, negative errno value otherwise and rte_errno is set. 556 */ 557 static int 558 mlx4_args(struct rte_devargs *devargs, struct mlx4_conf *conf) 559 { 560 struct rte_kvargs *kvlist; 561 unsigned int arg_count; 562 int ret = 0; 563 int i; 564 565 if (devargs == NULL) 566 return 0; 567 kvlist = rte_kvargs_parse(devargs->args, pmd_mlx4_init_params); 568 if (kvlist == NULL) { 569 rte_errno = EINVAL; 570 ERROR("failed to parse kvargs"); 571 return -rte_errno; 572 } 573 /* Process parameters. */ 574 for (i = 0; pmd_mlx4_init_params[i]; ++i) { 575 arg_count = rte_kvargs_count(kvlist, pmd_mlx4_init_params[i]); 576 while (arg_count-- > 0) { 577 ret = rte_kvargs_process(kvlist, 578 pmd_mlx4_init_params[i], 579 (int (*)(const char *, 580 const char *, 581 void *)) 582 mlx4_arg_parse, 583 conf); 584 if (ret != 0) 585 goto free_kvlist; 586 } 587 } 588 free_kvlist: 589 rte_kvargs_free(kvlist); 590 return ret; 591 } 592 593 /** 594 * Interpret RSS capabilities reported by device. 595 * 596 * This function returns the set of usable Verbs RSS hash fields, kernel 597 * quirks taken into account. 598 * 599 * @param ctx 600 * Verbs context. 601 * @param pd 602 * Verbs protection domain. 603 * @param device_attr_ex 604 * Extended device attributes to interpret. 605 * 606 * @return 607 * Usable RSS hash fields mask in Verbs format. 608 */ 609 static uint64_t 610 mlx4_hw_rss_sup(struct ibv_context *ctx, struct ibv_pd *pd, 611 struct ibv_device_attr_ex *device_attr_ex) 612 { 613 uint64_t hw_rss_sup = device_attr_ex->rss_caps.rx_hash_fields_mask; 614 struct ibv_cq *cq = NULL; 615 struct ibv_wq *wq = NULL; 616 struct ibv_rwq_ind_table *ind = NULL; 617 struct ibv_qp *qp = NULL; 618 619 if (!hw_rss_sup) { 620 WARN("no RSS capabilities reported; disabling support for UDP" 621 " RSS and inner VXLAN RSS"); 622 return IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4 | 623 IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6 | 624 IBV_RX_HASH_SRC_PORT_TCP | IBV_RX_HASH_DST_PORT_TCP; 625 } 626 if (!(hw_rss_sup & IBV_RX_HASH_INNER)) 627 return hw_rss_sup; 628 /* 629 * Although reported as supported, missing code in some Linux 630 * versions (v4.15, v4.16) prevents the creation of hash QPs with 631 * inner capability. 632 * 633 * There is no choice but to attempt to instantiate a temporary RSS 634 * context in order to confirm its support. 635 */ 636 cq = mlx4_glue->create_cq(ctx, 1, NULL, NULL, 0); 637 wq = cq ? mlx4_glue->create_wq 638 (ctx, 639 &(struct ibv_wq_init_attr){ 640 .wq_type = IBV_WQT_RQ, 641 .max_wr = 1, 642 .max_sge = 1, 643 .pd = pd, 644 .cq = cq, 645 }) : NULL; 646 ind = wq ? mlx4_glue->create_rwq_ind_table 647 (ctx, 648 &(struct ibv_rwq_ind_table_init_attr){ 649 .log_ind_tbl_size = 0, 650 .ind_tbl = &wq, 651 .comp_mask = 0, 652 }) : NULL; 653 qp = ind ? mlx4_glue->create_qp_ex 654 (ctx, 655 &(struct ibv_qp_init_attr_ex){ 656 .comp_mask = 657 (IBV_QP_INIT_ATTR_PD | 658 IBV_QP_INIT_ATTR_RX_HASH | 659 IBV_QP_INIT_ATTR_IND_TABLE), 660 .qp_type = IBV_QPT_RAW_PACKET, 661 .pd = pd, 662 .rwq_ind_tbl = ind, 663 .rx_hash_conf = { 664 .rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ, 665 .rx_hash_key_len = MLX4_RSS_HASH_KEY_SIZE, 666 .rx_hash_key = mlx4_rss_hash_key_default, 667 .rx_hash_fields_mask = hw_rss_sup, 668 }, 669 }) : NULL; 670 if (!qp) { 671 WARN("disabling unusable inner RSS capability due to kernel" 672 " quirk"); 673 hw_rss_sup &= ~IBV_RX_HASH_INNER; 674 } else { 675 claim_zero(mlx4_glue->destroy_qp(qp)); 676 } 677 if (ind) 678 claim_zero(mlx4_glue->destroy_rwq_ind_table(ind)); 679 if (wq) 680 claim_zero(mlx4_glue->destroy_wq(wq)); 681 if (cq) 682 claim_zero(mlx4_glue->destroy_cq(cq)); 683 return hw_rss_sup; 684 } 685 686 static struct rte_pci_driver mlx4_driver; 687 688 /** 689 * PMD global initialization. 690 * 691 * Independent from individual device, this function initializes global 692 * per-PMD data structures distinguishing primary and secondary processes. 693 * Hence, each initialization is called once per a process. 694 * 695 * @return 696 * 0 on success, a negative errno value otherwise and rte_errno is set. 697 */ 698 static int 699 mlx4_init_once(void) 700 { 701 struct mlx4_shared_data *sd; 702 struct mlx4_local_data *ld = &mlx4_local_data; 703 int ret = 0; 704 705 if (mlx4_init_shared_data()) 706 return -rte_errno; 707 sd = mlx4_shared_data; 708 MLX4_ASSERT(sd); 709 rte_spinlock_lock(&sd->lock); 710 switch (rte_eal_process_type()) { 711 case RTE_PROC_PRIMARY: 712 if (sd->init_done) 713 break; 714 LIST_INIT(&sd->mem_event_cb_list); 715 rte_rwlock_init(&sd->mem_event_rwlock); 716 rte_mem_event_callback_register("MLX4_MEM_EVENT_CB", 717 mlx4_mr_mem_event_cb, NULL); 718 ret = mlx4_mp_init_primary(); 719 if (ret) 720 goto out; 721 sd->init_done = 1; 722 break; 723 case RTE_PROC_SECONDARY: 724 if (ld->init_done) 725 break; 726 ret = mlx4_mp_init_secondary(); 727 if (ret) 728 goto out; 729 ++sd->secondary_cnt; 730 ld->init_done = 1; 731 break; 732 default: 733 break; 734 } 735 out: 736 rte_spinlock_unlock(&sd->lock); 737 return ret; 738 } 739 740 /** 741 * DPDK callback to register a PCI device. 742 * 743 * This function creates an Ethernet device for each port of a given 744 * PCI device. 745 * 746 * @param[in] pci_drv 747 * PCI driver structure (mlx4_driver). 748 * @param[in] pci_dev 749 * PCI device information. 750 * 751 * @return 752 * 0 on success, negative errno value otherwise and rte_errno is set. 753 */ 754 static int 755 mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) 756 { 757 struct ibv_device **list; 758 struct ibv_device *ibv_dev; 759 int err = 0; 760 struct ibv_context *attr_ctx = NULL; 761 struct ibv_device_attr device_attr; 762 struct ibv_device_attr_ex device_attr_ex; 763 struct mlx4_conf conf = { 764 .ports.present = 0, 765 .mr_ext_memseg_en = 1, 766 }; 767 unsigned int vf; 768 int i; 769 char ifname[IF_NAMESIZE]; 770 771 (void)pci_drv; 772 err = mlx4_init_once(); 773 if (err) { 774 ERROR("unable to init PMD global data: %s", 775 strerror(rte_errno)); 776 return -rte_errno; 777 } 778 MLX4_ASSERT(pci_drv == &mlx4_driver); 779 list = mlx4_glue->get_device_list(&i); 780 if (list == NULL) { 781 rte_errno = errno; 782 MLX4_ASSERT(rte_errno); 783 if (rte_errno == ENOSYS) 784 ERROR("cannot list devices, is ib_uverbs loaded?"); 785 return -rte_errno; 786 } 787 MLX4_ASSERT(i >= 0); 788 /* 789 * For each listed device, check related sysfs entry against 790 * the provided PCI ID. 791 */ 792 while (i != 0) { 793 struct rte_pci_addr pci_addr; 794 795 --i; 796 DEBUG("checking device \"%s\"", list[i]->name); 797 if (mlx4_ibv_device_to_pci_addr(list[i], &pci_addr)) 798 continue; 799 if ((pci_dev->addr.domain != pci_addr.domain) || 800 (pci_dev->addr.bus != pci_addr.bus) || 801 (pci_dev->addr.devid != pci_addr.devid) || 802 (pci_dev->addr.function != pci_addr.function)) 803 continue; 804 vf = (pci_dev->id.device_id == 805 PCI_DEVICE_ID_MELLANOX_CONNECTX3VF); 806 INFO("PCI information matches, using device \"%s\" (VF: %s)", 807 list[i]->name, (vf ? "true" : "false")); 808 attr_ctx = mlx4_glue->open_device(list[i]); 809 err = errno; 810 break; 811 } 812 if (attr_ctx == NULL) { 813 mlx4_glue->free_device_list(list); 814 switch (err) { 815 case 0: 816 rte_errno = ENODEV; 817 ERROR("cannot access device, is mlx4_ib loaded?"); 818 return -rte_errno; 819 case EINVAL: 820 rte_errno = EINVAL; 821 ERROR("cannot use device, are drivers up to date?"); 822 return -rte_errno; 823 } 824 MLX4_ASSERT(err > 0); 825 rte_errno = err; 826 return -rte_errno; 827 } 828 ibv_dev = list[i]; 829 DEBUG("device opened"); 830 if (mlx4_glue->query_device(attr_ctx, &device_attr)) { 831 err = ENODEV; 832 goto error; 833 } 834 INFO("%u port(s) detected", device_attr.phys_port_cnt); 835 conf.ports.present |= (UINT64_C(1) << device_attr.phys_port_cnt) - 1; 836 if (mlx4_args(pci_dev->device.devargs, &conf)) { 837 ERROR("failed to process device arguments"); 838 err = EINVAL; 839 goto error; 840 } 841 /* Use all ports when none are defined */ 842 if (!conf.ports.enabled) 843 conf.ports.enabled = conf.ports.present; 844 /* Retrieve extended device attributes. */ 845 if (mlx4_glue->query_device_ex(attr_ctx, NULL, &device_attr_ex)) { 846 err = ENODEV; 847 goto error; 848 } 849 MLX4_ASSERT(device_attr.max_sge >= MLX4_MAX_SGE); 850 for (i = 0; i < device_attr.phys_port_cnt; i++) { 851 uint32_t port = i + 1; /* ports are indexed from one */ 852 struct ibv_context *ctx = NULL; 853 struct ibv_port_attr port_attr; 854 struct ibv_pd *pd = NULL; 855 struct mlx4_priv *priv = NULL; 856 struct rte_eth_dev *eth_dev = NULL; 857 struct rte_ether_addr mac; 858 char name[RTE_ETH_NAME_MAX_LEN]; 859 860 /* If port is not enabled, skip. */ 861 if (!(conf.ports.enabled & (1 << i))) 862 continue; 863 DEBUG("using port %u", port); 864 ctx = mlx4_glue->open_device(ibv_dev); 865 if (ctx == NULL) { 866 err = ENODEV; 867 goto port_error; 868 } 869 snprintf(name, sizeof(name), "%s port %u", 870 mlx4_glue->get_device_name(ibv_dev), port); 871 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 872 eth_dev = rte_eth_dev_attach_secondary(name); 873 if (eth_dev == NULL) { 874 ERROR("can not attach rte ethdev"); 875 rte_errno = ENOMEM; 876 err = rte_errno; 877 goto error; 878 } 879 priv = eth_dev->data->dev_private; 880 if (!priv->verbs_alloc_ctx.enabled) { 881 ERROR("secondary process is not supported" 882 " due to lack of external allocator" 883 " from Verbs"); 884 rte_errno = ENOTSUP; 885 err = rte_errno; 886 goto error; 887 } 888 eth_dev->device = &pci_dev->device; 889 eth_dev->dev_ops = &mlx4_dev_sec_ops; 890 err = mlx4_proc_priv_init(eth_dev); 891 if (err) 892 goto error; 893 /* Receive command fd from primary process. */ 894 err = mlx4_mp_req_verbs_cmd_fd(eth_dev); 895 if (err < 0) { 896 err = rte_errno; 897 goto error; 898 } 899 /* Remap UAR for Tx queues. */ 900 err = mlx4_tx_uar_init_secondary(eth_dev, err); 901 if (err) { 902 err = rte_errno; 903 goto error; 904 } 905 /* 906 * Ethdev pointer is still required as input since 907 * the primary device is not accessible from the 908 * secondary process. 909 */ 910 eth_dev->tx_pkt_burst = mlx4_tx_burst; 911 eth_dev->rx_pkt_burst = mlx4_rx_burst; 912 claim_zero(mlx4_glue->close_device(ctx)); 913 rte_eth_copy_pci_info(eth_dev, pci_dev); 914 rte_eth_dev_probing_finish(eth_dev); 915 continue; 916 } 917 /* Check port status. */ 918 err = mlx4_glue->query_port(ctx, port, &port_attr); 919 if (err) { 920 err = ENODEV; 921 ERROR("port query failed: %s", strerror(err)); 922 goto port_error; 923 } 924 if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) { 925 err = ENOTSUP; 926 ERROR("port %d is not configured in Ethernet mode", 927 port); 928 goto port_error; 929 } 930 if (port_attr.state != IBV_PORT_ACTIVE) 931 DEBUG("port %d is not active: \"%s\" (%d)", 932 port, mlx4_glue->port_state_str(port_attr.state), 933 port_attr.state); 934 /* Make asynchronous FD non-blocking to handle interrupts. */ 935 err = mlx4_fd_set_non_blocking(ctx->async_fd); 936 if (err) { 937 ERROR("cannot make asynchronous FD non-blocking: %s", 938 strerror(err)); 939 goto port_error; 940 } 941 /* Allocate protection domain. */ 942 pd = mlx4_glue->alloc_pd(ctx); 943 if (pd == NULL) { 944 err = ENOMEM; 945 ERROR("PD allocation failure"); 946 goto port_error; 947 } 948 /* from rte_ethdev.c */ 949 priv = rte_zmalloc("ethdev private structure", 950 sizeof(*priv), 951 RTE_CACHE_LINE_SIZE); 952 if (priv == NULL) { 953 err = ENOMEM; 954 ERROR("priv allocation failure"); 955 goto port_error; 956 } 957 priv->ctx = ctx; 958 priv->device_attr = device_attr; 959 priv->port = port; 960 priv->pd = pd; 961 priv->mtu = RTE_ETHER_MTU; 962 priv->vf = vf; 963 priv->hw_csum = !!(device_attr.device_cap_flags & 964 IBV_DEVICE_RAW_IP_CSUM); 965 DEBUG("checksum offloading is %ssupported", 966 (priv->hw_csum ? "" : "not ")); 967 /* Only ConnectX-3 Pro supports tunneling. */ 968 priv->hw_csum_l2tun = 969 priv->hw_csum && 970 (device_attr.vendor_part_id == 971 PCI_DEVICE_ID_MELLANOX_CONNECTX3PRO); 972 DEBUG("L2 tunnel checksum offloads are %ssupported", 973 priv->hw_csum_l2tun ? "" : "not "); 974 priv->hw_rss_sup = mlx4_hw_rss_sup(priv->ctx, priv->pd, 975 &device_attr_ex); 976 DEBUG("supported RSS hash fields mask: %016" PRIx64, 977 priv->hw_rss_sup); 978 priv->hw_rss_max_qps = 979 device_attr_ex.rss_caps.max_rwq_indirection_table_size; 980 DEBUG("MAX RSS queues %d", priv->hw_rss_max_qps); 981 priv->hw_fcs_strip = !!(device_attr_ex.raw_packet_caps & 982 IBV_RAW_PACKET_CAP_SCATTER_FCS); 983 DEBUG("FCS stripping toggling is %ssupported", 984 priv->hw_fcs_strip ? "" : "not "); 985 priv->tso = 986 ((device_attr_ex.tso_caps.max_tso > 0) && 987 (device_attr_ex.tso_caps.supported_qpts & 988 (1 << IBV_QPT_RAW_PACKET))); 989 if (priv->tso) 990 priv->tso_max_payload_sz = 991 device_attr_ex.tso_caps.max_tso; 992 DEBUG("TSO is %ssupported", 993 priv->tso ? "" : "not "); 994 priv->mr_ext_memseg_en = conf.mr_ext_memseg_en; 995 /* Configure the first MAC address by default. */ 996 err = mlx4_get_mac(priv, &mac.addr_bytes); 997 if (err) { 998 ERROR("cannot get MAC address, is mlx4_en loaded?" 999 " (error: %s)", strerror(err)); 1000 goto port_error; 1001 } 1002 INFO("port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x", 1003 priv->port, 1004 mac.addr_bytes[0], mac.addr_bytes[1], 1005 mac.addr_bytes[2], mac.addr_bytes[3], 1006 mac.addr_bytes[4], mac.addr_bytes[5]); 1007 /* Register MAC address. */ 1008 priv->mac[0] = mac; 1009 1010 if (mlx4_get_ifname(priv, &ifname) == 0) { 1011 DEBUG("port %u ifname is \"%s\"", 1012 priv->port, ifname); 1013 priv->if_index = if_nametoindex(ifname); 1014 } else { 1015 DEBUG("port %u ifname is unknown", priv->port); 1016 } 1017 1018 /* Get actual MTU if possible. */ 1019 mlx4_mtu_get(priv, &priv->mtu); 1020 DEBUG("port %u MTU is %u", priv->port, priv->mtu); 1021 eth_dev = rte_eth_dev_allocate(name); 1022 if (eth_dev == NULL) { 1023 err = ENOMEM; 1024 ERROR("can not allocate rte ethdev"); 1025 goto port_error; 1026 } 1027 eth_dev->data->dev_private = priv; 1028 eth_dev->data->mac_addrs = priv->mac; 1029 eth_dev->device = &pci_dev->device; 1030 rte_eth_copy_pci_info(eth_dev, pci_dev); 1031 /* Initialize local interrupt handle for current port. */ 1032 memset(&priv->intr_handle, 0, sizeof(struct rte_intr_handle)); 1033 priv->intr_handle.fd = -1; 1034 priv->intr_handle.type = RTE_INTR_HANDLE_EXT; 1035 /* 1036 * Override ethdev interrupt handle pointer with private 1037 * handle instead of that of the parent PCI device used by 1038 * default. This prevents it from being shared between all 1039 * ports of the same PCI device since each of them is 1040 * associated its own Verbs context. 1041 * 1042 * Rx interrupts in particular require this as the PMD has 1043 * no control over the registration of queue interrupts 1044 * besides setting up eth_dev->intr_handle, the rest is 1045 * handled by rte_intr_rx_ctl(). 1046 */ 1047 eth_dev->intr_handle = &priv->intr_handle; 1048 priv->dev_data = eth_dev->data; 1049 eth_dev->dev_ops = &mlx4_dev_ops; 1050 #ifdef HAVE_IBV_MLX4_BUF_ALLOCATORS 1051 /* Hint libmlx4 to use PMD allocator for data plane resources */ 1052 err = mlx4_glue->dv_set_context_attr 1053 (ctx, MLX4DV_SET_CTX_ATTR_BUF_ALLOCATORS, 1054 (void *)((uintptr_t)&(struct mlx4dv_ctx_allocators){ 1055 .alloc = &mlx4_alloc_verbs_buf, 1056 .free = &mlx4_free_verbs_buf, 1057 .data = priv, 1058 })); 1059 if (err) 1060 WARN("Verbs external allocator is not supported"); 1061 else 1062 priv->verbs_alloc_ctx.enabled = 1; 1063 #endif 1064 /* Bring Ethernet device up. */ 1065 DEBUG("forcing Ethernet interface up"); 1066 mlx4_dev_set_link_up(eth_dev); 1067 /* Update link status once if waiting for LSC. */ 1068 if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) 1069 mlx4_link_update(eth_dev, 0); 1070 /* 1071 * Once the device is added to the list of memory event 1072 * callback, its global MR cache table cannot be expanded 1073 * on the fly because of deadlock. If it overflows, lookup 1074 * should be done by searching MR list linearly, which is slow. 1075 */ 1076 err = mlx4_mr_btree_init(&priv->mr.cache, 1077 MLX4_MR_BTREE_CACHE_N * 2, 1078 eth_dev->device->numa_node); 1079 if (err) { 1080 /* rte_errno is already set. */ 1081 goto port_error; 1082 } 1083 /* Add device to memory callback list. */ 1084 rte_rwlock_write_lock(&mlx4_shared_data->mem_event_rwlock); 1085 LIST_INSERT_HEAD(&mlx4_shared_data->mem_event_cb_list, 1086 priv, mem_event_cb); 1087 rte_rwlock_write_unlock(&mlx4_shared_data->mem_event_rwlock); 1088 rte_eth_dev_probing_finish(eth_dev); 1089 continue; 1090 port_error: 1091 rte_free(priv); 1092 if (eth_dev != NULL) 1093 eth_dev->data->dev_private = NULL; 1094 if (pd) 1095 claim_zero(mlx4_glue->dealloc_pd(pd)); 1096 if (ctx) 1097 claim_zero(mlx4_glue->close_device(ctx)); 1098 if (eth_dev != NULL) { 1099 /* mac_addrs must not be freed because part of dev_private */ 1100 eth_dev->data->mac_addrs = NULL; 1101 rte_eth_dev_release_port(eth_dev); 1102 } 1103 break; 1104 } 1105 /* 1106 * XXX if something went wrong in the loop above, there is a resource 1107 * leak (ctx, pd, priv, dpdk ethdev) but we can do nothing about it as 1108 * long as the dpdk does not provide a way to deallocate a ethdev and a 1109 * way to enumerate the registered ethdevs to free the previous ones. 1110 */ 1111 error: 1112 if (attr_ctx) 1113 claim_zero(mlx4_glue->close_device(attr_ctx)); 1114 if (list) 1115 mlx4_glue->free_device_list(list); 1116 if (err) 1117 rte_errno = err; 1118 return -err; 1119 } 1120 1121 static const struct rte_pci_id mlx4_pci_id_map[] = { 1122 { 1123 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 1124 PCI_DEVICE_ID_MELLANOX_CONNECTX3) 1125 }, 1126 { 1127 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 1128 PCI_DEVICE_ID_MELLANOX_CONNECTX3PRO) 1129 }, 1130 { 1131 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, 1132 PCI_DEVICE_ID_MELLANOX_CONNECTX3VF) 1133 }, 1134 { 1135 .vendor_id = 0 1136 } 1137 }; 1138 1139 static struct rte_pci_driver mlx4_driver = { 1140 .driver = { 1141 .name = MLX4_DRIVER_NAME 1142 }, 1143 .id_table = mlx4_pci_id_map, 1144 .probe = mlx4_pci_probe, 1145 .drv_flags = RTE_PCI_DRV_INTR_LSC | RTE_PCI_DRV_INTR_RMV, 1146 }; 1147 1148 #ifdef RTE_IBVERBS_LINK_DLOPEN 1149 1150 /** 1151 * Suffix RTE_EAL_PMD_PATH with "-glue". 1152 * 1153 * This function performs a sanity check on RTE_EAL_PMD_PATH before 1154 * suffixing its last component. 1155 * 1156 * @param buf[out] 1157 * Output buffer, should be large enough otherwise NULL is returned. 1158 * @param size 1159 * Size of @p out. 1160 * 1161 * @return 1162 * Pointer to @p buf or @p NULL in case suffix cannot be appended. 1163 */ 1164 static char * 1165 mlx4_glue_path(char *buf, size_t size) 1166 { 1167 static const char *const bad[] = { "/", ".", "..", NULL }; 1168 const char *path = RTE_EAL_PMD_PATH; 1169 size_t len = strlen(path); 1170 size_t off; 1171 int i; 1172 1173 while (len && path[len - 1] == '/') 1174 --len; 1175 for (off = len; off && path[off - 1] != '/'; --off) 1176 ; 1177 for (i = 0; bad[i]; ++i) 1178 if (!strncmp(path + off, bad[i], (int)(len - off))) 1179 goto error; 1180 i = snprintf(buf, size, "%.*s-glue", (int)len, path); 1181 if (i == -1 || (size_t)i >= size) 1182 goto error; 1183 return buf; 1184 error: 1185 ERROR("unable to append \"-glue\" to last component of" 1186 " RTE_EAL_PMD_PATH (\"" RTE_EAL_PMD_PATH "\")," 1187 " please re-configure DPDK"); 1188 return NULL; 1189 } 1190 1191 /** 1192 * Initialization routine for run-time dependency on rdma-core. 1193 */ 1194 static int 1195 mlx4_glue_init(void) 1196 { 1197 char glue_path[sizeof(RTE_EAL_PMD_PATH) - 1 + sizeof("-glue")]; 1198 const char *path[] = { 1199 /* 1200 * A basic security check is necessary before trusting 1201 * MLX4_GLUE_PATH, which may override RTE_EAL_PMD_PATH. 1202 */ 1203 (geteuid() == getuid() && getegid() == getgid() ? 1204 getenv("MLX4_GLUE_PATH") : NULL), 1205 /* 1206 * When RTE_EAL_PMD_PATH is set, use its glue-suffixed 1207 * variant, otherwise let dlopen() look up libraries on its 1208 * own. 1209 */ 1210 (*RTE_EAL_PMD_PATH ? 1211 mlx4_glue_path(glue_path, sizeof(glue_path)) : ""), 1212 }; 1213 unsigned int i = 0; 1214 void *handle = NULL; 1215 void **sym; 1216 const char *dlmsg; 1217 1218 while (!handle && i != RTE_DIM(path)) { 1219 const char *end; 1220 size_t len; 1221 int ret; 1222 1223 if (!path[i]) { 1224 ++i; 1225 continue; 1226 } 1227 end = strpbrk(path[i], ":;"); 1228 if (!end) 1229 end = path[i] + strlen(path[i]); 1230 len = end - path[i]; 1231 ret = 0; 1232 do { 1233 char name[ret + 1]; 1234 1235 ret = snprintf(name, sizeof(name), "%.*s%s" MLX4_GLUE, 1236 (int)len, path[i], 1237 (!len || *(end - 1) == '/') ? "" : "/"); 1238 if (ret == -1) 1239 break; 1240 if (sizeof(name) != (size_t)ret + 1) 1241 continue; 1242 DEBUG("looking for rdma-core glue as \"%s\"", name); 1243 handle = dlopen(name, RTLD_LAZY); 1244 break; 1245 } while (1); 1246 path[i] = end + 1; 1247 if (!*end) 1248 ++i; 1249 } 1250 if (!handle) { 1251 rte_errno = EINVAL; 1252 dlmsg = dlerror(); 1253 if (dlmsg) 1254 WARN("cannot load glue library: %s", dlmsg); 1255 goto glue_error; 1256 } 1257 sym = dlsym(handle, "mlx4_glue"); 1258 if (!sym || !*sym) { 1259 rte_errno = EINVAL; 1260 dlmsg = dlerror(); 1261 if (dlmsg) 1262 ERROR("cannot resolve glue symbol: %s", dlmsg); 1263 goto glue_error; 1264 } 1265 mlx4_glue = *sym; 1266 return 0; 1267 glue_error: 1268 if (handle) 1269 dlclose(handle); 1270 WARN("cannot initialize PMD due to missing run-time" 1271 " dependency on rdma-core libraries (libibverbs," 1272 " libmlx4)"); 1273 return -rte_errno; 1274 } 1275 1276 #endif 1277 1278 /* Initialize driver log type. */ 1279 RTE_LOG_REGISTER(mlx4_logtype, pmd.net.mlx4, NOTICE) 1280 1281 /** 1282 * Driver initialization routine. 1283 */ 1284 RTE_INIT(rte_mlx4_pmd_init) 1285 { 1286 /* 1287 * MLX4_DEVICE_FATAL_CLEANUP tells ibv_destroy functions we 1288 * want to get success errno value in case of calling them 1289 * when the device was removed. 1290 */ 1291 setenv("MLX4_DEVICE_FATAL_CLEANUP", "1", 1); 1292 /* 1293 * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use 1294 * huge pages. Calling ibv_fork_init() during init allows 1295 * applications to use fork() safely for purposes other than 1296 * using this PMD, which is not supported in forked processes. 1297 */ 1298 setenv("RDMAV_HUGEPAGES_SAFE", "1", 1); 1299 #ifdef RTE_IBVERBS_LINK_DLOPEN 1300 if (mlx4_glue_init()) 1301 return; 1302 MLX4_ASSERT(mlx4_glue); 1303 #endif 1304 #ifdef RTE_LIBRTE_MLX4_DEBUG 1305 /* Glue structure must not contain any NULL pointers. */ 1306 { 1307 unsigned int i; 1308 1309 for (i = 0; i != sizeof(*mlx4_glue) / sizeof(void *); ++i) 1310 MLX4_ASSERT(((const void *const *)mlx4_glue)[i]); 1311 } 1312 #endif 1313 if (strcmp(mlx4_glue->version, MLX4_GLUE_VERSION)) { 1314 ERROR("rdma-core glue \"%s\" mismatch: \"%s\" is required", 1315 mlx4_glue->version, MLX4_GLUE_VERSION); 1316 return; 1317 } 1318 mlx4_glue->fork_init(); 1319 rte_pci_register(&mlx4_driver); 1320 } 1321 1322 RTE_PMD_EXPORT_NAME(net_mlx4, __COUNTER__); 1323 RTE_PMD_REGISTER_PCI_TABLE(net_mlx4, mlx4_pci_id_map); 1324 RTE_PMD_REGISTER_KMOD_DEP(net_mlx4, 1325 "* ib_uverbs & mlx4_en & mlx4_core & mlx4_ib"); 1326