1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2020 Xilinx, Inc. 4 * Copyright(c) 2016-2019 Solarflare Communications Inc. 5 * 6 * This software was jointly developed between OKTET Labs (under contract 7 * for Solarflare) and Solarflare Communications, Inc. 8 */ 9 10 /* sysconf() */ 11 #include <unistd.h> 12 13 #include <rte_errno.h> 14 #include <rte_alarm.h> 15 16 #include "efx.h" 17 18 #include "sfc.h" 19 #include "sfc_debug.h" 20 #include "sfc_log.h" 21 #include "sfc_ev.h" 22 #include "sfc_rx.h" 23 #include "sfc_tx.h" 24 #include "sfc_kvargs.h" 25 #include "sfc_tweak.h" 26 27 28 int 29 sfc_dma_alloc(const struct sfc_adapter *sa, const char *name, uint16_t id, 30 size_t len, int socket_id, efsys_mem_t *esmp) 31 { 32 const struct rte_memzone *mz; 33 34 sfc_log_init(sa, "name=%s id=%u len=%zu socket_id=%d", 35 name, id, len, socket_id); 36 37 mz = rte_eth_dma_zone_reserve(sa->eth_dev, name, id, len, 38 sysconf(_SC_PAGESIZE), socket_id); 39 if (mz == NULL) { 40 sfc_err(sa, "cannot reserve DMA zone for %s:%u %#x@%d: %s", 41 name, (unsigned int)id, (unsigned int)len, socket_id, 42 rte_strerror(rte_errno)); 43 return ENOMEM; 44 } 45 46 esmp->esm_addr = mz->iova; 47 if (esmp->esm_addr == RTE_BAD_IOVA) { 48 (void)rte_memzone_free(mz); 49 return EFAULT; 50 } 51 52 esmp->esm_mz = mz; 53 esmp->esm_base = mz->addr; 54 55 return 0; 56 } 57 58 void 59 sfc_dma_free(const struct sfc_adapter *sa, efsys_mem_t *esmp) 60 { 61 int rc; 62 63 sfc_log_init(sa, "name=%s", esmp->esm_mz->name); 64 65 rc = rte_memzone_free(esmp->esm_mz); 66 if (rc != 0) 67 sfc_err(sa, "rte_memzone_free(() failed: %d", rc); 68 69 memset(esmp, 0, sizeof(*esmp)); 70 } 71 72 static uint32_t 73 sfc_phy_cap_from_link_speeds(uint32_t speeds) 74 { 75 uint32_t phy_caps = 0; 76 77 if (~speeds & ETH_LINK_SPEED_FIXED) { 78 phy_caps |= (1 << EFX_PHY_CAP_AN); 79 /* 80 * If no speeds are specified in the mask, any supported 81 * may be negotiated 82 */ 83 if (speeds == ETH_LINK_SPEED_AUTONEG) 84 phy_caps |= 85 (1 << EFX_PHY_CAP_1000FDX) | 86 (1 << EFX_PHY_CAP_10000FDX) | 87 (1 << EFX_PHY_CAP_25000FDX) | 88 (1 << EFX_PHY_CAP_40000FDX) | 89 (1 << EFX_PHY_CAP_50000FDX) | 90 (1 << EFX_PHY_CAP_100000FDX); 91 } 92 if (speeds & ETH_LINK_SPEED_1G) 93 phy_caps |= (1 << EFX_PHY_CAP_1000FDX); 94 if (speeds & ETH_LINK_SPEED_10G) 95 phy_caps |= (1 << EFX_PHY_CAP_10000FDX); 96 if (speeds & ETH_LINK_SPEED_25G) 97 phy_caps |= (1 << EFX_PHY_CAP_25000FDX); 98 if (speeds & ETH_LINK_SPEED_40G) 99 phy_caps |= (1 << EFX_PHY_CAP_40000FDX); 100 if (speeds & ETH_LINK_SPEED_50G) 101 phy_caps |= (1 << EFX_PHY_CAP_50000FDX); 102 if (speeds & ETH_LINK_SPEED_100G) 103 phy_caps |= (1 << EFX_PHY_CAP_100000FDX); 104 105 return phy_caps; 106 } 107 108 /* 109 * Check requested device level configuration. 110 * Receive and transmit configuration is checked in corresponding 111 * modules. 112 */ 113 static int 114 sfc_check_conf(struct sfc_adapter *sa) 115 { 116 const struct rte_eth_conf *conf = &sa->eth_dev->data->dev_conf; 117 int rc = 0; 118 119 sa->port.phy_adv_cap = 120 sfc_phy_cap_from_link_speeds(conf->link_speeds) & 121 sa->port.phy_adv_cap_mask; 122 if ((sa->port.phy_adv_cap & ~(1 << EFX_PHY_CAP_AN)) == 0) { 123 sfc_err(sa, "No link speeds from mask %#x are supported", 124 conf->link_speeds); 125 rc = EINVAL; 126 } 127 128 #if !EFSYS_OPT_LOOPBACK 129 if (conf->lpbk_mode != 0) { 130 sfc_err(sa, "Loopback not supported"); 131 rc = EINVAL; 132 } 133 #endif 134 135 if (conf->dcb_capability_en != 0) { 136 sfc_err(sa, "Priority-based flow control not supported"); 137 rc = EINVAL; 138 } 139 140 if (conf->fdir_conf.mode != RTE_FDIR_MODE_NONE) { 141 sfc_err(sa, "Flow Director not supported"); 142 rc = EINVAL; 143 } 144 145 if ((conf->intr_conf.lsc != 0) && 146 (sa->intr.type != EFX_INTR_LINE) && 147 (sa->intr.type != EFX_INTR_MESSAGE)) { 148 sfc_err(sa, "Link status change interrupt not supported"); 149 rc = EINVAL; 150 } 151 152 if (conf->intr_conf.rxq != 0 && 153 (sa->priv.dp_rx->features & SFC_DP_RX_FEAT_INTR) == 0) { 154 sfc_err(sa, "Receive queue interrupt not supported"); 155 rc = EINVAL; 156 } 157 158 return rc; 159 } 160 161 /* 162 * Find out maximum number of receive and transmit queues which could be 163 * advertised. 164 * 165 * NIC is kept initialized on success to allow other modules acquire 166 * defaults and capabilities. 167 */ 168 static int 169 sfc_estimate_resource_limits(struct sfc_adapter *sa) 170 { 171 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); 172 efx_drv_limits_t limits; 173 int rc; 174 uint32_t evq_allocated; 175 uint32_t rxq_allocated; 176 uint32_t txq_allocated; 177 178 memset(&limits, 0, sizeof(limits)); 179 180 /* Request at least one Rx and Tx queue */ 181 limits.edl_min_rxq_count = 1; 182 limits.edl_min_txq_count = 1; 183 /* Management event queue plus event queue for each Tx and Rx queue */ 184 limits.edl_min_evq_count = 185 1 + limits.edl_min_rxq_count + limits.edl_min_txq_count; 186 187 /* Divide by number of functions to guarantee that all functions 188 * will get promised resources 189 */ 190 /* FIXME Divide by number of functions (not 2) below */ 191 limits.edl_max_evq_count = encp->enc_evq_limit / 2; 192 SFC_ASSERT(limits.edl_max_evq_count >= limits.edl_min_rxq_count); 193 194 /* Split equally between receive and transmit */ 195 limits.edl_max_rxq_count = 196 MIN(encp->enc_rxq_limit, (limits.edl_max_evq_count - 1) / 2); 197 SFC_ASSERT(limits.edl_max_rxq_count >= limits.edl_min_rxq_count); 198 199 limits.edl_max_txq_count = 200 MIN(encp->enc_txq_limit, 201 limits.edl_max_evq_count - 1 - limits.edl_max_rxq_count); 202 203 if (sa->tso) 204 limits.edl_max_txq_count = 205 MIN(limits.edl_max_txq_count, 206 encp->enc_fw_assisted_tso_v2_n_contexts / 207 encp->enc_hw_pf_count); 208 209 SFC_ASSERT(limits.edl_max_txq_count >= limits.edl_min_rxq_count); 210 211 /* Configure the minimum required resources needed for the 212 * driver to operate, and the maximum desired resources that the 213 * driver is capable of using. 214 */ 215 efx_nic_set_drv_limits(sa->nic, &limits); 216 217 sfc_log_init(sa, "init nic"); 218 rc = efx_nic_init(sa->nic); 219 if (rc != 0) 220 goto fail_nic_init; 221 222 /* Find resource dimensions assigned by firmware to this function */ 223 rc = efx_nic_get_vi_pool(sa->nic, &evq_allocated, &rxq_allocated, 224 &txq_allocated); 225 if (rc != 0) 226 goto fail_get_vi_pool; 227 228 /* It still may allocate more than maximum, ensure limit */ 229 evq_allocated = MIN(evq_allocated, limits.edl_max_evq_count); 230 rxq_allocated = MIN(rxq_allocated, limits.edl_max_rxq_count); 231 txq_allocated = MIN(txq_allocated, limits.edl_max_txq_count); 232 233 /* Subtract management EVQ not used for traffic */ 234 SFC_ASSERT(evq_allocated > 0); 235 evq_allocated--; 236 237 /* Right now we use separate EVQ for Rx and Tx */ 238 sa->rxq_max = MIN(rxq_allocated, evq_allocated / 2); 239 sa->txq_max = MIN(txq_allocated, evq_allocated - sa->rxq_max); 240 241 /* Keep NIC initialized */ 242 return 0; 243 244 fail_get_vi_pool: 245 efx_nic_fini(sa->nic); 246 fail_nic_init: 247 return rc; 248 } 249 250 static int 251 sfc_set_drv_limits(struct sfc_adapter *sa) 252 { 253 const struct rte_eth_dev_data *data = sa->eth_dev->data; 254 efx_drv_limits_t lim; 255 256 memset(&lim, 0, sizeof(lim)); 257 258 /* Limits are strict since take into account initial estimation */ 259 lim.edl_min_evq_count = lim.edl_max_evq_count = 260 1 + data->nb_rx_queues + data->nb_tx_queues; 261 lim.edl_min_rxq_count = lim.edl_max_rxq_count = data->nb_rx_queues; 262 lim.edl_min_txq_count = lim.edl_max_txq_count = data->nb_tx_queues; 263 264 return efx_nic_set_drv_limits(sa->nic, &lim); 265 } 266 267 static int 268 sfc_set_fw_subvariant(struct sfc_adapter *sa) 269 { 270 struct sfc_adapter_shared *sas = sfc_sa2shared(sa); 271 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); 272 uint64_t tx_offloads = sa->eth_dev->data->dev_conf.txmode.offloads; 273 unsigned int txq_index; 274 efx_nic_fw_subvariant_t req_fw_subvariant; 275 efx_nic_fw_subvariant_t cur_fw_subvariant; 276 int rc; 277 278 if (!encp->enc_fw_subvariant_no_tx_csum_supported) { 279 sfc_info(sa, "no-Tx-checksum subvariant not supported"); 280 return 0; 281 } 282 283 for (txq_index = 0; txq_index < sas->txq_count; ++txq_index) { 284 struct sfc_txq_info *txq_info = &sas->txq_info[txq_index]; 285 286 if (txq_info->state & SFC_TXQ_INITIALIZED) 287 tx_offloads |= txq_info->offloads; 288 } 289 290 if (tx_offloads & (DEV_TX_OFFLOAD_IPV4_CKSUM | 291 DEV_TX_OFFLOAD_TCP_CKSUM | 292 DEV_TX_OFFLOAD_UDP_CKSUM | 293 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) 294 req_fw_subvariant = EFX_NIC_FW_SUBVARIANT_DEFAULT; 295 else 296 req_fw_subvariant = EFX_NIC_FW_SUBVARIANT_NO_TX_CSUM; 297 298 rc = efx_nic_get_fw_subvariant(sa->nic, &cur_fw_subvariant); 299 if (rc != 0) { 300 sfc_err(sa, "failed to get FW subvariant: %d", rc); 301 return rc; 302 } 303 sfc_info(sa, "FW subvariant is %u vs required %u", 304 cur_fw_subvariant, req_fw_subvariant); 305 306 if (cur_fw_subvariant == req_fw_subvariant) 307 return 0; 308 309 rc = efx_nic_set_fw_subvariant(sa->nic, req_fw_subvariant); 310 if (rc != 0) { 311 sfc_err(sa, "failed to set FW subvariant %u: %d", 312 req_fw_subvariant, rc); 313 return rc; 314 } 315 sfc_info(sa, "FW subvariant set to %u", req_fw_subvariant); 316 317 return 0; 318 } 319 320 static int 321 sfc_try_start(struct sfc_adapter *sa) 322 { 323 const efx_nic_cfg_t *encp; 324 int rc; 325 326 sfc_log_init(sa, "entry"); 327 328 SFC_ASSERT(sfc_adapter_is_locked(sa)); 329 SFC_ASSERT(sa->state == SFC_ADAPTER_STARTING); 330 331 sfc_log_init(sa, "set FW subvariant"); 332 rc = sfc_set_fw_subvariant(sa); 333 if (rc != 0) 334 goto fail_set_fw_subvariant; 335 336 sfc_log_init(sa, "set resource limits"); 337 rc = sfc_set_drv_limits(sa); 338 if (rc != 0) 339 goto fail_set_drv_limits; 340 341 sfc_log_init(sa, "init nic"); 342 rc = efx_nic_init(sa->nic); 343 if (rc != 0) 344 goto fail_nic_init; 345 346 encp = efx_nic_cfg_get(sa->nic); 347 348 /* 349 * Refresh (since it may change on NIC reset/restart) a copy of 350 * supported tunnel encapsulations in shared memory to be used 351 * on supported Rx packet type classes get. 352 */ 353 sa->priv.shared->tunnel_encaps = 354 encp->enc_tunnel_encapsulations_supported; 355 356 if (encp->enc_tunnel_encapsulations_supported != 0) { 357 sfc_log_init(sa, "apply tunnel config"); 358 rc = efx_tunnel_reconfigure(sa->nic); 359 if (rc != 0) 360 goto fail_tunnel_reconfigure; 361 } 362 363 rc = sfc_intr_start(sa); 364 if (rc != 0) 365 goto fail_intr_start; 366 367 rc = sfc_ev_start(sa); 368 if (rc != 0) 369 goto fail_ev_start; 370 371 rc = sfc_port_start(sa); 372 if (rc != 0) 373 goto fail_port_start; 374 375 rc = sfc_rx_start(sa); 376 if (rc != 0) 377 goto fail_rx_start; 378 379 rc = sfc_tx_start(sa); 380 if (rc != 0) 381 goto fail_tx_start; 382 383 rc = sfc_flow_start(sa); 384 if (rc != 0) 385 goto fail_flows_insert; 386 387 sfc_log_init(sa, "done"); 388 return 0; 389 390 fail_flows_insert: 391 sfc_tx_stop(sa); 392 393 fail_tx_start: 394 sfc_rx_stop(sa); 395 396 fail_rx_start: 397 sfc_port_stop(sa); 398 399 fail_port_start: 400 sfc_ev_stop(sa); 401 402 fail_ev_start: 403 sfc_intr_stop(sa); 404 405 fail_intr_start: 406 fail_tunnel_reconfigure: 407 efx_nic_fini(sa->nic); 408 409 fail_nic_init: 410 fail_set_drv_limits: 411 fail_set_fw_subvariant: 412 sfc_log_init(sa, "failed %d", rc); 413 return rc; 414 } 415 416 int 417 sfc_start(struct sfc_adapter *sa) 418 { 419 unsigned int start_tries = 3; 420 int rc; 421 422 sfc_log_init(sa, "entry"); 423 424 SFC_ASSERT(sfc_adapter_is_locked(sa)); 425 426 switch (sa->state) { 427 case SFC_ADAPTER_CONFIGURED: 428 break; 429 case SFC_ADAPTER_STARTED: 430 sfc_notice(sa, "already started"); 431 return 0; 432 default: 433 rc = EINVAL; 434 goto fail_bad_state; 435 } 436 437 sa->state = SFC_ADAPTER_STARTING; 438 439 do { 440 rc = sfc_try_start(sa); 441 } while ((--start_tries > 0) && 442 (rc == EIO || rc == EAGAIN || rc == ENOENT || rc == EINVAL)); 443 444 if (rc != 0) 445 goto fail_try_start; 446 447 sa->state = SFC_ADAPTER_STARTED; 448 sfc_log_init(sa, "done"); 449 return 0; 450 451 fail_try_start: 452 sa->state = SFC_ADAPTER_CONFIGURED; 453 fail_bad_state: 454 sfc_log_init(sa, "failed %d", rc); 455 return rc; 456 } 457 458 void 459 sfc_stop(struct sfc_adapter *sa) 460 { 461 sfc_log_init(sa, "entry"); 462 463 SFC_ASSERT(sfc_adapter_is_locked(sa)); 464 465 switch (sa->state) { 466 case SFC_ADAPTER_STARTED: 467 break; 468 case SFC_ADAPTER_CONFIGURED: 469 sfc_notice(sa, "already stopped"); 470 return; 471 default: 472 sfc_err(sa, "stop in unexpected state %u", sa->state); 473 SFC_ASSERT(B_FALSE); 474 return; 475 } 476 477 sa->state = SFC_ADAPTER_STOPPING; 478 479 sfc_flow_stop(sa); 480 sfc_tx_stop(sa); 481 sfc_rx_stop(sa); 482 sfc_port_stop(sa); 483 sfc_ev_stop(sa); 484 sfc_intr_stop(sa); 485 efx_nic_fini(sa->nic); 486 487 sa->state = SFC_ADAPTER_CONFIGURED; 488 sfc_log_init(sa, "done"); 489 } 490 491 static int 492 sfc_restart(struct sfc_adapter *sa) 493 { 494 int rc; 495 496 SFC_ASSERT(sfc_adapter_is_locked(sa)); 497 498 if (sa->state != SFC_ADAPTER_STARTED) 499 return EINVAL; 500 501 sfc_stop(sa); 502 503 rc = sfc_start(sa); 504 if (rc != 0) 505 sfc_err(sa, "restart failed"); 506 507 return rc; 508 } 509 510 static void 511 sfc_restart_if_required(void *arg) 512 { 513 struct sfc_adapter *sa = arg; 514 515 /* If restart is scheduled, clear the flag and do it */ 516 if (rte_atomic32_cmpset((volatile uint32_t *)&sa->restart_required, 517 1, 0)) { 518 sfc_adapter_lock(sa); 519 if (sa->state == SFC_ADAPTER_STARTED) 520 (void)sfc_restart(sa); 521 sfc_adapter_unlock(sa); 522 } 523 } 524 525 void 526 sfc_schedule_restart(struct sfc_adapter *sa) 527 { 528 int rc; 529 530 /* Schedule restart alarm if it is not scheduled yet */ 531 if (!rte_atomic32_test_and_set(&sa->restart_required)) 532 return; 533 534 rc = rte_eal_alarm_set(1, sfc_restart_if_required, sa); 535 if (rc == -ENOTSUP) 536 sfc_warn(sa, "alarms are not supported, restart is pending"); 537 else if (rc != 0) 538 sfc_err(sa, "cannot arm restart alarm (rc=%d)", rc); 539 else 540 sfc_notice(sa, "restart scheduled"); 541 } 542 543 int 544 sfc_configure(struct sfc_adapter *sa) 545 { 546 int rc; 547 548 sfc_log_init(sa, "entry"); 549 550 SFC_ASSERT(sfc_adapter_is_locked(sa)); 551 552 SFC_ASSERT(sa->state == SFC_ADAPTER_INITIALIZED || 553 sa->state == SFC_ADAPTER_CONFIGURED); 554 sa->state = SFC_ADAPTER_CONFIGURING; 555 556 rc = sfc_check_conf(sa); 557 if (rc != 0) 558 goto fail_check_conf; 559 560 rc = sfc_intr_configure(sa); 561 if (rc != 0) 562 goto fail_intr_configure; 563 564 rc = sfc_port_configure(sa); 565 if (rc != 0) 566 goto fail_port_configure; 567 568 rc = sfc_rx_configure(sa); 569 if (rc != 0) 570 goto fail_rx_configure; 571 572 rc = sfc_tx_configure(sa); 573 if (rc != 0) 574 goto fail_tx_configure; 575 576 sa->state = SFC_ADAPTER_CONFIGURED; 577 sfc_log_init(sa, "done"); 578 return 0; 579 580 fail_tx_configure: 581 sfc_rx_close(sa); 582 583 fail_rx_configure: 584 sfc_port_close(sa); 585 586 fail_port_configure: 587 sfc_intr_close(sa); 588 589 fail_intr_configure: 590 fail_check_conf: 591 sa->state = SFC_ADAPTER_INITIALIZED; 592 sfc_log_init(sa, "failed %d", rc); 593 return rc; 594 } 595 596 void 597 sfc_close(struct sfc_adapter *sa) 598 { 599 sfc_log_init(sa, "entry"); 600 601 SFC_ASSERT(sfc_adapter_is_locked(sa)); 602 603 SFC_ASSERT(sa->state == SFC_ADAPTER_CONFIGURED); 604 sa->state = SFC_ADAPTER_CLOSING; 605 606 sfc_tx_close(sa); 607 sfc_rx_close(sa); 608 sfc_port_close(sa); 609 sfc_intr_close(sa); 610 611 sa->state = SFC_ADAPTER_INITIALIZED; 612 sfc_log_init(sa, "done"); 613 } 614 615 static int 616 sfc_mem_bar_init(struct sfc_adapter *sa, unsigned int membar) 617 { 618 struct rte_eth_dev *eth_dev = sa->eth_dev; 619 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 620 efsys_bar_t *ebp = &sa->mem_bar; 621 struct rte_mem_resource *res = &pci_dev->mem_resource[membar]; 622 623 SFC_BAR_LOCK_INIT(ebp, eth_dev->data->name); 624 ebp->esb_rid = membar; 625 ebp->esb_dev = pci_dev; 626 ebp->esb_base = res->addr; 627 return 0; 628 } 629 630 static void 631 sfc_mem_bar_fini(struct sfc_adapter *sa) 632 { 633 efsys_bar_t *ebp = &sa->mem_bar; 634 635 SFC_BAR_LOCK_DESTROY(ebp); 636 memset(ebp, 0, sizeof(*ebp)); 637 } 638 639 /* 640 * A fixed RSS key which has a property of being symmetric 641 * (symmetrical flows are distributed to the same CPU) 642 * and also known to give a uniform distribution 643 * (a good distribution of traffic between different CPUs) 644 */ 645 static const uint8_t default_rss_key[EFX_RSS_KEY_SIZE] = { 646 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 647 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 648 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 649 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 650 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 651 }; 652 653 static int 654 sfc_rss_attach(struct sfc_adapter *sa) 655 { 656 struct sfc_rss *rss = &sfc_sa2shared(sa)->rss; 657 int rc; 658 659 rc = efx_intr_init(sa->nic, sa->intr.type, NULL); 660 if (rc != 0) 661 goto fail_intr_init; 662 663 rc = efx_ev_init(sa->nic); 664 if (rc != 0) 665 goto fail_ev_init; 666 667 rc = efx_rx_init(sa->nic); 668 if (rc != 0) 669 goto fail_rx_init; 670 671 rc = efx_rx_scale_default_support_get(sa->nic, &rss->context_type); 672 if (rc != 0) 673 goto fail_scale_support_get; 674 675 rc = efx_rx_hash_default_support_get(sa->nic, &rss->hash_support); 676 if (rc != 0) 677 goto fail_hash_support_get; 678 679 rc = sfc_rx_hash_init(sa); 680 if (rc != 0) 681 goto fail_rx_hash_init; 682 683 efx_rx_fini(sa->nic); 684 efx_ev_fini(sa->nic); 685 efx_intr_fini(sa->nic); 686 687 rte_memcpy(rss->key, default_rss_key, sizeof(rss->key)); 688 689 return 0; 690 691 fail_rx_hash_init: 692 fail_hash_support_get: 693 fail_scale_support_get: 694 efx_rx_fini(sa->nic); 695 696 fail_rx_init: 697 efx_ev_fini(sa->nic); 698 699 fail_ev_init: 700 efx_intr_fini(sa->nic); 701 702 fail_intr_init: 703 return rc; 704 } 705 706 static void 707 sfc_rss_detach(struct sfc_adapter *sa) 708 { 709 sfc_rx_hash_fini(sa); 710 } 711 712 int 713 sfc_attach(struct sfc_adapter *sa) 714 { 715 const efx_nic_cfg_t *encp; 716 efx_nic_t *enp = sa->nic; 717 int rc; 718 719 sfc_log_init(sa, "entry"); 720 721 SFC_ASSERT(sfc_adapter_is_locked(sa)); 722 723 efx_mcdi_new_epoch(enp); 724 725 sfc_log_init(sa, "reset nic"); 726 rc = efx_nic_reset(enp); 727 if (rc != 0) 728 goto fail_nic_reset; 729 730 /* 731 * Probed NIC is sufficient for tunnel init. 732 * Initialize tunnel support to be able to use libefx 733 * efx_tunnel_config_udp_{add,remove}() in any state and 734 * efx_tunnel_reconfigure() on start up. 735 */ 736 rc = efx_tunnel_init(enp); 737 if (rc != 0) 738 goto fail_tunnel_init; 739 740 encp = efx_nic_cfg_get(sa->nic); 741 742 /* 743 * Make a copy of supported tunnel encapsulations in shared 744 * memory to be used on supported Rx packet type classes get. 745 */ 746 sa->priv.shared->tunnel_encaps = 747 encp->enc_tunnel_encapsulations_supported; 748 749 if (sfc_dp_tx_offload_capa(sa->priv.dp_tx) & DEV_TX_OFFLOAD_TCP_TSO) { 750 sa->tso = encp->enc_fw_assisted_tso_v2_enabled; 751 if (!sa->tso) 752 sfc_info(sa, "TSO support isn't available on this adapter"); 753 } 754 755 if (sa->tso && 756 (sfc_dp_tx_offload_capa(sa->priv.dp_tx) & 757 (DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 758 DEV_TX_OFFLOAD_GENEVE_TNL_TSO)) != 0) { 759 sa->tso_encap = encp->enc_fw_assisted_tso_v2_encap_enabled; 760 if (!sa->tso_encap) 761 sfc_info(sa, "Encapsulated TSO support isn't available on this adapter"); 762 } 763 764 sfc_log_init(sa, "estimate resource limits"); 765 rc = sfc_estimate_resource_limits(sa); 766 if (rc != 0) 767 goto fail_estimate_rsrc_limits; 768 769 sa->evq_max_entries = encp->enc_evq_max_nevs; 770 SFC_ASSERT(rte_is_power_of_2(sa->evq_max_entries)); 771 772 sa->evq_min_entries = encp->enc_evq_min_nevs; 773 SFC_ASSERT(rte_is_power_of_2(sa->evq_min_entries)); 774 775 sa->rxq_max_entries = encp->enc_rxq_max_ndescs; 776 SFC_ASSERT(rte_is_power_of_2(sa->rxq_max_entries)); 777 778 sa->rxq_min_entries = encp->enc_rxq_min_ndescs; 779 SFC_ASSERT(rte_is_power_of_2(sa->rxq_min_entries)); 780 781 sa->txq_max_entries = encp->enc_txq_max_ndescs; 782 SFC_ASSERT(rte_is_power_of_2(sa->txq_max_entries)); 783 784 sa->txq_min_entries = encp->enc_txq_min_ndescs; 785 SFC_ASSERT(rte_is_power_of_2(sa->txq_min_entries)); 786 787 rc = sfc_intr_attach(sa); 788 if (rc != 0) 789 goto fail_intr_attach; 790 791 rc = sfc_ev_attach(sa); 792 if (rc != 0) 793 goto fail_ev_attach; 794 795 rc = sfc_port_attach(sa); 796 if (rc != 0) 797 goto fail_port_attach; 798 799 rc = sfc_rss_attach(sa); 800 if (rc != 0) 801 goto fail_rss_attach; 802 803 rc = sfc_filter_attach(sa); 804 if (rc != 0) 805 goto fail_filter_attach; 806 807 sfc_log_init(sa, "fini nic"); 808 efx_nic_fini(enp); 809 810 sfc_flow_init(sa); 811 812 sa->state = SFC_ADAPTER_INITIALIZED; 813 814 sfc_log_init(sa, "done"); 815 return 0; 816 817 fail_filter_attach: 818 sfc_rss_detach(sa); 819 820 fail_rss_attach: 821 sfc_port_detach(sa); 822 823 fail_port_attach: 824 sfc_ev_detach(sa); 825 826 fail_ev_attach: 827 sfc_intr_detach(sa); 828 829 fail_intr_attach: 830 efx_nic_fini(sa->nic); 831 832 fail_estimate_rsrc_limits: 833 fail_tunnel_init: 834 efx_tunnel_fini(sa->nic); 835 836 fail_nic_reset: 837 838 sfc_log_init(sa, "failed %d", rc); 839 return rc; 840 } 841 842 void 843 sfc_detach(struct sfc_adapter *sa) 844 { 845 sfc_log_init(sa, "entry"); 846 847 SFC_ASSERT(sfc_adapter_is_locked(sa)); 848 849 sfc_flow_fini(sa); 850 851 sfc_filter_detach(sa); 852 sfc_rss_detach(sa); 853 sfc_port_detach(sa); 854 sfc_ev_detach(sa); 855 sfc_intr_detach(sa); 856 efx_tunnel_fini(sa->nic); 857 858 sa->state = SFC_ADAPTER_UNINITIALIZED; 859 } 860 861 static int 862 sfc_kvarg_fv_variant_handler(__rte_unused const char *key, 863 const char *value_str, void *opaque) 864 { 865 uint32_t *value = opaque; 866 867 if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_DONT_CARE) == 0) 868 *value = EFX_FW_VARIANT_DONT_CARE; 869 else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_FULL_FEATURED) == 0) 870 *value = EFX_FW_VARIANT_FULL_FEATURED; 871 else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_LOW_LATENCY) == 0) 872 *value = EFX_FW_VARIANT_LOW_LATENCY; 873 else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_PACKED_STREAM) == 0) 874 *value = EFX_FW_VARIANT_PACKED_STREAM; 875 else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_DPDK) == 0) 876 *value = EFX_FW_VARIANT_DPDK; 877 else 878 return -EINVAL; 879 880 return 0; 881 } 882 883 static int 884 sfc_get_fw_variant(struct sfc_adapter *sa, efx_fw_variant_t *efv) 885 { 886 efx_nic_fw_info_t enfi; 887 int rc; 888 889 rc = efx_nic_get_fw_version(sa->nic, &enfi); 890 if (rc != 0) 891 return rc; 892 else if (!enfi.enfi_dpcpu_fw_ids_valid) 893 return ENOTSUP; 894 895 /* 896 * Firmware variant can be uniquely identified by the RxDPCPU 897 * firmware id 898 */ 899 switch (enfi.enfi_rx_dpcpu_fw_id) { 900 case EFX_RXDP_FULL_FEATURED_FW_ID: 901 *efv = EFX_FW_VARIANT_FULL_FEATURED; 902 break; 903 904 case EFX_RXDP_LOW_LATENCY_FW_ID: 905 *efv = EFX_FW_VARIANT_LOW_LATENCY; 906 break; 907 908 case EFX_RXDP_PACKED_STREAM_FW_ID: 909 *efv = EFX_FW_VARIANT_PACKED_STREAM; 910 break; 911 912 case EFX_RXDP_DPDK_FW_ID: 913 *efv = EFX_FW_VARIANT_DPDK; 914 break; 915 916 default: 917 /* 918 * Other firmware variants are not considered, since they are 919 * not supported in the device parameters 920 */ 921 *efv = EFX_FW_VARIANT_DONT_CARE; 922 break; 923 } 924 925 return 0; 926 } 927 928 static const char * 929 sfc_fw_variant2str(efx_fw_variant_t efv) 930 { 931 switch (efv) { 932 case EFX_RXDP_FULL_FEATURED_FW_ID: 933 return SFC_KVARG_FW_VARIANT_FULL_FEATURED; 934 case EFX_RXDP_LOW_LATENCY_FW_ID: 935 return SFC_KVARG_FW_VARIANT_LOW_LATENCY; 936 case EFX_RXDP_PACKED_STREAM_FW_ID: 937 return SFC_KVARG_FW_VARIANT_PACKED_STREAM; 938 case EFX_RXDP_DPDK_FW_ID: 939 return SFC_KVARG_FW_VARIANT_DPDK; 940 default: 941 return "unknown"; 942 } 943 } 944 945 static int 946 sfc_kvarg_rxd_wait_timeout_ns(struct sfc_adapter *sa) 947 { 948 int rc; 949 long value; 950 951 value = SFC_RXD_WAIT_TIMEOUT_NS_DEF; 952 953 rc = sfc_kvargs_process(sa, SFC_KVARG_RXD_WAIT_TIMEOUT_NS, 954 sfc_kvarg_long_handler, &value); 955 if (rc != 0) 956 return rc; 957 958 if (value < 0 || 959 (unsigned long)value > EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX) { 960 sfc_err(sa, "wrong '" SFC_KVARG_RXD_WAIT_TIMEOUT_NS "' " 961 "was set (%ld);", value); 962 sfc_err(sa, "it must not be less than 0 or greater than %u", 963 EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX); 964 return EINVAL; 965 } 966 967 sa->rxd_wait_timeout_ns = value; 968 return 0; 969 } 970 971 static int 972 sfc_nic_probe(struct sfc_adapter *sa) 973 { 974 efx_nic_t *enp = sa->nic; 975 efx_fw_variant_t preferred_efv; 976 efx_fw_variant_t efv; 977 int rc; 978 979 preferred_efv = EFX_FW_VARIANT_DONT_CARE; 980 rc = sfc_kvargs_process(sa, SFC_KVARG_FW_VARIANT, 981 sfc_kvarg_fv_variant_handler, 982 &preferred_efv); 983 if (rc != 0) { 984 sfc_err(sa, "invalid %s parameter value", SFC_KVARG_FW_VARIANT); 985 return rc; 986 } 987 988 rc = sfc_kvarg_rxd_wait_timeout_ns(sa); 989 if (rc != 0) 990 return rc; 991 992 rc = efx_nic_probe(enp, preferred_efv); 993 if (rc == EACCES) { 994 /* Unprivileged functions cannot set FW variant */ 995 rc = efx_nic_probe(enp, EFX_FW_VARIANT_DONT_CARE); 996 } 997 if (rc != 0) 998 return rc; 999 1000 rc = sfc_get_fw_variant(sa, &efv); 1001 if (rc == ENOTSUP) { 1002 sfc_warn(sa, "FW variant can not be obtained"); 1003 return 0; 1004 } 1005 if (rc != 0) 1006 return rc; 1007 1008 /* Check that firmware variant was changed to the requested one */ 1009 if (preferred_efv != EFX_FW_VARIANT_DONT_CARE && preferred_efv != efv) { 1010 sfc_warn(sa, "FW variant has not changed to the requested %s", 1011 sfc_fw_variant2str(preferred_efv)); 1012 } 1013 1014 sfc_notice(sa, "running FW variant is %s", sfc_fw_variant2str(efv)); 1015 1016 return 0; 1017 } 1018 1019 int 1020 sfc_probe(struct sfc_adapter *sa) 1021 { 1022 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(sa->eth_dev); 1023 unsigned int membar; 1024 efx_nic_t *enp; 1025 int rc; 1026 1027 sfc_log_init(sa, "entry"); 1028 1029 SFC_ASSERT(sfc_adapter_is_locked(sa)); 1030 1031 sa->socket_id = rte_socket_id(); 1032 rte_atomic32_init(&sa->restart_required); 1033 1034 sfc_log_init(sa, "get family"); 1035 rc = efx_family(pci_dev->id.vendor_id, pci_dev->id.device_id, 1036 &sa->family, &membar); 1037 if (rc != 0) 1038 goto fail_family; 1039 sfc_log_init(sa, "family is %u, membar is %u", sa->family, membar); 1040 1041 sfc_log_init(sa, "init mem bar"); 1042 rc = sfc_mem_bar_init(sa, membar); 1043 if (rc != 0) 1044 goto fail_mem_bar_init; 1045 1046 sfc_log_init(sa, "create nic"); 1047 rte_spinlock_init(&sa->nic_lock); 1048 rc = efx_nic_create(sa->family, (efsys_identifier_t *)sa, 1049 &sa->mem_bar, &sa->nic_lock, &enp); 1050 if (rc != 0) 1051 goto fail_nic_create; 1052 sa->nic = enp; 1053 1054 rc = sfc_mcdi_init(sa); 1055 if (rc != 0) 1056 goto fail_mcdi_init; 1057 1058 sfc_log_init(sa, "probe nic"); 1059 rc = sfc_nic_probe(sa); 1060 if (rc != 0) 1061 goto fail_nic_probe; 1062 1063 sfc_log_init(sa, "done"); 1064 return 0; 1065 1066 fail_nic_probe: 1067 sfc_mcdi_fini(sa); 1068 1069 fail_mcdi_init: 1070 sfc_log_init(sa, "destroy nic"); 1071 sa->nic = NULL; 1072 efx_nic_destroy(enp); 1073 1074 fail_nic_create: 1075 sfc_mem_bar_fini(sa); 1076 1077 fail_mem_bar_init: 1078 fail_family: 1079 sfc_log_init(sa, "failed %d", rc); 1080 return rc; 1081 } 1082 1083 void 1084 sfc_unprobe(struct sfc_adapter *sa) 1085 { 1086 efx_nic_t *enp = sa->nic; 1087 1088 sfc_log_init(sa, "entry"); 1089 1090 SFC_ASSERT(sfc_adapter_is_locked(sa)); 1091 1092 sfc_log_init(sa, "unprobe nic"); 1093 efx_nic_unprobe(enp); 1094 1095 sfc_mcdi_fini(sa); 1096 1097 /* 1098 * Make sure there is no pending alarm to restart since we are 1099 * going to free device private which is passed as the callback 1100 * opaque data. A new alarm cannot be scheduled since MCDI is 1101 * shut down. 1102 */ 1103 rte_eal_alarm_cancel(sfc_restart_if_required, sa); 1104 1105 sfc_log_init(sa, "destroy nic"); 1106 sa->nic = NULL; 1107 efx_nic_destroy(enp); 1108 1109 sfc_mem_bar_fini(sa); 1110 1111 sfc_flow_fini(sa); 1112 sa->state = SFC_ADAPTER_UNINITIALIZED; 1113 } 1114 1115 uint32_t 1116 sfc_register_logtype(const struct rte_pci_addr *pci_addr, 1117 const char *lt_prefix_str, uint32_t ll_default) 1118 { 1119 size_t lt_prefix_str_size = strlen(lt_prefix_str); 1120 size_t lt_str_size_max; 1121 char *lt_str = NULL; 1122 int ret; 1123 1124 if (SIZE_MAX - PCI_PRI_STR_SIZE - 1 > lt_prefix_str_size) { 1125 ++lt_prefix_str_size; /* Reserve space for prefix separator */ 1126 lt_str_size_max = lt_prefix_str_size + PCI_PRI_STR_SIZE + 1; 1127 } else { 1128 return sfc_logtype_driver; 1129 } 1130 1131 lt_str = rte_zmalloc("logtype_str", lt_str_size_max, 0); 1132 if (lt_str == NULL) 1133 return sfc_logtype_driver; 1134 1135 strncpy(lt_str, lt_prefix_str, lt_prefix_str_size); 1136 lt_str[lt_prefix_str_size - 1] = '.'; 1137 rte_pci_device_name(pci_addr, lt_str + lt_prefix_str_size, 1138 lt_str_size_max - lt_prefix_str_size); 1139 lt_str[lt_str_size_max - 1] = '\0'; 1140 1141 ret = rte_log_register_type_and_pick_level(lt_str, ll_default); 1142 rte_free(lt_str); 1143 1144 if (ret < 0) 1145 return sfc_logtype_driver; 1146 1147 return ret; 1148 } 1149