1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2021 Xilinx, Inc. 4 * Copyright(c) 2016-2019 Solarflare Communications Inc. 5 * 6 * This software was jointly developed between OKTET Labs (under contract 7 * for Solarflare) and Solarflare Communications, Inc. 8 */ 9 10 /* sysconf() */ 11 #include <unistd.h> 12 13 #include <rte_errno.h> 14 #include <rte_alarm.h> 15 16 #include "efx.h" 17 18 #include "sfc.h" 19 #include "sfc_debug.h" 20 #include "sfc_log.h" 21 #include "sfc_ev.h" 22 #include "sfc_rx.h" 23 #include "sfc_mae_counter.h" 24 #include "sfc_tx.h" 25 #include "sfc_kvargs.h" 26 #include "sfc_tweak.h" 27 28 29 int 30 sfc_dma_alloc(const struct sfc_adapter *sa, const char *name, uint16_t id, 31 size_t len, int socket_id, efsys_mem_t *esmp) 32 { 33 const struct rte_memzone *mz; 34 35 sfc_log_init(sa, "name=%s id=%u len=%zu socket_id=%d", 36 name, id, len, socket_id); 37 38 mz = rte_eth_dma_zone_reserve(sa->eth_dev, name, id, len, 39 sysconf(_SC_PAGESIZE), socket_id); 40 if (mz == NULL) { 41 sfc_err(sa, "cannot reserve DMA zone for %s:%u %#x@%d: %s", 42 name, (unsigned int)id, (unsigned int)len, socket_id, 43 rte_strerror(rte_errno)); 44 return ENOMEM; 45 } 46 47 esmp->esm_addr = mz->iova; 48 if (esmp->esm_addr == RTE_BAD_IOVA) { 49 (void)rte_memzone_free(mz); 50 return EFAULT; 51 } 52 53 esmp->esm_mz = mz; 54 esmp->esm_base = mz->addr; 55 56 sfc_info(sa, 57 "DMA name=%s id=%u len=%lu socket_id=%d => virt=%p iova=%lx", 58 name, id, len, socket_id, esmp->esm_base, 59 (unsigned long)esmp->esm_addr); 60 61 return 0; 62 } 63 64 void 65 sfc_dma_free(const struct sfc_adapter *sa, efsys_mem_t *esmp) 66 { 67 int rc; 68 69 sfc_log_init(sa, "name=%s", esmp->esm_mz->name); 70 71 rc = rte_memzone_free(esmp->esm_mz); 72 if (rc != 0) 73 sfc_err(sa, "rte_memzone_free(() failed: %d", rc); 74 75 memset(esmp, 0, sizeof(*esmp)); 76 } 77 78 static uint32_t 79 sfc_phy_cap_from_link_speeds(uint32_t speeds) 80 { 81 uint32_t phy_caps = 0; 82 83 if (~speeds & ETH_LINK_SPEED_FIXED) { 84 phy_caps |= (1 << EFX_PHY_CAP_AN); 85 /* 86 * If no speeds are specified in the mask, any supported 87 * may be negotiated 88 */ 89 if (speeds == ETH_LINK_SPEED_AUTONEG) 90 phy_caps |= 91 (1 << EFX_PHY_CAP_1000FDX) | 92 (1 << EFX_PHY_CAP_10000FDX) | 93 (1 << EFX_PHY_CAP_25000FDX) | 94 (1 << EFX_PHY_CAP_40000FDX) | 95 (1 << EFX_PHY_CAP_50000FDX) | 96 (1 << EFX_PHY_CAP_100000FDX); 97 } 98 if (speeds & ETH_LINK_SPEED_1G) 99 phy_caps |= (1 << EFX_PHY_CAP_1000FDX); 100 if (speeds & ETH_LINK_SPEED_10G) 101 phy_caps |= (1 << EFX_PHY_CAP_10000FDX); 102 if (speeds & ETH_LINK_SPEED_25G) 103 phy_caps |= (1 << EFX_PHY_CAP_25000FDX); 104 if (speeds & ETH_LINK_SPEED_40G) 105 phy_caps |= (1 << EFX_PHY_CAP_40000FDX); 106 if (speeds & ETH_LINK_SPEED_50G) 107 phy_caps |= (1 << EFX_PHY_CAP_50000FDX); 108 if (speeds & ETH_LINK_SPEED_100G) 109 phy_caps |= (1 << EFX_PHY_CAP_100000FDX); 110 111 return phy_caps; 112 } 113 114 /* 115 * Check requested device level configuration. 116 * Receive and transmit configuration is checked in corresponding 117 * modules. 118 */ 119 static int 120 sfc_check_conf(struct sfc_adapter *sa) 121 { 122 const struct rte_eth_conf *conf = &sa->eth_dev->data->dev_conf; 123 int rc = 0; 124 125 sa->port.phy_adv_cap = 126 sfc_phy_cap_from_link_speeds(conf->link_speeds) & 127 sa->port.phy_adv_cap_mask; 128 if ((sa->port.phy_adv_cap & ~(1 << EFX_PHY_CAP_AN)) == 0) { 129 sfc_err(sa, "No link speeds from mask %#x are supported", 130 conf->link_speeds); 131 rc = EINVAL; 132 } 133 134 #if !EFSYS_OPT_LOOPBACK 135 if (conf->lpbk_mode != 0) { 136 sfc_err(sa, "Loopback not supported"); 137 rc = EINVAL; 138 } 139 #endif 140 141 if (conf->dcb_capability_en != 0) { 142 sfc_err(sa, "Priority-based flow control not supported"); 143 rc = EINVAL; 144 } 145 146 if (conf->fdir_conf.mode != RTE_FDIR_MODE_NONE) { 147 sfc_err(sa, "Flow Director not supported"); 148 rc = EINVAL; 149 } 150 151 if ((conf->intr_conf.lsc != 0) && 152 (sa->intr.type != EFX_INTR_LINE) && 153 (sa->intr.type != EFX_INTR_MESSAGE)) { 154 sfc_err(sa, "Link status change interrupt not supported"); 155 rc = EINVAL; 156 } 157 158 if (conf->intr_conf.rxq != 0 && 159 (sa->priv.dp_rx->features & SFC_DP_RX_FEAT_INTR) == 0) { 160 sfc_err(sa, "Receive queue interrupt not supported"); 161 rc = EINVAL; 162 } 163 164 return rc; 165 } 166 167 /* 168 * Find out maximum number of receive and transmit queues which could be 169 * advertised. 170 * 171 * NIC is kept initialized on success to allow other modules acquire 172 * defaults and capabilities. 173 */ 174 static int 175 sfc_estimate_resource_limits(struct sfc_adapter *sa) 176 { 177 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); 178 struct sfc_adapter_shared *sas = sfc_sa2shared(sa); 179 efx_drv_limits_t limits; 180 int rc; 181 uint32_t evq_allocated; 182 uint32_t rxq_allocated; 183 uint32_t txq_allocated; 184 185 memset(&limits, 0, sizeof(limits)); 186 187 /* Request at least one Rx and Tx queue */ 188 limits.edl_min_rxq_count = 1; 189 limits.edl_min_txq_count = 1; 190 /* Management event queue plus event queue for each Tx and Rx queue */ 191 limits.edl_min_evq_count = 192 1 + limits.edl_min_rxq_count + limits.edl_min_txq_count; 193 194 /* Divide by number of functions to guarantee that all functions 195 * will get promised resources 196 */ 197 /* FIXME Divide by number of functions (not 2) below */ 198 limits.edl_max_evq_count = encp->enc_evq_limit / 2; 199 SFC_ASSERT(limits.edl_max_evq_count >= limits.edl_min_rxq_count); 200 201 /* Split equally between receive and transmit */ 202 limits.edl_max_rxq_count = 203 MIN(encp->enc_rxq_limit, (limits.edl_max_evq_count - 1) / 2); 204 SFC_ASSERT(limits.edl_max_rxq_count >= limits.edl_min_rxq_count); 205 206 limits.edl_max_txq_count = 207 MIN(encp->enc_txq_limit, 208 limits.edl_max_evq_count - 1 - limits.edl_max_rxq_count); 209 210 if (sa->tso && encp->enc_fw_assisted_tso_v2_enabled) 211 limits.edl_max_txq_count = 212 MIN(limits.edl_max_txq_count, 213 encp->enc_fw_assisted_tso_v2_n_contexts / 214 encp->enc_hw_pf_count); 215 216 SFC_ASSERT(limits.edl_max_txq_count >= limits.edl_min_rxq_count); 217 218 /* Configure the minimum required resources needed for the 219 * driver to operate, and the maximum desired resources that the 220 * driver is capable of using. 221 */ 222 efx_nic_set_drv_limits(sa->nic, &limits); 223 224 sfc_log_init(sa, "init nic"); 225 rc = efx_nic_init(sa->nic); 226 if (rc != 0) 227 goto fail_nic_init; 228 229 /* Find resource dimensions assigned by firmware to this function */ 230 rc = efx_nic_get_vi_pool(sa->nic, &evq_allocated, &rxq_allocated, 231 &txq_allocated); 232 if (rc != 0) 233 goto fail_get_vi_pool; 234 235 /* It still may allocate more than maximum, ensure limit */ 236 evq_allocated = MIN(evq_allocated, limits.edl_max_evq_count); 237 rxq_allocated = MIN(rxq_allocated, limits.edl_max_rxq_count); 238 txq_allocated = MIN(txq_allocated, limits.edl_max_txq_count); 239 240 /* 241 * Subtract management EVQ not used for traffic 242 * The resource allocation strategy is as follows: 243 * - one EVQ for management 244 * - one EVQ for each ethdev RXQ 245 * - one EVQ for each ethdev TXQ 246 * - one EVQ and one RXQ for optional MAE counters. 247 */ 248 if (evq_allocated == 0) { 249 sfc_err(sa, "count of allocated EvQ is 0"); 250 rc = ENOMEM; 251 goto fail_allocate_evq; 252 } 253 evq_allocated--; 254 255 /* 256 * Reserve absolutely required minimum. 257 * Right now we use separate EVQ for Rx and Tx. 258 */ 259 if (rxq_allocated > 0 && evq_allocated > 0) { 260 sa->rxq_max = 1; 261 rxq_allocated--; 262 evq_allocated--; 263 } 264 if (txq_allocated > 0 && evq_allocated > 0) { 265 sa->txq_max = 1; 266 txq_allocated--; 267 evq_allocated--; 268 } 269 270 if (sfc_mae_counter_rxq_required(sa) && 271 rxq_allocated > 0 && evq_allocated > 0) { 272 rxq_allocated--; 273 evq_allocated--; 274 sas->counters_rxq_allocated = true; 275 } else { 276 sas->counters_rxq_allocated = false; 277 } 278 279 /* Add remaining allocated queues */ 280 sa->rxq_max += MIN(rxq_allocated, evq_allocated / 2); 281 sa->txq_max += MIN(txq_allocated, evq_allocated - sa->rxq_max); 282 283 /* Keep NIC initialized */ 284 return 0; 285 286 fail_allocate_evq: 287 fail_get_vi_pool: 288 efx_nic_fini(sa->nic); 289 fail_nic_init: 290 return rc; 291 } 292 293 static int 294 sfc_set_drv_limits(struct sfc_adapter *sa) 295 { 296 const struct rte_eth_dev_data *data = sa->eth_dev->data; 297 uint32_t rxq_reserved = sfc_nb_reserved_rxq(sfc_sa2shared(sa)); 298 efx_drv_limits_t lim; 299 300 memset(&lim, 0, sizeof(lim)); 301 302 /* 303 * Limits are strict since take into account initial estimation. 304 * Resource allocation stategy is described in 305 * sfc_estimate_resource_limits(). 306 */ 307 lim.edl_min_evq_count = lim.edl_max_evq_count = 308 1 + data->nb_rx_queues + data->nb_tx_queues + rxq_reserved; 309 lim.edl_min_rxq_count = lim.edl_max_rxq_count = 310 data->nb_rx_queues + rxq_reserved; 311 lim.edl_min_txq_count = lim.edl_max_txq_count = data->nb_tx_queues; 312 313 return efx_nic_set_drv_limits(sa->nic, &lim); 314 } 315 316 static int 317 sfc_set_fw_subvariant(struct sfc_adapter *sa) 318 { 319 struct sfc_adapter_shared *sas = sfc_sa2shared(sa); 320 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic); 321 uint64_t tx_offloads = sa->eth_dev->data->dev_conf.txmode.offloads; 322 unsigned int txq_index; 323 efx_nic_fw_subvariant_t req_fw_subvariant; 324 efx_nic_fw_subvariant_t cur_fw_subvariant; 325 int rc; 326 327 if (!encp->enc_fw_subvariant_no_tx_csum_supported) { 328 sfc_info(sa, "no-Tx-checksum subvariant not supported"); 329 return 0; 330 } 331 332 for (txq_index = 0; txq_index < sas->txq_count; ++txq_index) { 333 struct sfc_txq_info *txq_info = &sas->txq_info[txq_index]; 334 335 if (txq_info->state & SFC_TXQ_INITIALIZED) 336 tx_offloads |= txq_info->offloads; 337 } 338 339 if (tx_offloads & (DEV_TX_OFFLOAD_IPV4_CKSUM | 340 DEV_TX_OFFLOAD_TCP_CKSUM | 341 DEV_TX_OFFLOAD_UDP_CKSUM | 342 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) 343 req_fw_subvariant = EFX_NIC_FW_SUBVARIANT_DEFAULT; 344 else 345 req_fw_subvariant = EFX_NIC_FW_SUBVARIANT_NO_TX_CSUM; 346 347 rc = efx_nic_get_fw_subvariant(sa->nic, &cur_fw_subvariant); 348 if (rc != 0) { 349 sfc_err(sa, "failed to get FW subvariant: %d", rc); 350 return rc; 351 } 352 sfc_info(sa, "FW subvariant is %u vs required %u", 353 cur_fw_subvariant, req_fw_subvariant); 354 355 if (cur_fw_subvariant == req_fw_subvariant) 356 return 0; 357 358 rc = efx_nic_set_fw_subvariant(sa->nic, req_fw_subvariant); 359 if (rc != 0) { 360 sfc_err(sa, "failed to set FW subvariant %u: %d", 361 req_fw_subvariant, rc); 362 return rc; 363 } 364 sfc_info(sa, "FW subvariant set to %u", req_fw_subvariant); 365 366 return 0; 367 } 368 369 static int 370 sfc_try_start(struct sfc_adapter *sa) 371 { 372 const efx_nic_cfg_t *encp; 373 int rc; 374 375 sfc_log_init(sa, "entry"); 376 377 SFC_ASSERT(sfc_adapter_is_locked(sa)); 378 SFC_ASSERT(sa->state == SFC_ADAPTER_STARTING); 379 380 sfc_log_init(sa, "set FW subvariant"); 381 rc = sfc_set_fw_subvariant(sa); 382 if (rc != 0) 383 goto fail_set_fw_subvariant; 384 385 sfc_log_init(sa, "set resource limits"); 386 rc = sfc_set_drv_limits(sa); 387 if (rc != 0) 388 goto fail_set_drv_limits; 389 390 sfc_log_init(sa, "init nic"); 391 rc = efx_nic_init(sa->nic); 392 if (rc != 0) 393 goto fail_nic_init; 394 395 encp = efx_nic_cfg_get(sa->nic); 396 397 /* 398 * Refresh (since it may change on NIC reset/restart) a copy of 399 * supported tunnel encapsulations in shared memory to be used 400 * on supported Rx packet type classes get. 401 */ 402 sa->priv.shared->tunnel_encaps = 403 encp->enc_tunnel_encapsulations_supported; 404 405 if (encp->enc_tunnel_encapsulations_supported != 0) { 406 sfc_log_init(sa, "apply tunnel config"); 407 rc = efx_tunnel_reconfigure(sa->nic); 408 if (rc != 0) 409 goto fail_tunnel_reconfigure; 410 } 411 412 rc = sfc_intr_start(sa); 413 if (rc != 0) 414 goto fail_intr_start; 415 416 rc = sfc_ev_start(sa); 417 if (rc != 0) 418 goto fail_ev_start; 419 420 rc = sfc_port_start(sa); 421 if (rc != 0) 422 goto fail_port_start; 423 424 rc = sfc_rx_start(sa); 425 if (rc != 0) 426 goto fail_rx_start; 427 428 rc = sfc_tx_start(sa); 429 if (rc != 0) 430 goto fail_tx_start; 431 432 rc = sfc_flow_start(sa); 433 if (rc != 0) 434 goto fail_flows_insert; 435 436 sfc_log_init(sa, "done"); 437 return 0; 438 439 fail_flows_insert: 440 sfc_tx_stop(sa); 441 442 fail_tx_start: 443 sfc_rx_stop(sa); 444 445 fail_rx_start: 446 sfc_port_stop(sa); 447 448 fail_port_start: 449 sfc_ev_stop(sa); 450 451 fail_ev_start: 452 sfc_intr_stop(sa); 453 454 fail_intr_start: 455 fail_tunnel_reconfigure: 456 efx_nic_fini(sa->nic); 457 458 fail_nic_init: 459 fail_set_drv_limits: 460 fail_set_fw_subvariant: 461 sfc_log_init(sa, "failed %d", rc); 462 return rc; 463 } 464 465 int 466 sfc_start(struct sfc_adapter *sa) 467 { 468 unsigned int start_tries = 3; 469 int rc; 470 471 sfc_log_init(sa, "entry"); 472 473 SFC_ASSERT(sfc_adapter_is_locked(sa)); 474 475 switch (sa->state) { 476 case SFC_ADAPTER_CONFIGURED: 477 break; 478 case SFC_ADAPTER_STARTED: 479 sfc_notice(sa, "already started"); 480 return 0; 481 default: 482 rc = EINVAL; 483 goto fail_bad_state; 484 } 485 486 sa->state = SFC_ADAPTER_STARTING; 487 488 rc = 0; 489 do { 490 /* 491 * FIXME Try to recreate vSwitch on start retry. 492 * vSwitch is absent after MC reboot like events and 493 * we should recreate it. May be we need proper 494 * indication instead of guessing. 495 */ 496 if (rc != 0) { 497 sfc_sriov_vswitch_destroy(sa); 498 rc = sfc_sriov_vswitch_create(sa); 499 if (rc != 0) 500 goto fail_sriov_vswitch_create; 501 } 502 rc = sfc_try_start(sa); 503 } while ((--start_tries > 0) && 504 (rc == EIO || rc == EAGAIN || rc == ENOENT || rc == EINVAL)); 505 506 if (rc != 0) 507 goto fail_try_start; 508 509 sa->state = SFC_ADAPTER_STARTED; 510 sfc_log_init(sa, "done"); 511 return 0; 512 513 fail_try_start: 514 fail_sriov_vswitch_create: 515 sa->state = SFC_ADAPTER_CONFIGURED; 516 fail_bad_state: 517 sfc_log_init(sa, "failed %d", rc); 518 return rc; 519 } 520 521 void 522 sfc_stop(struct sfc_adapter *sa) 523 { 524 sfc_log_init(sa, "entry"); 525 526 SFC_ASSERT(sfc_adapter_is_locked(sa)); 527 528 switch (sa->state) { 529 case SFC_ADAPTER_STARTED: 530 break; 531 case SFC_ADAPTER_CONFIGURED: 532 sfc_notice(sa, "already stopped"); 533 return; 534 default: 535 sfc_err(sa, "stop in unexpected state %u", sa->state); 536 SFC_ASSERT(B_FALSE); 537 return; 538 } 539 540 sa->state = SFC_ADAPTER_STOPPING; 541 542 sfc_flow_stop(sa); 543 sfc_tx_stop(sa); 544 sfc_rx_stop(sa); 545 sfc_port_stop(sa); 546 sfc_ev_stop(sa); 547 sfc_intr_stop(sa); 548 efx_nic_fini(sa->nic); 549 550 sa->state = SFC_ADAPTER_CONFIGURED; 551 sfc_log_init(sa, "done"); 552 } 553 554 static int 555 sfc_restart(struct sfc_adapter *sa) 556 { 557 int rc; 558 559 SFC_ASSERT(sfc_adapter_is_locked(sa)); 560 561 if (sa->state != SFC_ADAPTER_STARTED) 562 return EINVAL; 563 564 sfc_stop(sa); 565 566 rc = sfc_start(sa); 567 if (rc != 0) 568 sfc_err(sa, "restart failed"); 569 570 return rc; 571 } 572 573 static void 574 sfc_restart_if_required(void *arg) 575 { 576 struct sfc_adapter *sa = arg; 577 578 /* If restart is scheduled, clear the flag and do it */ 579 if (rte_atomic32_cmpset((volatile uint32_t *)&sa->restart_required, 580 1, 0)) { 581 sfc_adapter_lock(sa); 582 if (sa->state == SFC_ADAPTER_STARTED) 583 (void)sfc_restart(sa); 584 sfc_adapter_unlock(sa); 585 } 586 } 587 588 void 589 sfc_schedule_restart(struct sfc_adapter *sa) 590 { 591 int rc; 592 593 /* Schedule restart alarm if it is not scheduled yet */ 594 if (!rte_atomic32_test_and_set(&sa->restart_required)) 595 return; 596 597 rc = rte_eal_alarm_set(1, sfc_restart_if_required, sa); 598 if (rc == -ENOTSUP) 599 sfc_warn(sa, "alarms are not supported, restart is pending"); 600 else if (rc != 0) 601 sfc_err(sa, "cannot arm restart alarm (rc=%d)", rc); 602 else 603 sfc_notice(sa, "restart scheduled"); 604 } 605 606 int 607 sfc_configure(struct sfc_adapter *sa) 608 { 609 int rc; 610 611 sfc_log_init(sa, "entry"); 612 613 SFC_ASSERT(sfc_adapter_is_locked(sa)); 614 615 SFC_ASSERT(sa->state == SFC_ADAPTER_INITIALIZED || 616 sa->state == SFC_ADAPTER_CONFIGURED); 617 sa->state = SFC_ADAPTER_CONFIGURING; 618 619 rc = sfc_check_conf(sa); 620 if (rc != 0) 621 goto fail_check_conf; 622 623 rc = sfc_intr_configure(sa); 624 if (rc != 0) 625 goto fail_intr_configure; 626 627 rc = sfc_port_configure(sa); 628 if (rc != 0) 629 goto fail_port_configure; 630 631 rc = sfc_rx_configure(sa); 632 if (rc != 0) 633 goto fail_rx_configure; 634 635 rc = sfc_tx_configure(sa); 636 if (rc != 0) 637 goto fail_tx_configure; 638 639 sa->state = SFC_ADAPTER_CONFIGURED; 640 sfc_log_init(sa, "done"); 641 return 0; 642 643 fail_tx_configure: 644 sfc_rx_close(sa); 645 646 fail_rx_configure: 647 sfc_port_close(sa); 648 649 fail_port_configure: 650 sfc_intr_close(sa); 651 652 fail_intr_configure: 653 fail_check_conf: 654 sa->state = SFC_ADAPTER_INITIALIZED; 655 sfc_log_init(sa, "failed %d", rc); 656 return rc; 657 } 658 659 void 660 sfc_close(struct sfc_adapter *sa) 661 { 662 sfc_log_init(sa, "entry"); 663 664 SFC_ASSERT(sfc_adapter_is_locked(sa)); 665 666 SFC_ASSERT(sa->state == SFC_ADAPTER_CONFIGURED); 667 sa->state = SFC_ADAPTER_CLOSING; 668 669 sfc_tx_close(sa); 670 sfc_rx_close(sa); 671 sfc_port_close(sa); 672 sfc_intr_close(sa); 673 674 sa->state = SFC_ADAPTER_INITIALIZED; 675 sfc_log_init(sa, "done"); 676 } 677 678 static int 679 sfc_mem_bar_init(struct sfc_adapter *sa, const efx_bar_region_t *mem_ebrp) 680 { 681 struct rte_eth_dev *eth_dev = sa->eth_dev; 682 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 683 efsys_bar_t *ebp = &sa->mem_bar; 684 struct rte_mem_resource *res = 685 &pci_dev->mem_resource[mem_ebrp->ebr_index]; 686 687 SFC_BAR_LOCK_INIT(ebp, eth_dev->data->name); 688 ebp->esb_rid = mem_ebrp->ebr_index; 689 ebp->esb_dev = pci_dev; 690 ebp->esb_base = res->addr; 691 692 sa->fcw_offset = mem_ebrp->ebr_offset; 693 694 return 0; 695 } 696 697 static void 698 sfc_mem_bar_fini(struct sfc_adapter *sa) 699 { 700 efsys_bar_t *ebp = &sa->mem_bar; 701 702 SFC_BAR_LOCK_DESTROY(ebp); 703 memset(ebp, 0, sizeof(*ebp)); 704 } 705 706 /* 707 * A fixed RSS key which has a property of being symmetric 708 * (symmetrical flows are distributed to the same CPU) 709 * and also known to give a uniform distribution 710 * (a good distribution of traffic between different CPUs) 711 */ 712 static const uint8_t default_rss_key[EFX_RSS_KEY_SIZE] = { 713 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 714 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 715 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 716 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 717 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 718 }; 719 720 static int 721 sfc_rss_attach(struct sfc_adapter *sa) 722 { 723 struct sfc_rss *rss = &sfc_sa2shared(sa)->rss; 724 int rc; 725 726 rc = efx_intr_init(sa->nic, sa->intr.type, NULL); 727 if (rc != 0) 728 goto fail_intr_init; 729 730 rc = efx_ev_init(sa->nic); 731 if (rc != 0) 732 goto fail_ev_init; 733 734 rc = efx_rx_init(sa->nic); 735 if (rc != 0) 736 goto fail_rx_init; 737 738 rc = efx_rx_scale_default_support_get(sa->nic, &rss->context_type); 739 if (rc != 0) 740 goto fail_scale_support_get; 741 742 rc = efx_rx_hash_default_support_get(sa->nic, &rss->hash_support); 743 if (rc != 0) 744 goto fail_hash_support_get; 745 746 rc = sfc_rx_hash_init(sa); 747 if (rc != 0) 748 goto fail_rx_hash_init; 749 750 efx_rx_fini(sa->nic); 751 efx_ev_fini(sa->nic); 752 efx_intr_fini(sa->nic); 753 754 rte_memcpy(rss->key, default_rss_key, sizeof(rss->key)); 755 rss->dummy_rss_context = EFX_RSS_CONTEXT_DEFAULT; 756 757 return 0; 758 759 fail_rx_hash_init: 760 fail_hash_support_get: 761 fail_scale_support_get: 762 efx_rx_fini(sa->nic); 763 764 fail_rx_init: 765 efx_ev_fini(sa->nic); 766 767 fail_ev_init: 768 efx_intr_fini(sa->nic); 769 770 fail_intr_init: 771 return rc; 772 } 773 774 static void 775 sfc_rss_detach(struct sfc_adapter *sa) 776 { 777 sfc_rx_hash_fini(sa); 778 } 779 780 int 781 sfc_attach(struct sfc_adapter *sa) 782 { 783 const efx_nic_cfg_t *encp; 784 efx_nic_t *enp = sa->nic; 785 int rc; 786 787 sfc_log_init(sa, "entry"); 788 789 SFC_ASSERT(sfc_adapter_is_locked(sa)); 790 791 efx_mcdi_new_epoch(enp); 792 793 sfc_log_init(sa, "reset nic"); 794 rc = efx_nic_reset(enp); 795 if (rc != 0) 796 goto fail_nic_reset; 797 798 rc = sfc_sriov_attach(sa); 799 if (rc != 0) 800 goto fail_sriov_attach; 801 802 /* 803 * Probed NIC is sufficient for tunnel init. 804 * Initialize tunnel support to be able to use libefx 805 * efx_tunnel_config_udp_{add,remove}() in any state and 806 * efx_tunnel_reconfigure() on start up. 807 */ 808 rc = efx_tunnel_init(enp); 809 if (rc != 0) 810 goto fail_tunnel_init; 811 812 encp = efx_nic_cfg_get(sa->nic); 813 814 /* 815 * Make a copy of supported tunnel encapsulations in shared 816 * memory to be used on supported Rx packet type classes get. 817 */ 818 sa->priv.shared->tunnel_encaps = 819 encp->enc_tunnel_encapsulations_supported; 820 821 if (sfc_dp_tx_offload_capa(sa->priv.dp_tx) & DEV_TX_OFFLOAD_TCP_TSO) { 822 sa->tso = encp->enc_fw_assisted_tso_v2_enabled || 823 encp->enc_tso_v3_enabled; 824 if (!sa->tso) 825 sfc_info(sa, "TSO support isn't available on this adapter"); 826 } 827 828 if (sa->tso && 829 (sfc_dp_tx_offload_capa(sa->priv.dp_tx) & 830 (DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 831 DEV_TX_OFFLOAD_GENEVE_TNL_TSO)) != 0) { 832 sa->tso_encap = encp->enc_fw_assisted_tso_v2_encap_enabled || 833 encp->enc_tso_v3_enabled; 834 if (!sa->tso_encap) 835 sfc_info(sa, "Encapsulated TSO support isn't available on this adapter"); 836 } 837 838 sfc_log_init(sa, "estimate resource limits"); 839 rc = sfc_estimate_resource_limits(sa); 840 if (rc != 0) 841 goto fail_estimate_rsrc_limits; 842 843 sa->evq_max_entries = encp->enc_evq_max_nevs; 844 SFC_ASSERT(rte_is_power_of_2(sa->evq_max_entries)); 845 846 sa->evq_min_entries = encp->enc_evq_min_nevs; 847 SFC_ASSERT(rte_is_power_of_2(sa->evq_min_entries)); 848 849 sa->rxq_max_entries = encp->enc_rxq_max_ndescs; 850 SFC_ASSERT(rte_is_power_of_2(sa->rxq_max_entries)); 851 852 sa->rxq_min_entries = encp->enc_rxq_min_ndescs; 853 SFC_ASSERT(rte_is_power_of_2(sa->rxq_min_entries)); 854 855 sa->txq_max_entries = encp->enc_txq_max_ndescs; 856 SFC_ASSERT(rte_is_power_of_2(sa->txq_max_entries)); 857 858 sa->txq_min_entries = encp->enc_txq_min_ndescs; 859 SFC_ASSERT(rte_is_power_of_2(sa->txq_min_entries)); 860 861 rc = sfc_intr_attach(sa); 862 if (rc != 0) 863 goto fail_intr_attach; 864 865 rc = sfc_ev_attach(sa); 866 if (rc != 0) 867 goto fail_ev_attach; 868 869 rc = sfc_port_attach(sa); 870 if (rc != 0) 871 goto fail_port_attach; 872 873 rc = sfc_rss_attach(sa); 874 if (rc != 0) 875 goto fail_rss_attach; 876 877 rc = sfc_filter_attach(sa); 878 if (rc != 0) 879 goto fail_filter_attach; 880 881 rc = sfc_mae_counter_rxq_attach(sa); 882 if (rc != 0) 883 goto fail_mae_counter_rxq_attach; 884 885 rc = sfc_mae_attach(sa); 886 if (rc != 0) 887 goto fail_mae_attach; 888 889 sfc_log_init(sa, "fini nic"); 890 efx_nic_fini(enp); 891 892 sfc_flow_init(sa); 893 894 /* 895 * Create vSwitch to be able to use VFs when PF is not started yet 896 * as DPDK port. VFs should be able to talk to each other even 897 * if PF is down. 898 */ 899 rc = sfc_sriov_vswitch_create(sa); 900 if (rc != 0) 901 goto fail_sriov_vswitch_create; 902 903 sa->state = SFC_ADAPTER_INITIALIZED; 904 905 sfc_log_init(sa, "done"); 906 return 0; 907 908 fail_sriov_vswitch_create: 909 sfc_flow_fini(sa); 910 sfc_mae_detach(sa); 911 912 fail_mae_attach: 913 sfc_mae_counter_rxq_detach(sa); 914 915 fail_mae_counter_rxq_attach: 916 sfc_filter_detach(sa); 917 918 fail_filter_attach: 919 sfc_rss_detach(sa); 920 921 fail_rss_attach: 922 sfc_port_detach(sa); 923 924 fail_port_attach: 925 sfc_ev_detach(sa); 926 927 fail_ev_attach: 928 sfc_intr_detach(sa); 929 930 fail_intr_attach: 931 efx_nic_fini(sa->nic); 932 933 fail_estimate_rsrc_limits: 934 fail_tunnel_init: 935 efx_tunnel_fini(sa->nic); 936 sfc_sriov_detach(sa); 937 938 fail_sriov_attach: 939 fail_nic_reset: 940 941 sfc_log_init(sa, "failed %d", rc); 942 return rc; 943 } 944 945 void 946 sfc_detach(struct sfc_adapter *sa) 947 { 948 sfc_log_init(sa, "entry"); 949 950 SFC_ASSERT(sfc_adapter_is_locked(sa)); 951 952 sfc_sriov_vswitch_destroy(sa); 953 954 sfc_flow_fini(sa); 955 956 sfc_mae_detach(sa); 957 sfc_mae_counter_rxq_detach(sa); 958 sfc_filter_detach(sa); 959 sfc_rss_detach(sa); 960 sfc_port_detach(sa); 961 sfc_ev_detach(sa); 962 sfc_intr_detach(sa); 963 efx_tunnel_fini(sa->nic); 964 sfc_sriov_detach(sa); 965 966 sa->state = SFC_ADAPTER_UNINITIALIZED; 967 } 968 969 static int 970 sfc_kvarg_fv_variant_handler(__rte_unused const char *key, 971 const char *value_str, void *opaque) 972 { 973 uint32_t *value = opaque; 974 975 if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_DONT_CARE) == 0) 976 *value = EFX_FW_VARIANT_DONT_CARE; 977 else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_FULL_FEATURED) == 0) 978 *value = EFX_FW_VARIANT_FULL_FEATURED; 979 else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_LOW_LATENCY) == 0) 980 *value = EFX_FW_VARIANT_LOW_LATENCY; 981 else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_PACKED_STREAM) == 0) 982 *value = EFX_FW_VARIANT_PACKED_STREAM; 983 else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_DPDK) == 0) 984 *value = EFX_FW_VARIANT_DPDK; 985 else 986 return -EINVAL; 987 988 return 0; 989 } 990 991 static int 992 sfc_get_fw_variant(struct sfc_adapter *sa, efx_fw_variant_t *efv) 993 { 994 efx_nic_fw_info_t enfi; 995 int rc; 996 997 rc = efx_nic_get_fw_version(sa->nic, &enfi); 998 if (rc != 0) 999 return rc; 1000 else if (!enfi.enfi_dpcpu_fw_ids_valid) 1001 return ENOTSUP; 1002 1003 /* 1004 * Firmware variant can be uniquely identified by the RxDPCPU 1005 * firmware id 1006 */ 1007 switch (enfi.enfi_rx_dpcpu_fw_id) { 1008 case EFX_RXDP_FULL_FEATURED_FW_ID: 1009 *efv = EFX_FW_VARIANT_FULL_FEATURED; 1010 break; 1011 1012 case EFX_RXDP_LOW_LATENCY_FW_ID: 1013 *efv = EFX_FW_VARIANT_LOW_LATENCY; 1014 break; 1015 1016 case EFX_RXDP_PACKED_STREAM_FW_ID: 1017 *efv = EFX_FW_VARIANT_PACKED_STREAM; 1018 break; 1019 1020 case EFX_RXDP_DPDK_FW_ID: 1021 *efv = EFX_FW_VARIANT_DPDK; 1022 break; 1023 1024 default: 1025 /* 1026 * Other firmware variants are not considered, since they are 1027 * not supported in the device parameters 1028 */ 1029 *efv = EFX_FW_VARIANT_DONT_CARE; 1030 break; 1031 } 1032 1033 return 0; 1034 } 1035 1036 static const char * 1037 sfc_fw_variant2str(efx_fw_variant_t efv) 1038 { 1039 switch (efv) { 1040 case EFX_RXDP_FULL_FEATURED_FW_ID: 1041 return SFC_KVARG_FW_VARIANT_FULL_FEATURED; 1042 case EFX_RXDP_LOW_LATENCY_FW_ID: 1043 return SFC_KVARG_FW_VARIANT_LOW_LATENCY; 1044 case EFX_RXDP_PACKED_STREAM_FW_ID: 1045 return SFC_KVARG_FW_VARIANT_PACKED_STREAM; 1046 case EFX_RXDP_DPDK_FW_ID: 1047 return SFC_KVARG_FW_VARIANT_DPDK; 1048 default: 1049 return "unknown"; 1050 } 1051 } 1052 1053 static int 1054 sfc_kvarg_rxd_wait_timeout_ns(struct sfc_adapter *sa) 1055 { 1056 int rc; 1057 long value; 1058 1059 value = SFC_RXD_WAIT_TIMEOUT_NS_DEF; 1060 1061 rc = sfc_kvargs_process(sa, SFC_KVARG_RXD_WAIT_TIMEOUT_NS, 1062 sfc_kvarg_long_handler, &value); 1063 if (rc != 0) 1064 return rc; 1065 1066 if (value < 0 || 1067 (unsigned long)value > EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX) { 1068 sfc_err(sa, "wrong '" SFC_KVARG_RXD_WAIT_TIMEOUT_NS "' " 1069 "was set (%ld);", value); 1070 sfc_err(sa, "it must not be less than 0 or greater than %u", 1071 EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX); 1072 return EINVAL; 1073 } 1074 1075 sa->rxd_wait_timeout_ns = value; 1076 return 0; 1077 } 1078 1079 static int 1080 sfc_nic_probe(struct sfc_adapter *sa) 1081 { 1082 efx_nic_t *enp = sa->nic; 1083 efx_fw_variant_t preferred_efv; 1084 efx_fw_variant_t efv; 1085 int rc; 1086 1087 preferred_efv = EFX_FW_VARIANT_DONT_CARE; 1088 rc = sfc_kvargs_process(sa, SFC_KVARG_FW_VARIANT, 1089 sfc_kvarg_fv_variant_handler, 1090 &preferred_efv); 1091 if (rc != 0) { 1092 sfc_err(sa, "invalid %s parameter value", SFC_KVARG_FW_VARIANT); 1093 return rc; 1094 } 1095 1096 rc = sfc_kvarg_rxd_wait_timeout_ns(sa); 1097 if (rc != 0) 1098 return rc; 1099 1100 rc = efx_nic_probe(enp, preferred_efv); 1101 if (rc == EACCES) { 1102 /* Unprivileged functions cannot set FW variant */ 1103 rc = efx_nic_probe(enp, EFX_FW_VARIANT_DONT_CARE); 1104 } 1105 if (rc != 0) 1106 return rc; 1107 1108 rc = sfc_get_fw_variant(sa, &efv); 1109 if (rc == ENOTSUP) { 1110 sfc_warn(sa, "FW variant can not be obtained"); 1111 return 0; 1112 } 1113 if (rc != 0) 1114 return rc; 1115 1116 /* Check that firmware variant was changed to the requested one */ 1117 if (preferred_efv != EFX_FW_VARIANT_DONT_CARE && preferred_efv != efv) { 1118 sfc_warn(sa, "FW variant has not changed to the requested %s", 1119 sfc_fw_variant2str(preferred_efv)); 1120 } 1121 1122 sfc_notice(sa, "running FW variant is %s", sfc_fw_variant2str(efv)); 1123 1124 return 0; 1125 } 1126 1127 int 1128 sfc_probe(struct sfc_adapter *sa) 1129 { 1130 efx_bar_region_t mem_ebrp; 1131 struct rte_eth_dev *eth_dev = sa->eth_dev; 1132 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1133 efx_nic_t *enp; 1134 int rc; 1135 1136 sfc_log_init(sa, "entry"); 1137 1138 SFC_ASSERT(sfc_adapter_is_locked(sa)); 1139 1140 sa->socket_id = rte_socket_id(); 1141 rte_atomic32_init(&sa->restart_required); 1142 1143 sfc_log_init(sa, "get family"); 1144 rc = sfc_efx_family(pci_dev, &mem_ebrp, &sa->family); 1145 1146 if (rc != 0) 1147 goto fail_family; 1148 sfc_log_init(sa, 1149 "family is %u, membar is %u, function control window offset is %lu", 1150 sa->family, mem_ebrp.ebr_index, mem_ebrp.ebr_offset); 1151 1152 sfc_log_init(sa, "init mem bar"); 1153 rc = sfc_mem_bar_init(sa, &mem_ebrp); 1154 if (rc != 0) 1155 goto fail_mem_bar_init; 1156 1157 sfc_log_init(sa, "create nic"); 1158 rte_spinlock_init(&sa->nic_lock); 1159 rc = efx_nic_create(sa->family, (efsys_identifier_t *)sa, 1160 &sa->mem_bar, mem_ebrp.ebr_offset, 1161 &sa->nic_lock, &enp); 1162 if (rc != 0) 1163 goto fail_nic_create; 1164 sa->nic = enp; 1165 1166 rc = sfc_mcdi_init(sa); 1167 if (rc != 0) 1168 goto fail_mcdi_init; 1169 1170 sfc_log_init(sa, "probe nic"); 1171 rc = sfc_nic_probe(sa); 1172 if (rc != 0) 1173 goto fail_nic_probe; 1174 1175 sfc_log_init(sa, "done"); 1176 return 0; 1177 1178 fail_nic_probe: 1179 sfc_mcdi_fini(sa); 1180 1181 fail_mcdi_init: 1182 sfc_log_init(sa, "destroy nic"); 1183 sa->nic = NULL; 1184 efx_nic_destroy(enp); 1185 1186 fail_nic_create: 1187 sfc_mem_bar_fini(sa); 1188 1189 fail_mem_bar_init: 1190 fail_family: 1191 sfc_log_init(sa, "failed %d", rc); 1192 return rc; 1193 } 1194 1195 void 1196 sfc_unprobe(struct sfc_adapter *sa) 1197 { 1198 efx_nic_t *enp = sa->nic; 1199 1200 sfc_log_init(sa, "entry"); 1201 1202 SFC_ASSERT(sfc_adapter_is_locked(sa)); 1203 1204 sfc_log_init(sa, "unprobe nic"); 1205 efx_nic_unprobe(enp); 1206 1207 sfc_mcdi_fini(sa); 1208 1209 /* 1210 * Make sure there is no pending alarm to restart since we are 1211 * going to free device private which is passed as the callback 1212 * opaque data. A new alarm cannot be scheduled since MCDI is 1213 * shut down. 1214 */ 1215 rte_eal_alarm_cancel(sfc_restart_if_required, sa); 1216 1217 sfc_log_init(sa, "destroy nic"); 1218 sa->nic = NULL; 1219 efx_nic_destroy(enp); 1220 1221 sfc_mem_bar_fini(sa); 1222 1223 sfc_flow_fini(sa); 1224 sa->state = SFC_ADAPTER_UNINITIALIZED; 1225 } 1226 1227 uint32_t 1228 sfc_register_logtype(const struct rte_pci_addr *pci_addr, 1229 const char *lt_prefix_str, uint32_t ll_default) 1230 { 1231 size_t lt_prefix_str_size = strlen(lt_prefix_str); 1232 size_t lt_str_size_max; 1233 char *lt_str = NULL; 1234 int ret; 1235 1236 if (SIZE_MAX - PCI_PRI_STR_SIZE - 1 > lt_prefix_str_size) { 1237 ++lt_prefix_str_size; /* Reserve space for prefix separator */ 1238 lt_str_size_max = lt_prefix_str_size + PCI_PRI_STR_SIZE + 1; 1239 } else { 1240 return sfc_logtype_driver; 1241 } 1242 1243 lt_str = rte_zmalloc("logtype_str", lt_str_size_max, 0); 1244 if (lt_str == NULL) 1245 return sfc_logtype_driver; 1246 1247 strncpy(lt_str, lt_prefix_str, lt_prefix_str_size); 1248 lt_str[lt_prefix_str_size - 1] = '.'; 1249 rte_pci_device_name(pci_addr, lt_str + lt_prefix_str_size, 1250 lt_str_size_max - lt_prefix_str_size); 1251 lt_str[lt_str_size_max - 1] = '\0'; 1252 1253 ret = rte_log_register_type_and_pick_level(lt_str, ll_default); 1254 rte_free(lt_str); 1255 1256 if (ret < 0) 1257 return sfc_logtype_driver; 1258 1259 return ret; 1260 } 1261 1262 struct sfc_hw_switch_id { 1263 char board_sn[RTE_SIZEOF_FIELD(efx_nic_board_info_t, enbi_serial)]; 1264 }; 1265 1266 int 1267 sfc_hw_switch_id_init(struct sfc_adapter *sa, 1268 struct sfc_hw_switch_id **idp) 1269 { 1270 efx_nic_board_info_t board_info; 1271 struct sfc_hw_switch_id *id; 1272 int rc; 1273 1274 if (idp == NULL) 1275 return EINVAL; 1276 1277 id = rte_zmalloc("sfc_hw_switch_id", sizeof(*id), 0); 1278 if (id == NULL) 1279 return ENOMEM; 1280 1281 rc = efx_nic_get_board_info(sa->nic, &board_info); 1282 if (rc != 0) 1283 return rc; 1284 1285 memcpy(id->board_sn, board_info.enbi_serial, sizeof(id->board_sn)); 1286 1287 *idp = id; 1288 1289 return 0; 1290 } 1291 1292 void 1293 sfc_hw_switch_id_fini(__rte_unused struct sfc_adapter *sa, 1294 struct sfc_hw_switch_id *id) 1295 { 1296 rte_free(id); 1297 } 1298 1299 bool 1300 sfc_hw_switch_ids_equal(const struct sfc_hw_switch_id *left, 1301 const struct sfc_hw_switch_id *right) 1302 { 1303 return strncmp(left->board_sn, right->board_sn, 1304 sizeof(left->board_sn)) == 0; 1305 } 1306