1 /* 2 * Copyright (c) 2014, 2015 Netronome Systems, Inc. 3 * All rights reserved. 4 * 5 * Small portions derived from code Copyright(c) 2010-2015 Intel Corporation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution 16 * 17 * 3. Neither the name of the copyright holder nor the names of its 18 * contributors may be used to endorse or promote products derived from this 19 * software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * vim:shiftwidth=8:noexpandtab 36 * 37 * @file dpdk/pmd/nfp_net.c 38 * 39 * Netronome vNIC DPDK Poll-Mode Driver: Main entry point 40 */ 41 42 #include <rte_byteorder.h> 43 #include <rte_common.h> 44 #include <rte_log.h> 45 #include <rte_debug.h> 46 #include <rte_ethdev.h> 47 #include <rte_ethdev_pci.h> 48 #include <rte_dev.h> 49 #include <rte_ether.h> 50 #include <rte_malloc.h> 51 #include <rte_memzone.h> 52 #include <rte_mempool.h> 53 #include <rte_version.h> 54 #include <rte_string_fns.h> 55 #include <rte_alarm.h> 56 #include <rte_spinlock.h> 57 58 #include "nfp_nfpu.h" 59 #include "nfp_net_pmd.h" 60 #include "nfp_net_logs.h" 61 #include "nfp_net_ctrl.h" 62 63 /* Prototypes */ 64 static void nfp_net_close(struct rte_eth_dev *dev); 65 static int nfp_net_configure(struct rte_eth_dev *dev); 66 static void nfp_net_dev_interrupt_handler(void *param); 67 static void nfp_net_dev_interrupt_delayed_handler(void *param); 68 static int nfp_net_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 69 static void nfp_net_infos_get(struct rte_eth_dev *dev, 70 struct rte_eth_dev_info *dev_info); 71 static int nfp_net_init(struct rte_eth_dev *eth_dev); 72 static int nfp_net_link_update(struct rte_eth_dev *dev, int wait_to_complete); 73 static void nfp_net_promisc_enable(struct rte_eth_dev *dev); 74 static void nfp_net_promisc_disable(struct rte_eth_dev *dev); 75 static int nfp_net_rx_fill_freelist(struct nfp_net_rxq *rxq); 76 static uint32_t nfp_net_rx_queue_count(struct rte_eth_dev *dev, 77 uint16_t queue_idx); 78 static uint16_t nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 79 uint16_t nb_pkts); 80 static void nfp_net_rx_queue_release(void *rxq); 81 static int nfp_net_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 82 uint16_t nb_desc, unsigned int socket_id, 83 const struct rte_eth_rxconf *rx_conf, 84 struct rte_mempool *mp); 85 static int nfp_net_tx_free_bufs(struct nfp_net_txq *txq); 86 static void nfp_net_tx_queue_release(void *txq); 87 static int nfp_net_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 88 uint16_t nb_desc, unsigned int socket_id, 89 const struct rte_eth_txconf *tx_conf); 90 static int nfp_net_start(struct rte_eth_dev *dev); 91 static int nfp_net_stats_get(struct rte_eth_dev *dev, 92 struct rte_eth_stats *stats); 93 static void nfp_net_stats_reset(struct rte_eth_dev *dev); 94 static void nfp_net_stop(struct rte_eth_dev *dev); 95 static uint16_t nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 96 uint16_t nb_pkts); 97 98 /* 99 * The offset of the queue controller queues in the PCIe Target. These 100 * happen to be at the same offset on the NFP6000 and the NFP3200 so 101 * we use a single macro here. 102 */ 103 #define NFP_PCIE_QUEUE(_q) (0x800 * ((_q) & 0xff)) 104 105 /* Maximum value which can be added to a queue with one transaction */ 106 #define NFP_QCP_MAX_ADD 0x7f 107 108 #define RTE_MBUF_DMA_ADDR_DEFAULT(mb) \ 109 (uint64_t)((mb)->buf_iova + RTE_PKTMBUF_HEADROOM) 110 111 /* nfp_qcp_ptr - Read or Write Pointer of a queue */ 112 enum nfp_qcp_ptr { 113 NFP_QCP_READ_PTR = 0, 114 NFP_QCP_WRITE_PTR 115 }; 116 117 /* 118 * nfp_qcp_ptr_add - Add the value to the selected pointer of a queue 119 * @q: Base address for queue structure 120 * @ptr: Add to the Read or Write pointer 121 * @val: Value to add to the queue pointer 122 * 123 * If @val is greater than @NFP_QCP_MAX_ADD multiple writes are performed. 124 */ 125 static inline void 126 nfp_qcp_ptr_add(uint8_t *q, enum nfp_qcp_ptr ptr, uint32_t val) 127 { 128 uint32_t off; 129 130 if (ptr == NFP_QCP_READ_PTR) 131 off = NFP_QCP_QUEUE_ADD_RPTR; 132 else 133 off = NFP_QCP_QUEUE_ADD_WPTR; 134 135 while (val > NFP_QCP_MAX_ADD) { 136 nn_writel(rte_cpu_to_le_32(NFP_QCP_MAX_ADD), q + off); 137 val -= NFP_QCP_MAX_ADD; 138 } 139 140 nn_writel(rte_cpu_to_le_32(val), q + off); 141 } 142 143 /* 144 * nfp_qcp_read - Read the current Read/Write pointer value for a queue 145 * @q: Base address for queue structure 146 * @ptr: Read or Write pointer 147 */ 148 static inline uint32_t 149 nfp_qcp_read(uint8_t *q, enum nfp_qcp_ptr ptr) 150 { 151 uint32_t off; 152 uint32_t val; 153 154 if (ptr == NFP_QCP_READ_PTR) 155 off = NFP_QCP_QUEUE_STS_LO; 156 else 157 off = NFP_QCP_QUEUE_STS_HI; 158 159 val = rte_cpu_to_le_32(nn_readl(q + off)); 160 161 if (ptr == NFP_QCP_READ_PTR) 162 return val & NFP_QCP_QUEUE_STS_LO_READPTR_mask; 163 else 164 return val & NFP_QCP_QUEUE_STS_HI_WRITEPTR_mask; 165 } 166 167 /* 168 * Functions to read/write from/to Config BAR 169 * Performs any endian conversion necessary. 170 */ 171 static inline uint8_t 172 nn_cfg_readb(struct nfp_net_hw *hw, int off) 173 { 174 return nn_readb(hw->ctrl_bar + off); 175 } 176 177 static inline void 178 nn_cfg_writeb(struct nfp_net_hw *hw, int off, uint8_t val) 179 { 180 nn_writeb(val, hw->ctrl_bar + off); 181 } 182 183 static inline uint32_t 184 nn_cfg_readl(struct nfp_net_hw *hw, int off) 185 { 186 return rte_le_to_cpu_32(nn_readl(hw->ctrl_bar + off)); 187 } 188 189 static inline void 190 nn_cfg_writel(struct nfp_net_hw *hw, int off, uint32_t val) 191 { 192 nn_writel(rte_cpu_to_le_32(val), hw->ctrl_bar + off); 193 } 194 195 static inline uint64_t 196 nn_cfg_readq(struct nfp_net_hw *hw, int off) 197 { 198 return rte_le_to_cpu_64(nn_readq(hw->ctrl_bar + off)); 199 } 200 201 static inline void 202 nn_cfg_writeq(struct nfp_net_hw *hw, int off, uint64_t val) 203 { 204 nn_writeq(rte_cpu_to_le_64(val), hw->ctrl_bar + off); 205 } 206 207 /* 208 * Atomically reads link status information from global structure rte_eth_dev. 209 * 210 * @param dev 211 * - Pointer to the structure rte_eth_dev to read from. 212 * - Pointer to the buffer to be saved with the link status. 213 * 214 * @return 215 * - On success, zero. 216 * - On failure, negative value. 217 */ 218 static inline int 219 nfp_net_dev_atomic_read_link_status(struct rte_eth_dev *dev, 220 struct rte_eth_link *link) 221 { 222 struct rte_eth_link *dst = link; 223 struct rte_eth_link *src = &dev->data->dev_link; 224 225 if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, 226 *(uint64_t *)src) == 0) 227 return -1; 228 229 return 0; 230 } 231 232 /* 233 * Atomically writes the link status information into global 234 * structure rte_eth_dev. 235 * 236 * @param dev 237 * - Pointer to the structure rte_eth_dev to read from. 238 * - Pointer to the buffer to be saved with the link status. 239 * 240 * @return 241 * - On success, zero. 242 * - On failure, negative value. 243 */ 244 static inline int 245 nfp_net_dev_atomic_write_link_status(struct rte_eth_dev *dev, 246 struct rte_eth_link *link) 247 { 248 struct rte_eth_link *dst = &dev->data->dev_link; 249 struct rte_eth_link *src = link; 250 251 if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, 252 *(uint64_t *)src) == 0) 253 return -1; 254 255 return 0; 256 } 257 258 static void 259 nfp_net_rx_queue_release_mbufs(struct nfp_net_rxq *rxq) 260 { 261 unsigned i; 262 263 if (rxq->rxbufs == NULL) 264 return; 265 266 for (i = 0; i < rxq->rx_count; i++) { 267 if (rxq->rxbufs[i].mbuf) { 268 rte_pktmbuf_free_seg(rxq->rxbufs[i].mbuf); 269 rxq->rxbufs[i].mbuf = NULL; 270 } 271 } 272 } 273 274 static void 275 nfp_net_rx_queue_release(void *rx_queue) 276 { 277 struct nfp_net_rxq *rxq = rx_queue; 278 279 if (rxq) { 280 nfp_net_rx_queue_release_mbufs(rxq); 281 rte_free(rxq->rxbufs); 282 rte_free(rxq); 283 } 284 } 285 286 static void 287 nfp_net_reset_rx_queue(struct nfp_net_rxq *rxq) 288 { 289 nfp_net_rx_queue_release_mbufs(rxq); 290 rxq->rd_p = 0; 291 rxq->nb_rx_hold = 0; 292 } 293 294 static void 295 nfp_net_tx_queue_release_mbufs(struct nfp_net_txq *txq) 296 { 297 unsigned i; 298 299 if (txq->txbufs == NULL) 300 return; 301 302 for (i = 0; i < txq->tx_count; i++) { 303 if (txq->txbufs[i].mbuf) { 304 rte_pktmbuf_free_seg(txq->txbufs[i].mbuf); 305 txq->txbufs[i].mbuf = NULL; 306 } 307 } 308 } 309 310 static void 311 nfp_net_tx_queue_release(void *tx_queue) 312 { 313 struct nfp_net_txq *txq = tx_queue; 314 315 if (txq) { 316 nfp_net_tx_queue_release_mbufs(txq); 317 rte_free(txq->txbufs); 318 rte_free(txq); 319 } 320 } 321 322 static void 323 nfp_net_reset_tx_queue(struct nfp_net_txq *txq) 324 { 325 nfp_net_tx_queue_release_mbufs(txq); 326 txq->wr_p = 0; 327 txq->rd_p = 0; 328 } 329 330 static int 331 __nfp_net_reconfig(struct nfp_net_hw *hw, uint32_t update) 332 { 333 int cnt; 334 uint32_t new; 335 struct timespec wait; 336 337 PMD_DRV_LOG(DEBUG, "Writing to the configuration queue (%p)...\n", 338 hw->qcp_cfg); 339 340 if (hw->qcp_cfg == NULL) 341 rte_panic("Bad configuration queue pointer\n"); 342 343 nfp_qcp_ptr_add(hw->qcp_cfg, NFP_QCP_WRITE_PTR, 1); 344 345 wait.tv_sec = 0; 346 wait.tv_nsec = 1000000; 347 348 PMD_DRV_LOG(DEBUG, "Polling for update ack...\n"); 349 350 /* Poll update field, waiting for NFP to ack the config */ 351 for (cnt = 0; ; cnt++) { 352 new = nn_cfg_readl(hw, NFP_NET_CFG_UPDATE); 353 if (new == 0) 354 break; 355 if (new & NFP_NET_CFG_UPDATE_ERR) { 356 PMD_INIT_LOG(ERR, "Reconfig error: 0x%08x", new); 357 return -1; 358 } 359 if (cnt >= NFP_NET_POLL_TIMEOUT) { 360 PMD_INIT_LOG(ERR, "Reconfig timeout for 0x%08x after" 361 " %dms", update, cnt); 362 rte_panic("Exiting\n"); 363 } 364 nanosleep(&wait, 0); /* waiting for a 1ms */ 365 } 366 PMD_DRV_LOG(DEBUG, "Ack DONE\n"); 367 return 0; 368 } 369 370 /* 371 * Reconfigure the NIC 372 * @nn: device to reconfigure 373 * @ctrl: The value for the ctrl field in the BAR config 374 * @update: The value for the update field in the BAR config 375 * 376 * Write the update word to the BAR and ping the reconfig queue. Then poll 377 * until the firmware has acknowledged the update by zeroing the update word. 378 */ 379 static int 380 nfp_net_reconfig(struct nfp_net_hw *hw, uint32_t ctrl, uint32_t update) 381 { 382 uint32_t err; 383 384 PMD_DRV_LOG(DEBUG, "nfp_net_reconfig: ctrl=%08x update=%08x\n", 385 ctrl, update); 386 387 rte_spinlock_lock(&hw->reconfig_lock); 388 389 nn_cfg_writel(hw, NFP_NET_CFG_CTRL, ctrl); 390 nn_cfg_writel(hw, NFP_NET_CFG_UPDATE, update); 391 392 rte_wmb(); 393 394 err = __nfp_net_reconfig(hw, update); 395 396 rte_spinlock_unlock(&hw->reconfig_lock); 397 398 if (!err) 399 return 0; 400 401 /* 402 * Reconfig errors imply situations where they can be handled. 403 * Otherwise, rte_panic is called inside __nfp_net_reconfig 404 */ 405 PMD_INIT_LOG(ERR, "Error nfp_net reconfig for ctrl: %x update: %x", 406 ctrl, update); 407 return -EIO; 408 } 409 410 /* 411 * Configure an Ethernet device. This function must be invoked first 412 * before any other function in the Ethernet API. This function can 413 * also be re-invoked when a device is in the stopped state. 414 */ 415 static int 416 nfp_net_configure(struct rte_eth_dev *dev) 417 { 418 struct rte_eth_conf *dev_conf; 419 struct rte_eth_rxmode *rxmode; 420 struct rte_eth_txmode *txmode; 421 uint32_t new_ctrl = 0; 422 uint32_t update = 0; 423 struct nfp_net_hw *hw; 424 425 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 426 427 /* 428 * A DPDK app sends info about how many queues to use and how 429 * those queues need to be configured. This is used by the 430 * DPDK core and it makes sure no more queues than those 431 * advertised by the driver are requested. This function is 432 * called after that internal process 433 */ 434 435 PMD_INIT_LOG(DEBUG, "Configure"); 436 437 dev_conf = &dev->data->dev_conf; 438 rxmode = &dev_conf->rxmode; 439 txmode = &dev_conf->txmode; 440 441 /* Checking TX mode */ 442 if (txmode->mq_mode) { 443 PMD_INIT_LOG(INFO, "TX mq_mode DCB and VMDq not supported"); 444 return -EINVAL; 445 } 446 447 /* Checking RX mode */ 448 if (rxmode->mq_mode & ETH_MQ_RX_RSS) { 449 if (hw->cap & NFP_NET_CFG_CTRL_RSS) { 450 update = NFP_NET_CFG_UPDATE_RSS; 451 new_ctrl = NFP_NET_CFG_CTRL_RSS; 452 } else { 453 PMD_INIT_LOG(INFO, "RSS not supported"); 454 return -EINVAL; 455 } 456 } 457 458 if (rxmode->split_hdr_size) { 459 PMD_INIT_LOG(INFO, "rxmode does not support split header"); 460 return -EINVAL; 461 } 462 463 if (rxmode->hw_ip_checksum) { 464 if (hw->cap & NFP_NET_CFG_CTRL_RXCSUM) { 465 new_ctrl |= NFP_NET_CFG_CTRL_RXCSUM; 466 } else { 467 PMD_INIT_LOG(INFO, "RXCSUM not supported"); 468 return -EINVAL; 469 } 470 } 471 472 if (rxmode->hw_vlan_filter) { 473 PMD_INIT_LOG(INFO, "VLAN filter not supported"); 474 return -EINVAL; 475 } 476 477 if (rxmode->hw_vlan_strip) { 478 if (hw->cap & NFP_NET_CFG_CTRL_RXVLAN) { 479 new_ctrl |= NFP_NET_CFG_CTRL_RXVLAN; 480 } else { 481 PMD_INIT_LOG(INFO, "hw vlan strip not supported"); 482 return -EINVAL; 483 } 484 } 485 486 if (rxmode->hw_vlan_extend) { 487 PMD_INIT_LOG(INFO, "VLAN extended not supported"); 488 return -EINVAL; 489 } 490 491 if (rxmode->jumbo_frame) 492 hw->mtu = rxmode->max_rx_pkt_len; 493 494 if (!rxmode->hw_strip_crc) 495 PMD_INIT_LOG(INFO, "HW does strip CRC and it is not configurable"); 496 497 if (rxmode->enable_scatter) { 498 PMD_INIT_LOG(INFO, "Scatter not supported"); 499 return -EINVAL; 500 } 501 502 /* If next capabilities are supported, configure them by default */ 503 504 /* VLAN insertion */ 505 if (hw->cap & NFP_NET_CFG_CTRL_TXVLAN) 506 new_ctrl |= NFP_NET_CFG_CTRL_TXVLAN; 507 508 /* L2 broadcast */ 509 if (hw->cap & NFP_NET_CFG_CTRL_L2BC) 510 new_ctrl |= NFP_NET_CFG_CTRL_L2BC; 511 512 /* L2 multicast */ 513 if (hw->cap & NFP_NET_CFG_CTRL_L2MC) 514 new_ctrl |= NFP_NET_CFG_CTRL_L2MC; 515 516 /* TX checksum offload */ 517 if (hw->cap & NFP_NET_CFG_CTRL_TXCSUM) 518 new_ctrl |= NFP_NET_CFG_CTRL_TXCSUM; 519 520 /* LSO offload */ 521 if (hw->cap & NFP_NET_CFG_CTRL_LSO) 522 new_ctrl |= NFP_NET_CFG_CTRL_LSO; 523 524 /* RX gather */ 525 if (hw->cap & NFP_NET_CFG_CTRL_GATHER) 526 new_ctrl |= NFP_NET_CFG_CTRL_GATHER; 527 528 if (!new_ctrl) 529 return 0; 530 531 update |= NFP_NET_CFG_UPDATE_GEN; 532 533 nn_cfg_writel(hw, NFP_NET_CFG_CTRL, new_ctrl); 534 if (nfp_net_reconfig(hw, new_ctrl, update) < 0) 535 return -EIO; 536 537 hw->ctrl = new_ctrl; 538 539 return 0; 540 } 541 542 static void 543 nfp_net_enable_queues(struct rte_eth_dev *dev) 544 { 545 struct nfp_net_hw *hw; 546 uint64_t enabled_queues = 0; 547 int i; 548 549 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 550 551 /* Enabling the required TX queues in the device */ 552 for (i = 0; i < dev->data->nb_tx_queues; i++) 553 enabled_queues |= (1 << i); 554 555 nn_cfg_writeq(hw, NFP_NET_CFG_TXRS_ENABLE, enabled_queues); 556 557 enabled_queues = 0; 558 559 /* Enabling the required RX queues in the device */ 560 for (i = 0; i < dev->data->nb_rx_queues; i++) 561 enabled_queues |= (1 << i); 562 563 nn_cfg_writeq(hw, NFP_NET_CFG_RXRS_ENABLE, enabled_queues); 564 } 565 566 static void 567 nfp_net_disable_queues(struct rte_eth_dev *dev) 568 { 569 struct nfp_net_hw *hw; 570 uint32_t new_ctrl, update = 0; 571 572 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 573 574 nn_cfg_writeq(hw, NFP_NET_CFG_TXRS_ENABLE, 0); 575 nn_cfg_writeq(hw, NFP_NET_CFG_RXRS_ENABLE, 0); 576 577 new_ctrl = hw->ctrl & ~NFP_NET_CFG_CTRL_ENABLE; 578 update = NFP_NET_CFG_UPDATE_GEN | NFP_NET_CFG_UPDATE_RING | 579 NFP_NET_CFG_UPDATE_MSIX; 580 581 if (hw->cap & NFP_NET_CFG_CTRL_RINGCFG) 582 new_ctrl &= ~NFP_NET_CFG_CTRL_RINGCFG; 583 584 /* If an error when reconfig we avoid to change hw state */ 585 if (nfp_net_reconfig(hw, new_ctrl, update) < 0) 586 return; 587 588 hw->ctrl = new_ctrl; 589 } 590 591 static int 592 nfp_net_rx_freelist_setup(struct rte_eth_dev *dev) 593 { 594 int i; 595 596 for (i = 0; i < dev->data->nb_rx_queues; i++) { 597 if (nfp_net_rx_fill_freelist(dev->data->rx_queues[i]) < 0) 598 return -1; 599 } 600 return 0; 601 } 602 603 static void 604 nfp_net_params_setup(struct nfp_net_hw *hw) 605 { 606 nn_cfg_writel(hw, NFP_NET_CFG_MTU, hw->mtu); 607 nn_cfg_writel(hw, NFP_NET_CFG_FLBUFSZ, hw->flbufsz); 608 } 609 610 static void 611 nfp_net_cfg_queue_setup(struct nfp_net_hw *hw) 612 { 613 hw->qcp_cfg = hw->tx_bar + NFP_QCP_QUEUE_ADDR_SZ; 614 } 615 616 #define ETH_ADDR_LEN 6 617 618 static void 619 nfp_eth_copy_mac_reverse(uint8_t *dst, const uint8_t *src) 620 { 621 int i; 622 623 for (i = 0; i < ETH_ADDR_LEN; i++) 624 dst[ETH_ADDR_LEN - i - 1] = src[i]; 625 } 626 627 static int 628 nfp_net_pf_read_mac(struct nfp_net_hw *hw, int port) 629 { 630 union eth_table_entry *entry; 631 int idx, i; 632 633 idx = port; 634 entry = hw->eth_table; 635 636 /* Reading NFP ethernet table obtained before */ 637 for (i = 0; i < NSP_ETH_MAX_COUNT; i++) { 638 if (!(entry->port & NSP_ETH_PORT_LANES_MASK)) { 639 /* port not in use */ 640 entry++; 641 continue; 642 } 643 if (idx == 0) 644 break; 645 idx--; 646 entry++; 647 } 648 649 if (i == NSP_ETH_MAX_COUNT) 650 return -EINVAL; 651 652 /* 653 * hw points to port0 private data. We need hw now pointing to 654 * right port. 655 */ 656 hw += port; 657 nfp_eth_copy_mac_reverse((uint8_t *)&hw->mac_addr, 658 (uint8_t *)&entry->mac_addr); 659 660 return 0; 661 } 662 663 static void 664 nfp_net_vf_read_mac(struct nfp_net_hw *hw) 665 { 666 uint32_t tmp; 667 668 tmp = rte_be_to_cpu_32(nn_cfg_readl(hw, NFP_NET_CFG_MACADDR)); 669 memcpy(&hw->mac_addr[0], &tmp, sizeof(struct ether_addr)); 670 671 tmp = rte_be_to_cpu_32(nn_cfg_readl(hw, NFP_NET_CFG_MACADDR + 4)); 672 memcpy(&hw->mac_addr[4], &tmp, 2); 673 } 674 675 static void 676 nfp_net_write_mac(struct nfp_net_hw *hw, uint8_t *mac) 677 { 678 uint32_t mac0 = *(uint32_t *)mac; 679 uint16_t mac1; 680 681 nn_writel(rte_cpu_to_be_32(mac0), hw->ctrl_bar + NFP_NET_CFG_MACADDR); 682 683 mac += 4; 684 mac1 = *(uint16_t *)mac; 685 nn_writew(rte_cpu_to_be_16(mac1), 686 hw->ctrl_bar + NFP_NET_CFG_MACADDR + 6); 687 } 688 689 static int 690 nfp_configure_rx_interrupt(struct rte_eth_dev *dev, 691 struct rte_intr_handle *intr_handle) 692 { 693 struct nfp_net_hw *hw; 694 int i; 695 696 if (!intr_handle->intr_vec) { 697 intr_handle->intr_vec = 698 rte_zmalloc("intr_vec", 699 dev->data->nb_rx_queues * sizeof(int), 0); 700 if (!intr_handle->intr_vec) { 701 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 702 " intr_vec", dev->data->nb_rx_queues); 703 return -ENOMEM; 704 } 705 } 706 707 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 708 709 if (intr_handle->type == RTE_INTR_HANDLE_UIO) { 710 PMD_INIT_LOG(INFO, "VF: enabling RX interrupt with UIO"); 711 /* UIO just supports one queue and no LSC*/ 712 nn_cfg_writeb(hw, NFP_NET_CFG_RXR_VEC(0), 0); 713 intr_handle->intr_vec[0] = 0; 714 } else { 715 PMD_INIT_LOG(INFO, "VF: enabling RX interrupt with VFIO"); 716 for (i = 0; i < dev->data->nb_rx_queues; i++) { 717 /* 718 * The first msix vector is reserved for non 719 * efd interrupts 720 */ 721 nn_cfg_writeb(hw, NFP_NET_CFG_RXR_VEC(i), i + 1); 722 intr_handle->intr_vec[i] = i + 1; 723 PMD_INIT_LOG(DEBUG, "intr_vec[%d]= %d\n", i, 724 intr_handle->intr_vec[i]); 725 } 726 } 727 728 /* Avoiding TX interrupts */ 729 hw->ctrl |= NFP_NET_CFG_CTRL_MSIX_TX_OFF; 730 return 0; 731 } 732 733 static int 734 nfp_net_start(struct rte_eth_dev *dev) 735 { 736 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 737 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 738 uint32_t new_ctrl, update = 0; 739 struct nfp_net_hw *hw; 740 uint32_t intr_vector; 741 int ret; 742 743 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 744 745 PMD_INIT_LOG(DEBUG, "Start"); 746 747 /* Disabling queues just in case... */ 748 nfp_net_disable_queues(dev); 749 750 /* Writing configuration parameters in the device */ 751 nfp_net_params_setup(hw); 752 753 /* Enabling the required queues in the device */ 754 nfp_net_enable_queues(dev); 755 756 /* check and configure queue intr-vector mapping */ 757 if (dev->data->dev_conf.intr_conf.rxq != 0) { 758 if (hw->pf_multiport_enabled) { 759 PMD_INIT_LOG(ERR, "PMD rx interrupt is not supported " 760 "with NFP multiport PF"); 761 return -EINVAL; 762 } 763 if (intr_handle->type == RTE_INTR_HANDLE_UIO) { 764 /* 765 * Better not to share LSC with RX interrupts. 766 * Unregistering LSC interrupt handler 767 */ 768 rte_intr_callback_unregister(&pci_dev->intr_handle, 769 nfp_net_dev_interrupt_handler, (void *)dev); 770 771 if (dev->data->nb_rx_queues > 1) { 772 PMD_INIT_LOG(ERR, "PMD rx interrupt only " 773 "supports 1 queue with UIO"); 774 return -EIO; 775 } 776 } 777 intr_vector = dev->data->nb_rx_queues; 778 if (rte_intr_efd_enable(intr_handle, intr_vector)) 779 return -1; 780 781 nfp_configure_rx_interrupt(dev, intr_handle); 782 update = NFP_NET_CFG_UPDATE_MSIX; 783 } 784 785 rte_intr_enable(intr_handle); 786 787 /* Enable device */ 788 new_ctrl = hw->ctrl | NFP_NET_CFG_CTRL_ENABLE; 789 790 update |= NFP_NET_CFG_UPDATE_GEN | NFP_NET_CFG_UPDATE_RING; 791 792 if (hw->cap & NFP_NET_CFG_CTRL_RINGCFG) 793 new_ctrl |= NFP_NET_CFG_CTRL_RINGCFG; 794 795 nn_cfg_writel(hw, NFP_NET_CFG_CTRL, new_ctrl); 796 if (nfp_net_reconfig(hw, new_ctrl, update) < 0) 797 return -EIO; 798 799 /* 800 * Allocating rte mbuffs for configured rx queues. 801 * This requires queues being enabled before 802 */ 803 if (nfp_net_rx_freelist_setup(dev) < 0) { 804 ret = -ENOMEM; 805 goto error; 806 } 807 808 if (hw->is_pf) 809 /* Configure the physical port up */ 810 nfp_nsp_eth_config(hw->nspu_desc, hw->pf_port_idx, 1); 811 812 hw->ctrl = new_ctrl; 813 814 return 0; 815 816 error: 817 /* 818 * An error returned by this function should mean the app 819 * exiting and then the system releasing all the memory 820 * allocated even memory coming from hugepages. 821 * 822 * The device could be enabled at this point with some queues 823 * ready for getting packets. This is true if the call to 824 * nfp_net_rx_freelist_setup() succeeds for some queues but 825 * fails for subsequent queues. 826 * 827 * This should make the app exiting but better if we tell the 828 * device first. 829 */ 830 nfp_net_disable_queues(dev); 831 832 return ret; 833 } 834 835 /* Stop device: disable rx and tx functions to allow for reconfiguring. */ 836 static void 837 nfp_net_stop(struct rte_eth_dev *dev) 838 { 839 int i; 840 struct nfp_net_hw *hw; 841 842 PMD_INIT_LOG(DEBUG, "Stop"); 843 844 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 845 846 nfp_net_disable_queues(dev); 847 848 /* Clear queues */ 849 for (i = 0; i < dev->data->nb_tx_queues; i++) { 850 nfp_net_reset_tx_queue( 851 (struct nfp_net_txq *)dev->data->tx_queues[i]); 852 } 853 854 for (i = 0; i < dev->data->nb_rx_queues; i++) { 855 nfp_net_reset_rx_queue( 856 (struct nfp_net_rxq *)dev->data->rx_queues[i]); 857 } 858 859 if (hw->is_pf) 860 /* Configure the physical port down */ 861 nfp_nsp_eth_config(hw->nspu_desc, hw->pf_port_idx, 0); 862 } 863 864 /* Reset and stop device. The device can not be restarted. */ 865 static void 866 nfp_net_close(struct rte_eth_dev *dev) 867 { 868 struct nfp_net_hw *hw; 869 struct rte_pci_device *pci_dev; 870 int i; 871 872 PMD_INIT_LOG(DEBUG, "Close"); 873 874 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 875 pci_dev = RTE_ETH_DEV_TO_PCI(dev); 876 877 /* 878 * We assume that the DPDK application is stopping all the 879 * threads/queues before calling the device close function. 880 */ 881 882 nfp_net_disable_queues(dev); 883 884 /* Clear queues */ 885 for (i = 0; i < dev->data->nb_tx_queues; i++) { 886 nfp_net_reset_tx_queue( 887 (struct nfp_net_txq *)dev->data->tx_queues[i]); 888 } 889 890 for (i = 0; i < dev->data->nb_rx_queues; i++) { 891 nfp_net_reset_rx_queue( 892 (struct nfp_net_rxq *)dev->data->rx_queues[i]); 893 } 894 895 rte_intr_disable(&pci_dev->intr_handle); 896 nn_cfg_writeb(hw, NFP_NET_CFG_LSC, 0xff); 897 898 /* unregister callback func from eal lib */ 899 rte_intr_callback_unregister(&pci_dev->intr_handle, 900 nfp_net_dev_interrupt_handler, 901 (void *)dev); 902 903 /* 904 * The ixgbe PMD driver disables the pcie master on the 905 * device. The i40e does not... 906 */ 907 } 908 909 static void 910 nfp_net_promisc_enable(struct rte_eth_dev *dev) 911 { 912 uint32_t new_ctrl, update = 0; 913 struct nfp_net_hw *hw; 914 915 PMD_DRV_LOG(DEBUG, "Promiscuous mode enable\n"); 916 917 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 918 919 if (!(hw->cap & NFP_NET_CFG_CTRL_PROMISC)) { 920 PMD_INIT_LOG(INFO, "Promiscuous mode not supported"); 921 return; 922 } 923 924 if (hw->ctrl & NFP_NET_CFG_CTRL_PROMISC) { 925 PMD_DRV_LOG(INFO, "Promiscuous mode already enabled\n"); 926 return; 927 } 928 929 new_ctrl = hw->ctrl | NFP_NET_CFG_CTRL_PROMISC; 930 update = NFP_NET_CFG_UPDATE_GEN; 931 932 /* 933 * DPDK sets promiscuous mode on just after this call assuming 934 * it can not fail ... 935 */ 936 if (nfp_net_reconfig(hw, new_ctrl, update) < 0) 937 return; 938 939 hw->ctrl = new_ctrl; 940 } 941 942 static void 943 nfp_net_promisc_disable(struct rte_eth_dev *dev) 944 { 945 uint32_t new_ctrl, update = 0; 946 struct nfp_net_hw *hw; 947 948 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 949 950 if ((hw->ctrl & NFP_NET_CFG_CTRL_PROMISC) == 0) { 951 PMD_DRV_LOG(INFO, "Promiscuous mode already disabled\n"); 952 return; 953 } 954 955 new_ctrl = hw->ctrl & ~NFP_NET_CFG_CTRL_PROMISC; 956 update = NFP_NET_CFG_UPDATE_GEN; 957 958 /* 959 * DPDK sets promiscuous mode off just before this call 960 * assuming it can not fail ... 961 */ 962 if (nfp_net_reconfig(hw, new_ctrl, update) < 0) 963 return; 964 965 hw->ctrl = new_ctrl; 966 } 967 968 /* 969 * return 0 means link status changed, -1 means not changed 970 * 971 * Wait to complete is needed as it can take up to 9 seconds to get the Link 972 * status. 973 */ 974 static int 975 nfp_net_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) 976 { 977 struct nfp_net_hw *hw; 978 struct rte_eth_link link, old; 979 uint32_t nn_link_status; 980 981 static const uint32_t ls_to_ethtool[] = { 982 [NFP_NET_CFG_STS_LINK_RATE_UNSUPPORTED] = ETH_SPEED_NUM_NONE, 983 [NFP_NET_CFG_STS_LINK_RATE_UNKNOWN] = ETH_SPEED_NUM_NONE, 984 [NFP_NET_CFG_STS_LINK_RATE_1G] = ETH_SPEED_NUM_1G, 985 [NFP_NET_CFG_STS_LINK_RATE_10G] = ETH_SPEED_NUM_10G, 986 [NFP_NET_CFG_STS_LINK_RATE_25G] = ETH_SPEED_NUM_25G, 987 [NFP_NET_CFG_STS_LINK_RATE_40G] = ETH_SPEED_NUM_40G, 988 [NFP_NET_CFG_STS_LINK_RATE_50G] = ETH_SPEED_NUM_50G, 989 [NFP_NET_CFG_STS_LINK_RATE_100G] = ETH_SPEED_NUM_100G, 990 }; 991 992 PMD_DRV_LOG(DEBUG, "Link update\n"); 993 994 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 995 996 memset(&old, 0, sizeof(old)); 997 nfp_net_dev_atomic_read_link_status(dev, &old); 998 999 nn_link_status = nn_cfg_readl(hw, NFP_NET_CFG_STS); 1000 1001 memset(&link, 0, sizeof(struct rte_eth_link)); 1002 1003 if (nn_link_status & NFP_NET_CFG_STS_LINK) 1004 link.link_status = ETH_LINK_UP; 1005 1006 link.link_duplex = ETH_LINK_FULL_DUPLEX; 1007 1008 nn_link_status = (nn_link_status >> NFP_NET_CFG_STS_LINK_RATE_SHIFT) & 1009 NFP_NET_CFG_STS_LINK_RATE_MASK; 1010 1011 if (nn_link_status >= RTE_DIM(ls_to_ethtool)) 1012 link.link_speed = ETH_SPEED_NUM_NONE; 1013 else 1014 link.link_speed = ls_to_ethtool[nn_link_status]; 1015 1016 if (old.link_status != link.link_status) { 1017 nfp_net_dev_atomic_write_link_status(dev, &link); 1018 if (link.link_status) 1019 PMD_DRV_LOG(INFO, "NIC Link is Up\n"); 1020 else 1021 PMD_DRV_LOG(INFO, "NIC Link is Down\n"); 1022 return 0; 1023 } 1024 1025 return -1; 1026 } 1027 1028 static int 1029 nfp_net_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 1030 { 1031 int i; 1032 struct nfp_net_hw *hw; 1033 struct rte_eth_stats nfp_dev_stats; 1034 1035 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1036 1037 /* RTE_ETHDEV_QUEUE_STAT_CNTRS default value is 16 */ 1038 1039 memset(&nfp_dev_stats, 0, sizeof(nfp_dev_stats)); 1040 1041 /* reading per RX ring stats */ 1042 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1043 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS) 1044 break; 1045 1046 nfp_dev_stats.q_ipackets[i] = 1047 nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i)); 1048 1049 nfp_dev_stats.q_ipackets[i] -= 1050 hw->eth_stats_base.q_ipackets[i]; 1051 1052 nfp_dev_stats.q_ibytes[i] = 1053 nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i) + 0x8); 1054 1055 nfp_dev_stats.q_ibytes[i] -= 1056 hw->eth_stats_base.q_ibytes[i]; 1057 } 1058 1059 /* reading per TX ring stats */ 1060 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1061 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS) 1062 break; 1063 1064 nfp_dev_stats.q_opackets[i] = 1065 nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i)); 1066 1067 nfp_dev_stats.q_opackets[i] -= 1068 hw->eth_stats_base.q_opackets[i]; 1069 1070 nfp_dev_stats.q_obytes[i] = 1071 nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i) + 0x8); 1072 1073 nfp_dev_stats.q_obytes[i] -= 1074 hw->eth_stats_base.q_obytes[i]; 1075 } 1076 1077 nfp_dev_stats.ipackets = 1078 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_FRAMES); 1079 1080 nfp_dev_stats.ipackets -= hw->eth_stats_base.ipackets; 1081 1082 nfp_dev_stats.ibytes = 1083 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_OCTETS); 1084 1085 nfp_dev_stats.ibytes -= hw->eth_stats_base.ibytes; 1086 1087 nfp_dev_stats.opackets = 1088 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_FRAMES); 1089 1090 nfp_dev_stats.opackets -= hw->eth_stats_base.opackets; 1091 1092 nfp_dev_stats.obytes = 1093 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_OCTETS); 1094 1095 nfp_dev_stats.obytes -= hw->eth_stats_base.obytes; 1096 1097 /* reading general device stats */ 1098 nfp_dev_stats.ierrors = 1099 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_ERRORS); 1100 1101 nfp_dev_stats.ierrors -= hw->eth_stats_base.ierrors; 1102 1103 nfp_dev_stats.oerrors = 1104 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_ERRORS); 1105 1106 nfp_dev_stats.oerrors -= hw->eth_stats_base.oerrors; 1107 1108 /* RX ring mbuf allocation failures */ 1109 nfp_dev_stats.rx_nombuf = dev->data->rx_mbuf_alloc_failed; 1110 1111 nfp_dev_stats.imissed = 1112 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_DISCARDS); 1113 1114 nfp_dev_stats.imissed -= hw->eth_stats_base.imissed; 1115 1116 if (stats) { 1117 memcpy(stats, &nfp_dev_stats, sizeof(*stats)); 1118 return 0; 1119 } 1120 return -EINVAL; 1121 } 1122 1123 static void 1124 nfp_net_stats_reset(struct rte_eth_dev *dev) 1125 { 1126 int i; 1127 struct nfp_net_hw *hw; 1128 1129 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1130 1131 /* 1132 * hw->eth_stats_base records the per counter starting point. 1133 * Lets update it now 1134 */ 1135 1136 /* reading per RX ring stats */ 1137 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1138 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS) 1139 break; 1140 1141 hw->eth_stats_base.q_ipackets[i] = 1142 nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i)); 1143 1144 hw->eth_stats_base.q_ibytes[i] = 1145 nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i) + 0x8); 1146 } 1147 1148 /* reading per TX ring stats */ 1149 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1150 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS) 1151 break; 1152 1153 hw->eth_stats_base.q_opackets[i] = 1154 nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i)); 1155 1156 hw->eth_stats_base.q_obytes[i] = 1157 nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i) + 0x8); 1158 } 1159 1160 hw->eth_stats_base.ipackets = 1161 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_FRAMES); 1162 1163 hw->eth_stats_base.ibytes = 1164 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_OCTETS); 1165 1166 hw->eth_stats_base.opackets = 1167 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_FRAMES); 1168 1169 hw->eth_stats_base.obytes = 1170 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_OCTETS); 1171 1172 /* reading general device stats */ 1173 hw->eth_stats_base.ierrors = 1174 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_ERRORS); 1175 1176 hw->eth_stats_base.oerrors = 1177 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_ERRORS); 1178 1179 /* RX ring mbuf allocation failures */ 1180 dev->data->rx_mbuf_alloc_failed = 0; 1181 1182 hw->eth_stats_base.imissed = 1183 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_DISCARDS); 1184 } 1185 1186 static void 1187 nfp_net_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 1188 { 1189 struct nfp_net_hw *hw; 1190 1191 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1192 1193 dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1194 dev_info->max_rx_queues = (uint16_t)hw->max_rx_queues; 1195 dev_info->max_tx_queues = (uint16_t)hw->max_tx_queues; 1196 dev_info->min_rx_bufsize = ETHER_MIN_MTU; 1197 dev_info->max_rx_pktlen = hw->max_mtu; 1198 /* Next should change when PF support is implemented */ 1199 dev_info->max_mac_addrs = 1; 1200 1201 if (hw->cap & NFP_NET_CFG_CTRL_RXVLAN) 1202 dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP; 1203 1204 if (hw->cap & NFP_NET_CFG_CTRL_RXCSUM) 1205 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_IPV4_CKSUM | 1206 DEV_RX_OFFLOAD_UDP_CKSUM | 1207 DEV_RX_OFFLOAD_TCP_CKSUM; 1208 1209 if (hw->cap & NFP_NET_CFG_CTRL_TXVLAN) 1210 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT; 1211 1212 if (hw->cap & NFP_NET_CFG_CTRL_TXCSUM) 1213 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_IPV4_CKSUM | 1214 DEV_TX_OFFLOAD_UDP_CKSUM | 1215 DEV_TX_OFFLOAD_TCP_CKSUM; 1216 1217 dev_info->default_rxconf = (struct rte_eth_rxconf) { 1218 .rx_thresh = { 1219 .pthresh = DEFAULT_RX_PTHRESH, 1220 .hthresh = DEFAULT_RX_HTHRESH, 1221 .wthresh = DEFAULT_RX_WTHRESH, 1222 }, 1223 .rx_free_thresh = DEFAULT_RX_FREE_THRESH, 1224 .rx_drop_en = 0, 1225 }; 1226 1227 dev_info->default_txconf = (struct rte_eth_txconf) { 1228 .tx_thresh = { 1229 .pthresh = DEFAULT_TX_PTHRESH, 1230 .hthresh = DEFAULT_TX_HTHRESH, 1231 .wthresh = DEFAULT_TX_WTHRESH, 1232 }, 1233 .tx_free_thresh = DEFAULT_TX_FREE_THRESH, 1234 .tx_rs_thresh = DEFAULT_TX_RSBIT_THRESH, 1235 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS | 1236 ETH_TXQ_FLAGS_NOOFFLOADS, 1237 }; 1238 1239 dev_info->flow_type_rss_offloads = ETH_RSS_NONFRAG_IPV4_TCP | 1240 ETH_RSS_NONFRAG_IPV4_UDP | 1241 ETH_RSS_NONFRAG_IPV6_TCP | 1242 ETH_RSS_NONFRAG_IPV6_UDP; 1243 1244 dev_info->reta_size = NFP_NET_CFG_RSS_ITBL_SZ; 1245 dev_info->hash_key_size = NFP_NET_CFG_RSS_KEY_SZ; 1246 1247 dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G | 1248 ETH_LINK_SPEED_25G | ETH_LINK_SPEED_40G | 1249 ETH_LINK_SPEED_50G | ETH_LINK_SPEED_100G; 1250 1251 if (hw->cap & NFP_NET_CFG_CTRL_LSO) 1252 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; 1253 } 1254 1255 static const uint32_t * 1256 nfp_net_supported_ptypes_get(struct rte_eth_dev *dev) 1257 { 1258 static const uint32_t ptypes[] = { 1259 /* refers to nfp_net_set_hash() */ 1260 RTE_PTYPE_INNER_L3_IPV4, 1261 RTE_PTYPE_INNER_L3_IPV6, 1262 RTE_PTYPE_INNER_L3_IPV6_EXT, 1263 RTE_PTYPE_INNER_L4_MASK, 1264 RTE_PTYPE_UNKNOWN 1265 }; 1266 1267 if (dev->rx_pkt_burst == nfp_net_recv_pkts) 1268 return ptypes; 1269 return NULL; 1270 } 1271 1272 static uint32_t 1273 nfp_net_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx) 1274 { 1275 struct nfp_net_rxq *rxq; 1276 struct nfp_net_rx_desc *rxds; 1277 uint32_t idx; 1278 uint32_t count; 1279 1280 rxq = (struct nfp_net_rxq *)dev->data->rx_queues[queue_idx]; 1281 1282 idx = rxq->rd_p; 1283 1284 count = 0; 1285 1286 /* 1287 * Other PMDs are just checking the DD bit in intervals of 4 1288 * descriptors and counting all four if the first has the DD 1289 * bit on. Of course, this is not accurate but can be good for 1290 * performance. But ideally that should be done in descriptors 1291 * chunks belonging to the same cache line 1292 */ 1293 1294 while (count < rxq->rx_count) { 1295 rxds = &rxq->rxds[idx]; 1296 if ((rxds->rxd.meta_len_dd & PCIE_DESC_RX_DD) == 0) 1297 break; 1298 1299 count++; 1300 idx++; 1301 1302 /* Wrapping? */ 1303 if ((idx) == rxq->rx_count) 1304 idx = 0; 1305 } 1306 1307 return count; 1308 } 1309 1310 static int 1311 nfp_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 1312 { 1313 struct rte_pci_device *pci_dev; 1314 struct nfp_net_hw *hw; 1315 int base = 0; 1316 1317 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1318 pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1319 1320 if (pci_dev->intr_handle.type != RTE_INTR_HANDLE_UIO) 1321 base = 1; 1322 1323 /* Make sure all updates are written before un-masking */ 1324 rte_wmb(); 1325 nn_cfg_writeb(hw, NFP_NET_CFG_ICR(base + queue_id), 1326 NFP_NET_CFG_ICR_UNMASKED); 1327 return 0; 1328 } 1329 1330 static int 1331 nfp_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 1332 { 1333 struct rte_pci_device *pci_dev; 1334 struct nfp_net_hw *hw; 1335 int base = 0; 1336 1337 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1338 pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1339 1340 if (pci_dev->intr_handle.type != RTE_INTR_HANDLE_UIO) 1341 base = 1; 1342 1343 /* Make sure all updates are written before un-masking */ 1344 rte_wmb(); 1345 nn_cfg_writeb(hw, NFP_NET_CFG_ICR(base + queue_id), 0x1); 1346 return 0; 1347 } 1348 1349 static void 1350 nfp_net_dev_link_status_print(struct rte_eth_dev *dev) 1351 { 1352 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1353 struct rte_eth_link link; 1354 1355 memset(&link, 0, sizeof(link)); 1356 nfp_net_dev_atomic_read_link_status(dev, &link); 1357 if (link.link_status) 1358 RTE_LOG(INFO, PMD, "Port %d: Link Up - speed %u Mbps - %s\n", 1359 dev->data->port_id, link.link_speed, 1360 link.link_duplex == ETH_LINK_FULL_DUPLEX 1361 ? "full-duplex" : "half-duplex"); 1362 else 1363 RTE_LOG(INFO, PMD, " Port %d: Link Down\n", 1364 dev->data->port_id); 1365 1366 RTE_LOG(INFO, PMD, "PCI Address: %04d:%02d:%02d:%d\n", 1367 pci_dev->addr.domain, pci_dev->addr.bus, 1368 pci_dev->addr.devid, pci_dev->addr.function); 1369 } 1370 1371 /* Interrupt configuration and handling */ 1372 1373 /* 1374 * nfp_net_irq_unmask - Unmask an interrupt 1375 * 1376 * If MSI-X auto-masking is enabled clear the mask bit, otherwise 1377 * clear the ICR for the entry. 1378 */ 1379 static void 1380 nfp_net_irq_unmask(struct rte_eth_dev *dev) 1381 { 1382 struct nfp_net_hw *hw; 1383 struct rte_pci_device *pci_dev; 1384 1385 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1386 pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1387 1388 if (hw->ctrl & NFP_NET_CFG_CTRL_MSIXAUTO) { 1389 /* If MSI-X auto-masking is used, clear the entry */ 1390 rte_wmb(); 1391 rte_intr_enable(&pci_dev->intr_handle); 1392 } else { 1393 /* Make sure all updates are written before un-masking */ 1394 rte_wmb(); 1395 nn_cfg_writeb(hw, NFP_NET_CFG_ICR(NFP_NET_IRQ_LSC_IDX), 1396 NFP_NET_CFG_ICR_UNMASKED); 1397 } 1398 } 1399 1400 static void 1401 nfp_net_dev_interrupt_handler(void *param) 1402 { 1403 int64_t timeout; 1404 struct rte_eth_link link; 1405 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 1406 1407 PMD_DRV_LOG(DEBUG, "We got a LSC interrupt!!!\n"); 1408 1409 /* get the link status */ 1410 memset(&link, 0, sizeof(link)); 1411 nfp_net_dev_atomic_read_link_status(dev, &link); 1412 1413 nfp_net_link_update(dev, 0); 1414 1415 /* likely to up */ 1416 if (!link.link_status) { 1417 /* handle it 1 sec later, wait it being stable */ 1418 timeout = NFP_NET_LINK_UP_CHECK_TIMEOUT; 1419 /* likely to down */ 1420 } else { 1421 /* handle it 4 sec later, wait it being stable */ 1422 timeout = NFP_NET_LINK_DOWN_CHECK_TIMEOUT; 1423 } 1424 1425 if (rte_eal_alarm_set(timeout * 1000, 1426 nfp_net_dev_interrupt_delayed_handler, 1427 (void *)dev) < 0) { 1428 RTE_LOG(ERR, PMD, "Error setting alarm"); 1429 /* Unmasking */ 1430 nfp_net_irq_unmask(dev); 1431 } 1432 } 1433 1434 /* 1435 * Interrupt handler which shall be registered for alarm callback for delayed 1436 * handling specific interrupt to wait for the stable nic state. As the NIC 1437 * interrupt state is not stable for nfp after link is just down, it needs 1438 * to wait 4 seconds to get the stable status. 1439 * 1440 * @param handle Pointer to interrupt handle. 1441 * @param param The address of parameter (struct rte_eth_dev *) 1442 * 1443 * @return void 1444 */ 1445 static void 1446 nfp_net_dev_interrupt_delayed_handler(void *param) 1447 { 1448 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 1449 1450 nfp_net_link_update(dev, 0); 1451 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, NULL); 1452 1453 nfp_net_dev_link_status_print(dev); 1454 1455 /* Unmasking */ 1456 nfp_net_irq_unmask(dev); 1457 } 1458 1459 static int 1460 nfp_net_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 1461 { 1462 struct nfp_net_hw *hw; 1463 1464 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1465 1466 /* check that mtu is within the allowed range */ 1467 if ((mtu < ETHER_MIN_MTU) || ((uint32_t)mtu > hw->max_mtu)) 1468 return -EINVAL; 1469 1470 /* mtu setting is forbidden if port is started */ 1471 if (dev->data->dev_started) { 1472 PMD_DRV_LOG(ERR, "port %d must be stopped before configuration", 1473 dev->data->port_id); 1474 return -EBUSY; 1475 } 1476 1477 /* switch to jumbo mode if needed */ 1478 if ((uint32_t)mtu > ETHER_MAX_LEN) 1479 dev->data->dev_conf.rxmode.jumbo_frame = 1; 1480 else 1481 dev->data->dev_conf.rxmode.jumbo_frame = 0; 1482 1483 /* update max frame size */ 1484 dev->data->dev_conf.rxmode.max_rx_pkt_len = (uint32_t)mtu; 1485 1486 /* writing to configuration space */ 1487 nn_cfg_writel(hw, NFP_NET_CFG_MTU, (uint32_t)mtu); 1488 1489 hw->mtu = mtu; 1490 1491 return 0; 1492 } 1493 1494 static int 1495 nfp_net_rx_queue_setup(struct rte_eth_dev *dev, 1496 uint16_t queue_idx, uint16_t nb_desc, 1497 unsigned int socket_id, 1498 const struct rte_eth_rxconf *rx_conf, 1499 struct rte_mempool *mp) 1500 { 1501 const struct rte_memzone *tz; 1502 struct nfp_net_rxq *rxq; 1503 struct nfp_net_hw *hw; 1504 1505 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1506 1507 PMD_INIT_FUNC_TRACE(); 1508 1509 /* Validating number of descriptors */ 1510 if (((nb_desc * sizeof(struct nfp_net_rx_desc)) % 128) != 0 || 1511 (nb_desc > NFP_NET_MAX_RX_DESC) || 1512 (nb_desc < NFP_NET_MIN_RX_DESC)) { 1513 RTE_LOG(ERR, PMD, "Wrong nb_desc value\n"); 1514 return -EINVAL; 1515 } 1516 1517 /* 1518 * Free memory prior to re-allocation if needed. This is the case after 1519 * calling nfp_net_stop 1520 */ 1521 if (dev->data->rx_queues[queue_idx]) { 1522 nfp_net_rx_queue_release(dev->data->rx_queues[queue_idx]); 1523 dev->data->rx_queues[queue_idx] = NULL; 1524 } 1525 1526 /* Allocating rx queue data structure */ 1527 rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct nfp_net_rxq), 1528 RTE_CACHE_LINE_SIZE, socket_id); 1529 if (rxq == NULL) 1530 return -ENOMEM; 1531 1532 /* Hw queues mapping based on firmware confifguration */ 1533 rxq->qidx = queue_idx; 1534 rxq->fl_qcidx = queue_idx * hw->stride_rx; 1535 rxq->rx_qcidx = rxq->fl_qcidx + (hw->stride_rx - 1); 1536 rxq->qcp_fl = hw->rx_bar + NFP_QCP_QUEUE_OFF(rxq->fl_qcidx); 1537 rxq->qcp_rx = hw->rx_bar + NFP_QCP_QUEUE_OFF(rxq->rx_qcidx); 1538 1539 /* 1540 * Tracking mbuf size for detecting a potential mbuf overflow due to 1541 * RX offset 1542 */ 1543 rxq->mem_pool = mp; 1544 rxq->mbuf_size = rxq->mem_pool->elt_size; 1545 rxq->mbuf_size -= (sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM); 1546 hw->flbufsz = rxq->mbuf_size; 1547 1548 rxq->rx_count = nb_desc; 1549 rxq->port_id = dev->data->port_id; 1550 rxq->rx_free_thresh = rx_conf->rx_free_thresh; 1551 rxq->crc_len = (uint8_t) ((dev->data->dev_conf.rxmode.hw_strip_crc) ? 0 1552 : ETHER_CRC_LEN); 1553 rxq->drop_en = rx_conf->rx_drop_en; 1554 1555 /* 1556 * Allocate RX ring hardware descriptors. A memzone large enough to 1557 * handle the maximum ring size is allocated in order to allow for 1558 * resizing in later calls to the queue setup function. 1559 */ 1560 tz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, 1561 sizeof(struct nfp_net_rx_desc) * 1562 NFP_NET_MAX_RX_DESC, NFP_MEMZONE_ALIGN, 1563 socket_id); 1564 1565 if (tz == NULL) { 1566 RTE_LOG(ERR, PMD, "Error allocatig rx dma\n"); 1567 nfp_net_rx_queue_release(rxq); 1568 return -ENOMEM; 1569 } 1570 1571 /* Saving physical and virtual addresses for the RX ring */ 1572 rxq->dma = (uint64_t)tz->iova; 1573 rxq->rxds = (struct nfp_net_rx_desc *)tz->addr; 1574 1575 /* mbuf pointers array for referencing mbufs linked to RX descriptors */ 1576 rxq->rxbufs = rte_zmalloc_socket("rxq->rxbufs", 1577 sizeof(*rxq->rxbufs) * nb_desc, 1578 RTE_CACHE_LINE_SIZE, socket_id); 1579 if (rxq->rxbufs == NULL) { 1580 nfp_net_rx_queue_release(rxq); 1581 return -ENOMEM; 1582 } 1583 1584 PMD_RX_LOG(DEBUG, "rxbufs=%p hw_ring=%p dma_addr=0x%" PRIx64 "\n", 1585 rxq->rxbufs, rxq->rxds, (unsigned long int)rxq->dma); 1586 1587 nfp_net_reset_rx_queue(rxq); 1588 1589 dev->data->rx_queues[queue_idx] = rxq; 1590 rxq->hw = hw; 1591 1592 /* 1593 * Telling the HW about the physical address of the RX ring and number 1594 * of descriptors in log2 format 1595 */ 1596 nn_cfg_writeq(hw, NFP_NET_CFG_RXR_ADDR(queue_idx), rxq->dma); 1597 nn_cfg_writeb(hw, NFP_NET_CFG_RXR_SZ(queue_idx), rte_log2_u32(nb_desc)); 1598 1599 return 0; 1600 } 1601 1602 static int 1603 nfp_net_rx_fill_freelist(struct nfp_net_rxq *rxq) 1604 { 1605 struct nfp_net_rx_buff *rxe = rxq->rxbufs; 1606 uint64_t dma_addr; 1607 unsigned i; 1608 1609 PMD_RX_LOG(DEBUG, "nfp_net_rx_fill_freelist for %u descriptors\n", 1610 rxq->rx_count); 1611 1612 for (i = 0; i < rxq->rx_count; i++) { 1613 struct nfp_net_rx_desc *rxd; 1614 struct rte_mbuf *mbuf = rte_pktmbuf_alloc(rxq->mem_pool); 1615 1616 if (mbuf == NULL) { 1617 RTE_LOG(ERR, PMD, "RX mbuf alloc failed queue_id=%u\n", 1618 (unsigned)rxq->qidx); 1619 return -ENOMEM; 1620 } 1621 1622 dma_addr = rte_cpu_to_le_64(RTE_MBUF_DMA_ADDR_DEFAULT(mbuf)); 1623 1624 rxd = &rxq->rxds[i]; 1625 rxd->fld.dd = 0; 1626 rxd->fld.dma_addr_hi = (dma_addr >> 32) & 0xff; 1627 rxd->fld.dma_addr_lo = dma_addr & 0xffffffff; 1628 rxe[i].mbuf = mbuf; 1629 PMD_RX_LOG(DEBUG, "[%d]: %" PRIx64 "\n", i, dma_addr); 1630 } 1631 1632 /* Make sure all writes are flushed before telling the hardware */ 1633 rte_wmb(); 1634 1635 /* Not advertising the whole ring as the firmware gets confused if so */ 1636 PMD_RX_LOG(DEBUG, "Increment FL write pointer in %u\n", 1637 rxq->rx_count - 1); 1638 1639 nfp_qcp_ptr_add(rxq->qcp_fl, NFP_QCP_WRITE_PTR, rxq->rx_count - 1); 1640 1641 return 0; 1642 } 1643 1644 static int 1645 nfp_net_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 1646 uint16_t nb_desc, unsigned int socket_id, 1647 const struct rte_eth_txconf *tx_conf) 1648 { 1649 const struct rte_memzone *tz; 1650 struct nfp_net_txq *txq; 1651 uint16_t tx_free_thresh; 1652 struct nfp_net_hw *hw; 1653 1654 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1655 1656 PMD_INIT_FUNC_TRACE(); 1657 1658 /* Validating number of descriptors */ 1659 if (((nb_desc * sizeof(struct nfp_net_tx_desc)) % 128) != 0 || 1660 (nb_desc > NFP_NET_MAX_TX_DESC) || 1661 (nb_desc < NFP_NET_MIN_TX_DESC)) { 1662 RTE_LOG(ERR, PMD, "Wrong nb_desc value\n"); 1663 return -EINVAL; 1664 } 1665 1666 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ? 1667 tx_conf->tx_free_thresh : 1668 DEFAULT_TX_FREE_THRESH); 1669 1670 if (tx_free_thresh > (nb_desc)) { 1671 RTE_LOG(ERR, PMD, 1672 "tx_free_thresh must be less than the number of TX " 1673 "descriptors. (tx_free_thresh=%u port=%d " 1674 "queue=%d)\n", (unsigned int)tx_free_thresh, 1675 dev->data->port_id, (int)queue_idx); 1676 return -(EINVAL); 1677 } 1678 1679 /* 1680 * Free memory prior to re-allocation if needed. This is the case after 1681 * calling nfp_net_stop 1682 */ 1683 if (dev->data->tx_queues[queue_idx]) { 1684 PMD_TX_LOG(DEBUG, "Freeing memory prior to re-allocation %d\n", 1685 queue_idx); 1686 nfp_net_tx_queue_release(dev->data->tx_queues[queue_idx]); 1687 dev->data->tx_queues[queue_idx] = NULL; 1688 } 1689 1690 /* Allocating tx queue data structure */ 1691 txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct nfp_net_txq), 1692 RTE_CACHE_LINE_SIZE, socket_id); 1693 if (txq == NULL) { 1694 RTE_LOG(ERR, PMD, "Error allocating tx dma\n"); 1695 return -ENOMEM; 1696 } 1697 1698 /* 1699 * Allocate TX ring hardware descriptors. A memzone large enough to 1700 * handle the maximum ring size is allocated in order to allow for 1701 * resizing in later calls to the queue setup function. 1702 */ 1703 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx, 1704 sizeof(struct nfp_net_tx_desc) * 1705 NFP_NET_MAX_TX_DESC, NFP_MEMZONE_ALIGN, 1706 socket_id); 1707 if (tz == NULL) { 1708 RTE_LOG(ERR, PMD, "Error allocating tx dma\n"); 1709 nfp_net_tx_queue_release(txq); 1710 return -ENOMEM; 1711 } 1712 1713 txq->tx_count = nb_desc; 1714 txq->tx_free_thresh = tx_free_thresh; 1715 txq->tx_pthresh = tx_conf->tx_thresh.pthresh; 1716 txq->tx_hthresh = tx_conf->tx_thresh.hthresh; 1717 txq->tx_wthresh = tx_conf->tx_thresh.wthresh; 1718 1719 /* queue mapping based on firmware configuration */ 1720 txq->qidx = queue_idx; 1721 txq->tx_qcidx = queue_idx * hw->stride_tx; 1722 txq->qcp_q = hw->tx_bar + NFP_QCP_QUEUE_OFF(txq->tx_qcidx); 1723 1724 txq->port_id = dev->data->port_id; 1725 txq->txq_flags = tx_conf->txq_flags; 1726 1727 /* Saving physical and virtual addresses for the TX ring */ 1728 txq->dma = (uint64_t)tz->iova; 1729 txq->txds = (struct nfp_net_tx_desc *)tz->addr; 1730 1731 /* mbuf pointers array for referencing mbufs linked to TX descriptors */ 1732 txq->txbufs = rte_zmalloc_socket("txq->txbufs", 1733 sizeof(*txq->txbufs) * nb_desc, 1734 RTE_CACHE_LINE_SIZE, socket_id); 1735 if (txq->txbufs == NULL) { 1736 nfp_net_tx_queue_release(txq); 1737 return -ENOMEM; 1738 } 1739 PMD_TX_LOG(DEBUG, "txbufs=%p hw_ring=%p dma_addr=0x%" PRIx64 "\n", 1740 txq->txbufs, txq->txds, (unsigned long int)txq->dma); 1741 1742 nfp_net_reset_tx_queue(txq); 1743 1744 dev->data->tx_queues[queue_idx] = txq; 1745 txq->hw = hw; 1746 1747 /* 1748 * Telling the HW about the physical address of the TX ring and number 1749 * of descriptors in log2 format 1750 */ 1751 nn_cfg_writeq(hw, NFP_NET_CFG_TXR_ADDR(queue_idx), txq->dma); 1752 nn_cfg_writeb(hw, NFP_NET_CFG_TXR_SZ(queue_idx), rte_log2_u32(nb_desc)); 1753 1754 return 0; 1755 } 1756 1757 /* nfp_net_tx_tso - Set TX descriptor for TSO */ 1758 static inline void 1759 nfp_net_tx_tso(struct nfp_net_txq *txq, struct nfp_net_tx_desc *txd, 1760 struct rte_mbuf *mb) 1761 { 1762 uint64_t ol_flags; 1763 struct nfp_net_hw *hw = txq->hw; 1764 1765 if (!(hw->cap & NFP_NET_CFG_CTRL_LSO)) 1766 goto clean_txd; 1767 1768 ol_flags = mb->ol_flags; 1769 1770 if (!(ol_flags & PKT_TX_TCP_SEG)) 1771 goto clean_txd; 1772 1773 txd->l4_offset = mb->l2_len + mb->l3_len + mb->l4_len; 1774 txd->lso = rte_cpu_to_le_16(mb->tso_segsz); 1775 txd->flags = PCIE_DESC_TX_LSO; 1776 return; 1777 1778 clean_txd: 1779 txd->flags = 0; 1780 txd->l4_offset = 0; 1781 txd->lso = 0; 1782 } 1783 1784 /* nfp_net_tx_cksum - Set TX CSUM offload flags in TX descriptor */ 1785 static inline void 1786 nfp_net_tx_cksum(struct nfp_net_txq *txq, struct nfp_net_tx_desc *txd, 1787 struct rte_mbuf *mb) 1788 { 1789 uint64_t ol_flags; 1790 struct nfp_net_hw *hw = txq->hw; 1791 1792 if (!(hw->cap & NFP_NET_CFG_CTRL_TXCSUM)) 1793 return; 1794 1795 ol_flags = mb->ol_flags; 1796 1797 /* IPv6 does not need checksum */ 1798 if (ol_flags & PKT_TX_IP_CKSUM) 1799 txd->flags |= PCIE_DESC_TX_IP4_CSUM; 1800 1801 switch (ol_flags & PKT_TX_L4_MASK) { 1802 case PKT_TX_UDP_CKSUM: 1803 txd->flags |= PCIE_DESC_TX_UDP_CSUM; 1804 break; 1805 case PKT_TX_TCP_CKSUM: 1806 txd->flags |= PCIE_DESC_TX_TCP_CSUM; 1807 break; 1808 } 1809 1810 if (ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_L4_MASK)) 1811 txd->flags |= PCIE_DESC_TX_CSUM; 1812 } 1813 1814 /* nfp_net_rx_cksum - set mbuf checksum flags based on RX descriptor flags */ 1815 static inline void 1816 nfp_net_rx_cksum(struct nfp_net_rxq *rxq, struct nfp_net_rx_desc *rxd, 1817 struct rte_mbuf *mb) 1818 { 1819 struct nfp_net_hw *hw = rxq->hw; 1820 1821 if (!(hw->ctrl & NFP_NET_CFG_CTRL_RXCSUM)) 1822 return; 1823 1824 /* If IPv4 and IP checksum error, fail */ 1825 if ((rxd->rxd.flags & PCIE_DESC_RX_IP4_CSUM) && 1826 !(rxd->rxd.flags & PCIE_DESC_RX_IP4_CSUM_OK)) 1827 mb->ol_flags |= PKT_RX_IP_CKSUM_BAD; 1828 1829 /* If neither UDP nor TCP return */ 1830 if (!(rxd->rxd.flags & PCIE_DESC_RX_TCP_CSUM) && 1831 !(rxd->rxd.flags & PCIE_DESC_RX_UDP_CSUM)) 1832 return; 1833 1834 if ((rxd->rxd.flags & PCIE_DESC_RX_TCP_CSUM) && 1835 !(rxd->rxd.flags & PCIE_DESC_RX_TCP_CSUM_OK)) 1836 mb->ol_flags |= PKT_RX_L4_CKSUM_BAD; 1837 1838 if ((rxd->rxd.flags & PCIE_DESC_RX_UDP_CSUM) && 1839 !(rxd->rxd.flags & PCIE_DESC_RX_UDP_CSUM_OK)) 1840 mb->ol_flags |= PKT_RX_L4_CKSUM_BAD; 1841 } 1842 1843 #define NFP_HASH_OFFSET ((uint8_t *)mbuf->buf_addr + mbuf->data_off - 4) 1844 #define NFP_HASH_TYPE_OFFSET ((uint8_t *)mbuf->buf_addr + mbuf->data_off - 8) 1845 1846 #define NFP_DESC_META_LEN(d) (d->rxd.meta_len_dd & PCIE_DESC_RX_META_LEN_MASK) 1847 1848 /* 1849 * nfp_net_set_hash - Set mbuf hash data 1850 * 1851 * The RSS hash and hash-type are pre-pended to the packet data. 1852 * Extract and decode it and set the mbuf fields. 1853 */ 1854 static inline void 1855 nfp_net_set_hash(struct nfp_net_rxq *rxq, struct nfp_net_rx_desc *rxd, 1856 struct rte_mbuf *mbuf) 1857 { 1858 struct nfp_net_hw *hw = rxq->hw; 1859 uint8_t *meta_offset; 1860 uint32_t meta_info; 1861 uint32_t hash = 0; 1862 uint32_t hash_type = 0; 1863 1864 if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS)) 1865 return; 1866 1867 if (NFD_CFG_MAJOR_VERSION_of(hw->ver) <= 3) { 1868 if (!(rxd->rxd.flags & PCIE_DESC_RX_RSS)) 1869 return; 1870 1871 hash = rte_be_to_cpu_32(*(uint32_t *)NFP_HASH_OFFSET); 1872 hash_type = rte_be_to_cpu_32(*(uint32_t *)NFP_HASH_TYPE_OFFSET); 1873 1874 } else if (NFP_DESC_META_LEN(rxd)) { 1875 /* 1876 * new metadata api: 1877 * <---- 32 bit -----> 1878 * m field type word 1879 * e data field #2 1880 * t data field #1 1881 * a data field #0 1882 * ==================== 1883 * packet data 1884 * 1885 * Field type word contains up to 8 4bit field types 1886 * A 4bit field type refers to a data field word 1887 * A data field word can have several 4bit field types 1888 */ 1889 meta_offset = rte_pktmbuf_mtod(mbuf, uint8_t *); 1890 meta_offset -= NFP_DESC_META_LEN(rxd); 1891 meta_info = rte_be_to_cpu_32(*(uint32_t *)meta_offset); 1892 meta_offset += 4; 1893 /* NFP PMD just supports metadata for hashing */ 1894 switch (meta_info & NFP_NET_META_FIELD_MASK) { 1895 case NFP_NET_META_HASH: 1896 /* next field type is about the hash type */ 1897 meta_info >>= NFP_NET_META_FIELD_SIZE; 1898 /* hash value is in the data field */ 1899 hash = rte_be_to_cpu_32(*(uint32_t *)meta_offset); 1900 hash_type = meta_info & NFP_NET_META_FIELD_MASK; 1901 break; 1902 default: 1903 /* Unsupported metadata can be a performance issue */ 1904 return; 1905 } 1906 } else { 1907 return; 1908 } 1909 1910 mbuf->hash.rss = hash; 1911 mbuf->ol_flags |= PKT_RX_RSS_HASH; 1912 1913 switch (hash_type) { 1914 case NFP_NET_RSS_IPV4: 1915 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV4; 1916 break; 1917 case NFP_NET_RSS_IPV6: 1918 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6; 1919 break; 1920 case NFP_NET_RSS_IPV6_EX: 1921 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT; 1922 break; 1923 default: 1924 mbuf->packet_type |= RTE_PTYPE_INNER_L4_MASK; 1925 } 1926 } 1927 1928 static inline void 1929 nfp_net_mbuf_alloc_failed(struct nfp_net_rxq *rxq) 1930 { 1931 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++; 1932 } 1933 1934 #define NFP_DESC_META_LEN(d) (d->rxd.meta_len_dd & PCIE_DESC_RX_META_LEN_MASK) 1935 1936 /* 1937 * RX path design: 1938 * 1939 * There are some decissions to take: 1940 * 1) How to check DD RX descriptors bit 1941 * 2) How and when to allocate new mbufs 1942 * 1943 * Current implementation checks just one single DD bit each loop. As each 1944 * descriptor is 8 bytes, it is likely a good idea to check descriptors in 1945 * a single cache line instead. Tests with this change have not shown any 1946 * performance improvement but it requires further investigation. For example, 1947 * depending on which descriptor is next, the number of descriptors could be 1948 * less than 8 for just checking those in the same cache line. This implies 1949 * extra work which could be counterproductive by itself. Indeed, last firmware 1950 * changes are just doing this: writing several descriptors with the DD bit 1951 * for saving PCIe bandwidth and DMA operations from the NFP. 1952 * 1953 * Mbuf allocation is done when a new packet is received. Then the descriptor 1954 * is automatically linked with the new mbuf and the old one is given to the 1955 * user. The main drawback with this design is mbuf allocation is heavier than 1956 * using bulk allocations allowed by DPDK with rte_mempool_get_bulk. From the 1957 * cache point of view it does not seem allocating the mbuf early on as we are 1958 * doing now have any benefit at all. Again, tests with this change have not 1959 * shown any improvement. Also, rte_mempool_get_bulk returns all or nothing 1960 * so looking at the implications of this type of allocation should be studied 1961 * deeply 1962 */ 1963 1964 static uint16_t 1965 nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 1966 { 1967 struct nfp_net_rxq *rxq; 1968 struct nfp_net_rx_desc *rxds; 1969 struct nfp_net_rx_buff *rxb; 1970 struct nfp_net_hw *hw; 1971 struct rte_mbuf *mb; 1972 struct rte_mbuf *new_mb; 1973 uint16_t nb_hold; 1974 uint64_t dma_addr; 1975 int avail; 1976 1977 rxq = rx_queue; 1978 if (unlikely(rxq == NULL)) { 1979 /* 1980 * DPDK just checks the queue is lower than max queues 1981 * enabled. But the queue needs to be configured 1982 */ 1983 RTE_LOG_DP(ERR, PMD, "RX Bad queue\n"); 1984 return -EINVAL; 1985 } 1986 1987 hw = rxq->hw; 1988 avail = 0; 1989 nb_hold = 0; 1990 1991 while (avail < nb_pkts) { 1992 rxb = &rxq->rxbufs[rxq->rd_p]; 1993 if (unlikely(rxb == NULL)) { 1994 RTE_LOG_DP(ERR, PMD, "rxb does not exist!\n"); 1995 break; 1996 } 1997 1998 rxds = &rxq->rxds[rxq->rd_p]; 1999 if ((rxds->rxd.meta_len_dd & PCIE_DESC_RX_DD) == 0) 2000 break; 2001 2002 /* 2003 * Memory barrier to ensure that we won't do other 2004 * reads before the DD bit. 2005 */ 2006 rte_rmb(); 2007 2008 /* 2009 * We got a packet. Let's alloc a new mbuff for refilling the 2010 * free descriptor ring as soon as possible 2011 */ 2012 new_mb = rte_pktmbuf_alloc(rxq->mem_pool); 2013 if (unlikely(new_mb == NULL)) { 2014 RTE_LOG_DP(DEBUG, PMD, 2015 "RX mbuf alloc failed port_id=%u queue_id=%u\n", 2016 rxq->port_id, (unsigned int)rxq->qidx); 2017 nfp_net_mbuf_alloc_failed(rxq); 2018 break; 2019 } 2020 2021 nb_hold++; 2022 2023 /* 2024 * Grab the mbuff and refill the descriptor with the 2025 * previously allocated mbuff 2026 */ 2027 mb = rxb->mbuf; 2028 rxb->mbuf = new_mb; 2029 2030 PMD_RX_LOG(DEBUG, "Packet len: %u, mbuf_size: %u\n", 2031 rxds->rxd.data_len, rxq->mbuf_size); 2032 2033 /* Size of this segment */ 2034 mb->data_len = rxds->rxd.data_len - NFP_DESC_META_LEN(rxds); 2035 /* Size of the whole packet. We just support 1 segment */ 2036 mb->pkt_len = rxds->rxd.data_len - NFP_DESC_META_LEN(rxds); 2037 2038 if (unlikely((mb->data_len + hw->rx_offset) > 2039 rxq->mbuf_size)) { 2040 /* 2041 * This should not happen and the user has the 2042 * responsibility of avoiding it. But we have 2043 * to give some info about the error 2044 */ 2045 RTE_LOG_DP(ERR, PMD, 2046 "mbuf overflow likely due to the RX offset.\n" 2047 "\t\tYour mbuf size should have extra space for" 2048 " RX offset=%u bytes.\n" 2049 "\t\tCurrently you just have %u bytes available" 2050 " but the received packet is %u bytes long", 2051 hw->rx_offset, 2052 rxq->mbuf_size - hw->rx_offset, 2053 mb->data_len); 2054 return -EINVAL; 2055 } 2056 2057 /* Filling the received mbuff with packet info */ 2058 if (hw->rx_offset) 2059 mb->data_off = RTE_PKTMBUF_HEADROOM + hw->rx_offset; 2060 else 2061 mb->data_off = RTE_PKTMBUF_HEADROOM + 2062 NFP_DESC_META_LEN(rxds); 2063 2064 /* No scatter mode supported */ 2065 mb->nb_segs = 1; 2066 mb->next = NULL; 2067 2068 mb->port = rxq->port_id; 2069 2070 /* Checking the RSS flag */ 2071 nfp_net_set_hash(rxq, rxds, mb); 2072 2073 /* Checking the checksum flag */ 2074 nfp_net_rx_cksum(rxq, rxds, mb); 2075 2076 if ((rxds->rxd.flags & PCIE_DESC_RX_VLAN) && 2077 (hw->ctrl & NFP_NET_CFG_CTRL_RXVLAN)) { 2078 mb->vlan_tci = rte_cpu_to_le_32(rxds->rxd.vlan); 2079 mb->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 2080 } 2081 2082 /* Adding the mbuff to the mbuff array passed by the app */ 2083 rx_pkts[avail++] = mb; 2084 2085 /* Now resetting and updating the descriptor */ 2086 rxds->vals[0] = 0; 2087 rxds->vals[1] = 0; 2088 dma_addr = rte_cpu_to_le_64(RTE_MBUF_DMA_ADDR_DEFAULT(new_mb)); 2089 rxds->fld.dd = 0; 2090 rxds->fld.dma_addr_hi = (dma_addr >> 32) & 0xff; 2091 rxds->fld.dma_addr_lo = dma_addr & 0xffffffff; 2092 2093 rxq->rd_p++; 2094 if (unlikely(rxq->rd_p == rxq->rx_count)) /* wrapping?*/ 2095 rxq->rd_p = 0; 2096 } 2097 2098 if (nb_hold == 0) 2099 return nb_hold; 2100 2101 PMD_RX_LOG(DEBUG, "RX port_id=%u queue_id=%u, %d packets received\n", 2102 rxq->port_id, (unsigned int)rxq->qidx, nb_hold); 2103 2104 nb_hold += rxq->nb_rx_hold; 2105 2106 /* 2107 * FL descriptors needs to be written before incrementing the 2108 * FL queue WR pointer 2109 */ 2110 rte_wmb(); 2111 if (nb_hold > rxq->rx_free_thresh) { 2112 PMD_RX_LOG(DEBUG, "port=%u queue=%u nb_hold=%u avail=%u\n", 2113 rxq->port_id, (unsigned int)rxq->qidx, 2114 (unsigned)nb_hold, (unsigned)avail); 2115 nfp_qcp_ptr_add(rxq->qcp_fl, NFP_QCP_WRITE_PTR, nb_hold); 2116 nb_hold = 0; 2117 } 2118 rxq->nb_rx_hold = nb_hold; 2119 2120 return avail; 2121 } 2122 2123 /* 2124 * nfp_net_tx_free_bufs - Check for descriptors with a complete 2125 * status 2126 * @txq: TX queue to work with 2127 * Returns number of descriptors freed 2128 */ 2129 int 2130 nfp_net_tx_free_bufs(struct nfp_net_txq *txq) 2131 { 2132 uint32_t qcp_rd_p; 2133 int todo; 2134 2135 PMD_TX_LOG(DEBUG, "queue %u. Check for descriptor with a complete" 2136 " status\n", txq->qidx); 2137 2138 /* Work out how many packets have been sent */ 2139 qcp_rd_p = nfp_qcp_read(txq->qcp_q, NFP_QCP_READ_PTR); 2140 2141 if (qcp_rd_p == txq->rd_p) { 2142 PMD_TX_LOG(DEBUG, "queue %u: It seems harrier is not sending " 2143 "packets (%u, %u)\n", txq->qidx, 2144 qcp_rd_p, txq->rd_p); 2145 return 0; 2146 } 2147 2148 if (qcp_rd_p > txq->rd_p) 2149 todo = qcp_rd_p - txq->rd_p; 2150 else 2151 todo = qcp_rd_p + txq->tx_count - txq->rd_p; 2152 2153 PMD_TX_LOG(DEBUG, "qcp_rd_p %u, txq->rd_p: %u, qcp->rd_p: %u\n", 2154 qcp_rd_p, txq->rd_p, txq->rd_p); 2155 2156 if (todo == 0) 2157 return todo; 2158 2159 txq->rd_p += todo; 2160 if (unlikely(txq->rd_p >= txq->tx_count)) 2161 txq->rd_p -= txq->tx_count; 2162 2163 return todo; 2164 } 2165 2166 /* Leaving always free descriptors for avoiding wrapping confusion */ 2167 static inline 2168 uint32_t nfp_free_tx_desc(struct nfp_net_txq *txq) 2169 { 2170 if (txq->wr_p >= txq->rd_p) 2171 return txq->tx_count - (txq->wr_p - txq->rd_p) - 8; 2172 else 2173 return txq->rd_p - txq->wr_p - 8; 2174 } 2175 2176 /* 2177 * nfp_net_txq_full - Check if the TX queue free descriptors 2178 * is below tx_free_threshold 2179 * 2180 * @txq: TX queue to check 2181 * 2182 * This function uses the host copy* of read/write pointers 2183 */ 2184 static inline 2185 uint32_t nfp_net_txq_full(struct nfp_net_txq *txq) 2186 { 2187 return (nfp_free_tx_desc(txq) < txq->tx_free_thresh); 2188 } 2189 2190 static uint16_t 2191 nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 2192 { 2193 struct nfp_net_txq *txq; 2194 struct nfp_net_hw *hw; 2195 struct nfp_net_tx_desc *txds, txd; 2196 struct rte_mbuf *pkt; 2197 uint64_t dma_addr; 2198 int pkt_size, dma_size; 2199 uint16_t free_descs, issued_descs; 2200 struct rte_mbuf **lmbuf; 2201 int i; 2202 2203 txq = tx_queue; 2204 hw = txq->hw; 2205 txds = &txq->txds[txq->wr_p]; 2206 2207 PMD_TX_LOG(DEBUG, "working for queue %u at pos %d and %u packets\n", 2208 txq->qidx, txq->wr_p, nb_pkts); 2209 2210 if ((nfp_free_tx_desc(txq) < nb_pkts) || (nfp_net_txq_full(txq))) 2211 nfp_net_tx_free_bufs(txq); 2212 2213 free_descs = (uint16_t)nfp_free_tx_desc(txq); 2214 if (unlikely(free_descs == 0)) 2215 return 0; 2216 2217 pkt = *tx_pkts; 2218 2219 i = 0; 2220 issued_descs = 0; 2221 PMD_TX_LOG(DEBUG, "queue: %u. Sending %u packets\n", 2222 txq->qidx, nb_pkts); 2223 /* Sending packets */ 2224 while ((i < nb_pkts) && free_descs) { 2225 /* Grabbing the mbuf linked to the current descriptor */ 2226 lmbuf = &txq->txbufs[txq->wr_p].mbuf; 2227 /* Warming the cache for releasing the mbuf later on */ 2228 RTE_MBUF_PREFETCH_TO_FREE(*lmbuf); 2229 2230 pkt = *(tx_pkts + i); 2231 2232 if (unlikely((pkt->nb_segs > 1) && 2233 !(hw->cap & NFP_NET_CFG_CTRL_GATHER))) { 2234 PMD_INIT_LOG(INFO, "NFP_NET_CFG_CTRL_GATHER not set"); 2235 rte_panic("Multisegment packet unsupported\n"); 2236 } 2237 2238 /* Checking if we have enough descriptors */ 2239 if (unlikely(pkt->nb_segs > free_descs)) 2240 goto xmit_end; 2241 2242 /* 2243 * Checksum and VLAN flags just in the first descriptor for a 2244 * multisegment packet, but TSO info needs to be in all of them. 2245 */ 2246 txd.data_len = pkt->pkt_len; 2247 nfp_net_tx_tso(txq, &txd, pkt); 2248 nfp_net_tx_cksum(txq, &txd, pkt); 2249 2250 if ((pkt->ol_flags & PKT_TX_VLAN_PKT) && 2251 (hw->cap & NFP_NET_CFG_CTRL_TXVLAN)) { 2252 txd.flags |= PCIE_DESC_TX_VLAN; 2253 txd.vlan = pkt->vlan_tci; 2254 } 2255 2256 /* 2257 * mbuf data_len is the data in one segment and pkt_len data 2258 * in the whole packet. When the packet is just one segment, 2259 * then data_len = pkt_len 2260 */ 2261 pkt_size = pkt->pkt_len; 2262 2263 while (pkt) { 2264 /* Copying TSO, VLAN and cksum info */ 2265 *txds = txd; 2266 2267 /* Releasing mbuf used by this descriptor previously*/ 2268 if (*lmbuf) 2269 rte_pktmbuf_free_seg(*lmbuf); 2270 2271 /* 2272 * Linking mbuf with descriptor for being released 2273 * next time descriptor is used 2274 */ 2275 *lmbuf = pkt; 2276 2277 dma_size = pkt->data_len; 2278 dma_addr = rte_mbuf_data_iova(pkt); 2279 PMD_TX_LOG(DEBUG, "Working with mbuf at dma address:" 2280 "%" PRIx64 "\n", dma_addr); 2281 2282 /* Filling descriptors fields */ 2283 txds->dma_len = dma_size; 2284 txds->data_len = txd.data_len; 2285 txds->dma_addr_hi = (dma_addr >> 32) & 0xff; 2286 txds->dma_addr_lo = (dma_addr & 0xffffffff); 2287 ASSERT(free_descs > 0); 2288 free_descs--; 2289 2290 txq->wr_p++; 2291 if (unlikely(txq->wr_p == txq->tx_count)) /* wrapping?*/ 2292 txq->wr_p = 0; 2293 2294 pkt_size -= dma_size; 2295 2296 /* 2297 * Making the EOP, packets with just one segment 2298 * the priority 2299 */ 2300 if (likely(!pkt_size)) 2301 txds->offset_eop = PCIE_DESC_TX_EOP; 2302 else 2303 txds->offset_eop = 0; 2304 2305 pkt = pkt->next; 2306 /* Referencing next free TX descriptor */ 2307 txds = &txq->txds[txq->wr_p]; 2308 lmbuf = &txq->txbufs[txq->wr_p].mbuf; 2309 issued_descs++; 2310 } 2311 i++; 2312 } 2313 2314 xmit_end: 2315 /* Increment write pointers. Force memory write before we let HW know */ 2316 rte_wmb(); 2317 nfp_qcp_ptr_add(txq->qcp_q, NFP_QCP_WRITE_PTR, issued_descs); 2318 2319 return i; 2320 } 2321 2322 static int 2323 nfp_net_vlan_offload_set(struct rte_eth_dev *dev, int mask) 2324 { 2325 uint32_t new_ctrl, update; 2326 struct nfp_net_hw *hw; 2327 int ret; 2328 2329 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2330 new_ctrl = 0; 2331 2332 if ((mask & ETH_VLAN_FILTER_OFFLOAD) || 2333 (mask & ETH_VLAN_EXTEND_OFFLOAD)) 2334 RTE_LOG(INFO, PMD, "No support for ETH_VLAN_FILTER_OFFLOAD or" 2335 " ETH_VLAN_EXTEND_OFFLOAD"); 2336 2337 /* Enable vlan strip if it is not configured yet */ 2338 if ((mask & ETH_VLAN_STRIP_OFFLOAD) && 2339 !(hw->ctrl & NFP_NET_CFG_CTRL_RXVLAN)) 2340 new_ctrl = hw->ctrl | NFP_NET_CFG_CTRL_RXVLAN; 2341 2342 /* Disable vlan strip just if it is configured */ 2343 if (!(mask & ETH_VLAN_STRIP_OFFLOAD) && 2344 (hw->ctrl & NFP_NET_CFG_CTRL_RXVLAN)) 2345 new_ctrl = hw->ctrl & ~NFP_NET_CFG_CTRL_RXVLAN; 2346 2347 if (new_ctrl == 0) 2348 return 0; 2349 2350 update = NFP_NET_CFG_UPDATE_GEN; 2351 2352 ret = nfp_net_reconfig(hw, new_ctrl, update); 2353 if (!ret) 2354 hw->ctrl = new_ctrl; 2355 2356 return ret; 2357 } 2358 2359 /* Update Redirection Table(RETA) of Receive Side Scaling of Ethernet device */ 2360 static int 2361 nfp_net_reta_update(struct rte_eth_dev *dev, 2362 struct rte_eth_rss_reta_entry64 *reta_conf, 2363 uint16_t reta_size) 2364 { 2365 uint32_t reta, mask; 2366 int i, j; 2367 int idx, shift; 2368 uint32_t update; 2369 struct nfp_net_hw *hw = 2370 NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2371 2372 if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS)) 2373 return -EINVAL; 2374 2375 if (reta_size != NFP_NET_CFG_RSS_ITBL_SZ) { 2376 RTE_LOG(ERR, PMD, "The size of hash lookup table configured " 2377 "(%d) doesn't match the number hardware can supported " 2378 "(%d)\n", reta_size, NFP_NET_CFG_RSS_ITBL_SZ); 2379 return -EINVAL; 2380 } 2381 2382 /* 2383 * Update Redirection Table. There are 128 8bit-entries which can be 2384 * manage as 32 32bit-entries 2385 */ 2386 for (i = 0; i < reta_size; i += 4) { 2387 /* Handling 4 RSS entries per loop */ 2388 idx = i / RTE_RETA_GROUP_SIZE; 2389 shift = i % RTE_RETA_GROUP_SIZE; 2390 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 0xF); 2391 2392 if (!mask) 2393 continue; 2394 2395 reta = 0; 2396 /* If all 4 entries were set, don't need read RETA register */ 2397 if (mask != 0xF) 2398 reta = nn_cfg_readl(hw, NFP_NET_CFG_RSS_ITBL + i); 2399 2400 for (j = 0; j < 4; j++) { 2401 if (!(mask & (0x1 << j))) 2402 continue; 2403 if (mask != 0xF) 2404 /* Clearing the entry bits */ 2405 reta &= ~(0xFF << (8 * j)); 2406 reta |= reta_conf[idx].reta[shift + j] << (8 * j); 2407 } 2408 nn_cfg_writel(hw, NFP_NET_CFG_RSS_ITBL + (idx * 64) + shift, 2409 reta); 2410 } 2411 2412 update = NFP_NET_CFG_UPDATE_RSS; 2413 2414 if (nfp_net_reconfig(hw, hw->ctrl, update) < 0) 2415 return -EIO; 2416 2417 return 0; 2418 } 2419 2420 /* Query Redirection Table(RETA) of Receive Side Scaling of Ethernet device. */ 2421 static int 2422 nfp_net_reta_query(struct rte_eth_dev *dev, 2423 struct rte_eth_rss_reta_entry64 *reta_conf, 2424 uint16_t reta_size) 2425 { 2426 uint8_t i, j, mask; 2427 int idx, shift; 2428 uint32_t reta; 2429 struct nfp_net_hw *hw; 2430 2431 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2432 2433 if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS)) 2434 return -EINVAL; 2435 2436 if (reta_size != NFP_NET_CFG_RSS_ITBL_SZ) { 2437 RTE_LOG(ERR, PMD, "The size of hash lookup table configured " 2438 "(%d) doesn't match the number hardware can supported " 2439 "(%d)\n", reta_size, NFP_NET_CFG_RSS_ITBL_SZ); 2440 return -EINVAL; 2441 } 2442 2443 /* 2444 * Reading Redirection Table. There are 128 8bit-entries which can be 2445 * manage as 32 32bit-entries 2446 */ 2447 for (i = 0; i < reta_size; i += 4) { 2448 /* Handling 4 RSS entries per loop */ 2449 idx = i / RTE_RETA_GROUP_SIZE; 2450 shift = i % RTE_RETA_GROUP_SIZE; 2451 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 0xF); 2452 2453 if (!mask) 2454 continue; 2455 2456 reta = nn_cfg_readl(hw, NFP_NET_CFG_RSS_ITBL + (idx * 64) + 2457 shift); 2458 for (j = 0; j < 4; j++) { 2459 if (!(mask & (0x1 << j))) 2460 continue; 2461 reta_conf->reta[shift + j] = 2462 (uint8_t)((reta >> (8 * j)) & 0xF); 2463 } 2464 } 2465 return 0; 2466 } 2467 2468 static int 2469 nfp_net_rss_hash_update(struct rte_eth_dev *dev, 2470 struct rte_eth_rss_conf *rss_conf) 2471 { 2472 uint32_t update; 2473 uint32_t cfg_rss_ctrl = 0; 2474 uint8_t key; 2475 uint64_t rss_hf; 2476 int i; 2477 struct nfp_net_hw *hw; 2478 2479 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2480 2481 rss_hf = rss_conf->rss_hf; 2482 2483 /* Checking if RSS is enabled */ 2484 if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS)) { 2485 if (rss_hf != 0) { /* Enable RSS? */ 2486 RTE_LOG(ERR, PMD, "RSS unsupported\n"); 2487 return -EINVAL; 2488 } 2489 return 0; /* Nothing to do */ 2490 } 2491 2492 if (rss_conf->rss_key_len > NFP_NET_CFG_RSS_KEY_SZ) { 2493 RTE_LOG(ERR, PMD, "hash key too long\n"); 2494 return -EINVAL; 2495 } 2496 2497 if (rss_hf & ETH_RSS_IPV4) 2498 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4 | 2499 NFP_NET_CFG_RSS_IPV4_TCP | 2500 NFP_NET_CFG_RSS_IPV4_UDP; 2501 2502 if (rss_hf & ETH_RSS_IPV6) 2503 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6 | 2504 NFP_NET_CFG_RSS_IPV6_TCP | 2505 NFP_NET_CFG_RSS_IPV6_UDP; 2506 2507 cfg_rss_ctrl |= NFP_NET_CFG_RSS_MASK; 2508 cfg_rss_ctrl |= NFP_NET_CFG_RSS_TOEPLITZ; 2509 2510 /* configuring where to apply the RSS hash */ 2511 nn_cfg_writel(hw, NFP_NET_CFG_RSS_CTRL, cfg_rss_ctrl); 2512 2513 /* Writing the key byte a byte */ 2514 for (i = 0; i < rss_conf->rss_key_len; i++) { 2515 memcpy(&key, &rss_conf->rss_key[i], 1); 2516 nn_cfg_writeb(hw, NFP_NET_CFG_RSS_KEY + i, key); 2517 } 2518 2519 /* Writing the key size */ 2520 nn_cfg_writeb(hw, NFP_NET_CFG_RSS_KEY_SZ, rss_conf->rss_key_len); 2521 2522 update = NFP_NET_CFG_UPDATE_RSS; 2523 2524 if (nfp_net_reconfig(hw, hw->ctrl, update) < 0) 2525 return -EIO; 2526 2527 return 0; 2528 } 2529 2530 static int 2531 nfp_net_rss_hash_conf_get(struct rte_eth_dev *dev, 2532 struct rte_eth_rss_conf *rss_conf) 2533 { 2534 uint64_t rss_hf; 2535 uint32_t cfg_rss_ctrl; 2536 uint8_t key; 2537 int i; 2538 struct nfp_net_hw *hw; 2539 2540 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2541 2542 if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS)) 2543 return -EINVAL; 2544 2545 rss_hf = rss_conf->rss_hf; 2546 cfg_rss_ctrl = nn_cfg_readl(hw, NFP_NET_CFG_RSS_CTRL); 2547 2548 if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4) 2549 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_NONFRAG_IPV4_UDP; 2550 2551 if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4_TCP) 2552 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP; 2553 2554 if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV6_TCP) 2555 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP; 2556 2557 if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4_UDP) 2558 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP; 2559 2560 if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV6_UDP) 2561 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP; 2562 2563 if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV6) 2564 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_NONFRAG_IPV6_UDP; 2565 2566 /* Reading the key size */ 2567 rss_conf->rss_key_len = nn_cfg_readl(hw, NFP_NET_CFG_RSS_KEY_SZ); 2568 2569 /* Reading the key byte a byte */ 2570 for (i = 0; i < rss_conf->rss_key_len; i++) { 2571 key = nn_cfg_readb(hw, NFP_NET_CFG_RSS_KEY + i); 2572 memcpy(&rss_conf->rss_key[i], &key, 1); 2573 } 2574 2575 return 0; 2576 } 2577 2578 /* Initialise and register driver with DPDK Application */ 2579 static const struct eth_dev_ops nfp_net_eth_dev_ops = { 2580 .dev_configure = nfp_net_configure, 2581 .dev_start = nfp_net_start, 2582 .dev_stop = nfp_net_stop, 2583 .dev_close = nfp_net_close, 2584 .promiscuous_enable = nfp_net_promisc_enable, 2585 .promiscuous_disable = nfp_net_promisc_disable, 2586 .link_update = nfp_net_link_update, 2587 .stats_get = nfp_net_stats_get, 2588 .stats_reset = nfp_net_stats_reset, 2589 .dev_infos_get = nfp_net_infos_get, 2590 .dev_supported_ptypes_get = nfp_net_supported_ptypes_get, 2591 .mtu_set = nfp_net_dev_mtu_set, 2592 .vlan_offload_set = nfp_net_vlan_offload_set, 2593 .reta_update = nfp_net_reta_update, 2594 .reta_query = nfp_net_reta_query, 2595 .rss_hash_update = nfp_net_rss_hash_update, 2596 .rss_hash_conf_get = nfp_net_rss_hash_conf_get, 2597 .rx_queue_setup = nfp_net_rx_queue_setup, 2598 .rx_queue_release = nfp_net_rx_queue_release, 2599 .rx_queue_count = nfp_net_rx_queue_count, 2600 .tx_queue_setup = nfp_net_tx_queue_setup, 2601 .tx_queue_release = nfp_net_tx_queue_release, 2602 .rx_queue_intr_enable = nfp_rx_queue_intr_enable, 2603 .rx_queue_intr_disable = nfp_rx_queue_intr_disable, 2604 }; 2605 2606 /* 2607 * All eth_dev created got its private data, but before nfp_net_init, that 2608 * private data is referencing private data for all the PF ports. This is due 2609 * to how the vNIC bars are mapped based on first port, so all ports need info 2610 * about port 0 private data. Inside nfp_net_init the private data pointer is 2611 * changed to the right address for each port once the bars have been mapped. 2612 * 2613 * This functions helps to find out which port and therefore which offset 2614 * inside the private data array to use. 2615 */ 2616 static int 2617 get_pf_port_number(char *name) 2618 { 2619 char *pf_str = name; 2620 int size = 0; 2621 2622 while ((*pf_str != '_') && (*pf_str != '\0') && (size++ < 30)) 2623 pf_str++; 2624 2625 if (size == 30) 2626 /* 2627 * This should not happen at all and it would mean major 2628 * implementation fault. 2629 */ 2630 rte_panic("nfp_net: problem with pf device name\n"); 2631 2632 /* Expecting _portX with X within [0,7] */ 2633 pf_str += 5; 2634 2635 return (int)strtol(pf_str, NULL, 10); 2636 } 2637 2638 static int 2639 nfp_net_init(struct rte_eth_dev *eth_dev) 2640 { 2641 struct rte_pci_device *pci_dev; 2642 struct nfp_net_hw *hw, *hwport0; 2643 2644 uint64_t tx_bar_off = 0, rx_bar_off = 0; 2645 uint32_t start_q; 2646 int stride = 4; 2647 2648 nspu_desc_t *nspu_desc = NULL; 2649 uint64_t bar_offset; 2650 int port = 0; 2651 2652 PMD_INIT_FUNC_TRACE(); 2653 2654 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 2655 2656 /* NFP can not handle DMA addresses requiring more than 40 bits */ 2657 if (rte_eal_check_dma_mask(40) < 0) { 2658 RTE_LOG(INFO, PMD, "device %s can not be used:", 2659 pci_dev->device.name); 2660 RTE_LOG(INFO, PMD, "\trestricted dma mask to 40 bits!\n"); 2661 return -ENODEV; 2662 }; 2663 2664 if ((pci_dev->id.device_id == PCI_DEVICE_ID_NFP4000_PF_NIC) || 2665 (pci_dev->id.device_id == PCI_DEVICE_ID_NFP6000_PF_NIC)) { 2666 port = get_pf_port_number(eth_dev->data->name); 2667 if (port < 0 || port > 7) { 2668 RTE_LOG(ERR, PMD, "Port value is wrong\n"); 2669 return -ENODEV; 2670 } 2671 2672 PMD_INIT_LOG(DEBUG, "Working with PF port value %d\n", port); 2673 2674 /* This points to port 0 private data */ 2675 hwport0 = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 2676 2677 /* This points to the specific port private data */ 2678 hw = &hwport0[port]; 2679 hw->pf_port_idx = port; 2680 } else { 2681 hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 2682 hwport0 = 0; 2683 } 2684 2685 eth_dev->dev_ops = &nfp_net_eth_dev_ops; 2686 eth_dev->rx_pkt_burst = &nfp_net_recv_pkts; 2687 eth_dev->tx_pkt_burst = &nfp_net_xmit_pkts; 2688 2689 /* For secondary processes, the primary has done all the work */ 2690 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2691 return 0; 2692 2693 rte_eth_copy_pci_info(eth_dev, pci_dev); 2694 2695 hw->device_id = pci_dev->id.device_id; 2696 hw->vendor_id = pci_dev->id.vendor_id; 2697 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 2698 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 2699 2700 PMD_INIT_LOG(DEBUG, "nfp_net: device (%u:%u) %u:%u:%u:%u", 2701 pci_dev->id.vendor_id, pci_dev->id.device_id, 2702 pci_dev->addr.domain, pci_dev->addr.bus, 2703 pci_dev->addr.devid, pci_dev->addr.function); 2704 2705 hw->ctrl_bar = (uint8_t *)pci_dev->mem_resource[0].addr; 2706 if (hw->ctrl_bar == NULL) { 2707 RTE_LOG(ERR, PMD, 2708 "hw->ctrl_bar is NULL. BAR0 not configured\n"); 2709 return -ENODEV; 2710 } 2711 2712 if (hw->is_pf && port == 0) { 2713 nspu_desc = hw->nspu_desc; 2714 2715 if (nfp_nsp_map_ctrl_bar(nspu_desc, &bar_offset) != 0) { 2716 /* 2717 * A firmware should be there after PF probe so this 2718 * should not happen. 2719 */ 2720 RTE_LOG(ERR, PMD, "PF BAR symbol resolution failed\n"); 2721 return -ENODEV; 2722 } 2723 2724 /* vNIC PF control BAR is a subset of PF PCI device BAR */ 2725 hw->ctrl_bar += bar_offset; 2726 PMD_INIT_LOG(DEBUG, "ctrl bar: %p\n", hw->ctrl_bar); 2727 } 2728 2729 if (port > 0) { 2730 if (!hwport0->ctrl_bar) 2731 return -ENODEV; 2732 2733 /* address based on port0 offset */ 2734 hw->ctrl_bar = hwport0->ctrl_bar + 2735 (port * NFP_PF_CSR_SLICE_SIZE); 2736 } 2737 2738 PMD_INIT_LOG(DEBUG, "ctrl bar: %p\n", hw->ctrl_bar); 2739 2740 hw->max_rx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_RXRINGS); 2741 hw->max_tx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_TXRINGS); 2742 2743 /* Work out where in the BAR the queues start. */ 2744 switch (pci_dev->id.device_id) { 2745 case PCI_DEVICE_ID_NFP4000_PF_NIC: 2746 case PCI_DEVICE_ID_NFP6000_PF_NIC: 2747 case PCI_DEVICE_ID_NFP6000_VF_NIC: 2748 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_TXQ); 2749 tx_bar_off = NFP_PCIE_QUEUE(start_q); 2750 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_RXQ); 2751 rx_bar_off = NFP_PCIE_QUEUE(start_q); 2752 break; 2753 default: 2754 RTE_LOG(ERR, PMD, "nfp_net: no device ID matching\n"); 2755 return -ENODEV; 2756 } 2757 2758 PMD_INIT_LOG(DEBUG, "tx_bar_off: 0x%" PRIx64 "\n", tx_bar_off); 2759 PMD_INIT_LOG(DEBUG, "rx_bar_off: 0x%" PRIx64 "\n", rx_bar_off); 2760 2761 if (hw->is_pf && port == 0) { 2762 /* configure access to tx/rx vNIC BARs */ 2763 nfp_nsp_map_queues_bar(nspu_desc, &bar_offset); 2764 PMD_INIT_LOG(DEBUG, "tx/rx bar_offset: %" PRIx64 "\n", 2765 bar_offset); 2766 hwport0->hw_queues = (uint8_t *)pci_dev->mem_resource[0].addr; 2767 2768 /* vNIC PF tx/rx BARs are a subset of PF PCI device */ 2769 hwport0->hw_queues += bar_offset; 2770 2771 /* Lets seize the chance to read eth table from hw */ 2772 if (nfp_nsp_eth_read_table(nspu_desc, &hw->eth_table)) 2773 return -ENODEV; 2774 } 2775 2776 if (hw->is_pf) { 2777 hw->tx_bar = hwport0->hw_queues + tx_bar_off; 2778 hw->rx_bar = hwport0->hw_queues + rx_bar_off; 2779 eth_dev->data->dev_private = hw; 2780 } else { 2781 hw->tx_bar = (uint8_t *)pci_dev->mem_resource[2].addr + 2782 tx_bar_off; 2783 hw->rx_bar = (uint8_t *)pci_dev->mem_resource[2].addr + 2784 rx_bar_off; 2785 } 2786 2787 PMD_INIT_LOG(DEBUG, "ctrl_bar: %p, tx_bar: %p, rx_bar: %p", 2788 hw->ctrl_bar, hw->tx_bar, hw->rx_bar); 2789 2790 nfp_net_cfg_queue_setup(hw); 2791 2792 /* Get some of the read-only fields from the config BAR */ 2793 hw->ver = nn_cfg_readl(hw, NFP_NET_CFG_VERSION); 2794 hw->cap = nn_cfg_readl(hw, NFP_NET_CFG_CAP); 2795 hw->max_mtu = nn_cfg_readl(hw, NFP_NET_CFG_MAX_MTU); 2796 hw->mtu = ETHER_MTU; 2797 2798 if (NFD_CFG_MAJOR_VERSION_of(hw->ver) < 2) 2799 hw->rx_offset = NFP_NET_RX_OFFSET; 2800 else 2801 hw->rx_offset = nn_cfg_readl(hw, NFP_NET_CFG_RX_OFFSET_ADDR); 2802 2803 PMD_INIT_LOG(INFO, "VER: %#x, Maximum supported MTU: %d", 2804 hw->ver, hw->max_mtu); 2805 PMD_INIT_LOG(INFO, "CAP: %#x, %s%s%s%s%s%s%s%s%s%s%s", hw->cap, 2806 hw->cap & NFP_NET_CFG_CTRL_PROMISC ? "PROMISC " : "", 2807 hw->cap & NFP_NET_CFG_CTRL_L2BC ? "L2BCFILT " : "", 2808 hw->cap & NFP_NET_CFG_CTRL_L2MC ? "L2MCFILT " : "", 2809 hw->cap & NFP_NET_CFG_CTRL_RXCSUM ? "RXCSUM " : "", 2810 hw->cap & NFP_NET_CFG_CTRL_TXCSUM ? "TXCSUM " : "", 2811 hw->cap & NFP_NET_CFG_CTRL_RXVLAN ? "RXVLAN " : "", 2812 hw->cap & NFP_NET_CFG_CTRL_TXVLAN ? "TXVLAN " : "", 2813 hw->cap & NFP_NET_CFG_CTRL_SCATTER ? "SCATTER " : "", 2814 hw->cap & NFP_NET_CFG_CTRL_GATHER ? "GATHER " : "", 2815 hw->cap & NFP_NET_CFG_CTRL_LSO ? "TSO " : "", 2816 hw->cap & NFP_NET_CFG_CTRL_RSS ? "RSS " : ""); 2817 2818 hw->ctrl = 0; 2819 2820 hw->stride_rx = stride; 2821 hw->stride_tx = stride; 2822 2823 PMD_INIT_LOG(INFO, "max_rx_queues: %u, max_tx_queues: %u", 2824 hw->max_rx_queues, hw->max_tx_queues); 2825 2826 /* Initializing spinlock for reconfigs */ 2827 rte_spinlock_init(&hw->reconfig_lock); 2828 2829 /* Allocating memory for mac addr */ 2830 eth_dev->data->mac_addrs = rte_zmalloc("mac_addr", ETHER_ADDR_LEN, 0); 2831 if (eth_dev->data->mac_addrs == NULL) { 2832 PMD_INIT_LOG(ERR, "Failed to space for MAC address"); 2833 return -ENOMEM; 2834 } 2835 2836 if (hw->is_pf) { 2837 nfp_net_pf_read_mac(hwport0, port); 2838 nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr); 2839 } else { 2840 nfp_net_vf_read_mac(hw); 2841 } 2842 2843 if (!is_valid_assigned_ether_addr((struct ether_addr *)&hw->mac_addr)) { 2844 /* Using random mac addresses for VFs */ 2845 eth_random_addr(&hw->mac_addr[0]); 2846 nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr); 2847 } 2848 2849 /* Copying mac address to DPDK eth_dev struct */ 2850 ether_addr_copy((struct ether_addr *)hw->mac_addr, 2851 ð_dev->data->mac_addrs[0]); 2852 2853 PMD_INIT_LOG(INFO, "port %d VendorID=0x%x DeviceID=0x%x " 2854 "mac=%02x:%02x:%02x:%02x:%02x:%02x", 2855 eth_dev->data->port_id, pci_dev->id.vendor_id, 2856 pci_dev->id.device_id, 2857 hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2], 2858 hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]); 2859 2860 /* Registering LSC interrupt handler */ 2861 rte_intr_callback_register(&pci_dev->intr_handle, 2862 nfp_net_dev_interrupt_handler, 2863 (void *)eth_dev); 2864 2865 /* Telling the firmware about the LSC interrupt entry */ 2866 nn_cfg_writeb(hw, NFP_NET_CFG_LSC, NFP_NET_IRQ_LSC_IDX); 2867 2868 /* Recording current stats counters values */ 2869 nfp_net_stats_reset(eth_dev); 2870 2871 return 0; 2872 } 2873 2874 static int 2875 nfp_pf_create_dev(struct rte_pci_device *dev, int port, int ports, 2876 nfpu_desc_t *nfpu_desc, void **priv) 2877 { 2878 struct rte_eth_dev *eth_dev; 2879 struct nfp_net_hw *hw; 2880 char *port_name; 2881 int ret; 2882 2883 port_name = rte_zmalloc("nfp_pf_port_name", 100, 0); 2884 if (!port_name) 2885 return -ENOMEM; 2886 2887 if (ports > 1) 2888 sprintf(port_name, "%s_port%d", dev->device.name, port); 2889 else 2890 sprintf(port_name, "%s", dev->device.name); 2891 2892 eth_dev = rte_eth_dev_allocate(port_name); 2893 if (!eth_dev) 2894 return -ENOMEM; 2895 2896 if (port == 0) { 2897 *priv = rte_zmalloc(port_name, 2898 sizeof(struct nfp_net_adapter) * ports, 2899 RTE_CACHE_LINE_SIZE); 2900 if (!*priv) { 2901 rte_eth_dev_release_port(eth_dev); 2902 return -ENOMEM; 2903 } 2904 } 2905 2906 eth_dev->data->dev_private = *priv; 2907 2908 /* 2909 * dev_private pointing to port0 dev_private because we need 2910 * to configure vNIC bars based on port0 at nfp_net_init. 2911 * Then dev_private is adjusted per port. 2912 */ 2913 hw = (struct nfp_net_hw *)(eth_dev->data->dev_private) + port; 2914 hw->nspu_desc = nfpu_desc->nspu; 2915 hw->nfpu_desc = nfpu_desc; 2916 hw->is_pf = 1; 2917 if (ports > 1) 2918 hw->pf_multiport_enabled = 1; 2919 2920 eth_dev->device = &dev->device; 2921 rte_eth_copy_pci_info(eth_dev, dev); 2922 2923 ret = nfp_net_init(eth_dev); 2924 2925 if (ret) 2926 rte_eth_dev_release_port(eth_dev); 2927 2928 rte_free(port_name); 2929 2930 return ret; 2931 } 2932 2933 static int nfp_pf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 2934 struct rte_pci_device *dev) 2935 { 2936 nfpu_desc_t *nfpu_desc; 2937 nspu_desc_t *nspu_desc; 2938 uint64_t offset_symbol; 2939 uint8_t *bar_offset; 2940 int major, minor; 2941 int total_ports; 2942 void *priv = 0; 2943 int ret = -ENODEV; 2944 int i; 2945 2946 if (!dev) 2947 return ret; 2948 2949 nfpu_desc = rte_malloc("nfp nfpu", sizeof(nfpu_desc_t), 0); 2950 if (!nfpu_desc) 2951 return -ENOMEM; 2952 2953 if (nfpu_open(dev, nfpu_desc, 0) < 0) { 2954 RTE_LOG(ERR, PMD, 2955 "nfpu_open failed\n"); 2956 goto nfpu_error; 2957 } 2958 2959 nspu_desc = nfpu_desc->nspu; 2960 2961 2962 /* Check NSP ABI version */ 2963 if (nfp_nsp_get_abi_version(nspu_desc, &major, &minor) < 0) { 2964 RTE_LOG(INFO, PMD, "NFP NSP not present\n"); 2965 goto error; 2966 } 2967 PMD_INIT_LOG(INFO, "nspu ABI version: %d.%d\n", major, minor); 2968 2969 if ((major == 0) && (minor < 20)) { 2970 RTE_LOG(INFO, PMD, "NFP NSP ABI version too old. Required 0.20 or higher\n"); 2971 goto error; 2972 } 2973 2974 ret = nfp_nsp_fw_setup(nspu_desc, "nfd_cfg_pf0_num_ports", 2975 &offset_symbol); 2976 if (ret) 2977 goto error; 2978 2979 bar_offset = (uint8_t *)dev->mem_resource[0].addr; 2980 bar_offset += offset_symbol; 2981 total_ports = (uint32_t)*bar_offset; 2982 PMD_INIT_LOG(INFO, "Total pf ports: %d\n", total_ports); 2983 2984 if (total_ports <= 0 || total_ports > 8) { 2985 RTE_LOG(ERR, PMD, "nfd_cfg_pf0_num_ports symbol with wrong value"); 2986 ret = -ENODEV; 2987 goto error; 2988 } 2989 2990 for (i = 0; i < total_ports; i++) { 2991 ret = nfp_pf_create_dev(dev, i, total_ports, nfpu_desc, &priv); 2992 if (ret) 2993 goto error; 2994 } 2995 2996 return 0; 2997 2998 error: 2999 nfpu_close(nfpu_desc); 3000 nfpu_error: 3001 rte_free(nfpu_desc); 3002 3003 return ret; 3004 } 3005 3006 static const struct rte_pci_id pci_id_nfp_pf_net_map[] = { 3007 { 3008 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME, 3009 PCI_DEVICE_ID_NFP4000_PF_NIC) 3010 }, 3011 { 3012 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME, 3013 PCI_DEVICE_ID_NFP6000_PF_NIC) 3014 }, 3015 { 3016 .vendor_id = 0, 3017 }, 3018 }; 3019 3020 static const struct rte_pci_id pci_id_nfp_vf_net_map[] = { 3021 { 3022 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME, 3023 PCI_DEVICE_ID_NFP6000_VF_NIC) 3024 }, 3025 { 3026 .vendor_id = 0, 3027 }, 3028 }; 3029 3030 static int eth_nfp_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 3031 struct rte_pci_device *pci_dev) 3032 { 3033 return rte_eth_dev_pci_generic_probe(pci_dev, 3034 sizeof(struct nfp_net_adapter), nfp_net_init); 3035 } 3036 3037 static int eth_nfp_pci_remove(struct rte_pci_device *pci_dev) 3038 { 3039 struct rte_eth_dev *eth_dev; 3040 struct nfp_net_hw *hw, *hwport0; 3041 int port = 0; 3042 3043 eth_dev = rte_eth_dev_allocated(pci_dev->device.name); 3044 if ((pci_dev->id.device_id == PCI_DEVICE_ID_NFP4000_PF_NIC) || 3045 (pci_dev->id.device_id == PCI_DEVICE_ID_NFP6000_PF_NIC)) { 3046 port = get_pf_port_number(eth_dev->data->name); 3047 hwport0 = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 3048 hw = &hwport0[port]; 3049 } else { 3050 hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 3051 } 3052 /* hotplug is not possible with multiport PF */ 3053 if (hw->pf_multiport_enabled) 3054 return -ENOTSUP; 3055 return rte_eth_dev_pci_generic_remove(pci_dev, NULL); 3056 } 3057 3058 static struct rte_pci_driver rte_nfp_net_pf_pmd = { 3059 .id_table = pci_id_nfp_pf_net_map, 3060 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | 3061 RTE_PCI_DRV_IOVA_AS_VA, 3062 .probe = nfp_pf_pci_probe, 3063 .remove = eth_nfp_pci_remove, 3064 }; 3065 3066 static struct rte_pci_driver rte_nfp_net_vf_pmd = { 3067 .id_table = pci_id_nfp_vf_net_map, 3068 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | 3069 RTE_PCI_DRV_IOVA_AS_VA, 3070 .probe = eth_nfp_pci_probe, 3071 .remove = eth_nfp_pci_remove, 3072 }; 3073 3074 RTE_PMD_REGISTER_PCI(net_nfp_pf, rte_nfp_net_pf_pmd); 3075 RTE_PMD_REGISTER_PCI(net_nfp_vf, rte_nfp_net_vf_pmd); 3076 RTE_PMD_REGISTER_PCI_TABLE(net_nfp_pf, pci_id_nfp_pf_net_map); 3077 RTE_PMD_REGISTER_PCI_TABLE(net_nfp_vf, pci_id_nfp_vf_net_map); 3078 RTE_PMD_REGISTER_KMOD_DEP(net_nfp_pf, "* igb_uio | uio_pci_generic | vfio"); 3079 RTE_PMD_REGISTER_KMOD_DEP(net_nfp_vf, "* igb_uio | uio_pci_generic | vfio"); 3080 3081 /* 3082 * Local variables: 3083 * c-file-style: "Linux" 3084 * indent-tabs-mode: t 3085 * End: 3086 */ 3087