1 /* 2 * Copyright (c) 2014-2018 Netronome Systems, Inc. 3 * All rights reserved. 4 * 5 * Small portions derived from code Copyright(c) 2010-2015 Intel Corporation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution 16 * 17 * 3. Neither the name of the copyright holder nor the names of its 18 * contributors may be used to endorse or promote products derived from this 19 * software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * vim:shiftwidth=8:noexpandtab 36 * 37 * @file dpdk/pmd/nfp_net.c 38 * 39 * Netronome vNIC DPDK Poll-Mode Driver: Main entry point 40 */ 41 42 #include <rte_byteorder.h> 43 #include <rte_common.h> 44 #include <rte_log.h> 45 #include <rte_debug.h> 46 #include <rte_ethdev_driver.h> 47 #include <rte_ethdev_pci.h> 48 #include <rte_dev.h> 49 #include <rte_ether.h> 50 #include <rte_malloc.h> 51 #include <rte_memzone.h> 52 #include <rte_mempool.h> 53 #include <rte_version.h> 54 #include <rte_string_fns.h> 55 #include <rte_alarm.h> 56 #include <rte_spinlock.h> 57 58 #include "nfpcore/nfp_cpp.h" 59 #include "nfpcore/nfp_nffw.h" 60 #include "nfpcore/nfp_hwinfo.h" 61 #include "nfpcore/nfp_mip.h" 62 #include "nfpcore/nfp_rtsym.h" 63 #include "nfpcore/nfp_nsp.h" 64 65 #include "nfp_net_pmd.h" 66 #include "nfp_net_logs.h" 67 #include "nfp_net_ctrl.h" 68 69 /* Prototypes */ 70 static void nfp_net_close(struct rte_eth_dev *dev); 71 static int nfp_net_configure(struct rte_eth_dev *dev); 72 static void nfp_net_dev_interrupt_handler(void *param); 73 static void nfp_net_dev_interrupt_delayed_handler(void *param); 74 static int nfp_net_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 75 static void nfp_net_infos_get(struct rte_eth_dev *dev, 76 struct rte_eth_dev_info *dev_info); 77 static int nfp_net_init(struct rte_eth_dev *eth_dev); 78 static int nfp_net_link_update(struct rte_eth_dev *dev, int wait_to_complete); 79 static void nfp_net_promisc_enable(struct rte_eth_dev *dev); 80 static void nfp_net_promisc_disable(struct rte_eth_dev *dev); 81 static int nfp_net_rx_fill_freelist(struct nfp_net_rxq *rxq); 82 static uint32_t nfp_net_rx_queue_count(struct rte_eth_dev *dev, 83 uint16_t queue_idx); 84 static uint16_t nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 85 uint16_t nb_pkts); 86 static void nfp_net_rx_queue_release(void *rxq); 87 static int nfp_net_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 88 uint16_t nb_desc, unsigned int socket_id, 89 const struct rte_eth_rxconf *rx_conf, 90 struct rte_mempool *mp); 91 static int nfp_net_tx_free_bufs(struct nfp_net_txq *txq); 92 static void nfp_net_tx_queue_release(void *txq); 93 static int nfp_net_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 94 uint16_t nb_desc, unsigned int socket_id, 95 const struct rte_eth_txconf *tx_conf); 96 static int nfp_net_start(struct rte_eth_dev *dev); 97 static int nfp_net_stats_get(struct rte_eth_dev *dev, 98 struct rte_eth_stats *stats); 99 static void nfp_net_stats_reset(struct rte_eth_dev *dev); 100 static void nfp_net_stop(struct rte_eth_dev *dev); 101 static uint16_t nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 102 uint16_t nb_pkts); 103 104 static int nfp_net_rss_config_default(struct rte_eth_dev *dev); 105 static int nfp_net_rss_hash_update(struct rte_eth_dev *dev, 106 struct rte_eth_rss_conf *rss_conf); 107 static int nfp_net_rss_reta_write(struct rte_eth_dev *dev, 108 struct rte_eth_rss_reta_entry64 *reta_conf, 109 uint16_t reta_size); 110 static int nfp_net_rss_hash_write(struct rte_eth_dev *dev, 111 struct rte_eth_rss_conf *rss_conf); 112 static int nfp_set_mac_addr(struct rte_eth_dev *dev, 113 struct ether_addr *mac_addr); 114 115 /* The offset of the queue controller queues in the PCIe Target */ 116 #define NFP_PCIE_QUEUE(_q) (0x80000 + (NFP_QCP_QUEUE_ADDR_SZ * ((_q) & 0xff))) 117 118 /* Maximum value which can be added to a queue with one transaction */ 119 #define NFP_QCP_MAX_ADD 0x7f 120 121 #define RTE_MBUF_DMA_ADDR_DEFAULT(mb) \ 122 (uint64_t)((mb)->buf_iova + RTE_PKTMBUF_HEADROOM) 123 124 /* nfp_qcp_ptr - Read or Write Pointer of a queue */ 125 enum nfp_qcp_ptr { 126 NFP_QCP_READ_PTR = 0, 127 NFP_QCP_WRITE_PTR 128 }; 129 130 /* 131 * nfp_qcp_ptr_add - Add the value to the selected pointer of a queue 132 * @q: Base address for queue structure 133 * @ptr: Add to the Read or Write pointer 134 * @val: Value to add to the queue pointer 135 * 136 * If @val is greater than @NFP_QCP_MAX_ADD multiple writes are performed. 137 */ 138 static inline void 139 nfp_qcp_ptr_add(uint8_t *q, enum nfp_qcp_ptr ptr, uint32_t val) 140 { 141 uint32_t off; 142 143 if (ptr == NFP_QCP_READ_PTR) 144 off = NFP_QCP_QUEUE_ADD_RPTR; 145 else 146 off = NFP_QCP_QUEUE_ADD_WPTR; 147 148 while (val > NFP_QCP_MAX_ADD) { 149 nn_writel(rte_cpu_to_le_32(NFP_QCP_MAX_ADD), q + off); 150 val -= NFP_QCP_MAX_ADD; 151 } 152 153 nn_writel(rte_cpu_to_le_32(val), q + off); 154 } 155 156 /* 157 * nfp_qcp_read - Read the current Read/Write pointer value for a queue 158 * @q: Base address for queue structure 159 * @ptr: Read or Write pointer 160 */ 161 static inline uint32_t 162 nfp_qcp_read(uint8_t *q, enum nfp_qcp_ptr ptr) 163 { 164 uint32_t off; 165 uint32_t val; 166 167 if (ptr == NFP_QCP_READ_PTR) 168 off = NFP_QCP_QUEUE_STS_LO; 169 else 170 off = NFP_QCP_QUEUE_STS_HI; 171 172 val = rte_cpu_to_le_32(nn_readl(q + off)); 173 174 if (ptr == NFP_QCP_READ_PTR) 175 return val & NFP_QCP_QUEUE_STS_LO_READPTR_mask; 176 else 177 return val & NFP_QCP_QUEUE_STS_HI_WRITEPTR_mask; 178 } 179 180 /* 181 * Functions to read/write from/to Config BAR 182 * Performs any endian conversion necessary. 183 */ 184 static inline uint8_t 185 nn_cfg_readb(struct nfp_net_hw *hw, int off) 186 { 187 return nn_readb(hw->ctrl_bar + off); 188 } 189 190 static inline void 191 nn_cfg_writeb(struct nfp_net_hw *hw, int off, uint8_t val) 192 { 193 nn_writeb(val, hw->ctrl_bar + off); 194 } 195 196 static inline uint32_t 197 nn_cfg_readl(struct nfp_net_hw *hw, int off) 198 { 199 return rte_le_to_cpu_32(nn_readl(hw->ctrl_bar + off)); 200 } 201 202 static inline void 203 nn_cfg_writel(struct nfp_net_hw *hw, int off, uint32_t val) 204 { 205 nn_writel(rte_cpu_to_le_32(val), hw->ctrl_bar + off); 206 } 207 208 static inline uint64_t 209 nn_cfg_readq(struct nfp_net_hw *hw, int off) 210 { 211 return rte_le_to_cpu_64(nn_readq(hw->ctrl_bar + off)); 212 } 213 214 static inline void 215 nn_cfg_writeq(struct nfp_net_hw *hw, int off, uint64_t val) 216 { 217 nn_writeq(rte_cpu_to_le_64(val), hw->ctrl_bar + off); 218 } 219 220 static void 221 nfp_net_rx_queue_release_mbufs(struct nfp_net_rxq *rxq) 222 { 223 unsigned i; 224 225 if (rxq->rxbufs == NULL) 226 return; 227 228 for (i = 0; i < rxq->rx_count; i++) { 229 if (rxq->rxbufs[i].mbuf) { 230 rte_pktmbuf_free_seg(rxq->rxbufs[i].mbuf); 231 rxq->rxbufs[i].mbuf = NULL; 232 } 233 } 234 } 235 236 static void 237 nfp_net_rx_queue_release(void *rx_queue) 238 { 239 struct nfp_net_rxq *rxq = rx_queue; 240 241 if (rxq) { 242 nfp_net_rx_queue_release_mbufs(rxq); 243 rte_free(rxq->rxbufs); 244 rte_free(rxq); 245 } 246 } 247 248 static void 249 nfp_net_reset_rx_queue(struct nfp_net_rxq *rxq) 250 { 251 nfp_net_rx_queue_release_mbufs(rxq); 252 rxq->rd_p = 0; 253 rxq->nb_rx_hold = 0; 254 } 255 256 static void 257 nfp_net_tx_queue_release_mbufs(struct nfp_net_txq *txq) 258 { 259 unsigned i; 260 261 if (txq->txbufs == NULL) 262 return; 263 264 for (i = 0; i < txq->tx_count; i++) { 265 if (txq->txbufs[i].mbuf) { 266 rte_pktmbuf_free_seg(txq->txbufs[i].mbuf); 267 txq->txbufs[i].mbuf = NULL; 268 } 269 } 270 } 271 272 static void 273 nfp_net_tx_queue_release(void *tx_queue) 274 { 275 struct nfp_net_txq *txq = tx_queue; 276 277 if (txq) { 278 nfp_net_tx_queue_release_mbufs(txq); 279 rte_free(txq->txbufs); 280 rte_free(txq); 281 } 282 } 283 284 static void 285 nfp_net_reset_tx_queue(struct nfp_net_txq *txq) 286 { 287 nfp_net_tx_queue_release_mbufs(txq); 288 txq->wr_p = 0; 289 txq->rd_p = 0; 290 } 291 292 static int 293 __nfp_net_reconfig(struct nfp_net_hw *hw, uint32_t update) 294 { 295 int cnt; 296 uint32_t new; 297 struct timespec wait; 298 299 PMD_DRV_LOG(DEBUG, "Writing to the configuration queue (%p)...", 300 hw->qcp_cfg); 301 302 if (hw->qcp_cfg == NULL) 303 rte_panic("Bad configuration queue pointer\n"); 304 305 nfp_qcp_ptr_add(hw->qcp_cfg, NFP_QCP_WRITE_PTR, 1); 306 307 wait.tv_sec = 0; 308 wait.tv_nsec = 1000000; 309 310 PMD_DRV_LOG(DEBUG, "Polling for update ack..."); 311 312 /* Poll update field, waiting for NFP to ack the config */ 313 for (cnt = 0; ; cnt++) { 314 new = nn_cfg_readl(hw, NFP_NET_CFG_UPDATE); 315 if (new == 0) 316 break; 317 if (new & NFP_NET_CFG_UPDATE_ERR) { 318 PMD_INIT_LOG(ERR, "Reconfig error: 0x%08x", new); 319 return -1; 320 } 321 if (cnt >= NFP_NET_POLL_TIMEOUT) { 322 PMD_INIT_LOG(ERR, "Reconfig timeout for 0x%08x after" 323 " %dms", update, cnt); 324 rte_panic("Exiting\n"); 325 } 326 nanosleep(&wait, 0); /* waiting for a 1ms */ 327 } 328 PMD_DRV_LOG(DEBUG, "Ack DONE"); 329 return 0; 330 } 331 332 /* 333 * Reconfigure the NIC 334 * @nn: device to reconfigure 335 * @ctrl: The value for the ctrl field in the BAR config 336 * @update: The value for the update field in the BAR config 337 * 338 * Write the update word to the BAR and ping the reconfig queue. Then poll 339 * until the firmware has acknowledged the update by zeroing the update word. 340 */ 341 static int 342 nfp_net_reconfig(struct nfp_net_hw *hw, uint32_t ctrl, uint32_t update) 343 { 344 uint32_t err; 345 346 PMD_DRV_LOG(DEBUG, "nfp_net_reconfig: ctrl=%08x update=%08x", 347 ctrl, update); 348 349 rte_spinlock_lock(&hw->reconfig_lock); 350 351 nn_cfg_writel(hw, NFP_NET_CFG_CTRL, ctrl); 352 nn_cfg_writel(hw, NFP_NET_CFG_UPDATE, update); 353 354 rte_wmb(); 355 356 err = __nfp_net_reconfig(hw, update); 357 358 rte_spinlock_unlock(&hw->reconfig_lock); 359 360 if (!err) 361 return 0; 362 363 /* 364 * Reconfig errors imply situations where they can be handled. 365 * Otherwise, rte_panic is called inside __nfp_net_reconfig 366 */ 367 PMD_INIT_LOG(ERR, "Error nfp_net reconfig for ctrl: %x update: %x", 368 ctrl, update); 369 return -EIO; 370 } 371 372 /* 373 * Configure an Ethernet device. This function must be invoked first 374 * before any other function in the Ethernet API. This function can 375 * also be re-invoked when a device is in the stopped state. 376 */ 377 static int 378 nfp_net_configure(struct rte_eth_dev *dev) 379 { 380 struct rte_eth_conf *dev_conf; 381 struct rte_eth_rxmode *rxmode; 382 struct rte_eth_txmode *txmode; 383 struct nfp_net_hw *hw; 384 385 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 386 387 /* 388 * A DPDK app sends info about how many queues to use and how 389 * those queues need to be configured. This is used by the 390 * DPDK core and it makes sure no more queues than those 391 * advertised by the driver are requested. This function is 392 * called after that internal process 393 */ 394 395 PMD_INIT_LOG(DEBUG, "Configure"); 396 397 dev_conf = &dev->data->dev_conf; 398 rxmode = &dev_conf->rxmode; 399 txmode = &dev_conf->txmode; 400 401 /* Checking TX mode */ 402 if (txmode->mq_mode) { 403 PMD_INIT_LOG(INFO, "TX mq_mode DCB and VMDq not supported"); 404 return -EINVAL; 405 } 406 407 /* Checking RX mode */ 408 if (rxmode->mq_mode & ETH_MQ_RX_RSS && 409 !(hw->cap & NFP_NET_CFG_CTRL_RSS)) { 410 PMD_INIT_LOG(INFO, "RSS not supported"); 411 return -EINVAL; 412 } 413 414 return 0; 415 } 416 417 static void 418 nfp_net_enable_queues(struct rte_eth_dev *dev) 419 { 420 struct nfp_net_hw *hw; 421 uint64_t enabled_queues = 0; 422 int i; 423 424 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 425 426 /* Enabling the required TX queues in the device */ 427 for (i = 0; i < dev->data->nb_tx_queues; i++) 428 enabled_queues |= (1 << i); 429 430 nn_cfg_writeq(hw, NFP_NET_CFG_TXRS_ENABLE, enabled_queues); 431 432 enabled_queues = 0; 433 434 /* Enabling the required RX queues in the device */ 435 for (i = 0; i < dev->data->nb_rx_queues; i++) 436 enabled_queues |= (1 << i); 437 438 nn_cfg_writeq(hw, NFP_NET_CFG_RXRS_ENABLE, enabled_queues); 439 } 440 441 static void 442 nfp_net_disable_queues(struct rte_eth_dev *dev) 443 { 444 struct nfp_net_hw *hw; 445 uint32_t new_ctrl, update = 0; 446 447 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 448 449 nn_cfg_writeq(hw, NFP_NET_CFG_TXRS_ENABLE, 0); 450 nn_cfg_writeq(hw, NFP_NET_CFG_RXRS_ENABLE, 0); 451 452 new_ctrl = hw->ctrl & ~NFP_NET_CFG_CTRL_ENABLE; 453 update = NFP_NET_CFG_UPDATE_GEN | NFP_NET_CFG_UPDATE_RING | 454 NFP_NET_CFG_UPDATE_MSIX; 455 456 if (hw->cap & NFP_NET_CFG_CTRL_RINGCFG) 457 new_ctrl &= ~NFP_NET_CFG_CTRL_RINGCFG; 458 459 /* If an error when reconfig we avoid to change hw state */ 460 if (nfp_net_reconfig(hw, new_ctrl, update) < 0) 461 return; 462 463 hw->ctrl = new_ctrl; 464 } 465 466 static int 467 nfp_net_rx_freelist_setup(struct rte_eth_dev *dev) 468 { 469 int i; 470 471 for (i = 0; i < dev->data->nb_rx_queues; i++) { 472 if (nfp_net_rx_fill_freelist(dev->data->rx_queues[i]) < 0) 473 return -1; 474 } 475 return 0; 476 } 477 478 static void 479 nfp_net_params_setup(struct nfp_net_hw *hw) 480 { 481 nn_cfg_writel(hw, NFP_NET_CFG_MTU, hw->mtu); 482 nn_cfg_writel(hw, NFP_NET_CFG_FLBUFSZ, hw->flbufsz); 483 } 484 485 static void 486 nfp_net_cfg_queue_setup(struct nfp_net_hw *hw) 487 { 488 hw->qcp_cfg = hw->tx_bar + NFP_QCP_QUEUE_ADDR_SZ; 489 } 490 491 #define ETH_ADDR_LEN 6 492 493 static void 494 nfp_eth_copy_mac(uint8_t *dst, const uint8_t *src) 495 { 496 int i; 497 498 for (i = 0; i < ETH_ADDR_LEN; i++) 499 dst[i] = src[i]; 500 } 501 502 static int 503 nfp_net_pf_read_mac(struct nfp_net_hw *hw, int port) 504 { 505 struct nfp_eth_table *nfp_eth_table; 506 507 nfp_eth_table = nfp_eth_read_ports(hw->cpp); 508 /* 509 * hw points to port0 private data. We need hw now pointing to 510 * right port. 511 */ 512 hw += port; 513 nfp_eth_copy_mac((uint8_t *)&hw->mac_addr, 514 (uint8_t *)&nfp_eth_table->ports[port].mac_addr); 515 516 free(nfp_eth_table); 517 return 0; 518 } 519 520 static void 521 nfp_net_vf_read_mac(struct nfp_net_hw *hw) 522 { 523 uint32_t tmp; 524 525 tmp = rte_be_to_cpu_32(nn_cfg_readl(hw, NFP_NET_CFG_MACADDR)); 526 memcpy(&hw->mac_addr[0], &tmp, 4); 527 528 tmp = rte_be_to_cpu_32(nn_cfg_readl(hw, NFP_NET_CFG_MACADDR + 4)); 529 memcpy(&hw->mac_addr[4], &tmp, 2); 530 } 531 532 static void 533 nfp_net_write_mac(struct nfp_net_hw *hw, uint8_t *mac) 534 { 535 uint32_t mac0 = *(uint32_t *)mac; 536 uint16_t mac1; 537 538 nn_writel(rte_cpu_to_be_32(mac0), hw->ctrl_bar + NFP_NET_CFG_MACADDR); 539 540 mac += 4; 541 mac1 = *(uint16_t *)mac; 542 nn_writew(rte_cpu_to_be_16(mac1), 543 hw->ctrl_bar + NFP_NET_CFG_MACADDR + 6); 544 } 545 546 int 547 nfp_set_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr) 548 { 549 struct nfp_net_hw *hw; 550 uint32_t update, ctrl; 551 552 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 553 if ((hw->ctrl & NFP_NET_CFG_CTRL_ENABLE) && 554 !(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR)) { 555 PMD_INIT_LOG(INFO, "MAC address unable to change when" 556 " port enabled"); 557 return -EBUSY; 558 } 559 560 if ((hw->ctrl & NFP_NET_CFG_CTRL_ENABLE) && 561 !(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR)) 562 return -EBUSY; 563 564 /* Writing new MAC to the specific port BAR address */ 565 nfp_net_write_mac(hw, (uint8_t *)mac_addr); 566 567 /* Signal the NIC about the change */ 568 update = NFP_NET_CFG_UPDATE_MACADDR; 569 ctrl = hw->ctrl | NFP_NET_CFG_CTRL_LIVE_ADDR; 570 if (nfp_net_reconfig(hw, ctrl, update) < 0) { 571 PMD_INIT_LOG(INFO, "MAC address update failed"); 572 return -EIO; 573 } 574 return 0; 575 } 576 577 static int 578 nfp_configure_rx_interrupt(struct rte_eth_dev *dev, 579 struct rte_intr_handle *intr_handle) 580 { 581 struct nfp_net_hw *hw; 582 int i; 583 584 if (!intr_handle->intr_vec) { 585 intr_handle->intr_vec = 586 rte_zmalloc("intr_vec", 587 dev->data->nb_rx_queues * sizeof(int), 0); 588 if (!intr_handle->intr_vec) { 589 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 590 " intr_vec", dev->data->nb_rx_queues); 591 return -ENOMEM; 592 } 593 } 594 595 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 596 597 if (intr_handle->type == RTE_INTR_HANDLE_UIO) { 598 PMD_INIT_LOG(INFO, "VF: enabling RX interrupt with UIO"); 599 /* UIO just supports one queue and no LSC*/ 600 nn_cfg_writeb(hw, NFP_NET_CFG_RXR_VEC(0), 0); 601 intr_handle->intr_vec[0] = 0; 602 } else { 603 PMD_INIT_LOG(INFO, "VF: enabling RX interrupt with VFIO"); 604 for (i = 0; i < dev->data->nb_rx_queues; i++) { 605 /* 606 * The first msix vector is reserved for non 607 * efd interrupts 608 */ 609 nn_cfg_writeb(hw, NFP_NET_CFG_RXR_VEC(i), i + 1); 610 intr_handle->intr_vec[i] = i + 1; 611 PMD_INIT_LOG(DEBUG, "intr_vec[%d]= %d", i, 612 intr_handle->intr_vec[i]); 613 } 614 } 615 616 /* Avoiding TX interrupts */ 617 hw->ctrl |= NFP_NET_CFG_CTRL_MSIX_TX_OFF; 618 return 0; 619 } 620 621 static uint32_t 622 nfp_check_offloads(struct rte_eth_dev *dev) 623 { 624 struct nfp_net_hw *hw; 625 struct rte_eth_conf *dev_conf; 626 struct rte_eth_rxmode *rxmode; 627 struct rte_eth_txmode *txmode; 628 uint32_t ctrl = 0; 629 630 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 631 632 dev_conf = &dev->data->dev_conf; 633 rxmode = &dev_conf->rxmode; 634 txmode = &dev_conf->txmode; 635 636 if (rxmode->offloads & DEV_RX_OFFLOAD_IPV4_CKSUM) { 637 if (hw->cap & NFP_NET_CFG_CTRL_RXCSUM) 638 ctrl |= NFP_NET_CFG_CTRL_RXCSUM; 639 } 640 641 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) { 642 if (hw->cap & NFP_NET_CFG_CTRL_RXVLAN) 643 ctrl |= NFP_NET_CFG_CTRL_RXVLAN; 644 } 645 646 if (rxmode->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 647 hw->mtu = rxmode->max_rx_pkt_len; 648 649 if (txmode->offloads & DEV_TX_OFFLOAD_VLAN_INSERT) 650 ctrl |= NFP_NET_CFG_CTRL_TXVLAN; 651 652 /* L2 broadcast */ 653 if (hw->cap & NFP_NET_CFG_CTRL_L2BC) 654 ctrl |= NFP_NET_CFG_CTRL_L2BC; 655 656 /* L2 multicast */ 657 if (hw->cap & NFP_NET_CFG_CTRL_L2MC) 658 ctrl |= NFP_NET_CFG_CTRL_L2MC; 659 660 /* TX checksum offload */ 661 if (txmode->offloads & DEV_TX_OFFLOAD_IPV4_CKSUM || 662 txmode->offloads & DEV_TX_OFFLOAD_UDP_CKSUM || 663 txmode->offloads & DEV_TX_OFFLOAD_TCP_CKSUM) 664 ctrl |= NFP_NET_CFG_CTRL_TXCSUM; 665 666 /* LSO offload */ 667 if (txmode->offloads & DEV_TX_OFFLOAD_TCP_TSO) { 668 if (hw->cap & NFP_NET_CFG_CTRL_LSO) 669 ctrl |= NFP_NET_CFG_CTRL_LSO; 670 else 671 ctrl |= NFP_NET_CFG_CTRL_LSO2; 672 } 673 674 /* RX gather */ 675 if (txmode->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) 676 ctrl |= NFP_NET_CFG_CTRL_GATHER; 677 678 return ctrl; 679 } 680 681 static int 682 nfp_net_start(struct rte_eth_dev *dev) 683 { 684 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 685 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 686 uint32_t new_ctrl, update = 0; 687 struct nfp_net_hw *hw; 688 struct rte_eth_conf *dev_conf; 689 struct rte_eth_rxmode *rxmode; 690 uint32_t intr_vector; 691 int ret; 692 693 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 694 695 PMD_INIT_LOG(DEBUG, "Start"); 696 697 /* Disabling queues just in case... */ 698 nfp_net_disable_queues(dev); 699 700 /* Enabling the required queues in the device */ 701 nfp_net_enable_queues(dev); 702 703 /* check and configure queue intr-vector mapping */ 704 if (dev->data->dev_conf.intr_conf.rxq != 0) { 705 if (hw->pf_multiport_enabled) { 706 PMD_INIT_LOG(ERR, "PMD rx interrupt is not supported " 707 "with NFP multiport PF"); 708 return -EINVAL; 709 } 710 if (intr_handle->type == RTE_INTR_HANDLE_UIO) { 711 /* 712 * Better not to share LSC with RX interrupts. 713 * Unregistering LSC interrupt handler 714 */ 715 rte_intr_callback_unregister(&pci_dev->intr_handle, 716 nfp_net_dev_interrupt_handler, (void *)dev); 717 718 if (dev->data->nb_rx_queues > 1) { 719 PMD_INIT_LOG(ERR, "PMD rx interrupt only " 720 "supports 1 queue with UIO"); 721 return -EIO; 722 } 723 } 724 intr_vector = dev->data->nb_rx_queues; 725 if (rte_intr_efd_enable(intr_handle, intr_vector)) 726 return -1; 727 728 nfp_configure_rx_interrupt(dev, intr_handle); 729 update = NFP_NET_CFG_UPDATE_MSIX; 730 } 731 732 rte_intr_enable(intr_handle); 733 734 new_ctrl = nfp_check_offloads(dev); 735 736 /* Writing configuration parameters in the device */ 737 nfp_net_params_setup(hw); 738 739 dev_conf = &dev->data->dev_conf; 740 rxmode = &dev_conf->rxmode; 741 742 if (rxmode->mq_mode & ETH_MQ_RX_RSS) { 743 nfp_net_rss_config_default(dev); 744 update |= NFP_NET_CFG_UPDATE_RSS; 745 new_ctrl |= NFP_NET_CFG_CTRL_RSS; 746 } 747 748 /* Enable device */ 749 new_ctrl |= NFP_NET_CFG_CTRL_ENABLE; 750 751 update |= NFP_NET_CFG_UPDATE_GEN | NFP_NET_CFG_UPDATE_RING; 752 753 if (hw->cap & NFP_NET_CFG_CTRL_RINGCFG) 754 new_ctrl |= NFP_NET_CFG_CTRL_RINGCFG; 755 756 nn_cfg_writel(hw, NFP_NET_CFG_CTRL, new_ctrl); 757 if (nfp_net_reconfig(hw, new_ctrl, update) < 0) 758 return -EIO; 759 760 /* 761 * Allocating rte mbuffs for configured rx queues. 762 * This requires queues being enabled before 763 */ 764 if (nfp_net_rx_freelist_setup(dev) < 0) { 765 ret = -ENOMEM; 766 goto error; 767 } 768 769 if (hw->is_pf) 770 /* Configure the physical port up */ 771 nfp_eth_set_configured(hw->cpp, hw->pf_port_idx, 1); 772 773 hw->ctrl = new_ctrl; 774 775 return 0; 776 777 error: 778 /* 779 * An error returned by this function should mean the app 780 * exiting and then the system releasing all the memory 781 * allocated even memory coming from hugepages. 782 * 783 * The device could be enabled at this point with some queues 784 * ready for getting packets. This is true if the call to 785 * nfp_net_rx_freelist_setup() succeeds for some queues but 786 * fails for subsequent queues. 787 * 788 * This should make the app exiting but better if we tell the 789 * device first. 790 */ 791 nfp_net_disable_queues(dev); 792 793 return ret; 794 } 795 796 /* Stop device: disable rx and tx functions to allow for reconfiguring. */ 797 static void 798 nfp_net_stop(struct rte_eth_dev *dev) 799 { 800 int i; 801 struct nfp_net_hw *hw; 802 803 PMD_INIT_LOG(DEBUG, "Stop"); 804 805 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 806 807 nfp_net_disable_queues(dev); 808 809 /* Clear queues */ 810 for (i = 0; i < dev->data->nb_tx_queues; i++) { 811 nfp_net_reset_tx_queue( 812 (struct nfp_net_txq *)dev->data->tx_queues[i]); 813 } 814 815 for (i = 0; i < dev->data->nb_rx_queues; i++) { 816 nfp_net_reset_rx_queue( 817 (struct nfp_net_rxq *)dev->data->rx_queues[i]); 818 } 819 820 if (hw->is_pf) 821 /* Configure the physical port down */ 822 nfp_eth_set_configured(hw->cpp, hw->pf_port_idx, 0); 823 } 824 825 /* Reset and stop device. The device can not be restarted. */ 826 static void 827 nfp_net_close(struct rte_eth_dev *dev) 828 { 829 struct nfp_net_hw *hw; 830 struct rte_pci_device *pci_dev; 831 int i; 832 833 PMD_INIT_LOG(DEBUG, "Close"); 834 835 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 836 pci_dev = RTE_ETH_DEV_TO_PCI(dev); 837 838 /* 839 * We assume that the DPDK application is stopping all the 840 * threads/queues before calling the device close function. 841 */ 842 843 nfp_net_disable_queues(dev); 844 845 /* Clear queues */ 846 for (i = 0; i < dev->data->nb_tx_queues; i++) { 847 nfp_net_reset_tx_queue( 848 (struct nfp_net_txq *)dev->data->tx_queues[i]); 849 } 850 851 for (i = 0; i < dev->data->nb_rx_queues; i++) { 852 nfp_net_reset_rx_queue( 853 (struct nfp_net_rxq *)dev->data->rx_queues[i]); 854 } 855 856 rte_intr_disable(&pci_dev->intr_handle); 857 nn_cfg_writeb(hw, NFP_NET_CFG_LSC, 0xff); 858 859 /* unregister callback func from eal lib */ 860 rte_intr_callback_unregister(&pci_dev->intr_handle, 861 nfp_net_dev_interrupt_handler, 862 (void *)dev); 863 864 /* 865 * The ixgbe PMD driver disables the pcie master on the 866 * device. The i40e does not... 867 */ 868 } 869 870 static void 871 nfp_net_promisc_enable(struct rte_eth_dev *dev) 872 { 873 uint32_t new_ctrl, update = 0; 874 struct nfp_net_hw *hw; 875 876 PMD_DRV_LOG(DEBUG, "Promiscuous mode enable"); 877 878 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 879 880 if (!(hw->cap & NFP_NET_CFG_CTRL_PROMISC)) { 881 PMD_INIT_LOG(INFO, "Promiscuous mode not supported"); 882 return; 883 } 884 885 if (hw->ctrl & NFP_NET_CFG_CTRL_PROMISC) { 886 PMD_DRV_LOG(INFO, "Promiscuous mode already enabled"); 887 return; 888 } 889 890 new_ctrl = hw->ctrl | NFP_NET_CFG_CTRL_PROMISC; 891 update = NFP_NET_CFG_UPDATE_GEN; 892 893 /* 894 * DPDK sets promiscuous mode on just after this call assuming 895 * it can not fail ... 896 */ 897 if (nfp_net_reconfig(hw, new_ctrl, update) < 0) 898 return; 899 900 hw->ctrl = new_ctrl; 901 } 902 903 static void 904 nfp_net_promisc_disable(struct rte_eth_dev *dev) 905 { 906 uint32_t new_ctrl, update = 0; 907 struct nfp_net_hw *hw; 908 909 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 910 911 if ((hw->ctrl & NFP_NET_CFG_CTRL_PROMISC) == 0) { 912 PMD_DRV_LOG(INFO, "Promiscuous mode already disabled"); 913 return; 914 } 915 916 new_ctrl = hw->ctrl & ~NFP_NET_CFG_CTRL_PROMISC; 917 update = NFP_NET_CFG_UPDATE_GEN; 918 919 /* 920 * DPDK sets promiscuous mode off just before this call 921 * assuming it can not fail ... 922 */ 923 if (nfp_net_reconfig(hw, new_ctrl, update) < 0) 924 return; 925 926 hw->ctrl = new_ctrl; 927 } 928 929 /* 930 * return 0 means link status changed, -1 means not changed 931 * 932 * Wait to complete is needed as it can take up to 9 seconds to get the Link 933 * status. 934 */ 935 static int 936 nfp_net_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) 937 { 938 struct nfp_net_hw *hw; 939 struct rte_eth_link link; 940 uint32_t nn_link_status; 941 int ret; 942 943 static const uint32_t ls_to_ethtool[] = { 944 [NFP_NET_CFG_STS_LINK_RATE_UNSUPPORTED] = ETH_SPEED_NUM_NONE, 945 [NFP_NET_CFG_STS_LINK_RATE_UNKNOWN] = ETH_SPEED_NUM_NONE, 946 [NFP_NET_CFG_STS_LINK_RATE_1G] = ETH_SPEED_NUM_1G, 947 [NFP_NET_CFG_STS_LINK_RATE_10G] = ETH_SPEED_NUM_10G, 948 [NFP_NET_CFG_STS_LINK_RATE_25G] = ETH_SPEED_NUM_25G, 949 [NFP_NET_CFG_STS_LINK_RATE_40G] = ETH_SPEED_NUM_40G, 950 [NFP_NET_CFG_STS_LINK_RATE_50G] = ETH_SPEED_NUM_50G, 951 [NFP_NET_CFG_STS_LINK_RATE_100G] = ETH_SPEED_NUM_100G, 952 }; 953 954 PMD_DRV_LOG(DEBUG, "Link update"); 955 956 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 957 958 nn_link_status = nn_cfg_readl(hw, NFP_NET_CFG_STS); 959 960 memset(&link, 0, sizeof(struct rte_eth_link)); 961 962 if (nn_link_status & NFP_NET_CFG_STS_LINK) 963 link.link_status = ETH_LINK_UP; 964 965 link.link_duplex = ETH_LINK_FULL_DUPLEX; 966 967 nn_link_status = (nn_link_status >> NFP_NET_CFG_STS_LINK_RATE_SHIFT) & 968 NFP_NET_CFG_STS_LINK_RATE_MASK; 969 970 if (nn_link_status >= RTE_DIM(ls_to_ethtool)) 971 link.link_speed = ETH_SPEED_NUM_NONE; 972 else 973 link.link_speed = ls_to_ethtool[nn_link_status]; 974 975 ret = rte_eth_linkstatus_set(dev, &link); 976 if (ret == 0) { 977 if (link.link_status) 978 PMD_DRV_LOG(INFO, "NIC Link is Up"); 979 else 980 PMD_DRV_LOG(INFO, "NIC Link is Down"); 981 } 982 return ret; 983 } 984 985 static int 986 nfp_net_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 987 { 988 int i; 989 struct nfp_net_hw *hw; 990 struct rte_eth_stats nfp_dev_stats; 991 992 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 993 994 /* RTE_ETHDEV_QUEUE_STAT_CNTRS default value is 16 */ 995 996 memset(&nfp_dev_stats, 0, sizeof(nfp_dev_stats)); 997 998 /* reading per RX ring stats */ 999 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1000 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS) 1001 break; 1002 1003 nfp_dev_stats.q_ipackets[i] = 1004 nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i)); 1005 1006 nfp_dev_stats.q_ipackets[i] -= 1007 hw->eth_stats_base.q_ipackets[i]; 1008 1009 nfp_dev_stats.q_ibytes[i] = 1010 nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i) + 0x8); 1011 1012 nfp_dev_stats.q_ibytes[i] -= 1013 hw->eth_stats_base.q_ibytes[i]; 1014 } 1015 1016 /* reading per TX ring stats */ 1017 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1018 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS) 1019 break; 1020 1021 nfp_dev_stats.q_opackets[i] = 1022 nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i)); 1023 1024 nfp_dev_stats.q_opackets[i] -= 1025 hw->eth_stats_base.q_opackets[i]; 1026 1027 nfp_dev_stats.q_obytes[i] = 1028 nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i) + 0x8); 1029 1030 nfp_dev_stats.q_obytes[i] -= 1031 hw->eth_stats_base.q_obytes[i]; 1032 } 1033 1034 nfp_dev_stats.ipackets = 1035 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_FRAMES); 1036 1037 nfp_dev_stats.ipackets -= hw->eth_stats_base.ipackets; 1038 1039 nfp_dev_stats.ibytes = 1040 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_OCTETS); 1041 1042 nfp_dev_stats.ibytes -= hw->eth_stats_base.ibytes; 1043 1044 nfp_dev_stats.opackets = 1045 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_FRAMES); 1046 1047 nfp_dev_stats.opackets -= hw->eth_stats_base.opackets; 1048 1049 nfp_dev_stats.obytes = 1050 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_OCTETS); 1051 1052 nfp_dev_stats.obytes -= hw->eth_stats_base.obytes; 1053 1054 /* reading general device stats */ 1055 nfp_dev_stats.ierrors = 1056 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_ERRORS); 1057 1058 nfp_dev_stats.ierrors -= hw->eth_stats_base.ierrors; 1059 1060 nfp_dev_stats.oerrors = 1061 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_ERRORS); 1062 1063 nfp_dev_stats.oerrors -= hw->eth_stats_base.oerrors; 1064 1065 /* RX ring mbuf allocation failures */ 1066 nfp_dev_stats.rx_nombuf = dev->data->rx_mbuf_alloc_failed; 1067 1068 nfp_dev_stats.imissed = 1069 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_DISCARDS); 1070 1071 nfp_dev_stats.imissed -= hw->eth_stats_base.imissed; 1072 1073 if (stats) { 1074 memcpy(stats, &nfp_dev_stats, sizeof(*stats)); 1075 return 0; 1076 } 1077 return -EINVAL; 1078 } 1079 1080 static void 1081 nfp_net_stats_reset(struct rte_eth_dev *dev) 1082 { 1083 int i; 1084 struct nfp_net_hw *hw; 1085 1086 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1087 1088 /* 1089 * hw->eth_stats_base records the per counter starting point. 1090 * Lets update it now 1091 */ 1092 1093 /* reading per RX ring stats */ 1094 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1095 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS) 1096 break; 1097 1098 hw->eth_stats_base.q_ipackets[i] = 1099 nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i)); 1100 1101 hw->eth_stats_base.q_ibytes[i] = 1102 nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i) + 0x8); 1103 } 1104 1105 /* reading per TX ring stats */ 1106 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1107 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS) 1108 break; 1109 1110 hw->eth_stats_base.q_opackets[i] = 1111 nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i)); 1112 1113 hw->eth_stats_base.q_obytes[i] = 1114 nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i) + 0x8); 1115 } 1116 1117 hw->eth_stats_base.ipackets = 1118 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_FRAMES); 1119 1120 hw->eth_stats_base.ibytes = 1121 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_OCTETS); 1122 1123 hw->eth_stats_base.opackets = 1124 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_FRAMES); 1125 1126 hw->eth_stats_base.obytes = 1127 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_OCTETS); 1128 1129 /* reading general device stats */ 1130 hw->eth_stats_base.ierrors = 1131 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_ERRORS); 1132 1133 hw->eth_stats_base.oerrors = 1134 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_ERRORS); 1135 1136 /* RX ring mbuf allocation failures */ 1137 dev->data->rx_mbuf_alloc_failed = 0; 1138 1139 hw->eth_stats_base.imissed = 1140 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_DISCARDS); 1141 } 1142 1143 static void 1144 nfp_net_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 1145 { 1146 struct nfp_net_hw *hw; 1147 1148 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1149 1150 dev_info->max_rx_queues = (uint16_t)hw->max_rx_queues; 1151 dev_info->max_tx_queues = (uint16_t)hw->max_tx_queues; 1152 dev_info->min_rx_bufsize = ETHER_MIN_MTU; 1153 dev_info->max_rx_pktlen = hw->max_mtu; 1154 /* Next should change when PF support is implemented */ 1155 dev_info->max_mac_addrs = 1; 1156 1157 if (hw->cap & NFP_NET_CFG_CTRL_RXVLAN) 1158 dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP; 1159 1160 if (hw->cap & NFP_NET_CFG_CTRL_RXCSUM) 1161 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_IPV4_CKSUM | 1162 DEV_RX_OFFLOAD_UDP_CKSUM | 1163 DEV_RX_OFFLOAD_TCP_CKSUM; 1164 1165 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_JUMBO_FRAME; 1166 1167 if (hw->cap & NFP_NET_CFG_CTRL_TXVLAN) 1168 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT; 1169 1170 if (hw->cap & NFP_NET_CFG_CTRL_TXCSUM) 1171 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_IPV4_CKSUM | 1172 DEV_TX_OFFLOAD_UDP_CKSUM | 1173 DEV_TX_OFFLOAD_TCP_CKSUM; 1174 1175 if (hw->cap & NFP_NET_CFG_CTRL_LSO_ANY) 1176 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; 1177 1178 if (hw->cap & NFP_NET_CFG_CTRL_GATHER) 1179 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_MULTI_SEGS; 1180 1181 dev_info->default_rxconf = (struct rte_eth_rxconf) { 1182 .rx_thresh = { 1183 .pthresh = DEFAULT_RX_PTHRESH, 1184 .hthresh = DEFAULT_RX_HTHRESH, 1185 .wthresh = DEFAULT_RX_WTHRESH, 1186 }, 1187 .rx_free_thresh = DEFAULT_RX_FREE_THRESH, 1188 .rx_drop_en = 0, 1189 }; 1190 1191 dev_info->default_txconf = (struct rte_eth_txconf) { 1192 .tx_thresh = { 1193 .pthresh = DEFAULT_TX_PTHRESH, 1194 .hthresh = DEFAULT_TX_HTHRESH, 1195 .wthresh = DEFAULT_TX_WTHRESH, 1196 }, 1197 .tx_free_thresh = DEFAULT_TX_FREE_THRESH, 1198 .tx_rs_thresh = DEFAULT_TX_RSBIT_THRESH, 1199 }; 1200 1201 dev_info->flow_type_rss_offloads = ETH_RSS_IPV4 | 1202 ETH_RSS_NONFRAG_IPV4_TCP | 1203 ETH_RSS_NONFRAG_IPV4_UDP | 1204 ETH_RSS_IPV6 | 1205 ETH_RSS_NONFRAG_IPV6_TCP | 1206 ETH_RSS_NONFRAG_IPV6_UDP; 1207 1208 dev_info->reta_size = NFP_NET_CFG_RSS_ITBL_SZ; 1209 dev_info->hash_key_size = NFP_NET_CFG_RSS_KEY_SZ; 1210 1211 dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G | 1212 ETH_LINK_SPEED_25G | ETH_LINK_SPEED_40G | 1213 ETH_LINK_SPEED_50G | ETH_LINK_SPEED_100G; 1214 } 1215 1216 static const uint32_t * 1217 nfp_net_supported_ptypes_get(struct rte_eth_dev *dev) 1218 { 1219 static const uint32_t ptypes[] = { 1220 /* refers to nfp_net_set_hash() */ 1221 RTE_PTYPE_INNER_L3_IPV4, 1222 RTE_PTYPE_INNER_L3_IPV6, 1223 RTE_PTYPE_INNER_L3_IPV6_EXT, 1224 RTE_PTYPE_INNER_L4_MASK, 1225 RTE_PTYPE_UNKNOWN 1226 }; 1227 1228 if (dev->rx_pkt_burst == nfp_net_recv_pkts) 1229 return ptypes; 1230 return NULL; 1231 } 1232 1233 static uint32_t 1234 nfp_net_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx) 1235 { 1236 struct nfp_net_rxq *rxq; 1237 struct nfp_net_rx_desc *rxds; 1238 uint32_t idx; 1239 uint32_t count; 1240 1241 rxq = (struct nfp_net_rxq *)dev->data->rx_queues[queue_idx]; 1242 1243 idx = rxq->rd_p; 1244 1245 count = 0; 1246 1247 /* 1248 * Other PMDs are just checking the DD bit in intervals of 4 1249 * descriptors and counting all four if the first has the DD 1250 * bit on. Of course, this is not accurate but can be good for 1251 * performance. But ideally that should be done in descriptors 1252 * chunks belonging to the same cache line 1253 */ 1254 1255 while (count < rxq->rx_count) { 1256 rxds = &rxq->rxds[idx]; 1257 if ((rxds->rxd.meta_len_dd & PCIE_DESC_RX_DD) == 0) 1258 break; 1259 1260 count++; 1261 idx++; 1262 1263 /* Wrapping? */ 1264 if ((idx) == rxq->rx_count) 1265 idx = 0; 1266 } 1267 1268 return count; 1269 } 1270 1271 static int 1272 nfp_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 1273 { 1274 struct rte_pci_device *pci_dev; 1275 struct nfp_net_hw *hw; 1276 int base = 0; 1277 1278 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1279 pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1280 1281 if (pci_dev->intr_handle.type != RTE_INTR_HANDLE_UIO) 1282 base = 1; 1283 1284 /* Make sure all updates are written before un-masking */ 1285 rte_wmb(); 1286 nn_cfg_writeb(hw, NFP_NET_CFG_ICR(base + queue_id), 1287 NFP_NET_CFG_ICR_UNMASKED); 1288 return 0; 1289 } 1290 1291 static int 1292 nfp_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 1293 { 1294 struct rte_pci_device *pci_dev; 1295 struct nfp_net_hw *hw; 1296 int base = 0; 1297 1298 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1299 pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1300 1301 if (pci_dev->intr_handle.type != RTE_INTR_HANDLE_UIO) 1302 base = 1; 1303 1304 /* Make sure all updates are written before un-masking */ 1305 rte_wmb(); 1306 nn_cfg_writeb(hw, NFP_NET_CFG_ICR(base + queue_id), 0x1); 1307 return 0; 1308 } 1309 1310 static void 1311 nfp_net_dev_link_status_print(struct rte_eth_dev *dev) 1312 { 1313 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1314 struct rte_eth_link link; 1315 1316 rte_eth_linkstatus_get(dev, &link); 1317 if (link.link_status) 1318 PMD_DRV_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s", 1319 dev->data->port_id, link.link_speed, 1320 link.link_duplex == ETH_LINK_FULL_DUPLEX 1321 ? "full-duplex" : "half-duplex"); 1322 else 1323 PMD_DRV_LOG(INFO, " Port %d: Link Down", 1324 dev->data->port_id); 1325 1326 PMD_DRV_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d", 1327 pci_dev->addr.domain, pci_dev->addr.bus, 1328 pci_dev->addr.devid, pci_dev->addr.function); 1329 } 1330 1331 /* Interrupt configuration and handling */ 1332 1333 /* 1334 * nfp_net_irq_unmask - Unmask an interrupt 1335 * 1336 * If MSI-X auto-masking is enabled clear the mask bit, otherwise 1337 * clear the ICR for the entry. 1338 */ 1339 static void 1340 nfp_net_irq_unmask(struct rte_eth_dev *dev) 1341 { 1342 struct nfp_net_hw *hw; 1343 struct rte_pci_device *pci_dev; 1344 1345 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1346 pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1347 1348 if (hw->ctrl & NFP_NET_CFG_CTRL_MSIXAUTO) { 1349 /* If MSI-X auto-masking is used, clear the entry */ 1350 rte_wmb(); 1351 rte_intr_enable(&pci_dev->intr_handle); 1352 } else { 1353 /* Make sure all updates are written before un-masking */ 1354 rte_wmb(); 1355 nn_cfg_writeb(hw, NFP_NET_CFG_ICR(NFP_NET_IRQ_LSC_IDX), 1356 NFP_NET_CFG_ICR_UNMASKED); 1357 } 1358 } 1359 1360 static void 1361 nfp_net_dev_interrupt_handler(void *param) 1362 { 1363 int64_t timeout; 1364 struct rte_eth_link link; 1365 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 1366 1367 PMD_DRV_LOG(DEBUG, "We got a LSC interrupt!!!"); 1368 1369 rte_eth_linkstatus_get(dev, &link); 1370 1371 nfp_net_link_update(dev, 0); 1372 1373 /* likely to up */ 1374 if (!link.link_status) { 1375 /* handle it 1 sec later, wait it being stable */ 1376 timeout = NFP_NET_LINK_UP_CHECK_TIMEOUT; 1377 /* likely to down */ 1378 } else { 1379 /* handle it 4 sec later, wait it being stable */ 1380 timeout = NFP_NET_LINK_DOWN_CHECK_TIMEOUT; 1381 } 1382 1383 if (rte_eal_alarm_set(timeout * 1000, 1384 nfp_net_dev_interrupt_delayed_handler, 1385 (void *)dev) < 0) { 1386 PMD_INIT_LOG(ERR, "Error setting alarm"); 1387 /* Unmasking */ 1388 nfp_net_irq_unmask(dev); 1389 } 1390 } 1391 1392 /* 1393 * Interrupt handler which shall be registered for alarm callback for delayed 1394 * handling specific interrupt to wait for the stable nic state. As the NIC 1395 * interrupt state is not stable for nfp after link is just down, it needs 1396 * to wait 4 seconds to get the stable status. 1397 * 1398 * @param handle Pointer to interrupt handle. 1399 * @param param The address of parameter (struct rte_eth_dev *) 1400 * 1401 * @return void 1402 */ 1403 static void 1404 nfp_net_dev_interrupt_delayed_handler(void *param) 1405 { 1406 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 1407 1408 nfp_net_link_update(dev, 0); 1409 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); 1410 1411 nfp_net_dev_link_status_print(dev); 1412 1413 /* Unmasking */ 1414 nfp_net_irq_unmask(dev); 1415 } 1416 1417 static int 1418 nfp_net_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 1419 { 1420 struct nfp_net_hw *hw; 1421 1422 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1423 1424 /* check that mtu is within the allowed range */ 1425 if ((mtu < ETHER_MIN_MTU) || ((uint32_t)mtu > hw->max_mtu)) 1426 return -EINVAL; 1427 1428 /* mtu setting is forbidden if port is started */ 1429 if (dev->data->dev_started) { 1430 PMD_DRV_LOG(ERR, "port %d must be stopped before configuration", 1431 dev->data->port_id); 1432 return -EBUSY; 1433 } 1434 1435 /* switch to jumbo mode if needed */ 1436 if ((uint32_t)mtu > ETHER_MAX_LEN) 1437 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 1438 else 1439 dev->data->dev_conf.rxmode.offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME; 1440 1441 /* update max frame size */ 1442 dev->data->dev_conf.rxmode.max_rx_pkt_len = (uint32_t)mtu; 1443 1444 /* writing to configuration space */ 1445 nn_cfg_writel(hw, NFP_NET_CFG_MTU, (uint32_t)mtu); 1446 1447 hw->mtu = mtu; 1448 1449 return 0; 1450 } 1451 1452 static int 1453 nfp_net_rx_queue_setup(struct rte_eth_dev *dev, 1454 uint16_t queue_idx, uint16_t nb_desc, 1455 unsigned int socket_id, 1456 const struct rte_eth_rxconf *rx_conf, 1457 struct rte_mempool *mp) 1458 { 1459 const struct rte_memzone *tz; 1460 struct nfp_net_rxq *rxq; 1461 struct nfp_net_hw *hw; 1462 1463 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1464 1465 PMD_INIT_FUNC_TRACE(); 1466 1467 /* Validating number of descriptors */ 1468 if (((nb_desc * sizeof(struct nfp_net_rx_desc)) % 128) != 0 || 1469 (nb_desc > NFP_NET_MAX_RX_DESC) || 1470 (nb_desc < NFP_NET_MIN_RX_DESC)) { 1471 PMD_DRV_LOG(ERR, "Wrong nb_desc value"); 1472 return -EINVAL; 1473 } 1474 1475 /* 1476 * Free memory prior to re-allocation if needed. This is the case after 1477 * calling nfp_net_stop 1478 */ 1479 if (dev->data->rx_queues[queue_idx]) { 1480 nfp_net_rx_queue_release(dev->data->rx_queues[queue_idx]); 1481 dev->data->rx_queues[queue_idx] = NULL; 1482 } 1483 1484 /* Allocating rx queue data structure */ 1485 rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct nfp_net_rxq), 1486 RTE_CACHE_LINE_SIZE, socket_id); 1487 if (rxq == NULL) 1488 return -ENOMEM; 1489 1490 /* Hw queues mapping based on firmware confifguration */ 1491 rxq->qidx = queue_idx; 1492 rxq->fl_qcidx = queue_idx * hw->stride_rx; 1493 rxq->rx_qcidx = rxq->fl_qcidx + (hw->stride_rx - 1); 1494 rxq->qcp_fl = hw->rx_bar + NFP_QCP_QUEUE_OFF(rxq->fl_qcidx); 1495 rxq->qcp_rx = hw->rx_bar + NFP_QCP_QUEUE_OFF(rxq->rx_qcidx); 1496 1497 /* 1498 * Tracking mbuf size for detecting a potential mbuf overflow due to 1499 * RX offset 1500 */ 1501 rxq->mem_pool = mp; 1502 rxq->mbuf_size = rxq->mem_pool->elt_size; 1503 rxq->mbuf_size -= (sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM); 1504 hw->flbufsz = rxq->mbuf_size; 1505 1506 rxq->rx_count = nb_desc; 1507 rxq->port_id = dev->data->port_id; 1508 rxq->rx_free_thresh = rx_conf->rx_free_thresh; 1509 rxq->drop_en = rx_conf->rx_drop_en; 1510 1511 /* 1512 * Allocate RX ring hardware descriptors. A memzone large enough to 1513 * handle the maximum ring size is allocated in order to allow for 1514 * resizing in later calls to the queue setup function. 1515 */ 1516 tz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, 1517 sizeof(struct nfp_net_rx_desc) * 1518 NFP_NET_MAX_RX_DESC, NFP_MEMZONE_ALIGN, 1519 socket_id); 1520 1521 if (tz == NULL) { 1522 PMD_DRV_LOG(ERR, "Error allocatig rx dma"); 1523 nfp_net_rx_queue_release(rxq); 1524 return -ENOMEM; 1525 } 1526 1527 /* Saving physical and virtual addresses for the RX ring */ 1528 rxq->dma = (uint64_t)tz->iova; 1529 rxq->rxds = (struct nfp_net_rx_desc *)tz->addr; 1530 1531 /* mbuf pointers array for referencing mbufs linked to RX descriptors */ 1532 rxq->rxbufs = rte_zmalloc_socket("rxq->rxbufs", 1533 sizeof(*rxq->rxbufs) * nb_desc, 1534 RTE_CACHE_LINE_SIZE, socket_id); 1535 if (rxq->rxbufs == NULL) { 1536 nfp_net_rx_queue_release(rxq); 1537 return -ENOMEM; 1538 } 1539 1540 PMD_RX_LOG(DEBUG, "rxbufs=%p hw_ring=%p dma_addr=0x%" PRIx64, 1541 rxq->rxbufs, rxq->rxds, (unsigned long int)rxq->dma); 1542 1543 nfp_net_reset_rx_queue(rxq); 1544 1545 dev->data->rx_queues[queue_idx] = rxq; 1546 rxq->hw = hw; 1547 1548 /* 1549 * Telling the HW about the physical address of the RX ring and number 1550 * of descriptors in log2 format 1551 */ 1552 nn_cfg_writeq(hw, NFP_NET_CFG_RXR_ADDR(queue_idx), rxq->dma); 1553 nn_cfg_writeb(hw, NFP_NET_CFG_RXR_SZ(queue_idx), rte_log2_u32(nb_desc)); 1554 1555 return 0; 1556 } 1557 1558 static int 1559 nfp_net_rx_fill_freelist(struct nfp_net_rxq *rxq) 1560 { 1561 struct nfp_net_rx_buff *rxe = rxq->rxbufs; 1562 uint64_t dma_addr; 1563 unsigned i; 1564 1565 PMD_RX_LOG(DEBUG, "nfp_net_rx_fill_freelist for %u descriptors", 1566 rxq->rx_count); 1567 1568 for (i = 0; i < rxq->rx_count; i++) { 1569 struct nfp_net_rx_desc *rxd; 1570 struct rte_mbuf *mbuf = rte_pktmbuf_alloc(rxq->mem_pool); 1571 1572 if (mbuf == NULL) { 1573 PMD_DRV_LOG(ERR, "RX mbuf alloc failed queue_id=%u", 1574 (unsigned)rxq->qidx); 1575 return -ENOMEM; 1576 } 1577 1578 dma_addr = rte_cpu_to_le_64(RTE_MBUF_DMA_ADDR_DEFAULT(mbuf)); 1579 1580 rxd = &rxq->rxds[i]; 1581 rxd->fld.dd = 0; 1582 rxd->fld.dma_addr_hi = (dma_addr >> 32) & 0xff; 1583 rxd->fld.dma_addr_lo = dma_addr & 0xffffffff; 1584 rxe[i].mbuf = mbuf; 1585 PMD_RX_LOG(DEBUG, "[%d]: %" PRIx64, i, dma_addr); 1586 } 1587 1588 /* Make sure all writes are flushed before telling the hardware */ 1589 rte_wmb(); 1590 1591 /* Not advertising the whole ring as the firmware gets confused if so */ 1592 PMD_RX_LOG(DEBUG, "Increment FL write pointer in %u", 1593 rxq->rx_count - 1); 1594 1595 nfp_qcp_ptr_add(rxq->qcp_fl, NFP_QCP_WRITE_PTR, rxq->rx_count - 1); 1596 1597 return 0; 1598 } 1599 1600 static int 1601 nfp_net_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 1602 uint16_t nb_desc, unsigned int socket_id, 1603 const struct rte_eth_txconf *tx_conf) 1604 { 1605 const struct rte_memzone *tz; 1606 struct nfp_net_txq *txq; 1607 uint16_t tx_free_thresh; 1608 struct nfp_net_hw *hw; 1609 1610 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1611 1612 PMD_INIT_FUNC_TRACE(); 1613 1614 /* Validating number of descriptors */ 1615 if (((nb_desc * sizeof(struct nfp_net_tx_desc)) % 128) != 0 || 1616 (nb_desc > NFP_NET_MAX_TX_DESC) || 1617 (nb_desc < NFP_NET_MIN_TX_DESC)) { 1618 PMD_DRV_LOG(ERR, "Wrong nb_desc value"); 1619 return -EINVAL; 1620 } 1621 1622 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ? 1623 tx_conf->tx_free_thresh : 1624 DEFAULT_TX_FREE_THRESH); 1625 1626 if (tx_free_thresh > (nb_desc)) { 1627 PMD_DRV_LOG(ERR, 1628 "tx_free_thresh must be less than the number of TX " 1629 "descriptors. (tx_free_thresh=%u port=%d " 1630 "queue=%d)", (unsigned int)tx_free_thresh, 1631 dev->data->port_id, (int)queue_idx); 1632 return -(EINVAL); 1633 } 1634 1635 /* 1636 * Free memory prior to re-allocation if needed. This is the case after 1637 * calling nfp_net_stop 1638 */ 1639 if (dev->data->tx_queues[queue_idx]) { 1640 PMD_TX_LOG(DEBUG, "Freeing memory prior to re-allocation %d", 1641 queue_idx); 1642 nfp_net_tx_queue_release(dev->data->tx_queues[queue_idx]); 1643 dev->data->tx_queues[queue_idx] = NULL; 1644 } 1645 1646 /* Allocating tx queue data structure */ 1647 txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct nfp_net_txq), 1648 RTE_CACHE_LINE_SIZE, socket_id); 1649 if (txq == NULL) { 1650 PMD_DRV_LOG(ERR, "Error allocating tx dma"); 1651 return -ENOMEM; 1652 } 1653 1654 /* 1655 * Allocate TX ring hardware descriptors. A memzone large enough to 1656 * handle the maximum ring size is allocated in order to allow for 1657 * resizing in later calls to the queue setup function. 1658 */ 1659 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx, 1660 sizeof(struct nfp_net_tx_desc) * 1661 NFP_NET_MAX_TX_DESC, NFP_MEMZONE_ALIGN, 1662 socket_id); 1663 if (tz == NULL) { 1664 PMD_DRV_LOG(ERR, "Error allocating tx dma"); 1665 nfp_net_tx_queue_release(txq); 1666 return -ENOMEM; 1667 } 1668 1669 txq->tx_count = nb_desc; 1670 txq->tx_free_thresh = tx_free_thresh; 1671 txq->tx_pthresh = tx_conf->tx_thresh.pthresh; 1672 txq->tx_hthresh = tx_conf->tx_thresh.hthresh; 1673 txq->tx_wthresh = tx_conf->tx_thresh.wthresh; 1674 1675 /* queue mapping based on firmware configuration */ 1676 txq->qidx = queue_idx; 1677 txq->tx_qcidx = queue_idx * hw->stride_tx; 1678 txq->qcp_q = hw->tx_bar + NFP_QCP_QUEUE_OFF(txq->tx_qcidx); 1679 1680 txq->port_id = dev->data->port_id; 1681 1682 /* Saving physical and virtual addresses for the TX ring */ 1683 txq->dma = (uint64_t)tz->iova; 1684 txq->txds = (struct nfp_net_tx_desc *)tz->addr; 1685 1686 /* mbuf pointers array for referencing mbufs linked to TX descriptors */ 1687 txq->txbufs = rte_zmalloc_socket("txq->txbufs", 1688 sizeof(*txq->txbufs) * nb_desc, 1689 RTE_CACHE_LINE_SIZE, socket_id); 1690 if (txq->txbufs == NULL) { 1691 nfp_net_tx_queue_release(txq); 1692 return -ENOMEM; 1693 } 1694 PMD_TX_LOG(DEBUG, "txbufs=%p hw_ring=%p dma_addr=0x%" PRIx64, 1695 txq->txbufs, txq->txds, (unsigned long int)txq->dma); 1696 1697 nfp_net_reset_tx_queue(txq); 1698 1699 dev->data->tx_queues[queue_idx] = txq; 1700 txq->hw = hw; 1701 1702 /* 1703 * Telling the HW about the physical address of the TX ring and number 1704 * of descriptors in log2 format 1705 */ 1706 nn_cfg_writeq(hw, NFP_NET_CFG_TXR_ADDR(queue_idx), txq->dma); 1707 nn_cfg_writeb(hw, NFP_NET_CFG_TXR_SZ(queue_idx), rte_log2_u32(nb_desc)); 1708 1709 return 0; 1710 } 1711 1712 /* nfp_net_tx_tso - Set TX descriptor for TSO */ 1713 static inline void 1714 nfp_net_tx_tso(struct nfp_net_txq *txq, struct nfp_net_tx_desc *txd, 1715 struct rte_mbuf *mb) 1716 { 1717 uint64_t ol_flags; 1718 struct nfp_net_hw *hw = txq->hw; 1719 1720 if (!(hw->cap & NFP_NET_CFG_CTRL_LSO_ANY)) 1721 goto clean_txd; 1722 1723 ol_flags = mb->ol_flags; 1724 1725 if (!(ol_flags & PKT_TX_TCP_SEG)) 1726 goto clean_txd; 1727 1728 txd->l3_offset = mb->l2_len; 1729 txd->l4_offset = mb->l2_len + mb->l3_len; 1730 txd->lso_hdrlen = mb->l2_len + mb->l3_len + mb->l4_len; 1731 txd->mss = rte_cpu_to_le_16(mb->tso_segsz); 1732 txd->flags = PCIE_DESC_TX_LSO; 1733 return; 1734 1735 clean_txd: 1736 txd->flags = 0; 1737 txd->l3_offset = 0; 1738 txd->l4_offset = 0; 1739 txd->lso_hdrlen = 0; 1740 txd->mss = 0; 1741 } 1742 1743 /* nfp_net_tx_cksum - Set TX CSUM offload flags in TX descriptor */ 1744 static inline void 1745 nfp_net_tx_cksum(struct nfp_net_txq *txq, struct nfp_net_tx_desc *txd, 1746 struct rte_mbuf *mb) 1747 { 1748 uint64_t ol_flags; 1749 struct nfp_net_hw *hw = txq->hw; 1750 1751 if (!(hw->cap & NFP_NET_CFG_CTRL_TXCSUM)) 1752 return; 1753 1754 ol_flags = mb->ol_flags; 1755 1756 /* IPv6 does not need checksum */ 1757 if (ol_flags & PKT_TX_IP_CKSUM) 1758 txd->flags |= PCIE_DESC_TX_IP4_CSUM; 1759 1760 switch (ol_flags & PKT_TX_L4_MASK) { 1761 case PKT_TX_UDP_CKSUM: 1762 txd->flags |= PCIE_DESC_TX_UDP_CSUM; 1763 break; 1764 case PKT_TX_TCP_CKSUM: 1765 txd->flags |= PCIE_DESC_TX_TCP_CSUM; 1766 break; 1767 } 1768 1769 if (ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_L4_MASK)) 1770 txd->flags |= PCIE_DESC_TX_CSUM; 1771 } 1772 1773 /* nfp_net_rx_cksum - set mbuf checksum flags based on RX descriptor flags */ 1774 static inline void 1775 nfp_net_rx_cksum(struct nfp_net_rxq *rxq, struct nfp_net_rx_desc *rxd, 1776 struct rte_mbuf *mb) 1777 { 1778 struct nfp_net_hw *hw = rxq->hw; 1779 1780 if (!(hw->ctrl & NFP_NET_CFG_CTRL_RXCSUM)) 1781 return; 1782 1783 /* If IPv4 and IP checksum error, fail */ 1784 if (unlikely((rxd->rxd.flags & PCIE_DESC_RX_IP4_CSUM) && 1785 !(rxd->rxd.flags & PCIE_DESC_RX_IP4_CSUM_OK))) 1786 mb->ol_flags |= PKT_RX_IP_CKSUM_BAD; 1787 else 1788 mb->ol_flags |= PKT_RX_IP_CKSUM_GOOD; 1789 1790 /* If neither UDP nor TCP return */ 1791 if (!(rxd->rxd.flags & PCIE_DESC_RX_TCP_CSUM) && 1792 !(rxd->rxd.flags & PCIE_DESC_RX_UDP_CSUM)) 1793 return; 1794 1795 if (likely(rxd->rxd.flags & PCIE_DESC_RX_L4_CSUM_OK)) 1796 mb->ol_flags |= PKT_RX_L4_CKSUM_GOOD; 1797 else 1798 mb->ol_flags |= PKT_RX_L4_CKSUM_BAD; 1799 } 1800 1801 #define NFP_HASH_OFFSET ((uint8_t *)mbuf->buf_addr + mbuf->data_off - 4) 1802 #define NFP_HASH_TYPE_OFFSET ((uint8_t *)mbuf->buf_addr + mbuf->data_off - 8) 1803 1804 #define NFP_DESC_META_LEN(d) (d->rxd.meta_len_dd & PCIE_DESC_RX_META_LEN_MASK) 1805 1806 /* 1807 * nfp_net_set_hash - Set mbuf hash data 1808 * 1809 * The RSS hash and hash-type are pre-pended to the packet data. 1810 * Extract and decode it and set the mbuf fields. 1811 */ 1812 static inline void 1813 nfp_net_set_hash(struct nfp_net_rxq *rxq, struct nfp_net_rx_desc *rxd, 1814 struct rte_mbuf *mbuf) 1815 { 1816 struct nfp_net_hw *hw = rxq->hw; 1817 uint8_t *meta_offset; 1818 uint32_t meta_info; 1819 uint32_t hash = 0; 1820 uint32_t hash_type = 0; 1821 1822 if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS)) 1823 return; 1824 1825 /* this is true for new firmwares */ 1826 if (likely(((hw->cap & NFP_NET_CFG_CTRL_RSS2) || 1827 (NFD_CFG_MAJOR_VERSION_of(hw->ver) == 4)) && 1828 NFP_DESC_META_LEN(rxd))) { 1829 /* 1830 * new metadata api: 1831 * <---- 32 bit -----> 1832 * m field type word 1833 * e data field #2 1834 * t data field #1 1835 * a data field #0 1836 * ==================== 1837 * packet data 1838 * 1839 * Field type word contains up to 8 4bit field types 1840 * A 4bit field type refers to a data field word 1841 * A data field word can have several 4bit field types 1842 */ 1843 meta_offset = rte_pktmbuf_mtod(mbuf, uint8_t *); 1844 meta_offset -= NFP_DESC_META_LEN(rxd); 1845 meta_info = rte_be_to_cpu_32(*(uint32_t *)meta_offset); 1846 meta_offset += 4; 1847 /* NFP PMD just supports metadata for hashing */ 1848 switch (meta_info & NFP_NET_META_FIELD_MASK) { 1849 case NFP_NET_META_HASH: 1850 /* next field type is about the hash type */ 1851 meta_info >>= NFP_NET_META_FIELD_SIZE; 1852 /* hash value is in the data field */ 1853 hash = rte_be_to_cpu_32(*(uint32_t *)meta_offset); 1854 hash_type = meta_info & NFP_NET_META_FIELD_MASK; 1855 break; 1856 default: 1857 /* Unsupported metadata can be a performance issue */ 1858 return; 1859 } 1860 } else { 1861 if (!(rxd->rxd.flags & PCIE_DESC_RX_RSS)) 1862 return; 1863 1864 hash = rte_be_to_cpu_32(*(uint32_t *)NFP_HASH_OFFSET); 1865 hash_type = rte_be_to_cpu_32(*(uint32_t *)NFP_HASH_TYPE_OFFSET); 1866 } 1867 1868 mbuf->hash.rss = hash; 1869 mbuf->ol_flags |= PKT_RX_RSS_HASH; 1870 1871 switch (hash_type) { 1872 case NFP_NET_RSS_IPV4: 1873 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV4; 1874 break; 1875 case NFP_NET_RSS_IPV6: 1876 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6; 1877 break; 1878 case NFP_NET_RSS_IPV6_EX: 1879 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT; 1880 break; 1881 case NFP_NET_RSS_IPV4_TCP: 1882 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT; 1883 break; 1884 case NFP_NET_RSS_IPV6_TCP: 1885 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT; 1886 break; 1887 case NFP_NET_RSS_IPV4_UDP: 1888 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT; 1889 break; 1890 case NFP_NET_RSS_IPV6_UDP: 1891 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT; 1892 break; 1893 default: 1894 mbuf->packet_type |= RTE_PTYPE_INNER_L4_MASK; 1895 } 1896 } 1897 1898 static inline void 1899 nfp_net_mbuf_alloc_failed(struct nfp_net_rxq *rxq) 1900 { 1901 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++; 1902 } 1903 1904 #define NFP_DESC_META_LEN(d) (d->rxd.meta_len_dd & PCIE_DESC_RX_META_LEN_MASK) 1905 1906 /* 1907 * RX path design: 1908 * 1909 * There are some decissions to take: 1910 * 1) How to check DD RX descriptors bit 1911 * 2) How and when to allocate new mbufs 1912 * 1913 * Current implementation checks just one single DD bit each loop. As each 1914 * descriptor is 8 bytes, it is likely a good idea to check descriptors in 1915 * a single cache line instead. Tests with this change have not shown any 1916 * performance improvement but it requires further investigation. For example, 1917 * depending on which descriptor is next, the number of descriptors could be 1918 * less than 8 for just checking those in the same cache line. This implies 1919 * extra work which could be counterproductive by itself. Indeed, last firmware 1920 * changes are just doing this: writing several descriptors with the DD bit 1921 * for saving PCIe bandwidth and DMA operations from the NFP. 1922 * 1923 * Mbuf allocation is done when a new packet is received. Then the descriptor 1924 * is automatically linked with the new mbuf and the old one is given to the 1925 * user. The main drawback with this design is mbuf allocation is heavier than 1926 * using bulk allocations allowed by DPDK with rte_mempool_get_bulk. From the 1927 * cache point of view it does not seem allocating the mbuf early on as we are 1928 * doing now have any benefit at all. Again, tests with this change have not 1929 * shown any improvement. Also, rte_mempool_get_bulk returns all or nothing 1930 * so looking at the implications of this type of allocation should be studied 1931 * deeply 1932 */ 1933 1934 static uint16_t 1935 nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 1936 { 1937 struct nfp_net_rxq *rxq; 1938 struct nfp_net_rx_desc *rxds; 1939 struct nfp_net_rx_buff *rxb; 1940 struct nfp_net_hw *hw; 1941 struct rte_mbuf *mb; 1942 struct rte_mbuf *new_mb; 1943 uint16_t nb_hold; 1944 uint64_t dma_addr; 1945 int avail; 1946 1947 rxq = rx_queue; 1948 if (unlikely(rxq == NULL)) { 1949 /* 1950 * DPDK just checks the queue is lower than max queues 1951 * enabled. But the queue needs to be configured 1952 */ 1953 RTE_LOG_DP(ERR, PMD, "RX Bad queue\n"); 1954 return -EINVAL; 1955 } 1956 1957 hw = rxq->hw; 1958 avail = 0; 1959 nb_hold = 0; 1960 1961 while (avail < nb_pkts) { 1962 rxb = &rxq->rxbufs[rxq->rd_p]; 1963 if (unlikely(rxb == NULL)) { 1964 RTE_LOG_DP(ERR, PMD, "rxb does not exist!\n"); 1965 break; 1966 } 1967 1968 rxds = &rxq->rxds[rxq->rd_p]; 1969 if ((rxds->rxd.meta_len_dd & PCIE_DESC_RX_DD) == 0) 1970 break; 1971 1972 /* 1973 * Memory barrier to ensure that we won't do other 1974 * reads before the DD bit. 1975 */ 1976 rte_rmb(); 1977 1978 /* 1979 * We got a packet. Let's alloc a new mbuff for refilling the 1980 * free descriptor ring as soon as possible 1981 */ 1982 new_mb = rte_pktmbuf_alloc(rxq->mem_pool); 1983 if (unlikely(new_mb == NULL)) { 1984 RTE_LOG_DP(DEBUG, PMD, 1985 "RX mbuf alloc failed port_id=%u queue_id=%u\n", 1986 rxq->port_id, (unsigned int)rxq->qidx); 1987 nfp_net_mbuf_alloc_failed(rxq); 1988 break; 1989 } 1990 1991 nb_hold++; 1992 1993 /* 1994 * Grab the mbuff and refill the descriptor with the 1995 * previously allocated mbuff 1996 */ 1997 mb = rxb->mbuf; 1998 rxb->mbuf = new_mb; 1999 2000 PMD_RX_LOG(DEBUG, "Packet len: %u, mbuf_size: %u", 2001 rxds->rxd.data_len, rxq->mbuf_size); 2002 2003 /* Size of this segment */ 2004 mb->data_len = rxds->rxd.data_len - NFP_DESC_META_LEN(rxds); 2005 /* Size of the whole packet. We just support 1 segment */ 2006 mb->pkt_len = rxds->rxd.data_len - NFP_DESC_META_LEN(rxds); 2007 2008 if (unlikely((mb->data_len + hw->rx_offset) > 2009 rxq->mbuf_size)) { 2010 /* 2011 * This should not happen and the user has the 2012 * responsibility of avoiding it. But we have 2013 * to give some info about the error 2014 */ 2015 RTE_LOG_DP(ERR, PMD, 2016 "mbuf overflow likely due to the RX offset.\n" 2017 "\t\tYour mbuf size should have extra space for" 2018 " RX offset=%u bytes.\n" 2019 "\t\tCurrently you just have %u bytes available" 2020 " but the received packet is %u bytes long", 2021 hw->rx_offset, 2022 rxq->mbuf_size - hw->rx_offset, 2023 mb->data_len); 2024 return -EINVAL; 2025 } 2026 2027 /* Filling the received mbuff with packet info */ 2028 if (hw->rx_offset) 2029 mb->data_off = RTE_PKTMBUF_HEADROOM + hw->rx_offset; 2030 else 2031 mb->data_off = RTE_PKTMBUF_HEADROOM + 2032 NFP_DESC_META_LEN(rxds); 2033 2034 /* No scatter mode supported */ 2035 mb->nb_segs = 1; 2036 mb->next = NULL; 2037 2038 mb->port = rxq->port_id; 2039 2040 /* Checking the RSS flag */ 2041 nfp_net_set_hash(rxq, rxds, mb); 2042 2043 /* Checking the checksum flag */ 2044 nfp_net_rx_cksum(rxq, rxds, mb); 2045 2046 if ((rxds->rxd.flags & PCIE_DESC_RX_VLAN) && 2047 (hw->ctrl & NFP_NET_CFG_CTRL_RXVLAN)) { 2048 mb->vlan_tci = rte_cpu_to_le_32(rxds->rxd.vlan); 2049 mb->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 2050 } 2051 2052 /* Adding the mbuff to the mbuff array passed by the app */ 2053 rx_pkts[avail++] = mb; 2054 2055 /* Now resetting and updating the descriptor */ 2056 rxds->vals[0] = 0; 2057 rxds->vals[1] = 0; 2058 dma_addr = rte_cpu_to_le_64(RTE_MBUF_DMA_ADDR_DEFAULT(new_mb)); 2059 rxds->fld.dd = 0; 2060 rxds->fld.dma_addr_hi = (dma_addr >> 32) & 0xff; 2061 rxds->fld.dma_addr_lo = dma_addr & 0xffffffff; 2062 2063 rxq->rd_p++; 2064 if (unlikely(rxq->rd_p == rxq->rx_count)) /* wrapping?*/ 2065 rxq->rd_p = 0; 2066 } 2067 2068 if (nb_hold == 0) 2069 return nb_hold; 2070 2071 PMD_RX_LOG(DEBUG, "RX port_id=%u queue_id=%u, %d packets received", 2072 rxq->port_id, (unsigned int)rxq->qidx, nb_hold); 2073 2074 nb_hold += rxq->nb_rx_hold; 2075 2076 /* 2077 * FL descriptors needs to be written before incrementing the 2078 * FL queue WR pointer 2079 */ 2080 rte_wmb(); 2081 if (nb_hold > rxq->rx_free_thresh) { 2082 PMD_RX_LOG(DEBUG, "port=%u queue=%u nb_hold=%u avail=%u", 2083 rxq->port_id, (unsigned int)rxq->qidx, 2084 (unsigned)nb_hold, (unsigned)avail); 2085 nfp_qcp_ptr_add(rxq->qcp_fl, NFP_QCP_WRITE_PTR, nb_hold); 2086 nb_hold = 0; 2087 } 2088 rxq->nb_rx_hold = nb_hold; 2089 2090 return avail; 2091 } 2092 2093 /* 2094 * nfp_net_tx_free_bufs - Check for descriptors with a complete 2095 * status 2096 * @txq: TX queue to work with 2097 * Returns number of descriptors freed 2098 */ 2099 int 2100 nfp_net_tx_free_bufs(struct nfp_net_txq *txq) 2101 { 2102 uint32_t qcp_rd_p; 2103 int todo; 2104 2105 PMD_TX_LOG(DEBUG, "queue %u. Check for descriptor with a complete" 2106 " status", txq->qidx); 2107 2108 /* Work out how many packets have been sent */ 2109 qcp_rd_p = nfp_qcp_read(txq->qcp_q, NFP_QCP_READ_PTR); 2110 2111 if (qcp_rd_p == txq->rd_p) { 2112 PMD_TX_LOG(DEBUG, "queue %u: It seems harrier is not sending " 2113 "packets (%u, %u)", txq->qidx, 2114 qcp_rd_p, txq->rd_p); 2115 return 0; 2116 } 2117 2118 if (qcp_rd_p > txq->rd_p) 2119 todo = qcp_rd_p - txq->rd_p; 2120 else 2121 todo = qcp_rd_p + txq->tx_count - txq->rd_p; 2122 2123 PMD_TX_LOG(DEBUG, "qcp_rd_p %u, txq->rd_p: %u, qcp->rd_p: %u", 2124 qcp_rd_p, txq->rd_p, txq->rd_p); 2125 2126 if (todo == 0) 2127 return todo; 2128 2129 txq->rd_p += todo; 2130 if (unlikely(txq->rd_p >= txq->tx_count)) 2131 txq->rd_p -= txq->tx_count; 2132 2133 return todo; 2134 } 2135 2136 /* Leaving always free descriptors for avoiding wrapping confusion */ 2137 static inline 2138 uint32_t nfp_free_tx_desc(struct nfp_net_txq *txq) 2139 { 2140 if (txq->wr_p >= txq->rd_p) 2141 return txq->tx_count - (txq->wr_p - txq->rd_p) - 8; 2142 else 2143 return txq->rd_p - txq->wr_p - 8; 2144 } 2145 2146 /* 2147 * nfp_net_txq_full - Check if the TX queue free descriptors 2148 * is below tx_free_threshold 2149 * 2150 * @txq: TX queue to check 2151 * 2152 * This function uses the host copy* of read/write pointers 2153 */ 2154 static inline 2155 uint32_t nfp_net_txq_full(struct nfp_net_txq *txq) 2156 { 2157 return (nfp_free_tx_desc(txq) < txq->tx_free_thresh); 2158 } 2159 2160 static uint16_t 2161 nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 2162 { 2163 struct nfp_net_txq *txq; 2164 struct nfp_net_hw *hw; 2165 struct nfp_net_tx_desc *txds, txd; 2166 struct rte_mbuf *pkt; 2167 uint64_t dma_addr; 2168 int pkt_size, dma_size; 2169 uint16_t free_descs, issued_descs; 2170 struct rte_mbuf **lmbuf; 2171 int i; 2172 2173 txq = tx_queue; 2174 hw = txq->hw; 2175 txds = &txq->txds[txq->wr_p]; 2176 2177 PMD_TX_LOG(DEBUG, "working for queue %u at pos %d and %u packets", 2178 txq->qidx, txq->wr_p, nb_pkts); 2179 2180 if ((nfp_free_tx_desc(txq) < nb_pkts) || (nfp_net_txq_full(txq))) 2181 nfp_net_tx_free_bufs(txq); 2182 2183 free_descs = (uint16_t)nfp_free_tx_desc(txq); 2184 if (unlikely(free_descs == 0)) 2185 return 0; 2186 2187 pkt = *tx_pkts; 2188 2189 i = 0; 2190 issued_descs = 0; 2191 PMD_TX_LOG(DEBUG, "queue: %u. Sending %u packets", 2192 txq->qidx, nb_pkts); 2193 /* Sending packets */ 2194 while ((i < nb_pkts) && free_descs) { 2195 /* Grabbing the mbuf linked to the current descriptor */ 2196 lmbuf = &txq->txbufs[txq->wr_p].mbuf; 2197 /* Warming the cache for releasing the mbuf later on */ 2198 RTE_MBUF_PREFETCH_TO_FREE(*lmbuf); 2199 2200 pkt = *(tx_pkts + i); 2201 2202 if (unlikely((pkt->nb_segs > 1) && 2203 !(hw->cap & NFP_NET_CFG_CTRL_GATHER))) { 2204 PMD_INIT_LOG(INFO, "NFP_NET_CFG_CTRL_GATHER not set"); 2205 rte_panic("Multisegment packet unsupported\n"); 2206 } 2207 2208 /* Checking if we have enough descriptors */ 2209 if (unlikely(pkt->nb_segs > free_descs)) 2210 goto xmit_end; 2211 2212 /* 2213 * Checksum and VLAN flags just in the first descriptor for a 2214 * multisegment packet, but TSO info needs to be in all of them. 2215 */ 2216 txd.data_len = pkt->pkt_len; 2217 nfp_net_tx_tso(txq, &txd, pkt); 2218 nfp_net_tx_cksum(txq, &txd, pkt); 2219 2220 if ((pkt->ol_flags & PKT_TX_VLAN_PKT) && 2221 (hw->cap & NFP_NET_CFG_CTRL_TXVLAN)) { 2222 txd.flags |= PCIE_DESC_TX_VLAN; 2223 txd.vlan = pkt->vlan_tci; 2224 } 2225 2226 /* 2227 * mbuf data_len is the data in one segment and pkt_len data 2228 * in the whole packet. When the packet is just one segment, 2229 * then data_len = pkt_len 2230 */ 2231 pkt_size = pkt->pkt_len; 2232 2233 while (pkt) { 2234 /* Copying TSO, VLAN and cksum info */ 2235 *txds = txd; 2236 2237 /* Releasing mbuf used by this descriptor previously*/ 2238 if (*lmbuf) 2239 rte_pktmbuf_free_seg(*lmbuf); 2240 2241 /* 2242 * Linking mbuf with descriptor for being released 2243 * next time descriptor is used 2244 */ 2245 *lmbuf = pkt; 2246 2247 dma_size = pkt->data_len; 2248 dma_addr = rte_mbuf_data_iova(pkt); 2249 PMD_TX_LOG(DEBUG, "Working with mbuf at dma address:" 2250 "%" PRIx64 "", dma_addr); 2251 2252 /* Filling descriptors fields */ 2253 txds->dma_len = dma_size; 2254 txds->data_len = txd.data_len; 2255 txds->dma_addr_hi = (dma_addr >> 32) & 0xff; 2256 txds->dma_addr_lo = (dma_addr & 0xffffffff); 2257 ASSERT(free_descs > 0); 2258 free_descs--; 2259 2260 txq->wr_p++; 2261 if (unlikely(txq->wr_p == txq->tx_count)) /* wrapping?*/ 2262 txq->wr_p = 0; 2263 2264 pkt_size -= dma_size; 2265 2266 /* 2267 * Making the EOP, packets with just one segment 2268 * the priority 2269 */ 2270 if (likely(!pkt_size)) 2271 txds->offset_eop = PCIE_DESC_TX_EOP; 2272 else 2273 txds->offset_eop = 0; 2274 2275 pkt = pkt->next; 2276 /* Referencing next free TX descriptor */ 2277 txds = &txq->txds[txq->wr_p]; 2278 lmbuf = &txq->txbufs[txq->wr_p].mbuf; 2279 issued_descs++; 2280 } 2281 i++; 2282 } 2283 2284 xmit_end: 2285 /* Increment write pointers. Force memory write before we let HW know */ 2286 rte_wmb(); 2287 nfp_qcp_ptr_add(txq->qcp_q, NFP_QCP_WRITE_PTR, issued_descs); 2288 2289 return i; 2290 } 2291 2292 static int 2293 nfp_net_vlan_offload_set(struct rte_eth_dev *dev, int mask) 2294 { 2295 uint32_t new_ctrl, update; 2296 struct nfp_net_hw *hw; 2297 int ret; 2298 2299 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2300 new_ctrl = 0; 2301 2302 if ((mask & ETH_VLAN_FILTER_OFFLOAD) || 2303 (mask & ETH_VLAN_EXTEND_OFFLOAD)) 2304 PMD_DRV_LOG(INFO, "No support for ETH_VLAN_FILTER_OFFLOAD or" 2305 " ETH_VLAN_EXTEND_OFFLOAD"); 2306 2307 /* Enable vlan strip if it is not configured yet */ 2308 if ((mask & ETH_VLAN_STRIP_OFFLOAD) && 2309 !(hw->ctrl & NFP_NET_CFG_CTRL_RXVLAN)) 2310 new_ctrl = hw->ctrl | NFP_NET_CFG_CTRL_RXVLAN; 2311 2312 /* Disable vlan strip just if it is configured */ 2313 if (!(mask & ETH_VLAN_STRIP_OFFLOAD) && 2314 (hw->ctrl & NFP_NET_CFG_CTRL_RXVLAN)) 2315 new_ctrl = hw->ctrl & ~NFP_NET_CFG_CTRL_RXVLAN; 2316 2317 if (new_ctrl == 0) 2318 return 0; 2319 2320 update = NFP_NET_CFG_UPDATE_GEN; 2321 2322 ret = nfp_net_reconfig(hw, new_ctrl, update); 2323 if (!ret) 2324 hw->ctrl = new_ctrl; 2325 2326 return ret; 2327 } 2328 2329 static int 2330 nfp_net_rss_reta_write(struct rte_eth_dev *dev, 2331 struct rte_eth_rss_reta_entry64 *reta_conf, 2332 uint16_t reta_size) 2333 { 2334 uint32_t reta, mask; 2335 int i, j; 2336 int idx, shift; 2337 struct nfp_net_hw *hw = 2338 NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2339 2340 if (reta_size != NFP_NET_CFG_RSS_ITBL_SZ) { 2341 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 2342 "(%d) doesn't match the number hardware can supported " 2343 "(%d)", reta_size, NFP_NET_CFG_RSS_ITBL_SZ); 2344 return -EINVAL; 2345 } 2346 2347 /* 2348 * Update Redirection Table. There are 128 8bit-entries which can be 2349 * manage as 32 32bit-entries 2350 */ 2351 for (i = 0; i < reta_size; i += 4) { 2352 /* Handling 4 RSS entries per loop */ 2353 idx = i / RTE_RETA_GROUP_SIZE; 2354 shift = i % RTE_RETA_GROUP_SIZE; 2355 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 0xF); 2356 2357 if (!mask) 2358 continue; 2359 2360 reta = 0; 2361 /* If all 4 entries were set, don't need read RETA register */ 2362 if (mask != 0xF) 2363 reta = nn_cfg_readl(hw, NFP_NET_CFG_RSS_ITBL + i); 2364 2365 for (j = 0; j < 4; j++) { 2366 if (!(mask & (0x1 << j))) 2367 continue; 2368 if (mask != 0xF) 2369 /* Clearing the entry bits */ 2370 reta &= ~(0xFF << (8 * j)); 2371 reta |= reta_conf[idx].reta[shift + j] << (8 * j); 2372 } 2373 nn_cfg_writel(hw, NFP_NET_CFG_RSS_ITBL + (idx * 64) + shift, 2374 reta); 2375 } 2376 return 0; 2377 } 2378 2379 /* Update Redirection Table(RETA) of Receive Side Scaling of Ethernet device */ 2380 static int 2381 nfp_net_reta_update(struct rte_eth_dev *dev, 2382 struct rte_eth_rss_reta_entry64 *reta_conf, 2383 uint16_t reta_size) 2384 { 2385 struct nfp_net_hw *hw = 2386 NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2387 uint32_t update; 2388 int ret; 2389 2390 if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS)) 2391 return -EINVAL; 2392 2393 ret = nfp_net_rss_reta_write(dev, reta_conf, reta_size); 2394 if (ret != 0) 2395 return ret; 2396 2397 update = NFP_NET_CFG_UPDATE_RSS; 2398 2399 if (nfp_net_reconfig(hw, hw->ctrl, update) < 0) 2400 return -EIO; 2401 2402 return 0; 2403 } 2404 2405 /* Query Redirection Table(RETA) of Receive Side Scaling of Ethernet device. */ 2406 static int 2407 nfp_net_reta_query(struct rte_eth_dev *dev, 2408 struct rte_eth_rss_reta_entry64 *reta_conf, 2409 uint16_t reta_size) 2410 { 2411 uint8_t i, j, mask; 2412 int idx, shift; 2413 uint32_t reta; 2414 struct nfp_net_hw *hw; 2415 2416 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2417 2418 if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS)) 2419 return -EINVAL; 2420 2421 if (reta_size != NFP_NET_CFG_RSS_ITBL_SZ) { 2422 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 2423 "(%d) doesn't match the number hardware can supported " 2424 "(%d)", reta_size, NFP_NET_CFG_RSS_ITBL_SZ); 2425 return -EINVAL; 2426 } 2427 2428 /* 2429 * Reading Redirection Table. There are 128 8bit-entries which can be 2430 * manage as 32 32bit-entries 2431 */ 2432 for (i = 0; i < reta_size; i += 4) { 2433 /* Handling 4 RSS entries per loop */ 2434 idx = i / RTE_RETA_GROUP_SIZE; 2435 shift = i % RTE_RETA_GROUP_SIZE; 2436 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 0xF); 2437 2438 if (!mask) 2439 continue; 2440 2441 reta = nn_cfg_readl(hw, NFP_NET_CFG_RSS_ITBL + (idx * 64) + 2442 shift); 2443 for (j = 0; j < 4; j++) { 2444 if (!(mask & (0x1 << j))) 2445 continue; 2446 reta_conf->reta[shift + j] = 2447 (uint8_t)((reta >> (8 * j)) & 0xF); 2448 } 2449 } 2450 return 0; 2451 } 2452 2453 static int 2454 nfp_net_rss_hash_write(struct rte_eth_dev *dev, 2455 struct rte_eth_rss_conf *rss_conf) 2456 { 2457 struct nfp_net_hw *hw; 2458 uint64_t rss_hf; 2459 uint32_t cfg_rss_ctrl = 0; 2460 uint8_t key; 2461 int i; 2462 2463 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2464 2465 /* Writing the key byte a byte */ 2466 for (i = 0; i < rss_conf->rss_key_len; i++) { 2467 memcpy(&key, &rss_conf->rss_key[i], 1); 2468 nn_cfg_writeb(hw, NFP_NET_CFG_RSS_KEY + i, key); 2469 } 2470 2471 rss_hf = rss_conf->rss_hf; 2472 2473 if (rss_hf & ETH_RSS_IPV4) 2474 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4; 2475 2476 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) 2477 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4_TCP; 2478 2479 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) 2480 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4_UDP; 2481 2482 if (rss_hf & ETH_RSS_IPV6) 2483 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6; 2484 2485 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) 2486 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6_TCP; 2487 2488 if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) 2489 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6_UDP; 2490 2491 cfg_rss_ctrl |= NFP_NET_CFG_RSS_MASK; 2492 cfg_rss_ctrl |= NFP_NET_CFG_RSS_TOEPLITZ; 2493 2494 /* configuring where to apply the RSS hash */ 2495 nn_cfg_writel(hw, NFP_NET_CFG_RSS_CTRL, cfg_rss_ctrl); 2496 2497 /* Writing the key size */ 2498 nn_cfg_writeb(hw, NFP_NET_CFG_RSS_KEY_SZ, rss_conf->rss_key_len); 2499 2500 return 0; 2501 } 2502 2503 static int 2504 nfp_net_rss_hash_update(struct rte_eth_dev *dev, 2505 struct rte_eth_rss_conf *rss_conf) 2506 { 2507 uint32_t update; 2508 uint64_t rss_hf; 2509 struct nfp_net_hw *hw; 2510 2511 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2512 2513 rss_hf = rss_conf->rss_hf; 2514 2515 /* Checking if RSS is enabled */ 2516 if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS)) { 2517 if (rss_hf != 0) { /* Enable RSS? */ 2518 PMD_DRV_LOG(ERR, "RSS unsupported"); 2519 return -EINVAL; 2520 } 2521 return 0; /* Nothing to do */ 2522 } 2523 2524 if (rss_conf->rss_key_len > NFP_NET_CFG_RSS_KEY_SZ) { 2525 PMD_DRV_LOG(ERR, "hash key too long"); 2526 return -EINVAL; 2527 } 2528 2529 nfp_net_rss_hash_write(dev, rss_conf); 2530 2531 update = NFP_NET_CFG_UPDATE_RSS; 2532 2533 if (nfp_net_reconfig(hw, hw->ctrl, update) < 0) 2534 return -EIO; 2535 2536 return 0; 2537 } 2538 2539 static int 2540 nfp_net_rss_hash_conf_get(struct rte_eth_dev *dev, 2541 struct rte_eth_rss_conf *rss_conf) 2542 { 2543 uint64_t rss_hf; 2544 uint32_t cfg_rss_ctrl; 2545 uint8_t key; 2546 int i; 2547 struct nfp_net_hw *hw; 2548 2549 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2550 2551 if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS)) 2552 return -EINVAL; 2553 2554 rss_hf = rss_conf->rss_hf; 2555 cfg_rss_ctrl = nn_cfg_readl(hw, NFP_NET_CFG_RSS_CTRL); 2556 2557 if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4) 2558 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_NONFRAG_IPV4_UDP; 2559 2560 if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4_TCP) 2561 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP; 2562 2563 if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV6_TCP) 2564 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP; 2565 2566 if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4_UDP) 2567 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP; 2568 2569 if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV6_UDP) 2570 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP; 2571 2572 if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV6) 2573 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_NONFRAG_IPV6_UDP; 2574 2575 /* Reading the key size */ 2576 rss_conf->rss_key_len = nn_cfg_readl(hw, NFP_NET_CFG_RSS_KEY_SZ); 2577 2578 /* Reading the key byte a byte */ 2579 for (i = 0; i < rss_conf->rss_key_len; i++) { 2580 key = nn_cfg_readb(hw, NFP_NET_CFG_RSS_KEY + i); 2581 memcpy(&rss_conf->rss_key[i], &key, 1); 2582 } 2583 2584 return 0; 2585 } 2586 2587 static int 2588 nfp_net_rss_config_default(struct rte_eth_dev *dev) 2589 { 2590 struct rte_eth_conf *dev_conf; 2591 struct rte_eth_rss_conf rss_conf; 2592 struct rte_eth_rss_reta_entry64 nfp_reta_conf[2]; 2593 uint16_t rx_queues = dev->data->nb_rx_queues; 2594 uint16_t queue; 2595 int i, j, ret; 2596 2597 PMD_DRV_LOG(INFO, "setting default RSS conf for %u queues", 2598 rx_queues); 2599 2600 nfp_reta_conf[0].mask = ~0x0; 2601 nfp_reta_conf[1].mask = ~0x0; 2602 2603 queue = 0; 2604 for (i = 0; i < 0x40; i += 8) { 2605 for (j = i; j < (i + 8); j++) { 2606 nfp_reta_conf[0].reta[j] = queue; 2607 nfp_reta_conf[1].reta[j] = queue++; 2608 queue %= rx_queues; 2609 } 2610 } 2611 ret = nfp_net_rss_reta_write(dev, nfp_reta_conf, 0x80); 2612 if (ret != 0) 2613 return ret; 2614 2615 dev_conf = &dev->data->dev_conf; 2616 if (!dev_conf) { 2617 PMD_DRV_LOG(INFO, "wrong rss conf"); 2618 return -EINVAL; 2619 } 2620 rss_conf = dev_conf->rx_adv_conf.rss_conf; 2621 2622 ret = nfp_net_rss_hash_write(dev, &rss_conf); 2623 2624 return ret; 2625 } 2626 2627 2628 /* Initialise and register driver with DPDK Application */ 2629 static const struct eth_dev_ops nfp_net_eth_dev_ops = { 2630 .dev_configure = nfp_net_configure, 2631 .dev_start = nfp_net_start, 2632 .dev_stop = nfp_net_stop, 2633 .dev_close = nfp_net_close, 2634 .promiscuous_enable = nfp_net_promisc_enable, 2635 .promiscuous_disable = nfp_net_promisc_disable, 2636 .link_update = nfp_net_link_update, 2637 .stats_get = nfp_net_stats_get, 2638 .stats_reset = nfp_net_stats_reset, 2639 .dev_infos_get = nfp_net_infos_get, 2640 .dev_supported_ptypes_get = nfp_net_supported_ptypes_get, 2641 .mtu_set = nfp_net_dev_mtu_set, 2642 .mac_addr_set = nfp_set_mac_addr, 2643 .vlan_offload_set = nfp_net_vlan_offload_set, 2644 .reta_update = nfp_net_reta_update, 2645 .reta_query = nfp_net_reta_query, 2646 .rss_hash_update = nfp_net_rss_hash_update, 2647 .rss_hash_conf_get = nfp_net_rss_hash_conf_get, 2648 .rx_queue_setup = nfp_net_rx_queue_setup, 2649 .rx_queue_release = nfp_net_rx_queue_release, 2650 .rx_queue_count = nfp_net_rx_queue_count, 2651 .tx_queue_setup = nfp_net_tx_queue_setup, 2652 .tx_queue_release = nfp_net_tx_queue_release, 2653 .rx_queue_intr_enable = nfp_rx_queue_intr_enable, 2654 .rx_queue_intr_disable = nfp_rx_queue_intr_disable, 2655 }; 2656 2657 /* 2658 * All eth_dev created got its private data, but before nfp_net_init, that 2659 * private data is referencing private data for all the PF ports. This is due 2660 * to how the vNIC bars are mapped based on first port, so all ports need info 2661 * about port 0 private data. Inside nfp_net_init the private data pointer is 2662 * changed to the right address for each port once the bars have been mapped. 2663 * 2664 * This functions helps to find out which port and therefore which offset 2665 * inside the private data array to use. 2666 */ 2667 static int 2668 get_pf_port_number(char *name) 2669 { 2670 char *pf_str = name; 2671 int size = 0; 2672 2673 while ((*pf_str != '_') && (*pf_str != '\0') && (size++ < 30)) 2674 pf_str++; 2675 2676 if (size == 30) 2677 /* 2678 * This should not happen at all and it would mean major 2679 * implementation fault. 2680 */ 2681 rte_panic("nfp_net: problem with pf device name\n"); 2682 2683 /* Expecting _portX with X within [0,7] */ 2684 pf_str += 5; 2685 2686 return (int)strtol(pf_str, NULL, 10); 2687 } 2688 2689 static int 2690 nfp_net_init(struct rte_eth_dev *eth_dev) 2691 { 2692 struct rte_pci_device *pci_dev; 2693 struct nfp_net_hw *hw, *hwport0; 2694 2695 uint64_t tx_bar_off = 0, rx_bar_off = 0; 2696 uint32_t start_q; 2697 int stride = 4; 2698 int port = 0; 2699 int err; 2700 2701 PMD_INIT_FUNC_TRACE(); 2702 2703 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 2704 2705 /* NFP can not handle DMA addresses requiring more than 40 bits */ 2706 if (rte_mem_check_dma_mask(40)) { 2707 RTE_LOG(ERR, PMD, "device %s can not be used:", 2708 pci_dev->device.name); 2709 RTE_LOG(ERR, PMD, "\trestricted dma mask to 40 bits!\n"); 2710 return -ENODEV; 2711 }; 2712 2713 if ((pci_dev->id.device_id == PCI_DEVICE_ID_NFP4000_PF_NIC) || 2714 (pci_dev->id.device_id == PCI_DEVICE_ID_NFP6000_PF_NIC)) { 2715 port = get_pf_port_number(eth_dev->data->name); 2716 if (port < 0 || port > 7) { 2717 PMD_DRV_LOG(ERR, "Port value is wrong"); 2718 return -ENODEV; 2719 } 2720 2721 PMD_INIT_LOG(DEBUG, "Working with PF port value %d", port); 2722 2723 /* This points to port 0 private data */ 2724 hwport0 = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 2725 2726 /* This points to the specific port private data */ 2727 hw = &hwport0[port]; 2728 } else { 2729 hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 2730 hwport0 = 0; 2731 } 2732 2733 eth_dev->dev_ops = &nfp_net_eth_dev_ops; 2734 eth_dev->rx_pkt_burst = &nfp_net_recv_pkts; 2735 eth_dev->tx_pkt_burst = &nfp_net_xmit_pkts; 2736 2737 /* For secondary processes, the primary has done all the work */ 2738 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2739 return 0; 2740 2741 rte_eth_copy_pci_info(eth_dev, pci_dev); 2742 2743 hw->device_id = pci_dev->id.device_id; 2744 hw->vendor_id = pci_dev->id.vendor_id; 2745 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 2746 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 2747 2748 PMD_INIT_LOG(DEBUG, "nfp_net: device (%u:%u) %u:%u:%u:%u", 2749 pci_dev->id.vendor_id, pci_dev->id.device_id, 2750 pci_dev->addr.domain, pci_dev->addr.bus, 2751 pci_dev->addr.devid, pci_dev->addr.function); 2752 2753 hw->ctrl_bar = (uint8_t *)pci_dev->mem_resource[0].addr; 2754 if (hw->ctrl_bar == NULL) { 2755 PMD_DRV_LOG(ERR, 2756 "hw->ctrl_bar is NULL. BAR0 not configured"); 2757 return -ENODEV; 2758 } 2759 2760 if (hw->is_pf && port == 0) { 2761 hw->ctrl_bar = nfp_rtsym_map(hw->sym_tbl, "_pf0_net_bar0", 2762 hw->total_ports * 32768, 2763 &hw->ctrl_area); 2764 if (!hw->ctrl_bar) { 2765 printf("nfp_rtsym_map fails for _pf0_net_ctrl_bar"); 2766 return -EIO; 2767 } 2768 2769 PMD_INIT_LOG(DEBUG, "ctrl bar: %p", hw->ctrl_bar); 2770 } 2771 2772 if (port > 0) { 2773 if (!hwport0->ctrl_bar) 2774 return -ENODEV; 2775 2776 /* address based on port0 offset */ 2777 hw->ctrl_bar = hwport0->ctrl_bar + 2778 (port * NFP_PF_CSR_SLICE_SIZE); 2779 } 2780 2781 PMD_INIT_LOG(DEBUG, "ctrl bar: %p", hw->ctrl_bar); 2782 2783 hw->max_rx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_RXRINGS); 2784 hw->max_tx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_TXRINGS); 2785 2786 /* Work out where in the BAR the queues start. */ 2787 switch (pci_dev->id.device_id) { 2788 case PCI_DEVICE_ID_NFP4000_PF_NIC: 2789 case PCI_DEVICE_ID_NFP6000_PF_NIC: 2790 case PCI_DEVICE_ID_NFP6000_VF_NIC: 2791 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_TXQ); 2792 tx_bar_off = start_q * NFP_QCP_QUEUE_ADDR_SZ; 2793 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_RXQ); 2794 rx_bar_off = start_q * NFP_QCP_QUEUE_ADDR_SZ; 2795 break; 2796 default: 2797 PMD_DRV_LOG(ERR, "nfp_net: no device ID matching"); 2798 err = -ENODEV; 2799 goto dev_err_ctrl_map; 2800 } 2801 2802 PMD_INIT_LOG(DEBUG, "tx_bar_off: 0x%" PRIx64 "", tx_bar_off); 2803 PMD_INIT_LOG(DEBUG, "rx_bar_off: 0x%" PRIx64 "", rx_bar_off); 2804 2805 if (hw->is_pf && port == 0) { 2806 /* configure access to tx/rx vNIC BARs */ 2807 hwport0->hw_queues = nfp_cpp_map_area(hw->cpp, 0, 0, 2808 NFP_PCIE_QUEUE(0), 2809 NFP_QCP_QUEUE_AREA_SZ, 2810 &hw->hwqueues_area); 2811 2812 if (!hwport0->hw_queues) { 2813 printf("nfp_rtsym_map fails for net.qc"); 2814 err = -EIO; 2815 goto dev_err_ctrl_map; 2816 } 2817 2818 PMD_INIT_LOG(DEBUG, "tx/rx bar address: 0x%p", 2819 hwport0->hw_queues); 2820 } 2821 2822 if (hw->is_pf) { 2823 hw->tx_bar = hwport0->hw_queues + tx_bar_off; 2824 hw->rx_bar = hwport0->hw_queues + rx_bar_off; 2825 eth_dev->data->dev_private = hw; 2826 } else { 2827 hw->tx_bar = (uint8_t *)pci_dev->mem_resource[2].addr + 2828 tx_bar_off; 2829 hw->rx_bar = (uint8_t *)pci_dev->mem_resource[2].addr + 2830 rx_bar_off; 2831 } 2832 2833 PMD_INIT_LOG(DEBUG, "ctrl_bar: %p, tx_bar: %p, rx_bar: %p", 2834 hw->ctrl_bar, hw->tx_bar, hw->rx_bar); 2835 2836 nfp_net_cfg_queue_setup(hw); 2837 2838 /* Get some of the read-only fields from the config BAR */ 2839 hw->ver = nn_cfg_readl(hw, NFP_NET_CFG_VERSION); 2840 hw->cap = nn_cfg_readl(hw, NFP_NET_CFG_CAP); 2841 hw->max_mtu = nn_cfg_readl(hw, NFP_NET_CFG_MAX_MTU); 2842 hw->mtu = ETHER_MTU; 2843 2844 /* VLAN insertion is incompatible with LSOv2 */ 2845 if (hw->cap & NFP_NET_CFG_CTRL_LSO2) 2846 hw->cap &= ~NFP_NET_CFG_CTRL_TXVLAN; 2847 2848 if (NFD_CFG_MAJOR_VERSION_of(hw->ver) < 2) 2849 hw->rx_offset = NFP_NET_RX_OFFSET; 2850 else 2851 hw->rx_offset = nn_cfg_readl(hw, NFP_NET_CFG_RX_OFFSET_ADDR); 2852 2853 PMD_INIT_LOG(INFO, "VER: %u.%u, Maximum supported MTU: %d", 2854 NFD_CFG_MAJOR_VERSION_of(hw->ver), 2855 NFD_CFG_MINOR_VERSION_of(hw->ver), hw->max_mtu); 2856 2857 PMD_INIT_LOG(INFO, "CAP: %#x, %s%s%s%s%s%s%s%s%s%s%s%s%s%s", hw->cap, 2858 hw->cap & NFP_NET_CFG_CTRL_PROMISC ? "PROMISC " : "", 2859 hw->cap & NFP_NET_CFG_CTRL_L2BC ? "L2BCFILT " : "", 2860 hw->cap & NFP_NET_CFG_CTRL_L2MC ? "L2MCFILT " : "", 2861 hw->cap & NFP_NET_CFG_CTRL_RXCSUM ? "RXCSUM " : "", 2862 hw->cap & NFP_NET_CFG_CTRL_TXCSUM ? "TXCSUM " : "", 2863 hw->cap & NFP_NET_CFG_CTRL_RXVLAN ? "RXVLAN " : "", 2864 hw->cap & NFP_NET_CFG_CTRL_TXVLAN ? "TXVLAN " : "", 2865 hw->cap & NFP_NET_CFG_CTRL_SCATTER ? "SCATTER " : "", 2866 hw->cap & NFP_NET_CFG_CTRL_GATHER ? "GATHER " : "", 2867 hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR ? "LIVE_ADDR " : "", 2868 hw->cap & NFP_NET_CFG_CTRL_LSO ? "TSO " : "", 2869 hw->cap & NFP_NET_CFG_CTRL_LSO2 ? "TSOv2 " : "", 2870 hw->cap & NFP_NET_CFG_CTRL_RSS ? "RSS " : "", 2871 hw->cap & NFP_NET_CFG_CTRL_RSS2 ? "RSSv2 " : ""); 2872 2873 hw->ctrl = 0; 2874 2875 hw->stride_rx = stride; 2876 hw->stride_tx = stride; 2877 2878 PMD_INIT_LOG(INFO, "max_rx_queues: %u, max_tx_queues: %u", 2879 hw->max_rx_queues, hw->max_tx_queues); 2880 2881 /* Initializing spinlock for reconfigs */ 2882 rte_spinlock_init(&hw->reconfig_lock); 2883 2884 /* Allocating memory for mac addr */ 2885 eth_dev->data->mac_addrs = rte_zmalloc("mac_addr", ETHER_ADDR_LEN, 0); 2886 if (eth_dev->data->mac_addrs == NULL) { 2887 PMD_INIT_LOG(ERR, "Failed to space for MAC address"); 2888 err = -ENOMEM; 2889 goto dev_err_queues_map; 2890 } 2891 2892 if (hw->is_pf) { 2893 nfp_net_pf_read_mac(hwport0, port); 2894 nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr); 2895 } else { 2896 nfp_net_vf_read_mac(hw); 2897 } 2898 2899 if (!is_valid_assigned_ether_addr((struct ether_addr *)&hw->mac_addr)) { 2900 PMD_INIT_LOG(INFO, "Using random mac address for port %d", 2901 port); 2902 /* Using random mac addresses for VFs */ 2903 eth_random_addr(&hw->mac_addr[0]); 2904 nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr); 2905 } 2906 2907 /* Copying mac address to DPDK eth_dev struct */ 2908 ether_addr_copy((struct ether_addr *)hw->mac_addr, 2909 ð_dev->data->mac_addrs[0]); 2910 2911 if (!(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR)) 2912 eth_dev->data->dev_flags |= RTE_ETH_DEV_NOLIVE_MAC_ADDR; 2913 2914 PMD_INIT_LOG(INFO, "port %d VendorID=0x%x DeviceID=0x%x " 2915 "mac=%02x:%02x:%02x:%02x:%02x:%02x", 2916 eth_dev->data->port_id, pci_dev->id.vendor_id, 2917 pci_dev->id.device_id, 2918 hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2], 2919 hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]); 2920 2921 /* Registering LSC interrupt handler */ 2922 rte_intr_callback_register(&pci_dev->intr_handle, 2923 nfp_net_dev_interrupt_handler, 2924 (void *)eth_dev); 2925 2926 /* Telling the firmware about the LSC interrupt entry */ 2927 nn_cfg_writeb(hw, NFP_NET_CFG_LSC, NFP_NET_IRQ_LSC_IDX); 2928 2929 /* Recording current stats counters values */ 2930 nfp_net_stats_reset(eth_dev); 2931 2932 return 0; 2933 2934 dev_err_queues_map: 2935 nfp_cpp_area_free(hw->hwqueues_area); 2936 dev_err_ctrl_map: 2937 nfp_cpp_area_free(hw->ctrl_area); 2938 2939 return err; 2940 } 2941 2942 static int 2943 nfp_pf_create_dev(struct rte_pci_device *dev, int port, int ports, 2944 struct nfp_cpp *cpp, struct nfp_hwinfo *hwinfo, 2945 int phys_port, struct nfp_rtsym_table *sym_tbl, void **priv) 2946 { 2947 struct rte_eth_dev *eth_dev; 2948 struct nfp_net_hw *hw; 2949 char *port_name; 2950 int ret; 2951 2952 port_name = rte_zmalloc("nfp_pf_port_name", 100, 0); 2953 if (!port_name) 2954 return -ENOMEM; 2955 2956 if (ports > 1) 2957 sprintf(port_name, "%s_port%d", dev->device.name, port); 2958 else 2959 sprintf(port_name, "%s", dev->device.name); 2960 2961 eth_dev = rte_eth_dev_allocate(port_name); 2962 if (!eth_dev) 2963 return -ENOMEM; 2964 2965 if (port == 0) { 2966 *priv = rte_zmalloc(port_name, 2967 sizeof(struct nfp_net_adapter) * ports, 2968 RTE_CACHE_LINE_SIZE); 2969 if (!*priv) { 2970 rte_eth_dev_release_port(eth_dev); 2971 return -ENOMEM; 2972 } 2973 } 2974 2975 eth_dev->data->dev_private = *priv; 2976 2977 /* 2978 * dev_private pointing to port0 dev_private because we need 2979 * to configure vNIC bars based on port0 at nfp_net_init. 2980 * Then dev_private is adjusted per port. 2981 */ 2982 hw = (struct nfp_net_hw *)(eth_dev->data->dev_private) + port; 2983 hw->cpp = cpp; 2984 hw->hwinfo = hwinfo; 2985 hw->sym_tbl = sym_tbl; 2986 hw->pf_port_idx = phys_port; 2987 hw->is_pf = 1; 2988 if (ports > 1) 2989 hw->pf_multiport_enabled = 1; 2990 2991 hw->total_ports = ports; 2992 2993 eth_dev->device = &dev->device; 2994 rte_eth_copy_pci_info(eth_dev, dev); 2995 2996 ret = nfp_net_init(eth_dev); 2997 2998 if (ret) 2999 rte_eth_dev_release_port(eth_dev); 3000 else 3001 rte_eth_dev_probing_finish(eth_dev); 3002 3003 rte_free(port_name); 3004 3005 return ret; 3006 } 3007 3008 #define DEFAULT_FW_PATH "/lib/firmware/netronome" 3009 3010 static int 3011 nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card) 3012 { 3013 struct nfp_cpp *cpp = nsp->cpp; 3014 int fw_f; 3015 char *fw_buf; 3016 char fw_name[125]; 3017 char serial[40]; 3018 struct stat file_stat; 3019 off_t fsize, bytes; 3020 3021 /* Looking for firmware file in order of priority */ 3022 3023 /* First try to find a firmware image specific for this device */ 3024 sprintf(serial, "serial-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x", 3025 cpp->serial[0], cpp->serial[1], cpp->serial[2], cpp->serial[3], 3026 cpp->serial[4], cpp->serial[5], cpp->interface >> 8, 3027 cpp->interface & 0xff); 3028 3029 sprintf(fw_name, "%s/%s.nffw", DEFAULT_FW_PATH, serial); 3030 3031 PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name); 3032 fw_f = open(fw_name, O_RDONLY); 3033 if (fw_f > 0) 3034 goto read_fw; 3035 3036 /* Then try the PCI name */ 3037 sprintf(fw_name, "%s/pci-%s.nffw", DEFAULT_FW_PATH, dev->device.name); 3038 3039 PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name); 3040 fw_f = open(fw_name, O_RDONLY); 3041 if (fw_f > 0) 3042 goto read_fw; 3043 3044 /* Finally try the card type and media */ 3045 sprintf(fw_name, "%s/%s", DEFAULT_FW_PATH, card); 3046 PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name); 3047 fw_f = open(fw_name, O_RDONLY); 3048 if (fw_f < 0) { 3049 PMD_DRV_LOG(INFO, "Firmware file %s not found.", fw_name); 3050 return -ENOENT; 3051 } 3052 3053 read_fw: 3054 if (fstat(fw_f, &file_stat) < 0) { 3055 PMD_DRV_LOG(INFO, "Firmware file %s size is unknown", fw_name); 3056 close(fw_f); 3057 return -ENOENT; 3058 } 3059 3060 fsize = file_stat.st_size; 3061 PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %" PRIu64 "", 3062 fw_name, (uint64_t)fsize); 3063 3064 fw_buf = malloc((size_t)fsize); 3065 if (!fw_buf) { 3066 PMD_DRV_LOG(INFO, "malloc failed for fw buffer"); 3067 close(fw_f); 3068 return -ENOMEM; 3069 } 3070 memset(fw_buf, 0, fsize); 3071 3072 bytes = read(fw_f, fw_buf, fsize); 3073 if (bytes != fsize) { 3074 PMD_DRV_LOG(INFO, "Reading fw to buffer failed." 3075 "Just %" PRIu64 " of %" PRIu64 " bytes read", 3076 (uint64_t)bytes, (uint64_t)fsize); 3077 free(fw_buf); 3078 close(fw_f); 3079 return -EIO; 3080 } 3081 3082 PMD_DRV_LOG(INFO, "Uploading the firmware ..."); 3083 nfp_nsp_load_fw(nsp, fw_buf, bytes); 3084 PMD_DRV_LOG(INFO, "Done"); 3085 3086 free(fw_buf); 3087 close(fw_f); 3088 3089 return 0; 3090 } 3091 3092 static int 3093 nfp_fw_setup(struct rte_pci_device *dev, struct nfp_cpp *cpp, 3094 struct nfp_eth_table *nfp_eth_table, struct nfp_hwinfo *hwinfo) 3095 { 3096 struct nfp_nsp *nsp; 3097 const char *nfp_fw_model; 3098 char card_desc[100]; 3099 int err = 0; 3100 3101 nfp_fw_model = nfp_hwinfo_lookup(hwinfo, "assembly.partno"); 3102 3103 if (nfp_fw_model) { 3104 PMD_DRV_LOG(INFO, "firmware model found: %s", nfp_fw_model); 3105 } else { 3106 PMD_DRV_LOG(ERR, "firmware model NOT found"); 3107 return -EIO; 3108 } 3109 3110 if (nfp_eth_table->count == 0 || nfp_eth_table->count > 8) { 3111 PMD_DRV_LOG(ERR, "NFP ethernet table reports wrong ports: %u", 3112 nfp_eth_table->count); 3113 return -EIO; 3114 } 3115 3116 PMD_DRV_LOG(INFO, "NFP ethernet port table reports %u ports", 3117 nfp_eth_table->count); 3118 3119 PMD_DRV_LOG(INFO, "Port speed: %u", nfp_eth_table->ports[0].speed); 3120 3121 sprintf(card_desc, "nic_%s_%dx%d.nffw", nfp_fw_model, 3122 nfp_eth_table->count, nfp_eth_table->ports[0].speed / 1000); 3123 3124 nsp = nfp_nsp_open(cpp); 3125 if (!nsp) { 3126 PMD_DRV_LOG(ERR, "NFP error when obtaining NSP handle"); 3127 return -EIO; 3128 } 3129 3130 nfp_nsp_device_soft_reset(nsp); 3131 err = nfp_fw_upload(dev, nsp, card_desc); 3132 3133 nfp_nsp_close(nsp); 3134 return err; 3135 } 3136 3137 static int nfp_pf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 3138 struct rte_pci_device *dev) 3139 { 3140 struct nfp_cpp *cpp; 3141 struct nfp_hwinfo *hwinfo; 3142 struct nfp_rtsym_table *sym_tbl; 3143 struct nfp_eth_table *nfp_eth_table = NULL; 3144 int total_ports; 3145 void *priv = 0; 3146 int ret = -ENODEV; 3147 int err; 3148 int i; 3149 3150 if (!dev) 3151 return ret; 3152 3153 /* 3154 * When device bound to UIO, the device could be used, by mistake, 3155 * by two DPDK apps, and the UIO driver does not avoid it. This 3156 * could lead to a serious problem when configuring the NFP CPP 3157 * interface. Here we avoid this telling to the CPP init code to 3158 * use a lock file if UIO is being used. 3159 */ 3160 if (dev->kdrv == RTE_KDRV_VFIO) 3161 cpp = nfp_cpp_from_device_name(dev, 0); 3162 else 3163 cpp = nfp_cpp_from_device_name(dev, 1); 3164 3165 if (!cpp) { 3166 PMD_DRV_LOG(ERR, "A CPP handle can not be obtained"); 3167 ret = -EIO; 3168 goto error; 3169 } 3170 3171 hwinfo = nfp_hwinfo_read(cpp); 3172 if (!hwinfo) { 3173 PMD_DRV_LOG(ERR, "Error reading hwinfo table"); 3174 return -EIO; 3175 } 3176 3177 nfp_eth_table = nfp_eth_read_ports(cpp); 3178 if (!nfp_eth_table) { 3179 PMD_DRV_LOG(ERR, "Error reading NFP ethernet table"); 3180 return -EIO; 3181 } 3182 3183 if (nfp_fw_setup(dev, cpp, nfp_eth_table, hwinfo)) { 3184 PMD_DRV_LOG(INFO, "Error when uploading firmware"); 3185 ret = -EIO; 3186 goto error; 3187 } 3188 3189 /* Now the symbol table should be there */ 3190 sym_tbl = nfp_rtsym_table_read(cpp); 3191 if (!sym_tbl) { 3192 PMD_DRV_LOG(ERR, "Something is wrong with the firmware" 3193 " symbol table"); 3194 ret = -EIO; 3195 goto error; 3196 } 3197 3198 total_ports = nfp_rtsym_read_le(sym_tbl, "nfd_cfg_pf0_num_ports", &err); 3199 if (total_ports != (int)nfp_eth_table->count) { 3200 PMD_DRV_LOG(ERR, "Inconsistent number of ports"); 3201 ret = -EIO; 3202 goto error; 3203 } 3204 PMD_INIT_LOG(INFO, "Total pf ports: %d", total_ports); 3205 3206 if (total_ports <= 0 || total_ports > 8) { 3207 PMD_DRV_LOG(ERR, "nfd_cfg_pf0_num_ports symbol with wrong value"); 3208 ret = -ENODEV; 3209 goto error; 3210 } 3211 3212 for (i = 0; i < total_ports; i++) { 3213 ret = nfp_pf_create_dev(dev, i, total_ports, cpp, hwinfo, 3214 nfp_eth_table->ports[i].index, 3215 sym_tbl, &priv); 3216 if (ret) 3217 break; 3218 } 3219 3220 error: 3221 free(nfp_eth_table); 3222 return ret; 3223 } 3224 3225 int nfp_logtype_init; 3226 int nfp_logtype_driver; 3227 3228 static const struct rte_pci_id pci_id_nfp_pf_net_map[] = { 3229 { 3230 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME, 3231 PCI_DEVICE_ID_NFP4000_PF_NIC) 3232 }, 3233 { 3234 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME, 3235 PCI_DEVICE_ID_NFP6000_PF_NIC) 3236 }, 3237 { 3238 .vendor_id = 0, 3239 }, 3240 }; 3241 3242 static const struct rte_pci_id pci_id_nfp_vf_net_map[] = { 3243 { 3244 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME, 3245 PCI_DEVICE_ID_NFP6000_VF_NIC) 3246 }, 3247 { 3248 .vendor_id = 0, 3249 }, 3250 }; 3251 3252 static int eth_nfp_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 3253 struct rte_pci_device *pci_dev) 3254 { 3255 return rte_eth_dev_pci_generic_probe(pci_dev, 3256 sizeof(struct nfp_net_adapter), nfp_net_init); 3257 } 3258 3259 static int eth_nfp_pci_remove(struct rte_pci_device *pci_dev) 3260 { 3261 struct rte_eth_dev *eth_dev; 3262 struct nfp_net_hw *hw, *hwport0; 3263 int port = 0; 3264 3265 eth_dev = rte_eth_dev_allocated(pci_dev->device.name); 3266 if ((pci_dev->id.device_id == PCI_DEVICE_ID_NFP4000_PF_NIC) || 3267 (pci_dev->id.device_id == PCI_DEVICE_ID_NFP6000_PF_NIC)) { 3268 port = get_pf_port_number(eth_dev->data->name); 3269 /* 3270 * hotplug is not possible with multiport PF although freeing 3271 * data structures can be done for first port. 3272 */ 3273 if (port != 0) 3274 return -ENOTSUP; 3275 hwport0 = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 3276 hw = &hwport0[port]; 3277 nfp_cpp_area_free(hw->ctrl_area); 3278 nfp_cpp_area_free(hw->hwqueues_area); 3279 free(hw->hwinfo); 3280 free(hw->sym_tbl); 3281 nfp_cpp_free(hw->cpp); 3282 } else { 3283 hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 3284 } 3285 /* hotplug is not possible with multiport PF */ 3286 if (hw->pf_multiport_enabled) 3287 return -ENOTSUP; 3288 return rte_eth_dev_pci_generic_remove(pci_dev, NULL); 3289 } 3290 3291 static struct rte_pci_driver rte_nfp_net_pf_pmd = { 3292 .id_table = pci_id_nfp_pf_net_map, 3293 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | 3294 RTE_PCI_DRV_IOVA_AS_VA, 3295 .probe = nfp_pf_pci_probe, 3296 .remove = eth_nfp_pci_remove, 3297 }; 3298 3299 static struct rte_pci_driver rte_nfp_net_vf_pmd = { 3300 .id_table = pci_id_nfp_vf_net_map, 3301 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | 3302 RTE_PCI_DRV_IOVA_AS_VA, 3303 .probe = eth_nfp_pci_probe, 3304 .remove = eth_nfp_pci_remove, 3305 }; 3306 3307 RTE_PMD_REGISTER_PCI(net_nfp_pf, rte_nfp_net_pf_pmd); 3308 RTE_PMD_REGISTER_PCI(net_nfp_vf, rte_nfp_net_vf_pmd); 3309 RTE_PMD_REGISTER_PCI_TABLE(net_nfp_pf, pci_id_nfp_pf_net_map); 3310 RTE_PMD_REGISTER_PCI_TABLE(net_nfp_vf, pci_id_nfp_vf_net_map); 3311 RTE_PMD_REGISTER_KMOD_DEP(net_nfp_pf, "* igb_uio | uio_pci_generic | vfio"); 3312 RTE_PMD_REGISTER_KMOD_DEP(net_nfp_vf, "* igb_uio | uio_pci_generic | vfio"); 3313 3314 RTE_INIT(nfp_init_log) 3315 { 3316 nfp_logtype_init = rte_log_register("pmd.net.nfp.init"); 3317 if (nfp_logtype_init >= 0) 3318 rte_log_set_level(nfp_logtype_init, RTE_LOG_NOTICE); 3319 nfp_logtype_driver = rte_log_register("pmd.net.nfp.driver"); 3320 if (nfp_logtype_driver >= 0) 3321 rte_log_set_level(nfp_logtype_driver, RTE_LOG_NOTICE); 3322 } 3323 /* 3324 * Local variables: 3325 * c-file-style: "Linux" 3326 * indent-tabs-mode: t 3327 * End: 3328 */ 3329