1 /* 2 * Copyright (c) 2014-2018 Netronome Systems, Inc. 3 * All rights reserved. 4 * 5 * Small portions derived from code Copyright(c) 2010-2015 Intel Corporation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution 16 * 17 * 3. Neither the name of the copyright holder nor the names of its 18 * contributors may be used to endorse or promote products derived from this 19 * software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * vim:shiftwidth=8:noexpandtab 36 * 37 * @file dpdk/pmd/nfp_net.c 38 * 39 * Netronome vNIC DPDK Poll-Mode Driver: Main entry point 40 */ 41 42 #include <rte_byteorder.h> 43 #include <rte_common.h> 44 #include <rte_log.h> 45 #include <rte_debug.h> 46 #include <rte_ethdev_driver.h> 47 #include <rte_ethdev_pci.h> 48 #include <rte_dev.h> 49 #include <rte_ether.h> 50 #include <rte_malloc.h> 51 #include <rte_memzone.h> 52 #include <rte_mempool.h> 53 #include <rte_version.h> 54 #include <rte_string_fns.h> 55 #include <rte_alarm.h> 56 #include <rte_spinlock.h> 57 58 #include "nfpcore/nfp_cpp.h" 59 #include "nfpcore/nfp_nffw.h" 60 #include "nfpcore/nfp_hwinfo.h" 61 #include "nfpcore/nfp_mip.h" 62 #include "nfpcore/nfp_rtsym.h" 63 #include "nfpcore/nfp_nsp.h" 64 65 #include "nfp_net_pmd.h" 66 #include "nfp_net_logs.h" 67 #include "nfp_net_ctrl.h" 68 69 /* Prototypes */ 70 static void nfp_net_close(struct rte_eth_dev *dev); 71 static int nfp_net_configure(struct rte_eth_dev *dev); 72 static void nfp_net_dev_interrupt_handler(void *param); 73 static void nfp_net_dev_interrupt_delayed_handler(void *param); 74 static int nfp_net_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 75 static void nfp_net_infos_get(struct rte_eth_dev *dev, 76 struct rte_eth_dev_info *dev_info); 77 static int nfp_net_init(struct rte_eth_dev *eth_dev); 78 static int nfp_net_link_update(struct rte_eth_dev *dev, int wait_to_complete); 79 static void nfp_net_promisc_enable(struct rte_eth_dev *dev); 80 static void nfp_net_promisc_disable(struct rte_eth_dev *dev); 81 static int nfp_net_rx_fill_freelist(struct nfp_net_rxq *rxq); 82 static uint32_t nfp_net_rx_queue_count(struct rte_eth_dev *dev, 83 uint16_t queue_idx); 84 static uint16_t nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 85 uint16_t nb_pkts); 86 static void nfp_net_rx_queue_release(void *rxq); 87 static int nfp_net_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 88 uint16_t nb_desc, unsigned int socket_id, 89 const struct rte_eth_rxconf *rx_conf, 90 struct rte_mempool *mp); 91 static int nfp_net_tx_free_bufs(struct nfp_net_txq *txq); 92 static void nfp_net_tx_queue_release(void *txq); 93 static int nfp_net_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 94 uint16_t nb_desc, unsigned int socket_id, 95 const struct rte_eth_txconf *tx_conf); 96 static int nfp_net_start(struct rte_eth_dev *dev); 97 static int nfp_net_stats_get(struct rte_eth_dev *dev, 98 struct rte_eth_stats *stats); 99 static void nfp_net_stats_reset(struct rte_eth_dev *dev); 100 static void nfp_net_stop(struct rte_eth_dev *dev); 101 static uint16_t nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 102 uint16_t nb_pkts); 103 104 static int nfp_net_rss_config_default(struct rte_eth_dev *dev); 105 static int nfp_net_rss_hash_update(struct rte_eth_dev *dev, 106 struct rte_eth_rss_conf *rss_conf); 107 static int nfp_net_rss_reta_write(struct rte_eth_dev *dev, 108 struct rte_eth_rss_reta_entry64 *reta_conf, 109 uint16_t reta_size); 110 static int nfp_net_rss_hash_write(struct rte_eth_dev *dev, 111 struct rte_eth_rss_conf *rss_conf); 112 static int nfp_set_mac_addr(struct rte_eth_dev *dev, 113 struct ether_addr *mac_addr); 114 115 /* The offset of the queue controller queues in the PCIe Target */ 116 #define NFP_PCIE_QUEUE(_q) (0x80000 + (NFP_QCP_QUEUE_ADDR_SZ * ((_q) & 0xff))) 117 118 /* Maximum value which can be added to a queue with one transaction */ 119 #define NFP_QCP_MAX_ADD 0x7f 120 121 #define RTE_MBUF_DMA_ADDR_DEFAULT(mb) \ 122 (uint64_t)((mb)->buf_iova + RTE_PKTMBUF_HEADROOM) 123 124 /* nfp_qcp_ptr - Read or Write Pointer of a queue */ 125 enum nfp_qcp_ptr { 126 NFP_QCP_READ_PTR = 0, 127 NFP_QCP_WRITE_PTR 128 }; 129 130 /* 131 * nfp_qcp_ptr_add - Add the value to the selected pointer of a queue 132 * @q: Base address for queue structure 133 * @ptr: Add to the Read or Write pointer 134 * @val: Value to add to the queue pointer 135 * 136 * If @val is greater than @NFP_QCP_MAX_ADD multiple writes are performed. 137 */ 138 static inline void 139 nfp_qcp_ptr_add(uint8_t *q, enum nfp_qcp_ptr ptr, uint32_t val) 140 { 141 uint32_t off; 142 143 if (ptr == NFP_QCP_READ_PTR) 144 off = NFP_QCP_QUEUE_ADD_RPTR; 145 else 146 off = NFP_QCP_QUEUE_ADD_WPTR; 147 148 while (val > NFP_QCP_MAX_ADD) { 149 nn_writel(rte_cpu_to_le_32(NFP_QCP_MAX_ADD), q + off); 150 val -= NFP_QCP_MAX_ADD; 151 } 152 153 nn_writel(rte_cpu_to_le_32(val), q + off); 154 } 155 156 /* 157 * nfp_qcp_read - Read the current Read/Write pointer value for a queue 158 * @q: Base address for queue structure 159 * @ptr: Read or Write pointer 160 */ 161 static inline uint32_t 162 nfp_qcp_read(uint8_t *q, enum nfp_qcp_ptr ptr) 163 { 164 uint32_t off; 165 uint32_t val; 166 167 if (ptr == NFP_QCP_READ_PTR) 168 off = NFP_QCP_QUEUE_STS_LO; 169 else 170 off = NFP_QCP_QUEUE_STS_HI; 171 172 val = rte_cpu_to_le_32(nn_readl(q + off)); 173 174 if (ptr == NFP_QCP_READ_PTR) 175 return val & NFP_QCP_QUEUE_STS_LO_READPTR_mask; 176 else 177 return val & NFP_QCP_QUEUE_STS_HI_WRITEPTR_mask; 178 } 179 180 /* 181 * Functions to read/write from/to Config BAR 182 * Performs any endian conversion necessary. 183 */ 184 static inline uint8_t 185 nn_cfg_readb(struct nfp_net_hw *hw, int off) 186 { 187 return nn_readb(hw->ctrl_bar + off); 188 } 189 190 static inline void 191 nn_cfg_writeb(struct nfp_net_hw *hw, int off, uint8_t val) 192 { 193 nn_writeb(val, hw->ctrl_bar + off); 194 } 195 196 static inline uint32_t 197 nn_cfg_readl(struct nfp_net_hw *hw, int off) 198 { 199 return rte_le_to_cpu_32(nn_readl(hw->ctrl_bar + off)); 200 } 201 202 static inline void 203 nn_cfg_writel(struct nfp_net_hw *hw, int off, uint32_t val) 204 { 205 nn_writel(rte_cpu_to_le_32(val), hw->ctrl_bar + off); 206 } 207 208 static inline uint64_t 209 nn_cfg_readq(struct nfp_net_hw *hw, int off) 210 { 211 return rte_le_to_cpu_64(nn_readq(hw->ctrl_bar + off)); 212 } 213 214 static inline void 215 nn_cfg_writeq(struct nfp_net_hw *hw, int off, uint64_t val) 216 { 217 nn_writeq(rte_cpu_to_le_64(val), hw->ctrl_bar + off); 218 } 219 220 static void 221 nfp_net_rx_queue_release_mbufs(struct nfp_net_rxq *rxq) 222 { 223 unsigned i; 224 225 if (rxq->rxbufs == NULL) 226 return; 227 228 for (i = 0; i < rxq->rx_count; i++) { 229 if (rxq->rxbufs[i].mbuf) { 230 rte_pktmbuf_free_seg(rxq->rxbufs[i].mbuf); 231 rxq->rxbufs[i].mbuf = NULL; 232 } 233 } 234 } 235 236 static void 237 nfp_net_rx_queue_release(void *rx_queue) 238 { 239 struct nfp_net_rxq *rxq = rx_queue; 240 241 if (rxq) { 242 nfp_net_rx_queue_release_mbufs(rxq); 243 rte_free(rxq->rxbufs); 244 rte_free(rxq); 245 } 246 } 247 248 static void 249 nfp_net_reset_rx_queue(struct nfp_net_rxq *rxq) 250 { 251 nfp_net_rx_queue_release_mbufs(rxq); 252 rxq->rd_p = 0; 253 rxq->nb_rx_hold = 0; 254 } 255 256 static void 257 nfp_net_tx_queue_release_mbufs(struct nfp_net_txq *txq) 258 { 259 unsigned i; 260 261 if (txq->txbufs == NULL) 262 return; 263 264 for (i = 0; i < txq->tx_count; i++) { 265 if (txq->txbufs[i].mbuf) { 266 rte_pktmbuf_free_seg(txq->txbufs[i].mbuf); 267 txq->txbufs[i].mbuf = NULL; 268 } 269 } 270 } 271 272 static void 273 nfp_net_tx_queue_release(void *tx_queue) 274 { 275 struct nfp_net_txq *txq = tx_queue; 276 277 if (txq) { 278 nfp_net_tx_queue_release_mbufs(txq); 279 rte_free(txq->txbufs); 280 rte_free(txq); 281 } 282 } 283 284 static void 285 nfp_net_reset_tx_queue(struct nfp_net_txq *txq) 286 { 287 nfp_net_tx_queue_release_mbufs(txq); 288 txq->wr_p = 0; 289 txq->rd_p = 0; 290 } 291 292 static int 293 __nfp_net_reconfig(struct nfp_net_hw *hw, uint32_t update) 294 { 295 int cnt; 296 uint32_t new; 297 struct timespec wait; 298 299 PMD_DRV_LOG(DEBUG, "Writing to the configuration queue (%p)...", 300 hw->qcp_cfg); 301 302 if (hw->qcp_cfg == NULL) 303 rte_panic("Bad configuration queue pointer\n"); 304 305 nfp_qcp_ptr_add(hw->qcp_cfg, NFP_QCP_WRITE_PTR, 1); 306 307 wait.tv_sec = 0; 308 wait.tv_nsec = 1000000; 309 310 PMD_DRV_LOG(DEBUG, "Polling for update ack..."); 311 312 /* Poll update field, waiting for NFP to ack the config */ 313 for (cnt = 0; ; cnt++) { 314 new = nn_cfg_readl(hw, NFP_NET_CFG_UPDATE); 315 if (new == 0) 316 break; 317 if (new & NFP_NET_CFG_UPDATE_ERR) { 318 PMD_INIT_LOG(ERR, "Reconfig error: 0x%08x", new); 319 return -1; 320 } 321 if (cnt >= NFP_NET_POLL_TIMEOUT) { 322 PMD_INIT_LOG(ERR, "Reconfig timeout for 0x%08x after" 323 " %dms", update, cnt); 324 rte_panic("Exiting\n"); 325 } 326 nanosleep(&wait, 0); /* waiting for a 1ms */ 327 } 328 PMD_DRV_LOG(DEBUG, "Ack DONE"); 329 return 0; 330 } 331 332 /* 333 * Reconfigure the NIC 334 * @nn: device to reconfigure 335 * @ctrl: The value for the ctrl field in the BAR config 336 * @update: The value for the update field in the BAR config 337 * 338 * Write the update word to the BAR and ping the reconfig queue. Then poll 339 * until the firmware has acknowledged the update by zeroing the update word. 340 */ 341 static int 342 nfp_net_reconfig(struct nfp_net_hw *hw, uint32_t ctrl, uint32_t update) 343 { 344 uint32_t err; 345 346 PMD_DRV_LOG(DEBUG, "nfp_net_reconfig: ctrl=%08x update=%08x", 347 ctrl, update); 348 349 rte_spinlock_lock(&hw->reconfig_lock); 350 351 nn_cfg_writel(hw, NFP_NET_CFG_CTRL, ctrl); 352 nn_cfg_writel(hw, NFP_NET_CFG_UPDATE, update); 353 354 rte_wmb(); 355 356 err = __nfp_net_reconfig(hw, update); 357 358 rte_spinlock_unlock(&hw->reconfig_lock); 359 360 if (!err) 361 return 0; 362 363 /* 364 * Reconfig errors imply situations where they can be handled. 365 * Otherwise, rte_panic is called inside __nfp_net_reconfig 366 */ 367 PMD_INIT_LOG(ERR, "Error nfp_net reconfig for ctrl: %x update: %x", 368 ctrl, update); 369 return -EIO; 370 } 371 372 /* 373 * Configure an Ethernet device. This function must be invoked first 374 * before any other function in the Ethernet API. This function can 375 * also be re-invoked when a device is in the stopped state. 376 */ 377 static int 378 nfp_net_configure(struct rte_eth_dev *dev) 379 { 380 struct rte_eth_conf *dev_conf; 381 struct rte_eth_rxmode *rxmode; 382 struct rte_eth_txmode *txmode; 383 struct nfp_net_hw *hw; 384 385 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 386 387 /* 388 * A DPDK app sends info about how many queues to use and how 389 * those queues need to be configured. This is used by the 390 * DPDK core and it makes sure no more queues than those 391 * advertised by the driver are requested. This function is 392 * called after that internal process 393 */ 394 395 PMD_INIT_LOG(DEBUG, "Configure"); 396 397 dev_conf = &dev->data->dev_conf; 398 rxmode = &dev_conf->rxmode; 399 txmode = &dev_conf->txmode; 400 401 /* Checking TX mode */ 402 if (txmode->mq_mode) { 403 PMD_INIT_LOG(INFO, "TX mq_mode DCB and VMDq not supported"); 404 return -EINVAL; 405 } 406 407 /* Checking RX mode */ 408 if (rxmode->mq_mode & ETH_MQ_RX_RSS && 409 !(hw->cap & NFP_NET_CFG_CTRL_RSS)) { 410 PMD_INIT_LOG(INFO, "RSS not supported"); 411 return -EINVAL; 412 } 413 414 return 0; 415 } 416 417 static void 418 nfp_net_enable_queues(struct rte_eth_dev *dev) 419 { 420 struct nfp_net_hw *hw; 421 uint64_t enabled_queues = 0; 422 int i; 423 424 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 425 426 /* Enabling the required TX queues in the device */ 427 for (i = 0; i < dev->data->nb_tx_queues; i++) 428 enabled_queues |= (1 << i); 429 430 nn_cfg_writeq(hw, NFP_NET_CFG_TXRS_ENABLE, enabled_queues); 431 432 enabled_queues = 0; 433 434 /* Enabling the required RX queues in the device */ 435 for (i = 0; i < dev->data->nb_rx_queues; i++) 436 enabled_queues |= (1 << i); 437 438 nn_cfg_writeq(hw, NFP_NET_CFG_RXRS_ENABLE, enabled_queues); 439 } 440 441 static void 442 nfp_net_disable_queues(struct rte_eth_dev *dev) 443 { 444 struct nfp_net_hw *hw; 445 uint32_t new_ctrl, update = 0; 446 447 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 448 449 nn_cfg_writeq(hw, NFP_NET_CFG_TXRS_ENABLE, 0); 450 nn_cfg_writeq(hw, NFP_NET_CFG_RXRS_ENABLE, 0); 451 452 new_ctrl = hw->ctrl & ~NFP_NET_CFG_CTRL_ENABLE; 453 update = NFP_NET_CFG_UPDATE_GEN | NFP_NET_CFG_UPDATE_RING | 454 NFP_NET_CFG_UPDATE_MSIX; 455 456 if (hw->cap & NFP_NET_CFG_CTRL_RINGCFG) 457 new_ctrl &= ~NFP_NET_CFG_CTRL_RINGCFG; 458 459 /* If an error when reconfig we avoid to change hw state */ 460 if (nfp_net_reconfig(hw, new_ctrl, update) < 0) 461 return; 462 463 hw->ctrl = new_ctrl; 464 } 465 466 static int 467 nfp_net_rx_freelist_setup(struct rte_eth_dev *dev) 468 { 469 int i; 470 471 for (i = 0; i < dev->data->nb_rx_queues; i++) { 472 if (nfp_net_rx_fill_freelist(dev->data->rx_queues[i]) < 0) 473 return -1; 474 } 475 return 0; 476 } 477 478 static void 479 nfp_net_params_setup(struct nfp_net_hw *hw) 480 { 481 nn_cfg_writel(hw, NFP_NET_CFG_MTU, hw->mtu); 482 nn_cfg_writel(hw, NFP_NET_CFG_FLBUFSZ, hw->flbufsz); 483 } 484 485 static void 486 nfp_net_cfg_queue_setup(struct nfp_net_hw *hw) 487 { 488 hw->qcp_cfg = hw->tx_bar + NFP_QCP_QUEUE_ADDR_SZ; 489 } 490 491 #define ETH_ADDR_LEN 6 492 493 static void 494 nfp_eth_copy_mac(uint8_t *dst, const uint8_t *src) 495 { 496 int i; 497 498 for (i = 0; i < ETH_ADDR_LEN; i++) 499 dst[i] = src[i]; 500 } 501 502 static int 503 nfp_net_pf_read_mac(struct nfp_net_hw *hw, int port) 504 { 505 struct nfp_eth_table *nfp_eth_table; 506 507 nfp_eth_table = nfp_eth_read_ports(hw->cpp); 508 /* 509 * hw points to port0 private data. We need hw now pointing to 510 * right port. 511 */ 512 hw += port; 513 nfp_eth_copy_mac((uint8_t *)&hw->mac_addr, 514 (uint8_t *)&nfp_eth_table->ports[port].mac_addr); 515 516 free(nfp_eth_table); 517 return 0; 518 } 519 520 static void 521 nfp_net_vf_read_mac(struct nfp_net_hw *hw) 522 { 523 uint32_t tmp; 524 525 tmp = rte_be_to_cpu_32(nn_cfg_readl(hw, NFP_NET_CFG_MACADDR)); 526 memcpy(&hw->mac_addr[0], &tmp, 4); 527 528 tmp = rte_be_to_cpu_32(nn_cfg_readl(hw, NFP_NET_CFG_MACADDR + 4)); 529 memcpy(&hw->mac_addr[4], &tmp, 2); 530 } 531 532 static void 533 nfp_net_write_mac(struct nfp_net_hw *hw, uint8_t *mac) 534 { 535 uint32_t mac0 = *(uint32_t *)mac; 536 uint16_t mac1; 537 538 nn_writel(rte_cpu_to_be_32(mac0), hw->ctrl_bar + NFP_NET_CFG_MACADDR); 539 540 mac += 4; 541 mac1 = *(uint16_t *)mac; 542 nn_writew(rte_cpu_to_be_16(mac1), 543 hw->ctrl_bar + NFP_NET_CFG_MACADDR + 6); 544 } 545 546 int 547 nfp_set_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr) 548 { 549 struct nfp_net_hw *hw; 550 uint32_t update, ctrl; 551 552 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 553 if ((hw->ctrl & NFP_NET_CFG_CTRL_ENABLE) && 554 !(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR)) { 555 PMD_INIT_LOG(INFO, "MAC address unable to change when" 556 " port enabled"); 557 return -EBUSY; 558 } 559 560 if ((hw->ctrl & NFP_NET_CFG_CTRL_ENABLE) && 561 !(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR)) 562 return -EBUSY; 563 564 /* Writing new MAC to the specific port BAR address */ 565 nfp_net_write_mac(hw, (uint8_t *)mac_addr); 566 567 /* Signal the NIC about the change */ 568 update = NFP_NET_CFG_UPDATE_MACADDR; 569 ctrl = hw->ctrl; 570 if ((hw->ctrl & NFP_NET_CFG_CTRL_ENABLE) && 571 (hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR)) 572 ctrl |= NFP_NET_CFG_CTRL_LIVE_ADDR; 573 if (nfp_net_reconfig(hw, ctrl, update) < 0) { 574 PMD_INIT_LOG(INFO, "MAC address update failed"); 575 return -EIO; 576 } 577 return 0; 578 } 579 580 static int 581 nfp_configure_rx_interrupt(struct rte_eth_dev *dev, 582 struct rte_intr_handle *intr_handle) 583 { 584 struct nfp_net_hw *hw; 585 int i; 586 587 if (!intr_handle->intr_vec) { 588 intr_handle->intr_vec = 589 rte_zmalloc("intr_vec", 590 dev->data->nb_rx_queues * sizeof(int), 0); 591 if (!intr_handle->intr_vec) { 592 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 593 " intr_vec", dev->data->nb_rx_queues); 594 return -ENOMEM; 595 } 596 } 597 598 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 599 600 if (intr_handle->type == RTE_INTR_HANDLE_UIO) { 601 PMD_INIT_LOG(INFO, "VF: enabling RX interrupt with UIO"); 602 /* UIO just supports one queue and no LSC*/ 603 nn_cfg_writeb(hw, NFP_NET_CFG_RXR_VEC(0), 0); 604 intr_handle->intr_vec[0] = 0; 605 } else { 606 PMD_INIT_LOG(INFO, "VF: enabling RX interrupt with VFIO"); 607 for (i = 0; i < dev->data->nb_rx_queues; i++) { 608 /* 609 * The first msix vector is reserved for non 610 * efd interrupts 611 */ 612 nn_cfg_writeb(hw, NFP_NET_CFG_RXR_VEC(i), i + 1); 613 intr_handle->intr_vec[i] = i + 1; 614 PMD_INIT_LOG(DEBUG, "intr_vec[%d]= %d", i, 615 intr_handle->intr_vec[i]); 616 } 617 } 618 619 /* Avoiding TX interrupts */ 620 hw->ctrl |= NFP_NET_CFG_CTRL_MSIX_TX_OFF; 621 return 0; 622 } 623 624 static uint32_t 625 nfp_check_offloads(struct rte_eth_dev *dev) 626 { 627 struct nfp_net_hw *hw; 628 struct rte_eth_conf *dev_conf; 629 struct rte_eth_rxmode *rxmode; 630 struct rte_eth_txmode *txmode; 631 uint32_t ctrl = 0; 632 633 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 634 635 dev_conf = &dev->data->dev_conf; 636 rxmode = &dev_conf->rxmode; 637 txmode = &dev_conf->txmode; 638 639 if (rxmode->offloads & DEV_RX_OFFLOAD_IPV4_CKSUM) { 640 if (hw->cap & NFP_NET_CFG_CTRL_RXCSUM) 641 ctrl |= NFP_NET_CFG_CTRL_RXCSUM; 642 } 643 644 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) { 645 if (hw->cap & NFP_NET_CFG_CTRL_RXVLAN) 646 ctrl |= NFP_NET_CFG_CTRL_RXVLAN; 647 } 648 649 if (rxmode->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 650 hw->mtu = rxmode->max_rx_pkt_len; 651 652 if (txmode->offloads & DEV_TX_OFFLOAD_VLAN_INSERT) 653 ctrl |= NFP_NET_CFG_CTRL_TXVLAN; 654 655 /* L2 broadcast */ 656 if (hw->cap & NFP_NET_CFG_CTRL_L2BC) 657 ctrl |= NFP_NET_CFG_CTRL_L2BC; 658 659 /* L2 multicast */ 660 if (hw->cap & NFP_NET_CFG_CTRL_L2MC) 661 ctrl |= NFP_NET_CFG_CTRL_L2MC; 662 663 /* TX checksum offload */ 664 if (txmode->offloads & DEV_TX_OFFLOAD_IPV4_CKSUM || 665 txmode->offloads & DEV_TX_OFFLOAD_UDP_CKSUM || 666 txmode->offloads & DEV_TX_OFFLOAD_TCP_CKSUM) 667 ctrl |= NFP_NET_CFG_CTRL_TXCSUM; 668 669 /* LSO offload */ 670 if (txmode->offloads & DEV_TX_OFFLOAD_TCP_TSO) { 671 if (hw->cap & NFP_NET_CFG_CTRL_LSO) 672 ctrl |= NFP_NET_CFG_CTRL_LSO; 673 else 674 ctrl |= NFP_NET_CFG_CTRL_LSO2; 675 } 676 677 /* RX gather */ 678 if (txmode->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) 679 ctrl |= NFP_NET_CFG_CTRL_GATHER; 680 681 return ctrl; 682 } 683 684 static int 685 nfp_net_start(struct rte_eth_dev *dev) 686 { 687 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 688 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 689 uint32_t new_ctrl, update = 0; 690 struct nfp_net_hw *hw; 691 struct rte_eth_conf *dev_conf; 692 struct rte_eth_rxmode *rxmode; 693 uint32_t intr_vector; 694 int ret; 695 696 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 697 698 PMD_INIT_LOG(DEBUG, "Start"); 699 700 /* Disabling queues just in case... */ 701 nfp_net_disable_queues(dev); 702 703 /* Enabling the required queues in the device */ 704 nfp_net_enable_queues(dev); 705 706 /* check and configure queue intr-vector mapping */ 707 if (dev->data->dev_conf.intr_conf.rxq != 0) { 708 if (hw->pf_multiport_enabled) { 709 PMD_INIT_LOG(ERR, "PMD rx interrupt is not supported " 710 "with NFP multiport PF"); 711 return -EINVAL; 712 } 713 if (intr_handle->type == RTE_INTR_HANDLE_UIO) { 714 /* 715 * Better not to share LSC with RX interrupts. 716 * Unregistering LSC interrupt handler 717 */ 718 rte_intr_callback_unregister(&pci_dev->intr_handle, 719 nfp_net_dev_interrupt_handler, (void *)dev); 720 721 if (dev->data->nb_rx_queues > 1) { 722 PMD_INIT_LOG(ERR, "PMD rx interrupt only " 723 "supports 1 queue with UIO"); 724 return -EIO; 725 } 726 } 727 intr_vector = dev->data->nb_rx_queues; 728 if (rte_intr_efd_enable(intr_handle, intr_vector)) 729 return -1; 730 731 nfp_configure_rx_interrupt(dev, intr_handle); 732 update = NFP_NET_CFG_UPDATE_MSIX; 733 } 734 735 rte_intr_enable(intr_handle); 736 737 new_ctrl = nfp_check_offloads(dev); 738 739 /* Writing configuration parameters in the device */ 740 nfp_net_params_setup(hw); 741 742 dev_conf = &dev->data->dev_conf; 743 rxmode = &dev_conf->rxmode; 744 745 if (rxmode->mq_mode & ETH_MQ_RX_RSS) { 746 nfp_net_rss_config_default(dev); 747 update |= NFP_NET_CFG_UPDATE_RSS; 748 new_ctrl |= NFP_NET_CFG_CTRL_RSS; 749 } 750 751 /* Enable device */ 752 new_ctrl |= NFP_NET_CFG_CTRL_ENABLE; 753 754 update |= NFP_NET_CFG_UPDATE_GEN | NFP_NET_CFG_UPDATE_RING; 755 756 if (hw->cap & NFP_NET_CFG_CTRL_RINGCFG) 757 new_ctrl |= NFP_NET_CFG_CTRL_RINGCFG; 758 759 nn_cfg_writel(hw, NFP_NET_CFG_CTRL, new_ctrl); 760 if (nfp_net_reconfig(hw, new_ctrl, update) < 0) 761 return -EIO; 762 763 /* 764 * Allocating rte mbufs for configured rx queues. 765 * This requires queues being enabled before 766 */ 767 if (nfp_net_rx_freelist_setup(dev) < 0) { 768 ret = -ENOMEM; 769 goto error; 770 } 771 772 if (hw->is_pf) 773 /* Configure the physical port up */ 774 nfp_eth_set_configured(hw->cpp, hw->pf_port_idx, 1); 775 776 hw->ctrl = new_ctrl; 777 778 return 0; 779 780 error: 781 /* 782 * An error returned by this function should mean the app 783 * exiting and then the system releasing all the memory 784 * allocated even memory coming from hugepages. 785 * 786 * The device could be enabled at this point with some queues 787 * ready for getting packets. This is true if the call to 788 * nfp_net_rx_freelist_setup() succeeds for some queues but 789 * fails for subsequent queues. 790 * 791 * This should make the app exiting but better if we tell the 792 * device first. 793 */ 794 nfp_net_disable_queues(dev); 795 796 return ret; 797 } 798 799 /* Stop device: disable rx and tx functions to allow for reconfiguring. */ 800 static void 801 nfp_net_stop(struct rte_eth_dev *dev) 802 { 803 int i; 804 struct nfp_net_hw *hw; 805 806 PMD_INIT_LOG(DEBUG, "Stop"); 807 808 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 809 810 nfp_net_disable_queues(dev); 811 812 /* Clear queues */ 813 for (i = 0; i < dev->data->nb_tx_queues; i++) { 814 nfp_net_reset_tx_queue( 815 (struct nfp_net_txq *)dev->data->tx_queues[i]); 816 } 817 818 for (i = 0; i < dev->data->nb_rx_queues; i++) { 819 nfp_net_reset_rx_queue( 820 (struct nfp_net_rxq *)dev->data->rx_queues[i]); 821 } 822 823 if (hw->is_pf) 824 /* Configure the physical port down */ 825 nfp_eth_set_configured(hw->cpp, hw->pf_port_idx, 0); 826 } 827 828 /* Reset and stop device. The device can not be restarted. */ 829 static void 830 nfp_net_close(struct rte_eth_dev *dev) 831 { 832 struct nfp_net_hw *hw; 833 struct rte_pci_device *pci_dev; 834 int i; 835 836 PMD_INIT_LOG(DEBUG, "Close"); 837 838 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 839 pci_dev = RTE_ETH_DEV_TO_PCI(dev); 840 841 /* 842 * We assume that the DPDK application is stopping all the 843 * threads/queues before calling the device close function. 844 */ 845 846 nfp_net_disable_queues(dev); 847 848 /* Clear queues */ 849 for (i = 0; i < dev->data->nb_tx_queues; i++) { 850 nfp_net_reset_tx_queue( 851 (struct nfp_net_txq *)dev->data->tx_queues[i]); 852 } 853 854 for (i = 0; i < dev->data->nb_rx_queues; i++) { 855 nfp_net_reset_rx_queue( 856 (struct nfp_net_rxq *)dev->data->rx_queues[i]); 857 } 858 859 rte_intr_disable(&pci_dev->intr_handle); 860 nn_cfg_writeb(hw, NFP_NET_CFG_LSC, 0xff); 861 862 /* unregister callback func from eal lib */ 863 rte_intr_callback_unregister(&pci_dev->intr_handle, 864 nfp_net_dev_interrupt_handler, 865 (void *)dev); 866 867 /* 868 * The ixgbe PMD driver disables the pcie master on the 869 * device. The i40e does not... 870 */ 871 } 872 873 static void 874 nfp_net_promisc_enable(struct rte_eth_dev *dev) 875 { 876 uint32_t new_ctrl, update = 0; 877 struct nfp_net_hw *hw; 878 879 PMD_DRV_LOG(DEBUG, "Promiscuous mode enable"); 880 881 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 882 883 if (!(hw->cap & NFP_NET_CFG_CTRL_PROMISC)) { 884 PMD_INIT_LOG(INFO, "Promiscuous mode not supported"); 885 return; 886 } 887 888 if (hw->ctrl & NFP_NET_CFG_CTRL_PROMISC) { 889 PMD_DRV_LOG(INFO, "Promiscuous mode already enabled"); 890 return; 891 } 892 893 new_ctrl = hw->ctrl | NFP_NET_CFG_CTRL_PROMISC; 894 update = NFP_NET_CFG_UPDATE_GEN; 895 896 /* 897 * DPDK sets promiscuous mode on just after this call assuming 898 * it can not fail ... 899 */ 900 if (nfp_net_reconfig(hw, new_ctrl, update) < 0) 901 return; 902 903 hw->ctrl = new_ctrl; 904 } 905 906 static void 907 nfp_net_promisc_disable(struct rte_eth_dev *dev) 908 { 909 uint32_t new_ctrl, update = 0; 910 struct nfp_net_hw *hw; 911 912 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 913 914 if ((hw->ctrl & NFP_NET_CFG_CTRL_PROMISC) == 0) { 915 PMD_DRV_LOG(INFO, "Promiscuous mode already disabled"); 916 return; 917 } 918 919 new_ctrl = hw->ctrl & ~NFP_NET_CFG_CTRL_PROMISC; 920 update = NFP_NET_CFG_UPDATE_GEN; 921 922 /* 923 * DPDK sets promiscuous mode off just before this call 924 * assuming it can not fail ... 925 */ 926 if (nfp_net_reconfig(hw, new_ctrl, update) < 0) 927 return; 928 929 hw->ctrl = new_ctrl; 930 } 931 932 /* 933 * return 0 means link status changed, -1 means not changed 934 * 935 * Wait to complete is needed as it can take up to 9 seconds to get the Link 936 * status. 937 */ 938 static int 939 nfp_net_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) 940 { 941 struct nfp_net_hw *hw; 942 struct rte_eth_link link; 943 uint32_t nn_link_status; 944 int ret; 945 946 static const uint32_t ls_to_ethtool[] = { 947 [NFP_NET_CFG_STS_LINK_RATE_UNSUPPORTED] = ETH_SPEED_NUM_NONE, 948 [NFP_NET_CFG_STS_LINK_RATE_UNKNOWN] = ETH_SPEED_NUM_NONE, 949 [NFP_NET_CFG_STS_LINK_RATE_1G] = ETH_SPEED_NUM_1G, 950 [NFP_NET_CFG_STS_LINK_RATE_10G] = ETH_SPEED_NUM_10G, 951 [NFP_NET_CFG_STS_LINK_RATE_25G] = ETH_SPEED_NUM_25G, 952 [NFP_NET_CFG_STS_LINK_RATE_40G] = ETH_SPEED_NUM_40G, 953 [NFP_NET_CFG_STS_LINK_RATE_50G] = ETH_SPEED_NUM_50G, 954 [NFP_NET_CFG_STS_LINK_RATE_100G] = ETH_SPEED_NUM_100G, 955 }; 956 957 PMD_DRV_LOG(DEBUG, "Link update"); 958 959 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 960 961 nn_link_status = nn_cfg_readl(hw, NFP_NET_CFG_STS); 962 963 memset(&link, 0, sizeof(struct rte_eth_link)); 964 965 if (nn_link_status & NFP_NET_CFG_STS_LINK) 966 link.link_status = ETH_LINK_UP; 967 968 link.link_duplex = ETH_LINK_FULL_DUPLEX; 969 970 nn_link_status = (nn_link_status >> NFP_NET_CFG_STS_LINK_RATE_SHIFT) & 971 NFP_NET_CFG_STS_LINK_RATE_MASK; 972 973 if (nn_link_status >= RTE_DIM(ls_to_ethtool)) 974 link.link_speed = ETH_SPEED_NUM_NONE; 975 else 976 link.link_speed = ls_to_ethtool[nn_link_status]; 977 978 ret = rte_eth_linkstatus_set(dev, &link); 979 if (ret == 0) { 980 if (link.link_status) 981 PMD_DRV_LOG(INFO, "NIC Link is Up"); 982 else 983 PMD_DRV_LOG(INFO, "NIC Link is Down"); 984 } 985 return ret; 986 } 987 988 static int 989 nfp_net_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 990 { 991 int i; 992 struct nfp_net_hw *hw; 993 struct rte_eth_stats nfp_dev_stats; 994 995 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 996 997 /* RTE_ETHDEV_QUEUE_STAT_CNTRS default value is 16 */ 998 999 memset(&nfp_dev_stats, 0, sizeof(nfp_dev_stats)); 1000 1001 /* reading per RX ring stats */ 1002 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1003 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS) 1004 break; 1005 1006 nfp_dev_stats.q_ipackets[i] = 1007 nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i)); 1008 1009 nfp_dev_stats.q_ipackets[i] -= 1010 hw->eth_stats_base.q_ipackets[i]; 1011 1012 nfp_dev_stats.q_ibytes[i] = 1013 nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i) + 0x8); 1014 1015 nfp_dev_stats.q_ibytes[i] -= 1016 hw->eth_stats_base.q_ibytes[i]; 1017 } 1018 1019 /* reading per TX ring stats */ 1020 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1021 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS) 1022 break; 1023 1024 nfp_dev_stats.q_opackets[i] = 1025 nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i)); 1026 1027 nfp_dev_stats.q_opackets[i] -= 1028 hw->eth_stats_base.q_opackets[i]; 1029 1030 nfp_dev_stats.q_obytes[i] = 1031 nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i) + 0x8); 1032 1033 nfp_dev_stats.q_obytes[i] -= 1034 hw->eth_stats_base.q_obytes[i]; 1035 } 1036 1037 nfp_dev_stats.ipackets = 1038 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_FRAMES); 1039 1040 nfp_dev_stats.ipackets -= hw->eth_stats_base.ipackets; 1041 1042 nfp_dev_stats.ibytes = 1043 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_OCTETS); 1044 1045 nfp_dev_stats.ibytes -= hw->eth_stats_base.ibytes; 1046 1047 nfp_dev_stats.opackets = 1048 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_FRAMES); 1049 1050 nfp_dev_stats.opackets -= hw->eth_stats_base.opackets; 1051 1052 nfp_dev_stats.obytes = 1053 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_OCTETS); 1054 1055 nfp_dev_stats.obytes -= hw->eth_stats_base.obytes; 1056 1057 /* reading general device stats */ 1058 nfp_dev_stats.ierrors = 1059 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_ERRORS); 1060 1061 nfp_dev_stats.ierrors -= hw->eth_stats_base.ierrors; 1062 1063 nfp_dev_stats.oerrors = 1064 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_ERRORS); 1065 1066 nfp_dev_stats.oerrors -= hw->eth_stats_base.oerrors; 1067 1068 /* RX ring mbuf allocation failures */ 1069 nfp_dev_stats.rx_nombuf = dev->data->rx_mbuf_alloc_failed; 1070 1071 nfp_dev_stats.imissed = 1072 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_DISCARDS); 1073 1074 nfp_dev_stats.imissed -= hw->eth_stats_base.imissed; 1075 1076 if (stats) { 1077 memcpy(stats, &nfp_dev_stats, sizeof(*stats)); 1078 return 0; 1079 } 1080 return -EINVAL; 1081 } 1082 1083 static void 1084 nfp_net_stats_reset(struct rte_eth_dev *dev) 1085 { 1086 int i; 1087 struct nfp_net_hw *hw; 1088 1089 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1090 1091 /* 1092 * hw->eth_stats_base records the per counter starting point. 1093 * Lets update it now 1094 */ 1095 1096 /* reading per RX ring stats */ 1097 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1098 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS) 1099 break; 1100 1101 hw->eth_stats_base.q_ipackets[i] = 1102 nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i)); 1103 1104 hw->eth_stats_base.q_ibytes[i] = 1105 nn_cfg_readq(hw, NFP_NET_CFG_RXR_STATS(i) + 0x8); 1106 } 1107 1108 /* reading per TX ring stats */ 1109 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1110 if (i == RTE_ETHDEV_QUEUE_STAT_CNTRS) 1111 break; 1112 1113 hw->eth_stats_base.q_opackets[i] = 1114 nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i)); 1115 1116 hw->eth_stats_base.q_obytes[i] = 1117 nn_cfg_readq(hw, NFP_NET_CFG_TXR_STATS(i) + 0x8); 1118 } 1119 1120 hw->eth_stats_base.ipackets = 1121 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_FRAMES); 1122 1123 hw->eth_stats_base.ibytes = 1124 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_OCTETS); 1125 1126 hw->eth_stats_base.opackets = 1127 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_FRAMES); 1128 1129 hw->eth_stats_base.obytes = 1130 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_OCTETS); 1131 1132 /* reading general device stats */ 1133 hw->eth_stats_base.ierrors = 1134 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_ERRORS); 1135 1136 hw->eth_stats_base.oerrors = 1137 nn_cfg_readq(hw, NFP_NET_CFG_STATS_TX_ERRORS); 1138 1139 /* RX ring mbuf allocation failures */ 1140 dev->data->rx_mbuf_alloc_failed = 0; 1141 1142 hw->eth_stats_base.imissed = 1143 nn_cfg_readq(hw, NFP_NET_CFG_STATS_RX_DISCARDS); 1144 } 1145 1146 static void 1147 nfp_net_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 1148 { 1149 struct nfp_net_hw *hw; 1150 1151 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1152 1153 dev_info->max_rx_queues = (uint16_t)hw->max_rx_queues; 1154 dev_info->max_tx_queues = (uint16_t)hw->max_tx_queues; 1155 dev_info->min_rx_bufsize = ETHER_MIN_MTU; 1156 dev_info->max_rx_pktlen = hw->max_mtu; 1157 /* Next should change when PF support is implemented */ 1158 dev_info->max_mac_addrs = 1; 1159 1160 if (hw->cap & NFP_NET_CFG_CTRL_RXVLAN) 1161 dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP; 1162 1163 if (hw->cap & NFP_NET_CFG_CTRL_RXCSUM) 1164 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_IPV4_CKSUM | 1165 DEV_RX_OFFLOAD_UDP_CKSUM | 1166 DEV_RX_OFFLOAD_TCP_CKSUM; 1167 1168 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_JUMBO_FRAME; 1169 1170 if (hw->cap & NFP_NET_CFG_CTRL_TXVLAN) 1171 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT; 1172 1173 if (hw->cap & NFP_NET_CFG_CTRL_TXCSUM) 1174 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_IPV4_CKSUM | 1175 DEV_TX_OFFLOAD_UDP_CKSUM | 1176 DEV_TX_OFFLOAD_TCP_CKSUM; 1177 1178 if (hw->cap & NFP_NET_CFG_CTRL_LSO_ANY) 1179 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; 1180 1181 if (hw->cap & NFP_NET_CFG_CTRL_GATHER) 1182 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_MULTI_SEGS; 1183 1184 dev_info->default_rxconf = (struct rte_eth_rxconf) { 1185 .rx_thresh = { 1186 .pthresh = DEFAULT_RX_PTHRESH, 1187 .hthresh = DEFAULT_RX_HTHRESH, 1188 .wthresh = DEFAULT_RX_WTHRESH, 1189 }, 1190 .rx_free_thresh = DEFAULT_RX_FREE_THRESH, 1191 .rx_drop_en = 0, 1192 }; 1193 1194 dev_info->default_txconf = (struct rte_eth_txconf) { 1195 .tx_thresh = { 1196 .pthresh = DEFAULT_TX_PTHRESH, 1197 .hthresh = DEFAULT_TX_HTHRESH, 1198 .wthresh = DEFAULT_TX_WTHRESH, 1199 }, 1200 .tx_free_thresh = DEFAULT_TX_FREE_THRESH, 1201 .tx_rs_thresh = DEFAULT_TX_RSBIT_THRESH, 1202 }; 1203 1204 dev_info->flow_type_rss_offloads = ETH_RSS_IPV4 | 1205 ETH_RSS_NONFRAG_IPV4_TCP | 1206 ETH_RSS_NONFRAG_IPV4_UDP | 1207 ETH_RSS_IPV6 | 1208 ETH_RSS_NONFRAG_IPV6_TCP | 1209 ETH_RSS_NONFRAG_IPV6_UDP; 1210 1211 dev_info->reta_size = NFP_NET_CFG_RSS_ITBL_SZ; 1212 dev_info->hash_key_size = NFP_NET_CFG_RSS_KEY_SZ; 1213 1214 dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G | 1215 ETH_LINK_SPEED_25G | ETH_LINK_SPEED_40G | 1216 ETH_LINK_SPEED_50G | ETH_LINK_SPEED_100G; 1217 } 1218 1219 static const uint32_t * 1220 nfp_net_supported_ptypes_get(struct rte_eth_dev *dev) 1221 { 1222 static const uint32_t ptypes[] = { 1223 /* refers to nfp_net_set_hash() */ 1224 RTE_PTYPE_INNER_L3_IPV4, 1225 RTE_PTYPE_INNER_L3_IPV6, 1226 RTE_PTYPE_INNER_L3_IPV6_EXT, 1227 RTE_PTYPE_INNER_L4_MASK, 1228 RTE_PTYPE_UNKNOWN 1229 }; 1230 1231 if (dev->rx_pkt_burst == nfp_net_recv_pkts) 1232 return ptypes; 1233 return NULL; 1234 } 1235 1236 static uint32_t 1237 nfp_net_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx) 1238 { 1239 struct nfp_net_rxq *rxq; 1240 struct nfp_net_rx_desc *rxds; 1241 uint32_t idx; 1242 uint32_t count; 1243 1244 rxq = (struct nfp_net_rxq *)dev->data->rx_queues[queue_idx]; 1245 1246 idx = rxq->rd_p; 1247 1248 count = 0; 1249 1250 /* 1251 * Other PMDs are just checking the DD bit in intervals of 4 1252 * descriptors and counting all four if the first has the DD 1253 * bit on. Of course, this is not accurate but can be good for 1254 * performance. But ideally that should be done in descriptors 1255 * chunks belonging to the same cache line 1256 */ 1257 1258 while (count < rxq->rx_count) { 1259 rxds = &rxq->rxds[idx]; 1260 if ((rxds->rxd.meta_len_dd & PCIE_DESC_RX_DD) == 0) 1261 break; 1262 1263 count++; 1264 idx++; 1265 1266 /* Wrapping? */ 1267 if ((idx) == rxq->rx_count) 1268 idx = 0; 1269 } 1270 1271 return count; 1272 } 1273 1274 static int 1275 nfp_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 1276 { 1277 struct rte_pci_device *pci_dev; 1278 struct nfp_net_hw *hw; 1279 int base = 0; 1280 1281 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1282 pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1283 1284 if (pci_dev->intr_handle.type != RTE_INTR_HANDLE_UIO) 1285 base = 1; 1286 1287 /* Make sure all updates are written before un-masking */ 1288 rte_wmb(); 1289 nn_cfg_writeb(hw, NFP_NET_CFG_ICR(base + queue_id), 1290 NFP_NET_CFG_ICR_UNMASKED); 1291 return 0; 1292 } 1293 1294 static int 1295 nfp_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 1296 { 1297 struct rte_pci_device *pci_dev; 1298 struct nfp_net_hw *hw; 1299 int base = 0; 1300 1301 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1302 pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1303 1304 if (pci_dev->intr_handle.type != RTE_INTR_HANDLE_UIO) 1305 base = 1; 1306 1307 /* Make sure all updates are written before un-masking */ 1308 rte_wmb(); 1309 nn_cfg_writeb(hw, NFP_NET_CFG_ICR(base + queue_id), 0x1); 1310 return 0; 1311 } 1312 1313 static void 1314 nfp_net_dev_link_status_print(struct rte_eth_dev *dev) 1315 { 1316 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1317 struct rte_eth_link link; 1318 1319 rte_eth_linkstatus_get(dev, &link); 1320 if (link.link_status) 1321 PMD_DRV_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s", 1322 dev->data->port_id, link.link_speed, 1323 link.link_duplex == ETH_LINK_FULL_DUPLEX 1324 ? "full-duplex" : "half-duplex"); 1325 else 1326 PMD_DRV_LOG(INFO, " Port %d: Link Down", 1327 dev->data->port_id); 1328 1329 PMD_DRV_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d", 1330 pci_dev->addr.domain, pci_dev->addr.bus, 1331 pci_dev->addr.devid, pci_dev->addr.function); 1332 } 1333 1334 /* Interrupt configuration and handling */ 1335 1336 /* 1337 * nfp_net_irq_unmask - Unmask an interrupt 1338 * 1339 * If MSI-X auto-masking is enabled clear the mask bit, otherwise 1340 * clear the ICR for the entry. 1341 */ 1342 static void 1343 nfp_net_irq_unmask(struct rte_eth_dev *dev) 1344 { 1345 struct nfp_net_hw *hw; 1346 struct rte_pci_device *pci_dev; 1347 1348 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1349 pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1350 1351 if (hw->ctrl & NFP_NET_CFG_CTRL_MSIXAUTO) { 1352 /* If MSI-X auto-masking is used, clear the entry */ 1353 rte_wmb(); 1354 rte_intr_enable(&pci_dev->intr_handle); 1355 } else { 1356 /* Make sure all updates are written before un-masking */ 1357 rte_wmb(); 1358 nn_cfg_writeb(hw, NFP_NET_CFG_ICR(NFP_NET_IRQ_LSC_IDX), 1359 NFP_NET_CFG_ICR_UNMASKED); 1360 } 1361 } 1362 1363 static void 1364 nfp_net_dev_interrupt_handler(void *param) 1365 { 1366 int64_t timeout; 1367 struct rte_eth_link link; 1368 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 1369 1370 PMD_DRV_LOG(DEBUG, "We got a LSC interrupt!!!"); 1371 1372 rte_eth_linkstatus_get(dev, &link); 1373 1374 nfp_net_link_update(dev, 0); 1375 1376 /* likely to up */ 1377 if (!link.link_status) { 1378 /* handle it 1 sec later, wait it being stable */ 1379 timeout = NFP_NET_LINK_UP_CHECK_TIMEOUT; 1380 /* likely to down */ 1381 } else { 1382 /* handle it 4 sec later, wait it being stable */ 1383 timeout = NFP_NET_LINK_DOWN_CHECK_TIMEOUT; 1384 } 1385 1386 if (rte_eal_alarm_set(timeout * 1000, 1387 nfp_net_dev_interrupt_delayed_handler, 1388 (void *)dev) < 0) { 1389 PMD_INIT_LOG(ERR, "Error setting alarm"); 1390 /* Unmasking */ 1391 nfp_net_irq_unmask(dev); 1392 } 1393 } 1394 1395 /* 1396 * Interrupt handler which shall be registered for alarm callback for delayed 1397 * handling specific interrupt to wait for the stable nic state. As the NIC 1398 * interrupt state is not stable for nfp after link is just down, it needs 1399 * to wait 4 seconds to get the stable status. 1400 * 1401 * @param handle Pointer to interrupt handle. 1402 * @param param The address of parameter (struct rte_eth_dev *) 1403 * 1404 * @return void 1405 */ 1406 static void 1407 nfp_net_dev_interrupt_delayed_handler(void *param) 1408 { 1409 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 1410 1411 nfp_net_link_update(dev, 0); 1412 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); 1413 1414 nfp_net_dev_link_status_print(dev); 1415 1416 /* Unmasking */ 1417 nfp_net_irq_unmask(dev); 1418 } 1419 1420 static int 1421 nfp_net_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 1422 { 1423 struct nfp_net_hw *hw; 1424 1425 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1426 1427 /* check that mtu is within the allowed range */ 1428 if ((mtu < ETHER_MIN_MTU) || ((uint32_t)mtu > hw->max_mtu)) 1429 return -EINVAL; 1430 1431 /* mtu setting is forbidden if port is started */ 1432 if (dev->data->dev_started) { 1433 PMD_DRV_LOG(ERR, "port %d must be stopped before configuration", 1434 dev->data->port_id); 1435 return -EBUSY; 1436 } 1437 1438 /* switch to jumbo mode if needed */ 1439 if ((uint32_t)mtu > ETHER_MAX_LEN) 1440 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; 1441 else 1442 dev->data->dev_conf.rxmode.offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME; 1443 1444 /* update max frame size */ 1445 dev->data->dev_conf.rxmode.max_rx_pkt_len = (uint32_t)mtu; 1446 1447 /* writing to configuration space */ 1448 nn_cfg_writel(hw, NFP_NET_CFG_MTU, (uint32_t)mtu); 1449 1450 hw->mtu = mtu; 1451 1452 return 0; 1453 } 1454 1455 static int 1456 nfp_net_rx_queue_setup(struct rte_eth_dev *dev, 1457 uint16_t queue_idx, uint16_t nb_desc, 1458 unsigned int socket_id, 1459 const struct rte_eth_rxconf *rx_conf, 1460 struct rte_mempool *mp) 1461 { 1462 const struct rte_memzone *tz; 1463 struct nfp_net_rxq *rxq; 1464 struct nfp_net_hw *hw; 1465 1466 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1467 1468 PMD_INIT_FUNC_TRACE(); 1469 1470 /* Validating number of descriptors */ 1471 if (((nb_desc * sizeof(struct nfp_net_rx_desc)) % 128) != 0 || 1472 (nb_desc > NFP_NET_MAX_RX_DESC) || 1473 (nb_desc < NFP_NET_MIN_RX_DESC)) { 1474 PMD_DRV_LOG(ERR, "Wrong nb_desc value"); 1475 return -EINVAL; 1476 } 1477 1478 /* 1479 * Free memory prior to re-allocation if needed. This is the case after 1480 * calling nfp_net_stop 1481 */ 1482 if (dev->data->rx_queues[queue_idx]) { 1483 nfp_net_rx_queue_release(dev->data->rx_queues[queue_idx]); 1484 dev->data->rx_queues[queue_idx] = NULL; 1485 } 1486 1487 /* Allocating rx queue data structure */ 1488 rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct nfp_net_rxq), 1489 RTE_CACHE_LINE_SIZE, socket_id); 1490 if (rxq == NULL) 1491 return -ENOMEM; 1492 1493 /* Hw queues mapping based on firmware configuration */ 1494 rxq->qidx = queue_idx; 1495 rxq->fl_qcidx = queue_idx * hw->stride_rx; 1496 rxq->rx_qcidx = rxq->fl_qcidx + (hw->stride_rx - 1); 1497 rxq->qcp_fl = hw->rx_bar + NFP_QCP_QUEUE_OFF(rxq->fl_qcidx); 1498 rxq->qcp_rx = hw->rx_bar + NFP_QCP_QUEUE_OFF(rxq->rx_qcidx); 1499 1500 /* 1501 * Tracking mbuf size for detecting a potential mbuf overflow due to 1502 * RX offset 1503 */ 1504 rxq->mem_pool = mp; 1505 rxq->mbuf_size = rxq->mem_pool->elt_size; 1506 rxq->mbuf_size -= (sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM); 1507 hw->flbufsz = rxq->mbuf_size; 1508 1509 rxq->rx_count = nb_desc; 1510 rxq->port_id = dev->data->port_id; 1511 rxq->rx_free_thresh = rx_conf->rx_free_thresh; 1512 rxq->drop_en = rx_conf->rx_drop_en; 1513 1514 /* 1515 * Allocate RX ring hardware descriptors. A memzone large enough to 1516 * handle the maximum ring size is allocated in order to allow for 1517 * resizing in later calls to the queue setup function. 1518 */ 1519 tz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, 1520 sizeof(struct nfp_net_rx_desc) * 1521 NFP_NET_MAX_RX_DESC, NFP_MEMZONE_ALIGN, 1522 socket_id); 1523 1524 if (tz == NULL) { 1525 PMD_DRV_LOG(ERR, "Error allocating rx dma"); 1526 nfp_net_rx_queue_release(rxq); 1527 return -ENOMEM; 1528 } 1529 1530 /* Saving physical and virtual addresses for the RX ring */ 1531 rxq->dma = (uint64_t)tz->iova; 1532 rxq->rxds = (struct nfp_net_rx_desc *)tz->addr; 1533 1534 /* mbuf pointers array for referencing mbufs linked to RX descriptors */ 1535 rxq->rxbufs = rte_zmalloc_socket("rxq->rxbufs", 1536 sizeof(*rxq->rxbufs) * nb_desc, 1537 RTE_CACHE_LINE_SIZE, socket_id); 1538 if (rxq->rxbufs == NULL) { 1539 nfp_net_rx_queue_release(rxq); 1540 return -ENOMEM; 1541 } 1542 1543 PMD_RX_LOG(DEBUG, "rxbufs=%p hw_ring=%p dma_addr=0x%" PRIx64, 1544 rxq->rxbufs, rxq->rxds, (unsigned long int)rxq->dma); 1545 1546 nfp_net_reset_rx_queue(rxq); 1547 1548 dev->data->rx_queues[queue_idx] = rxq; 1549 rxq->hw = hw; 1550 1551 /* 1552 * Telling the HW about the physical address of the RX ring and number 1553 * of descriptors in log2 format 1554 */ 1555 nn_cfg_writeq(hw, NFP_NET_CFG_RXR_ADDR(queue_idx), rxq->dma); 1556 nn_cfg_writeb(hw, NFP_NET_CFG_RXR_SZ(queue_idx), rte_log2_u32(nb_desc)); 1557 1558 return 0; 1559 } 1560 1561 static int 1562 nfp_net_rx_fill_freelist(struct nfp_net_rxq *rxq) 1563 { 1564 struct nfp_net_rx_buff *rxe = rxq->rxbufs; 1565 uint64_t dma_addr; 1566 unsigned i; 1567 1568 PMD_RX_LOG(DEBUG, "nfp_net_rx_fill_freelist for %u descriptors", 1569 rxq->rx_count); 1570 1571 for (i = 0; i < rxq->rx_count; i++) { 1572 struct nfp_net_rx_desc *rxd; 1573 struct rte_mbuf *mbuf = rte_pktmbuf_alloc(rxq->mem_pool); 1574 1575 if (mbuf == NULL) { 1576 PMD_DRV_LOG(ERR, "RX mbuf alloc failed queue_id=%u", 1577 (unsigned)rxq->qidx); 1578 return -ENOMEM; 1579 } 1580 1581 dma_addr = rte_cpu_to_le_64(RTE_MBUF_DMA_ADDR_DEFAULT(mbuf)); 1582 1583 rxd = &rxq->rxds[i]; 1584 rxd->fld.dd = 0; 1585 rxd->fld.dma_addr_hi = (dma_addr >> 32) & 0xff; 1586 rxd->fld.dma_addr_lo = dma_addr & 0xffffffff; 1587 rxe[i].mbuf = mbuf; 1588 PMD_RX_LOG(DEBUG, "[%d]: %" PRIx64, i, dma_addr); 1589 } 1590 1591 /* Make sure all writes are flushed before telling the hardware */ 1592 rte_wmb(); 1593 1594 /* Not advertising the whole ring as the firmware gets confused if so */ 1595 PMD_RX_LOG(DEBUG, "Increment FL write pointer in %u", 1596 rxq->rx_count - 1); 1597 1598 nfp_qcp_ptr_add(rxq->qcp_fl, NFP_QCP_WRITE_PTR, rxq->rx_count - 1); 1599 1600 return 0; 1601 } 1602 1603 static int 1604 nfp_net_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 1605 uint16_t nb_desc, unsigned int socket_id, 1606 const struct rte_eth_txconf *tx_conf) 1607 { 1608 const struct rte_memzone *tz; 1609 struct nfp_net_txq *txq; 1610 uint16_t tx_free_thresh; 1611 struct nfp_net_hw *hw; 1612 1613 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1614 1615 PMD_INIT_FUNC_TRACE(); 1616 1617 /* Validating number of descriptors */ 1618 if (((nb_desc * sizeof(struct nfp_net_tx_desc)) % 128) != 0 || 1619 (nb_desc > NFP_NET_MAX_TX_DESC) || 1620 (nb_desc < NFP_NET_MIN_TX_DESC)) { 1621 PMD_DRV_LOG(ERR, "Wrong nb_desc value"); 1622 return -EINVAL; 1623 } 1624 1625 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ? 1626 tx_conf->tx_free_thresh : 1627 DEFAULT_TX_FREE_THRESH); 1628 1629 if (tx_free_thresh > (nb_desc)) { 1630 PMD_DRV_LOG(ERR, 1631 "tx_free_thresh must be less than the number of TX " 1632 "descriptors. (tx_free_thresh=%u port=%d " 1633 "queue=%d)", (unsigned int)tx_free_thresh, 1634 dev->data->port_id, (int)queue_idx); 1635 return -(EINVAL); 1636 } 1637 1638 /* 1639 * Free memory prior to re-allocation if needed. This is the case after 1640 * calling nfp_net_stop 1641 */ 1642 if (dev->data->tx_queues[queue_idx]) { 1643 PMD_TX_LOG(DEBUG, "Freeing memory prior to re-allocation %d", 1644 queue_idx); 1645 nfp_net_tx_queue_release(dev->data->tx_queues[queue_idx]); 1646 dev->data->tx_queues[queue_idx] = NULL; 1647 } 1648 1649 /* Allocating tx queue data structure */ 1650 txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct nfp_net_txq), 1651 RTE_CACHE_LINE_SIZE, socket_id); 1652 if (txq == NULL) { 1653 PMD_DRV_LOG(ERR, "Error allocating tx dma"); 1654 return -ENOMEM; 1655 } 1656 1657 /* 1658 * Allocate TX ring hardware descriptors. A memzone large enough to 1659 * handle the maximum ring size is allocated in order to allow for 1660 * resizing in later calls to the queue setup function. 1661 */ 1662 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx, 1663 sizeof(struct nfp_net_tx_desc) * 1664 NFP_NET_MAX_TX_DESC, NFP_MEMZONE_ALIGN, 1665 socket_id); 1666 if (tz == NULL) { 1667 PMD_DRV_LOG(ERR, "Error allocating tx dma"); 1668 nfp_net_tx_queue_release(txq); 1669 return -ENOMEM; 1670 } 1671 1672 txq->tx_count = nb_desc; 1673 txq->tx_free_thresh = tx_free_thresh; 1674 txq->tx_pthresh = tx_conf->tx_thresh.pthresh; 1675 txq->tx_hthresh = tx_conf->tx_thresh.hthresh; 1676 txq->tx_wthresh = tx_conf->tx_thresh.wthresh; 1677 1678 /* queue mapping based on firmware configuration */ 1679 txq->qidx = queue_idx; 1680 txq->tx_qcidx = queue_idx * hw->stride_tx; 1681 txq->qcp_q = hw->tx_bar + NFP_QCP_QUEUE_OFF(txq->tx_qcidx); 1682 1683 txq->port_id = dev->data->port_id; 1684 1685 /* Saving physical and virtual addresses for the TX ring */ 1686 txq->dma = (uint64_t)tz->iova; 1687 txq->txds = (struct nfp_net_tx_desc *)tz->addr; 1688 1689 /* mbuf pointers array for referencing mbufs linked to TX descriptors */ 1690 txq->txbufs = rte_zmalloc_socket("txq->txbufs", 1691 sizeof(*txq->txbufs) * nb_desc, 1692 RTE_CACHE_LINE_SIZE, socket_id); 1693 if (txq->txbufs == NULL) { 1694 nfp_net_tx_queue_release(txq); 1695 return -ENOMEM; 1696 } 1697 PMD_TX_LOG(DEBUG, "txbufs=%p hw_ring=%p dma_addr=0x%" PRIx64, 1698 txq->txbufs, txq->txds, (unsigned long int)txq->dma); 1699 1700 nfp_net_reset_tx_queue(txq); 1701 1702 dev->data->tx_queues[queue_idx] = txq; 1703 txq->hw = hw; 1704 1705 /* 1706 * Telling the HW about the physical address of the TX ring and number 1707 * of descriptors in log2 format 1708 */ 1709 nn_cfg_writeq(hw, NFP_NET_CFG_TXR_ADDR(queue_idx), txq->dma); 1710 nn_cfg_writeb(hw, NFP_NET_CFG_TXR_SZ(queue_idx), rte_log2_u32(nb_desc)); 1711 1712 return 0; 1713 } 1714 1715 /* nfp_net_tx_tso - Set TX descriptor for TSO */ 1716 static inline void 1717 nfp_net_tx_tso(struct nfp_net_txq *txq, struct nfp_net_tx_desc *txd, 1718 struct rte_mbuf *mb) 1719 { 1720 uint64_t ol_flags; 1721 struct nfp_net_hw *hw = txq->hw; 1722 1723 if (!(hw->cap & NFP_NET_CFG_CTRL_LSO_ANY)) 1724 goto clean_txd; 1725 1726 ol_flags = mb->ol_flags; 1727 1728 if (!(ol_flags & PKT_TX_TCP_SEG)) 1729 goto clean_txd; 1730 1731 txd->l3_offset = mb->l2_len; 1732 txd->l4_offset = mb->l2_len + mb->l3_len; 1733 txd->lso_hdrlen = mb->l2_len + mb->l3_len + mb->l4_len; 1734 txd->mss = rte_cpu_to_le_16(mb->tso_segsz); 1735 txd->flags = PCIE_DESC_TX_LSO; 1736 return; 1737 1738 clean_txd: 1739 txd->flags = 0; 1740 txd->l3_offset = 0; 1741 txd->l4_offset = 0; 1742 txd->lso_hdrlen = 0; 1743 txd->mss = 0; 1744 } 1745 1746 /* nfp_net_tx_cksum - Set TX CSUM offload flags in TX descriptor */ 1747 static inline void 1748 nfp_net_tx_cksum(struct nfp_net_txq *txq, struct nfp_net_tx_desc *txd, 1749 struct rte_mbuf *mb) 1750 { 1751 uint64_t ol_flags; 1752 struct nfp_net_hw *hw = txq->hw; 1753 1754 if (!(hw->cap & NFP_NET_CFG_CTRL_TXCSUM)) 1755 return; 1756 1757 ol_flags = mb->ol_flags; 1758 1759 /* IPv6 does not need checksum */ 1760 if (ol_flags & PKT_TX_IP_CKSUM) 1761 txd->flags |= PCIE_DESC_TX_IP4_CSUM; 1762 1763 switch (ol_flags & PKT_TX_L4_MASK) { 1764 case PKT_TX_UDP_CKSUM: 1765 txd->flags |= PCIE_DESC_TX_UDP_CSUM; 1766 break; 1767 case PKT_TX_TCP_CKSUM: 1768 txd->flags |= PCIE_DESC_TX_TCP_CSUM; 1769 break; 1770 } 1771 1772 if (ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_L4_MASK)) 1773 txd->flags |= PCIE_DESC_TX_CSUM; 1774 } 1775 1776 /* nfp_net_rx_cksum - set mbuf checksum flags based on RX descriptor flags */ 1777 static inline void 1778 nfp_net_rx_cksum(struct nfp_net_rxq *rxq, struct nfp_net_rx_desc *rxd, 1779 struct rte_mbuf *mb) 1780 { 1781 struct nfp_net_hw *hw = rxq->hw; 1782 1783 if (!(hw->ctrl & NFP_NET_CFG_CTRL_RXCSUM)) 1784 return; 1785 1786 /* If IPv4 and IP checksum error, fail */ 1787 if (unlikely((rxd->rxd.flags & PCIE_DESC_RX_IP4_CSUM) && 1788 !(rxd->rxd.flags & PCIE_DESC_RX_IP4_CSUM_OK))) 1789 mb->ol_flags |= PKT_RX_IP_CKSUM_BAD; 1790 else 1791 mb->ol_flags |= PKT_RX_IP_CKSUM_GOOD; 1792 1793 /* If neither UDP nor TCP return */ 1794 if (!(rxd->rxd.flags & PCIE_DESC_RX_TCP_CSUM) && 1795 !(rxd->rxd.flags & PCIE_DESC_RX_UDP_CSUM)) 1796 return; 1797 1798 if (likely(rxd->rxd.flags & PCIE_DESC_RX_L4_CSUM_OK)) 1799 mb->ol_flags |= PKT_RX_L4_CKSUM_GOOD; 1800 else 1801 mb->ol_flags |= PKT_RX_L4_CKSUM_BAD; 1802 } 1803 1804 #define NFP_HASH_OFFSET ((uint8_t *)mbuf->buf_addr + mbuf->data_off - 4) 1805 #define NFP_HASH_TYPE_OFFSET ((uint8_t *)mbuf->buf_addr + mbuf->data_off - 8) 1806 1807 #define NFP_DESC_META_LEN(d) (d->rxd.meta_len_dd & PCIE_DESC_RX_META_LEN_MASK) 1808 1809 /* 1810 * nfp_net_set_hash - Set mbuf hash data 1811 * 1812 * The RSS hash and hash-type are pre-pended to the packet data. 1813 * Extract and decode it and set the mbuf fields. 1814 */ 1815 static inline void 1816 nfp_net_set_hash(struct nfp_net_rxq *rxq, struct nfp_net_rx_desc *rxd, 1817 struct rte_mbuf *mbuf) 1818 { 1819 struct nfp_net_hw *hw = rxq->hw; 1820 uint8_t *meta_offset; 1821 uint32_t meta_info; 1822 uint32_t hash = 0; 1823 uint32_t hash_type = 0; 1824 1825 if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS)) 1826 return; 1827 1828 /* this is true for new firmwares */ 1829 if (likely(((hw->cap & NFP_NET_CFG_CTRL_RSS2) || 1830 (NFD_CFG_MAJOR_VERSION_of(hw->ver) == 4)) && 1831 NFP_DESC_META_LEN(rxd))) { 1832 /* 1833 * new metadata api: 1834 * <---- 32 bit -----> 1835 * m field type word 1836 * e data field #2 1837 * t data field #1 1838 * a data field #0 1839 * ==================== 1840 * packet data 1841 * 1842 * Field type word contains up to 8 4bit field types 1843 * A 4bit field type refers to a data field word 1844 * A data field word can have several 4bit field types 1845 */ 1846 meta_offset = rte_pktmbuf_mtod(mbuf, uint8_t *); 1847 meta_offset -= NFP_DESC_META_LEN(rxd); 1848 meta_info = rte_be_to_cpu_32(*(uint32_t *)meta_offset); 1849 meta_offset += 4; 1850 /* NFP PMD just supports metadata for hashing */ 1851 switch (meta_info & NFP_NET_META_FIELD_MASK) { 1852 case NFP_NET_META_HASH: 1853 /* next field type is about the hash type */ 1854 meta_info >>= NFP_NET_META_FIELD_SIZE; 1855 /* hash value is in the data field */ 1856 hash = rte_be_to_cpu_32(*(uint32_t *)meta_offset); 1857 hash_type = meta_info & NFP_NET_META_FIELD_MASK; 1858 break; 1859 default: 1860 /* Unsupported metadata can be a performance issue */ 1861 return; 1862 } 1863 } else { 1864 if (!(rxd->rxd.flags & PCIE_DESC_RX_RSS)) 1865 return; 1866 1867 hash = rte_be_to_cpu_32(*(uint32_t *)NFP_HASH_OFFSET); 1868 hash_type = rte_be_to_cpu_32(*(uint32_t *)NFP_HASH_TYPE_OFFSET); 1869 } 1870 1871 mbuf->hash.rss = hash; 1872 mbuf->ol_flags |= PKT_RX_RSS_HASH; 1873 1874 switch (hash_type) { 1875 case NFP_NET_RSS_IPV4: 1876 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV4; 1877 break; 1878 case NFP_NET_RSS_IPV6: 1879 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6; 1880 break; 1881 case NFP_NET_RSS_IPV6_EX: 1882 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT; 1883 break; 1884 case NFP_NET_RSS_IPV4_TCP: 1885 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT; 1886 break; 1887 case NFP_NET_RSS_IPV6_TCP: 1888 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT; 1889 break; 1890 case NFP_NET_RSS_IPV4_UDP: 1891 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT; 1892 break; 1893 case NFP_NET_RSS_IPV6_UDP: 1894 mbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT; 1895 break; 1896 default: 1897 mbuf->packet_type |= RTE_PTYPE_INNER_L4_MASK; 1898 } 1899 } 1900 1901 static inline void 1902 nfp_net_mbuf_alloc_failed(struct nfp_net_rxq *rxq) 1903 { 1904 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++; 1905 } 1906 1907 #define NFP_DESC_META_LEN(d) (d->rxd.meta_len_dd & PCIE_DESC_RX_META_LEN_MASK) 1908 1909 /* 1910 * RX path design: 1911 * 1912 * There are some decisions to take: 1913 * 1) How to check DD RX descriptors bit 1914 * 2) How and when to allocate new mbufs 1915 * 1916 * Current implementation checks just one single DD bit each loop. As each 1917 * descriptor is 8 bytes, it is likely a good idea to check descriptors in 1918 * a single cache line instead. Tests with this change have not shown any 1919 * performance improvement but it requires further investigation. For example, 1920 * depending on which descriptor is next, the number of descriptors could be 1921 * less than 8 for just checking those in the same cache line. This implies 1922 * extra work which could be counterproductive by itself. Indeed, last firmware 1923 * changes are just doing this: writing several descriptors with the DD bit 1924 * for saving PCIe bandwidth and DMA operations from the NFP. 1925 * 1926 * Mbuf allocation is done when a new packet is received. Then the descriptor 1927 * is automatically linked with the new mbuf and the old one is given to the 1928 * user. The main drawback with this design is mbuf allocation is heavier than 1929 * using bulk allocations allowed by DPDK with rte_mempool_get_bulk. From the 1930 * cache point of view it does not seem allocating the mbuf early on as we are 1931 * doing now have any benefit at all. Again, tests with this change have not 1932 * shown any improvement. Also, rte_mempool_get_bulk returns all or nothing 1933 * so looking at the implications of this type of allocation should be studied 1934 * deeply 1935 */ 1936 1937 static uint16_t 1938 nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 1939 { 1940 struct nfp_net_rxq *rxq; 1941 struct nfp_net_rx_desc *rxds; 1942 struct nfp_net_rx_buff *rxb; 1943 struct nfp_net_hw *hw; 1944 struct rte_mbuf *mb; 1945 struct rte_mbuf *new_mb; 1946 uint16_t nb_hold; 1947 uint64_t dma_addr; 1948 int avail; 1949 1950 rxq = rx_queue; 1951 if (unlikely(rxq == NULL)) { 1952 /* 1953 * DPDK just checks the queue is lower than max queues 1954 * enabled. But the queue needs to be configured 1955 */ 1956 RTE_LOG_DP(ERR, PMD, "RX Bad queue\n"); 1957 return -EINVAL; 1958 } 1959 1960 hw = rxq->hw; 1961 avail = 0; 1962 nb_hold = 0; 1963 1964 while (avail < nb_pkts) { 1965 rxb = &rxq->rxbufs[rxq->rd_p]; 1966 if (unlikely(rxb == NULL)) { 1967 RTE_LOG_DP(ERR, PMD, "rxb does not exist!\n"); 1968 break; 1969 } 1970 1971 rxds = &rxq->rxds[rxq->rd_p]; 1972 if ((rxds->rxd.meta_len_dd & PCIE_DESC_RX_DD) == 0) 1973 break; 1974 1975 /* 1976 * Memory barrier to ensure that we won't do other 1977 * reads before the DD bit. 1978 */ 1979 rte_rmb(); 1980 1981 /* 1982 * We got a packet. Let's alloc a new mbuf for refilling the 1983 * free descriptor ring as soon as possible 1984 */ 1985 new_mb = rte_pktmbuf_alloc(rxq->mem_pool); 1986 if (unlikely(new_mb == NULL)) { 1987 RTE_LOG_DP(DEBUG, PMD, 1988 "RX mbuf alloc failed port_id=%u queue_id=%u\n", 1989 rxq->port_id, (unsigned int)rxq->qidx); 1990 nfp_net_mbuf_alloc_failed(rxq); 1991 break; 1992 } 1993 1994 nb_hold++; 1995 1996 /* 1997 * Grab the mbuf and refill the descriptor with the 1998 * previously allocated mbuf 1999 */ 2000 mb = rxb->mbuf; 2001 rxb->mbuf = new_mb; 2002 2003 PMD_RX_LOG(DEBUG, "Packet len: %u, mbuf_size: %u", 2004 rxds->rxd.data_len, rxq->mbuf_size); 2005 2006 /* Size of this segment */ 2007 mb->data_len = rxds->rxd.data_len - NFP_DESC_META_LEN(rxds); 2008 /* Size of the whole packet. We just support 1 segment */ 2009 mb->pkt_len = rxds->rxd.data_len - NFP_DESC_META_LEN(rxds); 2010 2011 if (unlikely((mb->data_len + hw->rx_offset) > 2012 rxq->mbuf_size)) { 2013 /* 2014 * This should not happen and the user has the 2015 * responsibility of avoiding it. But we have 2016 * to give some info about the error 2017 */ 2018 RTE_LOG_DP(ERR, PMD, 2019 "mbuf overflow likely due to the RX offset.\n" 2020 "\t\tYour mbuf size should have extra space for" 2021 " RX offset=%u bytes.\n" 2022 "\t\tCurrently you just have %u bytes available" 2023 " but the received packet is %u bytes long", 2024 hw->rx_offset, 2025 rxq->mbuf_size - hw->rx_offset, 2026 mb->data_len); 2027 return -EINVAL; 2028 } 2029 2030 /* Filling the received mbuf with packet info */ 2031 if (hw->rx_offset) 2032 mb->data_off = RTE_PKTMBUF_HEADROOM + hw->rx_offset; 2033 else 2034 mb->data_off = RTE_PKTMBUF_HEADROOM + 2035 NFP_DESC_META_LEN(rxds); 2036 2037 /* No scatter mode supported */ 2038 mb->nb_segs = 1; 2039 mb->next = NULL; 2040 2041 mb->port = rxq->port_id; 2042 2043 /* Checking the RSS flag */ 2044 nfp_net_set_hash(rxq, rxds, mb); 2045 2046 /* Checking the checksum flag */ 2047 nfp_net_rx_cksum(rxq, rxds, mb); 2048 2049 if ((rxds->rxd.flags & PCIE_DESC_RX_VLAN) && 2050 (hw->ctrl & NFP_NET_CFG_CTRL_RXVLAN)) { 2051 mb->vlan_tci = rte_cpu_to_le_32(rxds->rxd.vlan); 2052 mb->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 2053 } 2054 2055 /* Adding the mbuf to the mbuf array passed by the app */ 2056 rx_pkts[avail++] = mb; 2057 2058 /* Now resetting and updating the descriptor */ 2059 rxds->vals[0] = 0; 2060 rxds->vals[1] = 0; 2061 dma_addr = rte_cpu_to_le_64(RTE_MBUF_DMA_ADDR_DEFAULT(new_mb)); 2062 rxds->fld.dd = 0; 2063 rxds->fld.dma_addr_hi = (dma_addr >> 32) & 0xff; 2064 rxds->fld.dma_addr_lo = dma_addr & 0xffffffff; 2065 2066 rxq->rd_p++; 2067 if (unlikely(rxq->rd_p == rxq->rx_count)) /* wrapping?*/ 2068 rxq->rd_p = 0; 2069 } 2070 2071 if (nb_hold == 0) 2072 return nb_hold; 2073 2074 PMD_RX_LOG(DEBUG, "RX port_id=%u queue_id=%u, %d packets received", 2075 rxq->port_id, (unsigned int)rxq->qidx, nb_hold); 2076 2077 nb_hold += rxq->nb_rx_hold; 2078 2079 /* 2080 * FL descriptors needs to be written before incrementing the 2081 * FL queue WR pointer 2082 */ 2083 rte_wmb(); 2084 if (nb_hold > rxq->rx_free_thresh) { 2085 PMD_RX_LOG(DEBUG, "port=%u queue=%u nb_hold=%u avail=%u", 2086 rxq->port_id, (unsigned int)rxq->qidx, 2087 (unsigned)nb_hold, (unsigned)avail); 2088 nfp_qcp_ptr_add(rxq->qcp_fl, NFP_QCP_WRITE_PTR, nb_hold); 2089 nb_hold = 0; 2090 } 2091 rxq->nb_rx_hold = nb_hold; 2092 2093 return avail; 2094 } 2095 2096 /* 2097 * nfp_net_tx_free_bufs - Check for descriptors with a complete 2098 * status 2099 * @txq: TX queue to work with 2100 * Returns number of descriptors freed 2101 */ 2102 int 2103 nfp_net_tx_free_bufs(struct nfp_net_txq *txq) 2104 { 2105 uint32_t qcp_rd_p; 2106 int todo; 2107 2108 PMD_TX_LOG(DEBUG, "queue %u. Check for descriptor with a complete" 2109 " status", txq->qidx); 2110 2111 /* Work out how many packets have been sent */ 2112 qcp_rd_p = nfp_qcp_read(txq->qcp_q, NFP_QCP_READ_PTR); 2113 2114 if (qcp_rd_p == txq->rd_p) { 2115 PMD_TX_LOG(DEBUG, "queue %u: It seems harrier is not sending " 2116 "packets (%u, %u)", txq->qidx, 2117 qcp_rd_p, txq->rd_p); 2118 return 0; 2119 } 2120 2121 if (qcp_rd_p > txq->rd_p) 2122 todo = qcp_rd_p - txq->rd_p; 2123 else 2124 todo = qcp_rd_p + txq->tx_count - txq->rd_p; 2125 2126 PMD_TX_LOG(DEBUG, "qcp_rd_p %u, txq->rd_p: %u, qcp->rd_p: %u", 2127 qcp_rd_p, txq->rd_p, txq->rd_p); 2128 2129 if (todo == 0) 2130 return todo; 2131 2132 txq->rd_p += todo; 2133 if (unlikely(txq->rd_p >= txq->tx_count)) 2134 txq->rd_p -= txq->tx_count; 2135 2136 return todo; 2137 } 2138 2139 /* Leaving always free descriptors for avoiding wrapping confusion */ 2140 static inline 2141 uint32_t nfp_free_tx_desc(struct nfp_net_txq *txq) 2142 { 2143 if (txq->wr_p >= txq->rd_p) 2144 return txq->tx_count - (txq->wr_p - txq->rd_p) - 8; 2145 else 2146 return txq->rd_p - txq->wr_p - 8; 2147 } 2148 2149 /* 2150 * nfp_net_txq_full - Check if the TX queue free descriptors 2151 * is below tx_free_threshold 2152 * 2153 * @txq: TX queue to check 2154 * 2155 * This function uses the host copy* of read/write pointers 2156 */ 2157 static inline 2158 uint32_t nfp_net_txq_full(struct nfp_net_txq *txq) 2159 { 2160 return (nfp_free_tx_desc(txq) < txq->tx_free_thresh); 2161 } 2162 2163 static uint16_t 2164 nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 2165 { 2166 struct nfp_net_txq *txq; 2167 struct nfp_net_hw *hw; 2168 struct nfp_net_tx_desc *txds, txd; 2169 struct rte_mbuf *pkt; 2170 uint64_t dma_addr; 2171 int pkt_size, dma_size; 2172 uint16_t free_descs, issued_descs; 2173 struct rte_mbuf **lmbuf; 2174 int i; 2175 2176 txq = tx_queue; 2177 hw = txq->hw; 2178 txds = &txq->txds[txq->wr_p]; 2179 2180 PMD_TX_LOG(DEBUG, "working for queue %u at pos %d and %u packets", 2181 txq->qidx, txq->wr_p, nb_pkts); 2182 2183 if ((nfp_free_tx_desc(txq) < nb_pkts) || (nfp_net_txq_full(txq))) 2184 nfp_net_tx_free_bufs(txq); 2185 2186 free_descs = (uint16_t)nfp_free_tx_desc(txq); 2187 if (unlikely(free_descs == 0)) 2188 return 0; 2189 2190 pkt = *tx_pkts; 2191 2192 i = 0; 2193 issued_descs = 0; 2194 PMD_TX_LOG(DEBUG, "queue: %u. Sending %u packets", 2195 txq->qidx, nb_pkts); 2196 /* Sending packets */ 2197 while ((i < nb_pkts) && free_descs) { 2198 /* Grabbing the mbuf linked to the current descriptor */ 2199 lmbuf = &txq->txbufs[txq->wr_p].mbuf; 2200 /* Warming the cache for releasing the mbuf later on */ 2201 RTE_MBUF_PREFETCH_TO_FREE(*lmbuf); 2202 2203 pkt = *(tx_pkts + i); 2204 2205 if (unlikely((pkt->nb_segs > 1) && 2206 !(hw->cap & NFP_NET_CFG_CTRL_GATHER))) { 2207 PMD_INIT_LOG(INFO, "NFP_NET_CFG_CTRL_GATHER not set"); 2208 rte_panic("Multisegment packet unsupported\n"); 2209 } 2210 2211 /* Checking if we have enough descriptors */ 2212 if (unlikely(pkt->nb_segs > free_descs)) 2213 goto xmit_end; 2214 2215 /* 2216 * Checksum and VLAN flags just in the first descriptor for a 2217 * multisegment packet, but TSO info needs to be in all of them. 2218 */ 2219 txd.data_len = pkt->pkt_len; 2220 nfp_net_tx_tso(txq, &txd, pkt); 2221 nfp_net_tx_cksum(txq, &txd, pkt); 2222 2223 if ((pkt->ol_flags & PKT_TX_VLAN_PKT) && 2224 (hw->cap & NFP_NET_CFG_CTRL_TXVLAN)) { 2225 txd.flags |= PCIE_DESC_TX_VLAN; 2226 txd.vlan = pkt->vlan_tci; 2227 } 2228 2229 /* 2230 * mbuf data_len is the data in one segment and pkt_len data 2231 * in the whole packet. When the packet is just one segment, 2232 * then data_len = pkt_len 2233 */ 2234 pkt_size = pkt->pkt_len; 2235 2236 while (pkt) { 2237 /* Copying TSO, VLAN and cksum info */ 2238 *txds = txd; 2239 2240 /* Releasing mbuf used by this descriptor previously*/ 2241 if (*lmbuf) 2242 rte_pktmbuf_free_seg(*lmbuf); 2243 2244 /* 2245 * Linking mbuf with descriptor for being released 2246 * next time descriptor is used 2247 */ 2248 *lmbuf = pkt; 2249 2250 dma_size = pkt->data_len; 2251 dma_addr = rte_mbuf_data_iova(pkt); 2252 PMD_TX_LOG(DEBUG, "Working with mbuf at dma address:" 2253 "%" PRIx64 "", dma_addr); 2254 2255 /* Filling descriptors fields */ 2256 txds->dma_len = dma_size; 2257 txds->data_len = txd.data_len; 2258 txds->dma_addr_hi = (dma_addr >> 32) & 0xff; 2259 txds->dma_addr_lo = (dma_addr & 0xffffffff); 2260 ASSERT(free_descs > 0); 2261 free_descs--; 2262 2263 txq->wr_p++; 2264 if (unlikely(txq->wr_p == txq->tx_count)) /* wrapping?*/ 2265 txq->wr_p = 0; 2266 2267 pkt_size -= dma_size; 2268 2269 /* 2270 * Making the EOP, packets with just one segment 2271 * the priority 2272 */ 2273 if (likely(!pkt_size)) 2274 txds->offset_eop = PCIE_DESC_TX_EOP; 2275 else 2276 txds->offset_eop = 0; 2277 2278 pkt = pkt->next; 2279 /* Referencing next free TX descriptor */ 2280 txds = &txq->txds[txq->wr_p]; 2281 lmbuf = &txq->txbufs[txq->wr_p].mbuf; 2282 issued_descs++; 2283 } 2284 i++; 2285 } 2286 2287 xmit_end: 2288 /* Increment write pointers. Force memory write before we let HW know */ 2289 rte_wmb(); 2290 nfp_qcp_ptr_add(txq->qcp_q, NFP_QCP_WRITE_PTR, issued_descs); 2291 2292 return i; 2293 } 2294 2295 static int 2296 nfp_net_vlan_offload_set(struct rte_eth_dev *dev, int mask) 2297 { 2298 uint32_t new_ctrl, update; 2299 struct nfp_net_hw *hw; 2300 int ret; 2301 2302 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2303 new_ctrl = 0; 2304 2305 if ((mask & ETH_VLAN_FILTER_OFFLOAD) || 2306 (mask & ETH_VLAN_EXTEND_OFFLOAD)) 2307 PMD_DRV_LOG(INFO, "No support for ETH_VLAN_FILTER_OFFLOAD or" 2308 " ETH_VLAN_EXTEND_OFFLOAD"); 2309 2310 /* Enable vlan strip if it is not configured yet */ 2311 if ((mask & ETH_VLAN_STRIP_OFFLOAD) && 2312 !(hw->ctrl & NFP_NET_CFG_CTRL_RXVLAN)) 2313 new_ctrl = hw->ctrl | NFP_NET_CFG_CTRL_RXVLAN; 2314 2315 /* Disable vlan strip just if it is configured */ 2316 if (!(mask & ETH_VLAN_STRIP_OFFLOAD) && 2317 (hw->ctrl & NFP_NET_CFG_CTRL_RXVLAN)) 2318 new_ctrl = hw->ctrl & ~NFP_NET_CFG_CTRL_RXVLAN; 2319 2320 if (new_ctrl == 0) 2321 return 0; 2322 2323 update = NFP_NET_CFG_UPDATE_GEN; 2324 2325 ret = nfp_net_reconfig(hw, new_ctrl, update); 2326 if (!ret) 2327 hw->ctrl = new_ctrl; 2328 2329 return ret; 2330 } 2331 2332 static int 2333 nfp_net_rss_reta_write(struct rte_eth_dev *dev, 2334 struct rte_eth_rss_reta_entry64 *reta_conf, 2335 uint16_t reta_size) 2336 { 2337 uint32_t reta, mask; 2338 int i, j; 2339 int idx, shift; 2340 struct nfp_net_hw *hw = 2341 NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2342 2343 if (reta_size != NFP_NET_CFG_RSS_ITBL_SZ) { 2344 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 2345 "(%d) doesn't match the number hardware can supported " 2346 "(%d)", reta_size, NFP_NET_CFG_RSS_ITBL_SZ); 2347 return -EINVAL; 2348 } 2349 2350 /* 2351 * Update Redirection Table. There are 128 8bit-entries which can be 2352 * manage as 32 32bit-entries 2353 */ 2354 for (i = 0; i < reta_size; i += 4) { 2355 /* Handling 4 RSS entries per loop */ 2356 idx = i / RTE_RETA_GROUP_SIZE; 2357 shift = i % RTE_RETA_GROUP_SIZE; 2358 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 0xF); 2359 2360 if (!mask) 2361 continue; 2362 2363 reta = 0; 2364 /* If all 4 entries were set, don't need read RETA register */ 2365 if (mask != 0xF) 2366 reta = nn_cfg_readl(hw, NFP_NET_CFG_RSS_ITBL + i); 2367 2368 for (j = 0; j < 4; j++) { 2369 if (!(mask & (0x1 << j))) 2370 continue; 2371 if (mask != 0xF) 2372 /* Clearing the entry bits */ 2373 reta &= ~(0xFF << (8 * j)); 2374 reta |= reta_conf[idx].reta[shift + j] << (8 * j); 2375 } 2376 nn_cfg_writel(hw, NFP_NET_CFG_RSS_ITBL + (idx * 64) + shift, 2377 reta); 2378 } 2379 return 0; 2380 } 2381 2382 /* Update Redirection Table(RETA) of Receive Side Scaling of Ethernet device */ 2383 static int 2384 nfp_net_reta_update(struct rte_eth_dev *dev, 2385 struct rte_eth_rss_reta_entry64 *reta_conf, 2386 uint16_t reta_size) 2387 { 2388 struct nfp_net_hw *hw = 2389 NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2390 uint32_t update; 2391 int ret; 2392 2393 if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS)) 2394 return -EINVAL; 2395 2396 ret = nfp_net_rss_reta_write(dev, reta_conf, reta_size); 2397 if (ret != 0) 2398 return ret; 2399 2400 update = NFP_NET_CFG_UPDATE_RSS; 2401 2402 if (nfp_net_reconfig(hw, hw->ctrl, update) < 0) 2403 return -EIO; 2404 2405 return 0; 2406 } 2407 2408 /* Query Redirection Table(RETA) of Receive Side Scaling of Ethernet device. */ 2409 static int 2410 nfp_net_reta_query(struct rte_eth_dev *dev, 2411 struct rte_eth_rss_reta_entry64 *reta_conf, 2412 uint16_t reta_size) 2413 { 2414 uint8_t i, j, mask; 2415 int idx, shift; 2416 uint32_t reta; 2417 struct nfp_net_hw *hw; 2418 2419 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2420 2421 if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS)) 2422 return -EINVAL; 2423 2424 if (reta_size != NFP_NET_CFG_RSS_ITBL_SZ) { 2425 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 2426 "(%d) doesn't match the number hardware can supported " 2427 "(%d)", reta_size, NFP_NET_CFG_RSS_ITBL_SZ); 2428 return -EINVAL; 2429 } 2430 2431 /* 2432 * Reading Redirection Table. There are 128 8bit-entries which can be 2433 * manage as 32 32bit-entries 2434 */ 2435 for (i = 0; i < reta_size; i += 4) { 2436 /* Handling 4 RSS entries per loop */ 2437 idx = i / RTE_RETA_GROUP_SIZE; 2438 shift = i % RTE_RETA_GROUP_SIZE; 2439 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 0xF); 2440 2441 if (!mask) 2442 continue; 2443 2444 reta = nn_cfg_readl(hw, NFP_NET_CFG_RSS_ITBL + (idx * 64) + 2445 shift); 2446 for (j = 0; j < 4; j++) { 2447 if (!(mask & (0x1 << j))) 2448 continue; 2449 reta_conf[idx].reta[shift + j] = 2450 (uint8_t)((reta >> (8 * j)) & 0xF); 2451 } 2452 } 2453 return 0; 2454 } 2455 2456 static int 2457 nfp_net_rss_hash_write(struct rte_eth_dev *dev, 2458 struct rte_eth_rss_conf *rss_conf) 2459 { 2460 struct nfp_net_hw *hw; 2461 uint64_t rss_hf; 2462 uint32_t cfg_rss_ctrl = 0; 2463 uint8_t key; 2464 int i; 2465 2466 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2467 2468 /* Writing the key byte a byte */ 2469 for (i = 0; i < rss_conf->rss_key_len; i++) { 2470 memcpy(&key, &rss_conf->rss_key[i], 1); 2471 nn_cfg_writeb(hw, NFP_NET_CFG_RSS_KEY + i, key); 2472 } 2473 2474 rss_hf = rss_conf->rss_hf; 2475 2476 if (rss_hf & ETH_RSS_IPV4) 2477 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4; 2478 2479 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) 2480 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4_TCP; 2481 2482 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) 2483 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV4_UDP; 2484 2485 if (rss_hf & ETH_RSS_IPV6) 2486 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6; 2487 2488 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) 2489 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6_TCP; 2490 2491 if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) 2492 cfg_rss_ctrl |= NFP_NET_CFG_RSS_IPV6_UDP; 2493 2494 cfg_rss_ctrl |= NFP_NET_CFG_RSS_MASK; 2495 cfg_rss_ctrl |= NFP_NET_CFG_RSS_TOEPLITZ; 2496 2497 /* configuring where to apply the RSS hash */ 2498 nn_cfg_writel(hw, NFP_NET_CFG_RSS_CTRL, cfg_rss_ctrl); 2499 2500 /* Writing the key size */ 2501 nn_cfg_writeb(hw, NFP_NET_CFG_RSS_KEY_SZ, rss_conf->rss_key_len); 2502 2503 return 0; 2504 } 2505 2506 static int 2507 nfp_net_rss_hash_update(struct rte_eth_dev *dev, 2508 struct rte_eth_rss_conf *rss_conf) 2509 { 2510 uint32_t update; 2511 uint64_t rss_hf; 2512 struct nfp_net_hw *hw; 2513 2514 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2515 2516 rss_hf = rss_conf->rss_hf; 2517 2518 /* Checking if RSS is enabled */ 2519 if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS)) { 2520 if (rss_hf != 0) { /* Enable RSS? */ 2521 PMD_DRV_LOG(ERR, "RSS unsupported"); 2522 return -EINVAL; 2523 } 2524 return 0; /* Nothing to do */ 2525 } 2526 2527 if (rss_conf->rss_key_len > NFP_NET_CFG_RSS_KEY_SZ) { 2528 PMD_DRV_LOG(ERR, "hash key too long"); 2529 return -EINVAL; 2530 } 2531 2532 nfp_net_rss_hash_write(dev, rss_conf); 2533 2534 update = NFP_NET_CFG_UPDATE_RSS; 2535 2536 if (nfp_net_reconfig(hw, hw->ctrl, update) < 0) 2537 return -EIO; 2538 2539 return 0; 2540 } 2541 2542 static int 2543 nfp_net_rss_hash_conf_get(struct rte_eth_dev *dev, 2544 struct rte_eth_rss_conf *rss_conf) 2545 { 2546 uint64_t rss_hf; 2547 uint32_t cfg_rss_ctrl; 2548 uint8_t key; 2549 int i; 2550 struct nfp_net_hw *hw; 2551 2552 hw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2553 2554 if (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS)) 2555 return -EINVAL; 2556 2557 rss_hf = rss_conf->rss_hf; 2558 cfg_rss_ctrl = nn_cfg_readl(hw, NFP_NET_CFG_RSS_CTRL); 2559 2560 if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4) 2561 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_NONFRAG_IPV4_UDP; 2562 2563 if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4_TCP) 2564 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP; 2565 2566 if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV6_TCP) 2567 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP; 2568 2569 if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV4_UDP) 2570 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP; 2571 2572 if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV6_UDP) 2573 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP; 2574 2575 if (cfg_rss_ctrl & NFP_NET_CFG_RSS_IPV6) 2576 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_NONFRAG_IPV6_UDP; 2577 2578 /* Reading the key size */ 2579 rss_conf->rss_key_len = nn_cfg_readl(hw, NFP_NET_CFG_RSS_KEY_SZ); 2580 2581 /* Reading the key byte a byte */ 2582 for (i = 0; i < rss_conf->rss_key_len; i++) { 2583 key = nn_cfg_readb(hw, NFP_NET_CFG_RSS_KEY + i); 2584 memcpy(&rss_conf->rss_key[i], &key, 1); 2585 } 2586 2587 return 0; 2588 } 2589 2590 static int 2591 nfp_net_rss_config_default(struct rte_eth_dev *dev) 2592 { 2593 struct rte_eth_conf *dev_conf; 2594 struct rte_eth_rss_conf rss_conf; 2595 struct rte_eth_rss_reta_entry64 nfp_reta_conf[2]; 2596 uint16_t rx_queues = dev->data->nb_rx_queues; 2597 uint16_t queue; 2598 int i, j, ret; 2599 2600 PMD_DRV_LOG(INFO, "setting default RSS conf for %u queues", 2601 rx_queues); 2602 2603 nfp_reta_conf[0].mask = ~0x0; 2604 nfp_reta_conf[1].mask = ~0x0; 2605 2606 queue = 0; 2607 for (i = 0; i < 0x40; i += 8) { 2608 for (j = i; j < (i + 8); j++) { 2609 nfp_reta_conf[0].reta[j] = queue; 2610 nfp_reta_conf[1].reta[j] = queue++; 2611 queue %= rx_queues; 2612 } 2613 } 2614 ret = nfp_net_rss_reta_write(dev, nfp_reta_conf, 0x80); 2615 if (ret != 0) 2616 return ret; 2617 2618 dev_conf = &dev->data->dev_conf; 2619 if (!dev_conf) { 2620 PMD_DRV_LOG(INFO, "wrong rss conf"); 2621 return -EINVAL; 2622 } 2623 rss_conf = dev_conf->rx_adv_conf.rss_conf; 2624 2625 ret = nfp_net_rss_hash_write(dev, &rss_conf); 2626 2627 return ret; 2628 } 2629 2630 2631 /* Initialise and register driver with DPDK Application */ 2632 static const struct eth_dev_ops nfp_net_eth_dev_ops = { 2633 .dev_configure = nfp_net_configure, 2634 .dev_start = nfp_net_start, 2635 .dev_stop = nfp_net_stop, 2636 .dev_close = nfp_net_close, 2637 .promiscuous_enable = nfp_net_promisc_enable, 2638 .promiscuous_disable = nfp_net_promisc_disable, 2639 .link_update = nfp_net_link_update, 2640 .stats_get = nfp_net_stats_get, 2641 .stats_reset = nfp_net_stats_reset, 2642 .dev_infos_get = nfp_net_infos_get, 2643 .dev_supported_ptypes_get = nfp_net_supported_ptypes_get, 2644 .mtu_set = nfp_net_dev_mtu_set, 2645 .mac_addr_set = nfp_set_mac_addr, 2646 .vlan_offload_set = nfp_net_vlan_offload_set, 2647 .reta_update = nfp_net_reta_update, 2648 .reta_query = nfp_net_reta_query, 2649 .rss_hash_update = nfp_net_rss_hash_update, 2650 .rss_hash_conf_get = nfp_net_rss_hash_conf_get, 2651 .rx_queue_setup = nfp_net_rx_queue_setup, 2652 .rx_queue_release = nfp_net_rx_queue_release, 2653 .rx_queue_count = nfp_net_rx_queue_count, 2654 .tx_queue_setup = nfp_net_tx_queue_setup, 2655 .tx_queue_release = nfp_net_tx_queue_release, 2656 .rx_queue_intr_enable = nfp_rx_queue_intr_enable, 2657 .rx_queue_intr_disable = nfp_rx_queue_intr_disable, 2658 }; 2659 2660 /* 2661 * All eth_dev created got its private data, but before nfp_net_init, that 2662 * private data is referencing private data for all the PF ports. This is due 2663 * to how the vNIC bars are mapped based on first port, so all ports need info 2664 * about port 0 private data. Inside nfp_net_init the private data pointer is 2665 * changed to the right address for each port once the bars have been mapped. 2666 * 2667 * This functions helps to find out which port and therefore which offset 2668 * inside the private data array to use. 2669 */ 2670 static int 2671 get_pf_port_number(char *name) 2672 { 2673 char *pf_str = name; 2674 int size = 0; 2675 2676 while ((*pf_str != '_') && (*pf_str != '\0') && (size++ < 30)) 2677 pf_str++; 2678 2679 if (size == 30) 2680 /* 2681 * This should not happen at all and it would mean major 2682 * implementation fault. 2683 */ 2684 rte_panic("nfp_net: problem with pf device name\n"); 2685 2686 /* Expecting _portX with X within [0,7] */ 2687 pf_str += 5; 2688 2689 return (int)strtol(pf_str, NULL, 10); 2690 } 2691 2692 static int 2693 nfp_net_init(struct rte_eth_dev *eth_dev) 2694 { 2695 struct rte_pci_device *pci_dev; 2696 struct nfp_net_hw *hw, *hwport0; 2697 2698 uint64_t tx_bar_off = 0, rx_bar_off = 0; 2699 uint32_t start_q; 2700 int stride = 4; 2701 int port = 0; 2702 int err; 2703 2704 PMD_INIT_FUNC_TRACE(); 2705 2706 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 2707 2708 /* NFP can not handle DMA addresses requiring more than 40 bits */ 2709 if (rte_mem_check_dma_mask(40)) { 2710 RTE_LOG(ERR, PMD, "device %s can not be used:", 2711 pci_dev->device.name); 2712 RTE_LOG(ERR, PMD, "\trestricted dma mask to 40 bits!\n"); 2713 return -ENODEV; 2714 }; 2715 2716 if ((pci_dev->id.device_id == PCI_DEVICE_ID_NFP4000_PF_NIC) || 2717 (pci_dev->id.device_id == PCI_DEVICE_ID_NFP6000_PF_NIC)) { 2718 port = get_pf_port_number(eth_dev->data->name); 2719 if (port < 0 || port > 7) { 2720 PMD_DRV_LOG(ERR, "Port value is wrong"); 2721 return -ENODEV; 2722 } 2723 2724 PMD_INIT_LOG(DEBUG, "Working with PF port value %d", port); 2725 2726 /* This points to port 0 private data */ 2727 hwport0 = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 2728 2729 /* This points to the specific port private data */ 2730 hw = &hwport0[port]; 2731 } else { 2732 hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 2733 hwport0 = 0; 2734 } 2735 2736 eth_dev->dev_ops = &nfp_net_eth_dev_ops; 2737 eth_dev->rx_pkt_burst = &nfp_net_recv_pkts; 2738 eth_dev->tx_pkt_burst = &nfp_net_xmit_pkts; 2739 2740 /* For secondary processes, the primary has done all the work */ 2741 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2742 return 0; 2743 2744 rte_eth_copy_pci_info(eth_dev, pci_dev); 2745 2746 hw->device_id = pci_dev->id.device_id; 2747 hw->vendor_id = pci_dev->id.vendor_id; 2748 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 2749 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 2750 2751 PMD_INIT_LOG(DEBUG, "nfp_net: device (%u:%u) %u:%u:%u:%u", 2752 pci_dev->id.vendor_id, pci_dev->id.device_id, 2753 pci_dev->addr.domain, pci_dev->addr.bus, 2754 pci_dev->addr.devid, pci_dev->addr.function); 2755 2756 hw->ctrl_bar = (uint8_t *)pci_dev->mem_resource[0].addr; 2757 if (hw->ctrl_bar == NULL) { 2758 PMD_DRV_LOG(ERR, 2759 "hw->ctrl_bar is NULL. BAR0 not configured"); 2760 return -ENODEV; 2761 } 2762 2763 if (hw->is_pf && port == 0) { 2764 hw->ctrl_bar = nfp_rtsym_map(hw->sym_tbl, "_pf0_net_bar0", 2765 hw->total_ports * 32768, 2766 &hw->ctrl_area); 2767 if (!hw->ctrl_bar) { 2768 printf("nfp_rtsym_map fails for _pf0_net_ctrl_bar"); 2769 return -EIO; 2770 } 2771 2772 PMD_INIT_LOG(DEBUG, "ctrl bar: %p", hw->ctrl_bar); 2773 } 2774 2775 if (port > 0) { 2776 if (!hwport0->ctrl_bar) 2777 return -ENODEV; 2778 2779 /* address based on port0 offset */ 2780 hw->ctrl_bar = hwport0->ctrl_bar + 2781 (port * NFP_PF_CSR_SLICE_SIZE); 2782 } 2783 2784 PMD_INIT_LOG(DEBUG, "ctrl bar: %p", hw->ctrl_bar); 2785 2786 hw->max_rx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_RXRINGS); 2787 hw->max_tx_queues = nn_cfg_readl(hw, NFP_NET_CFG_MAX_TXRINGS); 2788 2789 /* Work out where in the BAR the queues start. */ 2790 switch (pci_dev->id.device_id) { 2791 case PCI_DEVICE_ID_NFP4000_PF_NIC: 2792 case PCI_DEVICE_ID_NFP6000_PF_NIC: 2793 case PCI_DEVICE_ID_NFP6000_VF_NIC: 2794 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_TXQ); 2795 tx_bar_off = (uint64_t)start_q * NFP_QCP_QUEUE_ADDR_SZ; 2796 start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_RXQ); 2797 rx_bar_off = (uint64_t)start_q * NFP_QCP_QUEUE_ADDR_SZ; 2798 break; 2799 default: 2800 PMD_DRV_LOG(ERR, "nfp_net: no device ID matching"); 2801 err = -ENODEV; 2802 goto dev_err_ctrl_map; 2803 } 2804 2805 PMD_INIT_LOG(DEBUG, "tx_bar_off: 0x%" PRIx64 "", tx_bar_off); 2806 PMD_INIT_LOG(DEBUG, "rx_bar_off: 0x%" PRIx64 "", rx_bar_off); 2807 2808 if (hw->is_pf && port == 0) { 2809 /* configure access to tx/rx vNIC BARs */ 2810 hwport0->hw_queues = nfp_cpp_map_area(hw->cpp, 0, 0, 2811 NFP_PCIE_QUEUE(0), 2812 NFP_QCP_QUEUE_AREA_SZ, 2813 &hw->hwqueues_area); 2814 2815 if (!hwport0->hw_queues) { 2816 printf("nfp_rtsym_map fails for net.qc"); 2817 err = -EIO; 2818 goto dev_err_ctrl_map; 2819 } 2820 2821 PMD_INIT_LOG(DEBUG, "tx/rx bar address: 0x%p", 2822 hwport0->hw_queues); 2823 } 2824 2825 if (hw->is_pf) { 2826 hw->tx_bar = hwport0->hw_queues + tx_bar_off; 2827 hw->rx_bar = hwport0->hw_queues + rx_bar_off; 2828 eth_dev->data->dev_private = hw; 2829 } else { 2830 hw->tx_bar = (uint8_t *)pci_dev->mem_resource[2].addr + 2831 tx_bar_off; 2832 hw->rx_bar = (uint8_t *)pci_dev->mem_resource[2].addr + 2833 rx_bar_off; 2834 } 2835 2836 PMD_INIT_LOG(DEBUG, "ctrl_bar: %p, tx_bar: %p, rx_bar: %p", 2837 hw->ctrl_bar, hw->tx_bar, hw->rx_bar); 2838 2839 nfp_net_cfg_queue_setup(hw); 2840 2841 /* Get some of the read-only fields from the config BAR */ 2842 hw->ver = nn_cfg_readl(hw, NFP_NET_CFG_VERSION); 2843 hw->cap = nn_cfg_readl(hw, NFP_NET_CFG_CAP); 2844 hw->max_mtu = nn_cfg_readl(hw, NFP_NET_CFG_MAX_MTU); 2845 hw->mtu = ETHER_MTU; 2846 2847 /* VLAN insertion is incompatible with LSOv2 */ 2848 if (hw->cap & NFP_NET_CFG_CTRL_LSO2) 2849 hw->cap &= ~NFP_NET_CFG_CTRL_TXVLAN; 2850 2851 if (NFD_CFG_MAJOR_VERSION_of(hw->ver) < 2) 2852 hw->rx_offset = NFP_NET_RX_OFFSET; 2853 else 2854 hw->rx_offset = nn_cfg_readl(hw, NFP_NET_CFG_RX_OFFSET_ADDR); 2855 2856 PMD_INIT_LOG(INFO, "VER: %u.%u, Maximum supported MTU: %d", 2857 NFD_CFG_MAJOR_VERSION_of(hw->ver), 2858 NFD_CFG_MINOR_VERSION_of(hw->ver), hw->max_mtu); 2859 2860 PMD_INIT_LOG(INFO, "CAP: %#x, %s%s%s%s%s%s%s%s%s%s%s%s%s%s", hw->cap, 2861 hw->cap & NFP_NET_CFG_CTRL_PROMISC ? "PROMISC " : "", 2862 hw->cap & NFP_NET_CFG_CTRL_L2BC ? "L2BCFILT " : "", 2863 hw->cap & NFP_NET_CFG_CTRL_L2MC ? "L2MCFILT " : "", 2864 hw->cap & NFP_NET_CFG_CTRL_RXCSUM ? "RXCSUM " : "", 2865 hw->cap & NFP_NET_CFG_CTRL_TXCSUM ? "TXCSUM " : "", 2866 hw->cap & NFP_NET_CFG_CTRL_RXVLAN ? "RXVLAN " : "", 2867 hw->cap & NFP_NET_CFG_CTRL_TXVLAN ? "TXVLAN " : "", 2868 hw->cap & NFP_NET_CFG_CTRL_SCATTER ? "SCATTER " : "", 2869 hw->cap & NFP_NET_CFG_CTRL_GATHER ? "GATHER " : "", 2870 hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR ? "LIVE_ADDR " : "", 2871 hw->cap & NFP_NET_CFG_CTRL_LSO ? "TSO " : "", 2872 hw->cap & NFP_NET_CFG_CTRL_LSO2 ? "TSOv2 " : "", 2873 hw->cap & NFP_NET_CFG_CTRL_RSS ? "RSS " : "", 2874 hw->cap & NFP_NET_CFG_CTRL_RSS2 ? "RSSv2 " : ""); 2875 2876 hw->ctrl = 0; 2877 2878 hw->stride_rx = stride; 2879 hw->stride_tx = stride; 2880 2881 PMD_INIT_LOG(INFO, "max_rx_queues: %u, max_tx_queues: %u", 2882 hw->max_rx_queues, hw->max_tx_queues); 2883 2884 /* Initializing spinlock for reconfigs */ 2885 rte_spinlock_init(&hw->reconfig_lock); 2886 2887 /* Allocating memory for mac addr */ 2888 eth_dev->data->mac_addrs = rte_zmalloc("mac_addr", ETHER_ADDR_LEN, 0); 2889 if (eth_dev->data->mac_addrs == NULL) { 2890 PMD_INIT_LOG(ERR, "Failed to space for MAC address"); 2891 err = -ENOMEM; 2892 goto dev_err_queues_map; 2893 } 2894 2895 if (hw->is_pf) { 2896 nfp_net_pf_read_mac(hwport0, port); 2897 nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr); 2898 } else { 2899 nfp_net_vf_read_mac(hw); 2900 } 2901 2902 if (!is_valid_assigned_ether_addr((struct ether_addr *)&hw->mac_addr)) { 2903 PMD_INIT_LOG(INFO, "Using random mac address for port %d", 2904 port); 2905 /* Using random mac addresses for VFs */ 2906 eth_random_addr(&hw->mac_addr[0]); 2907 nfp_net_write_mac(hw, (uint8_t *)&hw->mac_addr); 2908 } 2909 2910 /* Copying mac address to DPDK eth_dev struct */ 2911 ether_addr_copy((struct ether_addr *)hw->mac_addr, 2912 ð_dev->data->mac_addrs[0]); 2913 2914 if (!(hw->cap & NFP_NET_CFG_CTRL_LIVE_ADDR)) 2915 eth_dev->data->dev_flags |= RTE_ETH_DEV_NOLIVE_MAC_ADDR; 2916 2917 PMD_INIT_LOG(INFO, "port %d VendorID=0x%x DeviceID=0x%x " 2918 "mac=%02x:%02x:%02x:%02x:%02x:%02x", 2919 eth_dev->data->port_id, pci_dev->id.vendor_id, 2920 pci_dev->id.device_id, 2921 hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2], 2922 hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]); 2923 2924 /* Registering LSC interrupt handler */ 2925 rte_intr_callback_register(&pci_dev->intr_handle, 2926 nfp_net_dev_interrupt_handler, 2927 (void *)eth_dev); 2928 2929 /* Telling the firmware about the LSC interrupt entry */ 2930 nn_cfg_writeb(hw, NFP_NET_CFG_LSC, NFP_NET_IRQ_LSC_IDX); 2931 2932 /* Recording current stats counters values */ 2933 nfp_net_stats_reset(eth_dev); 2934 2935 return 0; 2936 2937 dev_err_queues_map: 2938 nfp_cpp_area_free(hw->hwqueues_area); 2939 dev_err_ctrl_map: 2940 nfp_cpp_area_free(hw->ctrl_area); 2941 2942 return err; 2943 } 2944 2945 static int 2946 nfp_pf_create_dev(struct rte_pci_device *dev, int port, int ports, 2947 struct nfp_cpp *cpp, struct nfp_hwinfo *hwinfo, 2948 int phys_port, struct nfp_rtsym_table *sym_tbl, void **priv) 2949 { 2950 struct rte_eth_dev *eth_dev; 2951 struct nfp_net_hw *hw; 2952 char *port_name; 2953 int ret; 2954 2955 port_name = rte_zmalloc("nfp_pf_port_name", 100, 0); 2956 if (!port_name) 2957 return -ENOMEM; 2958 2959 if (ports > 1) 2960 snprintf(port_name, 100, "%s_port%d", dev->device.name, port); 2961 else 2962 strlcat(port_name, dev->device.name, 100); 2963 2964 eth_dev = rte_eth_dev_allocate(port_name); 2965 if (!eth_dev) 2966 return -ENOMEM; 2967 2968 if (port == 0) { 2969 *priv = rte_zmalloc(port_name, 2970 sizeof(struct nfp_net_adapter) * ports, 2971 RTE_CACHE_LINE_SIZE); 2972 if (!*priv) { 2973 rte_eth_dev_release_port(eth_dev); 2974 return -ENOMEM; 2975 } 2976 } 2977 2978 eth_dev->data->dev_private = *priv; 2979 2980 /* 2981 * dev_private pointing to port0 dev_private because we need 2982 * to configure vNIC bars based on port0 at nfp_net_init. 2983 * Then dev_private is adjusted per port. 2984 */ 2985 hw = (struct nfp_net_hw *)(eth_dev->data->dev_private) + port; 2986 hw->cpp = cpp; 2987 hw->hwinfo = hwinfo; 2988 hw->sym_tbl = sym_tbl; 2989 hw->pf_port_idx = phys_port; 2990 hw->is_pf = 1; 2991 if (ports > 1) 2992 hw->pf_multiport_enabled = 1; 2993 2994 hw->total_ports = ports; 2995 2996 eth_dev->device = &dev->device; 2997 rte_eth_copy_pci_info(eth_dev, dev); 2998 2999 ret = nfp_net_init(eth_dev); 3000 3001 if (ret) 3002 rte_eth_dev_release_port(eth_dev); 3003 else 3004 rte_eth_dev_probing_finish(eth_dev); 3005 3006 rte_free(port_name); 3007 3008 return ret; 3009 } 3010 3011 #define DEFAULT_FW_PATH "/lib/firmware/netronome" 3012 3013 static int 3014 nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card) 3015 { 3016 struct nfp_cpp *cpp = nsp->cpp; 3017 int fw_f; 3018 char *fw_buf; 3019 char fw_name[125]; 3020 char serial[40]; 3021 struct stat file_stat; 3022 off_t fsize, bytes; 3023 3024 /* Looking for firmware file in order of priority */ 3025 3026 /* First try to find a firmware image specific for this device */ 3027 snprintf(serial, sizeof(serial), 3028 "serial-%02x-%02x-%02x-%02x-%02x-%02x-%02x-%02x", 3029 cpp->serial[0], cpp->serial[1], cpp->serial[2], cpp->serial[3], 3030 cpp->serial[4], cpp->serial[5], cpp->interface >> 8, 3031 cpp->interface & 0xff); 3032 3033 snprintf(fw_name, sizeof(fw_name), "%s/%s.nffw", DEFAULT_FW_PATH, 3034 serial); 3035 3036 PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name); 3037 fw_f = open(fw_name, O_RDONLY); 3038 if (fw_f >= 0) 3039 goto read_fw; 3040 3041 /* Then try the PCI name */ 3042 snprintf(fw_name, sizeof(fw_name), "%s/pci-%s.nffw", DEFAULT_FW_PATH, 3043 dev->device.name); 3044 3045 PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name); 3046 fw_f = open(fw_name, O_RDONLY); 3047 if (fw_f >= 0) 3048 goto read_fw; 3049 3050 /* Finally try the card type and media */ 3051 snprintf(fw_name, sizeof(fw_name), "%s/%s", DEFAULT_FW_PATH, card); 3052 PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name); 3053 fw_f = open(fw_name, O_RDONLY); 3054 if (fw_f < 0) { 3055 PMD_DRV_LOG(INFO, "Firmware file %s not found.", fw_name); 3056 return -ENOENT; 3057 } 3058 3059 read_fw: 3060 if (fstat(fw_f, &file_stat) < 0) { 3061 PMD_DRV_LOG(INFO, "Firmware file %s size is unknown", fw_name); 3062 close(fw_f); 3063 return -ENOENT; 3064 } 3065 3066 fsize = file_stat.st_size; 3067 PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %" PRIu64 "", 3068 fw_name, (uint64_t)fsize); 3069 3070 fw_buf = malloc((size_t)fsize); 3071 if (!fw_buf) { 3072 PMD_DRV_LOG(INFO, "malloc failed for fw buffer"); 3073 close(fw_f); 3074 return -ENOMEM; 3075 } 3076 memset(fw_buf, 0, fsize); 3077 3078 bytes = read(fw_f, fw_buf, fsize); 3079 if (bytes != fsize) { 3080 PMD_DRV_LOG(INFO, "Reading fw to buffer failed." 3081 "Just %" PRIu64 " of %" PRIu64 " bytes read", 3082 (uint64_t)bytes, (uint64_t)fsize); 3083 free(fw_buf); 3084 close(fw_f); 3085 return -EIO; 3086 } 3087 3088 PMD_DRV_LOG(INFO, "Uploading the firmware ..."); 3089 nfp_nsp_load_fw(nsp, fw_buf, bytes); 3090 PMD_DRV_LOG(INFO, "Done"); 3091 3092 free(fw_buf); 3093 close(fw_f); 3094 3095 return 0; 3096 } 3097 3098 static int 3099 nfp_fw_setup(struct rte_pci_device *dev, struct nfp_cpp *cpp, 3100 struct nfp_eth_table *nfp_eth_table, struct nfp_hwinfo *hwinfo) 3101 { 3102 struct nfp_nsp *nsp; 3103 const char *nfp_fw_model; 3104 char card_desc[100]; 3105 int err = 0; 3106 3107 nfp_fw_model = nfp_hwinfo_lookup(hwinfo, "assembly.partno"); 3108 3109 if (nfp_fw_model) { 3110 PMD_DRV_LOG(INFO, "firmware model found: %s", nfp_fw_model); 3111 } else { 3112 PMD_DRV_LOG(ERR, "firmware model NOT found"); 3113 return -EIO; 3114 } 3115 3116 if (nfp_eth_table->count == 0 || nfp_eth_table->count > 8) { 3117 PMD_DRV_LOG(ERR, "NFP ethernet table reports wrong ports: %u", 3118 nfp_eth_table->count); 3119 return -EIO; 3120 } 3121 3122 PMD_DRV_LOG(INFO, "NFP ethernet port table reports %u ports", 3123 nfp_eth_table->count); 3124 3125 PMD_DRV_LOG(INFO, "Port speed: %u", nfp_eth_table->ports[0].speed); 3126 3127 snprintf(card_desc, sizeof(card_desc), "nic_%s_%dx%d.nffw", 3128 nfp_fw_model, nfp_eth_table->count, 3129 nfp_eth_table->ports[0].speed / 1000); 3130 3131 nsp = nfp_nsp_open(cpp); 3132 if (!nsp) { 3133 PMD_DRV_LOG(ERR, "NFP error when obtaining NSP handle"); 3134 return -EIO; 3135 } 3136 3137 nfp_nsp_device_soft_reset(nsp); 3138 err = nfp_fw_upload(dev, nsp, card_desc); 3139 3140 nfp_nsp_close(nsp); 3141 return err; 3142 } 3143 3144 static int nfp_pf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 3145 struct rte_pci_device *dev) 3146 { 3147 struct nfp_cpp *cpp; 3148 struct nfp_hwinfo *hwinfo; 3149 struct nfp_rtsym_table *sym_tbl; 3150 struct nfp_eth_table *nfp_eth_table = NULL; 3151 int total_ports; 3152 void *priv = 0; 3153 int ret = -ENODEV; 3154 int err; 3155 int i; 3156 3157 if (!dev) 3158 return ret; 3159 3160 /* 3161 * When device bound to UIO, the device could be used, by mistake, 3162 * by two DPDK apps, and the UIO driver does not avoid it. This 3163 * could lead to a serious problem when configuring the NFP CPP 3164 * interface. Here we avoid this telling to the CPP init code to 3165 * use a lock file if UIO is being used. 3166 */ 3167 if (dev->kdrv == RTE_KDRV_VFIO) 3168 cpp = nfp_cpp_from_device_name(dev, 0); 3169 else 3170 cpp = nfp_cpp_from_device_name(dev, 1); 3171 3172 if (!cpp) { 3173 PMD_DRV_LOG(ERR, "A CPP handle can not be obtained"); 3174 ret = -EIO; 3175 goto error; 3176 } 3177 3178 hwinfo = nfp_hwinfo_read(cpp); 3179 if (!hwinfo) { 3180 PMD_DRV_LOG(ERR, "Error reading hwinfo table"); 3181 return -EIO; 3182 } 3183 3184 nfp_eth_table = nfp_eth_read_ports(cpp); 3185 if (!nfp_eth_table) { 3186 PMD_DRV_LOG(ERR, "Error reading NFP ethernet table"); 3187 return -EIO; 3188 } 3189 3190 if (nfp_fw_setup(dev, cpp, nfp_eth_table, hwinfo)) { 3191 PMD_DRV_LOG(INFO, "Error when uploading firmware"); 3192 ret = -EIO; 3193 goto error; 3194 } 3195 3196 /* Now the symbol table should be there */ 3197 sym_tbl = nfp_rtsym_table_read(cpp); 3198 if (!sym_tbl) { 3199 PMD_DRV_LOG(ERR, "Something is wrong with the firmware" 3200 " symbol table"); 3201 ret = -EIO; 3202 goto error; 3203 } 3204 3205 total_ports = nfp_rtsym_read_le(sym_tbl, "nfd_cfg_pf0_num_ports", &err); 3206 if (total_ports != (int)nfp_eth_table->count) { 3207 PMD_DRV_LOG(ERR, "Inconsistent number of ports"); 3208 ret = -EIO; 3209 goto error; 3210 } 3211 PMD_INIT_LOG(INFO, "Total pf ports: %d", total_ports); 3212 3213 if (total_ports <= 0 || total_ports > 8) { 3214 PMD_DRV_LOG(ERR, "nfd_cfg_pf0_num_ports symbol with wrong value"); 3215 ret = -ENODEV; 3216 goto error; 3217 } 3218 3219 for (i = 0; i < total_ports; i++) { 3220 ret = nfp_pf_create_dev(dev, i, total_ports, cpp, hwinfo, 3221 nfp_eth_table->ports[i].index, 3222 sym_tbl, &priv); 3223 if (ret) 3224 break; 3225 } 3226 3227 error: 3228 free(nfp_eth_table); 3229 return ret; 3230 } 3231 3232 int nfp_logtype_init; 3233 int nfp_logtype_driver; 3234 3235 static const struct rte_pci_id pci_id_nfp_pf_net_map[] = { 3236 { 3237 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME, 3238 PCI_DEVICE_ID_NFP4000_PF_NIC) 3239 }, 3240 { 3241 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME, 3242 PCI_DEVICE_ID_NFP6000_PF_NIC) 3243 }, 3244 { 3245 .vendor_id = 0, 3246 }, 3247 }; 3248 3249 static const struct rte_pci_id pci_id_nfp_vf_net_map[] = { 3250 { 3251 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME, 3252 PCI_DEVICE_ID_NFP6000_VF_NIC) 3253 }, 3254 { 3255 .vendor_id = 0, 3256 }, 3257 }; 3258 3259 static int eth_nfp_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 3260 struct rte_pci_device *pci_dev) 3261 { 3262 return rte_eth_dev_pci_generic_probe(pci_dev, 3263 sizeof(struct nfp_net_adapter), nfp_net_init); 3264 } 3265 3266 static int eth_nfp_pci_remove(struct rte_pci_device *pci_dev) 3267 { 3268 struct rte_eth_dev *eth_dev; 3269 struct nfp_net_hw *hw, *hwport0; 3270 int port = 0; 3271 3272 eth_dev = rte_eth_dev_allocated(pci_dev->device.name); 3273 if ((pci_dev->id.device_id == PCI_DEVICE_ID_NFP4000_PF_NIC) || 3274 (pci_dev->id.device_id == PCI_DEVICE_ID_NFP6000_PF_NIC)) { 3275 port = get_pf_port_number(eth_dev->data->name); 3276 /* 3277 * hotplug is not possible with multiport PF although freeing 3278 * data structures can be done for first port. 3279 */ 3280 if (port != 0) 3281 return -ENOTSUP; 3282 hwport0 = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 3283 hw = &hwport0[port]; 3284 nfp_cpp_area_free(hw->ctrl_area); 3285 nfp_cpp_area_free(hw->hwqueues_area); 3286 free(hw->hwinfo); 3287 free(hw->sym_tbl); 3288 nfp_cpp_free(hw->cpp); 3289 } else { 3290 hw = NFP_NET_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 3291 } 3292 /* hotplug is not possible with multiport PF */ 3293 if (hw->pf_multiport_enabled) 3294 return -ENOTSUP; 3295 return rte_eth_dev_pci_generic_remove(pci_dev, NULL); 3296 } 3297 3298 static struct rte_pci_driver rte_nfp_net_pf_pmd = { 3299 .id_table = pci_id_nfp_pf_net_map, 3300 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | 3301 RTE_PCI_DRV_IOVA_AS_VA, 3302 .probe = nfp_pf_pci_probe, 3303 .remove = eth_nfp_pci_remove, 3304 }; 3305 3306 static struct rte_pci_driver rte_nfp_net_vf_pmd = { 3307 .id_table = pci_id_nfp_vf_net_map, 3308 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | 3309 RTE_PCI_DRV_IOVA_AS_VA, 3310 .probe = eth_nfp_pci_probe, 3311 .remove = eth_nfp_pci_remove, 3312 }; 3313 3314 RTE_PMD_REGISTER_PCI(net_nfp_pf, rte_nfp_net_pf_pmd); 3315 RTE_PMD_REGISTER_PCI(net_nfp_vf, rte_nfp_net_vf_pmd); 3316 RTE_PMD_REGISTER_PCI_TABLE(net_nfp_pf, pci_id_nfp_pf_net_map); 3317 RTE_PMD_REGISTER_PCI_TABLE(net_nfp_vf, pci_id_nfp_vf_net_map); 3318 RTE_PMD_REGISTER_KMOD_DEP(net_nfp_pf, "* igb_uio | uio_pci_generic | vfio"); 3319 RTE_PMD_REGISTER_KMOD_DEP(net_nfp_vf, "* igb_uio | uio_pci_generic | vfio"); 3320 3321 RTE_INIT(nfp_init_log) 3322 { 3323 nfp_logtype_init = rte_log_register("pmd.net.nfp.init"); 3324 if (nfp_logtype_init >= 0) 3325 rte_log_set_level(nfp_logtype_init, RTE_LOG_NOTICE); 3326 nfp_logtype_driver = rte_log_register("pmd.net.nfp.driver"); 3327 if (nfp_logtype_driver >= 0) 3328 rte_log_set_level(nfp_logtype_driver, RTE_LOG_NOTICE); 3329 } 3330 /* 3331 * Local variables: 3332 * c-file-style: "Linux" 3333 * indent-tabs-mode: t 3334 * End: 3335 */ 3336