1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2013-2016 Intel Corporation 3 */ 4 5 #include <ethdev_driver.h> 6 #include <ethdev_pci.h> 7 #include <rte_malloc.h> 8 #include <rte_memzone.h> 9 #include <rte_string_fns.h> 10 #include <rte_dev.h> 11 #include <rte_spinlock.h> 12 #include <rte_kvargs.h> 13 #include <rte_vect.h> 14 15 #include "fm10k.h" 16 #include "base/fm10k_api.h" 17 18 /* Default delay to acquire mailbox lock */ 19 #define FM10K_MBXLOCK_DELAY_US 20 20 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL 21 22 #define MAIN_VSI_POOL_NUMBER 0 23 24 /* Max try times to acquire switch status */ 25 #define MAX_QUERY_SWITCH_STATE_TIMES 10 26 /* Wait interval to get switch status */ 27 #define WAIT_SWITCH_MSG_US 100000 28 /* A period of quiescence for switch */ 29 #define FM10K_SWITCH_QUIESCE_US 100000 30 /* Number of chars per uint32 type */ 31 #define CHARS_PER_UINT32 (sizeof(uint32_t)) 32 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1) 33 34 /* default 1:1 map from queue ID to interrupt vector ID */ 35 #define Q2V(pci_dev, queue_id) ((pci_dev)->intr_handle.intr_vec[queue_id]) 36 37 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */ 38 #define MAX_LPORT_NUM 128 39 #define GLORT_FD_Q_BASE 0x40 40 #define GLORT_PF_MASK 0xFFC0 41 #define GLORT_FD_MASK GLORT_PF_MASK 42 #define GLORT_FD_INDEX GLORT_FD_Q_BASE 43 44 static void fm10k_close_mbx_service(struct fm10k_hw *hw); 45 static int fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev); 46 static int fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev); 47 static int fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev); 48 static int fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev); 49 static inline int fm10k_glort_valid(struct fm10k_hw *hw); 50 static int 51 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on); 52 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev, 53 const u8 *mac, bool add, uint32_t pool); 54 static void fm10k_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid); 55 static void fm10k_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid); 56 static void fm10k_set_rx_function(struct rte_eth_dev *dev); 57 static void fm10k_set_tx_function(struct rte_eth_dev *dev); 58 static int fm10k_check_ftag(struct rte_devargs *devargs); 59 static int fm10k_link_update(struct rte_eth_dev *dev, int wait_to_complete); 60 61 static int fm10k_dev_infos_get(struct rte_eth_dev *dev, 62 struct rte_eth_dev_info *dev_info); 63 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev); 64 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev); 65 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev); 66 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev); 67 68 struct fm10k_xstats_name_off { 69 char name[RTE_ETH_XSTATS_NAME_SIZE]; 70 unsigned offset; 71 }; 72 73 static const struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = { 74 {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)}, 75 {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)}, 76 {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)}, 77 {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)}, 78 {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)}, 79 {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)}, 80 {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)}, 81 {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats, 82 nodesc_drop)}, 83 }; 84 85 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \ 86 sizeof(fm10k_hw_stats_strings[0])) 87 88 static const struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = { 89 {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)}, 90 {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)}, 91 {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)}, 92 }; 93 94 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \ 95 sizeof(fm10k_hw_stats_rx_q_strings[0])) 96 97 static const struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = { 98 {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)}, 99 {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)}, 100 }; 101 102 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \ 103 sizeof(fm10k_hw_stats_tx_q_strings[0])) 104 105 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \ 106 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS)) 107 static int 108 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev); 109 110 static void 111 fm10k_mbx_initlock(struct fm10k_hw *hw) 112 { 113 rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)); 114 } 115 116 static void 117 fm10k_mbx_lock(struct fm10k_hw *hw) 118 { 119 while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back))) 120 rte_delay_us(FM10K_MBXLOCK_DELAY_US); 121 } 122 123 static void 124 fm10k_mbx_unlock(struct fm10k_hw *hw) 125 { 126 rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)); 127 } 128 129 /* Stubs needed for linkage when vPMD is disabled */ 130 __rte_weak int 131 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev) 132 { 133 return -1; 134 } 135 136 __rte_weak uint16_t 137 fm10k_recv_pkts_vec( 138 __rte_unused void *rx_queue, 139 __rte_unused struct rte_mbuf **rx_pkts, 140 __rte_unused uint16_t nb_pkts) 141 { 142 return 0; 143 } 144 145 __rte_weak uint16_t 146 fm10k_recv_scattered_pkts_vec( 147 __rte_unused void *rx_queue, 148 __rte_unused struct rte_mbuf **rx_pkts, 149 __rte_unused uint16_t nb_pkts) 150 { 151 return 0; 152 } 153 154 __rte_weak int 155 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq) 156 157 { 158 return -1; 159 } 160 161 __rte_weak void 162 fm10k_rx_queue_release_mbufs_vec( 163 __rte_unused struct fm10k_rx_queue *rxq) 164 { 165 return; 166 } 167 168 __rte_weak void 169 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq) 170 { 171 return; 172 } 173 174 __rte_weak int 175 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq) 176 { 177 return -1; 178 } 179 180 __rte_weak uint16_t 181 fm10k_xmit_fixed_burst_vec(__rte_unused void *tx_queue, 182 __rte_unused struct rte_mbuf **tx_pkts, 183 __rte_unused uint16_t nb_pkts) 184 { 185 return 0; 186 } 187 188 /* 189 * reset queue to initial state, allocate software buffers used when starting 190 * device. 191 * return 0 on success 192 * return -ENOMEM if buffers cannot be allocated 193 * return -EINVAL if buffers do not satisfy alignment condition 194 */ 195 static inline int 196 rx_queue_reset(struct fm10k_rx_queue *q) 197 { 198 static const union fm10k_rx_desc zero = {{0} }; 199 uint64_t dma_addr; 200 int i, diag; 201 PMD_INIT_FUNC_TRACE(); 202 203 diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc); 204 if (diag != 0) 205 return -ENOMEM; 206 207 for (i = 0; i < q->nb_desc; ++i) { 208 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id); 209 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) { 210 rte_mempool_put_bulk(q->mp, (void **)q->sw_ring, 211 q->nb_desc); 212 return -EINVAL; 213 } 214 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]); 215 q->hw_ring[i].q.pkt_addr = dma_addr; 216 q->hw_ring[i].q.hdr_addr = dma_addr; 217 } 218 219 /* initialize extra software ring entries. Space for these extra 220 * entries is always allocated. 221 */ 222 memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf)); 223 for (i = 0; i < q->nb_fake_desc; ++i) { 224 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf; 225 q->hw_ring[q->nb_desc + i] = zero; 226 } 227 228 q->next_dd = 0; 229 q->next_alloc = 0; 230 q->next_trigger = q->alloc_thresh - 1; 231 FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1); 232 q->rxrearm_start = 0; 233 q->rxrearm_nb = 0; 234 235 return 0; 236 } 237 238 /* 239 * clean queue, descriptor rings, free software buffers used when stopping 240 * device. 241 */ 242 static inline void 243 rx_queue_clean(struct fm10k_rx_queue *q) 244 { 245 union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} }; 246 uint32_t i; 247 PMD_INIT_FUNC_TRACE(); 248 249 /* zero descriptor rings */ 250 for (i = 0; i < q->nb_desc; ++i) 251 q->hw_ring[i] = zero; 252 253 /* zero faked descriptors */ 254 for (i = 0; i < q->nb_fake_desc; ++i) 255 q->hw_ring[q->nb_desc + i] = zero; 256 257 /* vPMD driver has a different way of releasing mbufs. */ 258 if (q->rx_using_sse) { 259 fm10k_rx_queue_release_mbufs_vec(q); 260 return; 261 } 262 263 /* free software buffers */ 264 for (i = 0; i < q->nb_desc; ++i) { 265 if (q->sw_ring[i]) { 266 rte_pktmbuf_free_seg(q->sw_ring[i]); 267 q->sw_ring[i] = NULL; 268 } 269 } 270 } 271 272 /* 273 * free all queue memory used when releasing the queue (i.e. configure) 274 */ 275 static inline void 276 rx_queue_free(struct fm10k_rx_queue *q) 277 { 278 PMD_INIT_FUNC_TRACE(); 279 if (q) { 280 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q); 281 rx_queue_clean(q); 282 if (q->sw_ring) { 283 rte_free(q->sw_ring); 284 q->sw_ring = NULL; 285 } 286 rte_free(q); 287 q = NULL; 288 } 289 } 290 291 /* 292 * disable RX queue, wait unitl HW finished necessary flush operation 293 */ 294 static inline int 295 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum) 296 { 297 uint32_t reg, i; 298 299 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum)); 300 FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum), 301 reg & ~FM10K_RXQCTL_ENABLE); 302 303 /* Wait 100us at most */ 304 for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) { 305 rte_delay_us(1); 306 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum)); 307 if (!(reg & FM10K_RXQCTL_ENABLE)) 308 break; 309 } 310 311 if (i == FM10K_QUEUE_DISABLE_TIMEOUT) 312 return -1; 313 314 return 0; 315 } 316 317 /* 318 * reset queue to initial state, allocate software buffers used when starting 319 * device 320 */ 321 static inline void 322 tx_queue_reset(struct fm10k_tx_queue *q) 323 { 324 PMD_INIT_FUNC_TRACE(); 325 q->last_free = 0; 326 q->next_free = 0; 327 q->nb_used = 0; 328 q->nb_free = q->nb_desc - 1; 329 fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh); 330 FM10K_PCI_REG_WRITE(q->tail_ptr, 0); 331 } 332 333 /* 334 * clean queue, descriptor rings, free software buffers used when stopping 335 * device 336 */ 337 static inline void 338 tx_queue_clean(struct fm10k_tx_queue *q) 339 { 340 struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0}; 341 uint32_t i; 342 PMD_INIT_FUNC_TRACE(); 343 344 /* zero descriptor rings */ 345 for (i = 0; i < q->nb_desc; ++i) 346 q->hw_ring[i] = zero; 347 348 /* free software buffers */ 349 for (i = 0; i < q->nb_desc; ++i) { 350 if (q->sw_ring[i]) { 351 rte_pktmbuf_free_seg(q->sw_ring[i]); 352 q->sw_ring[i] = NULL; 353 } 354 } 355 } 356 357 /* 358 * free all queue memory used when releasing the queue (i.e. configure) 359 */ 360 static inline void 361 tx_queue_free(struct fm10k_tx_queue *q) 362 { 363 PMD_INIT_FUNC_TRACE(); 364 if (q) { 365 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q); 366 tx_queue_clean(q); 367 if (q->rs_tracker.list) { 368 rte_free(q->rs_tracker.list); 369 q->rs_tracker.list = NULL; 370 } 371 if (q->sw_ring) { 372 rte_free(q->sw_ring); 373 q->sw_ring = NULL; 374 } 375 rte_free(q); 376 q = NULL; 377 } 378 } 379 380 /* 381 * disable TX queue, wait unitl HW finished necessary flush operation 382 */ 383 static inline int 384 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum) 385 { 386 uint32_t reg, i; 387 388 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum)); 389 FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum), 390 reg & ~FM10K_TXDCTL_ENABLE); 391 392 /* Wait 100us at most */ 393 for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) { 394 rte_delay_us(1); 395 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum)); 396 if (!(reg & FM10K_TXDCTL_ENABLE)) 397 break; 398 } 399 400 if (i == FM10K_QUEUE_DISABLE_TIMEOUT) 401 return -1; 402 403 return 0; 404 } 405 406 static int 407 fm10k_check_mq_mode(struct rte_eth_dev *dev) 408 { 409 enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode; 410 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 411 struct rte_eth_vmdq_rx_conf *vmdq_conf; 412 uint16_t nb_rx_q = dev->data->nb_rx_queues; 413 414 vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf; 415 416 if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) { 417 PMD_INIT_LOG(ERR, "DCB mode is not supported."); 418 return -EINVAL; 419 } 420 421 if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG)) 422 return 0; 423 424 if (hw->mac.type == fm10k_mac_vf) { 425 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF."); 426 return -EINVAL; 427 } 428 429 /* Check VMDQ queue pool number */ 430 if (vmdq_conf->nb_queue_pools > 431 sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT || 432 vmdq_conf->nb_queue_pools > nb_rx_q) { 433 PMD_INIT_LOG(ERR, "Too many of queue pools: %d", 434 vmdq_conf->nb_queue_pools); 435 return -EINVAL; 436 } 437 438 return 0; 439 } 440 441 static const struct fm10k_txq_ops def_txq_ops = { 442 .reset = tx_queue_reset, 443 }; 444 445 static int 446 fm10k_dev_configure(struct rte_eth_dev *dev) 447 { 448 int ret; 449 450 PMD_INIT_FUNC_TRACE(); 451 452 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) 453 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; 454 455 /* multipe queue mode checking */ 456 ret = fm10k_check_mq_mode(dev); 457 if (ret != 0) { 458 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.", 459 ret); 460 return ret; 461 } 462 463 dev->data->scattered_rx = 0; 464 465 return 0; 466 } 467 468 static void 469 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev) 470 { 471 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 472 struct rte_eth_vmdq_rx_conf *vmdq_conf; 473 uint32_t i; 474 475 vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf; 476 477 for (i = 0; i < vmdq_conf->nb_pool_maps; i++) { 478 if (!vmdq_conf->pool_map[i].pools) 479 continue; 480 fm10k_mbx_lock(hw); 481 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true); 482 fm10k_mbx_unlock(hw); 483 } 484 } 485 486 static void 487 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev) 488 { 489 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 490 491 /* Add default mac address */ 492 fm10k_MAC_filter_set(dev, hw->mac.addr, true, 493 MAIN_VSI_POOL_NUMBER); 494 } 495 496 static void 497 fm10k_dev_rss_configure(struct rte_eth_dev *dev) 498 { 499 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 500 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 501 uint32_t mrqc, *key, i, reta, j; 502 uint64_t hf; 503 504 #define RSS_KEY_SIZE 40 505 static uint8_t rss_intel_key[RSS_KEY_SIZE] = { 506 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 507 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0, 508 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4, 509 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, 510 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA, 511 }; 512 513 if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS || 514 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0) { 515 FM10K_WRITE_REG(hw, FM10K_MRQC(0), 0); 516 return; 517 } 518 519 /* random key is rss_intel_key (default) or user provided (rss_key) */ 520 if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL) 521 key = (uint32_t *)rss_intel_key; 522 else 523 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key; 524 525 /* Now fill our hash function seeds, 4 bytes at a time */ 526 for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i) 527 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]); 528 529 /* 530 * Fill in redirection table 531 * The byte-swap is needed because NIC registers are in 532 * little-endian order. 533 */ 534 reta = 0; 535 for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) { 536 if (j == dev->data->nb_rx_queues) 537 j = 0; 538 reta = (reta << CHAR_BIT) | j; 539 if ((i & 3) == 3) 540 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), 541 rte_bswap32(reta)); 542 } 543 544 /* 545 * Generate RSS hash based on packet types, TCP/UDP 546 * port numbers and/or IPv4/v6 src and dst addresses 547 */ 548 hf = dev_conf->rx_adv_conf.rss_conf.rss_hf; 549 mrqc = 0; 550 mrqc |= (hf & ETH_RSS_IPV4) ? FM10K_MRQC_IPV4 : 0; 551 mrqc |= (hf & ETH_RSS_IPV6) ? FM10K_MRQC_IPV6 : 0; 552 mrqc |= (hf & ETH_RSS_IPV6_EX) ? FM10K_MRQC_IPV6 : 0; 553 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP) ? FM10K_MRQC_TCP_IPV4 : 0; 554 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP) ? FM10K_MRQC_TCP_IPV6 : 0; 555 mrqc |= (hf & ETH_RSS_IPV6_TCP_EX) ? FM10K_MRQC_TCP_IPV6 : 0; 556 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP) ? FM10K_MRQC_UDP_IPV4 : 0; 557 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP) ? FM10K_MRQC_UDP_IPV6 : 0; 558 mrqc |= (hf & ETH_RSS_IPV6_UDP_EX) ? FM10K_MRQC_UDP_IPV6 : 0; 559 560 if (mrqc == 0) { 561 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not" 562 "supported", hf); 563 return; 564 } 565 566 FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc); 567 } 568 569 static void 570 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new) 571 { 572 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 573 uint32_t i; 574 575 for (i = 0; i < nb_lport_new; i++) { 576 /* Set unicast mode by default. App can change 577 * to other mode in other API func. 578 */ 579 fm10k_mbx_lock(hw); 580 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i, 581 FM10K_XCAST_MODE_NONE); 582 fm10k_mbx_unlock(hw); 583 } 584 } 585 586 static void 587 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev) 588 { 589 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 590 struct rte_eth_vmdq_rx_conf *vmdq_conf; 591 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 592 struct fm10k_macvlan_filter_info *macvlan; 593 uint16_t nb_queue_pools = 0; /* pool number in configuration */ 594 uint16_t nb_lport_new; 595 596 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); 597 vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf; 598 599 fm10k_dev_rss_configure(dev); 600 601 /* only PF supports VMDQ */ 602 if (hw->mac.type != fm10k_mac_pf) 603 return; 604 605 if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG) 606 nb_queue_pools = vmdq_conf->nb_queue_pools; 607 608 /* no pool number change, no need to update logic port and VLAN/MAC */ 609 if (macvlan->nb_queue_pools == nb_queue_pools) 610 return; 611 612 nb_lport_new = nb_queue_pools ? nb_queue_pools : 1; 613 fm10k_dev_logic_port_update(dev, nb_lport_new); 614 615 /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */ 616 memset(dev->data->mac_addrs, 0, 617 RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM); 618 rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr, 619 &dev->data->mac_addrs[0]); 620 memset(macvlan, 0, sizeof(*macvlan)); 621 macvlan->nb_queue_pools = nb_queue_pools; 622 623 if (nb_queue_pools) 624 fm10k_dev_vmdq_rx_configure(dev); 625 else 626 fm10k_dev_pf_main_vsi_reset(dev); 627 } 628 629 static int 630 fm10k_dev_tx_init(struct rte_eth_dev *dev) 631 { 632 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 633 int i, ret; 634 struct fm10k_tx_queue *txq; 635 uint64_t base_addr; 636 uint32_t size; 637 638 /* Disable TXINT to avoid possible interrupt */ 639 for (i = 0; i < hw->mac.max_queues; i++) 640 FM10K_WRITE_REG(hw, FM10K_TXINT(i), 641 3 << FM10K_TXINT_TIMER_SHIFT); 642 643 /* Setup TX queue */ 644 for (i = 0; i < dev->data->nb_tx_queues; ++i) { 645 txq = dev->data->tx_queues[i]; 646 base_addr = txq->hw_ring_phys_addr; 647 size = txq->nb_desc * sizeof(struct fm10k_tx_desc); 648 649 /* disable queue to avoid issues while updating state */ 650 ret = tx_queue_disable(hw, i); 651 if (ret) { 652 PMD_INIT_LOG(ERR, "failed to disable queue %d", i); 653 return -1; 654 } 655 /* Enable use of FTAG bit in TX descriptor, PFVTCTL 656 * register is read-only for VF. 657 */ 658 if (fm10k_check_ftag(dev->device->devargs)) { 659 if (hw->mac.type == fm10k_mac_pf) { 660 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i), 661 FM10K_PFVTCTL_FTAG_DESC_ENABLE); 662 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled"); 663 } else { 664 PMD_INIT_LOG(ERR, "VF FTAG is not supported."); 665 return -ENOTSUP; 666 } 667 } 668 669 /* set location and size for descriptor ring */ 670 FM10K_WRITE_REG(hw, FM10K_TDBAL(i), 671 base_addr & UINT64_LOWER_32BITS_MASK); 672 FM10K_WRITE_REG(hw, FM10K_TDBAH(i), 673 base_addr >> (CHAR_BIT * sizeof(uint32_t))); 674 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size); 675 676 /* assign default SGLORT for each TX queue by PF */ 677 if (hw->mac.type == fm10k_mac_pf) 678 FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map); 679 } 680 681 /* set up vector or scalar TX function as appropriate */ 682 fm10k_set_tx_function(dev); 683 684 return 0; 685 } 686 687 static int 688 fm10k_dev_rx_init(struct rte_eth_dev *dev) 689 { 690 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 691 struct fm10k_macvlan_filter_info *macvlan; 692 struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); 693 struct rte_intr_handle *intr_handle = &pdev->intr_handle; 694 int i, ret; 695 struct fm10k_rx_queue *rxq; 696 uint64_t base_addr; 697 uint32_t size; 698 uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY; 699 uint32_t logic_port = hw->mac.dglort_map; 700 uint16_t buf_size; 701 uint16_t queue_stride = 0; 702 703 /* enable RXINT for interrupt mode */ 704 i = 0; 705 if (rte_intr_dp_is_en(intr_handle)) { 706 for (; i < dev->data->nb_rx_queues; i++) { 707 FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(pdev, i)); 708 if (hw->mac.type == fm10k_mac_pf) 709 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)), 710 FM10K_ITR_AUTOMASK | 711 FM10K_ITR_MASK_CLEAR); 712 else 713 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)), 714 FM10K_ITR_AUTOMASK | 715 FM10K_ITR_MASK_CLEAR); 716 } 717 } 718 /* Disable other RXINT to avoid possible interrupt */ 719 for (; i < hw->mac.max_queues; i++) 720 FM10K_WRITE_REG(hw, FM10K_RXINT(i), 721 3 << FM10K_RXINT_TIMER_SHIFT); 722 723 /* Setup RX queues */ 724 for (i = 0; i < dev->data->nb_rx_queues; ++i) { 725 rxq = dev->data->rx_queues[i]; 726 base_addr = rxq->hw_ring_phys_addr; 727 size = rxq->nb_desc * sizeof(union fm10k_rx_desc); 728 729 /* disable queue to avoid issues while updating state */ 730 ret = rx_queue_disable(hw, i); 731 if (ret) { 732 PMD_INIT_LOG(ERR, "failed to disable queue %d", i); 733 return -1; 734 } 735 736 /* Setup the Base and Length of the Rx Descriptor Ring */ 737 FM10K_WRITE_REG(hw, FM10K_RDBAL(i), 738 base_addr & UINT64_LOWER_32BITS_MASK); 739 FM10K_WRITE_REG(hw, FM10K_RDBAH(i), 740 base_addr >> (CHAR_BIT * sizeof(uint32_t))); 741 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size); 742 743 /* Configure the Rx buffer size for one buff without split */ 744 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) - 745 RTE_PKTMBUF_HEADROOM); 746 /* As RX buffer is aligned to 512B within mbuf, some bytes are 747 * reserved for this purpose, and the worst case could be 511B. 748 * But SRR reg assumes all buffers have the same size. In order 749 * to fill the gap, we'll have to consider the worst case and 750 * assume 512B is reserved. If we don't do so, it's possible 751 * for HW to overwrite data to next mbuf. 752 */ 753 buf_size -= FM10K_RX_DATABUF_ALIGN; 754 755 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), 756 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) | 757 FM10K_SRRCTL_LOOPBACK_SUPPRESS); 758 759 /* It adds dual VLAN length for supporting dual VLAN */ 760 if ((dev->data->mtu + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + 761 2 * FM10K_VLAN_TAG_SIZE) > buf_size || 762 rxq->offloads & DEV_RX_OFFLOAD_SCATTER) { 763 uint32_t reg; 764 dev->data->scattered_rx = 1; 765 reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i)); 766 reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN; 767 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg); 768 } 769 770 /* Enable drop on empty, it's RO for VF */ 771 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en) 772 rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY; 773 774 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl); 775 FM10K_WRITE_FLUSH(hw); 776 } 777 778 /* Configure VMDQ/RSS if applicable */ 779 fm10k_dev_mq_rx_configure(dev); 780 781 /* Decide the best RX function */ 782 fm10k_set_rx_function(dev); 783 784 /* update RX_SGLORT for loopback suppress*/ 785 if (hw->mac.type != fm10k_mac_pf) 786 return 0; 787 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); 788 if (macvlan->nb_queue_pools) 789 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools; 790 for (i = 0; i < dev->data->nb_rx_queues; ++i) { 791 if (i && queue_stride && !(i % queue_stride)) 792 logic_port++; 793 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port); 794 } 795 796 return 0; 797 } 798 799 static int 800 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) 801 { 802 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 803 int err; 804 uint32_t reg; 805 struct fm10k_rx_queue *rxq; 806 807 PMD_INIT_FUNC_TRACE(); 808 809 rxq = dev->data->rx_queues[rx_queue_id]; 810 err = rx_queue_reset(rxq); 811 if (err == -ENOMEM) { 812 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err); 813 return err; 814 } else if (err == -EINVAL) { 815 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :" 816 " %d", err); 817 return err; 818 } 819 820 /* Setup the HW Rx Head and Tail Descriptor Pointers 821 * Note: this must be done AFTER the queue is enabled on real 822 * hardware, but BEFORE the queue is enabled when using the 823 * emulation platform. Do it in both places for now and remove 824 * this comment and the following two register writes when the 825 * emulation platform is no longer being used. 826 */ 827 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0); 828 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1); 829 830 /* Set PF ownership flag for PF devices */ 831 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id)); 832 if (hw->mac.type == fm10k_mac_pf) 833 reg |= FM10K_RXQCTL_PF; 834 reg |= FM10K_RXQCTL_ENABLE; 835 /* enable RX queue */ 836 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg); 837 FM10K_WRITE_FLUSH(hw); 838 839 /* Setup the HW Rx Head and Tail Descriptor Pointers 840 * Note: this must be done AFTER the queue is enabled 841 */ 842 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0); 843 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1); 844 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 845 846 return 0; 847 } 848 849 static int 850 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) 851 { 852 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 853 854 PMD_INIT_FUNC_TRACE(); 855 856 /* Disable RX queue */ 857 rx_queue_disable(hw, rx_queue_id); 858 859 /* Free mbuf and clean HW ring */ 860 rx_queue_clean(dev->data->rx_queues[rx_queue_id]); 861 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 862 863 return 0; 864 } 865 866 static int 867 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) 868 { 869 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 870 /** @todo - this should be defined in the shared code */ 871 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY 0x00010000 872 uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY; 873 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id]; 874 875 PMD_INIT_FUNC_TRACE(); 876 877 q->ops->reset(q); 878 879 /* reset head and tail pointers */ 880 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0); 881 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0); 882 883 /* enable TX queue */ 884 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id), 885 FM10K_TXDCTL_ENABLE | txdctl); 886 FM10K_WRITE_FLUSH(hw); 887 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 888 889 return 0; 890 } 891 892 static int 893 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) 894 { 895 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 896 897 PMD_INIT_FUNC_TRACE(); 898 899 tx_queue_disable(hw, tx_queue_id); 900 tx_queue_clean(dev->data->tx_queues[tx_queue_id]); 901 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 902 903 return 0; 904 } 905 906 static inline int fm10k_glort_valid(struct fm10k_hw *hw) 907 { 908 return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE) 909 != FM10K_DGLORTMAP_NONE); 910 } 911 912 static int 913 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev) 914 { 915 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 916 int status; 917 918 PMD_INIT_FUNC_TRACE(); 919 920 /* Return if it didn't acquire valid glort range */ 921 if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw)) 922 return 0; 923 924 fm10k_mbx_lock(hw); 925 status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map, 926 FM10K_XCAST_MODE_PROMISC); 927 fm10k_mbx_unlock(hw); 928 929 if (status != FM10K_SUCCESS) { 930 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode"); 931 return -EAGAIN; 932 } 933 934 return 0; 935 } 936 937 static int 938 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev) 939 { 940 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 941 uint8_t mode; 942 int status; 943 944 PMD_INIT_FUNC_TRACE(); 945 946 /* Return if it didn't acquire valid glort range */ 947 if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw)) 948 return 0; 949 950 if (dev->data->all_multicast == 1) 951 mode = FM10K_XCAST_MODE_ALLMULTI; 952 else 953 mode = FM10K_XCAST_MODE_NONE; 954 955 fm10k_mbx_lock(hw); 956 status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map, 957 mode); 958 fm10k_mbx_unlock(hw); 959 960 if (status != FM10K_SUCCESS) { 961 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode"); 962 return -EAGAIN; 963 } 964 965 return 0; 966 } 967 968 static int 969 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev) 970 { 971 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 972 int status; 973 974 PMD_INIT_FUNC_TRACE(); 975 976 /* Return if it didn't acquire valid glort range */ 977 if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw)) 978 return 0; 979 980 /* If promiscuous mode is enabled, it doesn't make sense to enable 981 * allmulticast and disable promiscuous since fm10k only can select 982 * one of the modes. 983 */ 984 if (dev->data->promiscuous) { 985 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\ 986 "needn't enable allmulticast"); 987 return 0; 988 } 989 990 fm10k_mbx_lock(hw); 991 status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map, 992 FM10K_XCAST_MODE_ALLMULTI); 993 fm10k_mbx_unlock(hw); 994 995 if (status != FM10K_SUCCESS) { 996 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode"); 997 return -EAGAIN; 998 } 999 1000 return 0; 1001 } 1002 1003 static int 1004 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev) 1005 { 1006 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1007 int status; 1008 1009 PMD_INIT_FUNC_TRACE(); 1010 1011 /* Return if it didn't acquire valid glort range */ 1012 if ((hw->mac.type == fm10k_mac_pf) && !fm10k_glort_valid(hw)) 1013 return 0; 1014 1015 if (dev->data->promiscuous) { 1016 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\ 1017 "since promisc mode is enabled"); 1018 return -EINVAL; 1019 } 1020 1021 fm10k_mbx_lock(hw); 1022 /* Change mode to unicast mode */ 1023 status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map, 1024 FM10K_XCAST_MODE_NONE); 1025 fm10k_mbx_unlock(hw); 1026 1027 if (status != FM10K_SUCCESS) { 1028 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode"); 1029 return -EAGAIN; 1030 } 1031 1032 return 0; 1033 } 1034 1035 static void 1036 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev) 1037 { 1038 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1039 uint32_t dglortdec, pool_len, rss_len, i, dglortmask; 1040 uint16_t nb_queue_pools; 1041 struct fm10k_macvlan_filter_info *macvlan; 1042 1043 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); 1044 nb_queue_pools = macvlan->nb_queue_pools; 1045 pool_len = nb_queue_pools ? rte_fls_u32(nb_queue_pools - 1) : 0; 1046 rss_len = rte_fls_u32(dev->data->nb_rx_queues - 1) - pool_len; 1047 1048 /* GLORT 0x0-0x3F are used by PF and VMDQ, 0x40-0x7F used by FD */ 1049 dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len; 1050 dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) | 1051 hw->mac.dglort_map; 1052 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask); 1053 /* Configure VMDQ/RSS DGlort Decoder */ 1054 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec); 1055 1056 /* Flow Director configurations, only queue number is valid. */ 1057 dglortdec = rte_fls_u32(dev->data->nb_rx_queues - 1); 1058 dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) | 1059 (hw->mac.dglort_map + GLORT_FD_Q_BASE); 1060 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask); 1061 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec); 1062 1063 /* Invalidate all other GLORT entries */ 1064 for (i = 2; i < FM10K_DGLORT_COUNT; i++) 1065 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i), 1066 FM10K_DGLORTMAP_NONE); 1067 } 1068 1069 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1) 1070 static int 1071 fm10k_dev_start(struct rte_eth_dev *dev) 1072 { 1073 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1074 int i, diag; 1075 1076 PMD_INIT_FUNC_TRACE(); 1077 1078 /* stop, init, then start the hw */ 1079 diag = fm10k_stop_hw(hw); 1080 if (diag != FM10K_SUCCESS) { 1081 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag); 1082 return -EIO; 1083 } 1084 1085 diag = fm10k_init_hw(hw); 1086 if (diag != FM10K_SUCCESS) { 1087 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag); 1088 return -EIO; 1089 } 1090 1091 diag = fm10k_start_hw(hw); 1092 if (diag != FM10K_SUCCESS) { 1093 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag); 1094 return -EIO; 1095 } 1096 1097 diag = fm10k_dev_tx_init(dev); 1098 if (diag) { 1099 PMD_INIT_LOG(ERR, "TX init failed: %d", diag); 1100 return diag; 1101 } 1102 1103 if (fm10k_dev_rxq_interrupt_setup(dev)) 1104 return -EIO; 1105 1106 diag = fm10k_dev_rx_init(dev); 1107 if (diag) { 1108 PMD_INIT_LOG(ERR, "RX init failed: %d", diag); 1109 return diag; 1110 } 1111 1112 if (hw->mac.type == fm10k_mac_pf) 1113 fm10k_dev_dglort_map_configure(dev); 1114 1115 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1116 struct fm10k_rx_queue *rxq; 1117 rxq = dev->data->rx_queues[i]; 1118 1119 if (rxq->rx_deferred_start) 1120 continue; 1121 diag = fm10k_dev_rx_queue_start(dev, i); 1122 if (diag != 0) { 1123 int j; 1124 for (j = 0; j < i; ++j) 1125 rx_queue_clean(dev->data->rx_queues[j]); 1126 return diag; 1127 } 1128 } 1129 1130 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1131 struct fm10k_tx_queue *txq; 1132 txq = dev->data->tx_queues[i]; 1133 1134 if (txq->tx_deferred_start) 1135 continue; 1136 diag = fm10k_dev_tx_queue_start(dev, i); 1137 if (diag != 0) { 1138 int j; 1139 for (j = 0; j < i; ++j) 1140 tx_queue_clean(dev->data->tx_queues[j]); 1141 for (j = 0; j < dev->data->nb_rx_queues; ++j) 1142 rx_queue_clean(dev->data->rx_queues[j]); 1143 return diag; 1144 } 1145 } 1146 1147 /* Update default vlan when not in VMDQ mode */ 1148 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)) 1149 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true); 1150 1151 fm10k_link_update(dev, 0); 1152 1153 return 0; 1154 } 1155 1156 static int 1157 fm10k_dev_stop(struct rte_eth_dev *dev) 1158 { 1159 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1160 struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); 1161 struct rte_intr_handle *intr_handle = &pdev->intr_handle; 1162 int i; 1163 1164 PMD_INIT_FUNC_TRACE(); 1165 dev->data->dev_started = 0; 1166 1167 if (dev->data->tx_queues) 1168 for (i = 0; i < dev->data->nb_tx_queues; i++) 1169 fm10k_dev_tx_queue_stop(dev, i); 1170 1171 if (dev->data->rx_queues) 1172 for (i = 0; i < dev->data->nb_rx_queues; i++) 1173 fm10k_dev_rx_queue_stop(dev, i); 1174 1175 /* Disable datapath event */ 1176 if (rte_intr_dp_is_en(intr_handle)) { 1177 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1178 FM10K_WRITE_REG(hw, FM10K_RXINT(i), 1179 3 << FM10K_RXINT_TIMER_SHIFT); 1180 if (hw->mac.type == fm10k_mac_pf) 1181 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, i)), 1182 FM10K_ITR_MASK_SET); 1183 else 1184 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, i)), 1185 FM10K_ITR_MASK_SET); 1186 } 1187 } 1188 /* Clean datapath event and queue/vec mapping */ 1189 rte_intr_efd_disable(intr_handle); 1190 rte_free(intr_handle->intr_vec); 1191 intr_handle->intr_vec = NULL; 1192 1193 return 0; 1194 } 1195 1196 static void 1197 fm10k_dev_queue_release(struct rte_eth_dev *dev) 1198 { 1199 int i; 1200 1201 PMD_INIT_FUNC_TRACE(); 1202 1203 if (dev->data->tx_queues) { 1204 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1205 struct fm10k_tx_queue *txq = dev->data->tx_queues[i]; 1206 1207 tx_queue_free(txq); 1208 } 1209 } 1210 1211 if (dev->data->rx_queues) { 1212 for (i = 0; i < dev->data->nb_rx_queues; i++) 1213 fm10k_rx_queue_release(dev, i); 1214 } 1215 } 1216 1217 static int 1218 fm10k_link_update(struct rte_eth_dev *dev, 1219 __rte_unused int wait_to_complete) 1220 { 1221 struct fm10k_dev_info *dev_info = 1222 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private); 1223 PMD_INIT_FUNC_TRACE(); 1224 1225 dev->data->dev_link.link_speed = ETH_SPEED_NUM_50G; 1226 dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX; 1227 dev->data->dev_link.link_status = 1228 dev_info->sm_down ? ETH_LINK_DOWN : ETH_LINK_UP; 1229 dev->data->dev_link.link_autoneg = ETH_LINK_FIXED; 1230 1231 return 0; 1232 } 1233 1234 static int fm10k_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 1235 struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit) 1236 { 1237 unsigned i, q; 1238 unsigned count = 0; 1239 1240 if (xstats_names != NULL) { 1241 /* Note: limit checked in rte_eth_xstats_names() */ 1242 1243 /* Global stats */ 1244 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) { 1245 snprintf(xstats_names[count].name, 1246 sizeof(xstats_names[count].name), 1247 "%s", fm10k_hw_stats_strings[count].name); 1248 count++; 1249 } 1250 1251 /* PF queue stats */ 1252 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) { 1253 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) { 1254 snprintf(xstats_names[count].name, 1255 sizeof(xstats_names[count].name), 1256 "rx_q%u_%s", q, 1257 fm10k_hw_stats_rx_q_strings[i].name); 1258 count++; 1259 } 1260 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) { 1261 snprintf(xstats_names[count].name, 1262 sizeof(xstats_names[count].name), 1263 "tx_q%u_%s", q, 1264 fm10k_hw_stats_tx_q_strings[i].name); 1265 count++; 1266 } 1267 } 1268 } 1269 return FM10K_NB_XSTATS; 1270 } 1271 1272 static int 1273 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 1274 unsigned n) 1275 { 1276 struct fm10k_hw_stats *hw_stats = 1277 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1278 unsigned i, q, count = 0; 1279 1280 if (n < FM10K_NB_XSTATS) 1281 return FM10K_NB_XSTATS; 1282 1283 /* Global stats */ 1284 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) { 1285 xstats[count].value = *(uint64_t *)(((char *)hw_stats) + 1286 fm10k_hw_stats_strings[count].offset); 1287 xstats[count].id = count; 1288 count++; 1289 } 1290 1291 /* PF queue stats */ 1292 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) { 1293 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) { 1294 xstats[count].value = 1295 *(uint64_t *)(((char *)&hw_stats->q[q]) + 1296 fm10k_hw_stats_rx_q_strings[i].offset); 1297 xstats[count].id = count; 1298 count++; 1299 } 1300 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) { 1301 xstats[count].value = 1302 *(uint64_t *)(((char *)&hw_stats->q[q]) + 1303 fm10k_hw_stats_tx_q_strings[i].offset); 1304 xstats[count].id = count; 1305 count++; 1306 } 1307 } 1308 1309 return FM10K_NB_XSTATS; 1310 } 1311 1312 static int 1313 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 1314 { 1315 uint64_t ipackets, opackets, ibytes, obytes, imissed; 1316 struct fm10k_hw *hw = 1317 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1318 struct fm10k_hw_stats *hw_stats = 1319 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1320 int i; 1321 1322 PMD_INIT_FUNC_TRACE(); 1323 1324 fm10k_update_hw_stats(hw, hw_stats); 1325 1326 ipackets = opackets = ibytes = obytes = imissed = 0; 1327 for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) && 1328 (i < hw->mac.max_queues); ++i) { 1329 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count; 1330 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count; 1331 stats->q_ibytes[i] = hw_stats->q[i].rx_bytes.count; 1332 stats->q_obytes[i] = hw_stats->q[i].tx_bytes.count; 1333 stats->q_errors[i] = hw_stats->q[i].rx_drops.count; 1334 ipackets += stats->q_ipackets[i]; 1335 opackets += stats->q_opackets[i]; 1336 ibytes += stats->q_ibytes[i]; 1337 obytes += stats->q_obytes[i]; 1338 imissed += stats->q_errors[i]; 1339 } 1340 stats->ipackets = ipackets; 1341 stats->opackets = opackets; 1342 stats->ibytes = ibytes; 1343 stats->obytes = obytes; 1344 stats->imissed = imissed; 1345 return 0; 1346 } 1347 1348 static int 1349 fm10k_stats_reset(struct rte_eth_dev *dev) 1350 { 1351 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1352 struct fm10k_hw_stats *hw_stats = 1353 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private); 1354 1355 PMD_INIT_FUNC_TRACE(); 1356 1357 memset(hw_stats, 0, sizeof(*hw_stats)); 1358 fm10k_rebind_hw_stats(hw, hw_stats); 1359 1360 return 0; 1361 } 1362 1363 static int 1364 fm10k_dev_infos_get(struct rte_eth_dev *dev, 1365 struct rte_eth_dev_info *dev_info) 1366 { 1367 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1368 struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); 1369 1370 PMD_INIT_FUNC_TRACE(); 1371 1372 dev_info->min_rx_bufsize = FM10K_MIN_RX_BUF_SIZE; 1373 dev_info->max_rx_pktlen = FM10K_MAX_PKT_SIZE; 1374 dev_info->max_rx_queues = hw->mac.max_queues; 1375 dev_info->max_tx_queues = hw->mac.max_queues; 1376 dev_info->max_mac_addrs = FM10K_MAX_MACADDR_NUM; 1377 dev_info->max_hash_mac_addrs = 0; 1378 dev_info->max_vfs = pdev->max_vfs; 1379 dev_info->vmdq_pool_base = 0; 1380 dev_info->vmdq_queue_base = 0; 1381 dev_info->max_vmdq_pools = ETH_32_POOLS; 1382 dev_info->vmdq_queue_num = FM10K_MAX_QUEUES_PF; 1383 dev_info->rx_queue_offload_capa = fm10k_get_rx_queue_offloads_capa(dev); 1384 dev_info->rx_offload_capa = fm10k_get_rx_port_offloads_capa(dev) | 1385 dev_info->rx_queue_offload_capa; 1386 dev_info->tx_queue_offload_capa = fm10k_get_tx_queue_offloads_capa(dev); 1387 dev_info->tx_offload_capa = fm10k_get_tx_port_offloads_capa(dev) | 1388 dev_info->tx_queue_offload_capa; 1389 1390 dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t); 1391 dev_info->reta_size = FM10K_MAX_RSS_INDICES; 1392 dev_info->flow_type_rss_offloads = ETH_RSS_IPV4 | 1393 ETH_RSS_IPV6 | 1394 ETH_RSS_IPV6_EX | 1395 ETH_RSS_NONFRAG_IPV4_TCP | 1396 ETH_RSS_NONFRAG_IPV6_TCP | 1397 ETH_RSS_IPV6_TCP_EX | 1398 ETH_RSS_NONFRAG_IPV4_UDP | 1399 ETH_RSS_NONFRAG_IPV6_UDP | 1400 ETH_RSS_IPV6_UDP_EX; 1401 1402 dev_info->default_rxconf = (struct rte_eth_rxconf) { 1403 .rx_thresh = { 1404 .pthresh = FM10K_DEFAULT_RX_PTHRESH, 1405 .hthresh = FM10K_DEFAULT_RX_HTHRESH, 1406 .wthresh = FM10K_DEFAULT_RX_WTHRESH, 1407 }, 1408 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0), 1409 .rx_drop_en = 0, 1410 .offloads = 0, 1411 }; 1412 1413 dev_info->default_txconf = (struct rte_eth_txconf) { 1414 .tx_thresh = { 1415 .pthresh = FM10K_DEFAULT_TX_PTHRESH, 1416 .hthresh = FM10K_DEFAULT_TX_HTHRESH, 1417 .wthresh = FM10K_DEFAULT_TX_WTHRESH, 1418 }, 1419 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0), 1420 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0), 1421 .offloads = 0, 1422 }; 1423 1424 dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { 1425 .nb_max = FM10K_MAX_RX_DESC, 1426 .nb_min = FM10K_MIN_RX_DESC, 1427 .nb_align = FM10K_MULT_RX_DESC, 1428 }; 1429 1430 dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { 1431 .nb_max = FM10K_MAX_TX_DESC, 1432 .nb_min = FM10K_MIN_TX_DESC, 1433 .nb_align = FM10K_MULT_TX_DESC, 1434 .nb_seg_max = FM10K_TX_MAX_SEG, 1435 .nb_mtu_seg_max = FM10K_TX_MAX_MTU_SEG, 1436 }; 1437 1438 dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G | 1439 ETH_LINK_SPEED_10G | ETH_LINK_SPEED_25G | 1440 ETH_LINK_SPEED_40G | ETH_LINK_SPEED_100G; 1441 1442 return 0; 1443 } 1444 1445 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE 1446 static const uint32_t * 1447 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev) 1448 { 1449 if (dev->rx_pkt_burst == fm10k_recv_pkts || 1450 dev->rx_pkt_burst == fm10k_recv_scattered_pkts) { 1451 static uint32_t ptypes[] = { 1452 /* refers to rx_desc_to_ol_flags() */ 1453 RTE_PTYPE_L2_ETHER, 1454 RTE_PTYPE_L3_IPV4, 1455 RTE_PTYPE_L3_IPV4_EXT, 1456 RTE_PTYPE_L3_IPV6, 1457 RTE_PTYPE_L3_IPV6_EXT, 1458 RTE_PTYPE_L4_TCP, 1459 RTE_PTYPE_L4_UDP, 1460 RTE_PTYPE_UNKNOWN 1461 }; 1462 1463 return ptypes; 1464 } else if (dev->rx_pkt_burst == fm10k_recv_pkts_vec || 1465 dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec) { 1466 static uint32_t ptypes_vec[] = { 1467 /* refers to fm10k_desc_to_pktype_v() */ 1468 RTE_PTYPE_L3_IPV4, 1469 RTE_PTYPE_L3_IPV4_EXT, 1470 RTE_PTYPE_L3_IPV6, 1471 RTE_PTYPE_L3_IPV6_EXT, 1472 RTE_PTYPE_L4_TCP, 1473 RTE_PTYPE_L4_UDP, 1474 RTE_PTYPE_TUNNEL_GENEVE, 1475 RTE_PTYPE_TUNNEL_NVGRE, 1476 RTE_PTYPE_TUNNEL_VXLAN, 1477 RTE_PTYPE_TUNNEL_GRE, 1478 RTE_PTYPE_UNKNOWN 1479 }; 1480 1481 return ptypes_vec; 1482 } 1483 1484 return NULL; 1485 } 1486 #else 1487 static const uint32_t * 1488 fm10k_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) 1489 { 1490 return NULL; 1491 } 1492 #endif 1493 1494 static int 1495 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 1496 { 1497 s32 result; 1498 uint16_t mac_num = 0; 1499 uint32_t vid_idx, vid_bit, mac_index; 1500 struct fm10k_hw *hw; 1501 struct fm10k_macvlan_filter_info *macvlan; 1502 struct rte_eth_dev_data *data = dev->data; 1503 1504 hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1505 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); 1506 1507 if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */ 1508 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode"); 1509 return -EINVAL; 1510 } 1511 1512 if (vlan_id > ETH_VLAN_ID_MAX) { 1513 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096"); 1514 return -EINVAL; 1515 } 1516 1517 vid_idx = FM10K_VFTA_IDX(vlan_id); 1518 vid_bit = FM10K_VFTA_BIT(vlan_id); 1519 /* this VLAN ID is already in the VLAN filter table, return SUCCESS */ 1520 if (on && (macvlan->vfta[vid_idx] & vid_bit)) 1521 return 0; 1522 /* this VLAN ID is NOT in the VLAN filter table, cannot remove */ 1523 if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) { 1524 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing " 1525 "in the VLAN filter table"); 1526 return -EINVAL; 1527 } 1528 1529 fm10k_mbx_lock(hw); 1530 result = fm10k_update_vlan(hw, vlan_id, 0, on); 1531 fm10k_mbx_unlock(hw); 1532 if (result != FM10K_SUCCESS) { 1533 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result); 1534 return -EIO; 1535 } 1536 1537 for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) && 1538 (result == FM10K_SUCCESS); mac_index++) { 1539 if (rte_is_zero_ether_addr(&data->mac_addrs[mac_index])) 1540 continue; 1541 if (mac_num > macvlan->mac_num - 1) { 1542 PMD_INIT_LOG(ERR, "MAC address number " 1543 "not match"); 1544 break; 1545 } 1546 fm10k_mbx_lock(hw); 1547 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map, 1548 data->mac_addrs[mac_index].addr_bytes, 1549 vlan_id, on, 0); 1550 fm10k_mbx_unlock(hw); 1551 mac_num++; 1552 } 1553 if (result != FM10K_SUCCESS) { 1554 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result); 1555 return -EIO; 1556 } 1557 1558 if (on) { 1559 macvlan->vlan_num++; 1560 macvlan->vfta[vid_idx] |= vid_bit; 1561 } else { 1562 macvlan->vlan_num--; 1563 macvlan->vfta[vid_idx] &= ~vid_bit; 1564 } 1565 return 0; 1566 } 1567 1568 static int 1569 fm10k_vlan_offload_set(struct rte_eth_dev *dev __rte_unused, 1570 int mask __rte_unused) 1571 { 1572 return 0; 1573 } 1574 1575 /* Add/Remove a MAC address, and update filters to main VSI */ 1576 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev, 1577 const u8 *mac, bool add, uint32_t pool) 1578 { 1579 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1580 struct fm10k_macvlan_filter_info *macvlan; 1581 uint32_t i, j, k; 1582 1583 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); 1584 1585 if (pool != MAIN_VSI_POOL_NUMBER) { 1586 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set " 1587 "mac to pool %u", pool); 1588 return; 1589 } 1590 for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) { 1591 if (!macvlan->vfta[j]) 1592 continue; 1593 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) { 1594 if (!(macvlan->vfta[j] & (1 << k))) 1595 continue; 1596 if (i + 1 > macvlan->vlan_num) { 1597 PMD_INIT_LOG(ERR, "vlan number not match"); 1598 return; 1599 } 1600 fm10k_mbx_lock(hw); 1601 fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac, 1602 j * FM10K_UINT32_BIT_SIZE + k, add, 0); 1603 fm10k_mbx_unlock(hw); 1604 i++; 1605 } 1606 } 1607 } 1608 1609 /* Add/Remove a MAC address, and update filters to VMDQ */ 1610 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev, 1611 const u8 *mac, bool add, uint32_t pool) 1612 { 1613 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1614 struct fm10k_macvlan_filter_info *macvlan; 1615 struct rte_eth_vmdq_rx_conf *vmdq_conf; 1616 uint32_t i; 1617 1618 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); 1619 vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf; 1620 1621 if (pool > macvlan->nb_queue_pools) { 1622 PMD_DRV_LOG(ERR, "Pool number %u invalid." 1623 " Max pool is %u", 1624 pool, macvlan->nb_queue_pools); 1625 return; 1626 } 1627 for (i = 0; i < vmdq_conf->nb_pool_maps; i++) { 1628 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool))) 1629 continue; 1630 fm10k_mbx_lock(hw); 1631 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac, 1632 vmdq_conf->pool_map[i].vlan_id, add, 0); 1633 fm10k_mbx_unlock(hw); 1634 } 1635 } 1636 1637 /* Add/Remove a MAC address, and update filters */ 1638 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev, 1639 const u8 *mac, bool add, uint32_t pool) 1640 { 1641 struct fm10k_macvlan_filter_info *macvlan; 1642 1643 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); 1644 1645 if (macvlan->nb_queue_pools > 0) /* VMDQ mode */ 1646 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool); 1647 else 1648 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool); 1649 1650 if (add) 1651 macvlan->mac_num++; 1652 else 1653 macvlan->mac_num--; 1654 } 1655 1656 /* Add a MAC address, and update filters */ 1657 static int 1658 fm10k_macaddr_add(struct rte_eth_dev *dev, 1659 struct rte_ether_addr *mac_addr, 1660 uint32_t index, 1661 uint32_t pool) 1662 { 1663 struct fm10k_macvlan_filter_info *macvlan; 1664 1665 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); 1666 fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool); 1667 macvlan->mac_vmdq_id[index] = pool; 1668 return 0; 1669 } 1670 1671 /* Remove a MAC address, and update filters */ 1672 static void 1673 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index) 1674 { 1675 struct rte_eth_dev_data *data = dev->data; 1676 struct fm10k_macvlan_filter_info *macvlan; 1677 1678 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); 1679 fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes, 1680 FALSE, macvlan->mac_vmdq_id[index]); 1681 macvlan->mac_vmdq_id[index] = 0; 1682 } 1683 1684 static inline int 1685 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request) 1686 { 1687 if ((request < min) || (request > max) || ((request % mult) != 0)) 1688 return -1; 1689 else 1690 return 0; 1691 } 1692 1693 1694 static inline int 1695 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request) 1696 { 1697 if ((request < min) || (request > max) || ((div % request) != 0)) 1698 return -1; 1699 else 1700 return 0; 1701 } 1702 1703 static inline int 1704 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf) 1705 { 1706 uint16_t rx_free_thresh; 1707 1708 if (conf->rx_free_thresh == 0) 1709 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q); 1710 else 1711 rx_free_thresh = conf->rx_free_thresh; 1712 1713 /* make sure the requested threshold satisfies the constraints */ 1714 if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q), 1715 FM10K_RX_FREE_THRESH_MAX(q), 1716 FM10K_RX_FREE_THRESH_DIV(q), 1717 rx_free_thresh)) { 1718 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be " 1719 "less than or equal to %u, " 1720 "greater than or equal to %u, " 1721 "and a divisor of %u", 1722 rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q), 1723 FM10K_RX_FREE_THRESH_MIN(q), 1724 FM10K_RX_FREE_THRESH_DIV(q)); 1725 return -EINVAL; 1726 } 1727 1728 q->alloc_thresh = rx_free_thresh; 1729 q->drop_en = conf->rx_drop_en; 1730 q->rx_deferred_start = conf->rx_deferred_start; 1731 1732 return 0; 1733 } 1734 1735 /* 1736 * Hardware requires specific alignment for Rx packet buffers. At 1737 * least one of the following two conditions must be satisfied. 1738 * 1. Address is 512B aligned 1739 * 2. Address is 8B aligned and buffer does not cross 4K boundary. 1740 * 1741 * As such, the driver may need to adjust the DMA address within the 1742 * buffer by up to 512B. 1743 * 1744 * return 1 if the element size is valid, otherwise return 0. 1745 */ 1746 static int 1747 mempool_element_size_valid(struct rte_mempool *mp) 1748 { 1749 uint32_t min_size; 1750 1751 /* elt_size includes mbuf header and headroom */ 1752 min_size = mp->elt_size - sizeof(struct rte_mbuf) - 1753 RTE_PKTMBUF_HEADROOM; 1754 1755 /* account for up to 512B of alignment */ 1756 min_size -= FM10K_RX_DATABUF_ALIGN; 1757 1758 /* sanity check for overflow */ 1759 if (min_size > mp->elt_size) 1760 return 0; 1761 1762 /* size is valid */ 1763 return 1; 1764 } 1765 1766 static uint64_t fm10k_get_rx_queue_offloads_capa(struct rte_eth_dev *dev) 1767 { 1768 RTE_SET_USED(dev); 1769 1770 return (uint64_t)(DEV_RX_OFFLOAD_SCATTER); 1771 } 1772 1773 static uint64_t fm10k_get_rx_port_offloads_capa(struct rte_eth_dev *dev) 1774 { 1775 RTE_SET_USED(dev); 1776 1777 return (uint64_t)(DEV_RX_OFFLOAD_VLAN_STRIP | 1778 DEV_RX_OFFLOAD_VLAN_FILTER | 1779 DEV_RX_OFFLOAD_IPV4_CKSUM | 1780 DEV_RX_OFFLOAD_UDP_CKSUM | 1781 DEV_RX_OFFLOAD_TCP_CKSUM | 1782 DEV_RX_OFFLOAD_HEADER_SPLIT | 1783 DEV_RX_OFFLOAD_RSS_HASH); 1784 } 1785 1786 static int 1787 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, 1788 uint16_t nb_desc, unsigned int socket_id, 1789 const struct rte_eth_rxconf *conf, struct rte_mempool *mp) 1790 { 1791 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1792 struct fm10k_dev_info *dev_info = 1793 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private); 1794 struct fm10k_rx_queue *q; 1795 const struct rte_memzone *mz; 1796 uint64_t offloads; 1797 1798 PMD_INIT_FUNC_TRACE(); 1799 1800 offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads; 1801 1802 /* make sure the mempool element size can account for alignment. */ 1803 if (!mempool_element_size_valid(mp)) { 1804 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small"); 1805 return -EINVAL; 1806 } 1807 1808 /* make sure a valid number of descriptors have been requested */ 1809 if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC, 1810 FM10K_MULT_RX_DESC, nb_desc)) { 1811 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be " 1812 "less than or equal to %"PRIu32", " 1813 "greater than or equal to %u, " 1814 "and a multiple of %u", 1815 nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC, 1816 FM10K_MULT_RX_DESC); 1817 return -EINVAL; 1818 } 1819 1820 /* 1821 * if this queue existed already, free the associated memory. The 1822 * queue cannot be reused in case we need to allocate memory on 1823 * different socket than was previously used. 1824 */ 1825 if (dev->data->rx_queues[queue_id] != NULL) { 1826 rx_queue_free(dev->data->rx_queues[queue_id]); 1827 dev->data->rx_queues[queue_id] = NULL; 1828 } 1829 1830 /* allocate memory for the queue structure */ 1831 q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE, 1832 socket_id); 1833 if (q == NULL) { 1834 PMD_INIT_LOG(ERR, "Cannot allocate queue structure"); 1835 return -ENOMEM; 1836 } 1837 1838 /* setup queue */ 1839 q->mp = mp; 1840 q->nb_desc = nb_desc; 1841 q->nb_fake_desc = FM10K_MULT_RX_DESC; 1842 q->port_id = dev->data->port_id; 1843 q->queue_id = queue_id; 1844 q->tail_ptr = (volatile uint32_t *) 1845 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)]; 1846 q->offloads = offloads; 1847 if (handle_rxconf(q, conf)) { 1848 rte_free(q); 1849 return -EINVAL; 1850 } 1851 /* allocate memory for the software ring */ 1852 q->sw_ring = rte_zmalloc_socket("fm10k sw ring", 1853 (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *), 1854 RTE_CACHE_LINE_SIZE, socket_id); 1855 if (q->sw_ring == NULL) { 1856 PMD_INIT_LOG(ERR, "Cannot allocate software ring"); 1857 rte_free(q); 1858 return -ENOMEM; 1859 } 1860 1861 /* 1862 * allocate memory for the hardware descriptor ring. A memzone large 1863 * enough to hold the maximum ring size is requested to allow for 1864 * resizing in later calls to the queue setup function. 1865 */ 1866 mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id, 1867 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC, 1868 socket_id); 1869 if (mz == NULL) { 1870 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring"); 1871 rte_free(q->sw_ring); 1872 rte_free(q); 1873 return -ENOMEM; 1874 } 1875 q->hw_ring = mz->addr; 1876 q->hw_ring_phys_addr = mz->iova; 1877 1878 /* Check if number of descs satisfied Vector requirement */ 1879 if (!rte_is_power_of_2(nb_desc)) { 1880 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx " 1881 "preconditions - canceling the feature for " 1882 "the whole port[%d]", 1883 q->queue_id, q->port_id); 1884 dev_info->rx_vec_allowed = false; 1885 } else 1886 fm10k_rxq_vec_setup(q); 1887 1888 dev->data->rx_queues[queue_id] = q; 1889 return 0; 1890 } 1891 1892 static void 1893 fm10k_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 1894 { 1895 PMD_INIT_FUNC_TRACE(); 1896 1897 rx_queue_free(dev->data->rx_queues[qid]); 1898 } 1899 1900 static inline int 1901 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf) 1902 { 1903 uint16_t tx_free_thresh; 1904 uint16_t tx_rs_thresh; 1905 1906 /* constraint MACROs require that tx_free_thresh is configured 1907 * before tx_rs_thresh */ 1908 if (conf->tx_free_thresh == 0) 1909 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q); 1910 else 1911 tx_free_thresh = conf->tx_free_thresh; 1912 1913 /* make sure the requested threshold satisfies the constraints */ 1914 if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q), 1915 FM10K_TX_FREE_THRESH_MAX(q), 1916 FM10K_TX_FREE_THRESH_DIV(q), 1917 tx_free_thresh)) { 1918 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be " 1919 "less than or equal to %u, " 1920 "greater than or equal to %u, " 1921 "and a divisor of %u", 1922 tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q), 1923 FM10K_TX_FREE_THRESH_MIN(q), 1924 FM10K_TX_FREE_THRESH_DIV(q)); 1925 return -EINVAL; 1926 } 1927 1928 q->free_thresh = tx_free_thresh; 1929 1930 if (conf->tx_rs_thresh == 0) 1931 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q); 1932 else 1933 tx_rs_thresh = conf->tx_rs_thresh; 1934 1935 q->tx_deferred_start = conf->tx_deferred_start; 1936 1937 /* make sure the requested threshold satisfies the constraints */ 1938 if (check_thresh(FM10K_TX_RS_THRESH_MIN(q), 1939 FM10K_TX_RS_THRESH_MAX(q), 1940 FM10K_TX_RS_THRESH_DIV(q), 1941 tx_rs_thresh)) { 1942 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be " 1943 "less than or equal to %u, " 1944 "greater than or equal to %u, " 1945 "and a divisor of %u", 1946 tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q), 1947 FM10K_TX_RS_THRESH_MIN(q), 1948 FM10K_TX_RS_THRESH_DIV(q)); 1949 return -EINVAL; 1950 } 1951 1952 q->rs_thresh = tx_rs_thresh; 1953 1954 return 0; 1955 } 1956 1957 static uint64_t fm10k_get_tx_queue_offloads_capa(struct rte_eth_dev *dev) 1958 { 1959 RTE_SET_USED(dev); 1960 1961 return 0; 1962 } 1963 1964 static uint64_t fm10k_get_tx_port_offloads_capa(struct rte_eth_dev *dev) 1965 { 1966 RTE_SET_USED(dev); 1967 1968 return (uint64_t)(DEV_TX_OFFLOAD_VLAN_INSERT | 1969 DEV_TX_OFFLOAD_MULTI_SEGS | 1970 DEV_TX_OFFLOAD_IPV4_CKSUM | 1971 DEV_TX_OFFLOAD_UDP_CKSUM | 1972 DEV_TX_OFFLOAD_TCP_CKSUM | 1973 DEV_TX_OFFLOAD_TCP_TSO); 1974 } 1975 1976 static int 1977 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, 1978 uint16_t nb_desc, unsigned int socket_id, 1979 const struct rte_eth_txconf *conf) 1980 { 1981 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1982 struct fm10k_tx_queue *q; 1983 const struct rte_memzone *mz; 1984 uint64_t offloads; 1985 1986 PMD_INIT_FUNC_TRACE(); 1987 1988 offloads = conf->offloads | dev->data->dev_conf.txmode.offloads; 1989 1990 /* make sure a valid number of descriptors have been requested */ 1991 if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC, 1992 FM10K_MULT_TX_DESC, nb_desc)) { 1993 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be " 1994 "less than or equal to %"PRIu32", " 1995 "greater than or equal to %u, " 1996 "and a multiple of %u", 1997 nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC, 1998 FM10K_MULT_TX_DESC); 1999 return -EINVAL; 2000 } 2001 2002 /* 2003 * if this queue existed already, free the associated memory. The 2004 * queue cannot be reused in case we need to allocate memory on 2005 * different socket than was previously used. 2006 */ 2007 if (dev->data->tx_queues[queue_id] != NULL) { 2008 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id]; 2009 2010 tx_queue_free(txq); 2011 dev->data->tx_queues[queue_id] = NULL; 2012 } 2013 2014 /* allocate memory for the queue structure */ 2015 q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE, 2016 socket_id); 2017 if (q == NULL) { 2018 PMD_INIT_LOG(ERR, "Cannot allocate queue structure"); 2019 return -ENOMEM; 2020 } 2021 2022 /* setup queue */ 2023 q->nb_desc = nb_desc; 2024 q->port_id = dev->data->port_id; 2025 q->queue_id = queue_id; 2026 q->offloads = offloads; 2027 q->ops = &def_txq_ops; 2028 q->tail_ptr = (volatile uint32_t *) 2029 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)]; 2030 if (handle_txconf(q, conf)) { 2031 rte_free(q); 2032 return -EINVAL; 2033 } 2034 2035 /* allocate memory for the software ring */ 2036 q->sw_ring = rte_zmalloc_socket("fm10k sw ring", 2037 nb_desc * sizeof(struct rte_mbuf *), 2038 RTE_CACHE_LINE_SIZE, socket_id); 2039 if (q->sw_ring == NULL) { 2040 PMD_INIT_LOG(ERR, "Cannot allocate software ring"); 2041 rte_free(q); 2042 return -ENOMEM; 2043 } 2044 2045 /* 2046 * allocate memory for the hardware descriptor ring. A memzone large 2047 * enough to hold the maximum ring size is requested to allow for 2048 * resizing in later calls to the queue setup function. 2049 */ 2050 mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id, 2051 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC, 2052 socket_id); 2053 if (mz == NULL) { 2054 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring"); 2055 rte_free(q->sw_ring); 2056 rte_free(q); 2057 return -ENOMEM; 2058 } 2059 q->hw_ring = mz->addr; 2060 q->hw_ring_phys_addr = mz->iova; 2061 2062 /* 2063 * allocate memory for the RS bit tracker. Enough slots to hold the 2064 * descriptor index for each RS bit needing to be set are required. 2065 */ 2066 q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker", 2067 ((nb_desc + 1) / q->rs_thresh) * 2068 sizeof(uint16_t), 2069 RTE_CACHE_LINE_SIZE, socket_id); 2070 if (q->rs_tracker.list == NULL) { 2071 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker"); 2072 rte_free(q->sw_ring); 2073 rte_free(q); 2074 return -ENOMEM; 2075 } 2076 2077 dev->data->tx_queues[queue_id] = q; 2078 return 0; 2079 } 2080 2081 static void 2082 fm10k_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 2083 { 2084 struct fm10k_tx_queue *q = dev->data->tx_queues[qid]; 2085 PMD_INIT_FUNC_TRACE(); 2086 2087 tx_queue_free(q); 2088 } 2089 2090 static int 2091 fm10k_reta_update(struct rte_eth_dev *dev, 2092 struct rte_eth_rss_reta_entry64 *reta_conf, 2093 uint16_t reta_size) 2094 { 2095 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2096 uint16_t i, j, idx, shift; 2097 uint8_t mask; 2098 uint32_t reta; 2099 2100 PMD_INIT_FUNC_TRACE(); 2101 2102 if (reta_size > FM10K_MAX_RSS_INDICES) { 2103 PMD_INIT_LOG(ERR, "The size of hash lookup table configured " 2104 "(%d) doesn't match the number hardware can supported " 2105 "(%d)", reta_size, FM10K_MAX_RSS_INDICES); 2106 return -EINVAL; 2107 } 2108 2109 /* 2110 * Update Redirection Table RETA[n], n=0..31. The redirection table has 2111 * 128-entries in 32 registers 2112 */ 2113 for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) { 2114 idx = i / RTE_RETA_GROUP_SIZE; 2115 shift = i % RTE_RETA_GROUP_SIZE; 2116 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 2117 BIT_MASK_PER_UINT32); 2118 if (mask == 0) 2119 continue; 2120 2121 reta = 0; 2122 if (mask != BIT_MASK_PER_UINT32) 2123 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2)); 2124 2125 for (j = 0; j < CHARS_PER_UINT32; j++) { 2126 if (mask & (0x1 << j)) { 2127 if (mask != 0xF) 2128 reta &= ~(UINT8_MAX << CHAR_BIT * j); 2129 reta |= reta_conf[idx].reta[shift + j] << 2130 (CHAR_BIT * j); 2131 } 2132 } 2133 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta); 2134 } 2135 2136 return 0; 2137 } 2138 2139 static int 2140 fm10k_reta_query(struct rte_eth_dev *dev, 2141 struct rte_eth_rss_reta_entry64 *reta_conf, 2142 uint16_t reta_size) 2143 { 2144 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2145 uint16_t i, j, idx, shift; 2146 uint8_t mask; 2147 uint32_t reta; 2148 2149 PMD_INIT_FUNC_TRACE(); 2150 2151 if (reta_size < FM10K_MAX_RSS_INDICES) { 2152 PMD_INIT_LOG(ERR, "The size of hash lookup table configured " 2153 "(%d) doesn't match the number hardware can supported " 2154 "(%d)", reta_size, FM10K_MAX_RSS_INDICES); 2155 return -EINVAL; 2156 } 2157 2158 /* 2159 * Read Redirection Table RETA[n], n=0..31. The redirection table has 2160 * 128-entries in 32 registers 2161 */ 2162 for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) { 2163 idx = i / RTE_RETA_GROUP_SIZE; 2164 shift = i % RTE_RETA_GROUP_SIZE; 2165 mask = (uint8_t)((reta_conf[idx].mask >> shift) & 2166 BIT_MASK_PER_UINT32); 2167 if (mask == 0) 2168 continue; 2169 2170 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2)); 2171 for (j = 0; j < CHARS_PER_UINT32; j++) { 2172 if (mask & (0x1 << j)) 2173 reta_conf[idx].reta[shift + j] = ((reta >> 2174 CHAR_BIT * j) & UINT8_MAX); 2175 } 2176 } 2177 2178 return 0; 2179 } 2180 2181 static int 2182 fm10k_rss_hash_update(struct rte_eth_dev *dev, 2183 struct rte_eth_rss_conf *rss_conf) 2184 { 2185 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2186 uint32_t *key = (uint32_t *)rss_conf->rss_key; 2187 uint32_t mrqc; 2188 uint64_t hf = rss_conf->rss_hf; 2189 int i; 2190 2191 PMD_INIT_FUNC_TRACE(); 2192 2193 if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE * 2194 FM10K_RSSRK_ENTRIES_PER_REG)) 2195 return -EINVAL; 2196 2197 if (hf == 0) 2198 return -EINVAL; 2199 2200 mrqc = 0; 2201 mrqc |= (hf & ETH_RSS_IPV4) ? FM10K_MRQC_IPV4 : 0; 2202 mrqc |= (hf & ETH_RSS_IPV6) ? FM10K_MRQC_IPV6 : 0; 2203 mrqc |= (hf & ETH_RSS_IPV6_EX) ? FM10K_MRQC_IPV6 : 0; 2204 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP) ? FM10K_MRQC_TCP_IPV4 : 0; 2205 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP) ? FM10K_MRQC_TCP_IPV6 : 0; 2206 mrqc |= (hf & ETH_RSS_IPV6_TCP_EX) ? FM10K_MRQC_TCP_IPV6 : 0; 2207 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP) ? FM10K_MRQC_UDP_IPV4 : 0; 2208 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP) ? FM10K_MRQC_UDP_IPV6 : 0; 2209 mrqc |= (hf & ETH_RSS_IPV6_UDP_EX) ? FM10K_MRQC_UDP_IPV6 : 0; 2210 2211 /* If the mapping doesn't fit any supported, return */ 2212 if (mrqc == 0) 2213 return -EINVAL; 2214 2215 if (key != NULL) 2216 for (i = 0; i < FM10K_RSSRK_SIZE; ++i) 2217 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]); 2218 2219 FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc); 2220 2221 return 0; 2222 } 2223 2224 static int 2225 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev, 2226 struct rte_eth_rss_conf *rss_conf) 2227 { 2228 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2229 uint32_t *key = (uint32_t *)rss_conf->rss_key; 2230 uint32_t mrqc; 2231 uint64_t hf; 2232 int i; 2233 2234 PMD_INIT_FUNC_TRACE(); 2235 2236 if (key && (rss_conf->rss_key_len < FM10K_RSSRK_SIZE * 2237 FM10K_RSSRK_ENTRIES_PER_REG)) 2238 return -EINVAL; 2239 2240 if (key != NULL) 2241 for (i = 0; i < FM10K_RSSRK_SIZE; ++i) 2242 key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i)); 2243 2244 mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0)); 2245 hf = 0; 2246 hf |= (mrqc & FM10K_MRQC_IPV4) ? ETH_RSS_IPV4 : 0; 2247 hf |= (mrqc & FM10K_MRQC_IPV6) ? ETH_RSS_IPV6 : 0; 2248 hf |= (mrqc & FM10K_MRQC_IPV6) ? ETH_RSS_IPV6_EX : 0; 2249 hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP : 0; 2250 hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP : 0; 2251 hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX : 0; 2252 hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP : 0; 2253 hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP : 0; 2254 hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX : 0; 2255 2256 rss_conf->rss_hf = hf; 2257 2258 return 0; 2259 } 2260 2261 static void 2262 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev) 2263 { 2264 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2265 uint32_t int_map = FM10K_INT_MAP_IMMEDIATE; 2266 2267 /* Bind all local non-queue interrupt to vector 0 */ 2268 int_map |= FM10K_MISC_VEC_ID; 2269 2270 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map); 2271 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map); 2272 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map); 2273 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map); 2274 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map); 2275 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map); 2276 2277 /* Enable misc causes */ 2278 FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) | 2279 FM10K_EIMR_ENABLE(THI_FAULT) | 2280 FM10K_EIMR_ENABLE(FUM_FAULT) | 2281 FM10K_EIMR_ENABLE(MAILBOX) | 2282 FM10K_EIMR_ENABLE(SWITCHREADY) | 2283 FM10K_EIMR_ENABLE(SWITCHNOTREADY) | 2284 FM10K_EIMR_ENABLE(SRAMERROR) | 2285 FM10K_EIMR_ENABLE(VFLR)); 2286 2287 /* Enable ITR 0 */ 2288 FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK | 2289 FM10K_ITR_MASK_CLEAR); 2290 FM10K_WRITE_FLUSH(hw); 2291 } 2292 2293 static void 2294 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev) 2295 { 2296 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2297 uint32_t int_map = FM10K_INT_MAP_DISABLE; 2298 2299 int_map |= FM10K_MISC_VEC_ID; 2300 2301 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map); 2302 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map); 2303 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map); 2304 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map); 2305 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map); 2306 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map); 2307 2308 /* Disable misc causes */ 2309 FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) | 2310 FM10K_EIMR_DISABLE(THI_FAULT) | 2311 FM10K_EIMR_DISABLE(FUM_FAULT) | 2312 FM10K_EIMR_DISABLE(MAILBOX) | 2313 FM10K_EIMR_DISABLE(SWITCHREADY) | 2314 FM10K_EIMR_DISABLE(SWITCHNOTREADY) | 2315 FM10K_EIMR_DISABLE(SRAMERROR) | 2316 FM10K_EIMR_DISABLE(VFLR)); 2317 2318 /* Disable ITR 0 */ 2319 FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET); 2320 FM10K_WRITE_FLUSH(hw); 2321 } 2322 2323 static void 2324 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev) 2325 { 2326 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2327 uint32_t int_map = FM10K_INT_MAP_IMMEDIATE; 2328 2329 /* Bind all local non-queue interrupt to vector 0 */ 2330 int_map |= FM10K_MISC_VEC_ID; 2331 2332 /* Only INT 0 available, other 15 are reserved. */ 2333 FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map); 2334 2335 /* Enable ITR 0 */ 2336 FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK | 2337 FM10K_ITR_MASK_CLEAR); 2338 FM10K_WRITE_FLUSH(hw); 2339 } 2340 2341 static void 2342 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev) 2343 { 2344 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2345 uint32_t int_map = FM10K_INT_MAP_DISABLE; 2346 2347 int_map |= FM10K_MISC_VEC_ID; 2348 2349 /* Only INT 0 available, other 15 are reserved. */ 2350 FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map); 2351 2352 /* Disable ITR 0 */ 2353 FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET); 2354 FM10K_WRITE_FLUSH(hw); 2355 } 2356 2357 static int 2358 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 2359 { 2360 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2361 struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); 2362 2363 /* Enable ITR */ 2364 if (hw->mac.type == fm10k_mac_pf) 2365 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)), 2366 FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR); 2367 else 2368 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)), 2369 FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR); 2370 rte_intr_ack(&pdev->intr_handle); 2371 return 0; 2372 } 2373 2374 static int 2375 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 2376 { 2377 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2378 struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); 2379 2380 /* Disable ITR */ 2381 if (hw->mac.type == fm10k_mac_pf) 2382 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(pdev, queue_id)), 2383 FM10K_ITR_MASK_SET); 2384 else 2385 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(pdev, queue_id)), 2386 FM10K_ITR_MASK_SET); 2387 return 0; 2388 } 2389 2390 static int 2391 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev) 2392 { 2393 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2394 struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); 2395 struct rte_intr_handle *intr_handle = &pdev->intr_handle; 2396 uint32_t intr_vector, vec; 2397 uint16_t queue_id; 2398 int result = 0; 2399 2400 /* fm10k needs one separate interrupt for mailbox, 2401 * so only drivers which support multiple interrupt vectors 2402 * e.g. vfio-pci can work for fm10k interrupt mode 2403 */ 2404 if (!rte_intr_cap_multiple(intr_handle) || 2405 dev->data->dev_conf.intr_conf.rxq == 0) 2406 return result; 2407 2408 intr_vector = dev->data->nb_rx_queues; 2409 2410 /* disable interrupt first */ 2411 rte_intr_disable(intr_handle); 2412 if (hw->mac.type == fm10k_mac_pf) 2413 fm10k_dev_disable_intr_pf(dev); 2414 else 2415 fm10k_dev_disable_intr_vf(dev); 2416 2417 if (rte_intr_efd_enable(intr_handle, intr_vector)) { 2418 PMD_INIT_LOG(ERR, "Failed to init event fd"); 2419 result = -EIO; 2420 } 2421 2422 if (rte_intr_dp_is_en(intr_handle) && !result) { 2423 intr_handle->intr_vec = rte_zmalloc("intr_vec", 2424 dev->data->nb_rx_queues * sizeof(int), 0); 2425 if (intr_handle->intr_vec) { 2426 for (queue_id = 0, vec = FM10K_RX_VEC_START; 2427 queue_id < dev->data->nb_rx_queues; 2428 queue_id++) { 2429 intr_handle->intr_vec[queue_id] = vec; 2430 if (vec < intr_handle->nb_efd - 1 2431 + FM10K_RX_VEC_START) 2432 vec++; 2433 } 2434 } else { 2435 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 2436 " intr_vec", dev->data->nb_rx_queues); 2437 rte_intr_efd_disable(intr_handle); 2438 result = -ENOMEM; 2439 } 2440 } 2441 2442 if (hw->mac.type == fm10k_mac_pf) 2443 fm10k_dev_enable_intr_pf(dev); 2444 else 2445 fm10k_dev_enable_intr_vf(dev); 2446 rte_intr_enable(intr_handle); 2447 hw->mac.ops.update_int_moderator(hw); 2448 return result; 2449 } 2450 2451 static int 2452 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr) 2453 { 2454 struct fm10k_fault fault; 2455 int err; 2456 const char *estr = "Unknown error"; 2457 2458 /* Process PCA fault */ 2459 if (eicr & FM10K_EICR_PCA_FAULT) { 2460 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault); 2461 if (err) 2462 goto error; 2463 switch (fault.type) { 2464 case PCA_NO_FAULT: 2465 estr = "PCA_NO_FAULT"; break; 2466 case PCA_UNMAPPED_ADDR: 2467 estr = "PCA_UNMAPPED_ADDR"; break; 2468 case PCA_BAD_QACCESS_PF: 2469 estr = "PCA_BAD_QACCESS_PF"; break; 2470 case PCA_BAD_QACCESS_VF: 2471 estr = "PCA_BAD_QACCESS_VF"; break; 2472 case PCA_MALICIOUS_REQ: 2473 estr = "PCA_MALICIOUS_REQ"; break; 2474 case PCA_POISONED_TLP: 2475 estr = "PCA_POISONED_TLP"; break; 2476 case PCA_TLP_ABORT: 2477 estr = "PCA_TLP_ABORT"; break; 2478 default: 2479 goto error; 2480 } 2481 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x", 2482 estr, fault.func ? "VF" : "PF", fault.func, 2483 fault.address, fault.specinfo); 2484 } 2485 2486 /* Process THI fault */ 2487 if (eicr & FM10K_EICR_THI_FAULT) { 2488 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault); 2489 if (err) 2490 goto error; 2491 switch (fault.type) { 2492 case THI_NO_FAULT: 2493 estr = "THI_NO_FAULT"; break; 2494 case THI_MAL_DIS_Q_FAULT: 2495 estr = "THI_MAL_DIS_Q_FAULT"; break; 2496 default: 2497 goto error; 2498 } 2499 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x", 2500 estr, fault.func ? "VF" : "PF", fault.func, 2501 fault.address, fault.specinfo); 2502 } 2503 2504 /* Process FUM fault */ 2505 if (eicr & FM10K_EICR_FUM_FAULT) { 2506 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault); 2507 if (err) 2508 goto error; 2509 switch (fault.type) { 2510 case FUM_NO_FAULT: 2511 estr = "FUM_NO_FAULT"; break; 2512 case FUM_UNMAPPED_ADDR: 2513 estr = "FUM_UNMAPPED_ADDR"; break; 2514 case FUM_POISONED_TLP: 2515 estr = "FUM_POISONED_TLP"; break; 2516 case FUM_BAD_VF_QACCESS: 2517 estr = "FUM_BAD_VF_QACCESS"; break; 2518 case FUM_ADD_DECODE_ERR: 2519 estr = "FUM_ADD_DECODE_ERR"; break; 2520 case FUM_RO_ERROR: 2521 estr = "FUM_RO_ERROR"; break; 2522 case FUM_QPRC_CRC_ERROR: 2523 estr = "FUM_QPRC_CRC_ERROR"; break; 2524 case FUM_CSR_TIMEOUT: 2525 estr = "FUM_CSR_TIMEOUT"; break; 2526 case FUM_INVALID_TYPE: 2527 estr = "FUM_INVALID_TYPE"; break; 2528 case FUM_INVALID_LENGTH: 2529 estr = "FUM_INVALID_LENGTH"; break; 2530 case FUM_INVALID_BE: 2531 estr = "FUM_INVALID_BE"; break; 2532 case FUM_INVALID_ALIGN: 2533 estr = "FUM_INVALID_ALIGN"; break; 2534 default: 2535 goto error; 2536 } 2537 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x", 2538 estr, fault.func ? "VF" : "PF", fault.func, 2539 fault.address, fault.specinfo); 2540 } 2541 2542 return 0; 2543 error: 2544 PMD_INIT_LOG(ERR, "Failed to handle fault event."); 2545 return err; 2546 } 2547 2548 /** 2549 * PF interrupt handler triggered by NIC for handling specific interrupt. 2550 * 2551 * @param handle 2552 * Pointer to interrupt handle. 2553 * @param param 2554 * The address of parameter (struct rte_eth_dev *) regsitered before. 2555 * 2556 * @return 2557 * void 2558 */ 2559 static void 2560 fm10k_dev_interrupt_handler_pf(void *param) 2561 { 2562 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2563 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2564 uint32_t cause, status; 2565 struct fm10k_dev_info *dev_info = 2566 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private); 2567 int status_mbx; 2568 s32 err; 2569 2570 if (hw->mac.type != fm10k_mac_pf) 2571 return; 2572 2573 cause = FM10K_READ_REG(hw, FM10K_EICR); 2574 2575 /* Handle PCI fault cases */ 2576 if (cause & FM10K_EICR_FAULT_MASK) { 2577 PMD_INIT_LOG(ERR, "INT: find fault!"); 2578 fm10k_dev_handle_fault(hw, cause); 2579 } 2580 2581 /* Handle switch up/down */ 2582 if (cause & FM10K_EICR_SWITCHNOTREADY) 2583 PMD_INIT_LOG(ERR, "INT: Switch is not ready"); 2584 2585 if (cause & FM10K_EICR_SWITCHREADY) { 2586 PMD_INIT_LOG(INFO, "INT: Switch is ready"); 2587 if (dev_info->sm_down == 1) { 2588 fm10k_mbx_lock(hw); 2589 2590 /* For recreating logical ports */ 2591 status_mbx = hw->mac.ops.update_lport_state(hw, 2592 hw->mac.dglort_map, MAX_LPORT_NUM, 1); 2593 if (status_mbx == FM10K_SUCCESS) 2594 PMD_INIT_LOG(INFO, 2595 "INT: Recreated Logical port"); 2596 else 2597 PMD_INIT_LOG(INFO, 2598 "INT: Logical ports weren't recreated"); 2599 2600 status_mbx = hw->mac.ops.update_xcast_mode(hw, 2601 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE); 2602 if (status_mbx != FM10K_SUCCESS) 2603 PMD_INIT_LOG(ERR, "Failed to set XCAST mode"); 2604 2605 fm10k_mbx_unlock(hw); 2606 2607 /* first clear the internal SW recording structure */ 2608 if (!(dev->data->dev_conf.rxmode.mq_mode & 2609 ETH_MQ_RX_VMDQ_FLAG)) 2610 fm10k_vlan_filter_set(dev, hw->mac.default_vid, 2611 false); 2612 2613 fm10k_MAC_filter_set(dev, hw->mac.addr, false, 2614 MAIN_VSI_POOL_NUMBER); 2615 2616 /* 2617 * Add default mac address and vlan for the logical 2618 * ports that have been created, leave to the 2619 * application to fully recover Rx filtering. 2620 */ 2621 fm10k_MAC_filter_set(dev, hw->mac.addr, true, 2622 MAIN_VSI_POOL_NUMBER); 2623 2624 if (!(dev->data->dev_conf.rxmode.mq_mode & 2625 ETH_MQ_RX_VMDQ_FLAG)) 2626 fm10k_vlan_filter_set(dev, hw->mac.default_vid, 2627 true); 2628 2629 dev_info->sm_down = 0; 2630 rte_eth_dev_callback_process(dev, 2631 RTE_ETH_EVENT_INTR_LSC, 2632 NULL); 2633 } 2634 } 2635 2636 /* Handle mailbox message */ 2637 fm10k_mbx_lock(hw); 2638 err = hw->mbx.ops.process(hw, &hw->mbx); 2639 fm10k_mbx_unlock(hw); 2640 2641 if (err == FM10K_ERR_RESET_REQUESTED) { 2642 PMD_INIT_LOG(INFO, "INT: Switch is down"); 2643 dev_info->sm_down = 1; 2644 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); 2645 } 2646 2647 /* Handle SRAM error */ 2648 if (cause & FM10K_EICR_SRAMERROR) { 2649 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP"); 2650 2651 status = FM10K_READ_REG(hw, FM10K_SRAM_IP); 2652 /* Write to clear pending bits */ 2653 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status); 2654 2655 /* Todo: print out error message after shared code updates */ 2656 } 2657 2658 /* Clear these 3 events if having any */ 2659 cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX | 2660 FM10K_EICR_SWITCHREADY; 2661 if (cause) 2662 FM10K_WRITE_REG(hw, FM10K_EICR, cause); 2663 2664 /* Re-enable interrupt from device side */ 2665 FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK | 2666 FM10K_ITR_MASK_CLEAR); 2667 /* Re-enable interrupt from host side */ 2668 rte_intr_ack(dev->intr_handle); 2669 } 2670 2671 /** 2672 * VF interrupt handler triggered by NIC for handling specific interrupt. 2673 * 2674 * @param handle 2675 * Pointer to interrupt handle. 2676 * @param param 2677 * The address of parameter (struct rte_eth_dev *) regsitered before. 2678 * 2679 * @return 2680 * void 2681 */ 2682 static void 2683 fm10k_dev_interrupt_handler_vf(void *param) 2684 { 2685 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2686 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2687 struct fm10k_mbx_info *mbx = &hw->mbx; 2688 struct fm10k_dev_info *dev_info = 2689 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private); 2690 const enum fm10k_mbx_state state = mbx->state; 2691 int status_mbx; 2692 2693 if (hw->mac.type != fm10k_mac_vf) 2694 return; 2695 2696 /* Handle mailbox message if lock is acquired */ 2697 fm10k_mbx_lock(hw); 2698 hw->mbx.ops.process(hw, &hw->mbx); 2699 fm10k_mbx_unlock(hw); 2700 2701 if (state == FM10K_STATE_OPEN && mbx->state == FM10K_STATE_CONNECT) { 2702 PMD_INIT_LOG(INFO, "INT: Switch has gone down"); 2703 2704 fm10k_mbx_lock(hw); 2705 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 2706 MAX_LPORT_NUM, 1); 2707 fm10k_mbx_unlock(hw); 2708 2709 /* Setting reset flag */ 2710 dev_info->sm_down = 1; 2711 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); 2712 } 2713 2714 if (dev_info->sm_down == 1 && 2715 hw->mac.dglort_map == FM10K_DGLORTMAP_ZERO) { 2716 PMD_INIT_LOG(INFO, "INT: Switch has gone up"); 2717 fm10k_mbx_lock(hw); 2718 status_mbx = hw->mac.ops.update_xcast_mode(hw, 2719 hw->mac.dglort_map, FM10K_XCAST_MODE_NONE); 2720 if (status_mbx != FM10K_SUCCESS) 2721 PMD_INIT_LOG(ERR, "Failed to set XCAST mode"); 2722 fm10k_mbx_unlock(hw); 2723 2724 /* first clear the internal SW recording structure */ 2725 fm10k_vlan_filter_set(dev, hw->mac.default_vid, false); 2726 fm10k_MAC_filter_set(dev, hw->mac.addr, false, 2727 MAIN_VSI_POOL_NUMBER); 2728 2729 /* 2730 * Add default mac address and vlan for the logical ports that 2731 * have been created, leave to the application to fully recover 2732 * Rx filtering. 2733 */ 2734 fm10k_MAC_filter_set(dev, hw->mac.addr, true, 2735 MAIN_VSI_POOL_NUMBER); 2736 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true); 2737 2738 dev_info->sm_down = 0; 2739 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); 2740 } 2741 2742 /* Re-enable interrupt from device side */ 2743 FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK | 2744 FM10K_ITR_MASK_CLEAR); 2745 /* Re-enable interrupt from host side */ 2746 rte_intr_ack(dev->intr_handle); 2747 } 2748 2749 /* Mailbox message handler in VF */ 2750 static const struct fm10k_msg_data fm10k_msgdata_vf[] = { 2751 FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test), 2752 FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf), 2753 FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf), 2754 FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error), 2755 }; 2756 2757 static int 2758 fm10k_setup_mbx_service(struct fm10k_hw *hw) 2759 { 2760 int err = 0; 2761 2762 /* Initialize mailbox lock */ 2763 fm10k_mbx_initlock(hw); 2764 2765 /* Replace default message handler with new ones */ 2766 if (hw->mac.type == fm10k_mac_vf) 2767 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf); 2768 2769 if (err) { 2770 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d", 2771 err); 2772 return err; 2773 } 2774 /* Connect to SM for PF device or PF for VF device */ 2775 return hw->mbx.ops.connect(hw, &hw->mbx); 2776 } 2777 2778 static void 2779 fm10k_close_mbx_service(struct fm10k_hw *hw) 2780 { 2781 /* Disconnect from SM for PF device or PF for VF device */ 2782 hw->mbx.ops.disconnect(hw, &hw->mbx); 2783 } 2784 2785 static int 2786 fm10k_dev_close(struct rte_eth_dev *dev) 2787 { 2788 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2789 struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); 2790 struct rte_intr_handle *intr_handle = &pdev->intr_handle; 2791 int ret; 2792 2793 PMD_INIT_FUNC_TRACE(); 2794 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2795 return 0; 2796 2797 fm10k_mbx_lock(hw); 2798 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 2799 MAX_LPORT_NUM, false); 2800 fm10k_mbx_unlock(hw); 2801 2802 /* allow 100ms for device to quiesce */ 2803 rte_delay_us(FM10K_SWITCH_QUIESCE_US); 2804 2805 /* Stop mailbox service first */ 2806 fm10k_close_mbx_service(hw); 2807 2808 ret = fm10k_dev_stop(dev); 2809 2810 fm10k_dev_queue_release(dev); 2811 fm10k_stop_hw(hw); 2812 2813 /* disable uio/vfio intr */ 2814 rte_intr_disable(intr_handle); 2815 2816 /*PF/VF has different interrupt handling mechanism */ 2817 if (hw->mac.type == fm10k_mac_pf) { 2818 /* disable interrupt */ 2819 fm10k_dev_disable_intr_pf(dev); 2820 2821 /* unregister callback func to eal lib */ 2822 rte_intr_callback_unregister(intr_handle, 2823 fm10k_dev_interrupt_handler_pf, (void *)dev); 2824 } else { 2825 /* disable interrupt */ 2826 fm10k_dev_disable_intr_vf(dev); 2827 2828 rte_intr_callback_unregister(intr_handle, 2829 fm10k_dev_interrupt_handler_vf, (void *)dev); 2830 } 2831 2832 return ret; 2833 } 2834 2835 static const struct eth_dev_ops fm10k_eth_dev_ops = { 2836 .dev_configure = fm10k_dev_configure, 2837 .dev_start = fm10k_dev_start, 2838 .dev_stop = fm10k_dev_stop, 2839 .dev_close = fm10k_dev_close, 2840 .promiscuous_enable = fm10k_dev_promiscuous_enable, 2841 .promiscuous_disable = fm10k_dev_promiscuous_disable, 2842 .allmulticast_enable = fm10k_dev_allmulticast_enable, 2843 .allmulticast_disable = fm10k_dev_allmulticast_disable, 2844 .stats_get = fm10k_stats_get, 2845 .xstats_get = fm10k_xstats_get, 2846 .xstats_get_names = fm10k_xstats_get_names, 2847 .stats_reset = fm10k_stats_reset, 2848 .xstats_reset = fm10k_stats_reset, 2849 .link_update = fm10k_link_update, 2850 .dev_infos_get = fm10k_dev_infos_get, 2851 .dev_supported_ptypes_get = fm10k_dev_supported_ptypes_get, 2852 .vlan_filter_set = fm10k_vlan_filter_set, 2853 .vlan_offload_set = fm10k_vlan_offload_set, 2854 .mac_addr_add = fm10k_macaddr_add, 2855 .mac_addr_remove = fm10k_macaddr_remove, 2856 .rx_queue_start = fm10k_dev_rx_queue_start, 2857 .rx_queue_stop = fm10k_dev_rx_queue_stop, 2858 .tx_queue_start = fm10k_dev_tx_queue_start, 2859 .tx_queue_stop = fm10k_dev_tx_queue_stop, 2860 .rx_queue_setup = fm10k_rx_queue_setup, 2861 .rx_queue_release = fm10k_rx_queue_release, 2862 .tx_queue_setup = fm10k_tx_queue_setup, 2863 .tx_queue_release = fm10k_tx_queue_release, 2864 .rx_queue_intr_enable = fm10k_dev_rx_queue_intr_enable, 2865 .rx_queue_intr_disable = fm10k_dev_rx_queue_intr_disable, 2866 .reta_update = fm10k_reta_update, 2867 .reta_query = fm10k_reta_query, 2868 .rss_hash_update = fm10k_rss_hash_update, 2869 .rss_hash_conf_get = fm10k_rss_hash_conf_get, 2870 }; 2871 2872 static int ftag_check_handler(__rte_unused const char *key, 2873 const char *value, __rte_unused void *opaque) 2874 { 2875 if (strcmp(value, "1")) 2876 return -1; 2877 2878 return 0; 2879 } 2880 2881 static int 2882 fm10k_check_ftag(struct rte_devargs *devargs) 2883 { 2884 struct rte_kvargs *kvlist; 2885 const char *ftag_key = "enable_ftag"; 2886 2887 if (devargs == NULL) 2888 return 0; 2889 2890 kvlist = rte_kvargs_parse(devargs->args, NULL); 2891 if (kvlist == NULL) 2892 return 0; 2893 2894 if (!rte_kvargs_count(kvlist, ftag_key)) { 2895 rte_kvargs_free(kvlist); 2896 return 0; 2897 } 2898 /* FTAG is enabled when there's key-value pair: enable_ftag=1 */ 2899 if (rte_kvargs_process(kvlist, ftag_key, 2900 ftag_check_handler, NULL) < 0) { 2901 rte_kvargs_free(kvlist); 2902 return 0; 2903 } 2904 rte_kvargs_free(kvlist); 2905 2906 return 1; 2907 } 2908 2909 static uint16_t 2910 fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, 2911 uint16_t nb_pkts) 2912 { 2913 uint16_t nb_tx = 0; 2914 struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue; 2915 2916 while (nb_pkts) { 2917 uint16_t ret, num; 2918 2919 num = (uint16_t)RTE_MIN(nb_pkts, txq->rs_thresh); 2920 ret = fm10k_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx], 2921 num); 2922 nb_tx += ret; 2923 nb_pkts -= ret; 2924 if (ret < num) 2925 break; 2926 } 2927 2928 return nb_tx; 2929 } 2930 2931 static void __rte_cold 2932 fm10k_set_tx_function(struct rte_eth_dev *dev) 2933 { 2934 struct fm10k_tx_queue *txq; 2935 int i; 2936 int use_sse = 1; 2937 uint16_t tx_ftag_en = 0; 2938 2939 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 2940 /* primary process has set the ftag flag and offloads */ 2941 txq = dev->data->tx_queues[0]; 2942 if (fm10k_tx_vec_condition_check(txq) || 2943 rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) { 2944 dev->tx_pkt_burst = fm10k_xmit_pkts; 2945 dev->tx_pkt_prepare = fm10k_prep_pkts; 2946 PMD_INIT_LOG(DEBUG, "Use regular Tx func"); 2947 } else { 2948 PMD_INIT_LOG(DEBUG, "Use vector Tx func"); 2949 dev->tx_pkt_burst = fm10k_xmit_pkts_vec; 2950 dev->tx_pkt_prepare = NULL; 2951 } 2952 return; 2953 } 2954 2955 if (fm10k_check_ftag(dev->device->devargs)) 2956 tx_ftag_en = 1; 2957 2958 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2959 txq = dev->data->tx_queues[i]; 2960 txq->tx_ftag_en = tx_ftag_en; 2961 /* Check if Vector Tx is satisfied */ 2962 if (fm10k_tx_vec_condition_check(txq) || 2963 rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) 2964 use_sse = 0; 2965 } 2966 2967 if (use_sse) { 2968 PMD_INIT_LOG(DEBUG, "Use vector Tx func"); 2969 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2970 txq = dev->data->tx_queues[i]; 2971 fm10k_txq_vec_setup(txq); 2972 } 2973 dev->tx_pkt_burst = fm10k_xmit_pkts_vec; 2974 dev->tx_pkt_prepare = NULL; 2975 } else { 2976 dev->tx_pkt_burst = fm10k_xmit_pkts; 2977 dev->tx_pkt_prepare = fm10k_prep_pkts; 2978 PMD_INIT_LOG(DEBUG, "Use regular Tx func"); 2979 } 2980 } 2981 2982 static void __rte_cold 2983 fm10k_set_rx_function(struct rte_eth_dev *dev) 2984 { 2985 struct fm10k_dev_info *dev_info = 2986 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private); 2987 uint16_t i, rx_using_sse; 2988 uint16_t rx_ftag_en = 0; 2989 2990 if (fm10k_check_ftag(dev->device->devargs)) 2991 rx_ftag_en = 1; 2992 2993 /* In order to allow Vector Rx there are a few configuration 2994 * conditions to be met. 2995 */ 2996 if (!fm10k_rx_vec_condition_check(dev) && 2997 dev_info->rx_vec_allowed && !rx_ftag_en && 2998 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) { 2999 if (dev->data->scattered_rx) 3000 dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec; 3001 else 3002 dev->rx_pkt_burst = fm10k_recv_pkts_vec; 3003 } else if (dev->data->scattered_rx) 3004 dev->rx_pkt_burst = fm10k_recv_scattered_pkts; 3005 else 3006 dev->rx_pkt_burst = fm10k_recv_pkts; 3007 3008 rx_using_sse = 3009 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec || 3010 dev->rx_pkt_burst == fm10k_recv_pkts_vec); 3011 3012 if (rx_using_sse) 3013 PMD_INIT_LOG(DEBUG, "Use vector Rx func"); 3014 else 3015 PMD_INIT_LOG(DEBUG, "Use regular Rx func"); 3016 3017 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 3018 return; 3019 3020 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3021 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i]; 3022 3023 rxq->rx_using_sse = rx_using_sse; 3024 rxq->rx_ftag_en = rx_ftag_en; 3025 } 3026 } 3027 3028 static void 3029 fm10k_params_init(struct rte_eth_dev *dev) 3030 { 3031 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3032 struct fm10k_dev_info *info = 3033 FM10K_DEV_PRIVATE_TO_INFO(dev->data->dev_private); 3034 3035 /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but 3036 * there is no way to get link status without reading BAR4. Until this 3037 * works, assume we have maximum bandwidth. 3038 * @todo - fix bus info 3039 */ 3040 hw->bus_caps.speed = fm10k_bus_speed_8000; 3041 hw->bus_caps.width = fm10k_bus_width_pcie_x8; 3042 hw->bus_caps.payload = fm10k_bus_payload_512; 3043 hw->bus.speed = fm10k_bus_speed_8000; 3044 hw->bus.width = fm10k_bus_width_pcie_x8; 3045 hw->bus.payload = fm10k_bus_payload_256; 3046 3047 info->rx_vec_allowed = true; 3048 info->sm_down = false; 3049 } 3050 3051 static int 3052 eth_fm10k_dev_init(struct rte_eth_dev *dev) 3053 { 3054 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3055 struct rte_pci_device *pdev = RTE_ETH_DEV_TO_PCI(dev); 3056 struct rte_intr_handle *intr_handle = &pdev->intr_handle; 3057 int diag, i; 3058 struct fm10k_macvlan_filter_info *macvlan; 3059 3060 PMD_INIT_FUNC_TRACE(); 3061 3062 dev->dev_ops = &fm10k_eth_dev_ops; 3063 dev->rx_queue_count = fm10k_dev_rx_queue_count; 3064 dev->rx_descriptor_status = fm10k_dev_rx_descriptor_status; 3065 dev->tx_descriptor_status = fm10k_dev_tx_descriptor_status; 3066 dev->rx_pkt_burst = &fm10k_recv_pkts; 3067 dev->tx_pkt_burst = &fm10k_xmit_pkts; 3068 dev->tx_pkt_prepare = &fm10k_prep_pkts; 3069 3070 /* 3071 * Primary process does the whole initialization, for secondary 3072 * processes, we just select the same Rx and Tx function as primary. 3073 */ 3074 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 3075 fm10k_set_rx_function(dev); 3076 fm10k_set_tx_function(dev); 3077 return 0; 3078 } 3079 3080 rte_eth_copy_pci_info(dev, pdev); 3081 dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 3082 3083 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private); 3084 memset(macvlan, 0, sizeof(*macvlan)); 3085 /* Vendor and Device ID need to be set before init of shared code */ 3086 memset(hw, 0, sizeof(*hw)); 3087 hw->device_id = pdev->id.device_id; 3088 hw->vendor_id = pdev->id.vendor_id; 3089 hw->subsystem_device_id = pdev->id.subsystem_device_id; 3090 hw->subsystem_vendor_id = pdev->id.subsystem_vendor_id; 3091 hw->revision_id = 0; 3092 hw->hw_addr = (void *)pdev->mem_resource[0].addr; 3093 if (hw->hw_addr == NULL) { 3094 PMD_INIT_LOG(ERR, "Bad mem resource." 3095 " Try to refuse unused devices."); 3096 return -EIO; 3097 } 3098 3099 /* Store fm10k_adapter pointer */ 3100 hw->back = dev->data->dev_private; 3101 3102 /* Initialize the shared code */ 3103 diag = fm10k_init_shared_code(hw); 3104 if (diag != FM10K_SUCCESS) { 3105 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag); 3106 return -EIO; 3107 } 3108 3109 /* Initialize parameters */ 3110 fm10k_params_init(dev); 3111 3112 /* Initialize the hw */ 3113 diag = fm10k_init_hw(hw); 3114 if (diag != FM10K_SUCCESS) { 3115 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag); 3116 return -EIO; 3117 } 3118 3119 /* Initialize MAC address(es) */ 3120 dev->data->mac_addrs = rte_zmalloc("fm10k", 3121 RTE_ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0); 3122 if (dev->data->mac_addrs == NULL) { 3123 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses"); 3124 return -ENOMEM; 3125 } 3126 3127 diag = fm10k_read_mac_addr(hw); 3128 3129 rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr, 3130 &dev->data->mac_addrs[0]); 3131 3132 if (diag != FM10K_SUCCESS || 3133 !rte_is_valid_assigned_ether_addr(dev->data->mac_addrs)) { 3134 3135 /* Generate a random addr */ 3136 rte_eth_random_addr(hw->mac.addr); 3137 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN); 3138 rte_ether_addr_copy((const struct rte_ether_addr *)hw->mac.addr, 3139 &dev->data->mac_addrs[0]); 3140 } 3141 3142 /* Reset the hw statistics */ 3143 diag = fm10k_stats_reset(dev); 3144 if (diag != 0) { 3145 PMD_INIT_LOG(ERR, "Stats reset failed: %d", diag); 3146 return diag; 3147 } 3148 3149 /* Reset the hw */ 3150 diag = fm10k_reset_hw(hw); 3151 if (diag != FM10K_SUCCESS) { 3152 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag); 3153 return -EIO; 3154 } 3155 3156 /* Setup mailbox service */ 3157 diag = fm10k_setup_mbx_service(hw); 3158 if (diag != FM10K_SUCCESS) { 3159 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag); 3160 return -EIO; 3161 } 3162 3163 /*PF/VF has different interrupt handling mechanism */ 3164 if (hw->mac.type == fm10k_mac_pf) { 3165 /* register callback func to eal lib */ 3166 rte_intr_callback_register(intr_handle, 3167 fm10k_dev_interrupt_handler_pf, (void *)dev); 3168 3169 /* enable MISC interrupt */ 3170 fm10k_dev_enable_intr_pf(dev); 3171 } else { /* VF */ 3172 rte_intr_callback_register(intr_handle, 3173 fm10k_dev_interrupt_handler_vf, (void *)dev); 3174 3175 fm10k_dev_enable_intr_vf(dev); 3176 } 3177 3178 /* Enable intr after callback registered */ 3179 rte_intr_enable(intr_handle); 3180 3181 hw->mac.ops.update_int_moderator(hw); 3182 3183 /* Make sure Switch Manager is ready before going forward. */ 3184 if (hw->mac.type == fm10k_mac_pf) { 3185 bool switch_ready = false; 3186 3187 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) { 3188 fm10k_mbx_lock(hw); 3189 hw->mac.ops.get_host_state(hw, &switch_ready); 3190 fm10k_mbx_unlock(hw); 3191 if (switch_ready == true) 3192 break; 3193 /* Delay some time to acquire async LPORT_MAP info. */ 3194 rte_delay_us(WAIT_SWITCH_MSG_US); 3195 } 3196 3197 if (switch_ready == false) { 3198 PMD_INIT_LOG(ERR, "switch is not ready"); 3199 return -1; 3200 } 3201 } 3202 3203 /* 3204 * Below function will trigger operations on mailbox, acquire lock to 3205 * avoid race condition from interrupt handler. Operations on mailbox 3206 * FIFO will trigger interrupt to PF/SM, in which interrupt handler 3207 * will handle and generate an interrupt to our side. Then, FIFO in 3208 * mailbox will be touched. 3209 */ 3210 fm10k_mbx_lock(hw); 3211 /* Enable port first */ 3212 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 3213 MAX_LPORT_NUM, 1); 3214 3215 /* Set unicast mode by default. App can change to other mode in other 3216 * API func. 3217 */ 3218 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map, 3219 FM10K_XCAST_MODE_NONE); 3220 3221 fm10k_mbx_unlock(hw); 3222 3223 /* Make sure default VID is ready before going forward. */ 3224 if (hw->mac.type == fm10k_mac_pf) { 3225 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) { 3226 if (hw->mac.default_vid) 3227 break; 3228 /* Delay some time to acquire async port VLAN info. */ 3229 rte_delay_us(WAIT_SWITCH_MSG_US); 3230 } 3231 3232 if (!hw->mac.default_vid) { 3233 PMD_INIT_LOG(ERR, "default VID is not ready"); 3234 return -1; 3235 } 3236 } 3237 3238 /* Add default mac address */ 3239 fm10k_MAC_filter_set(dev, hw->mac.addr, true, 3240 MAIN_VSI_POOL_NUMBER); 3241 3242 return 0; 3243 } 3244 3245 static int 3246 eth_fm10k_dev_uninit(struct rte_eth_dev *dev) 3247 { 3248 PMD_INIT_FUNC_TRACE(); 3249 fm10k_dev_close(dev); 3250 return 0; 3251 } 3252 3253 static int eth_fm10k_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 3254 struct rte_pci_device *pci_dev) 3255 { 3256 return rte_eth_dev_pci_generic_probe(pci_dev, 3257 sizeof(struct fm10k_adapter), eth_fm10k_dev_init); 3258 } 3259 3260 static int eth_fm10k_pci_remove(struct rte_pci_device *pci_dev) 3261 { 3262 return rte_eth_dev_pci_generic_remove(pci_dev, eth_fm10k_dev_uninit); 3263 } 3264 3265 /* 3266 * The set of PCI devices this driver supports. This driver will enable both PF 3267 * and SRIOV-VF devices. 3268 */ 3269 static const struct rte_pci_id pci_id_fm10k_map[] = { 3270 { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_PF) }, 3271 { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_SDI_FM10420_QDA2) }, 3272 { RTE_PCI_DEVICE(FM10K_INTEL_VENDOR_ID, FM10K_DEV_ID_VF) }, 3273 { .vendor_id = 0, /* sentinel */ }, 3274 }; 3275 3276 static struct rte_pci_driver rte_pmd_fm10k = { 3277 .id_table = pci_id_fm10k_map, 3278 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 3279 .probe = eth_fm10k_pci_probe, 3280 .remove = eth_fm10k_pci_remove, 3281 }; 3282 3283 RTE_PMD_REGISTER_PCI(net_fm10k, rte_pmd_fm10k); 3284 RTE_PMD_REGISTER_PCI_TABLE(net_fm10k, pci_id_fm10k_map); 3285 RTE_PMD_REGISTER_KMOD_DEP(net_fm10k, "* igb_uio | uio_pci_generic | vfio-pci"); 3286 RTE_LOG_REGISTER_SUFFIX(fm10k_logtype_init, init, NOTICE); 3287 RTE_LOG_REGISTER_SUFFIX(fm10k_logtype_driver, driver, NOTICE); 3288 #ifdef RTE_ETHDEV_DEBUG_RX 3289 RTE_LOG_REGISTER_SUFFIX(fm10k_logtype_rx, rx, DEBUG); 3290 #endif 3291 #ifdef RTE_ETHDEV_DEBUG_TX 3292 RTE_LOG_REGISTER_SUFFIX(fm10k_logtype_tx, tx, DEBUG); 3293 #endif 3294