1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2015-2020 3 */ 4 5 #include <sys/queue.h> 6 7 #include <stdio.h> 8 #include <stdlib.h> 9 #include <string.h> 10 #include <errno.h> 11 #include <stdint.h> 12 #include <stdarg.h> 13 #include <unistd.h> 14 #include <inttypes.h> 15 16 #include <rte_byteorder.h> 17 #include <rte_common.h> 18 #include <rte_cycles.h> 19 #include <rte_log.h> 20 #include <rte_debug.h> 21 #include <rte_ethdev.h> 22 #include <ethdev_driver.h> 23 #include <rte_security_driver.h> 24 #include <rte_memzone.h> 25 #include <rte_atomic.h> 26 #include <rte_mempool.h> 27 #include <rte_malloc.h> 28 #include <rte_mbuf.h> 29 #include <rte_ether.h> 30 #include <rte_prefetch.h> 31 #include <rte_udp.h> 32 #include <rte_tcp.h> 33 #include <rte_sctp.h> 34 #include <rte_string_fns.h> 35 #include <rte_errno.h> 36 #include <rte_ip.h> 37 #include <rte_net.h> 38 39 #include "txgbe_logs.h" 40 #include "base/txgbe.h" 41 #include "txgbe_ethdev.h" 42 #include "txgbe_rxtx.h" 43 44 #ifdef RTE_LIBRTE_IEEE1588 45 #define TXGBE_TX_IEEE1588_TMST PKT_TX_IEEE1588_TMST 46 #else 47 #define TXGBE_TX_IEEE1588_TMST 0 48 #endif 49 50 /* Bit Mask to indicate what bits required for building TX context */ 51 static const u64 TXGBE_TX_OFFLOAD_MASK = (PKT_TX_IP_CKSUM | 52 PKT_TX_OUTER_IPV6 | 53 PKT_TX_OUTER_IPV4 | 54 PKT_TX_IPV6 | 55 PKT_TX_IPV4 | 56 PKT_TX_VLAN_PKT | 57 PKT_TX_L4_MASK | 58 PKT_TX_TCP_SEG | 59 PKT_TX_TUNNEL_MASK | 60 PKT_TX_OUTER_IP_CKSUM | 61 #ifdef RTE_LIB_SECURITY 62 PKT_TX_SEC_OFFLOAD | 63 #endif 64 TXGBE_TX_IEEE1588_TMST); 65 66 #define TXGBE_TX_OFFLOAD_NOTSUP_MASK \ 67 (PKT_TX_OFFLOAD_MASK ^ TXGBE_TX_OFFLOAD_MASK) 68 69 /* 70 * Prefetch a cache line into all cache levels. 71 */ 72 #define rte_txgbe_prefetch(p) rte_prefetch0(p) 73 74 static int 75 txgbe_is_vf(struct rte_eth_dev *dev) 76 { 77 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 78 79 switch (hw->mac.type) { 80 case txgbe_mac_raptor_vf: 81 return 1; 82 default: 83 return 0; 84 } 85 } 86 87 /********************************************************************* 88 * 89 * TX functions 90 * 91 **********************************************************************/ 92 93 /* 94 * Check for descriptors with their DD bit set and free mbufs. 95 * Return the total number of buffers freed. 96 */ 97 static __rte_always_inline int 98 txgbe_tx_free_bufs(struct txgbe_tx_queue *txq) 99 { 100 struct txgbe_tx_entry *txep; 101 uint32_t status; 102 int i, nb_free = 0; 103 struct rte_mbuf *m, *free[RTE_TXGBE_TX_MAX_FREE_BUF_SZ]; 104 105 /* check DD bit on threshold descriptor */ 106 status = txq->tx_ring[txq->tx_next_dd].dw3; 107 if (!(status & rte_cpu_to_le_32(TXGBE_TXD_DD))) { 108 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh) 109 txgbe_set32_masked(txq->tdc_reg_addr, 110 TXGBE_TXCFG_FLUSH, TXGBE_TXCFG_FLUSH); 111 return 0; 112 } 113 114 /* 115 * first buffer to free from S/W ring is at index 116 * tx_next_dd - (tx_free_thresh-1) 117 */ 118 txep = &txq->sw_ring[txq->tx_next_dd - (txq->tx_free_thresh - 1)]; 119 for (i = 0; i < txq->tx_free_thresh; ++i, ++txep) { 120 /* free buffers one at a time */ 121 m = rte_pktmbuf_prefree_seg(txep->mbuf); 122 txep->mbuf = NULL; 123 124 if (unlikely(m == NULL)) 125 continue; 126 127 if (nb_free >= RTE_TXGBE_TX_MAX_FREE_BUF_SZ || 128 (nb_free > 0 && m->pool != free[0]->pool)) { 129 rte_mempool_put_bulk(free[0]->pool, 130 (void **)free, nb_free); 131 nb_free = 0; 132 } 133 134 free[nb_free++] = m; 135 } 136 137 if (nb_free > 0) 138 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); 139 140 /* buffers were freed, update counters */ 141 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_free_thresh); 142 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_free_thresh); 143 if (txq->tx_next_dd >= txq->nb_tx_desc) 144 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1); 145 146 return txq->tx_free_thresh; 147 } 148 149 /* Populate 4 descriptors with data from 4 mbufs */ 150 static inline void 151 tx4(volatile struct txgbe_tx_desc *txdp, struct rte_mbuf **pkts) 152 { 153 uint64_t buf_dma_addr; 154 uint32_t pkt_len; 155 int i; 156 157 for (i = 0; i < 4; ++i, ++txdp, ++pkts) { 158 buf_dma_addr = rte_mbuf_data_iova(*pkts); 159 pkt_len = (*pkts)->data_len; 160 161 /* write data to descriptor */ 162 txdp->qw0 = rte_cpu_to_le_64(buf_dma_addr); 163 txdp->dw2 = cpu_to_le32(TXGBE_TXD_FLAGS | 164 TXGBE_TXD_DATLEN(pkt_len)); 165 txdp->dw3 = cpu_to_le32(TXGBE_TXD_PAYLEN(pkt_len)); 166 167 rte_prefetch0(&(*pkts)->pool); 168 } 169 } 170 171 /* Populate 1 descriptor with data from 1 mbuf */ 172 static inline void 173 tx1(volatile struct txgbe_tx_desc *txdp, struct rte_mbuf **pkts) 174 { 175 uint64_t buf_dma_addr; 176 uint32_t pkt_len; 177 178 buf_dma_addr = rte_mbuf_data_iova(*pkts); 179 pkt_len = (*pkts)->data_len; 180 181 /* write data to descriptor */ 182 txdp->qw0 = cpu_to_le64(buf_dma_addr); 183 txdp->dw2 = cpu_to_le32(TXGBE_TXD_FLAGS | 184 TXGBE_TXD_DATLEN(pkt_len)); 185 txdp->dw3 = cpu_to_le32(TXGBE_TXD_PAYLEN(pkt_len)); 186 187 rte_prefetch0(&(*pkts)->pool); 188 } 189 190 /* 191 * Fill H/W descriptor ring with mbuf data. 192 * Copy mbuf pointers to the S/W ring. 193 */ 194 static inline void 195 txgbe_tx_fill_hw_ring(struct txgbe_tx_queue *txq, struct rte_mbuf **pkts, 196 uint16_t nb_pkts) 197 { 198 volatile struct txgbe_tx_desc *txdp = &txq->tx_ring[txq->tx_tail]; 199 struct txgbe_tx_entry *txep = &txq->sw_ring[txq->tx_tail]; 200 const int N_PER_LOOP = 4; 201 const int N_PER_LOOP_MASK = N_PER_LOOP - 1; 202 int mainpart, leftover; 203 int i, j; 204 205 /* 206 * Process most of the packets in chunks of N pkts. Any 207 * leftover packets will get processed one at a time. 208 */ 209 mainpart = (nb_pkts & ((uint32_t)~N_PER_LOOP_MASK)); 210 leftover = (nb_pkts & ((uint32_t)N_PER_LOOP_MASK)); 211 for (i = 0; i < mainpart; i += N_PER_LOOP) { 212 /* Copy N mbuf pointers to the S/W ring */ 213 for (j = 0; j < N_PER_LOOP; ++j) 214 (txep + i + j)->mbuf = *(pkts + i + j); 215 tx4(txdp + i, pkts + i); 216 } 217 218 if (unlikely(leftover > 0)) { 219 for (i = 0; i < leftover; ++i) { 220 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i); 221 tx1(txdp + mainpart + i, pkts + mainpart + i); 222 } 223 } 224 } 225 226 static inline uint16_t 227 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 228 uint16_t nb_pkts) 229 { 230 struct txgbe_tx_queue *txq = (struct txgbe_tx_queue *)tx_queue; 231 uint16_t n = 0; 232 233 /* 234 * Begin scanning the H/W ring for done descriptors when the 235 * number of available descriptors drops below tx_free_thresh. For 236 * each done descriptor, free the associated buffer. 237 */ 238 if (txq->nb_tx_free < txq->tx_free_thresh) 239 txgbe_tx_free_bufs(txq); 240 241 /* Only use descriptors that are available */ 242 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); 243 if (unlikely(nb_pkts == 0)) 244 return 0; 245 246 /* Use exactly nb_pkts descriptors */ 247 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); 248 249 /* 250 * At this point, we know there are enough descriptors in the 251 * ring to transmit all the packets. This assumes that each 252 * mbuf contains a single segment, and that no new offloads 253 * are expected, which would require a new context descriptor. 254 */ 255 256 /* 257 * See if we're going to wrap-around. If so, handle the top 258 * of the descriptor ring first, then do the bottom. If not, 259 * the processing looks just like the "bottom" part anyway... 260 */ 261 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) { 262 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail); 263 txgbe_tx_fill_hw_ring(txq, tx_pkts, n); 264 txq->tx_tail = 0; 265 } 266 267 /* Fill H/W descriptor ring with mbuf data */ 268 txgbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n)); 269 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n)); 270 271 /* 272 * Check for wrap-around. This would only happen if we used 273 * up to the last descriptor in the ring, no more, no less. 274 */ 275 if (txq->tx_tail >= txq->nb_tx_desc) 276 txq->tx_tail = 0; 277 278 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", 279 (uint16_t)txq->port_id, (uint16_t)txq->queue_id, 280 (uint16_t)txq->tx_tail, (uint16_t)nb_pkts); 281 282 /* update tail pointer */ 283 rte_wmb(); 284 txgbe_set32_relaxed(txq->tdt_reg_addr, txq->tx_tail); 285 286 return nb_pkts; 287 } 288 289 uint16_t 290 txgbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, 291 uint16_t nb_pkts) 292 { 293 uint16_t nb_tx; 294 295 /* Try to transmit at least chunks of TX_MAX_BURST pkts */ 296 if (likely(nb_pkts <= RTE_PMD_TXGBE_TX_MAX_BURST)) 297 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts); 298 299 /* transmit more than the max burst, in chunks of TX_MAX_BURST */ 300 nb_tx = 0; 301 while (nb_pkts) { 302 uint16_t ret, n; 303 304 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_TXGBE_TX_MAX_BURST); 305 ret = tx_xmit_pkts(tx_queue, &tx_pkts[nb_tx], n); 306 nb_tx = (uint16_t)(nb_tx + ret); 307 nb_pkts = (uint16_t)(nb_pkts - ret); 308 if (ret < n) 309 break; 310 } 311 312 return nb_tx; 313 } 314 315 static inline void 316 txgbe_set_xmit_ctx(struct txgbe_tx_queue *txq, 317 volatile struct txgbe_tx_ctx_desc *ctx_txd, 318 uint64_t ol_flags, union txgbe_tx_offload tx_offload, 319 __rte_unused uint64_t *mdata) 320 { 321 union txgbe_tx_offload tx_offload_mask; 322 uint32_t type_tucmd_mlhl; 323 uint32_t mss_l4len_idx; 324 uint32_t ctx_idx; 325 uint32_t vlan_macip_lens; 326 uint32_t tunnel_seed; 327 328 ctx_idx = txq->ctx_curr; 329 tx_offload_mask.data[0] = 0; 330 tx_offload_mask.data[1] = 0; 331 332 /* Specify which HW CTX to upload. */ 333 mss_l4len_idx = TXGBE_TXD_IDX(ctx_idx); 334 type_tucmd_mlhl = TXGBE_TXD_CTXT; 335 336 tx_offload_mask.ptid |= ~0; 337 type_tucmd_mlhl |= TXGBE_TXD_PTID(tx_offload.ptid); 338 339 /* check if TCP segmentation required for this packet */ 340 if (ol_flags & PKT_TX_TCP_SEG) { 341 tx_offload_mask.l2_len |= ~0; 342 tx_offload_mask.l3_len |= ~0; 343 tx_offload_mask.l4_len |= ~0; 344 tx_offload_mask.tso_segsz |= ~0; 345 mss_l4len_idx |= TXGBE_TXD_MSS(tx_offload.tso_segsz); 346 mss_l4len_idx |= TXGBE_TXD_L4LEN(tx_offload.l4_len); 347 } else { /* no TSO, check if hardware checksum is needed */ 348 if (ol_flags & PKT_TX_IP_CKSUM) { 349 tx_offload_mask.l2_len |= ~0; 350 tx_offload_mask.l3_len |= ~0; 351 } 352 353 switch (ol_flags & PKT_TX_L4_MASK) { 354 case PKT_TX_UDP_CKSUM: 355 mss_l4len_idx |= 356 TXGBE_TXD_L4LEN(sizeof(struct rte_udp_hdr)); 357 tx_offload_mask.l2_len |= ~0; 358 tx_offload_mask.l3_len |= ~0; 359 break; 360 case PKT_TX_TCP_CKSUM: 361 mss_l4len_idx |= 362 TXGBE_TXD_L4LEN(sizeof(struct rte_tcp_hdr)); 363 tx_offload_mask.l2_len |= ~0; 364 tx_offload_mask.l3_len |= ~0; 365 break; 366 case PKT_TX_SCTP_CKSUM: 367 mss_l4len_idx |= 368 TXGBE_TXD_L4LEN(sizeof(struct rte_sctp_hdr)); 369 tx_offload_mask.l2_len |= ~0; 370 tx_offload_mask.l3_len |= ~0; 371 break; 372 default: 373 break; 374 } 375 } 376 377 vlan_macip_lens = TXGBE_TXD_IPLEN(tx_offload.l3_len >> 1); 378 379 if (ol_flags & PKT_TX_TUNNEL_MASK) { 380 tx_offload_mask.outer_tun_len |= ~0; 381 tx_offload_mask.outer_l2_len |= ~0; 382 tx_offload_mask.outer_l3_len |= ~0; 383 tx_offload_mask.l2_len |= ~0; 384 tunnel_seed = TXGBE_TXD_ETUNLEN(tx_offload.outer_tun_len >> 1); 385 tunnel_seed |= TXGBE_TXD_EIPLEN(tx_offload.outer_l3_len >> 2); 386 387 switch (ol_flags & PKT_TX_TUNNEL_MASK) { 388 case PKT_TX_TUNNEL_IPIP: 389 /* for non UDP / GRE tunneling, set to 0b */ 390 break; 391 case PKT_TX_TUNNEL_VXLAN: 392 case PKT_TX_TUNNEL_GENEVE: 393 tunnel_seed |= TXGBE_TXD_ETYPE_UDP; 394 break; 395 case PKT_TX_TUNNEL_GRE: 396 tunnel_seed |= TXGBE_TXD_ETYPE_GRE; 397 break; 398 default: 399 PMD_TX_LOG(ERR, "Tunnel type not supported"); 400 return; 401 } 402 vlan_macip_lens |= TXGBE_TXD_MACLEN(tx_offload.outer_l2_len); 403 } else { 404 tunnel_seed = 0; 405 vlan_macip_lens |= TXGBE_TXD_MACLEN(tx_offload.l2_len); 406 } 407 408 if (ol_flags & PKT_TX_VLAN_PKT) { 409 tx_offload_mask.vlan_tci |= ~0; 410 vlan_macip_lens |= TXGBE_TXD_VLAN(tx_offload.vlan_tci); 411 } 412 413 #ifdef RTE_LIB_SECURITY 414 if (ol_flags & PKT_TX_SEC_OFFLOAD) { 415 union txgbe_crypto_tx_desc_md *md = 416 (union txgbe_crypto_tx_desc_md *)mdata; 417 tunnel_seed |= TXGBE_TXD_IPSEC_SAIDX(md->sa_idx); 418 type_tucmd_mlhl |= md->enc ? 419 (TXGBE_TXD_IPSEC_ESP | TXGBE_TXD_IPSEC_ESPENC) : 0; 420 type_tucmd_mlhl |= TXGBE_TXD_IPSEC_ESPLEN(md->pad_len); 421 tx_offload_mask.sa_idx |= ~0; 422 tx_offload_mask.sec_pad_len |= ~0; 423 } 424 #endif 425 426 txq->ctx_cache[ctx_idx].flags = ol_flags; 427 txq->ctx_cache[ctx_idx].tx_offload.data[0] = 428 tx_offload_mask.data[0] & tx_offload.data[0]; 429 txq->ctx_cache[ctx_idx].tx_offload.data[1] = 430 tx_offload_mask.data[1] & tx_offload.data[1]; 431 txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask; 432 433 ctx_txd->dw0 = rte_cpu_to_le_32(vlan_macip_lens); 434 ctx_txd->dw1 = rte_cpu_to_le_32(tunnel_seed); 435 ctx_txd->dw2 = rte_cpu_to_le_32(type_tucmd_mlhl); 436 ctx_txd->dw3 = rte_cpu_to_le_32(mss_l4len_idx); 437 } 438 439 /* 440 * Check which hardware context can be used. Use the existing match 441 * or create a new context descriptor. 442 */ 443 static inline uint32_t 444 what_ctx_update(struct txgbe_tx_queue *txq, uint64_t flags, 445 union txgbe_tx_offload tx_offload) 446 { 447 /* If match with the current used context */ 448 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags && 449 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] == 450 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0] 451 & tx_offload.data[0])) && 452 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] == 453 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1] 454 & tx_offload.data[1])))) 455 return txq->ctx_curr; 456 457 /* What if match with the next context */ 458 txq->ctx_curr ^= 1; 459 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags && 460 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] == 461 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0] 462 & tx_offload.data[0])) && 463 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] == 464 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1] 465 & tx_offload.data[1])))) 466 return txq->ctx_curr; 467 468 /* Mismatch, use the previous context */ 469 return TXGBE_CTX_NUM; 470 } 471 472 static inline uint32_t 473 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags) 474 { 475 uint32_t tmp = 0; 476 477 if ((ol_flags & PKT_TX_L4_MASK) != PKT_TX_L4_NO_CKSUM) { 478 tmp |= TXGBE_TXD_CC; 479 tmp |= TXGBE_TXD_L4CS; 480 } 481 if (ol_flags & PKT_TX_IP_CKSUM) { 482 tmp |= TXGBE_TXD_CC; 483 tmp |= TXGBE_TXD_IPCS; 484 } 485 if (ol_flags & PKT_TX_OUTER_IP_CKSUM) { 486 tmp |= TXGBE_TXD_CC; 487 tmp |= TXGBE_TXD_EIPCS; 488 } 489 if (ol_flags & PKT_TX_TCP_SEG) { 490 tmp |= TXGBE_TXD_CC; 491 /* implies IPv4 cksum */ 492 if (ol_flags & PKT_TX_IPV4) 493 tmp |= TXGBE_TXD_IPCS; 494 tmp |= TXGBE_TXD_L4CS; 495 } 496 if (ol_flags & PKT_TX_VLAN_PKT) 497 tmp |= TXGBE_TXD_CC; 498 499 return tmp; 500 } 501 502 static inline uint32_t 503 tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags) 504 { 505 uint32_t cmdtype = 0; 506 507 if (ol_flags & PKT_TX_VLAN_PKT) 508 cmdtype |= TXGBE_TXD_VLE; 509 if (ol_flags & PKT_TX_TCP_SEG) 510 cmdtype |= TXGBE_TXD_TSE; 511 if (ol_flags & PKT_TX_MACSEC) 512 cmdtype |= TXGBE_TXD_LINKSEC; 513 return cmdtype; 514 } 515 516 static inline uint8_t 517 tx_desc_ol_flags_to_ptid(uint64_t oflags, uint32_t ptype) 518 { 519 bool tun; 520 521 if (ptype) 522 return txgbe_encode_ptype(ptype); 523 524 /* Only support flags in TXGBE_TX_OFFLOAD_MASK */ 525 tun = !!(oflags & PKT_TX_TUNNEL_MASK); 526 527 /* L2 level */ 528 ptype = RTE_PTYPE_L2_ETHER; 529 if (oflags & PKT_TX_VLAN) 530 ptype |= RTE_PTYPE_L2_ETHER_VLAN; 531 532 /* L3 level */ 533 if (oflags & (PKT_TX_OUTER_IPV4 | PKT_TX_OUTER_IP_CKSUM)) 534 ptype |= RTE_PTYPE_L3_IPV4; 535 else if (oflags & (PKT_TX_OUTER_IPV6)) 536 ptype |= RTE_PTYPE_L3_IPV6; 537 538 if (oflags & (PKT_TX_IPV4 | PKT_TX_IP_CKSUM)) 539 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV4 : RTE_PTYPE_L3_IPV4); 540 else if (oflags & (PKT_TX_IPV6)) 541 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV6 : RTE_PTYPE_L3_IPV6); 542 543 /* L4 level */ 544 switch (oflags & (PKT_TX_L4_MASK)) { 545 case PKT_TX_TCP_CKSUM: 546 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP); 547 break; 548 case PKT_TX_UDP_CKSUM: 549 ptype |= (tun ? RTE_PTYPE_INNER_L4_UDP : RTE_PTYPE_L4_UDP); 550 break; 551 case PKT_TX_SCTP_CKSUM: 552 ptype |= (tun ? RTE_PTYPE_INNER_L4_SCTP : RTE_PTYPE_L4_SCTP); 553 break; 554 } 555 556 if (oflags & PKT_TX_TCP_SEG) 557 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP); 558 559 /* Tunnel */ 560 switch (oflags & PKT_TX_TUNNEL_MASK) { 561 case PKT_TX_TUNNEL_VXLAN: 562 ptype |= RTE_PTYPE_L2_ETHER | 563 RTE_PTYPE_L3_IPV4 | 564 RTE_PTYPE_TUNNEL_VXLAN; 565 ptype |= RTE_PTYPE_INNER_L2_ETHER; 566 break; 567 case PKT_TX_TUNNEL_GRE: 568 ptype |= RTE_PTYPE_L2_ETHER | 569 RTE_PTYPE_L3_IPV4 | 570 RTE_PTYPE_TUNNEL_GRE; 571 ptype |= RTE_PTYPE_INNER_L2_ETHER; 572 break; 573 case PKT_TX_TUNNEL_GENEVE: 574 ptype |= RTE_PTYPE_L2_ETHER | 575 RTE_PTYPE_L3_IPV4 | 576 RTE_PTYPE_TUNNEL_GENEVE; 577 ptype |= RTE_PTYPE_INNER_L2_ETHER; 578 break; 579 case PKT_TX_TUNNEL_VXLAN_GPE: 580 ptype |= RTE_PTYPE_L2_ETHER | 581 RTE_PTYPE_L3_IPV4 | 582 RTE_PTYPE_TUNNEL_VXLAN_GPE; 583 ptype |= RTE_PTYPE_INNER_L2_ETHER; 584 break; 585 case PKT_TX_TUNNEL_IPIP: 586 case PKT_TX_TUNNEL_IP: 587 ptype |= RTE_PTYPE_L2_ETHER | 588 RTE_PTYPE_L3_IPV4 | 589 RTE_PTYPE_TUNNEL_IP; 590 break; 591 } 592 593 return txgbe_encode_ptype(ptype); 594 } 595 596 #ifndef DEFAULT_TX_FREE_THRESH 597 #define DEFAULT_TX_FREE_THRESH 32 598 #endif 599 600 /* Reset transmit descriptors after they have been used */ 601 static inline int 602 txgbe_xmit_cleanup(struct txgbe_tx_queue *txq) 603 { 604 struct txgbe_tx_entry *sw_ring = txq->sw_ring; 605 volatile struct txgbe_tx_desc *txr = txq->tx_ring; 606 uint16_t last_desc_cleaned = txq->last_desc_cleaned; 607 uint16_t nb_tx_desc = txq->nb_tx_desc; 608 uint16_t desc_to_clean_to; 609 uint16_t nb_tx_to_clean; 610 uint32_t status; 611 612 /* Determine the last descriptor needing to be cleaned */ 613 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_free_thresh); 614 if (desc_to_clean_to >= nb_tx_desc) 615 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc); 616 617 /* Check to make sure the last descriptor to clean is done */ 618 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id; 619 status = txr[desc_to_clean_to].dw3; 620 if (!(status & rte_cpu_to_le_32(TXGBE_TXD_DD))) { 621 PMD_TX_FREE_LOG(DEBUG, 622 "TX descriptor %4u is not done" 623 "(port=%d queue=%d)", 624 desc_to_clean_to, 625 txq->port_id, txq->queue_id); 626 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh) 627 txgbe_set32_masked(txq->tdc_reg_addr, 628 TXGBE_TXCFG_FLUSH, TXGBE_TXCFG_FLUSH); 629 /* Failed to clean any descriptors, better luck next time */ 630 return -(1); 631 } 632 633 /* Figure out how many descriptors will be cleaned */ 634 if (last_desc_cleaned > desc_to_clean_to) 635 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) + 636 desc_to_clean_to); 637 else 638 nb_tx_to_clean = (uint16_t)(desc_to_clean_to - 639 last_desc_cleaned); 640 641 PMD_TX_FREE_LOG(DEBUG, 642 "Cleaning %4u TX descriptors: %4u to %4u " 643 "(port=%d queue=%d)", 644 nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to, 645 txq->port_id, txq->queue_id); 646 647 /* 648 * The last descriptor to clean is done, so that means all the 649 * descriptors from the last descriptor that was cleaned 650 * up to the last descriptor with the RS bit set 651 * are done. Only reset the threshold descriptor. 652 */ 653 txr[desc_to_clean_to].dw3 = 0; 654 655 /* Update the txq to reflect the last descriptor that was cleaned */ 656 txq->last_desc_cleaned = desc_to_clean_to; 657 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean); 658 659 /* No Error */ 660 return 0; 661 } 662 663 static inline uint8_t 664 txgbe_get_tun_len(struct rte_mbuf *mbuf) 665 { 666 struct txgbe_genevehdr genevehdr; 667 const struct txgbe_genevehdr *gh; 668 uint8_t tun_len; 669 670 switch (mbuf->ol_flags & PKT_TX_TUNNEL_MASK) { 671 case PKT_TX_TUNNEL_IPIP: 672 tun_len = 0; 673 break; 674 case PKT_TX_TUNNEL_VXLAN: 675 case PKT_TX_TUNNEL_VXLAN_GPE: 676 tun_len = sizeof(struct txgbe_udphdr) 677 + sizeof(struct txgbe_vxlanhdr); 678 break; 679 case PKT_TX_TUNNEL_GRE: 680 tun_len = sizeof(struct txgbe_nvgrehdr); 681 break; 682 case PKT_TX_TUNNEL_GENEVE: 683 gh = rte_pktmbuf_read(mbuf, 684 mbuf->outer_l2_len + mbuf->outer_l3_len, 685 sizeof(genevehdr), &genevehdr); 686 tun_len = sizeof(struct txgbe_udphdr) 687 + sizeof(struct txgbe_genevehdr) 688 + (gh->opt_len << 2); 689 break; 690 default: 691 tun_len = 0; 692 } 693 694 return tun_len; 695 } 696 697 uint16_t 698 txgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 699 uint16_t nb_pkts) 700 { 701 struct txgbe_tx_queue *txq; 702 struct txgbe_tx_entry *sw_ring; 703 struct txgbe_tx_entry *txe, *txn; 704 volatile struct txgbe_tx_desc *txr; 705 volatile struct txgbe_tx_desc *txd; 706 struct rte_mbuf *tx_pkt; 707 struct rte_mbuf *m_seg; 708 uint64_t buf_dma_addr; 709 uint32_t olinfo_status; 710 uint32_t cmd_type_len; 711 uint32_t pkt_len; 712 uint16_t slen; 713 uint64_t ol_flags; 714 uint16_t tx_id; 715 uint16_t tx_last; 716 uint16_t nb_tx; 717 uint16_t nb_used; 718 uint64_t tx_ol_req; 719 uint32_t ctx = 0; 720 uint32_t new_ctx; 721 union txgbe_tx_offload tx_offload; 722 #ifdef RTE_LIB_SECURITY 723 uint8_t use_ipsec; 724 #endif 725 726 tx_offload.data[0] = 0; 727 tx_offload.data[1] = 0; 728 txq = tx_queue; 729 sw_ring = txq->sw_ring; 730 txr = txq->tx_ring; 731 tx_id = txq->tx_tail; 732 txe = &sw_ring[tx_id]; 733 734 /* Determine if the descriptor ring needs to be cleaned. */ 735 if (txq->nb_tx_free < txq->tx_free_thresh) 736 txgbe_xmit_cleanup(txq); 737 738 rte_prefetch0(&txe->mbuf->pool); 739 740 /* TX loop */ 741 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 742 new_ctx = 0; 743 tx_pkt = *tx_pkts++; 744 pkt_len = tx_pkt->pkt_len; 745 746 /* 747 * Determine how many (if any) context descriptors 748 * are needed for offload functionality. 749 */ 750 ol_flags = tx_pkt->ol_flags; 751 #ifdef RTE_LIB_SECURITY 752 use_ipsec = txq->using_ipsec && (ol_flags & PKT_TX_SEC_OFFLOAD); 753 #endif 754 755 /* If hardware offload required */ 756 tx_ol_req = ol_flags & TXGBE_TX_OFFLOAD_MASK; 757 if (tx_ol_req) { 758 tx_offload.ptid = tx_desc_ol_flags_to_ptid(tx_ol_req, 759 tx_pkt->packet_type); 760 tx_offload.l2_len = tx_pkt->l2_len; 761 tx_offload.l3_len = tx_pkt->l3_len; 762 tx_offload.l4_len = tx_pkt->l4_len; 763 tx_offload.vlan_tci = tx_pkt->vlan_tci; 764 tx_offload.tso_segsz = tx_pkt->tso_segsz; 765 tx_offload.outer_l2_len = tx_pkt->outer_l2_len; 766 tx_offload.outer_l3_len = tx_pkt->outer_l3_len; 767 tx_offload.outer_tun_len = txgbe_get_tun_len(tx_pkt); 768 769 #ifdef RTE_LIB_SECURITY 770 if (use_ipsec) { 771 union txgbe_crypto_tx_desc_md *ipsec_mdata = 772 (union txgbe_crypto_tx_desc_md *) 773 rte_security_dynfield(tx_pkt); 774 tx_offload.sa_idx = ipsec_mdata->sa_idx; 775 tx_offload.sec_pad_len = ipsec_mdata->pad_len; 776 } 777 #endif 778 779 /* If new context need be built or reuse the exist ctx*/ 780 ctx = what_ctx_update(txq, tx_ol_req, tx_offload); 781 /* Only allocate context descriptor if required */ 782 new_ctx = (ctx == TXGBE_CTX_NUM); 783 ctx = txq->ctx_curr; 784 } 785 786 /* 787 * Keep track of how many descriptors are used this loop 788 * This will always be the number of segments + the number of 789 * Context descriptors required to transmit the packet 790 */ 791 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx); 792 793 /* 794 * The number of descriptors that must be allocated for a 795 * packet is the number of segments of that packet, plus 1 796 * Context Descriptor for the hardware offload, if any. 797 * Determine the last TX descriptor to allocate in the TX ring 798 * for the packet, starting from the current position (tx_id) 799 * in the ring. 800 */ 801 tx_last = (uint16_t)(tx_id + nb_used - 1); 802 803 /* Circular ring */ 804 if (tx_last >= txq->nb_tx_desc) 805 tx_last = (uint16_t)(tx_last - txq->nb_tx_desc); 806 807 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u" 808 " tx_first=%u tx_last=%u", 809 (uint16_t)txq->port_id, 810 (uint16_t)txq->queue_id, 811 (uint32_t)pkt_len, 812 (uint16_t)tx_id, 813 (uint16_t)tx_last); 814 815 /* 816 * Make sure there are enough TX descriptors available to 817 * transmit the entire packet. 818 * nb_used better be less than or equal to txq->tx_free_thresh 819 */ 820 if (nb_used > txq->nb_tx_free) { 821 PMD_TX_FREE_LOG(DEBUG, 822 "Not enough free TX descriptors " 823 "nb_used=%4u nb_free=%4u " 824 "(port=%d queue=%d)", 825 nb_used, txq->nb_tx_free, 826 txq->port_id, txq->queue_id); 827 828 if (txgbe_xmit_cleanup(txq) != 0) { 829 /* Could not clean any descriptors */ 830 if (nb_tx == 0) 831 return 0; 832 goto end_of_tx; 833 } 834 835 /* nb_used better be <= txq->tx_free_thresh */ 836 if (unlikely(nb_used > txq->tx_free_thresh)) { 837 PMD_TX_FREE_LOG(DEBUG, 838 "The number of descriptors needed to " 839 "transmit the packet exceeds the " 840 "RS bit threshold. This will impact " 841 "performance." 842 "nb_used=%4u nb_free=%4u " 843 "tx_free_thresh=%4u. " 844 "(port=%d queue=%d)", 845 nb_used, txq->nb_tx_free, 846 txq->tx_free_thresh, 847 txq->port_id, txq->queue_id); 848 /* 849 * Loop here until there are enough TX 850 * descriptors or until the ring cannot be 851 * cleaned. 852 */ 853 while (nb_used > txq->nb_tx_free) { 854 if (txgbe_xmit_cleanup(txq) != 0) { 855 /* 856 * Could not clean any 857 * descriptors 858 */ 859 if (nb_tx == 0) 860 return 0; 861 goto end_of_tx; 862 } 863 } 864 } 865 } 866 867 /* 868 * By now there are enough free TX descriptors to transmit 869 * the packet. 870 */ 871 872 /* 873 * Set common flags of all TX Data Descriptors. 874 * 875 * The following bits must be set in all Data Descriptors: 876 * - TXGBE_TXD_DTYP_DATA 877 * - TXGBE_TXD_DCMD_DEXT 878 * 879 * The following bits must be set in the first Data Descriptor 880 * and are ignored in the other ones: 881 * - TXGBE_TXD_DCMD_IFCS 882 * - TXGBE_TXD_MAC_1588 883 * - TXGBE_TXD_DCMD_VLE 884 * 885 * The following bits must only be set in the last Data 886 * Descriptor: 887 * - TXGBE_TXD_CMD_EOP 888 * 889 * The following bits can be set in any Data Descriptor, but 890 * are only set in the last Data Descriptor: 891 * - TXGBE_TXD_CMD_RS 892 */ 893 cmd_type_len = TXGBE_TXD_FCS; 894 895 #ifdef RTE_LIBRTE_IEEE1588 896 if (ol_flags & PKT_TX_IEEE1588_TMST) 897 cmd_type_len |= TXGBE_TXD_1588; 898 #endif 899 900 olinfo_status = 0; 901 if (tx_ol_req) { 902 if (ol_flags & PKT_TX_TCP_SEG) { 903 /* when TSO is on, paylen in descriptor is the 904 * not the packet len but the tcp payload len 905 */ 906 pkt_len -= (tx_offload.l2_len + 907 tx_offload.l3_len + tx_offload.l4_len); 908 pkt_len -= 909 (tx_pkt->ol_flags & PKT_TX_TUNNEL_MASK) 910 ? tx_offload.outer_l2_len + 911 tx_offload.outer_l3_len : 0; 912 } 913 914 /* 915 * Setup the TX Advanced Context Descriptor if required 916 */ 917 if (new_ctx) { 918 volatile struct txgbe_tx_ctx_desc *ctx_txd; 919 920 ctx_txd = (volatile struct txgbe_tx_ctx_desc *) 921 &txr[tx_id]; 922 923 txn = &sw_ring[txe->next_id]; 924 rte_prefetch0(&txn->mbuf->pool); 925 926 if (txe->mbuf != NULL) { 927 rte_pktmbuf_free_seg(txe->mbuf); 928 txe->mbuf = NULL; 929 } 930 931 txgbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req, 932 tx_offload, 933 rte_security_dynfield(tx_pkt)); 934 935 txe->last_id = tx_last; 936 tx_id = txe->next_id; 937 txe = txn; 938 } 939 940 /* 941 * Setup the TX Advanced Data Descriptor, 942 * This path will go through 943 * whatever new/reuse the context descriptor 944 */ 945 cmd_type_len |= tx_desc_ol_flags_to_cmdtype(ol_flags); 946 olinfo_status |= 947 tx_desc_cksum_flags_to_olinfo(ol_flags); 948 olinfo_status |= TXGBE_TXD_IDX(ctx); 949 } 950 951 olinfo_status |= TXGBE_TXD_PAYLEN(pkt_len); 952 #ifdef RTE_LIB_SECURITY 953 if (use_ipsec) 954 olinfo_status |= TXGBE_TXD_IPSEC; 955 #endif 956 957 m_seg = tx_pkt; 958 do { 959 txd = &txr[tx_id]; 960 txn = &sw_ring[txe->next_id]; 961 rte_prefetch0(&txn->mbuf->pool); 962 963 if (txe->mbuf != NULL) 964 rte_pktmbuf_free_seg(txe->mbuf); 965 txe->mbuf = m_seg; 966 967 /* 968 * Set up Transmit Data Descriptor. 969 */ 970 slen = m_seg->data_len; 971 buf_dma_addr = rte_mbuf_data_iova(m_seg); 972 txd->qw0 = rte_cpu_to_le_64(buf_dma_addr); 973 txd->dw2 = rte_cpu_to_le_32(cmd_type_len | slen); 974 txd->dw3 = rte_cpu_to_le_32(olinfo_status); 975 txe->last_id = tx_last; 976 tx_id = txe->next_id; 977 txe = txn; 978 m_seg = m_seg->next; 979 } while (m_seg != NULL); 980 981 /* 982 * The last packet data descriptor needs End Of Packet (EOP) 983 */ 984 cmd_type_len |= TXGBE_TXD_EOP; 985 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used); 986 987 txd->dw2 |= rte_cpu_to_le_32(cmd_type_len); 988 } 989 990 end_of_tx: 991 992 rte_wmb(); 993 994 /* 995 * Set the Transmit Descriptor Tail (TDT) 996 */ 997 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", 998 (uint16_t)txq->port_id, (uint16_t)txq->queue_id, 999 (uint16_t)tx_id, (uint16_t)nb_tx); 1000 txgbe_set32_relaxed(txq->tdt_reg_addr, tx_id); 1001 txq->tx_tail = tx_id; 1002 1003 return nb_tx; 1004 } 1005 1006 /********************************************************************* 1007 * 1008 * TX prep functions 1009 * 1010 **********************************************************************/ 1011 uint16_t 1012 txgbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 1013 { 1014 int i, ret; 1015 uint64_t ol_flags; 1016 struct rte_mbuf *m; 1017 struct txgbe_tx_queue *txq = (struct txgbe_tx_queue *)tx_queue; 1018 1019 for (i = 0; i < nb_pkts; i++) { 1020 m = tx_pkts[i]; 1021 ol_flags = m->ol_flags; 1022 1023 /** 1024 * Check if packet meets requirements for number of segments 1025 * 1026 * NOTE: for txgbe it's always (40 - WTHRESH) for both TSO and 1027 * non-TSO 1028 */ 1029 1030 if (m->nb_segs > TXGBE_TX_MAX_SEG - txq->wthresh) { 1031 rte_errno = -EINVAL; 1032 return i; 1033 } 1034 1035 if (ol_flags & TXGBE_TX_OFFLOAD_NOTSUP_MASK) { 1036 rte_errno = -ENOTSUP; 1037 return i; 1038 } 1039 1040 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 1041 ret = rte_validate_tx_offload(m); 1042 if (ret != 0) { 1043 rte_errno = ret; 1044 return i; 1045 } 1046 #endif 1047 ret = rte_net_intel_cksum_prepare(m); 1048 if (ret != 0) { 1049 rte_errno = ret; 1050 return i; 1051 } 1052 } 1053 1054 return i; 1055 } 1056 1057 /********************************************************************* 1058 * 1059 * RX functions 1060 * 1061 **********************************************************************/ 1062 /* @note: fix txgbe_dev_supported_ptypes_get() if any change here. */ 1063 static inline uint32_t 1064 txgbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptid_mask) 1065 { 1066 uint16_t ptid = TXGBE_RXD_PTID(pkt_info); 1067 1068 ptid &= ptid_mask; 1069 1070 return txgbe_decode_ptype(ptid); 1071 } 1072 1073 static inline uint64_t 1074 txgbe_rxd_pkt_info_to_pkt_flags(uint32_t pkt_info) 1075 { 1076 static uint64_t ip_rss_types_map[16] __rte_cache_aligned = { 1077 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, 1078 0, PKT_RX_RSS_HASH, 0, PKT_RX_RSS_HASH, 1079 PKT_RX_RSS_HASH, 0, 0, 0, 1080 0, 0, 0, PKT_RX_FDIR, 1081 }; 1082 #ifdef RTE_LIBRTE_IEEE1588 1083 static uint64_t ip_pkt_etqf_map[8] = { 1084 0, 0, 0, PKT_RX_IEEE1588_PTP, 1085 0, 0, 0, 0, 1086 }; 1087 int etfid = txgbe_etflt_id(TXGBE_RXD_PTID(pkt_info)); 1088 if (likely(-1 != etfid)) 1089 return ip_pkt_etqf_map[etfid] | 1090 ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)]; 1091 else 1092 return ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)]; 1093 #else 1094 return ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)]; 1095 #endif 1096 } 1097 1098 static inline uint64_t 1099 rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags) 1100 { 1101 uint64_t pkt_flags; 1102 1103 /* 1104 * Check if VLAN present only. 1105 * Do not check whether L3/L4 rx checksum done by NIC or not, 1106 * That can be found from rte_eth_rxmode.offloads flag 1107 */ 1108 pkt_flags = (rx_status & TXGBE_RXD_STAT_VLAN && 1109 vlan_flags & PKT_RX_VLAN_STRIPPED) 1110 ? vlan_flags : 0; 1111 1112 #ifdef RTE_LIBRTE_IEEE1588 1113 if (rx_status & TXGBE_RXD_STAT_1588) 1114 pkt_flags = pkt_flags | PKT_RX_IEEE1588_TMST; 1115 #endif 1116 return pkt_flags; 1117 } 1118 1119 static inline uint64_t 1120 rx_desc_error_to_pkt_flags(uint32_t rx_status) 1121 { 1122 uint64_t pkt_flags = 0; 1123 1124 /* checksum offload can't be disabled */ 1125 if (rx_status & TXGBE_RXD_STAT_IPCS) { 1126 pkt_flags |= (rx_status & TXGBE_RXD_ERR_IPCS 1127 ? PKT_RX_IP_CKSUM_BAD : PKT_RX_IP_CKSUM_GOOD); 1128 } 1129 1130 if (rx_status & TXGBE_RXD_STAT_L4CS) { 1131 pkt_flags |= (rx_status & TXGBE_RXD_ERR_L4CS 1132 ? PKT_RX_L4_CKSUM_BAD : PKT_RX_L4_CKSUM_GOOD); 1133 } 1134 1135 if (rx_status & TXGBE_RXD_STAT_EIPCS && 1136 rx_status & TXGBE_RXD_ERR_EIPCS) { 1137 pkt_flags |= PKT_RX_OUTER_IP_CKSUM_BAD; 1138 } 1139 1140 #ifdef RTE_LIB_SECURITY 1141 if (rx_status & TXGBE_RXD_STAT_SECP) { 1142 pkt_flags |= PKT_RX_SEC_OFFLOAD; 1143 if (rx_status & TXGBE_RXD_ERR_SECERR) 1144 pkt_flags |= PKT_RX_SEC_OFFLOAD_FAILED; 1145 } 1146 #endif 1147 1148 return pkt_flags; 1149 } 1150 1151 /* 1152 * LOOK_AHEAD defines how many desc statuses to check beyond the 1153 * current descriptor. 1154 * It must be a pound define for optimal performance. 1155 * Do not change the value of LOOK_AHEAD, as the txgbe_rx_scan_hw_ring 1156 * function only works with LOOK_AHEAD=8. 1157 */ 1158 #define LOOK_AHEAD 8 1159 #if (LOOK_AHEAD != 8) 1160 #error "PMD TXGBE: LOOK_AHEAD must be 8\n" 1161 #endif 1162 static inline int 1163 txgbe_rx_scan_hw_ring(struct txgbe_rx_queue *rxq) 1164 { 1165 volatile struct txgbe_rx_desc *rxdp; 1166 struct txgbe_rx_entry *rxep; 1167 struct rte_mbuf *mb; 1168 uint16_t pkt_len; 1169 uint64_t pkt_flags; 1170 int nb_dd; 1171 uint32_t s[LOOK_AHEAD]; 1172 uint32_t pkt_info[LOOK_AHEAD]; 1173 int i, j, nb_rx = 0; 1174 uint32_t status; 1175 1176 /* get references to current descriptor and S/W ring entry */ 1177 rxdp = &rxq->rx_ring[rxq->rx_tail]; 1178 rxep = &rxq->sw_ring[rxq->rx_tail]; 1179 1180 status = rxdp->qw1.lo.status; 1181 /* check to make sure there is at least 1 packet to receive */ 1182 if (!(status & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD))) 1183 return 0; 1184 1185 /* 1186 * Scan LOOK_AHEAD descriptors at a time to determine which descriptors 1187 * reference packets that are ready to be received. 1188 */ 1189 for (i = 0; i < RTE_PMD_TXGBE_RX_MAX_BURST; 1190 i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) { 1191 /* Read desc statuses backwards to avoid race condition */ 1192 for (j = 0; j < LOOK_AHEAD; j++) 1193 s[j] = rte_le_to_cpu_32(rxdp[j].qw1.lo.status); 1194 1195 rte_atomic_thread_fence(__ATOMIC_ACQUIRE); 1196 1197 /* Compute how many status bits were set */ 1198 for (nb_dd = 0; nb_dd < LOOK_AHEAD && 1199 (s[nb_dd] & TXGBE_RXD_STAT_DD); nb_dd++) 1200 ; 1201 1202 for (j = 0; j < nb_dd; j++) 1203 pkt_info[j] = rte_le_to_cpu_32(rxdp[j].qw0.dw0); 1204 1205 nb_rx += nb_dd; 1206 1207 /* Translate descriptor info to mbuf format */ 1208 for (j = 0; j < nb_dd; ++j) { 1209 mb = rxep[j].mbuf; 1210 pkt_len = rte_le_to_cpu_16(rxdp[j].qw1.hi.len) - 1211 rxq->crc_len; 1212 mb->data_len = pkt_len; 1213 mb->pkt_len = pkt_len; 1214 mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].qw1.hi.tag); 1215 1216 /* convert descriptor fields to rte mbuf flags */ 1217 pkt_flags = rx_desc_status_to_pkt_flags(s[j], 1218 rxq->vlan_flags); 1219 pkt_flags |= rx_desc_error_to_pkt_flags(s[j]); 1220 pkt_flags |= 1221 txgbe_rxd_pkt_info_to_pkt_flags(pkt_info[j]); 1222 mb->ol_flags = pkt_flags; 1223 mb->packet_type = 1224 txgbe_rxd_pkt_info_to_pkt_type(pkt_info[j], 1225 rxq->pkt_type_mask); 1226 1227 if (likely(pkt_flags & PKT_RX_RSS_HASH)) 1228 mb->hash.rss = 1229 rte_le_to_cpu_32(rxdp[j].qw0.dw1); 1230 else if (pkt_flags & PKT_RX_FDIR) { 1231 mb->hash.fdir.hash = 1232 rte_le_to_cpu_16(rxdp[j].qw0.hi.csum) & 1233 TXGBE_ATR_HASH_MASK; 1234 mb->hash.fdir.id = 1235 rte_le_to_cpu_16(rxdp[j].qw0.hi.ipid); 1236 } 1237 } 1238 1239 /* Move mbuf pointers from the S/W ring to the stage */ 1240 for (j = 0; j < LOOK_AHEAD; ++j) 1241 rxq->rx_stage[i + j] = rxep[j].mbuf; 1242 1243 /* stop if all requested packets could not be received */ 1244 if (nb_dd != LOOK_AHEAD) 1245 break; 1246 } 1247 1248 /* clear software ring entries so we can cleanup correctly */ 1249 for (i = 0; i < nb_rx; ++i) 1250 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL; 1251 1252 return nb_rx; 1253 } 1254 1255 static inline int 1256 txgbe_rx_alloc_bufs(struct txgbe_rx_queue *rxq, bool reset_mbuf) 1257 { 1258 volatile struct txgbe_rx_desc *rxdp; 1259 struct txgbe_rx_entry *rxep; 1260 struct rte_mbuf *mb; 1261 uint16_t alloc_idx; 1262 __le64 dma_addr; 1263 int diag, i; 1264 1265 /* allocate buffers in bulk directly into the S/W ring */ 1266 alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1); 1267 rxep = &rxq->sw_ring[alloc_idx]; 1268 diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep, 1269 rxq->rx_free_thresh); 1270 if (unlikely(diag != 0)) 1271 return -ENOMEM; 1272 1273 rxdp = &rxq->rx_ring[alloc_idx]; 1274 for (i = 0; i < rxq->rx_free_thresh; ++i) { 1275 /* populate the static rte mbuf fields */ 1276 mb = rxep[i].mbuf; 1277 if (reset_mbuf) 1278 mb->port = rxq->port_id; 1279 1280 rte_mbuf_refcnt_set(mb, 1); 1281 mb->data_off = RTE_PKTMBUF_HEADROOM; 1282 1283 /* populate the descriptors */ 1284 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb)); 1285 TXGBE_RXD_HDRADDR(&rxdp[i], 0); 1286 TXGBE_RXD_PKTADDR(&rxdp[i], dma_addr); 1287 } 1288 1289 /* update state of internal queue structure */ 1290 rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh; 1291 if (rxq->rx_free_trigger >= rxq->nb_rx_desc) 1292 rxq->rx_free_trigger = rxq->rx_free_thresh - 1; 1293 1294 /* no errors */ 1295 return 0; 1296 } 1297 1298 static inline uint16_t 1299 txgbe_rx_fill_from_stage(struct txgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, 1300 uint16_t nb_pkts) 1301 { 1302 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail]; 1303 int i; 1304 1305 /* how many packets are ready to return? */ 1306 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail); 1307 1308 /* copy mbuf pointers to the application's packet list */ 1309 for (i = 0; i < nb_pkts; ++i) 1310 rx_pkts[i] = stage[i]; 1311 1312 /* update internal queue state */ 1313 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts); 1314 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts); 1315 1316 return nb_pkts; 1317 } 1318 1319 static inline uint16_t 1320 txgbe_rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1321 uint16_t nb_pkts) 1322 { 1323 struct txgbe_rx_queue *rxq = (struct txgbe_rx_queue *)rx_queue; 1324 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 1325 uint16_t nb_rx = 0; 1326 1327 /* Any previously recv'd pkts will be returned from the Rx stage */ 1328 if (rxq->rx_nb_avail) 1329 return txgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 1330 1331 /* Scan the H/W ring for packets to receive */ 1332 nb_rx = (uint16_t)txgbe_rx_scan_hw_ring(rxq); 1333 1334 /* update internal queue state */ 1335 rxq->rx_next_avail = 0; 1336 rxq->rx_nb_avail = nb_rx; 1337 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx); 1338 1339 /* if required, allocate new buffers to replenish descriptors */ 1340 if (rxq->rx_tail > rxq->rx_free_trigger) { 1341 uint16_t cur_free_trigger = rxq->rx_free_trigger; 1342 1343 if (txgbe_rx_alloc_bufs(rxq, true) != 0) { 1344 int i, j; 1345 1346 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " 1347 "queue_id=%u", (uint16_t)rxq->port_id, 1348 (uint16_t)rxq->queue_id); 1349 1350 dev->data->rx_mbuf_alloc_failed += 1351 rxq->rx_free_thresh; 1352 1353 /* 1354 * Need to rewind any previous receives if we cannot 1355 * allocate new buffers to replenish the old ones. 1356 */ 1357 rxq->rx_nb_avail = 0; 1358 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx); 1359 for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j) 1360 rxq->sw_ring[j].mbuf = rxq->rx_stage[i]; 1361 1362 return 0; 1363 } 1364 1365 /* update tail pointer */ 1366 rte_wmb(); 1367 txgbe_set32_relaxed(rxq->rdt_reg_addr, cur_free_trigger); 1368 } 1369 1370 if (rxq->rx_tail >= rxq->nb_rx_desc) 1371 rxq->rx_tail = 0; 1372 1373 /* received any packets this loop? */ 1374 if (rxq->rx_nb_avail) 1375 return txgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 1376 1377 return 0; 1378 } 1379 1380 /* split requests into chunks of size RTE_PMD_TXGBE_RX_MAX_BURST */ 1381 uint16_t 1382 txgbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1383 uint16_t nb_pkts) 1384 { 1385 uint16_t nb_rx; 1386 1387 if (unlikely(nb_pkts == 0)) 1388 return 0; 1389 1390 if (likely(nb_pkts <= RTE_PMD_TXGBE_RX_MAX_BURST)) 1391 return txgbe_rx_recv_pkts(rx_queue, rx_pkts, nb_pkts); 1392 1393 /* request is relatively large, chunk it up */ 1394 nb_rx = 0; 1395 while (nb_pkts) { 1396 uint16_t ret, n; 1397 1398 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_TXGBE_RX_MAX_BURST); 1399 ret = txgbe_rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n); 1400 nb_rx = (uint16_t)(nb_rx + ret); 1401 nb_pkts = (uint16_t)(nb_pkts - ret); 1402 if (ret < n) 1403 break; 1404 } 1405 1406 return nb_rx; 1407 } 1408 1409 uint16_t 1410 txgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1411 uint16_t nb_pkts) 1412 { 1413 struct txgbe_rx_queue *rxq; 1414 volatile struct txgbe_rx_desc *rx_ring; 1415 volatile struct txgbe_rx_desc *rxdp; 1416 struct txgbe_rx_entry *sw_ring; 1417 struct txgbe_rx_entry *rxe; 1418 struct rte_mbuf *rxm; 1419 struct rte_mbuf *nmb; 1420 struct txgbe_rx_desc rxd; 1421 uint64_t dma_addr; 1422 uint32_t staterr; 1423 uint32_t pkt_info; 1424 uint16_t pkt_len; 1425 uint16_t rx_id; 1426 uint16_t nb_rx; 1427 uint16_t nb_hold; 1428 uint64_t pkt_flags; 1429 1430 nb_rx = 0; 1431 nb_hold = 0; 1432 rxq = rx_queue; 1433 rx_id = rxq->rx_tail; 1434 rx_ring = rxq->rx_ring; 1435 sw_ring = rxq->sw_ring; 1436 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 1437 while (nb_rx < nb_pkts) { 1438 /* 1439 * The order of operations here is important as the DD status 1440 * bit must not be read after any other descriptor fields. 1441 * rx_ring and rxdp are pointing to volatile data so the order 1442 * of accesses cannot be reordered by the compiler. If they were 1443 * not volatile, they could be reordered which could lead to 1444 * using invalid descriptor fields when read from rxd. 1445 */ 1446 rxdp = &rx_ring[rx_id]; 1447 staterr = rxdp->qw1.lo.status; 1448 if (!(staterr & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD))) 1449 break; 1450 rxd = *rxdp; 1451 1452 /* 1453 * End of packet. 1454 * 1455 * If the TXGBE_RXD_STAT_EOP flag is not set, the RX packet 1456 * is likely to be invalid and to be dropped by the various 1457 * validation checks performed by the network stack. 1458 * 1459 * Allocate a new mbuf to replenish the RX ring descriptor. 1460 * If the allocation fails: 1461 * - arrange for that RX descriptor to be the first one 1462 * being parsed the next time the receive function is 1463 * invoked [on the same queue]. 1464 * 1465 * - Stop parsing the RX ring and return immediately. 1466 * 1467 * This policy do not drop the packet received in the RX 1468 * descriptor for which the allocation of a new mbuf failed. 1469 * Thus, it allows that packet to be later retrieved if 1470 * mbuf have been freed in the mean time. 1471 * As a side effect, holding RX descriptors instead of 1472 * systematically giving them back to the NIC may lead to 1473 * RX ring exhaustion situations. 1474 * However, the NIC can gracefully prevent such situations 1475 * to happen by sending specific "back-pressure" flow control 1476 * frames to its peer(s). 1477 */ 1478 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u " 1479 "ext_err_stat=0x%08x pkt_len=%u", 1480 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id, 1481 (uint16_t)rx_id, (uint32_t)staterr, 1482 (uint16_t)rte_le_to_cpu_16(rxd.qw1.hi.len)); 1483 1484 nmb = rte_mbuf_raw_alloc(rxq->mb_pool); 1485 if (nmb == NULL) { 1486 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " 1487 "queue_id=%u", (uint16_t)rxq->port_id, 1488 (uint16_t)rxq->queue_id); 1489 dev->data->rx_mbuf_alloc_failed++; 1490 break; 1491 } 1492 1493 nb_hold++; 1494 rxe = &sw_ring[rx_id]; 1495 rx_id++; 1496 if (rx_id == rxq->nb_rx_desc) 1497 rx_id = 0; 1498 1499 /* Prefetch next mbuf while processing current one. */ 1500 rte_txgbe_prefetch(sw_ring[rx_id].mbuf); 1501 1502 /* 1503 * When next RX descriptor is on a cache-line boundary, 1504 * prefetch the next 4 RX descriptors and the next 8 pointers 1505 * to mbufs. 1506 */ 1507 if ((rx_id & 0x3) == 0) { 1508 rte_txgbe_prefetch(&rx_ring[rx_id]); 1509 rte_txgbe_prefetch(&sw_ring[rx_id]); 1510 } 1511 1512 rxm = rxe->mbuf; 1513 rxe->mbuf = nmb; 1514 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 1515 TXGBE_RXD_HDRADDR(rxdp, 0); 1516 TXGBE_RXD_PKTADDR(rxdp, dma_addr); 1517 1518 /* 1519 * Initialize the returned mbuf. 1520 * 1) setup generic mbuf fields: 1521 * - number of segments, 1522 * - next segment, 1523 * - packet length, 1524 * - RX port identifier. 1525 * 2) integrate hardware offload data, if any: 1526 * - RSS flag & hash, 1527 * - IP checksum flag, 1528 * - VLAN TCI, if any, 1529 * - error flags. 1530 */ 1531 pkt_len = (uint16_t)(rte_le_to_cpu_16(rxd.qw1.hi.len) - 1532 rxq->crc_len); 1533 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1534 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off); 1535 rxm->nb_segs = 1; 1536 rxm->next = NULL; 1537 rxm->pkt_len = pkt_len; 1538 rxm->data_len = pkt_len; 1539 rxm->port = rxq->port_id; 1540 1541 pkt_info = rte_le_to_cpu_32(rxd.qw0.dw0); 1542 /* Only valid if PKT_RX_VLAN set in pkt_flags */ 1543 rxm->vlan_tci = rte_le_to_cpu_16(rxd.qw1.hi.tag); 1544 1545 pkt_flags = rx_desc_status_to_pkt_flags(staterr, 1546 rxq->vlan_flags); 1547 pkt_flags |= rx_desc_error_to_pkt_flags(staterr); 1548 pkt_flags |= txgbe_rxd_pkt_info_to_pkt_flags(pkt_info); 1549 rxm->ol_flags = pkt_flags; 1550 rxm->packet_type = txgbe_rxd_pkt_info_to_pkt_type(pkt_info, 1551 rxq->pkt_type_mask); 1552 1553 if (likely(pkt_flags & PKT_RX_RSS_HASH)) { 1554 rxm->hash.rss = rte_le_to_cpu_32(rxd.qw0.dw1); 1555 } else if (pkt_flags & PKT_RX_FDIR) { 1556 rxm->hash.fdir.hash = 1557 rte_le_to_cpu_16(rxd.qw0.hi.csum) & 1558 TXGBE_ATR_HASH_MASK; 1559 rxm->hash.fdir.id = rte_le_to_cpu_16(rxd.qw0.hi.ipid); 1560 } 1561 /* 1562 * Store the mbuf address into the next entry of the array 1563 * of returned packets. 1564 */ 1565 rx_pkts[nb_rx++] = rxm; 1566 } 1567 rxq->rx_tail = rx_id; 1568 1569 /* 1570 * If the number of free RX descriptors is greater than the RX free 1571 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 1572 * register. 1573 * Update the RDT with the value of the last processed RX descriptor 1574 * minus 1, to guarantee that the RDT register is never equal to the 1575 * RDH register, which creates a "full" ring situation from the 1576 * hardware point of view... 1577 */ 1578 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold); 1579 if (nb_hold > rxq->rx_free_thresh) { 1580 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u " 1581 "nb_hold=%u nb_rx=%u", 1582 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id, 1583 (uint16_t)rx_id, (uint16_t)nb_hold, 1584 (uint16_t)nb_rx); 1585 rx_id = (uint16_t)((rx_id == 0) ? 1586 (rxq->nb_rx_desc - 1) : (rx_id - 1)); 1587 txgbe_set32(rxq->rdt_reg_addr, rx_id); 1588 nb_hold = 0; 1589 } 1590 rxq->nb_rx_hold = nb_hold; 1591 return nb_rx; 1592 } 1593 1594 /** 1595 * txgbe_fill_cluster_head_buf - fill the first mbuf of the returned packet 1596 * 1597 * Fill the following info in the HEAD buffer of the Rx cluster: 1598 * - RX port identifier 1599 * - hardware offload data, if any: 1600 * - RSS flag & hash 1601 * - IP checksum flag 1602 * - VLAN TCI, if any 1603 * - error flags 1604 * @head HEAD of the packet cluster 1605 * @desc HW descriptor to get data from 1606 * @rxq Pointer to the Rx queue 1607 */ 1608 static inline void 1609 txgbe_fill_cluster_head_buf(struct rte_mbuf *head, struct txgbe_rx_desc *desc, 1610 struct txgbe_rx_queue *rxq, uint32_t staterr) 1611 { 1612 uint32_t pkt_info; 1613 uint64_t pkt_flags; 1614 1615 head->port = rxq->port_id; 1616 1617 /* The vlan_tci field is only valid when PKT_RX_VLAN is 1618 * set in the pkt_flags field. 1619 */ 1620 head->vlan_tci = rte_le_to_cpu_16(desc->qw1.hi.tag); 1621 pkt_info = rte_le_to_cpu_32(desc->qw0.dw0); 1622 pkt_flags = rx_desc_status_to_pkt_flags(staterr, rxq->vlan_flags); 1623 pkt_flags |= rx_desc_error_to_pkt_flags(staterr); 1624 pkt_flags |= txgbe_rxd_pkt_info_to_pkt_flags(pkt_info); 1625 head->ol_flags = pkt_flags; 1626 head->packet_type = txgbe_rxd_pkt_info_to_pkt_type(pkt_info, 1627 rxq->pkt_type_mask); 1628 1629 if (likely(pkt_flags & PKT_RX_RSS_HASH)) { 1630 head->hash.rss = rte_le_to_cpu_32(desc->qw0.dw1); 1631 } else if (pkt_flags & PKT_RX_FDIR) { 1632 head->hash.fdir.hash = rte_le_to_cpu_16(desc->qw0.hi.csum) 1633 & TXGBE_ATR_HASH_MASK; 1634 head->hash.fdir.id = rte_le_to_cpu_16(desc->qw0.hi.ipid); 1635 } 1636 } 1637 1638 /** 1639 * txgbe_recv_pkts_lro - receive handler for and LRO case. 1640 * 1641 * @rx_queue Rx queue handle 1642 * @rx_pkts table of received packets 1643 * @nb_pkts size of rx_pkts table 1644 * @bulk_alloc if TRUE bulk allocation is used for a HW ring refilling 1645 * 1646 * Handles the Rx HW ring completions when RSC feature is configured. Uses an 1647 * additional ring of txgbe_rsc_entry's that will hold the relevant RSC info. 1648 * 1649 * We use the same logic as in Linux and in FreeBSD txgbe drivers: 1650 * 1) When non-EOP RSC completion arrives: 1651 * a) Update the HEAD of the current RSC aggregation cluster with the new 1652 * segment's data length. 1653 * b) Set the "next" pointer of the current segment to point to the segment 1654 * at the NEXTP index. 1655 * c) Pass the HEAD of RSC aggregation cluster on to the next NEXTP entry 1656 * in the sw_rsc_ring. 1657 * 2) When EOP arrives we just update the cluster's total length and offload 1658 * flags and deliver the cluster up to the upper layers. In our case - put it 1659 * in the rx_pkts table. 1660 * 1661 * Returns the number of received packets/clusters (according to the "bulk 1662 * receive" interface). 1663 */ 1664 static inline uint16_t 1665 txgbe_recv_pkts_lro(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts, 1666 bool bulk_alloc) 1667 { 1668 struct txgbe_rx_queue *rxq = rx_queue; 1669 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 1670 volatile struct txgbe_rx_desc *rx_ring = rxq->rx_ring; 1671 struct txgbe_rx_entry *sw_ring = rxq->sw_ring; 1672 struct txgbe_scattered_rx_entry *sw_sc_ring = rxq->sw_sc_ring; 1673 uint16_t rx_id = rxq->rx_tail; 1674 uint16_t nb_rx = 0; 1675 uint16_t nb_hold = rxq->nb_rx_hold; 1676 uint16_t prev_id = rxq->rx_tail; 1677 1678 while (nb_rx < nb_pkts) { 1679 bool eop; 1680 struct txgbe_rx_entry *rxe; 1681 struct txgbe_scattered_rx_entry *sc_entry; 1682 struct txgbe_scattered_rx_entry *next_sc_entry = NULL; 1683 struct txgbe_rx_entry *next_rxe = NULL; 1684 struct rte_mbuf *first_seg; 1685 struct rte_mbuf *rxm; 1686 struct rte_mbuf *nmb = NULL; 1687 struct txgbe_rx_desc rxd; 1688 uint16_t data_len; 1689 uint16_t next_id; 1690 volatile struct txgbe_rx_desc *rxdp; 1691 uint32_t staterr; 1692 1693 next_desc: 1694 /* 1695 * The code in this whole file uses the volatile pointer to 1696 * ensure the read ordering of the status and the rest of the 1697 * descriptor fields (on the compiler level only!!!). This is so 1698 * UGLY - why not to just use the compiler barrier instead? DPDK 1699 * even has the rte_compiler_barrier() for that. 1700 * 1701 * But most importantly this is just wrong because this doesn't 1702 * ensure memory ordering in a general case at all. For 1703 * instance, DPDK is supposed to work on Power CPUs where 1704 * compiler barrier may just not be enough! 1705 * 1706 * I tried to write only this function properly to have a 1707 * starting point (as a part of an LRO/RSC series) but the 1708 * compiler cursed at me when I tried to cast away the 1709 * "volatile" from rx_ring (yes, it's volatile too!!!). So, I'm 1710 * keeping it the way it is for now. 1711 * 1712 * The code in this file is broken in so many other places and 1713 * will just not work on a big endian CPU anyway therefore the 1714 * lines below will have to be revisited together with the rest 1715 * of the txgbe PMD. 1716 * 1717 * TODO: 1718 * - Get rid of "volatile" and let the compiler do its job. 1719 * - Use the proper memory barrier (rte_rmb()) to ensure the 1720 * memory ordering below. 1721 */ 1722 rxdp = &rx_ring[rx_id]; 1723 staterr = rte_le_to_cpu_32(rxdp->qw1.lo.status); 1724 1725 if (!(staterr & TXGBE_RXD_STAT_DD)) 1726 break; 1727 1728 rxd = *rxdp; 1729 1730 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u " 1731 "staterr=0x%x data_len=%u", 1732 rxq->port_id, rxq->queue_id, rx_id, staterr, 1733 rte_le_to_cpu_16(rxd.qw1.hi.len)); 1734 1735 if (!bulk_alloc) { 1736 nmb = rte_mbuf_raw_alloc(rxq->mb_pool); 1737 if (nmb == NULL) { 1738 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed " 1739 "port_id=%u queue_id=%u", 1740 rxq->port_id, rxq->queue_id); 1741 1742 dev->data->rx_mbuf_alloc_failed++; 1743 break; 1744 } 1745 } else if (nb_hold > rxq->rx_free_thresh) { 1746 uint16_t next_rdt = rxq->rx_free_trigger; 1747 1748 if (!txgbe_rx_alloc_bufs(rxq, false)) { 1749 rte_wmb(); 1750 txgbe_set32_relaxed(rxq->rdt_reg_addr, 1751 next_rdt); 1752 nb_hold -= rxq->rx_free_thresh; 1753 } else { 1754 PMD_RX_LOG(DEBUG, "RX bulk alloc failed " 1755 "port_id=%u queue_id=%u", 1756 rxq->port_id, rxq->queue_id); 1757 1758 dev->data->rx_mbuf_alloc_failed++; 1759 break; 1760 } 1761 } 1762 1763 nb_hold++; 1764 rxe = &sw_ring[rx_id]; 1765 eop = staterr & TXGBE_RXD_STAT_EOP; 1766 1767 next_id = rx_id + 1; 1768 if (next_id == rxq->nb_rx_desc) 1769 next_id = 0; 1770 1771 /* Prefetch next mbuf while processing current one. */ 1772 rte_txgbe_prefetch(sw_ring[next_id].mbuf); 1773 1774 /* 1775 * When next RX descriptor is on a cache-line boundary, 1776 * prefetch the next 4 RX descriptors and the next 4 pointers 1777 * to mbufs. 1778 */ 1779 if ((next_id & 0x3) == 0) { 1780 rte_txgbe_prefetch(&rx_ring[next_id]); 1781 rte_txgbe_prefetch(&sw_ring[next_id]); 1782 } 1783 1784 rxm = rxe->mbuf; 1785 1786 if (!bulk_alloc) { 1787 __le64 dma = 1788 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 1789 /* 1790 * Update RX descriptor with the physical address of the 1791 * new data buffer of the new allocated mbuf. 1792 */ 1793 rxe->mbuf = nmb; 1794 1795 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1796 TXGBE_RXD_HDRADDR(rxdp, 0); 1797 TXGBE_RXD_PKTADDR(rxdp, dma); 1798 } else { 1799 rxe->mbuf = NULL; 1800 } 1801 1802 /* 1803 * Set data length & data buffer address of mbuf. 1804 */ 1805 data_len = rte_le_to_cpu_16(rxd.qw1.hi.len); 1806 rxm->data_len = data_len; 1807 1808 if (!eop) { 1809 uint16_t nextp_id; 1810 /* 1811 * Get next descriptor index: 1812 * - For RSC it's in the NEXTP field. 1813 * - For a scattered packet - it's just a following 1814 * descriptor. 1815 */ 1816 if (TXGBE_RXD_RSCCNT(rxd.qw0.dw0)) 1817 nextp_id = TXGBE_RXD_NEXTP(staterr); 1818 else 1819 nextp_id = next_id; 1820 1821 next_sc_entry = &sw_sc_ring[nextp_id]; 1822 next_rxe = &sw_ring[nextp_id]; 1823 rte_txgbe_prefetch(next_rxe); 1824 } 1825 1826 sc_entry = &sw_sc_ring[rx_id]; 1827 first_seg = sc_entry->fbuf; 1828 sc_entry->fbuf = NULL; 1829 1830 /* 1831 * If this is the first buffer of the received packet, 1832 * set the pointer to the first mbuf of the packet and 1833 * initialize its context. 1834 * Otherwise, update the total length and the number of segments 1835 * of the current scattered packet, and update the pointer to 1836 * the last mbuf of the current packet. 1837 */ 1838 if (first_seg == NULL) { 1839 first_seg = rxm; 1840 first_seg->pkt_len = data_len; 1841 first_seg->nb_segs = 1; 1842 } else { 1843 first_seg->pkt_len += data_len; 1844 first_seg->nb_segs++; 1845 } 1846 1847 prev_id = rx_id; 1848 rx_id = next_id; 1849 1850 /* 1851 * If this is not the last buffer of the received packet, update 1852 * the pointer to the first mbuf at the NEXTP entry in the 1853 * sw_sc_ring and continue to parse the RX ring. 1854 */ 1855 if (!eop && next_rxe) { 1856 rxm->next = next_rxe->mbuf; 1857 next_sc_entry->fbuf = first_seg; 1858 goto next_desc; 1859 } 1860 1861 /* Initialize the first mbuf of the returned packet */ 1862 txgbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr); 1863 1864 /* 1865 * Deal with the case, when HW CRC srip is disabled. 1866 * That can't happen when LRO is enabled, but still could 1867 * happen for scattered RX mode. 1868 */ 1869 first_seg->pkt_len -= rxq->crc_len; 1870 if (unlikely(rxm->data_len <= rxq->crc_len)) { 1871 struct rte_mbuf *lp; 1872 1873 for (lp = first_seg; lp->next != rxm; lp = lp->next) 1874 ; 1875 1876 first_seg->nb_segs--; 1877 lp->data_len -= rxq->crc_len - rxm->data_len; 1878 lp->next = NULL; 1879 rte_pktmbuf_free_seg(rxm); 1880 } else { 1881 rxm->data_len -= rxq->crc_len; 1882 } 1883 1884 /* Prefetch data of first segment, if configured to do so. */ 1885 rte_packet_prefetch((char *)first_seg->buf_addr + 1886 first_seg->data_off); 1887 1888 /* 1889 * Store the mbuf address into the next entry of the array 1890 * of returned packets. 1891 */ 1892 rx_pkts[nb_rx++] = first_seg; 1893 } 1894 1895 /* 1896 * Record index of the next RX descriptor to probe. 1897 */ 1898 rxq->rx_tail = rx_id; 1899 1900 /* 1901 * If the number of free RX descriptors is greater than the RX free 1902 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 1903 * register. 1904 * Update the RDT with the value of the last processed RX descriptor 1905 * minus 1, to guarantee that the RDT register is never equal to the 1906 * RDH register, which creates a "full" ring situation from the 1907 * hardware point of view... 1908 */ 1909 if (!bulk_alloc && nb_hold > rxq->rx_free_thresh) { 1910 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u " 1911 "nb_hold=%u nb_rx=%u", 1912 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx); 1913 1914 rte_wmb(); 1915 txgbe_set32_relaxed(rxq->rdt_reg_addr, prev_id); 1916 nb_hold = 0; 1917 } 1918 1919 rxq->nb_rx_hold = nb_hold; 1920 return nb_rx; 1921 } 1922 1923 uint16_t 1924 txgbe_recv_pkts_lro_single_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1925 uint16_t nb_pkts) 1926 { 1927 return txgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, false); 1928 } 1929 1930 uint16_t 1931 txgbe_recv_pkts_lro_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1932 uint16_t nb_pkts) 1933 { 1934 return txgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, true); 1935 } 1936 1937 uint64_t 1938 txgbe_get_rx_queue_offloads(struct rte_eth_dev *dev __rte_unused) 1939 { 1940 return DEV_RX_OFFLOAD_VLAN_STRIP; 1941 } 1942 1943 uint64_t 1944 txgbe_get_rx_port_offloads(struct rte_eth_dev *dev) 1945 { 1946 uint64_t offloads; 1947 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 1948 struct rte_eth_dev_sriov *sriov = &RTE_ETH_DEV_SRIOV(dev); 1949 1950 offloads = DEV_RX_OFFLOAD_IPV4_CKSUM | 1951 DEV_RX_OFFLOAD_UDP_CKSUM | 1952 DEV_RX_OFFLOAD_TCP_CKSUM | 1953 DEV_RX_OFFLOAD_KEEP_CRC | 1954 DEV_RX_OFFLOAD_JUMBO_FRAME | 1955 DEV_RX_OFFLOAD_VLAN_FILTER | 1956 DEV_RX_OFFLOAD_RSS_HASH | 1957 DEV_RX_OFFLOAD_SCATTER; 1958 1959 if (!txgbe_is_vf(dev)) 1960 offloads |= (DEV_RX_OFFLOAD_VLAN_FILTER | 1961 DEV_RX_OFFLOAD_QINQ_STRIP | 1962 DEV_RX_OFFLOAD_VLAN_EXTEND); 1963 1964 /* 1965 * RSC is only supported by PF devices in a non-SR-IOV 1966 * mode. 1967 */ 1968 if (hw->mac.type == txgbe_mac_raptor && !sriov->active) 1969 offloads |= DEV_RX_OFFLOAD_TCP_LRO; 1970 1971 if (hw->mac.type == txgbe_mac_raptor) 1972 offloads |= DEV_RX_OFFLOAD_MACSEC_STRIP; 1973 1974 offloads |= DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM; 1975 1976 #ifdef RTE_LIB_SECURITY 1977 if (dev->security_ctx) 1978 offloads |= DEV_RX_OFFLOAD_SECURITY; 1979 #endif 1980 1981 return offloads; 1982 } 1983 1984 static void __rte_cold 1985 txgbe_tx_queue_release_mbufs(struct txgbe_tx_queue *txq) 1986 { 1987 unsigned int i; 1988 1989 if (txq->sw_ring != NULL) { 1990 for (i = 0; i < txq->nb_tx_desc; i++) { 1991 if (txq->sw_ring[i].mbuf != NULL) { 1992 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 1993 txq->sw_ring[i].mbuf = NULL; 1994 } 1995 } 1996 } 1997 } 1998 1999 static int 2000 txgbe_tx_done_cleanup_full(struct txgbe_tx_queue *txq, uint32_t free_cnt) 2001 { 2002 struct txgbe_tx_entry *swr_ring = txq->sw_ring; 2003 uint16_t i, tx_last, tx_id; 2004 uint16_t nb_tx_free_last; 2005 uint16_t nb_tx_to_clean; 2006 uint32_t pkt_cnt; 2007 2008 /* Start free mbuf from the next of tx_tail */ 2009 tx_last = txq->tx_tail; 2010 tx_id = swr_ring[tx_last].next_id; 2011 2012 if (txq->nb_tx_free == 0 && txgbe_xmit_cleanup(txq)) 2013 return 0; 2014 2015 nb_tx_to_clean = txq->nb_tx_free; 2016 nb_tx_free_last = txq->nb_tx_free; 2017 if (!free_cnt) 2018 free_cnt = txq->nb_tx_desc; 2019 2020 /* Loop through swr_ring to count the amount of 2021 * freeable mubfs and packets. 2022 */ 2023 for (pkt_cnt = 0; pkt_cnt < free_cnt; ) { 2024 for (i = 0; i < nb_tx_to_clean && 2025 pkt_cnt < free_cnt && 2026 tx_id != tx_last; i++) { 2027 if (swr_ring[tx_id].mbuf != NULL) { 2028 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf); 2029 swr_ring[tx_id].mbuf = NULL; 2030 2031 /* 2032 * last segment in the packet, 2033 * increment packet count 2034 */ 2035 pkt_cnt += (swr_ring[tx_id].last_id == tx_id); 2036 } 2037 2038 tx_id = swr_ring[tx_id].next_id; 2039 } 2040 2041 if (pkt_cnt < free_cnt) { 2042 if (txgbe_xmit_cleanup(txq)) 2043 break; 2044 2045 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last; 2046 nb_tx_free_last = txq->nb_tx_free; 2047 } 2048 } 2049 2050 return (int)pkt_cnt; 2051 } 2052 2053 static int 2054 txgbe_tx_done_cleanup_simple(struct txgbe_tx_queue *txq, 2055 uint32_t free_cnt) 2056 { 2057 int i, n, cnt; 2058 2059 if (free_cnt == 0 || free_cnt > txq->nb_tx_desc) 2060 free_cnt = txq->nb_tx_desc; 2061 2062 cnt = free_cnt - free_cnt % txq->tx_free_thresh; 2063 2064 for (i = 0; i < cnt; i += n) { 2065 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_free_thresh) 2066 break; 2067 2068 n = txgbe_tx_free_bufs(txq); 2069 2070 if (n == 0) 2071 break; 2072 } 2073 2074 return i; 2075 } 2076 2077 int 2078 txgbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt) 2079 { 2080 struct txgbe_tx_queue *txq = (struct txgbe_tx_queue *)tx_queue; 2081 if (txq->offloads == 0 && 2082 #ifdef RTE_LIB_SECURITY 2083 !(txq->using_ipsec) && 2084 #endif 2085 txq->tx_free_thresh >= RTE_PMD_TXGBE_TX_MAX_BURST) 2086 return txgbe_tx_done_cleanup_simple(txq, free_cnt); 2087 2088 return txgbe_tx_done_cleanup_full(txq, free_cnt); 2089 } 2090 2091 static void __rte_cold 2092 txgbe_tx_free_swring(struct txgbe_tx_queue *txq) 2093 { 2094 if (txq != NULL && 2095 txq->sw_ring != NULL) 2096 rte_free(txq->sw_ring); 2097 } 2098 2099 static void __rte_cold 2100 txgbe_tx_queue_release(struct txgbe_tx_queue *txq) 2101 { 2102 if (txq != NULL && txq->ops != NULL) { 2103 txq->ops->release_mbufs(txq); 2104 txq->ops->free_swring(txq); 2105 rte_free(txq); 2106 } 2107 } 2108 2109 void __rte_cold 2110 txgbe_dev_tx_queue_release(void *txq) 2111 { 2112 txgbe_tx_queue_release(txq); 2113 } 2114 2115 /* (Re)set dynamic txgbe_tx_queue fields to defaults */ 2116 static void __rte_cold 2117 txgbe_reset_tx_queue(struct txgbe_tx_queue *txq) 2118 { 2119 static const struct txgbe_tx_desc zeroed_desc = {0}; 2120 struct txgbe_tx_entry *txe = txq->sw_ring; 2121 uint16_t prev, i; 2122 2123 /* Zero out HW ring memory */ 2124 for (i = 0; i < txq->nb_tx_desc; i++) 2125 txq->tx_ring[i] = zeroed_desc; 2126 2127 /* Initialize SW ring entries */ 2128 prev = (uint16_t)(txq->nb_tx_desc - 1); 2129 for (i = 0; i < txq->nb_tx_desc; i++) { 2130 volatile struct txgbe_tx_desc *txd = &txq->tx_ring[i]; 2131 2132 txd->dw3 = rte_cpu_to_le_32(TXGBE_TXD_DD); 2133 txe[i].mbuf = NULL; 2134 txe[i].last_id = i; 2135 txe[prev].next_id = i; 2136 prev = i; 2137 } 2138 2139 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1); 2140 txq->tx_tail = 0; 2141 2142 /* 2143 * Always allow 1 descriptor to be un-allocated to avoid 2144 * a H/W race condition 2145 */ 2146 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1); 2147 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1); 2148 txq->ctx_curr = 0; 2149 memset((void *)&txq->ctx_cache, 0, 2150 TXGBE_CTX_NUM * sizeof(struct txgbe_ctx_info)); 2151 } 2152 2153 static const struct txgbe_txq_ops def_txq_ops = { 2154 .release_mbufs = txgbe_tx_queue_release_mbufs, 2155 .free_swring = txgbe_tx_free_swring, 2156 .reset = txgbe_reset_tx_queue, 2157 }; 2158 2159 /* Takes an ethdev and a queue and sets up the tx function to be used based on 2160 * the queue parameters. Used in tx_queue_setup by primary process and then 2161 * in dev_init by secondary process when attaching to an existing ethdev. 2162 */ 2163 void __rte_cold 2164 txgbe_set_tx_function(struct rte_eth_dev *dev, struct txgbe_tx_queue *txq) 2165 { 2166 /* Use a simple Tx queue (no offloads, no multi segs) if possible */ 2167 if (txq->offloads == 0 && 2168 #ifdef RTE_LIB_SECURITY 2169 !(txq->using_ipsec) && 2170 #endif 2171 txq->tx_free_thresh >= RTE_PMD_TXGBE_TX_MAX_BURST) { 2172 PMD_INIT_LOG(DEBUG, "Using simple tx code path"); 2173 dev->tx_pkt_burst = txgbe_xmit_pkts_simple; 2174 dev->tx_pkt_prepare = NULL; 2175 } else { 2176 PMD_INIT_LOG(DEBUG, "Using full-featured tx code path"); 2177 PMD_INIT_LOG(DEBUG, 2178 " - offloads = 0x%" PRIx64, 2179 txq->offloads); 2180 PMD_INIT_LOG(DEBUG, 2181 " - tx_free_thresh = %lu [RTE_PMD_TXGBE_TX_MAX_BURST=%lu]", 2182 (unsigned long)txq->tx_free_thresh, 2183 (unsigned long)RTE_PMD_TXGBE_TX_MAX_BURST); 2184 dev->tx_pkt_burst = txgbe_xmit_pkts; 2185 dev->tx_pkt_prepare = txgbe_prep_pkts; 2186 } 2187 } 2188 2189 uint64_t 2190 txgbe_get_tx_queue_offloads(struct rte_eth_dev *dev) 2191 { 2192 RTE_SET_USED(dev); 2193 2194 return 0; 2195 } 2196 2197 uint64_t 2198 txgbe_get_tx_port_offloads(struct rte_eth_dev *dev) 2199 { 2200 uint64_t tx_offload_capa; 2201 2202 tx_offload_capa = 2203 DEV_TX_OFFLOAD_VLAN_INSERT | 2204 DEV_TX_OFFLOAD_IPV4_CKSUM | 2205 DEV_TX_OFFLOAD_UDP_CKSUM | 2206 DEV_TX_OFFLOAD_TCP_CKSUM | 2207 DEV_TX_OFFLOAD_SCTP_CKSUM | 2208 DEV_TX_OFFLOAD_TCP_TSO | 2209 DEV_TX_OFFLOAD_UDP_TSO | 2210 DEV_TX_OFFLOAD_UDP_TNL_TSO | 2211 DEV_TX_OFFLOAD_IP_TNL_TSO | 2212 DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 2213 DEV_TX_OFFLOAD_GRE_TNL_TSO | 2214 DEV_TX_OFFLOAD_IPIP_TNL_TSO | 2215 DEV_TX_OFFLOAD_GENEVE_TNL_TSO | 2216 DEV_TX_OFFLOAD_MULTI_SEGS; 2217 2218 if (!txgbe_is_vf(dev)) 2219 tx_offload_capa |= DEV_TX_OFFLOAD_QINQ_INSERT; 2220 2221 tx_offload_capa |= DEV_TX_OFFLOAD_MACSEC_INSERT; 2222 2223 tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM; 2224 2225 #ifdef RTE_LIB_SECURITY 2226 if (dev->security_ctx) 2227 tx_offload_capa |= DEV_TX_OFFLOAD_SECURITY; 2228 #endif 2229 return tx_offload_capa; 2230 } 2231 2232 int __rte_cold 2233 txgbe_dev_tx_queue_setup(struct rte_eth_dev *dev, 2234 uint16_t queue_idx, 2235 uint16_t nb_desc, 2236 unsigned int socket_id, 2237 const struct rte_eth_txconf *tx_conf) 2238 { 2239 const struct rte_memzone *tz; 2240 struct txgbe_tx_queue *txq; 2241 struct txgbe_hw *hw; 2242 uint16_t tx_free_thresh; 2243 uint64_t offloads; 2244 2245 PMD_INIT_FUNC_TRACE(); 2246 hw = TXGBE_DEV_HW(dev); 2247 2248 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads; 2249 2250 /* 2251 * Validate number of transmit descriptors. 2252 * It must not exceed hardware maximum, and must be multiple 2253 * of TXGBE_ALIGN. 2254 */ 2255 if (nb_desc % TXGBE_TXD_ALIGN != 0 || 2256 nb_desc > TXGBE_RING_DESC_MAX || 2257 nb_desc < TXGBE_RING_DESC_MIN) { 2258 return -EINVAL; 2259 } 2260 2261 /* 2262 * The TX descriptor ring will be cleaned after txq->tx_free_thresh 2263 * descriptors are used or if the number of descriptors required 2264 * to transmit a packet is greater than the number of free TX 2265 * descriptors. 2266 * One descriptor in the TX ring is used as a sentinel to avoid a 2267 * H/W race condition, hence the maximum threshold constraints. 2268 * When set to zero use default values. 2269 */ 2270 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ? 2271 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH); 2272 if (tx_free_thresh >= (nb_desc - 3)) { 2273 PMD_INIT_LOG(ERR, "tx_free_thresh must be less than the number of " 2274 "TX descriptors minus 3. (tx_free_thresh=%u " 2275 "port=%d queue=%d)", 2276 (unsigned int)tx_free_thresh, 2277 (int)dev->data->port_id, (int)queue_idx); 2278 return -(EINVAL); 2279 } 2280 2281 if ((nb_desc % tx_free_thresh) != 0) { 2282 PMD_INIT_LOG(ERR, "tx_free_thresh must be a divisor of the " 2283 "number of TX descriptors. (tx_free_thresh=%u " 2284 "port=%d queue=%d)", (unsigned int)tx_free_thresh, 2285 (int)dev->data->port_id, (int)queue_idx); 2286 return -(EINVAL); 2287 } 2288 2289 /* Free memory prior to re-allocation if needed... */ 2290 if (dev->data->tx_queues[queue_idx] != NULL) { 2291 txgbe_tx_queue_release(dev->data->tx_queues[queue_idx]); 2292 dev->data->tx_queues[queue_idx] = NULL; 2293 } 2294 2295 /* First allocate the tx queue data structure */ 2296 txq = rte_zmalloc_socket("ethdev TX queue", 2297 sizeof(struct txgbe_tx_queue), 2298 RTE_CACHE_LINE_SIZE, socket_id); 2299 if (txq == NULL) 2300 return -ENOMEM; 2301 2302 /* 2303 * Allocate TX ring hardware descriptors. A memzone large enough to 2304 * handle the maximum ring size is allocated in order to allow for 2305 * resizing in later calls to the queue setup function. 2306 */ 2307 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx, 2308 sizeof(struct txgbe_tx_desc) * TXGBE_RING_DESC_MAX, 2309 TXGBE_ALIGN, socket_id); 2310 if (tz == NULL) { 2311 txgbe_tx_queue_release(txq); 2312 return -ENOMEM; 2313 } 2314 2315 txq->nb_tx_desc = nb_desc; 2316 txq->tx_free_thresh = tx_free_thresh; 2317 txq->pthresh = tx_conf->tx_thresh.pthresh; 2318 txq->hthresh = tx_conf->tx_thresh.hthresh; 2319 txq->wthresh = tx_conf->tx_thresh.wthresh; 2320 txq->queue_id = queue_idx; 2321 txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ? 2322 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx); 2323 txq->port_id = dev->data->port_id; 2324 txq->offloads = offloads; 2325 txq->ops = &def_txq_ops; 2326 txq->tx_deferred_start = tx_conf->tx_deferred_start; 2327 #ifdef RTE_LIB_SECURITY 2328 txq->using_ipsec = !!(dev->data->dev_conf.txmode.offloads & 2329 DEV_TX_OFFLOAD_SECURITY); 2330 #endif 2331 2332 /* Modification to set tail pointer for virtual function 2333 * if vf is detected. 2334 */ 2335 if (hw->mac.type == txgbe_mac_raptor_vf) { 2336 txq->tdt_reg_addr = TXGBE_REG_ADDR(hw, TXGBE_TXWP(queue_idx)); 2337 txq->tdc_reg_addr = TXGBE_REG_ADDR(hw, TXGBE_TXCFG(queue_idx)); 2338 } else { 2339 txq->tdt_reg_addr = TXGBE_REG_ADDR(hw, 2340 TXGBE_TXWP(txq->reg_idx)); 2341 txq->tdc_reg_addr = TXGBE_REG_ADDR(hw, 2342 TXGBE_TXCFG(txq->reg_idx)); 2343 } 2344 2345 txq->tx_ring_phys_addr = TMZ_PADDR(tz); 2346 txq->tx_ring = (struct txgbe_tx_desc *)TMZ_VADDR(tz); 2347 2348 /* Allocate software ring */ 2349 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring", 2350 sizeof(struct txgbe_tx_entry) * nb_desc, 2351 RTE_CACHE_LINE_SIZE, socket_id); 2352 if (txq->sw_ring == NULL) { 2353 txgbe_tx_queue_release(txq); 2354 return -ENOMEM; 2355 } 2356 PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64, 2357 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); 2358 2359 /* set up scalar TX function as appropriate */ 2360 txgbe_set_tx_function(dev, txq); 2361 2362 txq->ops->reset(txq); 2363 2364 dev->data->tx_queues[queue_idx] = txq; 2365 2366 return 0; 2367 } 2368 2369 /** 2370 * txgbe_free_sc_cluster - free the not-yet-completed scattered cluster 2371 * 2372 * The "next" pointer of the last segment of (not-yet-completed) RSC clusters 2373 * in the sw_rsc_ring is not set to NULL but rather points to the next 2374 * mbuf of this RSC aggregation (that has not been completed yet and still 2375 * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we 2376 * will just free first "nb_segs" segments of the cluster explicitly by calling 2377 * an rte_pktmbuf_free_seg(). 2378 * 2379 * @m scattered cluster head 2380 */ 2381 static void __rte_cold 2382 txgbe_free_sc_cluster(struct rte_mbuf *m) 2383 { 2384 uint16_t i, nb_segs = m->nb_segs; 2385 struct rte_mbuf *next_seg; 2386 2387 for (i = 0; i < nb_segs; i++) { 2388 next_seg = m->next; 2389 rte_pktmbuf_free_seg(m); 2390 m = next_seg; 2391 } 2392 } 2393 2394 static void __rte_cold 2395 txgbe_rx_queue_release_mbufs(struct txgbe_rx_queue *rxq) 2396 { 2397 unsigned int i; 2398 2399 if (rxq->sw_ring != NULL) { 2400 for (i = 0; i < rxq->nb_rx_desc; i++) { 2401 if (rxq->sw_ring[i].mbuf != NULL) { 2402 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); 2403 rxq->sw_ring[i].mbuf = NULL; 2404 } 2405 } 2406 if (rxq->rx_nb_avail) { 2407 for (i = 0; i < rxq->rx_nb_avail; ++i) { 2408 struct rte_mbuf *mb; 2409 2410 mb = rxq->rx_stage[rxq->rx_next_avail + i]; 2411 rte_pktmbuf_free_seg(mb); 2412 } 2413 rxq->rx_nb_avail = 0; 2414 } 2415 } 2416 2417 if (rxq->sw_sc_ring) 2418 for (i = 0; i < rxq->nb_rx_desc; i++) 2419 if (rxq->sw_sc_ring[i].fbuf) { 2420 txgbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf); 2421 rxq->sw_sc_ring[i].fbuf = NULL; 2422 } 2423 } 2424 2425 static void __rte_cold 2426 txgbe_rx_queue_release(struct txgbe_rx_queue *rxq) 2427 { 2428 if (rxq != NULL) { 2429 txgbe_rx_queue_release_mbufs(rxq); 2430 rte_free(rxq->sw_ring); 2431 rte_free(rxq->sw_sc_ring); 2432 rte_free(rxq); 2433 } 2434 } 2435 2436 void __rte_cold 2437 txgbe_dev_rx_queue_release(void *rxq) 2438 { 2439 txgbe_rx_queue_release(rxq); 2440 } 2441 2442 /* 2443 * Check if Rx Burst Bulk Alloc function can be used. 2444 * Return 2445 * 0: the preconditions are satisfied and the bulk allocation function 2446 * can be used. 2447 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst 2448 * function must be used. 2449 */ 2450 static inline int __rte_cold 2451 check_rx_burst_bulk_alloc_preconditions(struct txgbe_rx_queue *rxq) 2452 { 2453 int ret = 0; 2454 2455 /* 2456 * Make sure the following pre-conditions are satisfied: 2457 * rxq->rx_free_thresh >= RTE_PMD_TXGBE_RX_MAX_BURST 2458 * rxq->rx_free_thresh < rxq->nb_rx_desc 2459 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0 2460 * Scattered packets are not supported. This should be checked 2461 * outside of this function. 2462 */ 2463 if (!(rxq->rx_free_thresh >= RTE_PMD_TXGBE_RX_MAX_BURST)) { 2464 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 2465 "rxq->rx_free_thresh=%d, " 2466 "RTE_PMD_TXGBE_RX_MAX_BURST=%d", 2467 rxq->rx_free_thresh, RTE_PMD_TXGBE_RX_MAX_BURST); 2468 ret = -EINVAL; 2469 } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) { 2470 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 2471 "rxq->rx_free_thresh=%d, " 2472 "rxq->nb_rx_desc=%d", 2473 rxq->rx_free_thresh, rxq->nb_rx_desc); 2474 ret = -EINVAL; 2475 } else if (!((rxq->nb_rx_desc % rxq->rx_free_thresh) == 0)) { 2476 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 2477 "rxq->nb_rx_desc=%d, " 2478 "rxq->rx_free_thresh=%d", 2479 rxq->nb_rx_desc, rxq->rx_free_thresh); 2480 ret = -EINVAL; 2481 } 2482 2483 return ret; 2484 } 2485 2486 /* Reset dynamic txgbe_rx_queue fields back to defaults */ 2487 static void __rte_cold 2488 txgbe_reset_rx_queue(struct txgbe_adapter *adapter, struct txgbe_rx_queue *rxq) 2489 { 2490 static const struct txgbe_rx_desc zeroed_desc = { 2491 {{0}, {0} }, {{0}, {0} } }; 2492 unsigned int i; 2493 uint16_t len = rxq->nb_rx_desc; 2494 2495 /* 2496 * By default, the Rx queue setup function allocates enough memory for 2497 * TXGBE_RING_DESC_MAX. The Rx Burst bulk allocation function requires 2498 * extra memory at the end of the descriptor ring to be zero'd out. 2499 */ 2500 if (adapter->rx_bulk_alloc_allowed) 2501 /* zero out extra memory */ 2502 len += RTE_PMD_TXGBE_RX_MAX_BURST; 2503 2504 /* 2505 * Zero out HW ring memory. Zero out extra memory at the end of 2506 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function 2507 * reads extra memory as zeros. 2508 */ 2509 for (i = 0; i < len; i++) 2510 rxq->rx_ring[i] = zeroed_desc; 2511 2512 /* 2513 * initialize extra software ring entries. Space for these extra 2514 * entries is always allocated 2515 */ 2516 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf)); 2517 for (i = rxq->nb_rx_desc; i < len; ++i) 2518 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf; 2519 2520 rxq->rx_nb_avail = 0; 2521 rxq->rx_next_avail = 0; 2522 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); 2523 rxq->rx_tail = 0; 2524 rxq->nb_rx_hold = 0; 2525 rxq->pkt_first_seg = NULL; 2526 rxq->pkt_last_seg = NULL; 2527 } 2528 2529 int __rte_cold 2530 txgbe_dev_rx_queue_setup(struct rte_eth_dev *dev, 2531 uint16_t queue_idx, 2532 uint16_t nb_desc, 2533 unsigned int socket_id, 2534 const struct rte_eth_rxconf *rx_conf, 2535 struct rte_mempool *mp) 2536 { 2537 const struct rte_memzone *rz; 2538 struct txgbe_rx_queue *rxq; 2539 struct txgbe_hw *hw; 2540 uint16_t len; 2541 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev); 2542 uint64_t offloads; 2543 2544 PMD_INIT_FUNC_TRACE(); 2545 hw = TXGBE_DEV_HW(dev); 2546 2547 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads; 2548 2549 /* 2550 * Validate number of receive descriptors. 2551 * It must not exceed hardware maximum, and must be multiple 2552 * of TXGBE_ALIGN. 2553 */ 2554 if (nb_desc % TXGBE_RXD_ALIGN != 0 || 2555 nb_desc > TXGBE_RING_DESC_MAX || 2556 nb_desc < TXGBE_RING_DESC_MIN) { 2557 return -EINVAL; 2558 } 2559 2560 /* Free memory prior to re-allocation if needed... */ 2561 if (dev->data->rx_queues[queue_idx] != NULL) { 2562 txgbe_rx_queue_release(dev->data->rx_queues[queue_idx]); 2563 dev->data->rx_queues[queue_idx] = NULL; 2564 } 2565 2566 /* First allocate the rx queue data structure */ 2567 rxq = rte_zmalloc_socket("ethdev RX queue", 2568 sizeof(struct txgbe_rx_queue), 2569 RTE_CACHE_LINE_SIZE, socket_id); 2570 if (rxq == NULL) 2571 return -ENOMEM; 2572 rxq->mb_pool = mp; 2573 rxq->nb_rx_desc = nb_desc; 2574 rxq->rx_free_thresh = rx_conf->rx_free_thresh; 2575 rxq->queue_id = queue_idx; 2576 rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ? 2577 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx); 2578 rxq->port_id = dev->data->port_id; 2579 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) 2580 rxq->crc_len = RTE_ETHER_CRC_LEN; 2581 else 2582 rxq->crc_len = 0; 2583 rxq->drop_en = rx_conf->rx_drop_en; 2584 rxq->rx_deferred_start = rx_conf->rx_deferred_start; 2585 rxq->offloads = offloads; 2586 2587 /* 2588 * The packet type in RX descriptor is different for different NICs. 2589 * So set different masks for different NICs. 2590 */ 2591 rxq->pkt_type_mask = TXGBE_PTID_MASK; 2592 2593 /* 2594 * Allocate RX ring hardware descriptors. A memzone large enough to 2595 * handle the maximum ring size is allocated in order to allow for 2596 * resizing in later calls to the queue setup function. 2597 */ 2598 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, 2599 RX_RING_SZ, TXGBE_ALIGN, socket_id); 2600 if (rz == NULL) { 2601 txgbe_rx_queue_release(rxq); 2602 return -ENOMEM; 2603 } 2604 2605 /* 2606 * Zero init all the descriptors in the ring. 2607 */ 2608 memset(rz->addr, 0, RX_RING_SZ); 2609 2610 /* 2611 * Modified to setup VFRDT for Virtual Function 2612 */ 2613 if (hw->mac.type == txgbe_mac_raptor_vf) { 2614 rxq->rdt_reg_addr = 2615 TXGBE_REG_ADDR(hw, TXGBE_RXWP(queue_idx)); 2616 rxq->rdh_reg_addr = 2617 TXGBE_REG_ADDR(hw, TXGBE_RXRP(queue_idx)); 2618 } else { 2619 rxq->rdt_reg_addr = 2620 TXGBE_REG_ADDR(hw, TXGBE_RXWP(rxq->reg_idx)); 2621 rxq->rdh_reg_addr = 2622 TXGBE_REG_ADDR(hw, TXGBE_RXRP(rxq->reg_idx)); 2623 } 2624 2625 rxq->rx_ring_phys_addr = TMZ_PADDR(rz); 2626 rxq->rx_ring = (struct txgbe_rx_desc *)TMZ_VADDR(rz); 2627 2628 /* 2629 * Certain constraints must be met in order to use the bulk buffer 2630 * allocation Rx burst function. If any of Rx queues doesn't meet them 2631 * the feature should be disabled for the whole port. 2632 */ 2633 if (check_rx_burst_bulk_alloc_preconditions(rxq)) { 2634 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Rx Bulk Alloc " 2635 "preconditions - canceling the feature for " 2636 "the whole port[%d]", 2637 rxq->queue_id, rxq->port_id); 2638 adapter->rx_bulk_alloc_allowed = false; 2639 } 2640 2641 /* 2642 * Allocate software ring. Allow for space at the end of the 2643 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst 2644 * function does not access an invalid memory region. 2645 */ 2646 len = nb_desc; 2647 if (adapter->rx_bulk_alloc_allowed) 2648 len += RTE_PMD_TXGBE_RX_MAX_BURST; 2649 2650 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring", 2651 sizeof(struct txgbe_rx_entry) * len, 2652 RTE_CACHE_LINE_SIZE, socket_id); 2653 if (!rxq->sw_ring) { 2654 txgbe_rx_queue_release(rxq); 2655 return -ENOMEM; 2656 } 2657 2658 /* 2659 * Always allocate even if it's not going to be needed in order to 2660 * simplify the code. 2661 * 2662 * This ring is used in LRO and Scattered Rx cases and Scattered Rx may 2663 * be requested in txgbe_dev_rx_init(), which is called later from 2664 * dev_start() flow. 2665 */ 2666 rxq->sw_sc_ring = 2667 rte_zmalloc_socket("rxq->sw_sc_ring", 2668 sizeof(struct txgbe_scattered_rx_entry) * len, 2669 RTE_CACHE_LINE_SIZE, socket_id); 2670 if (!rxq->sw_sc_ring) { 2671 txgbe_rx_queue_release(rxq); 2672 return -ENOMEM; 2673 } 2674 2675 PMD_INIT_LOG(DEBUG, "sw_ring=%p sw_sc_ring=%p hw_ring=%p " 2676 "dma_addr=0x%" PRIx64, 2677 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring, 2678 rxq->rx_ring_phys_addr); 2679 2680 dev->data->rx_queues[queue_idx] = rxq; 2681 2682 txgbe_reset_rx_queue(adapter, rxq); 2683 2684 return 0; 2685 } 2686 2687 uint32_t 2688 txgbe_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id) 2689 { 2690 #define TXGBE_RXQ_SCAN_INTERVAL 4 2691 volatile struct txgbe_rx_desc *rxdp; 2692 struct txgbe_rx_queue *rxq; 2693 uint32_t desc = 0; 2694 2695 rxq = dev->data->rx_queues[rx_queue_id]; 2696 rxdp = &rxq->rx_ring[rxq->rx_tail]; 2697 2698 while ((desc < rxq->nb_rx_desc) && 2699 (rxdp->qw1.lo.status & 2700 rte_cpu_to_le_32(TXGBE_RXD_STAT_DD))) { 2701 desc += TXGBE_RXQ_SCAN_INTERVAL; 2702 rxdp += TXGBE_RXQ_SCAN_INTERVAL; 2703 if (rxq->rx_tail + desc >= rxq->nb_rx_desc) 2704 rxdp = &(rxq->rx_ring[rxq->rx_tail + 2705 desc - rxq->nb_rx_desc]); 2706 } 2707 2708 return desc; 2709 } 2710 2711 int 2712 txgbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset) 2713 { 2714 struct txgbe_rx_queue *rxq = rx_queue; 2715 volatile uint32_t *status; 2716 uint32_t nb_hold, desc; 2717 2718 if (unlikely(offset >= rxq->nb_rx_desc)) 2719 return -EINVAL; 2720 2721 nb_hold = rxq->nb_rx_hold; 2722 if (offset >= rxq->nb_rx_desc - nb_hold) 2723 return RTE_ETH_RX_DESC_UNAVAIL; 2724 2725 desc = rxq->rx_tail + offset; 2726 if (desc >= rxq->nb_rx_desc) 2727 desc -= rxq->nb_rx_desc; 2728 2729 status = &rxq->rx_ring[desc].qw1.lo.status; 2730 if (*status & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD)) 2731 return RTE_ETH_RX_DESC_DONE; 2732 2733 return RTE_ETH_RX_DESC_AVAIL; 2734 } 2735 2736 int 2737 txgbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset) 2738 { 2739 struct txgbe_tx_queue *txq = tx_queue; 2740 volatile uint32_t *status; 2741 uint32_t desc; 2742 2743 if (unlikely(offset >= txq->nb_tx_desc)) 2744 return -EINVAL; 2745 2746 desc = txq->tx_tail + offset; 2747 if (desc >= txq->nb_tx_desc) { 2748 desc -= txq->nb_tx_desc; 2749 if (desc >= txq->nb_tx_desc) 2750 desc -= txq->nb_tx_desc; 2751 } 2752 2753 status = &txq->tx_ring[desc].dw3; 2754 if (*status & rte_cpu_to_le_32(TXGBE_TXD_DD)) 2755 return RTE_ETH_TX_DESC_DONE; 2756 2757 return RTE_ETH_TX_DESC_FULL; 2758 } 2759 2760 void __rte_cold 2761 txgbe_dev_clear_queues(struct rte_eth_dev *dev) 2762 { 2763 unsigned int i; 2764 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev); 2765 2766 PMD_INIT_FUNC_TRACE(); 2767 2768 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2769 struct txgbe_tx_queue *txq = dev->data->tx_queues[i]; 2770 2771 if (txq != NULL) { 2772 txq->ops->release_mbufs(txq); 2773 txq->ops->reset(txq); 2774 } 2775 } 2776 2777 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2778 struct txgbe_rx_queue *rxq = dev->data->rx_queues[i]; 2779 2780 if (rxq != NULL) { 2781 txgbe_rx_queue_release_mbufs(rxq); 2782 txgbe_reset_rx_queue(adapter, rxq); 2783 } 2784 } 2785 } 2786 2787 void 2788 txgbe_dev_free_queues(struct rte_eth_dev *dev) 2789 { 2790 unsigned int i; 2791 2792 PMD_INIT_FUNC_TRACE(); 2793 2794 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2795 txgbe_dev_rx_queue_release(dev->data->rx_queues[i]); 2796 dev->data->rx_queues[i] = NULL; 2797 } 2798 dev->data->nb_rx_queues = 0; 2799 2800 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2801 txgbe_dev_tx_queue_release(dev->data->tx_queues[i]); 2802 dev->data->tx_queues[i] = NULL; 2803 } 2804 dev->data->nb_tx_queues = 0; 2805 } 2806 2807 /** 2808 * Receive Side Scaling (RSS) 2809 * 2810 * Principles: 2811 * The source and destination IP addresses of the IP header and the source 2812 * and destination ports of TCP/UDP headers, if any, of received packets are 2813 * hashed against a configurable random key to compute a 32-bit RSS hash result. 2814 * The seven (7) LSBs of the 32-bit hash result are used as an index into a 2815 * 128-entry redirection table (RETA). Each entry of the RETA provides a 3-bit 2816 * RSS output index which is used as the RX queue index where to store the 2817 * received packets. 2818 * The following output is supplied in the RX write-back descriptor: 2819 * - 32-bit result of the Microsoft RSS hash function, 2820 * - 4-bit RSS type field. 2821 */ 2822 2823 /* 2824 * Used as the default key. 2825 */ 2826 static uint8_t rss_intel_key[40] = { 2827 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 2828 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0, 2829 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4, 2830 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, 2831 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA, 2832 }; 2833 2834 static void 2835 txgbe_rss_disable(struct rte_eth_dev *dev) 2836 { 2837 struct txgbe_hw *hw; 2838 2839 hw = TXGBE_DEV_HW(dev); 2840 if (hw->mac.type == txgbe_mac_raptor_vf) 2841 wr32m(hw, TXGBE_VFPLCFG, TXGBE_VFPLCFG_RSSENA, 0); 2842 else 2843 wr32m(hw, TXGBE_RACTL, TXGBE_RACTL_RSSENA, 0); 2844 } 2845 2846 int 2847 txgbe_dev_rss_hash_update(struct rte_eth_dev *dev, 2848 struct rte_eth_rss_conf *rss_conf) 2849 { 2850 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 2851 uint8_t *hash_key; 2852 uint32_t mrqc; 2853 uint32_t rss_key; 2854 uint64_t rss_hf; 2855 uint16_t i; 2856 2857 if (!txgbe_rss_update_sp(hw->mac.type)) { 2858 PMD_DRV_LOG(ERR, "RSS hash update is not supported on this " 2859 "NIC."); 2860 return -ENOTSUP; 2861 } 2862 2863 hash_key = rss_conf->rss_key; 2864 if (hash_key) { 2865 /* Fill in RSS hash key */ 2866 for (i = 0; i < 10; i++) { 2867 rss_key = LS32(hash_key[(i * 4) + 0], 0, 0xFF); 2868 rss_key |= LS32(hash_key[(i * 4) + 1], 8, 0xFF); 2869 rss_key |= LS32(hash_key[(i * 4) + 2], 16, 0xFF); 2870 rss_key |= LS32(hash_key[(i * 4) + 3], 24, 0xFF); 2871 wr32at(hw, TXGBE_REG_RSSKEY, i, rss_key); 2872 } 2873 } 2874 2875 /* Set configured hashing protocols */ 2876 rss_hf = rss_conf->rss_hf & TXGBE_RSS_OFFLOAD_ALL; 2877 if (hw->mac.type == txgbe_mac_raptor_vf) { 2878 mrqc = rd32(hw, TXGBE_VFPLCFG); 2879 mrqc &= ~TXGBE_VFPLCFG_RSSMASK; 2880 if (rss_hf & ETH_RSS_IPV4) 2881 mrqc |= TXGBE_VFPLCFG_RSSIPV4; 2882 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) 2883 mrqc |= TXGBE_VFPLCFG_RSSIPV4TCP; 2884 if (rss_hf & ETH_RSS_IPV6 || 2885 rss_hf & ETH_RSS_IPV6_EX) 2886 mrqc |= TXGBE_VFPLCFG_RSSIPV6; 2887 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP || 2888 rss_hf & ETH_RSS_IPV6_TCP_EX) 2889 mrqc |= TXGBE_VFPLCFG_RSSIPV6TCP; 2890 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) 2891 mrqc |= TXGBE_VFPLCFG_RSSIPV4UDP; 2892 if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP || 2893 rss_hf & ETH_RSS_IPV6_UDP_EX) 2894 mrqc |= TXGBE_VFPLCFG_RSSIPV6UDP; 2895 2896 if (rss_hf) 2897 mrqc |= TXGBE_VFPLCFG_RSSENA; 2898 else 2899 mrqc &= ~TXGBE_VFPLCFG_RSSENA; 2900 2901 if (dev->data->nb_rx_queues > 3) 2902 mrqc |= TXGBE_VFPLCFG_RSSHASH(2); 2903 else if (dev->data->nb_rx_queues > 1) 2904 mrqc |= TXGBE_VFPLCFG_RSSHASH(1); 2905 2906 wr32(hw, TXGBE_VFPLCFG, mrqc); 2907 } else { 2908 mrqc = rd32(hw, TXGBE_RACTL); 2909 mrqc &= ~TXGBE_RACTL_RSSMASK; 2910 if (rss_hf & ETH_RSS_IPV4) 2911 mrqc |= TXGBE_RACTL_RSSIPV4; 2912 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) 2913 mrqc |= TXGBE_RACTL_RSSIPV4TCP; 2914 if (rss_hf & ETH_RSS_IPV6 || 2915 rss_hf & ETH_RSS_IPV6_EX) 2916 mrqc |= TXGBE_RACTL_RSSIPV6; 2917 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP || 2918 rss_hf & ETH_RSS_IPV6_TCP_EX) 2919 mrqc |= TXGBE_RACTL_RSSIPV6TCP; 2920 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) 2921 mrqc |= TXGBE_RACTL_RSSIPV4UDP; 2922 if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP || 2923 rss_hf & ETH_RSS_IPV6_UDP_EX) 2924 mrqc |= TXGBE_RACTL_RSSIPV6UDP; 2925 2926 if (rss_hf) 2927 mrqc |= TXGBE_RACTL_RSSENA; 2928 else 2929 mrqc &= ~TXGBE_RACTL_RSSENA; 2930 2931 wr32(hw, TXGBE_RACTL, mrqc); 2932 } 2933 2934 return 0; 2935 } 2936 2937 int 2938 txgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 2939 struct rte_eth_rss_conf *rss_conf) 2940 { 2941 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 2942 uint8_t *hash_key; 2943 uint32_t mrqc; 2944 uint32_t rss_key; 2945 uint64_t rss_hf; 2946 uint16_t i; 2947 2948 hash_key = rss_conf->rss_key; 2949 if (hash_key) { 2950 /* Return RSS hash key */ 2951 for (i = 0; i < 10; i++) { 2952 rss_key = rd32at(hw, TXGBE_REG_RSSKEY, i); 2953 hash_key[(i * 4) + 0] = RS32(rss_key, 0, 0xFF); 2954 hash_key[(i * 4) + 1] = RS32(rss_key, 8, 0xFF); 2955 hash_key[(i * 4) + 2] = RS32(rss_key, 16, 0xFF); 2956 hash_key[(i * 4) + 3] = RS32(rss_key, 24, 0xFF); 2957 } 2958 } 2959 2960 rss_hf = 0; 2961 if (hw->mac.type == txgbe_mac_raptor_vf) { 2962 mrqc = rd32(hw, TXGBE_VFPLCFG); 2963 if (mrqc & TXGBE_VFPLCFG_RSSIPV4) 2964 rss_hf |= ETH_RSS_IPV4; 2965 if (mrqc & TXGBE_VFPLCFG_RSSIPV4TCP) 2966 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP; 2967 if (mrqc & TXGBE_VFPLCFG_RSSIPV6) 2968 rss_hf |= ETH_RSS_IPV6 | 2969 ETH_RSS_IPV6_EX; 2970 if (mrqc & TXGBE_VFPLCFG_RSSIPV6TCP) 2971 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP | 2972 ETH_RSS_IPV6_TCP_EX; 2973 if (mrqc & TXGBE_VFPLCFG_RSSIPV4UDP) 2974 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP; 2975 if (mrqc & TXGBE_VFPLCFG_RSSIPV6UDP) 2976 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP | 2977 ETH_RSS_IPV6_UDP_EX; 2978 if (!(mrqc & TXGBE_VFPLCFG_RSSENA)) 2979 rss_hf = 0; 2980 } else { 2981 mrqc = rd32(hw, TXGBE_RACTL); 2982 if (mrqc & TXGBE_RACTL_RSSIPV4) 2983 rss_hf |= ETH_RSS_IPV4; 2984 if (mrqc & TXGBE_RACTL_RSSIPV4TCP) 2985 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP; 2986 if (mrqc & TXGBE_RACTL_RSSIPV6) 2987 rss_hf |= ETH_RSS_IPV6 | 2988 ETH_RSS_IPV6_EX; 2989 if (mrqc & TXGBE_RACTL_RSSIPV6TCP) 2990 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP | 2991 ETH_RSS_IPV6_TCP_EX; 2992 if (mrqc & TXGBE_RACTL_RSSIPV4UDP) 2993 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP; 2994 if (mrqc & TXGBE_RACTL_RSSIPV6UDP) 2995 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP | 2996 ETH_RSS_IPV6_UDP_EX; 2997 if (!(mrqc & TXGBE_RACTL_RSSENA)) 2998 rss_hf = 0; 2999 } 3000 3001 rss_hf &= TXGBE_RSS_OFFLOAD_ALL; 3002 3003 rss_conf->rss_hf = rss_hf; 3004 return 0; 3005 } 3006 3007 static void 3008 txgbe_rss_configure(struct rte_eth_dev *dev) 3009 { 3010 struct rte_eth_rss_conf rss_conf; 3011 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev); 3012 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 3013 uint32_t reta; 3014 uint16_t i; 3015 uint16_t j; 3016 3017 PMD_INIT_FUNC_TRACE(); 3018 3019 /* 3020 * Fill in redirection table 3021 * The byte-swap is needed because NIC registers are in 3022 * little-endian order. 3023 */ 3024 if (adapter->rss_reta_updated == 0) { 3025 reta = 0; 3026 for (i = 0, j = 0; i < ETH_RSS_RETA_SIZE_128; i++, j++) { 3027 if (j == dev->data->nb_rx_queues) 3028 j = 0; 3029 reta = (reta >> 8) | LS32(j, 24, 0xFF); 3030 if ((i & 3) == 3) 3031 wr32at(hw, TXGBE_REG_RSSTBL, i >> 2, reta); 3032 } 3033 } 3034 /* 3035 * Configure the RSS key and the RSS protocols used to compute 3036 * the RSS hash of input packets. 3037 */ 3038 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf; 3039 if (rss_conf.rss_key == NULL) 3040 rss_conf.rss_key = rss_intel_key; /* Default hash key */ 3041 txgbe_dev_rss_hash_update(dev, &rss_conf); 3042 } 3043 3044 #define NUM_VFTA_REGISTERS 128 3045 #define NIC_RX_BUFFER_SIZE 0x200 3046 3047 static void 3048 txgbe_vmdq_dcb_configure(struct rte_eth_dev *dev) 3049 { 3050 struct rte_eth_vmdq_dcb_conf *cfg; 3051 struct txgbe_hw *hw; 3052 enum rte_eth_nb_pools num_pools; 3053 uint32_t mrqc, vt_ctl, queue_mapping, vlanctrl; 3054 uint16_t pbsize; 3055 uint8_t nb_tcs; /* number of traffic classes */ 3056 int i; 3057 3058 PMD_INIT_FUNC_TRACE(); 3059 hw = TXGBE_DEV_HW(dev); 3060 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf; 3061 num_pools = cfg->nb_queue_pools; 3062 /* Check we have a valid number of pools */ 3063 if (num_pools != ETH_16_POOLS && num_pools != ETH_32_POOLS) { 3064 txgbe_rss_disable(dev); 3065 return; 3066 } 3067 /* 16 pools -> 8 traffic classes, 32 pools -> 4 traffic classes */ 3068 nb_tcs = (uint8_t)(ETH_VMDQ_DCB_NUM_QUEUES / (int)num_pools); 3069 3070 /* 3071 * split rx buffer up into sections, each for 1 traffic class 3072 */ 3073 pbsize = (uint16_t)(NIC_RX_BUFFER_SIZE / nb_tcs); 3074 for (i = 0; i < nb_tcs; i++) { 3075 uint32_t rxpbsize = rd32(hw, TXGBE_PBRXSIZE(i)); 3076 3077 rxpbsize &= (~(0x3FF << 10)); 3078 /* clear 10 bits. */ 3079 rxpbsize |= (pbsize << 10); /* set value */ 3080 wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize); 3081 } 3082 /* zero alloc all unused TCs */ 3083 for (i = nb_tcs; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3084 uint32_t rxpbsize = rd32(hw, TXGBE_PBRXSIZE(i)); 3085 3086 rxpbsize &= (~(0x3FF << 10)); 3087 /* clear 10 bits. */ 3088 wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize); 3089 } 3090 3091 if (num_pools == ETH_16_POOLS) { 3092 mrqc = TXGBE_PORTCTL_NUMTC_8; 3093 mrqc |= TXGBE_PORTCTL_NUMVT_16; 3094 } else { 3095 mrqc = TXGBE_PORTCTL_NUMTC_4; 3096 mrqc |= TXGBE_PORTCTL_NUMVT_32; 3097 } 3098 wr32m(hw, TXGBE_PORTCTL, 3099 TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK, mrqc); 3100 3101 vt_ctl = TXGBE_POOLCTL_RPLEN; 3102 if (cfg->enable_default_pool) 3103 vt_ctl |= TXGBE_POOLCTL_DEFPL(cfg->default_pool); 3104 else 3105 vt_ctl |= TXGBE_POOLCTL_DEFDSA; 3106 3107 wr32(hw, TXGBE_POOLCTL, vt_ctl); 3108 3109 queue_mapping = 0; 3110 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) 3111 /* 3112 * mapping is done with 3 bits per priority, 3113 * so shift by i*3 each time 3114 */ 3115 queue_mapping |= ((cfg->dcb_tc[i] & 0x07) << (i * 3)); 3116 3117 wr32(hw, TXGBE_RPUP2TC, queue_mapping); 3118 3119 wr32(hw, TXGBE_ARBRXCTL, TXGBE_ARBRXCTL_RRM); 3120 3121 /* enable vlan filtering and allow all vlan tags through */ 3122 vlanctrl = rd32(hw, TXGBE_VLANCTL); 3123 vlanctrl |= TXGBE_VLANCTL_VFE; /* enable vlan filters */ 3124 wr32(hw, TXGBE_VLANCTL, vlanctrl); 3125 3126 /* enable all vlan filters */ 3127 for (i = 0; i < NUM_VFTA_REGISTERS; i++) 3128 wr32(hw, TXGBE_VLANTBL(i), 0xFFFFFFFF); 3129 3130 wr32(hw, TXGBE_POOLRXENA(0), 3131 num_pools == ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF); 3132 3133 wr32(hw, TXGBE_ETHADDRIDX, 0); 3134 wr32(hw, TXGBE_ETHADDRASSL, 0xFFFFFFFF); 3135 wr32(hw, TXGBE_ETHADDRASSH, 0xFFFFFFFF); 3136 3137 /* set up filters for vlan tags as configured */ 3138 for (i = 0; i < cfg->nb_pool_maps; i++) { 3139 /* set vlan id in VF register and set the valid bit */ 3140 wr32(hw, TXGBE_PSRVLANIDX, i); 3141 wr32(hw, TXGBE_PSRVLAN, (TXGBE_PSRVLAN_EA | 3142 (cfg->pool_map[i].vlan_id & 0xFFF))); 3143 3144 wr32(hw, TXGBE_PSRVLANPLM(0), cfg->pool_map[i].pools); 3145 } 3146 } 3147 3148 /** 3149 * txgbe_dcb_config_tx_hw_config - Configure general DCB TX parameters 3150 * @dev: pointer to eth_dev structure 3151 * @dcb_config: pointer to txgbe_dcb_config structure 3152 */ 3153 static void 3154 txgbe_dcb_tx_hw_config(struct rte_eth_dev *dev, 3155 struct txgbe_dcb_config *dcb_config) 3156 { 3157 uint32_t reg; 3158 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 3159 3160 PMD_INIT_FUNC_TRACE(); 3161 3162 /* Disable the Tx desc arbiter */ 3163 reg = rd32(hw, TXGBE_ARBTXCTL); 3164 reg |= TXGBE_ARBTXCTL_DIA; 3165 wr32(hw, TXGBE_ARBTXCTL, reg); 3166 3167 /* Enable DCB for Tx with 8 TCs */ 3168 reg = rd32(hw, TXGBE_PORTCTL); 3169 reg &= TXGBE_PORTCTL_NUMTC_MASK; 3170 reg |= TXGBE_PORTCTL_DCB; 3171 if (dcb_config->num_tcs.pg_tcs == 8) 3172 reg |= TXGBE_PORTCTL_NUMTC_8; 3173 else 3174 reg |= TXGBE_PORTCTL_NUMTC_4; 3175 3176 wr32(hw, TXGBE_PORTCTL, reg); 3177 3178 /* Enable the Tx desc arbiter */ 3179 reg = rd32(hw, TXGBE_ARBTXCTL); 3180 reg &= ~TXGBE_ARBTXCTL_DIA; 3181 wr32(hw, TXGBE_ARBTXCTL, reg); 3182 } 3183 3184 /** 3185 * txgbe_vmdq_dcb_hw_tx_config - Configure general VMDQ+DCB TX parameters 3186 * @dev: pointer to rte_eth_dev structure 3187 * @dcb_config: pointer to txgbe_dcb_config structure 3188 */ 3189 static void 3190 txgbe_vmdq_dcb_hw_tx_config(struct rte_eth_dev *dev, 3191 struct txgbe_dcb_config *dcb_config) 3192 { 3193 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3194 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf; 3195 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 3196 3197 PMD_INIT_FUNC_TRACE(); 3198 /*PF VF Transmit Enable*/ 3199 wr32(hw, TXGBE_POOLTXENA(0), 3200 vmdq_tx_conf->nb_queue_pools == 3201 ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF); 3202 3203 /*Configure general DCB TX parameters*/ 3204 txgbe_dcb_tx_hw_config(dev, dcb_config); 3205 } 3206 3207 static void 3208 txgbe_vmdq_dcb_rx_config(struct rte_eth_dev *dev, 3209 struct txgbe_dcb_config *dcb_config) 3210 { 3211 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3212 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf; 3213 struct txgbe_dcb_tc_config *tc; 3214 uint8_t i, j; 3215 3216 /* convert rte_eth_conf.rx_adv_conf to struct txgbe_dcb_config */ 3217 if (vmdq_rx_conf->nb_queue_pools == ETH_16_POOLS) { 3218 dcb_config->num_tcs.pg_tcs = ETH_8_TCS; 3219 dcb_config->num_tcs.pfc_tcs = ETH_8_TCS; 3220 } else { 3221 dcb_config->num_tcs.pg_tcs = ETH_4_TCS; 3222 dcb_config->num_tcs.pfc_tcs = ETH_4_TCS; 3223 } 3224 3225 /* Initialize User Priority to Traffic Class mapping */ 3226 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) { 3227 tc = &dcb_config->tc_config[j]; 3228 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0; 3229 } 3230 3231 /* User Priority to Traffic Class mapping */ 3232 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3233 j = vmdq_rx_conf->dcb_tc[i]; 3234 tc = &dcb_config->tc_config[j]; 3235 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |= 3236 (uint8_t)(1 << i); 3237 } 3238 } 3239 3240 static void 3241 txgbe_dcb_vt_tx_config(struct rte_eth_dev *dev, 3242 struct txgbe_dcb_config *dcb_config) 3243 { 3244 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3245 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf; 3246 struct txgbe_dcb_tc_config *tc; 3247 uint8_t i, j; 3248 3249 /* convert rte_eth_conf.rx_adv_conf to struct txgbe_dcb_config */ 3250 if (vmdq_tx_conf->nb_queue_pools == ETH_16_POOLS) { 3251 dcb_config->num_tcs.pg_tcs = ETH_8_TCS; 3252 dcb_config->num_tcs.pfc_tcs = ETH_8_TCS; 3253 } else { 3254 dcb_config->num_tcs.pg_tcs = ETH_4_TCS; 3255 dcb_config->num_tcs.pfc_tcs = ETH_4_TCS; 3256 } 3257 3258 /* Initialize User Priority to Traffic Class mapping */ 3259 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) { 3260 tc = &dcb_config->tc_config[j]; 3261 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0; 3262 } 3263 3264 /* User Priority to Traffic Class mapping */ 3265 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3266 j = vmdq_tx_conf->dcb_tc[i]; 3267 tc = &dcb_config->tc_config[j]; 3268 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |= 3269 (uint8_t)(1 << i); 3270 } 3271 } 3272 3273 static void 3274 txgbe_dcb_rx_config(struct rte_eth_dev *dev, 3275 struct txgbe_dcb_config *dcb_config) 3276 { 3277 struct rte_eth_dcb_rx_conf *rx_conf = 3278 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf; 3279 struct txgbe_dcb_tc_config *tc; 3280 uint8_t i, j; 3281 3282 dcb_config->num_tcs.pg_tcs = (uint8_t)rx_conf->nb_tcs; 3283 dcb_config->num_tcs.pfc_tcs = (uint8_t)rx_conf->nb_tcs; 3284 3285 /* Initialize User Priority to Traffic Class mapping */ 3286 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) { 3287 tc = &dcb_config->tc_config[j]; 3288 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0; 3289 } 3290 3291 /* User Priority to Traffic Class mapping */ 3292 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3293 j = rx_conf->dcb_tc[i]; 3294 tc = &dcb_config->tc_config[j]; 3295 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |= 3296 (uint8_t)(1 << i); 3297 } 3298 } 3299 3300 static void 3301 txgbe_dcb_tx_config(struct rte_eth_dev *dev, 3302 struct txgbe_dcb_config *dcb_config) 3303 { 3304 struct rte_eth_dcb_tx_conf *tx_conf = 3305 &dev->data->dev_conf.tx_adv_conf.dcb_tx_conf; 3306 struct txgbe_dcb_tc_config *tc; 3307 uint8_t i, j; 3308 3309 dcb_config->num_tcs.pg_tcs = (uint8_t)tx_conf->nb_tcs; 3310 dcb_config->num_tcs.pfc_tcs = (uint8_t)tx_conf->nb_tcs; 3311 3312 /* Initialize User Priority to Traffic Class mapping */ 3313 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) { 3314 tc = &dcb_config->tc_config[j]; 3315 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0; 3316 } 3317 3318 /* User Priority to Traffic Class mapping */ 3319 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3320 j = tx_conf->dcb_tc[i]; 3321 tc = &dcb_config->tc_config[j]; 3322 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |= 3323 (uint8_t)(1 << i); 3324 } 3325 } 3326 3327 /** 3328 * txgbe_dcb_rx_hw_config - Configure general DCB RX HW parameters 3329 * @dev: pointer to eth_dev structure 3330 * @dcb_config: pointer to txgbe_dcb_config structure 3331 */ 3332 static void 3333 txgbe_dcb_rx_hw_config(struct rte_eth_dev *dev, 3334 struct txgbe_dcb_config *dcb_config) 3335 { 3336 uint32_t reg; 3337 uint32_t vlanctrl; 3338 uint8_t i; 3339 uint32_t q; 3340 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 3341 3342 PMD_INIT_FUNC_TRACE(); 3343 /* 3344 * Disable the arbiter before changing parameters 3345 * (always enable recycle mode; WSP) 3346 */ 3347 reg = TXGBE_ARBRXCTL_RRM | TXGBE_ARBRXCTL_WSP | TXGBE_ARBRXCTL_DIA; 3348 wr32(hw, TXGBE_ARBRXCTL, reg); 3349 3350 reg = rd32(hw, TXGBE_PORTCTL); 3351 reg &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK); 3352 if (dcb_config->num_tcs.pg_tcs == 4) { 3353 reg |= TXGBE_PORTCTL_NUMTC_4; 3354 if (dcb_config->vt_mode) 3355 reg |= TXGBE_PORTCTL_NUMVT_32; 3356 else 3357 wr32(hw, TXGBE_POOLCTL, 0); 3358 } 3359 3360 if (dcb_config->num_tcs.pg_tcs == 8) { 3361 reg |= TXGBE_PORTCTL_NUMTC_8; 3362 if (dcb_config->vt_mode) 3363 reg |= TXGBE_PORTCTL_NUMVT_16; 3364 else 3365 wr32(hw, TXGBE_POOLCTL, 0); 3366 } 3367 3368 wr32(hw, TXGBE_PORTCTL, reg); 3369 3370 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 3371 /* Disable drop for all queues in VMDQ mode*/ 3372 for (q = 0; q < TXGBE_MAX_RX_QUEUE_NUM; q++) { 3373 u32 val = 1 << (q % 32); 3374 wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val); 3375 } 3376 } else { 3377 /* Enable drop for all queues in SRIOV mode */ 3378 for (q = 0; q < TXGBE_MAX_RX_QUEUE_NUM; q++) { 3379 u32 val = 1 << (q % 32); 3380 wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val); 3381 } 3382 } 3383 3384 /* VLNCTL: enable vlan filtering and allow all vlan tags through */ 3385 vlanctrl = rd32(hw, TXGBE_VLANCTL); 3386 vlanctrl |= TXGBE_VLANCTL_VFE; /* enable vlan filters */ 3387 wr32(hw, TXGBE_VLANCTL, vlanctrl); 3388 3389 /* VLANTBL - enable all vlan filters */ 3390 for (i = 0; i < NUM_VFTA_REGISTERS; i++) 3391 wr32(hw, TXGBE_VLANTBL(i), 0xFFFFFFFF); 3392 3393 /* 3394 * Configure Rx packet plane (recycle mode; WSP) and 3395 * enable arbiter 3396 */ 3397 reg = TXGBE_ARBRXCTL_RRM | TXGBE_ARBRXCTL_WSP; 3398 wr32(hw, TXGBE_ARBRXCTL, reg); 3399 } 3400 3401 static void 3402 txgbe_dcb_hw_arbite_rx_config(struct txgbe_hw *hw, uint16_t *refill, 3403 uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map) 3404 { 3405 txgbe_dcb_config_rx_arbiter_raptor(hw, refill, max, bwg_id, 3406 tsa, map); 3407 } 3408 3409 static void 3410 txgbe_dcb_hw_arbite_tx_config(struct txgbe_hw *hw, uint16_t *refill, 3411 uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map) 3412 { 3413 switch (hw->mac.type) { 3414 case txgbe_mac_raptor: 3415 txgbe_dcb_config_tx_desc_arbiter_raptor(hw, refill, 3416 max, bwg_id, tsa); 3417 txgbe_dcb_config_tx_data_arbiter_raptor(hw, refill, 3418 max, bwg_id, tsa, map); 3419 break; 3420 default: 3421 break; 3422 } 3423 } 3424 3425 #define DCB_RX_CONFIG 1 3426 #define DCB_TX_CONFIG 1 3427 #define DCB_TX_PB 1024 3428 /** 3429 * txgbe_dcb_hw_configure - Enable DCB and configure 3430 * general DCB in VT mode and non-VT mode parameters 3431 * @dev: pointer to rte_eth_dev structure 3432 * @dcb_config: pointer to txgbe_dcb_config structure 3433 */ 3434 static int 3435 txgbe_dcb_hw_configure(struct rte_eth_dev *dev, 3436 struct txgbe_dcb_config *dcb_config) 3437 { 3438 int ret = 0; 3439 uint8_t i, pfc_en, nb_tcs; 3440 uint16_t pbsize, rx_buffer_size; 3441 uint8_t config_dcb_rx = 0; 3442 uint8_t config_dcb_tx = 0; 3443 uint8_t tsa[TXGBE_DCB_TC_MAX] = {0}; 3444 uint8_t bwgid[TXGBE_DCB_TC_MAX] = {0}; 3445 uint16_t refill[TXGBE_DCB_TC_MAX] = {0}; 3446 uint16_t max[TXGBE_DCB_TC_MAX] = {0}; 3447 uint8_t map[TXGBE_DCB_TC_MAX] = {0}; 3448 struct txgbe_dcb_tc_config *tc; 3449 uint32_t max_frame = dev->data->mtu + 3450 RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 3451 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 3452 struct txgbe_bw_conf *bw_conf = TXGBE_DEV_BW_CONF(dev); 3453 3454 switch (dev->data->dev_conf.rxmode.mq_mode) { 3455 case ETH_MQ_RX_VMDQ_DCB: 3456 dcb_config->vt_mode = true; 3457 config_dcb_rx = DCB_RX_CONFIG; 3458 /* 3459 * get dcb and VT rx configuration parameters 3460 * from rte_eth_conf 3461 */ 3462 txgbe_vmdq_dcb_rx_config(dev, dcb_config); 3463 /*Configure general VMDQ and DCB RX parameters*/ 3464 txgbe_vmdq_dcb_configure(dev); 3465 break; 3466 case ETH_MQ_RX_DCB: 3467 case ETH_MQ_RX_DCB_RSS: 3468 dcb_config->vt_mode = false; 3469 config_dcb_rx = DCB_RX_CONFIG; 3470 /* Get dcb TX configuration parameters from rte_eth_conf */ 3471 txgbe_dcb_rx_config(dev, dcb_config); 3472 /*Configure general DCB RX parameters*/ 3473 txgbe_dcb_rx_hw_config(dev, dcb_config); 3474 break; 3475 default: 3476 PMD_INIT_LOG(ERR, "Incorrect DCB RX mode configuration"); 3477 break; 3478 } 3479 switch (dev->data->dev_conf.txmode.mq_mode) { 3480 case ETH_MQ_TX_VMDQ_DCB: 3481 dcb_config->vt_mode = true; 3482 config_dcb_tx = DCB_TX_CONFIG; 3483 /* get DCB and VT TX configuration parameters 3484 * from rte_eth_conf 3485 */ 3486 txgbe_dcb_vt_tx_config(dev, dcb_config); 3487 /* Configure general VMDQ and DCB TX parameters */ 3488 txgbe_vmdq_dcb_hw_tx_config(dev, dcb_config); 3489 break; 3490 3491 case ETH_MQ_TX_DCB: 3492 dcb_config->vt_mode = false; 3493 config_dcb_tx = DCB_TX_CONFIG; 3494 /* get DCB TX configuration parameters from rte_eth_conf */ 3495 txgbe_dcb_tx_config(dev, dcb_config); 3496 /* Configure general DCB TX parameters */ 3497 txgbe_dcb_tx_hw_config(dev, dcb_config); 3498 break; 3499 default: 3500 PMD_INIT_LOG(ERR, "Incorrect DCB TX mode configuration"); 3501 break; 3502 } 3503 3504 nb_tcs = dcb_config->num_tcs.pfc_tcs; 3505 /* Unpack map */ 3506 txgbe_dcb_unpack_map_cee(dcb_config, TXGBE_DCB_RX_CONFIG, map); 3507 if (nb_tcs == ETH_4_TCS) { 3508 /* Avoid un-configured priority mapping to TC0 */ 3509 uint8_t j = 4; 3510 uint8_t mask = 0xFF; 3511 3512 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES - 4; i++) 3513 mask = (uint8_t)(mask & (~(1 << map[i]))); 3514 for (i = 0; mask && (i < TXGBE_DCB_TC_MAX); i++) { 3515 if ((mask & 0x1) && j < ETH_DCB_NUM_USER_PRIORITIES) 3516 map[j++] = i; 3517 mask >>= 1; 3518 } 3519 /* Re-configure 4 TCs BW */ 3520 for (i = 0; i < nb_tcs; i++) { 3521 tc = &dcb_config->tc_config[i]; 3522 if (bw_conf->tc_num != nb_tcs) 3523 tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent = 3524 (uint8_t)(100 / nb_tcs); 3525 tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent = 3526 (uint8_t)(100 / nb_tcs); 3527 } 3528 for (; i < TXGBE_DCB_TC_MAX; i++) { 3529 tc = &dcb_config->tc_config[i]; 3530 tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent = 0; 3531 tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent = 0; 3532 } 3533 } else { 3534 /* Re-configure 8 TCs BW */ 3535 for (i = 0; i < nb_tcs; i++) { 3536 tc = &dcb_config->tc_config[i]; 3537 if (bw_conf->tc_num != nb_tcs) 3538 tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent = 3539 (uint8_t)(100 / nb_tcs + (i & 1)); 3540 tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent = 3541 (uint8_t)(100 / nb_tcs + (i & 1)); 3542 } 3543 } 3544 3545 rx_buffer_size = NIC_RX_BUFFER_SIZE; 3546 3547 if (config_dcb_rx) { 3548 /* Set RX buffer size */ 3549 pbsize = (uint16_t)(rx_buffer_size / nb_tcs); 3550 uint32_t rxpbsize = pbsize << 10; 3551 3552 for (i = 0; i < nb_tcs; i++) 3553 wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize); 3554 3555 /* zero alloc all unused TCs */ 3556 for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++) 3557 wr32(hw, TXGBE_PBRXSIZE(i), 0); 3558 } 3559 if (config_dcb_tx) { 3560 /* Only support an equally distributed 3561 * Tx packet buffer strategy. 3562 */ 3563 uint32_t txpktsize = TXGBE_PBTXSIZE_MAX / nb_tcs; 3564 uint32_t txpbthresh = (txpktsize / DCB_TX_PB) - 3565 TXGBE_TXPKT_SIZE_MAX; 3566 3567 for (i = 0; i < nb_tcs; i++) { 3568 wr32(hw, TXGBE_PBTXSIZE(i), txpktsize); 3569 wr32(hw, TXGBE_PBTXDMATH(i), txpbthresh); 3570 } 3571 /* Clear unused TCs, if any, to zero buffer size*/ 3572 for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3573 wr32(hw, TXGBE_PBTXSIZE(i), 0); 3574 wr32(hw, TXGBE_PBTXDMATH(i), 0); 3575 } 3576 } 3577 3578 /*Calculates traffic class credits*/ 3579 txgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame, 3580 TXGBE_DCB_TX_CONFIG); 3581 txgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame, 3582 TXGBE_DCB_RX_CONFIG); 3583 3584 if (config_dcb_rx) { 3585 /* Unpack CEE standard containers */ 3586 txgbe_dcb_unpack_refill_cee(dcb_config, 3587 TXGBE_DCB_RX_CONFIG, refill); 3588 txgbe_dcb_unpack_max_cee(dcb_config, max); 3589 txgbe_dcb_unpack_bwgid_cee(dcb_config, 3590 TXGBE_DCB_RX_CONFIG, bwgid); 3591 txgbe_dcb_unpack_tsa_cee(dcb_config, 3592 TXGBE_DCB_RX_CONFIG, tsa); 3593 /* Configure PG(ETS) RX */ 3594 txgbe_dcb_hw_arbite_rx_config(hw, refill, max, bwgid, tsa, map); 3595 } 3596 3597 if (config_dcb_tx) { 3598 /* Unpack CEE standard containers */ 3599 txgbe_dcb_unpack_refill_cee(dcb_config, 3600 TXGBE_DCB_TX_CONFIG, refill); 3601 txgbe_dcb_unpack_max_cee(dcb_config, max); 3602 txgbe_dcb_unpack_bwgid_cee(dcb_config, 3603 TXGBE_DCB_TX_CONFIG, bwgid); 3604 txgbe_dcb_unpack_tsa_cee(dcb_config, 3605 TXGBE_DCB_TX_CONFIG, tsa); 3606 /* Configure PG(ETS) TX */ 3607 txgbe_dcb_hw_arbite_tx_config(hw, refill, max, bwgid, tsa, map); 3608 } 3609 3610 /* Configure queue statistics registers */ 3611 txgbe_dcb_config_tc_stats_raptor(hw, dcb_config); 3612 3613 /* Check if the PFC is supported */ 3614 if (dev->data->dev_conf.dcb_capability_en & ETH_DCB_PFC_SUPPORT) { 3615 pbsize = (uint16_t)(rx_buffer_size / nb_tcs); 3616 for (i = 0; i < nb_tcs; i++) { 3617 /* If the TC count is 8, 3618 * and the default high_water is 48, 3619 * the low_water is 16 as default. 3620 */ 3621 hw->fc.high_water[i] = (pbsize * 3) / 4; 3622 hw->fc.low_water[i] = pbsize / 4; 3623 /* Enable pfc for this TC */ 3624 tc = &dcb_config->tc_config[i]; 3625 tc->pfc = txgbe_dcb_pfc_enabled; 3626 } 3627 txgbe_dcb_unpack_pfc_cee(dcb_config, map, &pfc_en); 3628 if (dcb_config->num_tcs.pfc_tcs == ETH_4_TCS) 3629 pfc_en &= 0x0F; 3630 ret = txgbe_dcb_config_pfc(hw, pfc_en, map); 3631 } 3632 3633 return ret; 3634 } 3635 3636 void txgbe_configure_pb(struct rte_eth_dev *dev) 3637 { 3638 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 3639 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 3640 3641 int hdrm; 3642 int tc = dev_conf->rx_adv_conf.dcb_rx_conf.nb_tcs; 3643 3644 /* Reserve 256KB(/512KB) rx buffer for fdir */ 3645 hdrm = 256; /*KB*/ 3646 3647 hw->mac.setup_pba(hw, tc, hdrm, PBA_STRATEGY_EQUAL); 3648 } 3649 3650 void txgbe_configure_port(struct rte_eth_dev *dev) 3651 { 3652 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 3653 int i = 0; 3654 uint16_t tpids[8] = {RTE_ETHER_TYPE_VLAN, RTE_ETHER_TYPE_QINQ, 3655 0x9100, 0x9200, 3656 0x0000, 0x0000, 3657 0x0000, 0x0000}; 3658 3659 PMD_INIT_FUNC_TRACE(); 3660 3661 /* default outer vlan tpid */ 3662 wr32(hw, TXGBE_EXTAG, 3663 TXGBE_EXTAG_ETAG(RTE_ETHER_TYPE_ETAG) | 3664 TXGBE_EXTAG_VLAN(RTE_ETHER_TYPE_QINQ)); 3665 3666 /* default inner vlan tpid */ 3667 wr32m(hw, TXGBE_VLANCTL, 3668 TXGBE_VLANCTL_TPID_MASK, 3669 TXGBE_VLANCTL_TPID(RTE_ETHER_TYPE_VLAN)); 3670 wr32m(hw, TXGBE_DMATXCTRL, 3671 TXGBE_DMATXCTRL_TPID_MASK, 3672 TXGBE_DMATXCTRL_TPID(RTE_ETHER_TYPE_VLAN)); 3673 3674 /* default vlan tpid filters */ 3675 for (i = 0; i < 8; i++) { 3676 wr32m(hw, TXGBE_TAGTPID(i / 2), 3677 (i % 2 ? TXGBE_TAGTPID_MSB_MASK 3678 : TXGBE_TAGTPID_LSB_MASK), 3679 (i % 2 ? TXGBE_TAGTPID_MSB(tpids[i]) 3680 : TXGBE_TAGTPID_LSB(tpids[i]))); 3681 } 3682 3683 /* default vxlan port */ 3684 wr32(hw, TXGBE_VXLANPORT, 4789); 3685 } 3686 3687 /** 3688 * txgbe_configure_dcb - Configure DCB Hardware 3689 * @dev: pointer to rte_eth_dev 3690 */ 3691 void txgbe_configure_dcb(struct rte_eth_dev *dev) 3692 { 3693 struct txgbe_dcb_config *dcb_cfg = TXGBE_DEV_DCB_CONFIG(dev); 3694 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 3695 3696 PMD_INIT_FUNC_TRACE(); 3697 3698 /* check support mq_mode for DCB */ 3699 if (dev_conf->rxmode.mq_mode != ETH_MQ_RX_VMDQ_DCB && 3700 dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB && 3701 dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB_RSS) 3702 return; 3703 3704 if (dev->data->nb_rx_queues > ETH_DCB_NUM_QUEUES) 3705 return; 3706 3707 /** Configure DCB hardware **/ 3708 txgbe_dcb_hw_configure(dev, dcb_cfg); 3709 } 3710 3711 /* 3712 * VMDq only support for 10 GbE NIC. 3713 */ 3714 static void 3715 txgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev) 3716 { 3717 struct rte_eth_vmdq_rx_conf *cfg; 3718 struct txgbe_hw *hw; 3719 enum rte_eth_nb_pools num_pools; 3720 uint32_t mrqc, vt_ctl, vlanctrl; 3721 uint32_t vmolr = 0; 3722 int i; 3723 3724 PMD_INIT_FUNC_TRACE(); 3725 hw = TXGBE_DEV_HW(dev); 3726 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf; 3727 num_pools = cfg->nb_queue_pools; 3728 3729 txgbe_rss_disable(dev); 3730 3731 /* enable vmdq */ 3732 mrqc = TXGBE_PORTCTL_NUMVT_64; 3733 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, mrqc); 3734 3735 /* turn on virtualisation and set the default pool */ 3736 vt_ctl = TXGBE_POOLCTL_RPLEN; 3737 if (cfg->enable_default_pool) 3738 vt_ctl |= TXGBE_POOLCTL_DEFPL(cfg->default_pool); 3739 else 3740 vt_ctl |= TXGBE_POOLCTL_DEFDSA; 3741 3742 wr32(hw, TXGBE_POOLCTL, vt_ctl); 3743 3744 for (i = 0; i < (int)num_pools; i++) { 3745 vmolr = txgbe_convert_vm_rx_mask_to_val(cfg->rx_mode, vmolr); 3746 wr32(hw, TXGBE_POOLETHCTL(i), vmolr); 3747 } 3748 3749 /* enable vlan filtering and allow all vlan tags through */ 3750 vlanctrl = rd32(hw, TXGBE_VLANCTL); 3751 vlanctrl |= TXGBE_VLANCTL_VFE; /* enable vlan filters */ 3752 wr32(hw, TXGBE_VLANCTL, vlanctrl); 3753 3754 /* enable all vlan filters */ 3755 for (i = 0; i < NUM_VFTA_REGISTERS; i++) 3756 wr32(hw, TXGBE_VLANTBL(i), UINT32_MAX); 3757 3758 /* pool enabling for receive - 64 */ 3759 wr32(hw, TXGBE_POOLRXENA(0), UINT32_MAX); 3760 if (num_pools == ETH_64_POOLS) 3761 wr32(hw, TXGBE_POOLRXENA(1), UINT32_MAX); 3762 3763 /* 3764 * allow pools to read specific mac addresses 3765 * In this case, all pools should be able to read from mac addr 0 3766 */ 3767 wr32(hw, TXGBE_ETHADDRIDX, 0); 3768 wr32(hw, TXGBE_ETHADDRASSL, 0xFFFFFFFF); 3769 wr32(hw, TXGBE_ETHADDRASSH, 0xFFFFFFFF); 3770 3771 /* set up filters for vlan tags as configured */ 3772 for (i = 0; i < cfg->nb_pool_maps; i++) { 3773 /* set vlan id in VF register and set the valid bit */ 3774 wr32(hw, TXGBE_PSRVLANIDX, i); 3775 wr32(hw, TXGBE_PSRVLAN, (TXGBE_PSRVLAN_EA | 3776 TXGBE_PSRVLAN_VID(cfg->pool_map[i].vlan_id))); 3777 /* 3778 * Put the allowed pools in VFB reg. As we only have 16 or 64 3779 * pools, we only need to use the first half of the register 3780 * i.e. bits 0-31 3781 */ 3782 if (((cfg->pool_map[i].pools >> 32) & UINT32_MAX) == 0) 3783 wr32(hw, TXGBE_PSRVLANPLM(0), 3784 (cfg->pool_map[i].pools & UINT32_MAX)); 3785 else 3786 wr32(hw, TXGBE_PSRVLANPLM(1), 3787 ((cfg->pool_map[i].pools >> 32) & UINT32_MAX)); 3788 } 3789 3790 /* Tx General Switch Control Enables VMDQ loopback */ 3791 if (cfg->enable_loop_back) { 3792 wr32(hw, TXGBE_PSRCTL, TXGBE_PSRCTL_LBENA); 3793 for (i = 0; i < 64; i++) 3794 wr32m(hw, TXGBE_POOLETHCTL(i), 3795 TXGBE_POOLETHCTL_LLB, TXGBE_POOLETHCTL_LLB); 3796 } 3797 3798 txgbe_flush(hw); 3799 } 3800 3801 /* 3802 * txgbe_vmdq_tx_hw_configure - Configure general VMDq TX parameters 3803 * @hw: pointer to hardware structure 3804 */ 3805 static void 3806 txgbe_vmdq_tx_hw_configure(struct txgbe_hw *hw) 3807 { 3808 uint32_t reg; 3809 uint32_t q; 3810 3811 PMD_INIT_FUNC_TRACE(); 3812 /*PF VF Transmit Enable*/ 3813 wr32(hw, TXGBE_POOLTXENA(0), UINT32_MAX); 3814 wr32(hw, TXGBE_POOLTXENA(1), UINT32_MAX); 3815 3816 /* Disable the Tx desc arbiter */ 3817 reg = rd32(hw, TXGBE_ARBTXCTL); 3818 reg |= TXGBE_ARBTXCTL_DIA; 3819 wr32(hw, TXGBE_ARBTXCTL, reg); 3820 3821 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, 3822 TXGBE_PORTCTL_NUMVT_64); 3823 3824 /* Disable drop for all queues */ 3825 for (q = 0; q < 128; q++) { 3826 u32 val = 1 << (q % 32); 3827 wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val); 3828 } 3829 3830 /* Enable the Tx desc arbiter */ 3831 reg = rd32(hw, TXGBE_ARBTXCTL); 3832 reg &= ~TXGBE_ARBTXCTL_DIA; 3833 wr32(hw, TXGBE_ARBTXCTL, reg); 3834 3835 txgbe_flush(hw); 3836 } 3837 3838 static int __rte_cold 3839 txgbe_alloc_rx_queue_mbufs(struct txgbe_rx_queue *rxq) 3840 { 3841 struct txgbe_rx_entry *rxe = rxq->sw_ring; 3842 uint64_t dma_addr; 3843 unsigned int i; 3844 3845 /* Initialize software ring entries */ 3846 for (i = 0; i < rxq->nb_rx_desc; i++) { 3847 volatile struct txgbe_rx_desc *rxd; 3848 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool); 3849 3850 if (mbuf == NULL) { 3851 PMD_INIT_LOG(ERR, "RX mbuf alloc failed queue_id=%u", 3852 (unsigned int)rxq->queue_id); 3853 return -ENOMEM; 3854 } 3855 3856 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 3857 mbuf->port = rxq->port_id; 3858 3859 dma_addr = 3860 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 3861 rxd = &rxq->rx_ring[i]; 3862 TXGBE_RXD_HDRADDR(rxd, 0); 3863 TXGBE_RXD_PKTADDR(rxd, dma_addr); 3864 rxe[i].mbuf = mbuf; 3865 } 3866 3867 return 0; 3868 } 3869 3870 static int 3871 txgbe_config_vf_rss(struct rte_eth_dev *dev) 3872 { 3873 struct txgbe_hw *hw; 3874 uint32_t mrqc; 3875 3876 txgbe_rss_configure(dev); 3877 3878 hw = TXGBE_DEV_HW(dev); 3879 3880 /* enable VF RSS */ 3881 mrqc = rd32(hw, TXGBE_PORTCTL); 3882 mrqc &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK); 3883 switch (RTE_ETH_DEV_SRIOV(dev).active) { 3884 case ETH_64_POOLS: 3885 mrqc |= TXGBE_PORTCTL_NUMVT_64; 3886 break; 3887 3888 case ETH_32_POOLS: 3889 mrqc |= TXGBE_PORTCTL_NUMVT_32; 3890 break; 3891 3892 default: 3893 PMD_INIT_LOG(ERR, "Invalid pool number in IOV mode with VMDQ RSS"); 3894 return -EINVAL; 3895 } 3896 3897 wr32(hw, TXGBE_PORTCTL, mrqc); 3898 3899 return 0; 3900 } 3901 3902 static int 3903 txgbe_config_vf_default(struct rte_eth_dev *dev) 3904 { 3905 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 3906 uint32_t mrqc; 3907 3908 mrqc = rd32(hw, TXGBE_PORTCTL); 3909 mrqc &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK); 3910 switch (RTE_ETH_DEV_SRIOV(dev).active) { 3911 case ETH_64_POOLS: 3912 mrqc |= TXGBE_PORTCTL_NUMVT_64; 3913 break; 3914 3915 case ETH_32_POOLS: 3916 mrqc |= TXGBE_PORTCTL_NUMVT_32; 3917 break; 3918 3919 case ETH_16_POOLS: 3920 mrqc |= TXGBE_PORTCTL_NUMVT_16; 3921 break; 3922 default: 3923 PMD_INIT_LOG(ERR, 3924 "invalid pool number in IOV mode"); 3925 return 0; 3926 } 3927 3928 wr32(hw, TXGBE_PORTCTL, mrqc); 3929 3930 return 0; 3931 } 3932 3933 static int 3934 txgbe_dev_mq_rx_configure(struct rte_eth_dev *dev) 3935 { 3936 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 3937 /* 3938 * SRIOV inactive scheme 3939 * any DCB/RSS w/o VMDq multi-queue setting 3940 */ 3941 switch (dev->data->dev_conf.rxmode.mq_mode) { 3942 case ETH_MQ_RX_RSS: 3943 case ETH_MQ_RX_DCB_RSS: 3944 case ETH_MQ_RX_VMDQ_RSS: 3945 txgbe_rss_configure(dev); 3946 break; 3947 3948 case ETH_MQ_RX_VMDQ_DCB: 3949 txgbe_vmdq_dcb_configure(dev); 3950 break; 3951 3952 case ETH_MQ_RX_VMDQ_ONLY: 3953 txgbe_vmdq_rx_hw_configure(dev); 3954 break; 3955 3956 case ETH_MQ_RX_NONE: 3957 default: 3958 /* if mq_mode is none, disable rss mode.*/ 3959 txgbe_rss_disable(dev); 3960 break; 3961 } 3962 } else { 3963 /* SRIOV active scheme 3964 * Support RSS together with SRIOV. 3965 */ 3966 switch (dev->data->dev_conf.rxmode.mq_mode) { 3967 case ETH_MQ_RX_RSS: 3968 case ETH_MQ_RX_VMDQ_RSS: 3969 txgbe_config_vf_rss(dev); 3970 break; 3971 case ETH_MQ_RX_VMDQ_DCB: 3972 case ETH_MQ_RX_DCB: 3973 /* In SRIOV, the configuration is the same as VMDq case */ 3974 txgbe_vmdq_dcb_configure(dev); 3975 break; 3976 /* DCB/RSS together with SRIOV is not supported */ 3977 case ETH_MQ_RX_VMDQ_DCB_RSS: 3978 case ETH_MQ_RX_DCB_RSS: 3979 PMD_INIT_LOG(ERR, 3980 "Could not support DCB/RSS with VMDq & SRIOV"); 3981 return -1; 3982 default: 3983 txgbe_config_vf_default(dev); 3984 break; 3985 } 3986 } 3987 3988 return 0; 3989 } 3990 3991 static int 3992 txgbe_dev_mq_tx_configure(struct rte_eth_dev *dev) 3993 { 3994 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 3995 uint32_t mtqc; 3996 uint32_t rttdcs; 3997 3998 /* disable arbiter */ 3999 rttdcs = rd32(hw, TXGBE_ARBTXCTL); 4000 rttdcs |= TXGBE_ARBTXCTL_DIA; 4001 wr32(hw, TXGBE_ARBTXCTL, rttdcs); 4002 4003 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 4004 /* 4005 * SRIOV inactive scheme 4006 * any DCB w/o VMDq multi-queue setting 4007 */ 4008 if (dev->data->dev_conf.txmode.mq_mode == ETH_MQ_TX_VMDQ_ONLY) 4009 txgbe_vmdq_tx_hw_configure(hw); 4010 else 4011 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, 0); 4012 } else { 4013 switch (RTE_ETH_DEV_SRIOV(dev).active) { 4014 /* 4015 * SRIOV active scheme 4016 * FIXME if support DCB together with VMDq & SRIOV 4017 */ 4018 case ETH_64_POOLS: 4019 mtqc = TXGBE_PORTCTL_NUMVT_64; 4020 break; 4021 case ETH_32_POOLS: 4022 mtqc = TXGBE_PORTCTL_NUMVT_32; 4023 break; 4024 case ETH_16_POOLS: 4025 mtqc = TXGBE_PORTCTL_NUMVT_16; 4026 break; 4027 default: 4028 mtqc = 0; 4029 PMD_INIT_LOG(ERR, "invalid pool number in IOV mode"); 4030 } 4031 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, mtqc); 4032 } 4033 4034 /* re-enable arbiter */ 4035 rttdcs &= ~TXGBE_ARBTXCTL_DIA; 4036 wr32(hw, TXGBE_ARBTXCTL, rttdcs); 4037 4038 return 0; 4039 } 4040 4041 /** 4042 * txgbe_get_rscctl_maxdesc 4043 * 4044 * @pool Memory pool of the Rx queue 4045 */ 4046 static inline uint32_t 4047 txgbe_get_rscctl_maxdesc(struct rte_mempool *pool) 4048 { 4049 struct rte_pktmbuf_pool_private *mp_priv = rte_mempool_get_priv(pool); 4050 4051 uint16_t maxdesc = 4052 RTE_IPV4_MAX_PKT_LEN / 4053 (mp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM); 4054 4055 if (maxdesc >= 16) 4056 return TXGBE_RXCFG_RSCMAX_16; 4057 else if (maxdesc >= 8) 4058 return TXGBE_RXCFG_RSCMAX_8; 4059 else if (maxdesc >= 4) 4060 return TXGBE_RXCFG_RSCMAX_4; 4061 else 4062 return TXGBE_RXCFG_RSCMAX_1; 4063 } 4064 4065 /** 4066 * txgbe_set_rsc - configure RSC related port HW registers 4067 * 4068 * Configures the port's RSC related registers. 4069 * 4070 * @dev port handle 4071 * 4072 * Returns 0 in case of success or a non-zero error code 4073 */ 4074 static int 4075 txgbe_set_rsc(struct rte_eth_dev *dev) 4076 { 4077 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; 4078 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 4079 struct rte_eth_dev_info dev_info = { 0 }; 4080 bool rsc_capable = false; 4081 uint16_t i; 4082 uint32_t rdrxctl; 4083 uint32_t rfctl; 4084 4085 /* Sanity check */ 4086 dev->dev_ops->dev_infos_get(dev, &dev_info); 4087 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO) 4088 rsc_capable = true; 4089 4090 if (!rsc_capable && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) { 4091 PMD_INIT_LOG(CRIT, "LRO is requested on HW that doesn't " 4092 "support it"); 4093 return -EINVAL; 4094 } 4095 4096 /* RSC global configuration */ 4097 4098 if ((rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC) && 4099 (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) { 4100 PMD_INIT_LOG(CRIT, "LRO can't be enabled when HW CRC " 4101 "is disabled"); 4102 return -EINVAL; 4103 } 4104 4105 rfctl = rd32(hw, TXGBE_PSRCTL); 4106 if (rsc_capable && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) 4107 rfctl &= ~TXGBE_PSRCTL_RSCDIA; 4108 else 4109 rfctl |= TXGBE_PSRCTL_RSCDIA; 4110 wr32(hw, TXGBE_PSRCTL, rfctl); 4111 4112 /* If LRO hasn't been requested - we are done here. */ 4113 if (!(rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) 4114 return 0; 4115 4116 /* Set PSRCTL.RSCACK bit */ 4117 rdrxctl = rd32(hw, TXGBE_PSRCTL); 4118 rdrxctl |= TXGBE_PSRCTL_RSCACK; 4119 wr32(hw, TXGBE_PSRCTL, rdrxctl); 4120 4121 /* Per-queue RSC configuration */ 4122 for (i = 0; i < dev->data->nb_rx_queues; i++) { 4123 struct txgbe_rx_queue *rxq = dev->data->rx_queues[i]; 4124 uint32_t srrctl = 4125 rd32(hw, TXGBE_RXCFG(rxq->reg_idx)); 4126 uint32_t psrtype = 4127 rd32(hw, TXGBE_POOLRSS(rxq->reg_idx)); 4128 uint32_t eitr = 4129 rd32(hw, TXGBE_ITR(rxq->reg_idx)); 4130 4131 /* 4132 * txgbe PMD doesn't support header-split at the moment. 4133 */ 4134 srrctl &= ~TXGBE_RXCFG_HDRLEN_MASK; 4135 srrctl |= TXGBE_RXCFG_HDRLEN(128); 4136 4137 /* 4138 * TODO: Consider setting the Receive Descriptor Minimum 4139 * Threshold Size for an RSC case. This is not an obviously 4140 * beneficiary option but the one worth considering... 4141 */ 4142 4143 srrctl |= TXGBE_RXCFG_RSCENA; 4144 srrctl &= ~TXGBE_RXCFG_RSCMAX_MASK; 4145 srrctl |= txgbe_get_rscctl_maxdesc(rxq->mb_pool); 4146 psrtype |= TXGBE_POOLRSS_L4HDR; 4147 4148 /* 4149 * RSC: Set ITR interval corresponding to 2K ints/s. 4150 * 4151 * Full-sized RSC aggregations for a 10Gb/s link will 4152 * arrive at about 20K aggregation/s rate. 4153 * 4154 * 2K inst/s rate will make only 10% of the 4155 * aggregations to be closed due to the interrupt timer 4156 * expiration for a streaming at wire-speed case. 4157 * 4158 * For a sparse streaming case this setting will yield 4159 * at most 500us latency for a single RSC aggregation. 4160 */ 4161 eitr &= ~TXGBE_ITR_IVAL_MASK; 4162 eitr |= TXGBE_ITR_IVAL_10G(TXGBE_QUEUE_ITR_INTERVAL_DEFAULT); 4163 eitr |= TXGBE_ITR_WRDSA; 4164 4165 wr32(hw, TXGBE_RXCFG(rxq->reg_idx), srrctl); 4166 wr32(hw, TXGBE_POOLRSS(rxq->reg_idx), psrtype); 4167 wr32(hw, TXGBE_ITR(rxq->reg_idx), eitr); 4168 4169 /* 4170 * RSC requires the mapping of the queue to the 4171 * interrupt vector. 4172 */ 4173 txgbe_set_ivar_map(hw, 0, rxq->reg_idx, i); 4174 } 4175 4176 dev->data->lro = 1; 4177 4178 PMD_INIT_LOG(DEBUG, "enabling LRO mode"); 4179 4180 return 0; 4181 } 4182 4183 void __rte_cold 4184 txgbe_set_rx_function(struct rte_eth_dev *dev) 4185 { 4186 uint16_t i; 4187 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev); 4188 4189 /* 4190 * Initialize the appropriate LRO callback. 4191 * 4192 * If all queues satisfy the bulk allocation preconditions 4193 * (adapter->rx_bulk_alloc_allowed is TRUE) then we may use 4194 * bulk allocation. Otherwise use a single allocation version. 4195 */ 4196 if (dev->data->lro) { 4197 if (adapter->rx_bulk_alloc_allowed) { 4198 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a bulk " 4199 "allocation version"); 4200 dev->rx_pkt_burst = txgbe_recv_pkts_lro_bulk_alloc; 4201 } else { 4202 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a single " 4203 "allocation version"); 4204 dev->rx_pkt_burst = txgbe_recv_pkts_lro_single_alloc; 4205 } 4206 } else if (dev->data->scattered_rx) { 4207 /* 4208 * Set the non-LRO scattered callback: there are bulk and 4209 * single allocation versions. 4210 */ 4211 if (adapter->rx_bulk_alloc_allowed) { 4212 PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk " 4213 "allocation callback (port=%d).", 4214 dev->data->port_id); 4215 dev->rx_pkt_burst = txgbe_recv_pkts_lro_bulk_alloc; 4216 } else { 4217 PMD_INIT_LOG(DEBUG, "Using Regular (non-vector, " 4218 "single allocation) " 4219 "Scattered Rx callback " 4220 "(port=%d).", 4221 dev->data->port_id); 4222 4223 dev->rx_pkt_burst = txgbe_recv_pkts_lro_single_alloc; 4224 } 4225 /* 4226 * Below we set "simple" callbacks according to port/queues parameters. 4227 * If parameters allow we are going to choose between the following 4228 * callbacks: 4229 * - Bulk Allocation 4230 * - Single buffer allocation (the simplest one) 4231 */ 4232 } else if (adapter->rx_bulk_alloc_allowed) { 4233 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " 4234 "satisfied. Rx Burst Bulk Alloc function " 4235 "will be used on port=%d.", 4236 dev->data->port_id); 4237 4238 dev->rx_pkt_burst = txgbe_recv_pkts_bulk_alloc; 4239 } else { 4240 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not " 4241 "satisfied, or Scattered Rx is requested " 4242 "(port=%d).", 4243 dev->data->port_id); 4244 4245 dev->rx_pkt_burst = txgbe_recv_pkts; 4246 } 4247 4248 #ifdef RTE_LIB_SECURITY 4249 for (i = 0; i < dev->data->nb_rx_queues; i++) { 4250 struct txgbe_rx_queue *rxq = dev->data->rx_queues[i]; 4251 4252 rxq->using_ipsec = !!(dev->data->dev_conf.rxmode.offloads & 4253 DEV_RX_OFFLOAD_SECURITY); 4254 } 4255 #endif 4256 } 4257 4258 /* 4259 * Initializes Receive Unit. 4260 */ 4261 int __rte_cold 4262 txgbe_dev_rx_init(struct rte_eth_dev *dev) 4263 { 4264 struct txgbe_hw *hw; 4265 struct txgbe_rx_queue *rxq; 4266 uint64_t bus_addr; 4267 uint32_t fctrl; 4268 uint32_t hlreg0; 4269 uint32_t srrctl; 4270 uint32_t rdrxctl; 4271 uint32_t rxcsum; 4272 uint16_t buf_size; 4273 uint16_t i; 4274 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; 4275 int rc; 4276 4277 PMD_INIT_FUNC_TRACE(); 4278 hw = TXGBE_DEV_HW(dev); 4279 4280 /* 4281 * Make sure receives are disabled while setting 4282 * up the RX context (registers, descriptor rings, etc.). 4283 */ 4284 wr32m(hw, TXGBE_MACRXCFG, TXGBE_MACRXCFG_ENA, 0); 4285 wr32m(hw, TXGBE_PBRXCTL, TXGBE_PBRXCTL_ENA, 0); 4286 4287 /* Enable receipt of broadcasted frames */ 4288 fctrl = rd32(hw, TXGBE_PSRCTL); 4289 fctrl |= TXGBE_PSRCTL_BCA; 4290 wr32(hw, TXGBE_PSRCTL, fctrl); 4291 4292 /* 4293 * Configure CRC stripping, if any. 4294 */ 4295 hlreg0 = rd32(hw, TXGBE_SECRXCTL); 4296 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC) 4297 hlreg0 &= ~TXGBE_SECRXCTL_CRCSTRIP; 4298 else 4299 hlreg0 |= TXGBE_SECRXCTL_CRCSTRIP; 4300 wr32(hw, TXGBE_SECRXCTL, hlreg0); 4301 4302 /* 4303 * Configure jumbo frame support, if any. 4304 */ 4305 if (rx_conf->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) { 4306 wr32m(hw, TXGBE_FRMSZ, TXGBE_FRMSZ_MAX_MASK, 4307 TXGBE_FRMSZ_MAX(rx_conf->max_rx_pkt_len)); 4308 } else { 4309 wr32m(hw, TXGBE_FRMSZ, TXGBE_FRMSZ_MAX_MASK, 4310 TXGBE_FRMSZ_MAX(TXGBE_FRAME_SIZE_DFT)); 4311 } 4312 4313 /* 4314 * If loopback mode is configured, set LPBK bit. 4315 */ 4316 hlreg0 = rd32(hw, TXGBE_PSRCTL); 4317 if (hw->mac.type == txgbe_mac_raptor && 4318 dev->data->dev_conf.lpbk_mode) 4319 hlreg0 |= TXGBE_PSRCTL_LBENA; 4320 else 4321 hlreg0 &= ~TXGBE_PSRCTL_LBENA; 4322 4323 wr32(hw, TXGBE_PSRCTL, hlreg0); 4324 4325 /* 4326 * Assume no header split and no VLAN strip support 4327 * on any Rx queue first . 4328 */ 4329 rx_conf->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP; 4330 4331 /* Setup RX queues */ 4332 for (i = 0; i < dev->data->nb_rx_queues; i++) { 4333 rxq = dev->data->rx_queues[i]; 4334 4335 /* 4336 * Reset crc_len in case it was changed after queue setup by a 4337 * call to configure. 4338 */ 4339 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC) 4340 rxq->crc_len = RTE_ETHER_CRC_LEN; 4341 else 4342 rxq->crc_len = 0; 4343 4344 /* Setup the Base and Length of the Rx Descriptor Rings */ 4345 bus_addr = rxq->rx_ring_phys_addr; 4346 wr32(hw, TXGBE_RXBAL(rxq->reg_idx), 4347 (uint32_t)(bus_addr & BIT_MASK32)); 4348 wr32(hw, TXGBE_RXBAH(rxq->reg_idx), 4349 (uint32_t)(bus_addr >> 32)); 4350 wr32(hw, TXGBE_RXRP(rxq->reg_idx), 0); 4351 wr32(hw, TXGBE_RXWP(rxq->reg_idx), 0); 4352 4353 srrctl = TXGBE_RXCFG_RNGLEN(rxq->nb_rx_desc); 4354 4355 /* Set if packets are dropped when no descriptors available */ 4356 if (rxq->drop_en) 4357 srrctl |= TXGBE_RXCFG_DROP; 4358 4359 /* 4360 * Configure the RX buffer size in the PKTLEN field of 4361 * the RXCFG register of the queue. 4362 * The value is in 1 KB resolution. Valid values can be from 4363 * 1 KB to 16 KB. 4364 */ 4365 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - 4366 RTE_PKTMBUF_HEADROOM); 4367 buf_size = ROUND_UP(buf_size, 0x1 << 10); 4368 srrctl |= TXGBE_RXCFG_PKTLEN(buf_size); 4369 4370 wr32(hw, TXGBE_RXCFG(rxq->reg_idx), srrctl); 4371 4372 /* It adds dual VLAN length for supporting dual VLAN */ 4373 if (dev->data->dev_conf.rxmode.max_rx_pkt_len + 4374 2 * TXGBE_VLAN_TAG_SIZE > buf_size) 4375 dev->data->scattered_rx = 1; 4376 if (rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 4377 rx_conf->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP; 4378 } 4379 4380 if (rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER) 4381 dev->data->scattered_rx = 1; 4382 4383 /* 4384 * Device configured with multiple RX queues. 4385 */ 4386 txgbe_dev_mq_rx_configure(dev); 4387 4388 /* 4389 * Setup the Checksum Register. 4390 * Disable Full-Packet Checksum which is mutually exclusive with RSS. 4391 * Enable IP/L4 checksum computation by hardware if requested to do so. 4392 */ 4393 rxcsum = rd32(hw, TXGBE_PSRCTL); 4394 rxcsum |= TXGBE_PSRCTL_PCSD; 4395 if (rx_conf->offloads & DEV_RX_OFFLOAD_CHECKSUM) 4396 rxcsum |= TXGBE_PSRCTL_L4CSUM; 4397 else 4398 rxcsum &= ~TXGBE_PSRCTL_L4CSUM; 4399 4400 wr32(hw, TXGBE_PSRCTL, rxcsum); 4401 4402 if (hw->mac.type == txgbe_mac_raptor) { 4403 rdrxctl = rd32(hw, TXGBE_SECRXCTL); 4404 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC) 4405 rdrxctl &= ~TXGBE_SECRXCTL_CRCSTRIP; 4406 else 4407 rdrxctl |= TXGBE_SECRXCTL_CRCSTRIP; 4408 wr32(hw, TXGBE_SECRXCTL, rdrxctl); 4409 } 4410 4411 rc = txgbe_set_rsc(dev); 4412 if (rc) 4413 return rc; 4414 4415 txgbe_set_rx_function(dev); 4416 4417 return 0; 4418 } 4419 4420 /* 4421 * Initializes Transmit Unit. 4422 */ 4423 void __rte_cold 4424 txgbe_dev_tx_init(struct rte_eth_dev *dev) 4425 { 4426 struct txgbe_hw *hw; 4427 struct txgbe_tx_queue *txq; 4428 uint64_t bus_addr; 4429 uint16_t i; 4430 4431 PMD_INIT_FUNC_TRACE(); 4432 hw = TXGBE_DEV_HW(dev); 4433 4434 /* Setup the Base and Length of the Tx Descriptor Rings */ 4435 for (i = 0; i < dev->data->nb_tx_queues; i++) { 4436 txq = dev->data->tx_queues[i]; 4437 4438 bus_addr = txq->tx_ring_phys_addr; 4439 wr32(hw, TXGBE_TXBAL(txq->reg_idx), 4440 (uint32_t)(bus_addr & BIT_MASK32)); 4441 wr32(hw, TXGBE_TXBAH(txq->reg_idx), 4442 (uint32_t)(bus_addr >> 32)); 4443 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), TXGBE_TXCFG_BUFLEN_MASK, 4444 TXGBE_TXCFG_BUFLEN(txq->nb_tx_desc)); 4445 /* Setup the HW Tx Head and TX Tail descriptor pointers */ 4446 wr32(hw, TXGBE_TXRP(txq->reg_idx), 0); 4447 wr32(hw, TXGBE_TXWP(txq->reg_idx), 0); 4448 } 4449 4450 /* Device configured with multiple TX queues. */ 4451 txgbe_dev_mq_tx_configure(dev); 4452 } 4453 4454 /* 4455 * Set up link loopback mode Tx->Rx. 4456 */ 4457 static inline void __rte_cold 4458 txgbe_setup_loopback_link_raptor(struct txgbe_hw *hw) 4459 { 4460 PMD_INIT_FUNC_TRACE(); 4461 4462 wr32m(hw, TXGBE_MACRXCFG, TXGBE_MACRXCFG_LB, TXGBE_MACRXCFG_LB); 4463 4464 msec_delay(50); 4465 } 4466 4467 /* 4468 * Start Transmit and Receive Units. 4469 */ 4470 int __rte_cold 4471 txgbe_dev_rxtx_start(struct rte_eth_dev *dev) 4472 { 4473 struct txgbe_hw *hw; 4474 struct txgbe_tx_queue *txq; 4475 struct txgbe_rx_queue *rxq; 4476 uint32_t dmatxctl; 4477 uint32_t rxctrl; 4478 uint16_t i; 4479 int ret = 0; 4480 4481 PMD_INIT_FUNC_TRACE(); 4482 hw = TXGBE_DEV_HW(dev); 4483 4484 for (i = 0; i < dev->data->nb_tx_queues; i++) { 4485 txq = dev->data->tx_queues[i]; 4486 /* Setup Transmit Threshold Registers */ 4487 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), 4488 TXGBE_TXCFG_HTHRESH_MASK | 4489 TXGBE_TXCFG_WTHRESH_MASK, 4490 TXGBE_TXCFG_HTHRESH(txq->hthresh) | 4491 TXGBE_TXCFG_WTHRESH(txq->wthresh)); 4492 } 4493 4494 dmatxctl = rd32(hw, TXGBE_DMATXCTRL); 4495 dmatxctl |= TXGBE_DMATXCTRL_ENA; 4496 wr32(hw, TXGBE_DMATXCTRL, dmatxctl); 4497 4498 for (i = 0; i < dev->data->nb_tx_queues; i++) { 4499 txq = dev->data->tx_queues[i]; 4500 if (!txq->tx_deferred_start) { 4501 ret = txgbe_dev_tx_queue_start(dev, i); 4502 if (ret < 0) 4503 return ret; 4504 } 4505 } 4506 4507 for (i = 0; i < dev->data->nb_rx_queues; i++) { 4508 rxq = dev->data->rx_queues[i]; 4509 if (!rxq->rx_deferred_start) { 4510 ret = txgbe_dev_rx_queue_start(dev, i); 4511 if (ret < 0) 4512 return ret; 4513 } 4514 } 4515 4516 /* Enable Receive engine */ 4517 rxctrl = rd32(hw, TXGBE_PBRXCTL); 4518 rxctrl |= TXGBE_PBRXCTL_ENA; 4519 hw->mac.enable_rx_dma(hw, rxctrl); 4520 4521 /* If loopback mode is enabled, set up the link accordingly */ 4522 if (hw->mac.type == txgbe_mac_raptor && 4523 dev->data->dev_conf.lpbk_mode) 4524 txgbe_setup_loopback_link_raptor(hw); 4525 4526 #ifdef RTE_LIB_SECURITY 4527 if ((dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_SECURITY) || 4528 (dev->data->dev_conf.txmode.offloads & DEV_TX_OFFLOAD_SECURITY)) { 4529 ret = txgbe_crypto_enable_ipsec(dev); 4530 if (ret != 0) { 4531 PMD_DRV_LOG(ERR, 4532 "txgbe_crypto_enable_ipsec fails with %d.", 4533 ret); 4534 return ret; 4535 } 4536 } 4537 #endif 4538 4539 return 0; 4540 } 4541 4542 void 4543 txgbe_dev_save_rx_queue(struct txgbe_hw *hw, uint16_t rx_queue_id) 4544 { 4545 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8]; 4546 *(reg++) = rd32(hw, TXGBE_RXBAL(rx_queue_id)); 4547 *(reg++) = rd32(hw, TXGBE_RXBAH(rx_queue_id)); 4548 *(reg++) = rd32(hw, TXGBE_RXCFG(rx_queue_id)); 4549 } 4550 4551 void 4552 txgbe_dev_store_rx_queue(struct txgbe_hw *hw, uint16_t rx_queue_id) 4553 { 4554 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8]; 4555 wr32(hw, TXGBE_RXBAL(rx_queue_id), *(reg++)); 4556 wr32(hw, TXGBE_RXBAH(rx_queue_id), *(reg++)); 4557 wr32(hw, TXGBE_RXCFG(rx_queue_id), *(reg++) & ~TXGBE_RXCFG_ENA); 4558 } 4559 4560 void 4561 txgbe_dev_save_tx_queue(struct txgbe_hw *hw, uint16_t tx_queue_id) 4562 { 4563 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8]; 4564 *(reg++) = rd32(hw, TXGBE_TXBAL(tx_queue_id)); 4565 *(reg++) = rd32(hw, TXGBE_TXBAH(tx_queue_id)); 4566 *(reg++) = rd32(hw, TXGBE_TXCFG(tx_queue_id)); 4567 } 4568 4569 void 4570 txgbe_dev_store_tx_queue(struct txgbe_hw *hw, uint16_t tx_queue_id) 4571 { 4572 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8]; 4573 wr32(hw, TXGBE_TXBAL(tx_queue_id), *(reg++)); 4574 wr32(hw, TXGBE_TXBAH(tx_queue_id), *(reg++)); 4575 wr32(hw, TXGBE_TXCFG(tx_queue_id), *(reg++) & ~TXGBE_TXCFG_ENA); 4576 } 4577 4578 /* 4579 * Start Receive Units for specified queue. 4580 */ 4581 int __rte_cold 4582 txgbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) 4583 { 4584 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 4585 struct txgbe_rx_queue *rxq; 4586 uint32_t rxdctl; 4587 int poll_ms; 4588 4589 PMD_INIT_FUNC_TRACE(); 4590 4591 rxq = dev->data->rx_queues[rx_queue_id]; 4592 4593 /* Allocate buffers for descriptor rings */ 4594 if (txgbe_alloc_rx_queue_mbufs(rxq) != 0) { 4595 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d", 4596 rx_queue_id); 4597 return -1; 4598 } 4599 rxdctl = rd32(hw, TXGBE_RXCFG(rxq->reg_idx)); 4600 rxdctl |= TXGBE_RXCFG_ENA; 4601 wr32(hw, TXGBE_RXCFG(rxq->reg_idx), rxdctl); 4602 4603 /* Wait until RX Enable ready */ 4604 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS; 4605 do { 4606 rte_delay_ms(1); 4607 rxdctl = rd32(hw, TXGBE_RXCFG(rxq->reg_idx)); 4608 } while (--poll_ms && !(rxdctl & TXGBE_RXCFG_ENA)); 4609 if (!poll_ms) 4610 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id); 4611 rte_wmb(); 4612 wr32(hw, TXGBE_RXRP(rxq->reg_idx), 0); 4613 wr32(hw, TXGBE_RXWP(rxq->reg_idx), rxq->nb_rx_desc - 1); 4614 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 4615 4616 return 0; 4617 } 4618 4619 /* 4620 * Stop Receive Units for specified queue. 4621 */ 4622 int __rte_cold 4623 txgbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) 4624 { 4625 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 4626 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev); 4627 struct txgbe_rx_queue *rxq; 4628 uint32_t rxdctl; 4629 int poll_ms; 4630 4631 PMD_INIT_FUNC_TRACE(); 4632 4633 rxq = dev->data->rx_queues[rx_queue_id]; 4634 4635 txgbe_dev_save_rx_queue(hw, rxq->reg_idx); 4636 wr32m(hw, TXGBE_RXCFG(rxq->reg_idx), TXGBE_RXCFG_ENA, 0); 4637 4638 /* Wait until RX Enable bit clear */ 4639 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS; 4640 do { 4641 rte_delay_ms(1); 4642 rxdctl = rd32(hw, TXGBE_RXCFG(rxq->reg_idx)); 4643 } while (--poll_ms && (rxdctl & TXGBE_RXCFG_ENA)); 4644 if (!poll_ms) 4645 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id); 4646 4647 rte_delay_us(RTE_TXGBE_WAIT_100_US); 4648 txgbe_dev_store_rx_queue(hw, rxq->reg_idx); 4649 4650 txgbe_rx_queue_release_mbufs(rxq); 4651 txgbe_reset_rx_queue(adapter, rxq); 4652 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 4653 4654 return 0; 4655 } 4656 4657 /* 4658 * Start Transmit Units for specified queue. 4659 */ 4660 int __rte_cold 4661 txgbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) 4662 { 4663 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 4664 struct txgbe_tx_queue *txq; 4665 uint32_t txdctl; 4666 int poll_ms; 4667 4668 PMD_INIT_FUNC_TRACE(); 4669 4670 txq = dev->data->tx_queues[tx_queue_id]; 4671 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), TXGBE_TXCFG_ENA, TXGBE_TXCFG_ENA); 4672 4673 /* Wait until TX Enable ready */ 4674 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS; 4675 do { 4676 rte_delay_ms(1); 4677 txdctl = rd32(hw, TXGBE_TXCFG(txq->reg_idx)); 4678 } while (--poll_ms && !(txdctl & TXGBE_TXCFG_ENA)); 4679 if (!poll_ms) 4680 PMD_INIT_LOG(ERR, "Could not enable " 4681 "Tx Queue %d", tx_queue_id); 4682 4683 rte_wmb(); 4684 wr32(hw, TXGBE_TXWP(txq->reg_idx), txq->tx_tail); 4685 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 4686 4687 return 0; 4688 } 4689 4690 /* 4691 * Stop Transmit Units for specified queue. 4692 */ 4693 int __rte_cold 4694 txgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) 4695 { 4696 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 4697 struct txgbe_tx_queue *txq; 4698 uint32_t txdctl; 4699 uint32_t txtdh, txtdt; 4700 int poll_ms; 4701 4702 PMD_INIT_FUNC_TRACE(); 4703 4704 txq = dev->data->tx_queues[tx_queue_id]; 4705 4706 /* Wait until TX queue is empty */ 4707 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS; 4708 do { 4709 rte_delay_us(RTE_TXGBE_WAIT_100_US); 4710 txtdh = rd32(hw, TXGBE_TXRP(txq->reg_idx)); 4711 txtdt = rd32(hw, TXGBE_TXWP(txq->reg_idx)); 4712 } while (--poll_ms && (txtdh != txtdt)); 4713 if (!poll_ms) 4714 PMD_INIT_LOG(ERR, 4715 "Tx Queue %d is not empty when stopping.", 4716 tx_queue_id); 4717 4718 txgbe_dev_save_tx_queue(hw, txq->reg_idx); 4719 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), TXGBE_TXCFG_ENA, 0); 4720 4721 /* Wait until TX Enable bit clear */ 4722 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS; 4723 do { 4724 rte_delay_ms(1); 4725 txdctl = rd32(hw, TXGBE_TXCFG(txq->reg_idx)); 4726 } while (--poll_ms && (txdctl & TXGBE_TXCFG_ENA)); 4727 if (!poll_ms) 4728 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d", 4729 tx_queue_id); 4730 4731 rte_delay_us(RTE_TXGBE_WAIT_100_US); 4732 txgbe_dev_store_tx_queue(hw, txq->reg_idx); 4733 4734 if (txq->ops != NULL) { 4735 txq->ops->release_mbufs(txq); 4736 txq->ops->reset(txq); 4737 } 4738 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 4739 4740 return 0; 4741 } 4742 4743 void 4744 txgbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 4745 struct rte_eth_rxq_info *qinfo) 4746 { 4747 struct txgbe_rx_queue *rxq; 4748 4749 rxq = dev->data->rx_queues[queue_id]; 4750 4751 qinfo->mp = rxq->mb_pool; 4752 qinfo->scattered_rx = dev->data->scattered_rx; 4753 qinfo->nb_desc = rxq->nb_rx_desc; 4754 4755 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh; 4756 qinfo->conf.rx_drop_en = rxq->drop_en; 4757 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start; 4758 qinfo->conf.offloads = rxq->offloads; 4759 } 4760 4761 void 4762 txgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 4763 struct rte_eth_txq_info *qinfo) 4764 { 4765 struct txgbe_tx_queue *txq; 4766 4767 txq = dev->data->tx_queues[queue_id]; 4768 4769 qinfo->nb_desc = txq->nb_tx_desc; 4770 4771 qinfo->conf.tx_thresh.pthresh = txq->pthresh; 4772 qinfo->conf.tx_thresh.hthresh = txq->hthresh; 4773 qinfo->conf.tx_thresh.wthresh = txq->wthresh; 4774 4775 qinfo->conf.tx_free_thresh = txq->tx_free_thresh; 4776 qinfo->conf.offloads = txq->offloads; 4777 qinfo->conf.tx_deferred_start = txq->tx_deferred_start; 4778 } 4779 4780 /* 4781 * [VF] Initializes Receive Unit. 4782 */ 4783 int __rte_cold 4784 txgbevf_dev_rx_init(struct rte_eth_dev *dev) 4785 { 4786 struct txgbe_hw *hw; 4787 struct txgbe_rx_queue *rxq; 4788 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 4789 uint64_t bus_addr; 4790 uint32_t srrctl, psrtype; 4791 uint16_t buf_size; 4792 uint16_t i; 4793 int ret; 4794 4795 PMD_INIT_FUNC_TRACE(); 4796 hw = TXGBE_DEV_HW(dev); 4797 4798 if (rte_is_power_of_2(dev->data->nb_rx_queues) == 0) { 4799 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, " 4800 "it should be power of 2"); 4801 return -1; 4802 } 4803 4804 if (dev->data->nb_rx_queues > hw->mac.max_rx_queues) { 4805 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, " 4806 "it should be equal to or less than %d", 4807 hw->mac.max_rx_queues); 4808 return -1; 4809 } 4810 4811 /* 4812 * When the VF driver issues a TXGBE_VF_RESET request, the PF driver 4813 * disables the VF receipt of packets if the PF MTU is > 1500. 4814 * This is done to deal with limitations that imposes 4815 * the PF and all VFs to share the same MTU. 4816 * Then, the PF driver enables again the VF receipt of packet when 4817 * the VF driver issues a TXGBE_VF_SET_LPE request. 4818 * In the meantime, the VF device cannot be used, even if the VF driver 4819 * and the Guest VM network stack are ready to accept packets with a 4820 * size up to the PF MTU. 4821 * As a work-around to this PF behaviour, force the call to 4822 * txgbevf_rlpml_set_vf even if jumbo frames are not used. This way, 4823 * VF packets received can work in all cases. 4824 */ 4825 if (txgbevf_rlpml_set_vf(hw, 4826 (uint16_t)dev->data->dev_conf.rxmode.max_rx_pkt_len)) { 4827 PMD_INIT_LOG(ERR, "Set max packet length to %d failed.", 4828 dev->data->dev_conf.rxmode.max_rx_pkt_len); 4829 return -EINVAL; 4830 } 4831 4832 /* 4833 * Assume no header split and no VLAN strip support 4834 * on any Rx queue first . 4835 */ 4836 rxmode->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP; 4837 4838 /* Set PSR type for VF RSS according to max Rx queue */ 4839 psrtype = TXGBE_VFPLCFG_PSRL4HDR | 4840 TXGBE_VFPLCFG_PSRL4HDR | 4841 TXGBE_VFPLCFG_PSRL2HDR | 4842 TXGBE_VFPLCFG_PSRTUNHDR | 4843 TXGBE_VFPLCFG_PSRTUNMAC; 4844 wr32(hw, TXGBE_VFPLCFG, TXGBE_VFPLCFG_PSR(psrtype)); 4845 4846 /* Setup RX queues */ 4847 for (i = 0; i < dev->data->nb_rx_queues; i++) { 4848 rxq = dev->data->rx_queues[i]; 4849 4850 /* Allocate buffers for descriptor rings */ 4851 ret = txgbe_alloc_rx_queue_mbufs(rxq); 4852 if (ret) 4853 return ret; 4854 4855 /* Setup the Base and Length of the Rx Descriptor Rings */ 4856 bus_addr = rxq->rx_ring_phys_addr; 4857 4858 wr32(hw, TXGBE_RXBAL(i), 4859 (uint32_t)(bus_addr & BIT_MASK32)); 4860 wr32(hw, TXGBE_RXBAH(i), 4861 (uint32_t)(bus_addr >> 32)); 4862 wr32(hw, TXGBE_RXRP(i), 0); 4863 wr32(hw, TXGBE_RXWP(i), 0); 4864 4865 /* Configure the RXCFG register */ 4866 srrctl = TXGBE_RXCFG_RNGLEN(rxq->nb_rx_desc); 4867 4868 /* Set if packets are dropped when no descriptors available */ 4869 if (rxq->drop_en) 4870 srrctl |= TXGBE_RXCFG_DROP; 4871 4872 /* 4873 * Configure the RX buffer size in the PKTLEN field of 4874 * the RXCFG register of the queue. 4875 * The value is in 1 KB resolution. Valid values can be from 4876 * 1 KB to 16 KB. 4877 */ 4878 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - 4879 RTE_PKTMBUF_HEADROOM); 4880 buf_size = ROUND_UP(buf_size, 1 << 10); 4881 srrctl |= TXGBE_RXCFG_PKTLEN(buf_size); 4882 4883 /* 4884 * VF modification to write virtual function RXCFG register 4885 */ 4886 wr32(hw, TXGBE_RXCFG(i), srrctl); 4887 4888 if (rxmode->offloads & DEV_RX_OFFLOAD_SCATTER || 4889 /* It adds dual VLAN length for supporting dual VLAN */ 4890 (rxmode->max_rx_pkt_len + 4891 2 * TXGBE_VLAN_TAG_SIZE) > buf_size) { 4892 if (!dev->data->scattered_rx) 4893 PMD_INIT_LOG(DEBUG, "forcing scatter mode"); 4894 dev->data->scattered_rx = 1; 4895 } 4896 4897 if (rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 4898 rxmode->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP; 4899 } 4900 4901 /* 4902 * Device configured with multiple RX queues. 4903 */ 4904 txgbe_dev_mq_rx_configure(dev); 4905 4906 txgbe_set_rx_function(dev); 4907 4908 return 0; 4909 } 4910 4911 /* 4912 * [VF] Initializes Transmit Unit. 4913 */ 4914 void __rte_cold 4915 txgbevf_dev_tx_init(struct rte_eth_dev *dev) 4916 { 4917 struct txgbe_hw *hw; 4918 struct txgbe_tx_queue *txq; 4919 uint64_t bus_addr; 4920 uint16_t i; 4921 4922 PMD_INIT_FUNC_TRACE(); 4923 hw = TXGBE_DEV_HW(dev); 4924 4925 /* Setup the Base and Length of the Tx Descriptor Rings */ 4926 for (i = 0; i < dev->data->nb_tx_queues; i++) { 4927 txq = dev->data->tx_queues[i]; 4928 bus_addr = txq->tx_ring_phys_addr; 4929 wr32(hw, TXGBE_TXBAL(i), 4930 (uint32_t)(bus_addr & BIT_MASK32)); 4931 wr32(hw, TXGBE_TXBAH(i), 4932 (uint32_t)(bus_addr >> 32)); 4933 wr32m(hw, TXGBE_TXCFG(i), TXGBE_TXCFG_BUFLEN_MASK, 4934 TXGBE_TXCFG_BUFLEN(txq->nb_tx_desc)); 4935 /* Setup the HW Tx Head and TX Tail descriptor pointers */ 4936 wr32(hw, TXGBE_TXRP(i), 0); 4937 wr32(hw, TXGBE_TXWP(i), 0); 4938 } 4939 } 4940 4941 /* 4942 * [VF] Start Transmit and Receive Units. 4943 */ 4944 void __rte_cold 4945 txgbevf_dev_rxtx_start(struct rte_eth_dev *dev) 4946 { 4947 struct txgbe_hw *hw; 4948 struct txgbe_tx_queue *txq; 4949 struct txgbe_rx_queue *rxq; 4950 uint32_t txdctl; 4951 uint32_t rxdctl; 4952 uint16_t i; 4953 int poll_ms; 4954 4955 PMD_INIT_FUNC_TRACE(); 4956 hw = TXGBE_DEV_HW(dev); 4957 4958 for (i = 0; i < dev->data->nb_tx_queues; i++) { 4959 txq = dev->data->tx_queues[i]; 4960 /* Setup Transmit Threshold Registers */ 4961 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), 4962 TXGBE_TXCFG_HTHRESH_MASK | 4963 TXGBE_TXCFG_WTHRESH_MASK, 4964 TXGBE_TXCFG_HTHRESH(txq->hthresh) | 4965 TXGBE_TXCFG_WTHRESH(txq->wthresh)); 4966 } 4967 4968 for (i = 0; i < dev->data->nb_tx_queues; i++) { 4969 wr32m(hw, TXGBE_TXCFG(i), TXGBE_TXCFG_ENA, TXGBE_TXCFG_ENA); 4970 4971 poll_ms = 10; 4972 /* Wait until TX Enable ready */ 4973 do { 4974 rte_delay_ms(1); 4975 txdctl = rd32(hw, TXGBE_TXCFG(i)); 4976 } while (--poll_ms && !(txdctl & TXGBE_TXCFG_ENA)); 4977 if (!poll_ms) 4978 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", i); 4979 } 4980 for (i = 0; i < dev->data->nb_rx_queues; i++) { 4981 rxq = dev->data->rx_queues[i]; 4982 4983 wr32m(hw, TXGBE_RXCFG(i), TXGBE_RXCFG_ENA, TXGBE_RXCFG_ENA); 4984 4985 /* Wait until RX Enable ready */ 4986 poll_ms = 10; 4987 do { 4988 rte_delay_ms(1); 4989 rxdctl = rd32(hw, TXGBE_RXCFG(i)); 4990 } while (--poll_ms && !(rxdctl & TXGBE_RXCFG_ENA)); 4991 if (!poll_ms) 4992 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", i); 4993 rte_wmb(); 4994 wr32(hw, TXGBE_RXWP(i), rxq->nb_rx_desc - 1); 4995 } 4996 } 4997 4998 int 4999 txgbe_rss_conf_init(struct txgbe_rte_flow_rss_conf *out, 5000 const struct rte_flow_action_rss *in) 5001 { 5002 if (in->key_len > RTE_DIM(out->key) || 5003 in->queue_num > RTE_DIM(out->queue)) 5004 return -EINVAL; 5005 out->conf = (struct rte_flow_action_rss){ 5006 .func = in->func, 5007 .level = in->level, 5008 .types = in->types, 5009 .key_len = in->key_len, 5010 .queue_num = in->queue_num, 5011 .key = memcpy(out->key, in->key, in->key_len), 5012 .queue = memcpy(out->queue, in->queue, 5013 sizeof(*in->queue) * in->queue_num), 5014 }; 5015 return 0; 5016 } 5017 5018 int 5019 txgbe_action_rss_same(const struct rte_flow_action_rss *comp, 5020 const struct rte_flow_action_rss *with) 5021 { 5022 return (comp->func == with->func && 5023 comp->level == with->level && 5024 comp->types == with->types && 5025 comp->key_len == with->key_len && 5026 comp->queue_num == with->queue_num && 5027 !memcmp(comp->key, with->key, with->key_len) && 5028 !memcmp(comp->queue, with->queue, 5029 sizeof(*with->queue) * with->queue_num)); 5030 } 5031 5032 int 5033 txgbe_config_rss_filter(struct rte_eth_dev *dev, 5034 struct txgbe_rte_flow_rss_conf *conf, bool add) 5035 { 5036 struct txgbe_hw *hw; 5037 uint32_t reta; 5038 uint16_t i; 5039 uint16_t j; 5040 struct rte_eth_rss_conf rss_conf = { 5041 .rss_key = conf->conf.key_len ? 5042 (void *)(uintptr_t)conf->conf.key : NULL, 5043 .rss_key_len = conf->conf.key_len, 5044 .rss_hf = conf->conf.types, 5045 }; 5046 struct txgbe_filter_info *filter_info = TXGBE_DEV_FILTER(dev); 5047 5048 PMD_INIT_FUNC_TRACE(); 5049 hw = TXGBE_DEV_HW(dev); 5050 5051 if (!add) { 5052 if (txgbe_action_rss_same(&filter_info->rss_info.conf, 5053 &conf->conf)) { 5054 txgbe_rss_disable(dev); 5055 memset(&filter_info->rss_info, 0, 5056 sizeof(struct txgbe_rte_flow_rss_conf)); 5057 return 0; 5058 } 5059 return -EINVAL; 5060 } 5061 5062 if (filter_info->rss_info.conf.queue_num) 5063 return -EINVAL; 5064 /* Fill in redirection table 5065 * The byte-swap is needed because NIC registers are in 5066 * little-endian order. 5067 */ 5068 reta = 0; 5069 for (i = 0, j = 0; i < ETH_RSS_RETA_SIZE_128; i++, j++) { 5070 if (j == conf->conf.queue_num) 5071 j = 0; 5072 reta = (reta >> 8) | LS32(conf->conf.queue[j], 24, 0xFF); 5073 if ((i & 3) == 3) 5074 wr32at(hw, TXGBE_REG_RSSTBL, i >> 2, reta); 5075 } 5076 5077 /* Configure the RSS key and the RSS protocols used to compute 5078 * the RSS hash of input packets. 5079 */ 5080 if ((rss_conf.rss_hf & TXGBE_RSS_OFFLOAD_ALL) == 0) { 5081 txgbe_rss_disable(dev); 5082 return 0; 5083 } 5084 if (rss_conf.rss_key == NULL) 5085 rss_conf.rss_key = rss_intel_key; /* Default hash key */ 5086 txgbe_dev_rss_hash_update(dev, &rss_conf); 5087 5088 if (txgbe_rss_conf_init(&filter_info->rss_info, &conf->conf)) 5089 return -EINVAL; 5090 5091 return 0; 5092 } 5093 5094