1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2015-2020 Beijing WangXun Technology Co., Ltd. 3 * Copyright(c) 2010-2017 Intel Corporation 4 */ 5 6 #include <sys/queue.h> 7 8 #include <stdio.h> 9 #include <stdlib.h> 10 #include <string.h> 11 #include <errno.h> 12 #include <stdint.h> 13 #include <stdarg.h> 14 #include <unistd.h> 15 #include <inttypes.h> 16 17 #include <rte_byteorder.h> 18 #include <rte_common.h> 19 #include <rte_cycles.h> 20 #include <rte_log.h> 21 #include <rte_debug.h> 22 #include <rte_ethdev.h> 23 #include <ethdev_driver.h> 24 #include <rte_security_driver.h> 25 #include <rte_memzone.h> 26 #include <rte_atomic.h> 27 #include <rte_mempool.h> 28 #include <rte_malloc.h> 29 #include <rte_mbuf.h> 30 #include <rte_ether.h> 31 #include <rte_prefetch.h> 32 #include <rte_udp.h> 33 #include <rte_tcp.h> 34 #include <rte_sctp.h> 35 #include <rte_string_fns.h> 36 #include <rte_errno.h> 37 #include <rte_ip.h> 38 #include <rte_net.h> 39 40 #include "txgbe_logs.h" 41 #include "base/txgbe.h" 42 #include "txgbe_ethdev.h" 43 #include "txgbe_rxtx.h" 44 45 #ifdef RTE_LIBRTE_IEEE1588 46 #define TXGBE_TX_IEEE1588_TMST RTE_MBUF_F_TX_IEEE1588_TMST 47 #else 48 #define TXGBE_TX_IEEE1588_TMST 0 49 #endif 50 51 /* Bit Mask to indicate what bits required for building TX context */ 52 static const u64 TXGBE_TX_OFFLOAD_MASK = (RTE_MBUF_F_TX_IP_CKSUM | 53 RTE_MBUF_F_TX_OUTER_IPV6 | 54 RTE_MBUF_F_TX_OUTER_IPV4 | 55 RTE_MBUF_F_TX_IPV6 | 56 RTE_MBUF_F_TX_IPV4 | 57 RTE_MBUF_F_TX_VLAN | 58 RTE_MBUF_F_TX_L4_MASK | 59 RTE_MBUF_F_TX_TCP_SEG | 60 RTE_MBUF_F_TX_TUNNEL_MASK | 61 RTE_MBUF_F_TX_OUTER_IP_CKSUM | 62 RTE_MBUF_F_TX_OUTER_UDP_CKSUM | 63 #ifdef RTE_LIB_SECURITY 64 RTE_MBUF_F_TX_SEC_OFFLOAD | 65 #endif 66 TXGBE_TX_IEEE1588_TMST); 67 68 #define TXGBE_TX_OFFLOAD_NOTSUP_MASK \ 69 (RTE_MBUF_F_TX_OFFLOAD_MASK ^ TXGBE_TX_OFFLOAD_MASK) 70 71 /* 72 * Prefetch a cache line into all cache levels. 73 */ 74 #define rte_txgbe_prefetch(p) rte_prefetch0(p) 75 76 static int 77 txgbe_is_vf(struct rte_eth_dev *dev) 78 { 79 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 80 81 switch (hw->mac.type) { 82 case txgbe_mac_raptor_vf: 83 return 1; 84 default: 85 return 0; 86 } 87 } 88 89 /********************************************************************* 90 * 91 * TX functions 92 * 93 **********************************************************************/ 94 95 /* 96 * Check for descriptors with their DD bit set and free mbufs. 97 * Return the total number of buffers freed. 98 */ 99 static __rte_always_inline int 100 txgbe_tx_free_bufs(struct txgbe_tx_queue *txq) 101 { 102 struct txgbe_tx_entry *txep; 103 uint32_t status; 104 int i, nb_free = 0; 105 struct rte_mbuf *m, *free[RTE_TXGBE_TX_MAX_FREE_BUF_SZ]; 106 107 /* check DD bit on threshold descriptor */ 108 status = txq->tx_ring[txq->tx_next_dd].dw3; 109 if (!(status & rte_cpu_to_le_32(TXGBE_TXD_DD))) { 110 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh) 111 txgbe_set32_masked(txq->tdc_reg_addr, 112 TXGBE_TXCFG_FLUSH, TXGBE_TXCFG_FLUSH); 113 return 0; 114 } 115 116 /* 117 * first buffer to free from S/W ring is at index 118 * tx_next_dd - (tx_free_thresh-1) 119 */ 120 txep = &txq->sw_ring[txq->tx_next_dd - (txq->tx_free_thresh - 1)]; 121 for (i = 0; i < txq->tx_free_thresh; ++i, ++txep) { 122 /* free buffers one at a time */ 123 m = rte_pktmbuf_prefree_seg(txep->mbuf); 124 txep->mbuf = NULL; 125 126 if (unlikely(m == NULL)) 127 continue; 128 129 if (nb_free >= RTE_TXGBE_TX_MAX_FREE_BUF_SZ || 130 (nb_free > 0 && m->pool != free[0]->pool)) { 131 rte_mempool_put_bulk(free[0]->pool, 132 (void **)free, nb_free); 133 nb_free = 0; 134 } 135 136 free[nb_free++] = m; 137 } 138 139 if (nb_free > 0) 140 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); 141 142 /* buffers were freed, update counters */ 143 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_free_thresh); 144 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_free_thresh); 145 if (txq->tx_next_dd >= txq->nb_tx_desc) 146 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1); 147 148 return txq->tx_free_thresh; 149 } 150 151 /* Populate 4 descriptors with data from 4 mbufs */ 152 static inline void 153 tx4(volatile struct txgbe_tx_desc *txdp, struct rte_mbuf **pkts) 154 { 155 uint64_t buf_dma_addr; 156 uint32_t pkt_len; 157 int i; 158 159 for (i = 0; i < 4; ++i, ++txdp, ++pkts) { 160 buf_dma_addr = rte_mbuf_data_iova(*pkts); 161 pkt_len = (*pkts)->data_len; 162 163 /* write data to descriptor */ 164 txdp->qw0 = rte_cpu_to_le_64(buf_dma_addr); 165 txdp->dw2 = cpu_to_le32(TXGBE_TXD_FLAGS | 166 TXGBE_TXD_DATLEN(pkt_len)); 167 txdp->dw3 = cpu_to_le32(TXGBE_TXD_PAYLEN(pkt_len)); 168 169 rte_prefetch0(&(*pkts)->pool); 170 } 171 } 172 173 /* Populate 1 descriptor with data from 1 mbuf */ 174 static inline void 175 tx1(volatile struct txgbe_tx_desc *txdp, struct rte_mbuf **pkts) 176 { 177 uint64_t buf_dma_addr; 178 uint32_t pkt_len; 179 180 buf_dma_addr = rte_mbuf_data_iova(*pkts); 181 pkt_len = (*pkts)->data_len; 182 183 /* write data to descriptor */ 184 txdp->qw0 = cpu_to_le64(buf_dma_addr); 185 txdp->dw2 = cpu_to_le32(TXGBE_TXD_FLAGS | 186 TXGBE_TXD_DATLEN(pkt_len)); 187 txdp->dw3 = cpu_to_le32(TXGBE_TXD_PAYLEN(pkt_len)); 188 189 rte_prefetch0(&(*pkts)->pool); 190 } 191 192 /* 193 * Fill H/W descriptor ring with mbuf data. 194 * Copy mbuf pointers to the S/W ring. 195 */ 196 static inline void 197 txgbe_tx_fill_hw_ring(struct txgbe_tx_queue *txq, struct rte_mbuf **pkts, 198 uint16_t nb_pkts) 199 { 200 volatile struct txgbe_tx_desc *txdp = &txq->tx_ring[txq->tx_tail]; 201 struct txgbe_tx_entry *txep = &txq->sw_ring[txq->tx_tail]; 202 const int N_PER_LOOP = 4; 203 const int N_PER_LOOP_MASK = N_PER_LOOP - 1; 204 int mainpart, leftover; 205 int i, j; 206 207 /* 208 * Process most of the packets in chunks of N pkts. Any 209 * leftover packets will get processed one at a time. 210 */ 211 mainpart = (nb_pkts & ((uint32_t)~N_PER_LOOP_MASK)); 212 leftover = (nb_pkts & ((uint32_t)N_PER_LOOP_MASK)); 213 for (i = 0; i < mainpart; i += N_PER_LOOP) { 214 /* Copy N mbuf pointers to the S/W ring */ 215 for (j = 0; j < N_PER_LOOP; ++j) 216 (txep + i + j)->mbuf = *(pkts + i + j); 217 tx4(txdp + i, pkts + i); 218 } 219 220 if (unlikely(leftover > 0)) { 221 for (i = 0; i < leftover; ++i) { 222 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i); 223 tx1(txdp + mainpart + i, pkts + mainpart + i); 224 } 225 } 226 } 227 228 static inline uint16_t 229 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 230 uint16_t nb_pkts) 231 { 232 struct txgbe_tx_queue *txq = (struct txgbe_tx_queue *)tx_queue; 233 uint16_t n = 0; 234 235 /* 236 * Begin scanning the H/W ring for done descriptors when the 237 * number of available descriptors drops below tx_free_thresh. For 238 * each done descriptor, free the associated buffer. 239 */ 240 if (txq->nb_tx_free < txq->tx_free_thresh) 241 txgbe_tx_free_bufs(txq); 242 243 /* Only use descriptors that are available */ 244 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); 245 if (unlikely(nb_pkts == 0)) 246 return 0; 247 248 /* Use exactly nb_pkts descriptors */ 249 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); 250 251 /* 252 * At this point, we know there are enough descriptors in the 253 * ring to transmit all the packets. This assumes that each 254 * mbuf contains a single segment, and that no new offloads 255 * are expected, which would require a new context descriptor. 256 */ 257 258 /* 259 * See if we're going to wrap-around. If so, handle the top 260 * of the descriptor ring first, then do the bottom. If not, 261 * the processing looks just like the "bottom" part anyway... 262 */ 263 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) { 264 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail); 265 txgbe_tx_fill_hw_ring(txq, tx_pkts, n); 266 txq->tx_tail = 0; 267 } 268 269 /* Fill H/W descriptor ring with mbuf data */ 270 txgbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n)); 271 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n)); 272 273 /* 274 * Check for wrap-around. This would only happen if we used 275 * up to the last descriptor in the ring, no more, no less. 276 */ 277 if (txq->tx_tail >= txq->nb_tx_desc) 278 txq->tx_tail = 0; 279 280 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", 281 (uint16_t)txq->port_id, (uint16_t)txq->queue_id, 282 (uint16_t)txq->tx_tail, (uint16_t)nb_pkts); 283 284 /* update tail pointer */ 285 rte_wmb(); 286 txgbe_set32_relaxed(txq->tdt_reg_addr, txq->tx_tail); 287 288 return nb_pkts; 289 } 290 291 uint16_t 292 txgbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, 293 uint16_t nb_pkts) 294 { 295 uint16_t nb_tx; 296 297 /* Try to transmit at least chunks of TX_MAX_BURST pkts */ 298 if (likely(nb_pkts <= RTE_PMD_TXGBE_TX_MAX_BURST)) 299 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts); 300 301 /* transmit more than the max burst, in chunks of TX_MAX_BURST */ 302 nb_tx = 0; 303 while (nb_pkts) { 304 uint16_t ret, n; 305 306 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_TXGBE_TX_MAX_BURST); 307 ret = tx_xmit_pkts(tx_queue, &tx_pkts[nb_tx], n); 308 nb_tx = (uint16_t)(nb_tx + ret); 309 nb_pkts = (uint16_t)(nb_pkts - ret); 310 if (ret < n) 311 break; 312 } 313 314 return nb_tx; 315 } 316 317 static inline void 318 txgbe_set_xmit_ctx(struct txgbe_tx_queue *txq, 319 volatile struct txgbe_tx_ctx_desc *ctx_txd, 320 uint64_t ol_flags, union txgbe_tx_offload tx_offload, 321 __rte_unused uint64_t *mdata) 322 { 323 union txgbe_tx_offload tx_offload_mask; 324 uint32_t type_tucmd_mlhl; 325 uint32_t mss_l4len_idx; 326 uint32_t ctx_idx; 327 uint32_t vlan_macip_lens; 328 uint32_t tunnel_seed; 329 330 ctx_idx = txq->ctx_curr; 331 tx_offload_mask.data[0] = 0; 332 tx_offload_mask.data[1] = 0; 333 334 /* Specify which HW CTX to upload. */ 335 mss_l4len_idx = TXGBE_TXD_IDX(ctx_idx); 336 type_tucmd_mlhl = TXGBE_TXD_CTXT; 337 338 tx_offload_mask.ptid |= ~0; 339 type_tucmd_mlhl |= TXGBE_TXD_PTID(tx_offload.ptid); 340 341 /* check if TCP segmentation required for this packet */ 342 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 343 tx_offload_mask.l2_len |= ~0; 344 tx_offload_mask.l3_len |= ~0; 345 tx_offload_mask.l4_len |= ~0; 346 tx_offload_mask.tso_segsz |= ~0; 347 mss_l4len_idx |= TXGBE_TXD_MSS(tx_offload.tso_segsz); 348 mss_l4len_idx |= TXGBE_TXD_L4LEN(tx_offload.l4_len); 349 } else { /* no TSO, check if hardware checksum is needed */ 350 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { 351 tx_offload_mask.l2_len |= ~0; 352 tx_offload_mask.l3_len |= ~0; 353 } 354 355 switch (ol_flags & RTE_MBUF_F_TX_L4_MASK) { 356 case RTE_MBUF_F_TX_UDP_CKSUM: 357 mss_l4len_idx |= 358 TXGBE_TXD_L4LEN(sizeof(struct rte_udp_hdr)); 359 tx_offload_mask.l2_len |= ~0; 360 tx_offload_mask.l3_len |= ~0; 361 break; 362 case RTE_MBUF_F_TX_TCP_CKSUM: 363 mss_l4len_idx |= 364 TXGBE_TXD_L4LEN(sizeof(struct rte_tcp_hdr)); 365 tx_offload_mask.l2_len |= ~0; 366 tx_offload_mask.l3_len |= ~0; 367 break; 368 case RTE_MBUF_F_TX_SCTP_CKSUM: 369 mss_l4len_idx |= 370 TXGBE_TXD_L4LEN(sizeof(struct rte_sctp_hdr)); 371 tx_offload_mask.l2_len |= ~0; 372 tx_offload_mask.l3_len |= ~0; 373 break; 374 default: 375 break; 376 } 377 } 378 379 vlan_macip_lens = TXGBE_TXD_IPLEN(tx_offload.l3_len >> 1); 380 381 if (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { 382 tx_offload_mask.outer_tun_len |= ~0; 383 tx_offload_mask.outer_l2_len |= ~0; 384 tx_offload_mask.outer_l3_len |= ~0; 385 tx_offload_mask.l2_len |= ~0; 386 tunnel_seed = TXGBE_TXD_ETUNLEN(tx_offload.outer_tun_len >> 1); 387 tunnel_seed |= TXGBE_TXD_EIPLEN(tx_offload.outer_l3_len >> 2); 388 389 switch (ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { 390 case RTE_MBUF_F_TX_TUNNEL_IPIP: 391 /* for non UDP / GRE tunneling, set to 0b */ 392 break; 393 case RTE_MBUF_F_TX_TUNNEL_VXLAN: 394 case RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE: 395 case RTE_MBUF_F_TX_TUNNEL_GENEVE: 396 tunnel_seed |= TXGBE_TXD_ETYPE_UDP; 397 break; 398 case RTE_MBUF_F_TX_TUNNEL_GRE: 399 tunnel_seed |= TXGBE_TXD_ETYPE_GRE; 400 break; 401 default: 402 PMD_TX_LOG(ERR, "Tunnel type not supported"); 403 return; 404 } 405 vlan_macip_lens |= TXGBE_TXD_MACLEN(tx_offload.outer_l2_len); 406 } else { 407 tunnel_seed = 0; 408 vlan_macip_lens |= TXGBE_TXD_MACLEN(tx_offload.l2_len); 409 } 410 411 if (ol_flags & RTE_MBUF_F_TX_VLAN) { 412 tx_offload_mask.vlan_tci |= ~0; 413 vlan_macip_lens |= TXGBE_TXD_VLAN(tx_offload.vlan_tci); 414 } 415 416 #ifdef RTE_LIB_SECURITY 417 if (ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD) { 418 union txgbe_crypto_tx_desc_md *md = 419 (union txgbe_crypto_tx_desc_md *)mdata; 420 tunnel_seed |= TXGBE_TXD_IPSEC_SAIDX(md->sa_idx); 421 type_tucmd_mlhl |= md->enc ? 422 (TXGBE_TXD_IPSEC_ESP | TXGBE_TXD_IPSEC_ESPENC) : 0; 423 type_tucmd_mlhl |= TXGBE_TXD_IPSEC_ESPLEN(md->pad_len); 424 tx_offload_mask.sa_idx |= ~0; 425 tx_offload_mask.sec_pad_len |= ~0; 426 } 427 #endif 428 429 txq->ctx_cache[ctx_idx].flags = ol_flags; 430 txq->ctx_cache[ctx_idx].tx_offload.data[0] = 431 tx_offload_mask.data[0] & tx_offload.data[0]; 432 txq->ctx_cache[ctx_idx].tx_offload.data[1] = 433 tx_offload_mask.data[1] & tx_offload.data[1]; 434 txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask; 435 436 ctx_txd->dw0 = rte_cpu_to_le_32(vlan_macip_lens); 437 ctx_txd->dw1 = rte_cpu_to_le_32(tunnel_seed); 438 ctx_txd->dw2 = rte_cpu_to_le_32(type_tucmd_mlhl); 439 ctx_txd->dw3 = rte_cpu_to_le_32(mss_l4len_idx); 440 } 441 442 /* 443 * Check which hardware context can be used. Use the existing match 444 * or create a new context descriptor. 445 */ 446 static inline uint32_t 447 what_ctx_update(struct txgbe_tx_queue *txq, uint64_t flags, 448 union txgbe_tx_offload tx_offload) 449 { 450 /* If match with the current used context */ 451 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags && 452 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] == 453 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0] 454 & tx_offload.data[0])) && 455 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] == 456 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1] 457 & tx_offload.data[1])))) 458 return txq->ctx_curr; 459 460 /* What if match with the next context */ 461 txq->ctx_curr ^= 1; 462 if (likely(txq->ctx_cache[txq->ctx_curr].flags == flags && 463 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] == 464 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0] 465 & tx_offload.data[0])) && 466 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] == 467 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1] 468 & tx_offload.data[1])))) 469 return txq->ctx_curr; 470 471 /* Mismatch, use the previous context */ 472 return TXGBE_CTX_NUM; 473 } 474 475 static inline uint32_t 476 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags) 477 { 478 uint32_t tmp = 0; 479 480 if ((ol_flags & RTE_MBUF_F_TX_L4_MASK) != RTE_MBUF_F_TX_L4_NO_CKSUM) { 481 tmp |= TXGBE_TXD_CC; 482 tmp |= TXGBE_TXD_L4CS; 483 } 484 if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM) { 485 tmp |= TXGBE_TXD_CC; 486 tmp |= TXGBE_TXD_IPCS; 487 } 488 if (ol_flags & RTE_MBUF_F_TX_OUTER_IP_CKSUM) { 489 tmp |= TXGBE_TXD_CC; 490 tmp |= TXGBE_TXD_EIPCS; 491 } 492 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 493 tmp |= TXGBE_TXD_CC; 494 /* implies IPv4 cksum */ 495 if (ol_flags & RTE_MBUF_F_TX_IPV4) 496 tmp |= TXGBE_TXD_IPCS; 497 tmp |= TXGBE_TXD_L4CS; 498 } 499 if (ol_flags & RTE_MBUF_F_TX_VLAN) 500 tmp |= TXGBE_TXD_CC; 501 502 return tmp; 503 } 504 505 static inline uint32_t 506 tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags) 507 { 508 uint32_t cmdtype = 0; 509 510 if (ol_flags & RTE_MBUF_F_TX_VLAN) 511 cmdtype |= TXGBE_TXD_VLE; 512 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) 513 cmdtype |= TXGBE_TXD_TSE; 514 if (ol_flags & RTE_MBUF_F_TX_MACSEC) 515 cmdtype |= TXGBE_TXD_LINKSEC; 516 return cmdtype; 517 } 518 519 static inline uint8_t 520 tx_desc_ol_flags_to_ptid(uint64_t oflags, uint32_t ptype) 521 { 522 bool tun; 523 524 if (ptype) 525 return txgbe_encode_ptype(ptype); 526 527 /* Only support flags in TXGBE_TX_OFFLOAD_MASK */ 528 tun = !!(oflags & RTE_MBUF_F_TX_TUNNEL_MASK); 529 530 /* L2 level */ 531 ptype = RTE_PTYPE_L2_ETHER; 532 if (oflags & RTE_MBUF_F_TX_VLAN) 533 ptype |= RTE_PTYPE_L2_ETHER_VLAN; 534 535 /* L3 level */ 536 if (oflags & (RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IP_CKSUM)) 537 ptype |= RTE_PTYPE_L3_IPV4; 538 else if (oflags & (RTE_MBUF_F_TX_OUTER_IPV6)) 539 ptype |= RTE_PTYPE_L3_IPV6; 540 541 if (oflags & (RTE_MBUF_F_TX_IPV4 | RTE_MBUF_F_TX_IP_CKSUM)) 542 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV4 : RTE_PTYPE_L3_IPV4); 543 else if (oflags & (RTE_MBUF_F_TX_IPV6)) 544 ptype |= (tun ? RTE_PTYPE_INNER_L3_IPV6 : RTE_PTYPE_L3_IPV6); 545 546 /* L4 level */ 547 switch (oflags & (RTE_MBUF_F_TX_L4_MASK)) { 548 case RTE_MBUF_F_TX_TCP_CKSUM: 549 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP); 550 break; 551 case RTE_MBUF_F_TX_UDP_CKSUM: 552 ptype |= (tun ? RTE_PTYPE_INNER_L4_UDP : RTE_PTYPE_L4_UDP); 553 break; 554 case RTE_MBUF_F_TX_SCTP_CKSUM: 555 ptype |= (tun ? RTE_PTYPE_INNER_L4_SCTP : RTE_PTYPE_L4_SCTP); 556 break; 557 } 558 559 if (oflags & RTE_MBUF_F_TX_TCP_SEG) 560 ptype |= (tun ? RTE_PTYPE_INNER_L4_TCP : RTE_PTYPE_L4_TCP); 561 562 /* Tunnel */ 563 switch (oflags & RTE_MBUF_F_TX_TUNNEL_MASK) { 564 case RTE_MBUF_F_TX_TUNNEL_VXLAN: 565 case RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE: 566 ptype |= RTE_PTYPE_L2_ETHER | 567 RTE_PTYPE_L3_IPV4 | 568 RTE_PTYPE_TUNNEL_GRENAT; 569 break; 570 case RTE_MBUF_F_TX_TUNNEL_GRE: 571 ptype |= RTE_PTYPE_L2_ETHER | 572 RTE_PTYPE_L3_IPV4 | 573 RTE_PTYPE_TUNNEL_GRE; 574 ptype |= RTE_PTYPE_INNER_L2_ETHER; 575 break; 576 case RTE_MBUF_F_TX_TUNNEL_GENEVE: 577 ptype |= RTE_PTYPE_L2_ETHER | 578 RTE_PTYPE_L3_IPV4 | 579 RTE_PTYPE_TUNNEL_GENEVE; 580 ptype |= RTE_PTYPE_INNER_L2_ETHER; 581 break; 582 case RTE_MBUF_F_TX_TUNNEL_IPIP: 583 case RTE_MBUF_F_TX_TUNNEL_IP: 584 ptype |= RTE_PTYPE_L2_ETHER | 585 RTE_PTYPE_L3_IPV4 | 586 RTE_PTYPE_TUNNEL_IP; 587 break; 588 } 589 590 return txgbe_encode_ptype(ptype); 591 } 592 593 #ifndef DEFAULT_TX_FREE_THRESH 594 #define DEFAULT_TX_FREE_THRESH 32 595 #endif 596 597 /* Reset transmit descriptors after they have been used */ 598 static inline int 599 txgbe_xmit_cleanup(struct txgbe_tx_queue *txq) 600 { 601 struct txgbe_tx_entry *sw_ring = txq->sw_ring; 602 volatile struct txgbe_tx_desc *txr = txq->tx_ring; 603 uint16_t last_desc_cleaned = txq->last_desc_cleaned; 604 uint16_t nb_tx_desc = txq->nb_tx_desc; 605 uint16_t desc_to_clean_to; 606 uint16_t nb_tx_to_clean; 607 uint32_t status; 608 609 /* Determine the last descriptor needing to be cleaned */ 610 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_free_thresh); 611 if (desc_to_clean_to >= nb_tx_desc) 612 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc); 613 614 /* Check to make sure the last descriptor to clean is done */ 615 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id; 616 status = txr[desc_to_clean_to].dw3; 617 if (!(status & rte_cpu_to_le_32(TXGBE_TXD_DD))) { 618 PMD_TX_FREE_LOG(DEBUG, 619 "TX descriptor %4u is not done" 620 "(port=%d queue=%d)", 621 desc_to_clean_to, 622 txq->port_id, txq->queue_id); 623 if (txq->nb_tx_free >> 1 < txq->tx_free_thresh) 624 txgbe_set32_masked(txq->tdc_reg_addr, 625 TXGBE_TXCFG_FLUSH, TXGBE_TXCFG_FLUSH); 626 /* Failed to clean any descriptors, better luck next time */ 627 return -(1); 628 } 629 630 /* Figure out how many descriptors will be cleaned */ 631 if (last_desc_cleaned > desc_to_clean_to) 632 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) + 633 desc_to_clean_to); 634 else 635 nb_tx_to_clean = (uint16_t)(desc_to_clean_to - 636 last_desc_cleaned); 637 638 PMD_TX_FREE_LOG(DEBUG, 639 "Cleaning %4u TX descriptors: %4u to %4u " 640 "(port=%d queue=%d)", 641 nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to, 642 txq->port_id, txq->queue_id); 643 644 /* 645 * The last descriptor to clean is done, so that means all the 646 * descriptors from the last descriptor that was cleaned 647 * up to the last descriptor with the RS bit set 648 * are done. Only reset the threshold descriptor. 649 */ 650 txr[desc_to_clean_to].dw3 = 0; 651 652 /* Update the txq to reflect the last descriptor that was cleaned */ 653 txq->last_desc_cleaned = desc_to_clean_to; 654 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean); 655 656 /* No Error */ 657 return 0; 658 } 659 660 static inline uint8_t 661 txgbe_get_tun_len(struct rte_mbuf *mbuf) 662 { 663 struct txgbe_genevehdr genevehdr; 664 const struct txgbe_genevehdr *gh; 665 uint8_t tun_len; 666 667 switch (mbuf->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) { 668 case RTE_MBUF_F_TX_TUNNEL_IPIP: 669 tun_len = 0; 670 break; 671 case RTE_MBUF_F_TX_TUNNEL_VXLAN: 672 case RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE: 673 tun_len = sizeof(struct txgbe_udphdr) 674 + sizeof(struct txgbe_vxlanhdr); 675 break; 676 case RTE_MBUF_F_TX_TUNNEL_GRE: 677 tun_len = sizeof(struct txgbe_nvgrehdr); 678 break; 679 case RTE_MBUF_F_TX_TUNNEL_GENEVE: 680 gh = rte_pktmbuf_read(mbuf, 681 mbuf->outer_l2_len + mbuf->outer_l3_len, 682 sizeof(genevehdr), &genevehdr); 683 tun_len = sizeof(struct txgbe_udphdr) 684 + sizeof(struct txgbe_genevehdr) 685 + (gh->opt_len << 2); 686 break; 687 default: 688 tun_len = 0; 689 } 690 691 return tun_len; 692 } 693 694 static inline uint8_t 695 txgbe_parse_tun_ptid(struct rte_mbuf *tx_pkt) 696 { 697 uint64_t l2_none, l2_mac, l2_mac_vlan; 698 uint8_t ptid = 0; 699 700 if ((tx_pkt->ol_flags & (RTE_MBUF_F_TX_TUNNEL_VXLAN | 701 RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE)) == 0) 702 return ptid; 703 704 l2_none = sizeof(struct txgbe_udphdr) + sizeof(struct txgbe_vxlanhdr); 705 l2_mac = l2_none + sizeof(struct rte_ether_hdr); 706 l2_mac_vlan = l2_mac + sizeof(struct rte_vlan_hdr); 707 708 if (tx_pkt->l2_len == l2_none) 709 ptid = TXGBE_PTID_TUN_EIG; 710 else if (tx_pkt->l2_len == l2_mac) 711 ptid = TXGBE_PTID_TUN_EIGM; 712 else if (tx_pkt->l2_len == l2_mac_vlan) 713 ptid = TXGBE_PTID_TUN_EIGMV; 714 715 return ptid; 716 } 717 718 uint16_t 719 txgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 720 uint16_t nb_pkts) 721 { 722 struct txgbe_tx_queue *txq; 723 struct txgbe_tx_entry *sw_ring; 724 struct txgbe_tx_entry *txe, *txn; 725 volatile struct txgbe_tx_desc *txr; 726 volatile struct txgbe_tx_desc *txd; 727 struct rte_mbuf *tx_pkt; 728 struct rte_mbuf *m_seg; 729 uint64_t buf_dma_addr; 730 uint32_t olinfo_status; 731 uint32_t cmd_type_len; 732 uint32_t pkt_len; 733 uint16_t slen; 734 uint64_t ol_flags; 735 uint16_t tx_id; 736 uint16_t tx_last; 737 uint16_t nb_tx; 738 uint16_t nb_used; 739 uint64_t tx_ol_req; 740 uint32_t ctx = 0; 741 uint32_t new_ctx; 742 union txgbe_tx_offload tx_offload; 743 #ifdef RTE_LIB_SECURITY 744 uint8_t use_ipsec; 745 #endif 746 747 tx_offload.data[0] = 0; 748 tx_offload.data[1] = 0; 749 txq = tx_queue; 750 sw_ring = txq->sw_ring; 751 txr = txq->tx_ring; 752 tx_id = txq->tx_tail; 753 txe = &sw_ring[tx_id]; 754 755 /* Determine if the descriptor ring needs to be cleaned. */ 756 if (txq->nb_tx_free < txq->tx_free_thresh) 757 txgbe_xmit_cleanup(txq); 758 759 rte_prefetch0(&txe->mbuf->pool); 760 761 /* TX loop */ 762 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 763 new_ctx = 0; 764 tx_pkt = *tx_pkts++; 765 pkt_len = tx_pkt->pkt_len; 766 767 /* 768 * Determine how many (if any) context descriptors 769 * are needed for offload functionality. 770 */ 771 ol_flags = tx_pkt->ol_flags; 772 #ifdef RTE_LIB_SECURITY 773 use_ipsec = txq->using_ipsec && (ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD); 774 #endif 775 776 /* If hardware offload required */ 777 tx_ol_req = ol_flags & TXGBE_TX_OFFLOAD_MASK; 778 if (tx_ol_req) { 779 tx_offload.ptid = tx_desc_ol_flags_to_ptid(tx_ol_req, 780 tx_pkt->packet_type); 781 if (tx_offload.ptid & TXGBE_PTID_PKT_TUN) 782 tx_offload.ptid |= txgbe_parse_tun_ptid(tx_pkt); 783 tx_offload.l2_len = tx_pkt->l2_len; 784 tx_offload.l3_len = tx_pkt->l3_len; 785 tx_offload.l4_len = tx_pkt->l4_len; 786 tx_offload.vlan_tci = tx_pkt->vlan_tci; 787 tx_offload.tso_segsz = tx_pkt->tso_segsz; 788 tx_offload.outer_l2_len = tx_pkt->outer_l2_len; 789 tx_offload.outer_l3_len = tx_pkt->outer_l3_len; 790 tx_offload.outer_tun_len = txgbe_get_tun_len(tx_pkt); 791 792 #ifdef RTE_LIB_SECURITY 793 if (use_ipsec) { 794 union txgbe_crypto_tx_desc_md *ipsec_mdata = 795 (union txgbe_crypto_tx_desc_md *) 796 rte_security_dynfield(tx_pkt); 797 tx_offload.sa_idx = ipsec_mdata->sa_idx; 798 tx_offload.sec_pad_len = ipsec_mdata->pad_len; 799 } 800 #endif 801 802 /* If new context need be built or reuse the exist ctx*/ 803 ctx = what_ctx_update(txq, tx_ol_req, tx_offload); 804 /* Only allocate context descriptor if required */ 805 new_ctx = (ctx == TXGBE_CTX_NUM); 806 ctx = txq->ctx_curr; 807 } 808 809 /* 810 * Keep track of how many descriptors are used this loop 811 * This will always be the number of segments + the number of 812 * Context descriptors required to transmit the packet 813 */ 814 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx); 815 816 /* 817 * The number of descriptors that must be allocated for a 818 * packet is the number of segments of that packet, plus 1 819 * Context Descriptor for the hardware offload, if any. 820 * Determine the last TX descriptor to allocate in the TX ring 821 * for the packet, starting from the current position (tx_id) 822 * in the ring. 823 */ 824 tx_last = (uint16_t)(tx_id + nb_used - 1); 825 826 /* Circular ring */ 827 if (tx_last >= txq->nb_tx_desc) 828 tx_last = (uint16_t)(tx_last - txq->nb_tx_desc); 829 830 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u" 831 " tx_first=%u tx_last=%u", 832 (uint16_t)txq->port_id, 833 (uint16_t)txq->queue_id, 834 (uint32_t)pkt_len, 835 (uint16_t)tx_id, 836 (uint16_t)tx_last); 837 838 /* 839 * Make sure there are enough TX descriptors available to 840 * transmit the entire packet. 841 * nb_used better be less than or equal to txq->tx_free_thresh 842 */ 843 if (nb_used > txq->nb_tx_free) { 844 PMD_TX_FREE_LOG(DEBUG, 845 "Not enough free TX descriptors " 846 "nb_used=%4u nb_free=%4u " 847 "(port=%d queue=%d)", 848 nb_used, txq->nb_tx_free, 849 txq->port_id, txq->queue_id); 850 851 if (txgbe_xmit_cleanup(txq) != 0) { 852 /* Could not clean any descriptors */ 853 if (nb_tx == 0) 854 return 0; 855 goto end_of_tx; 856 } 857 858 /* nb_used better be <= txq->tx_free_thresh */ 859 if (unlikely(nb_used > txq->tx_free_thresh)) { 860 PMD_TX_FREE_LOG(DEBUG, 861 "The number of descriptors needed to " 862 "transmit the packet exceeds the " 863 "RS bit threshold. This will impact " 864 "performance." 865 "nb_used=%4u nb_free=%4u " 866 "tx_free_thresh=%4u. " 867 "(port=%d queue=%d)", 868 nb_used, txq->nb_tx_free, 869 txq->tx_free_thresh, 870 txq->port_id, txq->queue_id); 871 /* 872 * Loop here until there are enough TX 873 * descriptors or until the ring cannot be 874 * cleaned. 875 */ 876 while (nb_used > txq->nb_tx_free) { 877 if (txgbe_xmit_cleanup(txq) != 0) { 878 /* 879 * Could not clean any 880 * descriptors 881 */ 882 if (nb_tx == 0) 883 return 0; 884 goto end_of_tx; 885 } 886 } 887 } 888 } 889 890 /* 891 * By now there are enough free TX descriptors to transmit 892 * the packet. 893 */ 894 895 /* 896 * Set common flags of all TX Data Descriptors. 897 * 898 * The following bits must be set in all Data Descriptors: 899 * - TXGBE_TXD_DTYP_DATA 900 * - TXGBE_TXD_DCMD_DEXT 901 * 902 * The following bits must be set in the first Data Descriptor 903 * and are ignored in the other ones: 904 * - TXGBE_TXD_DCMD_IFCS 905 * - TXGBE_TXD_MAC_1588 906 * - TXGBE_TXD_DCMD_VLE 907 * 908 * The following bits must only be set in the last Data 909 * Descriptor: 910 * - TXGBE_TXD_CMD_EOP 911 * 912 * The following bits can be set in any Data Descriptor, but 913 * are only set in the last Data Descriptor: 914 * - TXGBE_TXD_CMD_RS 915 */ 916 cmd_type_len = TXGBE_TXD_FCS; 917 918 #ifdef RTE_LIBRTE_IEEE1588 919 if (ol_flags & RTE_MBUF_F_TX_IEEE1588_TMST) 920 cmd_type_len |= TXGBE_TXD_1588; 921 #endif 922 923 olinfo_status = 0; 924 if (tx_ol_req) { 925 if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { 926 /* when TSO is on, paylen in descriptor is the 927 * not the packet len but the tcp payload len 928 */ 929 pkt_len -= (tx_offload.l2_len + 930 tx_offload.l3_len + tx_offload.l4_len); 931 pkt_len -= 932 (tx_pkt->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) 933 ? tx_offload.outer_l2_len + 934 tx_offload.outer_l3_len : 0; 935 } 936 937 /* 938 * Setup the TX Advanced Context Descriptor if required 939 */ 940 if (new_ctx) { 941 volatile struct txgbe_tx_ctx_desc *ctx_txd; 942 943 ctx_txd = (volatile struct txgbe_tx_ctx_desc *) 944 &txr[tx_id]; 945 946 txn = &sw_ring[txe->next_id]; 947 rte_prefetch0(&txn->mbuf->pool); 948 949 if (txe->mbuf != NULL) { 950 rte_pktmbuf_free_seg(txe->mbuf); 951 txe->mbuf = NULL; 952 } 953 954 txgbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req, 955 tx_offload, 956 rte_security_dynfield(tx_pkt)); 957 958 txe->last_id = tx_last; 959 tx_id = txe->next_id; 960 txe = txn; 961 } 962 963 /* 964 * Setup the TX Advanced Data Descriptor, 965 * This path will go through 966 * whatever new/reuse the context descriptor 967 */ 968 cmd_type_len |= tx_desc_ol_flags_to_cmdtype(ol_flags); 969 olinfo_status |= 970 tx_desc_cksum_flags_to_olinfo(ol_flags); 971 olinfo_status |= TXGBE_TXD_IDX(ctx); 972 } 973 974 olinfo_status |= TXGBE_TXD_PAYLEN(pkt_len); 975 #ifdef RTE_LIB_SECURITY 976 if (use_ipsec) 977 olinfo_status |= TXGBE_TXD_IPSEC; 978 #endif 979 980 m_seg = tx_pkt; 981 do { 982 txd = &txr[tx_id]; 983 txn = &sw_ring[txe->next_id]; 984 rte_prefetch0(&txn->mbuf->pool); 985 986 if (txe->mbuf != NULL) 987 rte_pktmbuf_free_seg(txe->mbuf); 988 txe->mbuf = m_seg; 989 990 /* 991 * Set up Transmit Data Descriptor. 992 */ 993 slen = m_seg->data_len; 994 buf_dma_addr = rte_mbuf_data_iova(m_seg); 995 txd->qw0 = rte_cpu_to_le_64(buf_dma_addr); 996 txd->dw2 = rte_cpu_to_le_32(cmd_type_len | slen); 997 txd->dw3 = rte_cpu_to_le_32(olinfo_status); 998 txe->last_id = tx_last; 999 tx_id = txe->next_id; 1000 txe = txn; 1001 m_seg = m_seg->next; 1002 } while (m_seg != NULL); 1003 1004 /* 1005 * The last packet data descriptor needs End Of Packet (EOP) 1006 */ 1007 cmd_type_len |= TXGBE_TXD_EOP; 1008 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used); 1009 1010 txd->dw2 |= rte_cpu_to_le_32(cmd_type_len); 1011 } 1012 1013 end_of_tx: 1014 1015 rte_wmb(); 1016 1017 /* 1018 * Set the Transmit Descriptor Tail (TDT) 1019 */ 1020 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", 1021 (uint16_t)txq->port_id, (uint16_t)txq->queue_id, 1022 (uint16_t)tx_id, (uint16_t)nb_tx); 1023 txgbe_set32_relaxed(txq->tdt_reg_addr, tx_id); 1024 txq->tx_tail = tx_id; 1025 1026 return nb_tx; 1027 } 1028 1029 /********************************************************************* 1030 * 1031 * TX prep functions 1032 * 1033 **********************************************************************/ 1034 uint16_t 1035 txgbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 1036 { 1037 int i, ret; 1038 uint64_t ol_flags; 1039 struct rte_mbuf *m; 1040 struct txgbe_tx_queue *txq = (struct txgbe_tx_queue *)tx_queue; 1041 1042 for (i = 0; i < nb_pkts; i++) { 1043 m = tx_pkts[i]; 1044 ol_flags = m->ol_flags; 1045 1046 /** 1047 * Check if packet meets requirements for number of segments 1048 * 1049 * NOTE: for txgbe it's always (40 - WTHRESH) for both TSO and 1050 * non-TSO 1051 */ 1052 1053 if (m->nb_segs > TXGBE_TX_MAX_SEG - txq->wthresh) { 1054 rte_errno = -EINVAL; 1055 return i; 1056 } 1057 1058 if (ol_flags & TXGBE_TX_OFFLOAD_NOTSUP_MASK) { 1059 rte_errno = -ENOTSUP; 1060 return i; 1061 } 1062 1063 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 1064 ret = rte_validate_tx_offload(m); 1065 if (ret != 0) { 1066 rte_errno = ret; 1067 return i; 1068 } 1069 #endif 1070 ret = rte_net_intel_cksum_prepare(m); 1071 if (ret != 0) { 1072 rte_errno = ret; 1073 return i; 1074 } 1075 } 1076 1077 return i; 1078 } 1079 1080 /********************************************************************* 1081 * 1082 * RX functions 1083 * 1084 **********************************************************************/ 1085 /* @note: fix txgbe_dev_supported_ptypes_get() if any change here. */ 1086 static inline uint32_t 1087 txgbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptid_mask) 1088 { 1089 uint16_t ptid = TXGBE_RXD_PTID(pkt_info); 1090 1091 ptid &= ptid_mask; 1092 1093 return txgbe_decode_ptype(ptid); 1094 } 1095 1096 static inline uint64_t 1097 txgbe_rxd_pkt_info_to_pkt_flags(uint32_t pkt_info) 1098 { 1099 static uint64_t ip_rss_types_map[16] __rte_cache_aligned = { 1100 0, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, RTE_MBUF_F_RX_RSS_HASH, 1101 0, RTE_MBUF_F_RX_RSS_HASH, 0, RTE_MBUF_F_RX_RSS_HASH, 1102 RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0, 1103 0, 0, 0, RTE_MBUF_F_RX_FDIR, 1104 }; 1105 #ifdef RTE_LIBRTE_IEEE1588 1106 static uint64_t ip_pkt_etqf_map[8] = { 1107 0, 0, 0, RTE_MBUF_F_RX_IEEE1588_PTP, 1108 0, 0, 0, 0, 1109 }; 1110 int etfid = txgbe_etflt_id(TXGBE_RXD_PTID(pkt_info)); 1111 if (likely(-1 != etfid)) 1112 return ip_pkt_etqf_map[etfid] | 1113 ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)]; 1114 else 1115 return ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)]; 1116 #else 1117 return ip_rss_types_map[TXGBE_RXD_RSSTYPE(pkt_info)]; 1118 #endif 1119 } 1120 1121 static inline uint64_t 1122 rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags) 1123 { 1124 uint64_t pkt_flags; 1125 1126 /* 1127 * Check if VLAN present only. 1128 * Do not check whether L3/L4 rx checksum done by NIC or not, 1129 * That can be found from rte_eth_rxmode.offloads flag 1130 */ 1131 pkt_flags = (rx_status & TXGBE_RXD_STAT_VLAN && 1132 vlan_flags & RTE_MBUF_F_RX_VLAN_STRIPPED) 1133 ? vlan_flags : 0; 1134 1135 #ifdef RTE_LIBRTE_IEEE1588 1136 if (rx_status & TXGBE_RXD_STAT_1588) 1137 pkt_flags = pkt_flags | RTE_MBUF_F_RX_IEEE1588_TMST; 1138 #endif 1139 return pkt_flags; 1140 } 1141 1142 static inline uint64_t 1143 rx_desc_error_to_pkt_flags(uint32_t rx_status) 1144 { 1145 uint64_t pkt_flags = 0; 1146 1147 /* checksum offload can't be disabled */ 1148 if (rx_status & TXGBE_RXD_STAT_IPCS) { 1149 pkt_flags |= (rx_status & TXGBE_RXD_ERR_IPCS 1150 ? RTE_MBUF_F_RX_IP_CKSUM_BAD : RTE_MBUF_F_RX_IP_CKSUM_GOOD); 1151 } 1152 1153 if (rx_status & TXGBE_RXD_STAT_L4CS) { 1154 pkt_flags |= (rx_status & TXGBE_RXD_ERR_L4CS 1155 ? RTE_MBUF_F_RX_L4_CKSUM_BAD : RTE_MBUF_F_RX_L4_CKSUM_GOOD); 1156 } 1157 1158 if (rx_status & TXGBE_RXD_STAT_EIPCS && 1159 rx_status & TXGBE_RXD_ERR_EIPCS) { 1160 pkt_flags |= RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD; 1161 } 1162 1163 #ifdef RTE_LIB_SECURITY 1164 if (rx_status & TXGBE_RXD_STAT_SECP) { 1165 pkt_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD; 1166 if (rx_status & TXGBE_RXD_ERR_SECERR) 1167 pkt_flags |= RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED; 1168 } 1169 #endif 1170 1171 return pkt_flags; 1172 } 1173 1174 /* 1175 * LOOK_AHEAD defines how many desc statuses to check beyond the 1176 * current descriptor. 1177 * It must be a pound define for optimal performance. 1178 * Do not change the value of LOOK_AHEAD, as the txgbe_rx_scan_hw_ring 1179 * function only works with LOOK_AHEAD=8. 1180 */ 1181 #define LOOK_AHEAD 8 1182 #if (LOOK_AHEAD != 8) 1183 #error "PMD TXGBE: LOOK_AHEAD must be 8\n" 1184 #endif 1185 static inline int 1186 txgbe_rx_scan_hw_ring(struct txgbe_rx_queue *rxq) 1187 { 1188 volatile struct txgbe_rx_desc *rxdp; 1189 struct txgbe_rx_entry *rxep; 1190 struct rte_mbuf *mb; 1191 uint16_t pkt_len; 1192 uint64_t pkt_flags; 1193 int nb_dd; 1194 uint32_t s[LOOK_AHEAD]; 1195 uint32_t pkt_info[LOOK_AHEAD]; 1196 int i, j, nb_rx = 0; 1197 uint32_t status; 1198 1199 /* get references to current descriptor and S/W ring entry */ 1200 rxdp = &rxq->rx_ring[rxq->rx_tail]; 1201 rxep = &rxq->sw_ring[rxq->rx_tail]; 1202 1203 status = rxdp->qw1.lo.status; 1204 /* check to make sure there is at least 1 packet to receive */ 1205 if (!(status & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD))) 1206 return 0; 1207 1208 /* 1209 * Scan LOOK_AHEAD descriptors at a time to determine which descriptors 1210 * reference packets that are ready to be received. 1211 */ 1212 for (i = 0; i < RTE_PMD_TXGBE_RX_MAX_BURST; 1213 i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) { 1214 /* Read desc statuses backwards to avoid race condition */ 1215 for (j = 0; j < LOOK_AHEAD; j++) 1216 s[j] = rte_le_to_cpu_32(rxdp[j].qw1.lo.status); 1217 1218 rte_atomic_thread_fence(__ATOMIC_ACQUIRE); 1219 1220 /* Compute how many status bits were set */ 1221 for (nb_dd = 0; nb_dd < LOOK_AHEAD && 1222 (s[nb_dd] & TXGBE_RXD_STAT_DD); nb_dd++) 1223 ; 1224 1225 for (j = 0; j < nb_dd; j++) 1226 pkt_info[j] = rte_le_to_cpu_32(rxdp[j].qw0.dw0); 1227 1228 nb_rx += nb_dd; 1229 1230 /* Translate descriptor info to mbuf format */ 1231 for (j = 0; j < nb_dd; ++j) { 1232 mb = rxep[j].mbuf; 1233 pkt_len = rte_le_to_cpu_16(rxdp[j].qw1.hi.len) - 1234 rxq->crc_len; 1235 mb->data_len = pkt_len; 1236 mb->pkt_len = pkt_len; 1237 mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].qw1.hi.tag); 1238 1239 /* convert descriptor fields to rte mbuf flags */ 1240 pkt_flags = rx_desc_status_to_pkt_flags(s[j], 1241 rxq->vlan_flags); 1242 pkt_flags |= rx_desc_error_to_pkt_flags(s[j]); 1243 pkt_flags |= 1244 txgbe_rxd_pkt_info_to_pkt_flags(pkt_info[j]); 1245 mb->ol_flags = pkt_flags; 1246 mb->packet_type = 1247 txgbe_rxd_pkt_info_to_pkt_type(pkt_info[j], 1248 rxq->pkt_type_mask); 1249 1250 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) 1251 mb->hash.rss = 1252 rte_le_to_cpu_32(rxdp[j].qw0.dw1); 1253 else if (pkt_flags & RTE_MBUF_F_RX_FDIR) { 1254 mb->hash.fdir.hash = 1255 rte_le_to_cpu_16(rxdp[j].qw0.hi.csum) & 1256 TXGBE_ATR_HASH_MASK; 1257 mb->hash.fdir.id = 1258 rte_le_to_cpu_16(rxdp[j].qw0.hi.ipid); 1259 } 1260 } 1261 1262 /* Move mbuf pointers from the S/W ring to the stage */ 1263 for (j = 0; j < LOOK_AHEAD; ++j) 1264 rxq->rx_stage[i + j] = rxep[j].mbuf; 1265 1266 /* stop if all requested packets could not be received */ 1267 if (nb_dd != LOOK_AHEAD) 1268 break; 1269 } 1270 1271 /* clear software ring entries so we can cleanup correctly */ 1272 for (i = 0; i < nb_rx; ++i) 1273 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL; 1274 1275 return nb_rx; 1276 } 1277 1278 static inline int 1279 txgbe_rx_alloc_bufs(struct txgbe_rx_queue *rxq, bool reset_mbuf) 1280 { 1281 volatile struct txgbe_rx_desc *rxdp; 1282 struct txgbe_rx_entry *rxep; 1283 struct rte_mbuf *mb; 1284 uint16_t alloc_idx; 1285 __le64 dma_addr; 1286 int diag, i; 1287 1288 /* allocate buffers in bulk directly into the S/W ring */ 1289 alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1); 1290 rxep = &rxq->sw_ring[alloc_idx]; 1291 diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep, 1292 rxq->rx_free_thresh); 1293 if (unlikely(diag != 0)) 1294 return -ENOMEM; 1295 1296 rxdp = &rxq->rx_ring[alloc_idx]; 1297 for (i = 0; i < rxq->rx_free_thresh; ++i) { 1298 /* populate the static rte mbuf fields */ 1299 mb = rxep[i].mbuf; 1300 if (reset_mbuf) 1301 mb->port = rxq->port_id; 1302 1303 rte_mbuf_refcnt_set(mb, 1); 1304 mb->data_off = RTE_PKTMBUF_HEADROOM; 1305 1306 /* populate the descriptors */ 1307 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb)); 1308 TXGBE_RXD_HDRADDR(&rxdp[i], 0); 1309 TXGBE_RXD_PKTADDR(&rxdp[i], dma_addr); 1310 } 1311 1312 /* update state of internal queue structure */ 1313 rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh; 1314 if (rxq->rx_free_trigger >= rxq->nb_rx_desc) 1315 rxq->rx_free_trigger = rxq->rx_free_thresh - 1; 1316 1317 /* no errors */ 1318 return 0; 1319 } 1320 1321 static inline uint16_t 1322 txgbe_rx_fill_from_stage(struct txgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, 1323 uint16_t nb_pkts) 1324 { 1325 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail]; 1326 int i; 1327 1328 /* how many packets are ready to return? */ 1329 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail); 1330 1331 /* copy mbuf pointers to the application's packet list */ 1332 for (i = 0; i < nb_pkts; ++i) 1333 rx_pkts[i] = stage[i]; 1334 1335 /* update internal queue state */ 1336 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts); 1337 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts); 1338 1339 return nb_pkts; 1340 } 1341 1342 static inline uint16_t 1343 txgbe_rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1344 uint16_t nb_pkts) 1345 { 1346 struct txgbe_rx_queue *rxq = (struct txgbe_rx_queue *)rx_queue; 1347 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 1348 uint16_t nb_rx = 0; 1349 1350 /* Any previously recv'd pkts will be returned from the Rx stage */ 1351 if (rxq->rx_nb_avail) 1352 return txgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 1353 1354 /* Scan the H/W ring for packets to receive */ 1355 nb_rx = (uint16_t)txgbe_rx_scan_hw_ring(rxq); 1356 1357 /* update internal queue state */ 1358 rxq->rx_next_avail = 0; 1359 rxq->rx_nb_avail = nb_rx; 1360 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx); 1361 1362 /* if required, allocate new buffers to replenish descriptors */ 1363 if (rxq->rx_tail > rxq->rx_free_trigger) { 1364 uint16_t cur_free_trigger = rxq->rx_free_trigger; 1365 1366 if (txgbe_rx_alloc_bufs(rxq, true) != 0) { 1367 int i, j; 1368 1369 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " 1370 "queue_id=%u", (uint16_t)rxq->port_id, 1371 (uint16_t)rxq->queue_id); 1372 1373 dev->data->rx_mbuf_alloc_failed += 1374 rxq->rx_free_thresh; 1375 1376 /* 1377 * Need to rewind any previous receives if we cannot 1378 * allocate new buffers to replenish the old ones. 1379 */ 1380 rxq->rx_nb_avail = 0; 1381 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx); 1382 for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j) 1383 rxq->sw_ring[j].mbuf = rxq->rx_stage[i]; 1384 1385 return 0; 1386 } 1387 1388 /* update tail pointer */ 1389 rte_wmb(); 1390 txgbe_set32_relaxed(rxq->rdt_reg_addr, cur_free_trigger); 1391 } 1392 1393 if (rxq->rx_tail >= rxq->nb_rx_desc) 1394 rxq->rx_tail = 0; 1395 1396 /* received any packets this loop? */ 1397 if (rxq->rx_nb_avail) 1398 return txgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 1399 1400 return 0; 1401 } 1402 1403 /* split requests into chunks of size RTE_PMD_TXGBE_RX_MAX_BURST */ 1404 uint16_t 1405 txgbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1406 uint16_t nb_pkts) 1407 { 1408 uint16_t nb_rx; 1409 1410 if (unlikely(nb_pkts == 0)) 1411 return 0; 1412 1413 if (likely(nb_pkts <= RTE_PMD_TXGBE_RX_MAX_BURST)) 1414 return txgbe_rx_recv_pkts(rx_queue, rx_pkts, nb_pkts); 1415 1416 /* request is relatively large, chunk it up */ 1417 nb_rx = 0; 1418 while (nb_pkts) { 1419 uint16_t ret, n; 1420 1421 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_TXGBE_RX_MAX_BURST); 1422 ret = txgbe_rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n); 1423 nb_rx = (uint16_t)(nb_rx + ret); 1424 nb_pkts = (uint16_t)(nb_pkts - ret); 1425 if (ret < n) 1426 break; 1427 } 1428 1429 return nb_rx; 1430 } 1431 1432 uint16_t 1433 txgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1434 uint16_t nb_pkts) 1435 { 1436 struct txgbe_rx_queue *rxq; 1437 volatile struct txgbe_rx_desc *rx_ring; 1438 volatile struct txgbe_rx_desc *rxdp; 1439 struct txgbe_rx_entry *sw_ring; 1440 struct txgbe_rx_entry *rxe; 1441 struct rte_mbuf *rxm; 1442 struct rte_mbuf *nmb; 1443 struct txgbe_rx_desc rxd; 1444 uint64_t dma_addr; 1445 uint32_t staterr; 1446 uint32_t pkt_info; 1447 uint16_t pkt_len; 1448 uint16_t rx_id; 1449 uint16_t nb_rx; 1450 uint16_t nb_hold; 1451 uint64_t pkt_flags; 1452 1453 nb_rx = 0; 1454 nb_hold = 0; 1455 rxq = rx_queue; 1456 rx_id = rxq->rx_tail; 1457 rx_ring = rxq->rx_ring; 1458 sw_ring = rxq->sw_ring; 1459 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 1460 while (nb_rx < nb_pkts) { 1461 /* 1462 * The order of operations here is important as the DD status 1463 * bit must not be read after any other descriptor fields. 1464 * rx_ring and rxdp are pointing to volatile data so the order 1465 * of accesses cannot be reordered by the compiler. If they were 1466 * not volatile, they could be reordered which could lead to 1467 * using invalid descriptor fields when read from rxd. 1468 */ 1469 rxdp = &rx_ring[rx_id]; 1470 staterr = rxdp->qw1.lo.status; 1471 if (!(staterr & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD))) 1472 break; 1473 rxd = *rxdp; 1474 1475 /* 1476 * End of packet. 1477 * 1478 * If the TXGBE_RXD_STAT_EOP flag is not set, the RX packet 1479 * is likely to be invalid and to be dropped by the various 1480 * validation checks performed by the network stack. 1481 * 1482 * Allocate a new mbuf to replenish the RX ring descriptor. 1483 * If the allocation fails: 1484 * - arrange for that RX descriptor to be the first one 1485 * being parsed the next time the receive function is 1486 * invoked [on the same queue]. 1487 * 1488 * - Stop parsing the RX ring and return immediately. 1489 * 1490 * This policy do not drop the packet received in the RX 1491 * descriptor for which the allocation of a new mbuf failed. 1492 * Thus, it allows that packet to be later retrieved if 1493 * mbuf have been freed in the mean time. 1494 * As a side effect, holding RX descriptors instead of 1495 * systematically giving them back to the NIC may lead to 1496 * RX ring exhaustion situations. 1497 * However, the NIC can gracefully prevent such situations 1498 * to happen by sending specific "back-pressure" flow control 1499 * frames to its peer(s). 1500 */ 1501 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u " 1502 "ext_err_stat=0x%08x pkt_len=%u", 1503 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id, 1504 (uint16_t)rx_id, (uint32_t)staterr, 1505 (uint16_t)rte_le_to_cpu_16(rxd.qw1.hi.len)); 1506 1507 nmb = rte_mbuf_raw_alloc(rxq->mb_pool); 1508 if (nmb == NULL) { 1509 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " 1510 "queue_id=%u", (uint16_t)rxq->port_id, 1511 (uint16_t)rxq->queue_id); 1512 dev->data->rx_mbuf_alloc_failed++; 1513 break; 1514 } 1515 1516 nb_hold++; 1517 rxe = &sw_ring[rx_id]; 1518 rx_id++; 1519 if (rx_id == rxq->nb_rx_desc) 1520 rx_id = 0; 1521 1522 /* Prefetch next mbuf while processing current one. */ 1523 rte_txgbe_prefetch(sw_ring[rx_id].mbuf); 1524 1525 /* 1526 * When next RX descriptor is on a cache-line boundary, 1527 * prefetch the next 4 RX descriptors and the next 8 pointers 1528 * to mbufs. 1529 */ 1530 if ((rx_id & 0x3) == 0) { 1531 rte_txgbe_prefetch(&rx_ring[rx_id]); 1532 rte_txgbe_prefetch(&sw_ring[rx_id]); 1533 } 1534 1535 rxm = rxe->mbuf; 1536 rxe->mbuf = nmb; 1537 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 1538 TXGBE_RXD_HDRADDR(rxdp, 0); 1539 TXGBE_RXD_PKTADDR(rxdp, dma_addr); 1540 1541 /* 1542 * Initialize the returned mbuf. 1543 * 1) setup generic mbuf fields: 1544 * - number of segments, 1545 * - next segment, 1546 * - packet length, 1547 * - RX port identifier. 1548 * 2) integrate hardware offload data, if any: 1549 * - RSS flag & hash, 1550 * - IP checksum flag, 1551 * - VLAN TCI, if any, 1552 * - error flags. 1553 */ 1554 pkt_len = (uint16_t)(rte_le_to_cpu_16(rxd.qw1.hi.len) - 1555 rxq->crc_len); 1556 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1557 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off); 1558 rxm->nb_segs = 1; 1559 rxm->next = NULL; 1560 rxm->pkt_len = pkt_len; 1561 rxm->data_len = pkt_len; 1562 rxm->port = rxq->port_id; 1563 1564 pkt_info = rte_le_to_cpu_32(rxd.qw0.dw0); 1565 /* Only valid if RTE_MBUF_F_RX_VLAN set in pkt_flags */ 1566 rxm->vlan_tci = rte_le_to_cpu_16(rxd.qw1.hi.tag); 1567 1568 pkt_flags = rx_desc_status_to_pkt_flags(staterr, 1569 rxq->vlan_flags); 1570 pkt_flags |= rx_desc_error_to_pkt_flags(staterr); 1571 pkt_flags |= txgbe_rxd_pkt_info_to_pkt_flags(pkt_info); 1572 rxm->ol_flags = pkt_flags; 1573 rxm->packet_type = txgbe_rxd_pkt_info_to_pkt_type(pkt_info, 1574 rxq->pkt_type_mask); 1575 1576 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) { 1577 rxm->hash.rss = rte_le_to_cpu_32(rxd.qw0.dw1); 1578 } else if (pkt_flags & RTE_MBUF_F_RX_FDIR) { 1579 rxm->hash.fdir.hash = 1580 rte_le_to_cpu_16(rxd.qw0.hi.csum) & 1581 TXGBE_ATR_HASH_MASK; 1582 rxm->hash.fdir.id = rte_le_to_cpu_16(rxd.qw0.hi.ipid); 1583 } 1584 /* 1585 * Store the mbuf address into the next entry of the array 1586 * of returned packets. 1587 */ 1588 rx_pkts[nb_rx++] = rxm; 1589 } 1590 rxq->rx_tail = rx_id; 1591 1592 /* 1593 * If the number of free RX descriptors is greater than the RX free 1594 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 1595 * register. 1596 * Update the RDT with the value of the last processed RX descriptor 1597 * minus 1, to guarantee that the RDT register is never equal to the 1598 * RDH register, which creates a "full" ring situation from the 1599 * hardware point of view... 1600 */ 1601 nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold); 1602 if (nb_hold > rxq->rx_free_thresh) { 1603 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u " 1604 "nb_hold=%u nb_rx=%u", 1605 (uint16_t)rxq->port_id, (uint16_t)rxq->queue_id, 1606 (uint16_t)rx_id, (uint16_t)nb_hold, 1607 (uint16_t)nb_rx); 1608 rx_id = (uint16_t)((rx_id == 0) ? 1609 (rxq->nb_rx_desc - 1) : (rx_id - 1)); 1610 txgbe_set32(rxq->rdt_reg_addr, rx_id); 1611 nb_hold = 0; 1612 } 1613 rxq->nb_rx_hold = nb_hold; 1614 return nb_rx; 1615 } 1616 1617 /** 1618 * txgbe_fill_cluster_head_buf - fill the first mbuf of the returned packet 1619 * 1620 * Fill the following info in the HEAD buffer of the Rx cluster: 1621 * - RX port identifier 1622 * - hardware offload data, if any: 1623 * - RSS flag & hash 1624 * - IP checksum flag 1625 * - VLAN TCI, if any 1626 * - error flags 1627 * @head HEAD of the packet cluster 1628 * @desc HW descriptor to get data from 1629 * @rxq Pointer to the Rx queue 1630 */ 1631 static inline void 1632 txgbe_fill_cluster_head_buf(struct rte_mbuf *head, struct txgbe_rx_desc *desc, 1633 struct txgbe_rx_queue *rxq, uint32_t staterr) 1634 { 1635 uint32_t pkt_info; 1636 uint64_t pkt_flags; 1637 1638 head->port = rxq->port_id; 1639 1640 /* The vlan_tci field is only valid when RTE_MBUF_F_RX_VLAN is 1641 * set in the pkt_flags field. 1642 */ 1643 head->vlan_tci = rte_le_to_cpu_16(desc->qw1.hi.tag); 1644 pkt_info = rte_le_to_cpu_32(desc->qw0.dw0); 1645 pkt_flags = rx_desc_status_to_pkt_flags(staterr, rxq->vlan_flags); 1646 pkt_flags |= rx_desc_error_to_pkt_flags(staterr); 1647 pkt_flags |= txgbe_rxd_pkt_info_to_pkt_flags(pkt_info); 1648 head->ol_flags = pkt_flags; 1649 head->packet_type = txgbe_rxd_pkt_info_to_pkt_type(pkt_info, 1650 rxq->pkt_type_mask); 1651 1652 if (likely(pkt_flags & RTE_MBUF_F_RX_RSS_HASH)) { 1653 head->hash.rss = rte_le_to_cpu_32(desc->qw0.dw1); 1654 } else if (pkt_flags & RTE_MBUF_F_RX_FDIR) { 1655 head->hash.fdir.hash = rte_le_to_cpu_16(desc->qw0.hi.csum) 1656 & TXGBE_ATR_HASH_MASK; 1657 head->hash.fdir.id = rte_le_to_cpu_16(desc->qw0.hi.ipid); 1658 } 1659 } 1660 1661 /** 1662 * txgbe_recv_pkts_lro - receive handler for and LRO case. 1663 * 1664 * @rx_queue Rx queue handle 1665 * @rx_pkts table of received packets 1666 * @nb_pkts size of rx_pkts table 1667 * @bulk_alloc if TRUE bulk allocation is used for a HW ring refilling 1668 * 1669 * Handles the Rx HW ring completions when RSC feature is configured. Uses an 1670 * additional ring of txgbe_rsc_entry's that will hold the relevant RSC info. 1671 * 1672 * We use the same logic as in Linux and in FreeBSD txgbe drivers: 1673 * 1) When non-EOP RSC completion arrives: 1674 * a) Update the HEAD of the current RSC aggregation cluster with the new 1675 * segment's data length. 1676 * b) Set the "next" pointer of the current segment to point to the segment 1677 * at the NEXTP index. 1678 * c) Pass the HEAD of RSC aggregation cluster on to the next NEXTP entry 1679 * in the sw_rsc_ring. 1680 * 2) When EOP arrives we just update the cluster's total length and offload 1681 * flags and deliver the cluster up to the upper layers. In our case - put it 1682 * in the rx_pkts table. 1683 * 1684 * Returns the number of received packets/clusters (according to the "bulk 1685 * receive" interface). 1686 */ 1687 static inline uint16_t 1688 txgbe_recv_pkts_lro(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts, 1689 bool bulk_alloc) 1690 { 1691 struct txgbe_rx_queue *rxq = rx_queue; 1692 struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id]; 1693 volatile struct txgbe_rx_desc *rx_ring = rxq->rx_ring; 1694 struct txgbe_rx_entry *sw_ring = rxq->sw_ring; 1695 struct txgbe_scattered_rx_entry *sw_sc_ring = rxq->sw_sc_ring; 1696 uint16_t rx_id = rxq->rx_tail; 1697 uint16_t nb_rx = 0; 1698 uint16_t nb_hold = rxq->nb_rx_hold; 1699 uint16_t prev_id = rxq->rx_tail; 1700 1701 while (nb_rx < nb_pkts) { 1702 bool eop; 1703 struct txgbe_rx_entry *rxe; 1704 struct txgbe_scattered_rx_entry *sc_entry; 1705 struct txgbe_scattered_rx_entry *next_sc_entry = NULL; 1706 struct txgbe_rx_entry *next_rxe = NULL; 1707 struct rte_mbuf *first_seg; 1708 struct rte_mbuf *rxm; 1709 struct rte_mbuf *nmb = NULL; 1710 struct txgbe_rx_desc rxd; 1711 uint16_t data_len; 1712 uint16_t next_id; 1713 volatile struct txgbe_rx_desc *rxdp; 1714 uint32_t staterr; 1715 1716 next_desc: 1717 /* 1718 * The code in this whole file uses the volatile pointer to 1719 * ensure the read ordering of the status and the rest of the 1720 * descriptor fields (on the compiler level only!!!). This is so 1721 * UGLY - why not to just use the compiler barrier instead? DPDK 1722 * even has the rte_compiler_barrier() for that. 1723 * 1724 * But most importantly this is just wrong because this doesn't 1725 * ensure memory ordering in a general case at all. For 1726 * instance, DPDK is supposed to work on Power CPUs where 1727 * compiler barrier may just not be enough! 1728 * 1729 * I tried to write only this function properly to have a 1730 * starting point (as a part of an LRO/RSC series) but the 1731 * compiler cursed at me when I tried to cast away the 1732 * "volatile" from rx_ring (yes, it's volatile too!!!). So, I'm 1733 * keeping it the way it is for now. 1734 * 1735 * The code in this file is broken in so many other places and 1736 * will just not work on a big endian CPU anyway therefore the 1737 * lines below will have to be revisited together with the rest 1738 * of the txgbe PMD. 1739 * 1740 * TODO: 1741 * - Get rid of "volatile" and let the compiler do its job. 1742 * - Use the proper memory barrier (rte_rmb()) to ensure the 1743 * memory ordering below. 1744 */ 1745 rxdp = &rx_ring[rx_id]; 1746 staterr = rte_le_to_cpu_32(rxdp->qw1.lo.status); 1747 1748 if (!(staterr & TXGBE_RXD_STAT_DD)) 1749 break; 1750 1751 rxd = *rxdp; 1752 1753 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u " 1754 "staterr=0x%x data_len=%u", 1755 rxq->port_id, rxq->queue_id, rx_id, staterr, 1756 rte_le_to_cpu_16(rxd.qw1.hi.len)); 1757 1758 if (!bulk_alloc) { 1759 nmb = rte_mbuf_raw_alloc(rxq->mb_pool); 1760 if (nmb == NULL) { 1761 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed " 1762 "port_id=%u queue_id=%u", 1763 rxq->port_id, rxq->queue_id); 1764 1765 dev->data->rx_mbuf_alloc_failed++; 1766 break; 1767 } 1768 } else if (nb_hold > rxq->rx_free_thresh) { 1769 uint16_t next_rdt = rxq->rx_free_trigger; 1770 1771 if (!txgbe_rx_alloc_bufs(rxq, false)) { 1772 rte_wmb(); 1773 txgbe_set32_relaxed(rxq->rdt_reg_addr, 1774 next_rdt); 1775 nb_hold -= rxq->rx_free_thresh; 1776 } else { 1777 PMD_RX_LOG(DEBUG, "RX bulk alloc failed " 1778 "port_id=%u queue_id=%u", 1779 rxq->port_id, rxq->queue_id); 1780 1781 dev->data->rx_mbuf_alloc_failed++; 1782 break; 1783 } 1784 } 1785 1786 nb_hold++; 1787 rxe = &sw_ring[rx_id]; 1788 eop = staterr & TXGBE_RXD_STAT_EOP; 1789 1790 next_id = rx_id + 1; 1791 if (next_id == rxq->nb_rx_desc) 1792 next_id = 0; 1793 1794 /* Prefetch next mbuf while processing current one. */ 1795 rte_txgbe_prefetch(sw_ring[next_id].mbuf); 1796 1797 /* 1798 * When next RX descriptor is on a cache-line boundary, 1799 * prefetch the next 4 RX descriptors and the next 4 pointers 1800 * to mbufs. 1801 */ 1802 if ((next_id & 0x3) == 0) { 1803 rte_txgbe_prefetch(&rx_ring[next_id]); 1804 rte_txgbe_prefetch(&sw_ring[next_id]); 1805 } 1806 1807 rxm = rxe->mbuf; 1808 1809 if (!bulk_alloc) { 1810 __le64 dma = 1811 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 1812 /* 1813 * Update RX descriptor with the physical address of the 1814 * new data buffer of the new allocated mbuf. 1815 */ 1816 rxe->mbuf = nmb; 1817 1818 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1819 TXGBE_RXD_HDRADDR(rxdp, 0); 1820 TXGBE_RXD_PKTADDR(rxdp, dma); 1821 } else { 1822 rxe->mbuf = NULL; 1823 } 1824 1825 /* 1826 * Set data length & data buffer address of mbuf. 1827 */ 1828 data_len = rte_le_to_cpu_16(rxd.qw1.hi.len); 1829 rxm->data_len = data_len; 1830 1831 if (!eop) { 1832 uint16_t nextp_id; 1833 /* 1834 * Get next descriptor index: 1835 * - For RSC it's in the NEXTP field. 1836 * - For a scattered packet - it's just a following 1837 * descriptor. 1838 */ 1839 if (TXGBE_RXD_RSCCNT(rxd.qw0.dw0)) 1840 nextp_id = TXGBE_RXD_NEXTP(staterr); 1841 else 1842 nextp_id = next_id; 1843 1844 next_sc_entry = &sw_sc_ring[nextp_id]; 1845 next_rxe = &sw_ring[nextp_id]; 1846 rte_txgbe_prefetch(next_rxe); 1847 } 1848 1849 sc_entry = &sw_sc_ring[rx_id]; 1850 first_seg = sc_entry->fbuf; 1851 sc_entry->fbuf = NULL; 1852 1853 /* 1854 * If this is the first buffer of the received packet, 1855 * set the pointer to the first mbuf of the packet and 1856 * initialize its context. 1857 * Otherwise, update the total length and the number of segments 1858 * of the current scattered packet, and update the pointer to 1859 * the last mbuf of the current packet. 1860 */ 1861 if (first_seg == NULL) { 1862 first_seg = rxm; 1863 first_seg->pkt_len = data_len; 1864 first_seg->nb_segs = 1; 1865 } else { 1866 first_seg->pkt_len += data_len; 1867 first_seg->nb_segs++; 1868 } 1869 1870 prev_id = rx_id; 1871 rx_id = next_id; 1872 1873 /* 1874 * If this is not the last buffer of the received packet, update 1875 * the pointer to the first mbuf at the NEXTP entry in the 1876 * sw_sc_ring and continue to parse the RX ring. 1877 */ 1878 if (!eop && next_rxe) { 1879 rxm->next = next_rxe->mbuf; 1880 next_sc_entry->fbuf = first_seg; 1881 goto next_desc; 1882 } 1883 1884 /* Initialize the first mbuf of the returned packet */ 1885 txgbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr); 1886 1887 /* 1888 * Deal with the case, when HW CRC srip is disabled. 1889 * That can't happen when LRO is enabled, but still could 1890 * happen for scattered RX mode. 1891 */ 1892 first_seg->pkt_len -= rxq->crc_len; 1893 if (unlikely(rxm->data_len <= rxq->crc_len)) { 1894 struct rte_mbuf *lp; 1895 1896 for (lp = first_seg; lp->next != rxm; lp = lp->next) 1897 ; 1898 1899 first_seg->nb_segs--; 1900 lp->data_len -= rxq->crc_len - rxm->data_len; 1901 lp->next = NULL; 1902 rte_pktmbuf_free_seg(rxm); 1903 } else { 1904 rxm->data_len -= rxq->crc_len; 1905 } 1906 1907 /* Prefetch data of first segment, if configured to do so. */ 1908 rte_packet_prefetch((char *)first_seg->buf_addr + 1909 first_seg->data_off); 1910 1911 /* 1912 * Store the mbuf address into the next entry of the array 1913 * of returned packets. 1914 */ 1915 rx_pkts[nb_rx++] = first_seg; 1916 } 1917 1918 /* 1919 * Record index of the next RX descriptor to probe. 1920 */ 1921 rxq->rx_tail = rx_id; 1922 1923 /* 1924 * If the number of free RX descriptors is greater than the RX free 1925 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 1926 * register. 1927 * Update the RDT with the value of the last processed RX descriptor 1928 * minus 1, to guarantee that the RDT register is never equal to the 1929 * RDH register, which creates a "full" ring situation from the 1930 * hardware point of view... 1931 */ 1932 if (!bulk_alloc && nb_hold > rxq->rx_free_thresh) { 1933 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u " 1934 "nb_hold=%u nb_rx=%u", 1935 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx); 1936 1937 rte_wmb(); 1938 txgbe_set32_relaxed(rxq->rdt_reg_addr, prev_id); 1939 nb_hold = 0; 1940 } 1941 1942 rxq->nb_rx_hold = nb_hold; 1943 return nb_rx; 1944 } 1945 1946 uint16_t 1947 txgbe_recv_pkts_lro_single_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1948 uint16_t nb_pkts) 1949 { 1950 return txgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, false); 1951 } 1952 1953 uint16_t 1954 txgbe_recv_pkts_lro_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1955 uint16_t nb_pkts) 1956 { 1957 return txgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, true); 1958 } 1959 1960 uint64_t 1961 txgbe_get_rx_queue_offloads(struct rte_eth_dev *dev __rte_unused) 1962 { 1963 return RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 1964 } 1965 1966 uint64_t 1967 txgbe_get_rx_port_offloads(struct rte_eth_dev *dev) 1968 { 1969 uint64_t offloads; 1970 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 1971 struct rte_eth_dev_sriov *sriov = &RTE_ETH_DEV_SRIOV(dev); 1972 1973 offloads = RTE_ETH_RX_OFFLOAD_IPV4_CKSUM | 1974 RTE_ETH_RX_OFFLOAD_UDP_CKSUM | 1975 RTE_ETH_RX_OFFLOAD_TCP_CKSUM | 1976 RTE_ETH_RX_OFFLOAD_KEEP_CRC | 1977 RTE_ETH_RX_OFFLOAD_VLAN_FILTER | 1978 RTE_ETH_RX_OFFLOAD_RSS_HASH | 1979 RTE_ETH_RX_OFFLOAD_SCATTER; 1980 1981 if (!txgbe_is_vf(dev)) 1982 offloads |= (RTE_ETH_RX_OFFLOAD_VLAN_FILTER | 1983 RTE_ETH_RX_OFFLOAD_QINQ_STRIP | 1984 RTE_ETH_RX_OFFLOAD_VLAN_EXTEND); 1985 1986 /* 1987 * RSC is only supported by PF devices in a non-SR-IOV 1988 * mode. 1989 */ 1990 if (hw->mac.type == txgbe_mac_raptor && !sriov->active) 1991 offloads |= RTE_ETH_RX_OFFLOAD_TCP_LRO; 1992 1993 if (hw->mac.type == txgbe_mac_raptor) 1994 offloads |= RTE_ETH_RX_OFFLOAD_MACSEC_STRIP; 1995 1996 offloads |= RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM; 1997 1998 #ifdef RTE_LIB_SECURITY 1999 if (dev->security_ctx) 2000 offloads |= RTE_ETH_RX_OFFLOAD_SECURITY; 2001 #endif 2002 2003 return offloads; 2004 } 2005 2006 static void __rte_cold 2007 txgbe_tx_queue_release_mbufs(struct txgbe_tx_queue *txq) 2008 { 2009 unsigned int i; 2010 2011 if (txq->sw_ring != NULL) { 2012 for (i = 0; i < txq->nb_tx_desc; i++) { 2013 if (txq->sw_ring[i].mbuf != NULL) { 2014 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 2015 txq->sw_ring[i].mbuf = NULL; 2016 } 2017 } 2018 } 2019 } 2020 2021 static int 2022 txgbe_tx_done_cleanup_full(struct txgbe_tx_queue *txq, uint32_t free_cnt) 2023 { 2024 struct txgbe_tx_entry *swr_ring = txq->sw_ring; 2025 uint16_t i, tx_last, tx_id; 2026 uint16_t nb_tx_free_last; 2027 uint16_t nb_tx_to_clean; 2028 uint32_t pkt_cnt; 2029 2030 /* Start free mbuf from the next of tx_tail */ 2031 tx_last = txq->tx_tail; 2032 tx_id = swr_ring[tx_last].next_id; 2033 2034 if (txq->nb_tx_free == 0 && txgbe_xmit_cleanup(txq)) 2035 return 0; 2036 2037 nb_tx_to_clean = txq->nb_tx_free; 2038 nb_tx_free_last = txq->nb_tx_free; 2039 if (!free_cnt) 2040 free_cnt = txq->nb_tx_desc; 2041 2042 /* Loop through swr_ring to count the amount of 2043 * freeable mubfs and packets. 2044 */ 2045 for (pkt_cnt = 0; pkt_cnt < free_cnt; ) { 2046 for (i = 0; i < nb_tx_to_clean && 2047 pkt_cnt < free_cnt && 2048 tx_id != tx_last; i++) { 2049 if (swr_ring[tx_id].mbuf != NULL) { 2050 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf); 2051 swr_ring[tx_id].mbuf = NULL; 2052 2053 /* 2054 * last segment in the packet, 2055 * increment packet count 2056 */ 2057 pkt_cnt += (swr_ring[tx_id].last_id == tx_id); 2058 } 2059 2060 tx_id = swr_ring[tx_id].next_id; 2061 } 2062 2063 if (pkt_cnt < free_cnt) { 2064 if (txgbe_xmit_cleanup(txq)) 2065 break; 2066 2067 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last; 2068 nb_tx_free_last = txq->nb_tx_free; 2069 } 2070 } 2071 2072 return (int)pkt_cnt; 2073 } 2074 2075 static int 2076 txgbe_tx_done_cleanup_simple(struct txgbe_tx_queue *txq, 2077 uint32_t free_cnt) 2078 { 2079 int i, n, cnt; 2080 2081 if (free_cnt == 0 || free_cnt > txq->nb_tx_desc) 2082 free_cnt = txq->nb_tx_desc; 2083 2084 cnt = free_cnt - free_cnt % txq->tx_free_thresh; 2085 2086 for (i = 0; i < cnt; i += n) { 2087 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_free_thresh) 2088 break; 2089 2090 n = txgbe_tx_free_bufs(txq); 2091 2092 if (n == 0) 2093 break; 2094 } 2095 2096 return i; 2097 } 2098 2099 int 2100 txgbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt) 2101 { 2102 struct txgbe_tx_queue *txq = (struct txgbe_tx_queue *)tx_queue; 2103 if (txq->offloads == 0 && 2104 #ifdef RTE_LIB_SECURITY 2105 !(txq->using_ipsec) && 2106 #endif 2107 txq->tx_free_thresh >= RTE_PMD_TXGBE_TX_MAX_BURST) 2108 return txgbe_tx_done_cleanup_simple(txq, free_cnt); 2109 2110 return txgbe_tx_done_cleanup_full(txq, free_cnt); 2111 } 2112 2113 static void __rte_cold 2114 txgbe_tx_free_swring(struct txgbe_tx_queue *txq) 2115 { 2116 if (txq != NULL && 2117 txq->sw_ring != NULL) 2118 rte_free(txq->sw_ring); 2119 } 2120 2121 static void __rte_cold 2122 txgbe_tx_queue_release(struct txgbe_tx_queue *txq) 2123 { 2124 if (txq != NULL && txq->ops != NULL) { 2125 txq->ops->release_mbufs(txq); 2126 txq->ops->free_swring(txq); 2127 rte_free(txq); 2128 } 2129 } 2130 2131 void __rte_cold 2132 txgbe_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 2133 { 2134 txgbe_tx_queue_release(dev->data->tx_queues[qid]); 2135 } 2136 2137 /* (Re)set dynamic txgbe_tx_queue fields to defaults */ 2138 static void __rte_cold 2139 txgbe_reset_tx_queue(struct txgbe_tx_queue *txq) 2140 { 2141 static const struct txgbe_tx_desc zeroed_desc = {0}; 2142 struct txgbe_tx_entry *txe = txq->sw_ring; 2143 uint16_t prev, i; 2144 2145 /* Zero out HW ring memory */ 2146 for (i = 0; i < txq->nb_tx_desc; i++) 2147 txq->tx_ring[i] = zeroed_desc; 2148 2149 /* Initialize SW ring entries */ 2150 prev = (uint16_t)(txq->nb_tx_desc - 1); 2151 for (i = 0; i < txq->nb_tx_desc; i++) { 2152 volatile struct txgbe_tx_desc *txd = &txq->tx_ring[i]; 2153 2154 txd->dw3 = rte_cpu_to_le_32(TXGBE_TXD_DD); 2155 txe[i].mbuf = NULL; 2156 txe[i].last_id = i; 2157 txe[prev].next_id = i; 2158 prev = i; 2159 } 2160 2161 txq->tx_next_dd = (uint16_t)(txq->tx_free_thresh - 1); 2162 txq->tx_tail = 0; 2163 2164 /* 2165 * Always allow 1 descriptor to be un-allocated to avoid 2166 * a H/W race condition 2167 */ 2168 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1); 2169 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1); 2170 txq->ctx_curr = 0; 2171 memset((void *)&txq->ctx_cache, 0, 2172 TXGBE_CTX_NUM * sizeof(struct txgbe_ctx_info)); 2173 } 2174 2175 static const struct txgbe_txq_ops def_txq_ops = { 2176 .release_mbufs = txgbe_tx_queue_release_mbufs, 2177 .free_swring = txgbe_tx_free_swring, 2178 .reset = txgbe_reset_tx_queue, 2179 }; 2180 2181 /* Takes an ethdev and a queue and sets up the tx function to be used based on 2182 * the queue parameters. Used in tx_queue_setup by primary process and then 2183 * in dev_init by secondary process when attaching to an existing ethdev. 2184 */ 2185 void __rte_cold 2186 txgbe_set_tx_function(struct rte_eth_dev *dev, struct txgbe_tx_queue *txq) 2187 { 2188 /* Use a simple Tx queue (no offloads, no multi segs) if possible */ 2189 if (txq->offloads == 0 && 2190 #ifdef RTE_LIB_SECURITY 2191 !(txq->using_ipsec) && 2192 #endif 2193 txq->tx_free_thresh >= RTE_PMD_TXGBE_TX_MAX_BURST) { 2194 PMD_INIT_LOG(DEBUG, "Using simple tx code path"); 2195 dev->tx_pkt_burst = txgbe_xmit_pkts_simple; 2196 dev->tx_pkt_prepare = NULL; 2197 } else { 2198 PMD_INIT_LOG(DEBUG, "Using full-featured tx code path"); 2199 PMD_INIT_LOG(DEBUG, 2200 " - offloads = 0x%" PRIx64, 2201 txq->offloads); 2202 PMD_INIT_LOG(DEBUG, 2203 " - tx_free_thresh = %lu [RTE_PMD_TXGBE_TX_MAX_BURST=%lu]", 2204 (unsigned long)txq->tx_free_thresh, 2205 (unsigned long)RTE_PMD_TXGBE_TX_MAX_BURST); 2206 dev->tx_pkt_burst = txgbe_xmit_pkts; 2207 dev->tx_pkt_prepare = txgbe_prep_pkts; 2208 } 2209 } 2210 2211 uint64_t 2212 txgbe_get_tx_queue_offloads(struct rte_eth_dev *dev) 2213 { 2214 RTE_SET_USED(dev); 2215 2216 return 0; 2217 } 2218 2219 uint64_t 2220 txgbe_get_tx_port_offloads(struct rte_eth_dev *dev) 2221 { 2222 uint64_t tx_offload_capa; 2223 2224 tx_offload_capa = 2225 RTE_ETH_TX_OFFLOAD_VLAN_INSERT | 2226 RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | 2227 RTE_ETH_TX_OFFLOAD_UDP_CKSUM | 2228 RTE_ETH_TX_OFFLOAD_TCP_CKSUM | 2229 RTE_ETH_TX_OFFLOAD_SCTP_CKSUM | 2230 RTE_ETH_TX_OFFLOAD_TCP_TSO | 2231 RTE_ETH_TX_OFFLOAD_UDP_TSO | 2232 RTE_ETH_TX_OFFLOAD_UDP_TNL_TSO | 2233 RTE_ETH_TX_OFFLOAD_IP_TNL_TSO | 2234 RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO | 2235 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | 2236 RTE_ETH_TX_OFFLOAD_IPIP_TNL_TSO | 2237 RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO | 2238 RTE_ETH_TX_OFFLOAD_MULTI_SEGS; 2239 2240 if (!txgbe_is_vf(dev)) 2241 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_QINQ_INSERT; 2242 2243 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_MACSEC_INSERT; 2244 2245 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM | 2246 RTE_ETH_TX_OFFLOAD_OUTER_UDP_CKSUM; 2247 2248 #ifdef RTE_LIB_SECURITY 2249 if (dev->security_ctx) 2250 tx_offload_capa |= RTE_ETH_TX_OFFLOAD_SECURITY; 2251 #endif 2252 return tx_offload_capa; 2253 } 2254 2255 int __rte_cold 2256 txgbe_dev_tx_queue_setup(struct rte_eth_dev *dev, 2257 uint16_t queue_idx, 2258 uint16_t nb_desc, 2259 unsigned int socket_id, 2260 const struct rte_eth_txconf *tx_conf) 2261 { 2262 const struct rte_memzone *tz; 2263 struct txgbe_tx_queue *txq; 2264 struct txgbe_hw *hw; 2265 uint16_t tx_free_thresh; 2266 uint64_t offloads; 2267 2268 PMD_INIT_FUNC_TRACE(); 2269 hw = TXGBE_DEV_HW(dev); 2270 2271 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads; 2272 2273 /* 2274 * Validate number of transmit descriptors. 2275 * It must not exceed hardware maximum, and must be multiple 2276 * of TXGBE_ALIGN. 2277 */ 2278 if (nb_desc % TXGBE_TXD_ALIGN != 0 || 2279 nb_desc > TXGBE_RING_DESC_MAX || 2280 nb_desc < TXGBE_RING_DESC_MIN) { 2281 return -EINVAL; 2282 } 2283 2284 /* 2285 * The TX descriptor ring will be cleaned after txq->tx_free_thresh 2286 * descriptors are used or if the number of descriptors required 2287 * to transmit a packet is greater than the number of free TX 2288 * descriptors. 2289 * One descriptor in the TX ring is used as a sentinel to avoid a 2290 * H/W race condition, hence the maximum threshold constraints. 2291 * When set to zero use default values. 2292 */ 2293 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ? 2294 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH); 2295 if (tx_free_thresh >= (nb_desc - 3)) { 2296 PMD_INIT_LOG(ERR, "tx_free_thresh must be less than the number of " 2297 "TX descriptors minus 3. (tx_free_thresh=%u " 2298 "port=%d queue=%d)", 2299 (unsigned int)tx_free_thresh, 2300 (int)dev->data->port_id, (int)queue_idx); 2301 return -(EINVAL); 2302 } 2303 2304 if ((nb_desc % tx_free_thresh) != 0) { 2305 PMD_INIT_LOG(ERR, "tx_free_thresh must be a divisor of the " 2306 "number of TX descriptors. (tx_free_thresh=%u " 2307 "port=%d queue=%d)", (unsigned int)tx_free_thresh, 2308 (int)dev->data->port_id, (int)queue_idx); 2309 return -(EINVAL); 2310 } 2311 2312 /* Free memory prior to re-allocation if needed... */ 2313 if (dev->data->tx_queues[queue_idx] != NULL) { 2314 txgbe_tx_queue_release(dev->data->tx_queues[queue_idx]); 2315 dev->data->tx_queues[queue_idx] = NULL; 2316 } 2317 2318 /* First allocate the tx queue data structure */ 2319 txq = rte_zmalloc_socket("ethdev TX queue", 2320 sizeof(struct txgbe_tx_queue), 2321 RTE_CACHE_LINE_SIZE, socket_id); 2322 if (txq == NULL) 2323 return -ENOMEM; 2324 2325 /* 2326 * Allocate TX ring hardware descriptors. A memzone large enough to 2327 * handle the maximum ring size is allocated in order to allow for 2328 * resizing in later calls to the queue setup function. 2329 */ 2330 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx, 2331 sizeof(struct txgbe_tx_desc) * TXGBE_RING_DESC_MAX, 2332 TXGBE_ALIGN, socket_id); 2333 if (tz == NULL) { 2334 txgbe_tx_queue_release(txq); 2335 return -ENOMEM; 2336 } 2337 2338 txq->nb_tx_desc = nb_desc; 2339 txq->tx_free_thresh = tx_free_thresh; 2340 txq->pthresh = tx_conf->tx_thresh.pthresh; 2341 txq->hthresh = tx_conf->tx_thresh.hthresh; 2342 txq->wthresh = tx_conf->tx_thresh.wthresh; 2343 txq->queue_id = queue_idx; 2344 txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ? 2345 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx); 2346 txq->port_id = dev->data->port_id; 2347 txq->offloads = offloads; 2348 txq->ops = &def_txq_ops; 2349 txq->tx_deferred_start = tx_conf->tx_deferred_start; 2350 #ifdef RTE_LIB_SECURITY 2351 txq->using_ipsec = !!(dev->data->dev_conf.txmode.offloads & 2352 RTE_ETH_TX_OFFLOAD_SECURITY); 2353 #endif 2354 2355 /* Modification to set tail pointer for virtual function 2356 * if vf is detected. 2357 */ 2358 if (hw->mac.type == txgbe_mac_raptor_vf) { 2359 txq->tdt_reg_addr = TXGBE_REG_ADDR(hw, TXGBE_TXWP(queue_idx)); 2360 txq->tdc_reg_addr = TXGBE_REG_ADDR(hw, TXGBE_TXCFG(queue_idx)); 2361 } else { 2362 txq->tdt_reg_addr = TXGBE_REG_ADDR(hw, 2363 TXGBE_TXWP(txq->reg_idx)); 2364 txq->tdc_reg_addr = TXGBE_REG_ADDR(hw, 2365 TXGBE_TXCFG(txq->reg_idx)); 2366 } 2367 2368 txq->tx_ring_phys_addr = TMZ_PADDR(tz); 2369 txq->tx_ring = (struct txgbe_tx_desc *)TMZ_VADDR(tz); 2370 2371 /* Allocate software ring */ 2372 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring", 2373 sizeof(struct txgbe_tx_entry) * nb_desc, 2374 RTE_CACHE_LINE_SIZE, socket_id); 2375 if (txq->sw_ring == NULL) { 2376 txgbe_tx_queue_release(txq); 2377 return -ENOMEM; 2378 } 2379 PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64, 2380 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); 2381 2382 /* set up scalar TX function as appropriate */ 2383 txgbe_set_tx_function(dev, txq); 2384 2385 txq->ops->reset(txq); 2386 2387 dev->data->tx_queues[queue_idx] = txq; 2388 2389 return 0; 2390 } 2391 2392 /** 2393 * txgbe_free_sc_cluster - free the not-yet-completed scattered cluster 2394 * 2395 * The "next" pointer of the last segment of (not-yet-completed) RSC clusters 2396 * in the sw_rsc_ring is not set to NULL but rather points to the next 2397 * mbuf of this RSC aggregation (that has not been completed yet and still 2398 * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we 2399 * will just free first "nb_segs" segments of the cluster explicitly by calling 2400 * an rte_pktmbuf_free_seg(). 2401 * 2402 * @m scattered cluster head 2403 */ 2404 static void __rte_cold 2405 txgbe_free_sc_cluster(struct rte_mbuf *m) 2406 { 2407 uint16_t i, nb_segs = m->nb_segs; 2408 struct rte_mbuf *next_seg; 2409 2410 for (i = 0; i < nb_segs; i++) { 2411 next_seg = m->next; 2412 rte_pktmbuf_free_seg(m); 2413 m = next_seg; 2414 } 2415 } 2416 2417 static void __rte_cold 2418 txgbe_rx_queue_release_mbufs(struct txgbe_rx_queue *rxq) 2419 { 2420 unsigned int i; 2421 2422 if (rxq->sw_ring != NULL) { 2423 for (i = 0; i < rxq->nb_rx_desc; i++) { 2424 if (rxq->sw_ring[i].mbuf != NULL) { 2425 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); 2426 rxq->sw_ring[i].mbuf = NULL; 2427 } 2428 } 2429 if (rxq->rx_nb_avail) { 2430 for (i = 0; i < rxq->rx_nb_avail; ++i) { 2431 struct rte_mbuf *mb; 2432 2433 mb = rxq->rx_stage[rxq->rx_next_avail + i]; 2434 rte_pktmbuf_free_seg(mb); 2435 } 2436 rxq->rx_nb_avail = 0; 2437 } 2438 } 2439 2440 if (rxq->sw_sc_ring) 2441 for (i = 0; i < rxq->nb_rx_desc; i++) 2442 if (rxq->sw_sc_ring[i].fbuf) { 2443 txgbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf); 2444 rxq->sw_sc_ring[i].fbuf = NULL; 2445 } 2446 } 2447 2448 static void __rte_cold 2449 txgbe_rx_queue_release(struct txgbe_rx_queue *rxq) 2450 { 2451 if (rxq != NULL) { 2452 txgbe_rx_queue_release_mbufs(rxq); 2453 rte_free(rxq->sw_ring); 2454 rte_free(rxq->sw_sc_ring); 2455 rte_free(rxq); 2456 } 2457 } 2458 2459 void __rte_cold 2460 txgbe_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) 2461 { 2462 txgbe_rx_queue_release(dev->data->rx_queues[qid]); 2463 } 2464 2465 /* 2466 * Check if Rx Burst Bulk Alloc function can be used. 2467 * Return 2468 * 0: the preconditions are satisfied and the bulk allocation function 2469 * can be used. 2470 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst 2471 * function must be used. 2472 */ 2473 static inline int __rte_cold 2474 check_rx_burst_bulk_alloc_preconditions(struct txgbe_rx_queue *rxq) 2475 { 2476 int ret = 0; 2477 2478 /* 2479 * Make sure the following pre-conditions are satisfied: 2480 * rxq->rx_free_thresh >= RTE_PMD_TXGBE_RX_MAX_BURST 2481 * rxq->rx_free_thresh < rxq->nb_rx_desc 2482 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0 2483 * Scattered packets are not supported. This should be checked 2484 * outside of this function. 2485 */ 2486 if (!(rxq->rx_free_thresh >= RTE_PMD_TXGBE_RX_MAX_BURST)) { 2487 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 2488 "rxq->rx_free_thresh=%d, " 2489 "RTE_PMD_TXGBE_RX_MAX_BURST=%d", 2490 rxq->rx_free_thresh, RTE_PMD_TXGBE_RX_MAX_BURST); 2491 ret = -EINVAL; 2492 } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) { 2493 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 2494 "rxq->rx_free_thresh=%d, " 2495 "rxq->nb_rx_desc=%d", 2496 rxq->rx_free_thresh, rxq->nb_rx_desc); 2497 ret = -EINVAL; 2498 } else if (!((rxq->nb_rx_desc % rxq->rx_free_thresh) == 0)) { 2499 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 2500 "rxq->nb_rx_desc=%d, " 2501 "rxq->rx_free_thresh=%d", 2502 rxq->nb_rx_desc, rxq->rx_free_thresh); 2503 ret = -EINVAL; 2504 } 2505 2506 return ret; 2507 } 2508 2509 /* Reset dynamic txgbe_rx_queue fields back to defaults */ 2510 static void __rte_cold 2511 txgbe_reset_rx_queue(struct txgbe_adapter *adapter, struct txgbe_rx_queue *rxq) 2512 { 2513 static const struct txgbe_rx_desc zeroed_desc = { 2514 {{0}, {0} }, {{0}, {0} } }; 2515 unsigned int i; 2516 uint16_t len = rxq->nb_rx_desc; 2517 2518 /* 2519 * By default, the Rx queue setup function allocates enough memory for 2520 * TXGBE_RING_DESC_MAX. The Rx Burst bulk allocation function requires 2521 * extra memory at the end of the descriptor ring to be zero'd out. 2522 */ 2523 if (adapter->rx_bulk_alloc_allowed) 2524 /* zero out extra memory */ 2525 len += RTE_PMD_TXGBE_RX_MAX_BURST; 2526 2527 /* 2528 * Zero out HW ring memory. Zero out extra memory at the end of 2529 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function 2530 * reads extra memory as zeros. 2531 */ 2532 for (i = 0; i < len; i++) 2533 rxq->rx_ring[i] = zeroed_desc; 2534 2535 /* 2536 * initialize extra software ring entries. Space for these extra 2537 * entries is always allocated 2538 */ 2539 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf)); 2540 for (i = rxq->nb_rx_desc; i < len; ++i) 2541 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf; 2542 2543 rxq->rx_nb_avail = 0; 2544 rxq->rx_next_avail = 0; 2545 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); 2546 rxq->rx_tail = 0; 2547 rxq->nb_rx_hold = 0; 2548 rxq->pkt_first_seg = NULL; 2549 rxq->pkt_last_seg = NULL; 2550 } 2551 2552 int __rte_cold 2553 txgbe_dev_rx_queue_setup(struct rte_eth_dev *dev, 2554 uint16_t queue_idx, 2555 uint16_t nb_desc, 2556 unsigned int socket_id, 2557 const struct rte_eth_rxconf *rx_conf, 2558 struct rte_mempool *mp) 2559 { 2560 const struct rte_memzone *rz; 2561 struct txgbe_rx_queue *rxq; 2562 struct txgbe_hw *hw; 2563 uint16_t len; 2564 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev); 2565 uint64_t offloads; 2566 2567 PMD_INIT_FUNC_TRACE(); 2568 hw = TXGBE_DEV_HW(dev); 2569 2570 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads; 2571 2572 /* 2573 * Validate number of receive descriptors. 2574 * It must not exceed hardware maximum, and must be multiple 2575 * of TXGBE_ALIGN. 2576 */ 2577 if (nb_desc % TXGBE_RXD_ALIGN != 0 || 2578 nb_desc > TXGBE_RING_DESC_MAX || 2579 nb_desc < TXGBE_RING_DESC_MIN) { 2580 return -EINVAL; 2581 } 2582 2583 /* Free memory prior to re-allocation if needed... */ 2584 if (dev->data->rx_queues[queue_idx] != NULL) { 2585 txgbe_rx_queue_release(dev->data->rx_queues[queue_idx]); 2586 dev->data->rx_queues[queue_idx] = NULL; 2587 } 2588 2589 /* First allocate the rx queue data structure */ 2590 rxq = rte_zmalloc_socket("ethdev RX queue", 2591 sizeof(struct txgbe_rx_queue), 2592 RTE_CACHE_LINE_SIZE, socket_id); 2593 if (rxq == NULL) 2594 return -ENOMEM; 2595 rxq->mb_pool = mp; 2596 rxq->nb_rx_desc = nb_desc; 2597 rxq->rx_free_thresh = rx_conf->rx_free_thresh; 2598 rxq->queue_id = queue_idx; 2599 rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ? 2600 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx); 2601 rxq->port_id = dev->data->port_id; 2602 if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 2603 rxq->crc_len = RTE_ETHER_CRC_LEN; 2604 else 2605 rxq->crc_len = 0; 2606 rxq->drop_en = rx_conf->rx_drop_en; 2607 rxq->rx_deferred_start = rx_conf->rx_deferred_start; 2608 rxq->offloads = offloads; 2609 2610 /* 2611 * The packet type in RX descriptor is different for different NICs. 2612 * So set different masks for different NICs. 2613 */ 2614 rxq->pkt_type_mask = TXGBE_PTID_MASK; 2615 2616 /* 2617 * Allocate RX ring hardware descriptors. A memzone large enough to 2618 * handle the maximum ring size is allocated in order to allow for 2619 * resizing in later calls to the queue setup function. 2620 */ 2621 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, 2622 RX_RING_SZ, TXGBE_ALIGN, socket_id); 2623 if (rz == NULL) { 2624 txgbe_rx_queue_release(rxq); 2625 return -ENOMEM; 2626 } 2627 2628 /* 2629 * Zero init all the descriptors in the ring. 2630 */ 2631 memset(rz->addr, 0, RX_RING_SZ); 2632 2633 /* 2634 * Modified to setup VFRDT for Virtual Function 2635 */ 2636 if (hw->mac.type == txgbe_mac_raptor_vf) { 2637 rxq->rdt_reg_addr = 2638 TXGBE_REG_ADDR(hw, TXGBE_RXWP(queue_idx)); 2639 rxq->rdh_reg_addr = 2640 TXGBE_REG_ADDR(hw, TXGBE_RXRP(queue_idx)); 2641 } else { 2642 rxq->rdt_reg_addr = 2643 TXGBE_REG_ADDR(hw, TXGBE_RXWP(rxq->reg_idx)); 2644 rxq->rdh_reg_addr = 2645 TXGBE_REG_ADDR(hw, TXGBE_RXRP(rxq->reg_idx)); 2646 } 2647 2648 rxq->rx_ring_phys_addr = TMZ_PADDR(rz); 2649 rxq->rx_ring = (struct txgbe_rx_desc *)TMZ_VADDR(rz); 2650 2651 /* 2652 * Certain constraints must be met in order to use the bulk buffer 2653 * allocation Rx burst function. If any of Rx queues doesn't meet them 2654 * the feature should be disabled for the whole port. 2655 */ 2656 if (check_rx_burst_bulk_alloc_preconditions(rxq)) { 2657 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Rx Bulk Alloc " 2658 "preconditions - canceling the feature for " 2659 "the whole port[%d]", 2660 rxq->queue_id, rxq->port_id); 2661 adapter->rx_bulk_alloc_allowed = false; 2662 } 2663 2664 /* 2665 * Allocate software ring. Allow for space at the end of the 2666 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst 2667 * function does not access an invalid memory region. 2668 */ 2669 len = nb_desc; 2670 if (adapter->rx_bulk_alloc_allowed) 2671 len += RTE_PMD_TXGBE_RX_MAX_BURST; 2672 2673 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring", 2674 sizeof(struct txgbe_rx_entry) * len, 2675 RTE_CACHE_LINE_SIZE, socket_id); 2676 if (!rxq->sw_ring) { 2677 txgbe_rx_queue_release(rxq); 2678 return -ENOMEM; 2679 } 2680 2681 /* 2682 * Always allocate even if it's not going to be needed in order to 2683 * simplify the code. 2684 * 2685 * This ring is used in LRO and Scattered Rx cases and Scattered Rx may 2686 * be requested in txgbe_dev_rx_init(), which is called later from 2687 * dev_start() flow. 2688 */ 2689 rxq->sw_sc_ring = 2690 rte_zmalloc_socket("rxq->sw_sc_ring", 2691 sizeof(struct txgbe_scattered_rx_entry) * len, 2692 RTE_CACHE_LINE_SIZE, socket_id); 2693 if (!rxq->sw_sc_ring) { 2694 txgbe_rx_queue_release(rxq); 2695 return -ENOMEM; 2696 } 2697 2698 PMD_INIT_LOG(DEBUG, "sw_ring=%p sw_sc_ring=%p hw_ring=%p " 2699 "dma_addr=0x%" PRIx64, 2700 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring, 2701 rxq->rx_ring_phys_addr); 2702 2703 dev->data->rx_queues[queue_idx] = rxq; 2704 2705 txgbe_reset_rx_queue(adapter, rxq); 2706 2707 return 0; 2708 } 2709 2710 uint32_t 2711 txgbe_dev_rx_queue_count(void *rx_queue) 2712 { 2713 #define TXGBE_RXQ_SCAN_INTERVAL 4 2714 volatile struct txgbe_rx_desc *rxdp; 2715 struct txgbe_rx_queue *rxq; 2716 uint32_t desc = 0; 2717 2718 rxq = rx_queue; 2719 rxdp = &rxq->rx_ring[rxq->rx_tail]; 2720 2721 while ((desc < rxq->nb_rx_desc) && 2722 (rxdp->qw1.lo.status & 2723 rte_cpu_to_le_32(TXGBE_RXD_STAT_DD))) { 2724 desc += TXGBE_RXQ_SCAN_INTERVAL; 2725 rxdp += TXGBE_RXQ_SCAN_INTERVAL; 2726 if (rxq->rx_tail + desc >= rxq->nb_rx_desc) 2727 rxdp = &(rxq->rx_ring[rxq->rx_tail + 2728 desc - rxq->nb_rx_desc]); 2729 } 2730 2731 return desc; 2732 } 2733 2734 int 2735 txgbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset) 2736 { 2737 struct txgbe_rx_queue *rxq = rx_queue; 2738 volatile uint32_t *status; 2739 uint32_t nb_hold, desc; 2740 2741 if (unlikely(offset >= rxq->nb_rx_desc)) 2742 return -EINVAL; 2743 2744 nb_hold = rxq->nb_rx_hold; 2745 if (offset >= rxq->nb_rx_desc - nb_hold) 2746 return RTE_ETH_RX_DESC_UNAVAIL; 2747 2748 desc = rxq->rx_tail + offset; 2749 if (desc >= rxq->nb_rx_desc) 2750 desc -= rxq->nb_rx_desc; 2751 2752 status = &rxq->rx_ring[desc].qw1.lo.status; 2753 if (*status & rte_cpu_to_le_32(TXGBE_RXD_STAT_DD)) 2754 return RTE_ETH_RX_DESC_DONE; 2755 2756 return RTE_ETH_RX_DESC_AVAIL; 2757 } 2758 2759 int 2760 txgbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset) 2761 { 2762 struct txgbe_tx_queue *txq = tx_queue; 2763 volatile uint32_t *status; 2764 uint32_t desc; 2765 2766 if (unlikely(offset >= txq->nb_tx_desc)) 2767 return -EINVAL; 2768 2769 desc = txq->tx_tail + offset; 2770 if (desc >= txq->nb_tx_desc) { 2771 desc -= txq->nb_tx_desc; 2772 if (desc >= txq->nb_tx_desc) 2773 desc -= txq->nb_tx_desc; 2774 } 2775 2776 status = &txq->tx_ring[desc].dw3; 2777 if (*status & rte_cpu_to_le_32(TXGBE_TXD_DD)) 2778 return RTE_ETH_TX_DESC_DONE; 2779 2780 return RTE_ETH_TX_DESC_FULL; 2781 } 2782 2783 void __rte_cold 2784 txgbe_dev_clear_queues(struct rte_eth_dev *dev) 2785 { 2786 unsigned int i; 2787 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev); 2788 2789 PMD_INIT_FUNC_TRACE(); 2790 2791 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2792 struct txgbe_tx_queue *txq = dev->data->tx_queues[i]; 2793 2794 if (txq != NULL) { 2795 txq->ops->release_mbufs(txq); 2796 txq->ops->reset(txq); 2797 } 2798 } 2799 2800 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2801 struct txgbe_rx_queue *rxq = dev->data->rx_queues[i]; 2802 2803 if (rxq != NULL) { 2804 txgbe_rx_queue_release_mbufs(rxq); 2805 txgbe_reset_rx_queue(adapter, rxq); 2806 } 2807 } 2808 } 2809 2810 void 2811 txgbe_dev_free_queues(struct rte_eth_dev *dev) 2812 { 2813 unsigned int i; 2814 2815 PMD_INIT_FUNC_TRACE(); 2816 2817 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2818 txgbe_dev_rx_queue_release(dev, i); 2819 dev->data->rx_queues[i] = NULL; 2820 } 2821 dev->data->nb_rx_queues = 0; 2822 2823 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2824 txgbe_dev_tx_queue_release(dev, i); 2825 dev->data->tx_queues[i] = NULL; 2826 } 2827 dev->data->nb_tx_queues = 0; 2828 } 2829 2830 /** 2831 * Receive Side Scaling (RSS) 2832 * 2833 * Principles: 2834 * The source and destination IP addresses of the IP header and the source 2835 * and destination ports of TCP/UDP headers, if any, of received packets are 2836 * hashed against a configurable random key to compute a 32-bit RSS hash result. 2837 * The seven (7) LSBs of the 32-bit hash result are used as an index into a 2838 * 128-entry redirection table (RETA). Each entry of the RETA provides a 3-bit 2839 * RSS output index which is used as the RX queue index where to store the 2840 * received packets. 2841 * The following output is supplied in the RX write-back descriptor: 2842 * - 32-bit result of the Microsoft RSS hash function, 2843 * - 4-bit RSS type field. 2844 */ 2845 2846 /* 2847 * Used as the default key. 2848 */ 2849 static uint8_t rss_intel_key[40] = { 2850 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 2851 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0, 2852 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4, 2853 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, 2854 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA, 2855 }; 2856 2857 static void 2858 txgbe_rss_disable(struct rte_eth_dev *dev) 2859 { 2860 struct txgbe_hw *hw; 2861 2862 hw = TXGBE_DEV_HW(dev); 2863 if (hw->mac.type == txgbe_mac_raptor_vf) 2864 wr32m(hw, TXGBE_VFPLCFG, TXGBE_VFPLCFG_RSSENA, 0); 2865 else 2866 wr32m(hw, TXGBE_RACTL, TXGBE_RACTL_RSSENA, 0); 2867 } 2868 2869 int 2870 txgbe_dev_rss_hash_update(struct rte_eth_dev *dev, 2871 struct rte_eth_rss_conf *rss_conf) 2872 { 2873 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 2874 uint8_t *hash_key; 2875 uint32_t mrqc; 2876 uint32_t rss_key; 2877 uint64_t rss_hf; 2878 uint16_t i; 2879 2880 if (!txgbe_rss_update_sp(hw->mac.type)) { 2881 PMD_DRV_LOG(ERR, "RSS hash update is not supported on this " 2882 "NIC."); 2883 return -ENOTSUP; 2884 } 2885 2886 hash_key = rss_conf->rss_key; 2887 if (hash_key) { 2888 /* Fill in RSS hash key */ 2889 for (i = 0; i < 10; i++) { 2890 rss_key = LS32(hash_key[(i * 4) + 0], 0, 0xFF); 2891 rss_key |= LS32(hash_key[(i * 4) + 1], 8, 0xFF); 2892 rss_key |= LS32(hash_key[(i * 4) + 2], 16, 0xFF); 2893 rss_key |= LS32(hash_key[(i * 4) + 3], 24, 0xFF); 2894 wr32at(hw, TXGBE_REG_RSSKEY, i, rss_key); 2895 } 2896 } 2897 2898 /* Set configured hashing protocols */ 2899 rss_hf = rss_conf->rss_hf & TXGBE_RSS_OFFLOAD_ALL; 2900 if (hw->mac.type == txgbe_mac_raptor_vf) { 2901 mrqc = rd32(hw, TXGBE_VFPLCFG); 2902 mrqc &= ~TXGBE_VFPLCFG_RSSMASK; 2903 if (rss_hf & RTE_ETH_RSS_IPV4) 2904 mrqc |= TXGBE_VFPLCFG_RSSIPV4; 2905 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP) 2906 mrqc |= TXGBE_VFPLCFG_RSSIPV4TCP; 2907 if (rss_hf & RTE_ETH_RSS_IPV6 || 2908 rss_hf & RTE_ETH_RSS_IPV6_EX) 2909 mrqc |= TXGBE_VFPLCFG_RSSIPV6; 2910 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_TCP || 2911 rss_hf & RTE_ETH_RSS_IPV6_TCP_EX) 2912 mrqc |= TXGBE_VFPLCFG_RSSIPV6TCP; 2913 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP) 2914 mrqc |= TXGBE_VFPLCFG_RSSIPV4UDP; 2915 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_UDP || 2916 rss_hf & RTE_ETH_RSS_IPV6_UDP_EX) 2917 mrqc |= TXGBE_VFPLCFG_RSSIPV6UDP; 2918 2919 if (rss_hf) 2920 mrqc |= TXGBE_VFPLCFG_RSSENA; 2921 else 2922 mrqc &= ~TXGBE_VFPLCFG_RSSENA; 2923 2924 if (dev->data->nb_rx_queues > 3) 2925 mrqc |= TXGBE_VFPLCFG_RSSHASH(2); 2926 else if (dev->data->nb_rx_queues > 1) 2927 mrqc |= TXGBE_VFPLCFG_RSSHASH(1); 2928 2929 wr32(hw, TXGBE_VFPLCFG, mrqc); 2930 } else { 2931 mrqc = rd32(hw, TXGBE_RACTL); 2932 mrqc &= ~TXGBE_RACTL_RSSMASK; 2933 if (rss_hf & RTE_ETH_RSS_IPV4) 2934 mrqc |= TXGBE_RACTL_RSSIPV4; 2935 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP) 2936 mrqc |= TXGBE_RACTL_RSSIPV4TCP; 2937 if (rss_hf & RTE_ETH_RSS_IPV6 || 2938 rss_hf & RTE_ETH_RSS_IPV6_EX) 2939 mrqc |= TXGBE_RACTL_RSSIPV6; 2940 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_TCP || 2941 rss_hf & RTE_ETH_RSS_IPV6_TCP_EX) 2942 mrqc |= TXGBE_RACTL_RSSIPV6TCP; 2943 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP) 2944 mrqc |= TXGBE_RACTL_RSSIPV4UDP; 2945 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_UDP || 2946 rss_hf & RTE_ETH_RSS_IPV6_UDP_EX) 2947 mrqc |= TXGBE_RACTL_RSSIPV6UDP; 2948 2949 if (rss_hf) 2950 mrqc |= TXGBE_RACTL_RSSENA; 2951 else 2952 mrqc &= ~TXGBE_RACTL_RSSENA; 2953 2954 wr32(hw, TXGBE_RACTL, mrqc); 2955 } 2956 2957 return 0; 2958 } 2959 2960 int 2961 txgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 2962 struct rte_eth_rss_conf *rss_conf) 2963 { 2964 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 2965 uint8_t *hash_key; 2966 uint32_t mrqc; 2967 uint32_t rss_key; 2968 uint64_t rss_hf; 2969 uint16_t i; 2970 2971 hash_key = rss_conf->rss_key; 2972 if (hash_key) { 2973 /* Return RSS hash key */ 2974 for (i = 0; i < 10; i++) { 2975 rss_key = rd32at(hw, TXGBE_REG_RSSKEY, i); 2976 hash_key[(i * 4) + 0] = RS32(rss_key, 0, 0xFF); 2977 hash_key[(i * 4) + 1] = RS32(rss_key, 8, 0xFF); 2978 hash_key[(i * 4) + 2] = RS32(rss_key, 16, 0xFF); 2979 hash_key[(i * 4) + 3] = RS32(rss_key, 24, 0xFF); 2980 } 2981 } 2982 2983 rss_hf = 0; 2984 if (hw->mac.type == txgbe_mac_raptor_vf) { 2985 mrqc = rd32(hw, TXGBE_VFPLCFG); 2986 if (mrqc & TXGBE_VFPLCFG_RSSIPV4) 2987 rss_hf |= RTE_ETH_RSS_IPV4; 2988 if (mrqc & TXGBE_VFPLCFG_RSSIPV4TCP) 2989 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_TCP; 2990 if (mrqc & TXGBE_VFPLCFG_RSSIPV6) 2991 rss_hf |= RTE_ETH_RSS_IPV6 | 2992 RTE_ETH_RSS_IPV6_EX; 2993 if (mrqc & TXGBE_VFPLCFG_RSSIPV6TCP) 2994 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_TCP | 2995 RTE_ETH_RSS_IPV6_TCP_EX; 2996 if (mrqc & TXGBE_VFPLCFG_RSSIPV4UDP) 2997 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_UDP; 2998 if (mrqc & TXGBE_VFPLCFG_RSSIPV6UDP) 2999 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_UDP | 3000 RTE_ETH_RSS_IPV6_UDP_EX; 3001 if (!(mrqc & TXGBE_VFPLCFG_RSSENA)) 3002 rss_hf = 0; 3003 } else { 3004 mrqc = rd32(hw, TXGBE_RACTL); 3005 if (mrqc & TXGBE_RACTL_RSSIPV4) 3006 rss_hf |= RTE_ETH_RSS_IPV4; 3007 if (mrqc & TXGBE_RACTL_RSSIPV4TCP) 3008 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_TCP; 3009 if (mrqc & TXGBE_RACTL_RSSIPV6) 3010 rss_hf |= RTE_ETH_RSS_IPV6 | 3011 RTE_ETH_RSS_IPV6_EX; 3012 if (mrqc & TXGBE_RACTL_RSSIPV6TCP) 3013 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_TCP | 3014 RTE_ETH_RSS_IPV6_TCP_EX; 3015 if (mrqc & TXGBE_RACTL_RSSIPV4UDP) 3016 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_UDP; 3017 if (mrqc & TXGBE_RACTL_RSSIPV6UDP) 3018 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_UDP | 3019 RTE_ETH_RSS_IPV6_UDP_EX; 3020 if (!(mrqc & TXGBE_RACTL_RSSENA)) 3021 rss_hf = 0; 3022 } 3023 3024 rss_hf &= TXGBE_RSS_OFFLOAD_ALL; 3025 3026 rss_conf->rss_hf = rss_hf; 3027 return 0; 3028 } 3029 3030 static void 3031 txgbe_rss_configure(struct rte_eth_dev *dev) 3032 { 3033 struct rte_eth_rss_conf rss_conf; 3034 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev); 3035 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 3036 uint32_t reta; 3037 uint16_t i; 3038 uint16_t j; 3039 3040 PMD_INIT_FUNC_TRACE(); 3041 3042 /* 3043 * Fill in redirection table 3044 * The byte-swap is needed because NIC registers are in 3045 * little-endian order. 3046 */ 3047 if (adapter->rss_reta_updated == 0) { 3048 reta = 0; 3049 for (i = 0, j = 0; i < RTE_ETH_RSS_RETA_SIZE_128; i++, j++) { 3050 if (j == dev->data->nb_rx_queues) 3051 j = 0; 3052 reta = (reta >> 8) | LS32(j, 24, 0xFF); 3053 if ((i & 3) == 3) 3054 wr32at(hw, TXGBE_REG_RSSTBL, i >> 2, reta); 3055 } 3056 } 3057 /* 3058 * Configure the RSS key and the RSS protocols used to compute 3059 * the RSS hash of input packets. 3060 */ 3061 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf; 3062 if (rss_conf.rss_key == NULL) 3063 rss_conf.rss_key = rss_intel_key; /* Default hash key */ 3064 txgbe_dev_rss_hash_update(dev, &rss_conf); 3065 } 3066 3067 #define NUM_VFTA_REGISTERS 128 3068 #define NIC_RX_BUFFER_SIZE 0x200 3069 3070 static void 3071 txgbe_vmdq_dcb_configure(struct rte_eth_dev *dev) 3072 { 3073 struct rte_eth_vmdq_dcb_conf *cfg; 3074 struct txgbe_hw *hw; 3075 enum rte_eth_nb_pools num_pools; 3076 uint32_t mrqc, vt_ctl, queue_mapping, vlanctrl; 3077 uint16_t pbsize; 3078 uint8_t nb_tcs; /* number of traffic classes */ 3079 int i; 3080 3081 PMD_INIT_FUNC_TRACE(); 3082 hw = TXGBE_DEV_HW(dev); 3083 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf; 3084 num_pools = cfg->nb_queue_pools; 3085 /* Check we have a valid number of pools */ 3086 if (num_pools != RTE_ETH_16_POOLS && num_pools != RTE_ETH_32_POOLS) { 3087 txgbe_rss_disable(dev); 3088 return; 3089 } 3090 /* 16 pools -> 8 traffic classes, 32 pools -> 4 traffic classes */ 3091 nb_tcs = (uint8_t)(RTE_ETH_VMDQ_DCB_NUM_QUEUES / (int)num_pools); 3092 3093 /* 3094 * split rx buffer up into sections, each for 1 traffic class 3095 */ 3096 pbsize = (uint16_t)(NIC_RX_BUFFER_SIZE / nb_tcs); 3097 for (i = 0; i < nb_tcs; i++) { 3098 uint32_t rxpbsize = rd32(hw, TXGBE_PBRXSIZE(i)); 3099 3100 rxpbsize &= (~(0x3FF << 10)); 3101 /* clear 10 bits. */ 3102 rxpbsize |= (pbsize << 10); /* set value */ 3103 wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize); 3104 } 3105 /* zero alloc all unused TCs */ 3106 for (i = nb_tcs; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3107 uint32_t rxpbsize = rd32(hw, TXGBE_PBRXSIZE(i)); 3108 3109 rxpbsize &= (~(0x3FF << 10)); 3110 /* clear 10 bits. */ 3111 wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize); 3112 } 3113 3114 if (num_pools == RTE_ETH_16_POOLS) { 3115 mrqc = TXGBE_PORTCTL_NUMTC_8; 3116 mrqc |= TXGBE_PORTCTL_NUMVT_16; 3117 } else { 3118 mrqc = TXGBE_PORTCTL_NUMTC_4; 3119 mrqc |= TXGBE_PORTCTL_NUMVT_32; 3120 } 3121 wr32m(hw, TXGBE_PORTCTL, 3122 TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK, mrqc); 3123 3124 vt_ctl = TXGBE_POOLCTL_RPLEN; 3125 if (cfg->enable_default_pool) 3126 vt_ctl |= TXGBE_POOLCTL_DEFPL(cfg->default_pool); 3127 else 3128 vt_ctl |= TXGBE_POOLCTL_DEFDSA; 3129 3130 wr32(hw, TXGBE_POOLCTL, vt_ctl); 3131 3132 queue_mapping = 0; 3133 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) 3134 /* 3135 * mapping is done with 3 bits per priority, 3136 * so shift by i*3 each time 3137 */ 3138 queue_mapping |= ((cfg->dcb_tc[i] & 0x07) << (i * 3)); 3139 3140 wr32(hw, TXGBE_RPUP2TC, queue_mapping); 3141 3142 wr32(hw, TXGBE_ARBRXCTL, TXGBE_ARBRXCTL_RRM); 3143 3144 /* enable vlan filtering and allow all vlan tags through */ 3145 vlanctrl = rd32(hw, TXGBE_VLANCTL); 3146 vlanctrl |= TXGBE_VLANCTL_VFE; /* enable vlan filters */ 3147 wr32(hw, TXGBE_VLANCTL, vlanctrl); 3148 3149 /* enable all vlan filters */ 3150 for (i = 0; i < NUM_VFTA_REGISTERS; i++) 3151 wr32(hw, TXGBE_VLANTBL(i), 0xFFFFFFFF); 3152 3153 wr32(hw, TXGBE_POOLRXENA(0), 3154 num_pools == RTE_ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF); 3155 3156 wr32(hw, TXGBE_ETHADDRIDX, 0); 3157 wr32(hw, TXGBE_ETHADDRASSL, 0xFFFFFFFF); 3158 wr32(hw, TXGBE_ETHADDRASSH, 0xFFFFFFFF); 3159 3160 /* set up filters for vlan tags as configured */ 3161 for (i = 0; i < cfg->nb_pool_maps; i++) { 3162 /* set vlan id in VF register and set the valid bit */ 3163 wr32(hw, TXGBE_PSRVLANIDX, i); 3164 wr32(hw, TXGBE_PSRVLAN, (TXGBE_PSRVLAN_EA | 3165 (cfg->pool_map[i].vlan_id & 0xFFF))); 3166 3167 wr32(hw, TXGBE_PSRVLANPLM(0), cfg->pool_map[i].pools); 3168 } 3169 } 3170 3171 /** 3172 * txgbe_dcb_config_tx_hw_config - Configure general DCB TX parameters 3173 * @dev: pointer to eth_dev structure 3174 * @dcb_config: pointer to txgbe_dcb_config structure 3175 */ 3176 static void 3177 txgbe_dcb_tx_hw_config(struct rte_eth_dev *dev, 3178 struct txgbe_dcb_config *dcb_config) 3179 { 3180 uint32_t reg; 3181 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 3182 3183 PMD_INIT_FUNC_TRACE(); 3184 3185 /* Disable the Tx desc arbiter */ 3186 reg = rd32(hw, TXGBE_ARBTXCTL); 3187 reg |= TXGBE_ARBTXCTL_DIA; 3188 wr32(hw, TXGBE_ARBTXCTL, reg); 3189 3190 /* Enable DCB for Tx with 8 TCs */ 3191 reg = rd32(hw, TXGBE_PORTCTL); 3192 reg &= TXGBE_PORTCTL_NUMTC_MASK; 3193 reg |= TXGBE_PORTCTL_DCB; 3194 if (dcb_config->num_tcs.pg_tcs == 8) 3195 reg |= TXGBE_PORTCTL_NUMTC_8; 3196 else 3197 reg |= TXGBE_PORTCTL_NUMTC_4; 3198 3199 wr32(hw, TXGBE_PORTCTL, reg); 3200 3201 /* Enable the Tx desc arbiter */ 3202 reg = rd32(hw, TXGBE_ARBTXCTL); 3203 reg &= ~TXGBE_ARBTXCTL_DIA; 3204 wr32(hw, TXGBE_ARBTXCTL, reg); 3205 } 3206 3207 /** 3208 * txgbe_vmdq_dcb_hw_tx_config - Configure general VMDQ+DCB TX parameters 3209 * @dev: pointer to rte_eth_dev structure 3210 * @dcb_config: pointer to txgbe_dcb_config structure 3211 */ 3212 static void 3213 txgbe_vmdq_dcb_hw_tx_config(struct rte_eth_dev *dev, 3214 struct txgbe_dcb_config *dcb_config) 3215 { 3216 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3217 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf; 3218 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 3219 3220 PMD_INIT_FUNC_TRACE(); 3221 /*PF VF Transmit Enable*/ 3222 wr32(hw, TXGBE_POOLTXENA(0), 3223 vmdq_tx_conf->nb_queue_pools == 3224 RTE_ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF); 3225 3226 /*Configure general DCB TX parameters*/ 3227 txgbe_dcb_tx_hw_config(dev, dcb_config); 3228 } 3229 3230 static void 3231 txgbe_vmdq_dcb_rx_config(struct rte_eth_dev *dev, 3232 struct txgbe_dcb_config *dcb_config) 3233 { 3234 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3235 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf; 3236 struct txgbe_dcb_tc_config *tc; 3237 uint8_t i, j; 3238 3239 /* convert rte_eth_conf.rx_adv_conf to struct txgbe_dcb_config */ 3240 if (vmdq_rx_conf->nb_queue_pools == RTE_ETH_16_POOLS) { 3241 dcb_config->num_tcs.pg_tcs = RTE_ETH_8_TCS; 3242 dcb_config->num_tcs.pfc_tcs = RTE_ETH_8_TCS; 3243 } else { 3244 dcb_config->num_tcs.pg_tcs = RTE_ETH_4_TCS; 3245 dcb_config->num_tcs.pfc_tcs = RTE_ETH_4_TCS; 3246 } 3247 3248 /* Initialize User Priority to Traffic Class mapping */ 3249 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) { 3250 tc = &dcb_config->tc_config[j]; 3251 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0; 3252 } 3253 3254 /* User Priority to Traffic Class mapping */ 3255 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3256 j = vmdq_rx_conf->dcb_tc[i]; 3257 tc = &dcb_config->tc_config[j]; 3258 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |= 3259 (uint8_t)(1 << i); 3260 } 3261 } 3262 3263 static void 3264 txgbe_dcb_vt_tx_config(struct rte_eth_dev *dev, 3265 struct txgbe_dcb_config *dcb_config) 3266 { 3267 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3268 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf; 3269 struct txgbe_dcb_tc_config *tc; 3270 uint8_t i, j; 3271 3272 /* convert rte_eth_conf.rx_adv_conf to struct txgbe_dcb_config */ 3273 if (vmdq_tx_conf->nb_queue_pools == RTE_ETH_16_POOLS) { 3274 dcb_config->num_tcs.pg_tcs = RTE_ETH_8_TCS; 3275 dcb_config->num_tcs.pfc_tcs = RTE_ETH_8_TCS; 3276 } else { 3277 dcb_config->num_tcs.pg_tcs = RTE_ETH_4_TCS; 3278 dcb_config->num_tcs.pfc_tcs = RTE_ETH_4_TCS; 3279 } 3280 3281 /* Initialize User Priority to Traffic Class mapping */ 3282 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) { 3283 tc = &dcb_config->tc_config[j]; 3284 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0; 3285 } 3286 3287 /* User Priority to Traffic Class mapping */ 3288 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3289 j = vmdq_tx_conf->dcb_tc[i]; 3290 tc = &dcb_config->tc_config[j]; 3291 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |= 3292 (uint8_t)(1 << i); 3293 } 3294 } 3295 3296 static void 3297 txgbe_dcb_rx_config(struct rte_eth_dev *dev, 3298 struct txgbe_dcb_config *dcb_config) 3299 { 3300 struct rte_eth_dcb_rx_conf *rx_conf = 3301 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf; 3302 struct txgbe_dcb_tc_config *tc; 3303 uint8_t i, j; 3304 3305 dcb_config->num_tcs.pg_tcs = (uint8_t)rx_conf->nb_tcs; 3306 dcb_config->num_tcs.pfc_tcs = (uint8_t)rx_conf->nb_tcs; 3307 3308 /* Initialize User Priority to Traffic Class mapping */ 3309 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) { 3310 tc = &dcb_config->tc_config[j]; 3311 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0; 3312 } 3313 3314 /* User Priority to Traffic Class mapping */ 3315 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3316 j = rx_conf->dcb_tc[i]; 3317 tc = &dcb_config->tc_config[j]; 3318 tc->path[TXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |= 3319 (uint8_t)(1 << i); 3320 } 3321 } 3322 3323 static void 3324 txgbe_dcb_tx_config(struct rte_eth_dev *dev, 3325 struct txgbe_dcb_config *dcb_config) 3326 { 3327 struct rte_eth_dcb_tx_conf *tx_conf = 3328 &dev->data->dev_conf.tx_adv_conf.dcb_tx_conf; 3329 struct txgbe_dcb_tc_config *tc; 3330 uint8_t i, j; 3331 3332 dcb_config->num_tcs.pg_tcs = (uint8_t)tx_conf->nb_tcs; 3333 dcb_config->num_tcs.pfc_tcs = (uint8_t)tx_conf->nb_tcs; 3334 3335 /* Initialize User Priority to Traffic Class mapping */ 3336 for (j = 0; j < TXGBE_DCB_TC_MAX; j++) { 3337 tc = &dcb_config->tc_config[j]; 3338 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0; 3339 } 3340 3341 /* User Priority to Traffic Class mapping */ 3342 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3343 j = tx_conf->dcb_tc[i]; 3344 tc = &dcb_config->tc_config[j]; 3345 tc->path[TXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |= 3346 (uint8_t)(1 << i); 3347 } 3348 } 3349 3350 /** 3351 * txgbe_dcb_rx_hw_config - Configure general DCB RX HW parameters 3352 * @dev: pointer to eth_dev structure 3353 * @dcb_config: pointer to txgbe_dcb_config structure 3354 */ 3355 static void 3356 txgbe_dcb_rx_hw_config(struct rte_eth_dev *dev, 3357 struct txgbe_dcb_config *dcb_config) 3358 { 3359 uint32_t reg; 3360 uint32_t vlanctrl; 3361 uint8_t i; 3362 uint32_t q; 3363 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 3364 3365 PMD_INIT_FUNC_TRACE(); 3366 /* 3367 * Disable the arbiter before changing parameters 3368 * (always enable recycle mode; WSP) 3369 */ 3370 reg = TXGBE_ARBRXCTL_RRM | TXGBE_ARBRXCTL_WSP | TXGBE_ARBRXCTL_DIA; 3371 wr32(hw, TXGBE_ARBRXCTL, reg); 3372 3373 reg = rd32(hw, TXGBE_PORTCTL); 3374 reg &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK); 3375 if (dcb_config->num_tcs.pg_tcs == 4) { 3376 reg |= TXGBE_PORTCTL_NUMTC_4; 3377 if (dcb_config->vt_mode) 3378 reg |= TXGBE_PORTCTL_NUMVT_32; 3379 else 3380 wr32(hw, TXGBE_POOLCTL, 0); 3381 } 3382 3383 if (dcb_config->num_tcs.pg_tcs == 8) { 3384 reg |= TXGBE_PORTCTL_NUMTC_8; 3385 if (dcb_config->vt_mode) 3386 reg |= TXGBE_PORTCTL_NUMVT_16; 3387 else 3388 wr32(hw, TXGBE_POOLCTL, 0); 3389 } 3390 3391 wr32(hw, TXGBE_PORTCTL, reg); 3392 3393 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 3394 /* Disable drop for all queues in VMDQ mode*/ 3395 for (q = 0; q < TXGBE_MAX_RX_QUEUE_NUM; q++) { 3396 u32 val = 1 << (q % 32); 3397 wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val); 3398 } 3399 } else { 3400 /* Enable drop for all queues in SRIOV mode */ 3401 for (q = 0; q < TXGBE_MAX_RX_QUEUE_NUM; q++) { 3402 u32 val = 1 << (q % 32); 3403 wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val); 3404 } 3405 } 3406 3407 /* VLNCTL: enable vlan filtering and allow all vlan tags through */ 3408 vlanctrl = rd32(hw, TXGBE_VLANCTL); 3409 vlanctrl |= TXGBE_VLANCTL_VFE; /* enable vlan filters */ 3410 wr32(hw, TXGBE_VLANCTL, vlanctrl); 3411 3412 /* VLANTBL - enable all vlan filters */ 3413 for (i = 0; i < NUM_VFTA_REGISTERS; i++) 3414 wr32(hw, TXGBE_VLANTBL(i), 0xFFFFFFFF); 3415 3416 /* 3417 * Configure Rx packet plane (recycle mode; WSP) and 3418 * enable arbiter 3419 */ 3420 reg = TXGBE_ARBRXCTL_RRM | TXGBE_ARBRXCTL_WSP; 3421 wr32(hw, TXGBE_ARBRXCTL, reg); 3422 } 3423 3424 static void 3425 txgbe_dcb_hw_arbite_rx_config(struct txgbe_hw *hw, uint16_t *refill, 3426 uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map) 3427 { 3428 txgbe_dcb_config_rx_arbiter_raptor(hw, refill, max, bwg_id, 3429 tsa, map); 3430 } 3431 3432 static void 3433 txgbe_dcb_hw_arbite_tx_config(struct txgbe_hw *hw, uint16_t *refill, 3434 uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map) 3435 { 3436 switch (hw->mac.type) { 3437 case txgbe_mac_raptor: 3438 txgbe_dcb_config_tx_desc_arbiter_raptor(hw, refill, 3439 max, bwg_id, tsa); 3440 txgbe_dcb_config_tx_data_arbiter_raptor(hw, refill, 3441 max, bwg_id, tsa, map); 3442 break; 3443 default: 3444 break; 3445 } 3446 } 3447 3448 #define DCB_RX_CONFIG 1 3449 #define DCB_TX_CONFIG 1 3450 #define DCB_TX_PB 1024 3451 /** 3452 * txgbe_dcb_hw_configure - Enable DCB and configure 3453 * general DCB in VT mode and non-VT mode parameters 3454 * @dev: pointer to rte_eth_dev structure 3455 * @dcb_config: pointer to txgbe_dcb_config structure 3456 */ 3457 static int 3458 txgbe_dcb_hw_configure(struct rte_eth_dev *dev, 3459 struct txgbe_dcb_config *dcb_config) 3460 { 3461 int ret = 0; 3462 uint8_t i, pfc_en, nb_tcs; 3463 uint16_t pbsize, rx_buffer_size; 3464 uint8_t config_dcb_rx = 0; 3465 uint8_t config_dcb_tx = 0; 3466 uint8_t tsa[TXGBE_DCB_TC_MAX] = {0}; 3467 uint8_t bwgid[TXGBE_DCB_TC_MAX] = {0}; 3468 uint16_t refill[TXGBE_DCB_TC_MAX] = {0}; 3469 uint16_t max[TXGBE_DCB_TC_MAX] = {0}; 3470 uint8_t map[TXGBE_DCB_TC_MAX] = {0}; 3471 struct txgbe_dcb_tc_config *tc; 3472 uint32_t max_frame = dev->data->mtu + 3473 RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN; 3474 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 3475 struct txgbe_bw_conf *bw_conf = TXGBE_DEV_BW_CONF(dev); 3476 3477 switch (dev->data->dev_conf.rxmode.mq_mode) { 3478 case RTE_ETH_MQ_RX_VMDQ_DCB: 3479 dcb_config->vt_mode = true; 3480 config_dcb_rx = DCB_RX_CONFIG; 3481 /* 3482 * get dcb and VT rx configuration parameters 3483 * from rte_eth_conf 3484 */ 3485 txgbe_vmdq_dcb_rx_config(dev, dcb_config); 3486 /*Configure general VMDQ and DCB RX parameters*/ 3487 txgbe_vmdq_dcb_configure(dev); 3488 break; 3489 case RTE_ETH_MQ_RX_DCB: 3490 case RTE_ETH_MQ_RX_DCB_RSS: 3491 dcb_config->vt_mode = false; 3492 config_dcb_rx = DCB_RX_CONFIG; 3493 /* Get dcb TX configuration parameters from rte_eth_conf */ 3494 txgbe_dcb_rx_config(dev, dcb_config); 3495 /*Configure general DCB RX parameters*/ 3496 txgbe_dcb_rx_hw_config(dev, dcb_config); 3497 break; 3498 default: 3499 PMD_INIT_LOG(ERR, "Incorrect DCB RX mode configuration"); 3500 break; 3501 } 3502 switch (dev->data->dev_conf.txmode.mq_mode) { 3503 case RTE_ETH_MQ_TX_VMDQ_DCB: 3504 dcb_config->vt_mode = true; 3505 config_dcb_tx = DCB_TX_CONFIG; 3506 /* get DCB and VT TX configuration parameters 3507 * from rte_eth_conf 3508 */ 3509 txgbe_dcb_vt_tx_config(dev, dcb_config); 3510 /* Configure general VMDQ and DCB TX parameters */ 3511 txgbe_vmdq_dcb_hw_tx_config(dev, dcb_config); 3512 break; 3513 3514 case RTE_ETH_MQ_TX_DCB: 3515 dcb_config->vt_mode = false; 3516 config_dcb_tx = DCB_TX_CONFIG; 3517 /* get DCB TX configuration parameters from rte_eth_conf */ 3518 txgbe_dcb_tx_config(dev, dcb_config); 3519 /* Configure general DCB TX parameters */ 3520 txgbe_dcb_tx_hw_config(dev, dcb_config); 3521 break; 3522 default: 3523 PMD_INIT_LOG(ERR, "Incorrect DCB TX mode configuration"); 3524 break; 3525 } 3526 3527 nb_tcs = dcb_config->num_tcs.pfc_tcs; 3528 /* Unpack map */ 3529 txgbe_dcb_unpack_map_cee(dcb_config, TXGBE_DCB_RX_CONFIG, map); 3530 if (nb_tcs == RTE_ETH_4_TCS) { 3531 /* Avoid un-configured priority mapping to TC0 */ 3532 uint8_t j = 4; 3533 uint8_t mask = 0xFF; 3534 3535 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES - 4; i++) 3536 mask = (uint8_t)(mask & (~(1 << map[i]))); 3537 for (i = 0; mask && (i < TXGBE_DCB_TC_MAX); i++) { 3538 if ((mask & 0x1) && j < RTE_ETH_DCB_NUM_USER_PRIORITIES) 3539 map[j++] = i; 3540 mask >>= 1; 3541 } 3542 /* Re-configure 4 TCs BW */ 3543 for (i = 0; i < nb_tcs; i++) { 3544 tc = &dcb_config->tc_config[i]; 3545 if (bw_conf->tc_num != nb_tcs) 3546 tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent = 3547 (uint8_t)(100 / nb_tcs); 3548 tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent = 3549 (uint8_t)(100 / nb_tcs); 3550 } 3551 for (; i < TXGBE_DCB_TC_MAX; i++) { 3552 tc = &dcb_config->tc_config[i]; 3553 tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent = 0; 3554 tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent = 0; 3555 } 3556 } else { 3557 /* Re-configure 8 TCs BW */ 3558 for (i = 0; i < nb_tcs; i++) { 3559 tc = &dcb_config->tc_config[i]; 3560 if (bw_conf->tc_num != nb_tcs) 3561 tc->path[TXGBE_DCB_TX_CONFIG].bwg_percent = 3562 (uint8_t)(100 / nb_tcs + (i & 1)); 3563 tc->path[TXGBE_DCB_RX_CONFIG].bwg_percent = 3564 (uint8_t)(100 / nb_tcs + (i & 1)); 3565 } 3566 } 3567 3568 rx_buffer_size = NIC_RX_BUFFER_SIZE; 3569 3570 if (config_dcb_rx) { 3571 /* Set RX buffer size */ 3572 pbsize = (uint16_t)(rx_buffer_size / nb_tcs); 3573 uint32_t rxpbsize = pbsize << 10; 3574 3575 for (i = 0; i < nb_tcs; i++) 3576 wr32(hw, TXGBE_PBRXSIZE(i), rxpbsize); 3577 3578 /* zero alloc all unused TCs */ 3579 for (; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) 3580 wr32(hw, TXGBE_PBRXSIZE(i), 0); 3581 } 3582 if (config_dcb_tx) { 3583 /* Only support an equally distributed 3584 * Tx packet buffer strategy. 3585 */ 3586 uint32_t txpktsize = TXGBE_PBTXSIZE_MAX / nb_tcs; 3587 uint32_t txpbthresh = (txpktsize / DCB_TX_PB) - 3588 TXGBE_TXPKT_SIZE_MAX; 3589 3590 for (i = 0; i < nb_tcs; i++) { 3591 wr32(hw, TXGBE_PBTXSIZE(i), txpktsize); 3592 wr32(hw, TXGBE_PBTXDMATH(i), txpbthresh); 3593 } 3594 /* Clear unused TCs, if any, to zero buffer size*/ 3595 for (; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) { 3596 wr32(hw, TXGBE_PBTXSIZE(i), 0); 3597 wr32(hw, TXGBE_PBTXDMATH(i), 0); 3598 } 3599 } 3600 3601 /*Calculates traffic class credits*/ 3602 txgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame, 3603 TXGBE_DCB_TX_CONFIG); 3604 txgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame, 3605 TXGBE_DCB_RX_CONFIG); 3606 3607 if (config_dcb_rx) { 3608 /* Unpack CEE standard containers */ 3609 txgbe_dcb_unpack_refill_cee(dcb_config, 3610 TXGBE_DCB_RX_CONFIG, refill); 3611 txgbe_dcb_unpack_max_cee(dcb_config, max); 3612 txgbe_dcb_unpack_bwgid_cee(dcb_config, 3613 TXGBE_DCB_RX_CONFIG, bwgid); 3614 txgbe_dcb_unpack_tsa_cee(dcb_config, 3615 TXGBE_DCB_RX_CONFIG, tsa); 3616 /* Configure PG(ETS) RX */ 3617 txgbe_dcb_hw_arbite_rx_config(hw, refill, max, bwgid, tsa, map); 3618 } 3619 3620 if (config_dcb_tx) { 3621 /* Unpack CEE standard containers */ 3622 txgbe_dcb_unpack_refill_cee(dcb_config, 3623 TXGBE_DCB_TX_CONFIG, refill); 3624 txgbe_dcb_unpack_max_cee(dcb_config, max); 3625 txgbe_dcb_unpack_bwgid_cee(dcb_config, 3626 TXGBE_DCB_TX_CONFIG, bwgid); 3627 txgbe_dcb_unpack_tsa_cee(dcb_config, 3628 TXGBE_DCB_TX_CONFIG, tsa); 3629 /* Configure PG(ETS) TX */ 3630 txgbe_dcb_hw_arbite_tx_config(hw, refill, max, bwgid, tsa, map); 3631 } 3632 3633 /* Configure queue statistics registers */ 3634 txgbe_dcb_config_tc_stats_raptor(hw, dcb_config); 3635 3636 /* Check if the PFC is supported */ 3637 if (dev->data->dev_conf.dcb_capability_en & RTE_ETH_DCB_PFC_SUPPORT) { 3638 pbsize = (uint16_t)(rx_buffer_size / nb_tcs); 3639 for (i = 0; i < nb_tcs; i++) { 3640 /* If the TC count is 8, 3641 * and the default high_water is 48, 3642 * the low_water is 16 as default. 3643 */ 3644 hw->fc.high_water[i] = (pbsize * 3) / 4; 3645 hw->fc.low_water[i] = pbsize / 4; 3646 /* Enable pfc for this TC */ 3647 tc = &dcb_config->tc_config[i]; 3648 tc->pfc = txgbe_dcb_pfc_enabled; 3649 } 3650 txgbe_dcb_unpack_pfc_cee(dcb_config, map, &pfc_en); 3651 if (dcb_config->num_tcs.pfc_tcs == RTE_ETH_4_TCS) 3652 pfc_en &= 0x0F; 3653 ret = txgbe_dcb_config_pfc(hw, pfc_en, map); 3654 } 3655 3656 return ret; 3657 } 3658 3659 void txgbe_configure_pb(struct rte_eth_dev *dev) 3660 { 3661 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 3662 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 3663 3664 int hdrm; 3665 int tc = dev_conf->rx_adv_conf.dcb_rx_conf.nb_tcs; 3666 3667 /* Reserve 256KB(/512KB) rx buffer for fdir */ 3668 hdrm = 256; /*KB*/ 3669 3670 hw->mac.setup_pba(hw, tc, hdrm, PBA_STRATEGY_EQUAL); 3671 } 3672 3673 void txgbe_configure_port(struct rte_eth_dev *dev) 3674 { 3675 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 3676 int i = 0; 3677 uint16_t tpids[8] = {RTE_ETHER_TYPE_VLAN, RTE_ETHER_TYPE_QINQ, 3678 0x9100, 0x9200, 3679 0x0000, 0x0000, 3680 0x0000, 0x0000}; 3681 3682 PMD_INIT_FUNC_TRACE(); 3683 3684 /* default outer vlan tpid */ 3685 wr32(hw, TXGBE_EXTAG, 3686 TXGBE_EXTAG_ETAG(RTE_ETHER_TYPE_ETAG) | 3687 TXGBE_EXTAG_VLAN(RTE_ETHER_TYPE_QINQ)); 3688 3689 /* default inner vlan tpid */ 3690 wr32m(hw, TXGBE_VLANCTL, 3691 TXGBE_VLANCTL_TPID_MASK, 3692 TXGBE_VLANCTL_TPID(RTE_ETHER_TYPE_VLAN)); 3693 wr32m(hw, TXGBE_DMATXCTRL, 3694 TXGBE_DMATXCTRL_TPID_MASK, 3695 TXGBE_DMATXCTRL_TPID(RTE_ETHER_TYPE_VLAN)); 3696 3697 /* default vlan tpid filters */ 3698 for (i = 0; i < 8; i++) { 3699 wr32m(hw, TXGBE_TAGTPID(i / 2), 3700 (i % 2 ? TXGBE_TAGTPID_MSB_MASK 3701 : TXGBE_TAGTPID_LSB_MASK), 3702 (i % 2 ? TXGBE_TAGTPID_MSB(tpids[i]) 3703 : TXGBE_TAGTPID_LSB(tpids[i]))); 3704 } 3705 3706 /* default vxlan port */ 3707 wr32(hw, TXGBE_VXLANPORT, 4789); 3708 } 3709 3710 /** 3711 * txgbe_configure_dcb - Configure DCB Hardware 3712 * @dev: pointer to rte_eth_dev 3713 */ 3714 void txgbe_configure_dcb(struct rte_eth_dev *dev) 3715 { 3716 struct txgbe_dcb_config *dcb_cfg = TXGBE_DEV_DCB_CONFIG(dev); 3717 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 3718 3719 PMD_INIT_FUNC_TRACE(); 3720 3721 /* check support mq_mode for DCB */ 3722 if (dev_conf->rxmode.mq_mode != RTE_ETH_MQ_RX_VMDQ_DCB && 3723 dev_conf->rxmode.mq_mode != RTE_ETH_MQ_RX_DCB && 3724 dev_conf->rxmode.mq_mode != RTE_ETH_MQ_RX_DCB_RSS) 3725 return; 3726 3727 if (dev->data->nb_rx_queues > RTE_ETH_DCB_NUM_QUEUES) 3728 return; 3729 3730 /** Configure DCB hardware **/ 3731 txgbe_dcb_hw_configure(dev, dcb_cfg); 3732 } 3733 3734 /* 3735 * VMDq only support for 10 GbE NIC. 3736 */ 3737 static void 3738 txgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev) 3739 { 3740 struct rte_eth_vmdq_rx_conf *cfg; 3741 struct txgbe_hw *hw; 3742 enum rte_eth_nb_pools num_pools; 3743 uint32_t mrqc, vt_ctl, vlanctrl; 3744 uint32_t vmolr = 0; 3745 int i; 3746 3747 PMD_INIT_FUNC_TRACE(); 3748 hw = TXGBE_DEV_HW(dev); 3749 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf; 3750 num_pools = cfg->nb_queue_pools; 3751 3752 txgbe_rss_disable(dev); 3753 3754 /* enable vmdq */ 3755 mrqc = TXGBE_PORTCTL_NUMVT_64; 3756 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, mrqc); 3757 3758 /* turn on virtualisation and set the default pool */ 3759 vt_ctl = TXGBE_POOLCTL_RPLEN; 3760 if (cfg->enable_default_pool) 3761 vt_ctl |= TXGBE_POOLCTL_DEFPL(cfg->default_pool); 3762 else 3763 vt_ctl |= TXGBE_POOLCTL_DEFDSA; 3764 3765 wr32(hw, TXGBE_POOLCTL, vt_ctl); 3766 3767 for (i = 0; i < (int)num_pools; i++) { 3768 vmolr = txgbe_convert_vm_rx_mask_to_val(cfg->rx_mode, vmolr); 3769 wr32(hw, TXGBE_POOLETHCTL(i), vmolr); 3770 } 3771 3772 /* enable vlan filtering and allow all vlan tags through */ 3773 vlanctrl = rd32(hw, TXGBE_VLANCTL); 3774 vlanctrl |= TXGBE_VLANCTL_VFE; /* enable vlan filters */ 3775 wr32(hw, TXGBE_VLANCTL, vlanctrl); 3776 3777 /* enable all vlan filters */ 3778 for (i = 0; i < NUM_VFTA_REGISTERS; i++) 3779 wr32(hw, TXGBE_VLANTBL(i), UINT32_MAX); 3780 3781 /* pool enabling for receive - 64 */ 3782 wr32(hw, TXGBE_POOLRXENA(0), UINT32_MAX); 3783 if (num_pools == RTE_ETH_64_POOLS) 3784 wr32(hw, TXGBE_POOLRXENA(1), UINT32_MAX); 3785 3786 /* 3787 * allow pools to read specific mac addresses 3788 * In this case, all pools should be able to read from mac addr 0 3789 */ 3790 wr32(hw, TXGBE_ETHADDRIDX, 0); 3791 wr32(hw, TXGBE_ETHADDRASSL, 0xFFFFFFFF); 3792 wr32(hw, TXGBE_ETHADDRASSH, 0xFFFFFFFF); 3793 3794 /* set up filters for vlan tags as configured */ 3795 for (i = 0; i < cfg->nb_pool_maps; i++) { 3796 /* set vlan id in VF register and set the valid bit */ 3797 wr32(hw, TXGBE_PSRVLANIDX, i); 3798 wr32(hw, TXGBE_PSRVLAN, (TXGBE_PSRVLAN_EA | 3799 TXGBE_PSRVLAN_VID(cfg->pool_map[i].vlan_id))); 3800 /* 3801 * Put the allowed pools in VFB reg. As we only have 16 or 64 3802 * pools, we only need to use the first half of the register 3803 * i.e. bits 0-31 3804 */ 3805 if (((cfg->pool_map[i].pools >> 32) & UINT32_MAX) == 0) 3806 wr32(hw, TXGBE_PSRVLANPLM(0), 3807 (cfg->pool_map[i].pools & UINT32_MAX)); 3808 else 3809 wr32(hw, TXGBE_PSRVLANPLM(1), 3810 ((cfg->pool_map[i].pools >> 32) & UINT32_MAX)); 3811 } 3812 3813 /* Tx General Switch Control Enables VMDQ loopback */ 3814 if (cfg->enable_loop_back) { 3815 wr32(hw, TXGBE_PSRCTL, TXGBE_PSRCTL_LBENA); 3816 for (i = 0; i < 64; i++) 3817 wr32m(hw, TXGBE_POOLETHCTL(i), 3818 TXGBE_POOLETHCTL_LLB, TXGBE_POOLETHCTL_LLB); 3819 } 3820 3821 txgbe_flush(hw); 3822 } 3823 3824 /* 3825 * txgbe_vmdq_tx_hw_configure - Configure general VMDq TX parameters 3826 * @hw: pointer to hardware structure 3827 */ 3828 static void 3829 txgbe_vmdq_tx_hw_configure(struct txgbe_hw *hw) 3830 { 3831 uint32_t reg; 3832 uint32_t q; 3833 3834 PMD_INIT_FUNC_TRACE(); 3835 /*PF VF Transmit Enable*/ 3836 wr32(hw, TXGBE_POOLTXENA(0), UINT32_MAX); 3837 wr32(hw, TXGBE_POOLTXENA(1), UINT32_MAX); 3838 3839 /* Disable the Tx desc arbiter */ 3840 reg = rd32(hw, TXGBE_ARBTXCTL); 3841 reg |= TXGBE_ARBTXCTL_DIA; 3842 wr32(hw, TXGBE_ARBTXCTL, reg); 3843 3844 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, 3845 TXGBE_PORTCTL_NUMVT_64); 3846 3847 /* Disable drop for all queues */ 3848 for (q = 0; q < 128; q++) { 3849 u32 val = 1 << (q % 32); 3850 wr32m(hw, TXGBE_QPRXDROP(q / 32), val, val); 3851 } 3852 3853 /* Enable the Tx desc arbiter */ 3854 reg = rd32(hw, TXGBE_ARBTXCTL); 3855 reg &= ~TXGBE_ARBTXCTL_DIA; 3856 wr32(hw, TXGBE_ARBTXCTL, reg); 3857 3858 txgbe_flush(hw); 3859 } 3860 3861 static int __rte_cold 3862 txgbe_alloc_rx_queue_mbufs(struct txgbe_rx_queue *rxq) 3863 { 3864 struct txgbe_rx_entry *rxe = rxq->sw_ring; 3865 uint64_t dma_addr; 3866 unsigned int i; 3867 3868 /* Initialize software ring entries */ 3869 for (i = 0; i < rxq->nb_rx_desc; i++) { 3870 volatile struct txgbe_rx_desc *rxd; 3871 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool); 3872 3873 if (mbuf == NULL) { 3874 PMD_INIT_LOG(ERR, "RX mbuf alloc failed queue_id=%u", 3875 (unsigned int)rxq->queue_id); 3876 return -ENOMEM; 3877 } 3878 3879 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 3880 mbuf->port = rxq->port_id; 3881 3882 dma_addr = 3883 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 3884 rxd = &rxq->rx_ring[i]; 3885 TXGBE_RXD_HDRADDR(rxd, 0); 3886 TXGBE_RXD_PKTADDR(rxd, dma_addr); 3887 rxe[i].mbuf = mbuf; 3888 } 3889 3890 return 0; 3891 } 3892 3893 static int 3894 txgbe_config_vf_rss(struct rte_eth_dev *dev) 3895 { 3896 struct txgbe_hw *hw; 3897 uint32_t mrqc; 3898 3899 txgbe_rss_configure(dev); 3900 3901 hw = TXGBE_DEV_HW(dev); 3902 3903 /* enable VF RSS */ 3904 mrqc = rd32(hw, TXGBE_PORTCTL); 3905 mrqc &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK); 3906 switch (RTE_ETH_DEV_SRIOV(dev).active) { 3907 case RTE_ETH_64_POOLS: 3908 mrqc |= TXGBE_PORTCTL_NUMVT_64; 3909 break; 3910 3911 case RTE_ETH_32_POOLS: 3912 mrqc |= TXGBE_PORTCTL_NUMVT_32; 3913 break; 3914 3915 default: 3916 PMD_INIT_LOG(ERR, "Invalid pool number in IOV mode with VMDQ RSS"); 3917 return -EINVAL; 3918 } 3919 3920 wr32(hw, TXGBE_PORTCTL, mrqc); 3921 3922 return 0; 3923 } 3924 3925 static int 3926 txgbe_config_vf_default(struct rte_eth_dev *dev) 3927 { 3928 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 3929 uint32_t mrqc; 3930 3931 mrqc = rd32(hw, TXGBE_PORTCTL); 3932 mrqc &= ~(TXGBE_PORTCTL_NUMTC_MASK | TXGBE_PORTCTL_NUMVT_MASK); 3933 switch (RTE_ETH_DEV_SRIOV(dev).active) { 3934 case RTE_ETH_64_POOLS: 3935 mrqc |= TXGBE_PORTCTL_NUMVT_64; 3936 break; 3937 3938 case RTE_ETH_32_POOLS: 3939 mrqc |= TXGBE_PORTCTL_NUMVT_32; 3940 break; 3941 3942 case RTE_ETH_16_POOLS: 3943 mrqc |= TXGBE_PORTCTL_NUMVT_16; 3944 break; 3945 default: 3946 PMD_INIT_LOG(ERR, 3947 "invalid pool number in IOV mode"); 3948 return 0; 3949 } 3950 3951 wr32(hw, TXGBE_PORTCTL, mrqc); 3952 3953 return 0; 3954 } 3955 3956 static int 3957 txgbe_dev_mq_rx_configure(struct rte_eth_dev *dev) 3958 { 3959 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 3960 /* 3961 * SRIOV inactive scheme 3962 * any DCB/RSS w/o VMDq multi-queue setting 3963 */ 3964 switch (dev->data->dev_conf.rxmode.mq_mode) { 3965 case RTE_ETH_MQ_RX_RSS: 3966 case RTE_ETH_MQ_RX_DCB_RSS: 3967 case RTE_ETH_MQ_RX_VMDQ_RSS: 3968 txgbe_rss_configure(dev); 3969 break; 3970 3971 case RTE_ETH_MQ_RX_VMDQ_DCB: 3972 txgbe_vmdq_dcb_configure(dev); 3973 break; 3974 3975 case RTE_ETH_MQ_RX_VMDQ_ONLY: 3976 txgbe_vmdq_rx_hw_configure(dev); 3977 break; 3978 3979 case RTE_ETH_MQ_RX_NONE: 3980 default: 3981 /* if mq_mode is none, disable rss mode.*/ 3982 txgbe_rss_disable(dev); 3983 break; 3984 } 3985 } else { 3986 /* SRIOV active scheme 3987 * Support RSS together with SRIOV. 3988 */ 3989 switch (dev->data->dev_conf.rxmode.mq_mode) { 3990 case RTE_ETH_MQ_RX_RSS: 3991 case RTE_ETH_MQ_RX_VMDQ_RSS: 3992 txgbe_config_vf_rss(dev); 3993 break; 3994 case RTE_ETH_MQ_RX_VMDQ_DCB: 3995 case RTE_ETH_MQ_RX_DCB: 3996 /* In SRIOV, the configuration is the same as VMDq case */ 3997 txgbe_vmdq_dcb_configure(dev); 3998 break; 3999 /* DCB/RSS together with SRIOV is not supported */ 4000 case RTE_ETH_MQ_RX_VMDQ_DCB_RSS: 4001 case RTE_ETH_MQ_RX_DCB_RSS: 4002 PMD_INIT_LOG(ERR, 4003 "Could not support DCB/RSS with VMDq & SRIOV"); 4004 return -1; 4005 default: 4006 txgbe_config_vf_default(dev); 4007 break; 4008 } 4009 } 4010 4011 return 0; 4012 } 4013 4014 static int 4015 txgbe_dev_mq_tx_configure(struct rte_eth_dev *dev) 4016 { 4017 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 4018 uint32_t mtqc; 4019 uint32_t rttdcs; 4020 4021 /* disable arbiter */ 4022 rttdcs = rd32(hw, TXGBE_ARBTXCTL); 4023 rttdcs |= TXGBE_ARBTXCTL_DIA; 4024 wr32(hw, TXGBE_ARBTXCTL, rttdcs); 4025 4026 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 4027 /* 4028 * SRIOV inactive scheme 4029 * any DCB w/o VMDq multi-queue setting 4030 */ 4031 if (dev->data->dev_conf.txmode.mq_mode == RTE_ETH_MQ_TX_VMDQ_ONLY) 4032 txgbe_vmdq_tx_hw_configure(hw); 4033 else 4034 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, 0); 4035 } else { 4036 switch (RTE_ETH_DEV_SRIOV(dev).active) { 4037 /* 4038 * SRIOV active scheme 4039 * FIXME if support DCB together with VMDq & SRIOV 4040 */ 4041 case RTE_ETH_64_POOLS: 4042 mtqc = TXGBE_PORTCTL_NUMVT_64; 4043 break; 4044 case RTE_ETH_32_POOLS: 4045 mtqc = TXGBE_PORTCTL_NUMVT_32; 4046 break; 4047 case RTE_ETH_16_POOLS: 4048 mtqc = TXGBE_PORTCTL_NUMVT_16; 4049 break; 4050 default: 4051 mtqc = 0; 4052 PMD_INIT_LOG(ERR, "invalid pool number in IOV mode"); 4053 } 4054 wr32m(hw, TXGBE_PORTCTL, TXGBE_PORTCTL_NUMVT_MASK, mtqc); 4055 } 4056 4057 /* re-enable arbiter */ 4058 rttdcs &= ~TXGBE_ARBTXCTL_DIA; 4059 wr32(hw, TXGBE_ARBTXCTL, rttdcs); 4060 4061 return 0; 4062 } 4063 4064 /** 4065 * txgbe_get_rscctl_maxdesc 4066 * 4067 * @pool Memory pool of the Rx queue 4068 */ 4069 static inline uint32_t 4070 txgbe_get_rscctl_maxdesc(struct rte_mempool *pool) 4071 { 4072 struct rte_pktmbuf_pool_private *mp_priv = rte_mempool_get_priv(pool); 4073 4074 uint16_t maxdesc = 4075 RTE_IPV4_MAX_PKT_LEN / 4076 (mp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM); 4077 4078 if (maxdesc >= 16) 4079 return TXGBE_RXCFG_RSCMAX_16; 4080 else if (maxdesc >= 8) 4081 return TXGBE_RXCFG_RSCMAX_8; 4082 else if (maxdesc >= 4) 4083 return TXGBE_RXCFG_RSCMAX_4; 4084 else 4085 return TXGBE_RXCFG_RSCMAX_1; 4086 } 4087 4088 /** 4089 * txgbe_set_rsc - configure RSC related port HW registers 4090 * 4091 * Configures the port's RSC related registers. 4092 * 4093 * @dev port handle 4094 * 4095 * Returns 0 in case of success or a non-zero error code 4096 */ 4097 static int 4098 txgbe_set_rsc(struct rte_eth_dev *dev) 4099 { 4100 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; 4101 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 4102 struct rte_eth_dev_info dev_info = { 0 }; 4103 bool rsc_capable = false; 4104 uint16_t i; 4105 uint32_t rdrxctl; 4106 uint32_t rfctl; 4107 4108 /* Sanity check */ 4109 dev->dev_ops->dev_infos_get(dev, &dev_info); 4110 if (dev_info.rx_offload_capa & RTE_ETH_RX_OFFLOAD_TCP_LRO) 4111 rsc_capable = true; 4112 4113 if (!rsc_capable && (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO)) { 4114 PMD_INIT_LOG(CRIT, "LRO is requested on HW that doesn't " 4115 "support it"); 4116 return -EINVAL; 4117 } 4118 4119 /* RSC global configuration */ 4120 4121 if ((rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) && 4122 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO)) { 4123 PMD_INIT_LOG(CRIT, "LRO can't be enabled when HW CRC " 4124 "is disabled"); 4125 return -EINVAL; 4126 } 4127 4128 rfctl = rd32(hw, TXGBE_PSRCTL); 4129 if (rsc_capable && (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO)) 4130 rfctl &= ~TXGBE_PSRCTL_RSCDIA; 4131 else 4132 rfctl |= TXGBE_PSRCTL_RSCDIA; 4133 wr32(hw, TXGBE_PSRCTL, rfctl); 4134 4135 /* If LRO hasn't been requested - we are done here. */ 4136 if (!(rx_conf->offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO)) 4137 return 0; 4138 4139 /* Set PSRCTL.RSCACK bit */ 4140 rdrxctl = rd32(hw, TXGBE_PSRCTL); 4141 rdrxctl |= TXGBE_PSRCTL_RSCACK; 4142 wr32(hw, TXGBE_PSRCTL, rdrxctl); 4143 4144 /* Per-queue RSC configuration */ 4145 for (i = 0; i < dev->data->nb_rx_queues; i++) { 4146 struct txgbe_rx_queue *rxq = dev->data->rx_queues[i]; 4147 uint32_t srrctl = 4148 rd32(hw, TXGBE_RXCFG(rxq->reg_idx)); 4149 uint32_t psrtype = 4150 rd32(hw, TXGBE_POOLRSS(rxq->reg_idx)); 4151 uint32_t eitr = 4152 rd32(hw, TXGBE_ITR(rxq->reg_idx)); 4153 4154 /* 4155 * txgbe PMD doesn't support header-split at the moment. 4156 */ 4157 srrctl &= ~TXGBE_RXCFG_HDRLEN_MASK; 4158 srrctl |= TXGBE_RXCFG_HDRLEN(128); 4159 4160 /* 4161 * TODO: Consider setting the Receive Descriptor Minimum 4162 * Threshold Size for an RSC case. This is not an obviously 4163 * beneficiary option but the one worth considering... 4164 */ 4165 4166 srrctl |= TXGBE_RXCFG_RSCENA; 4167 srrctl &= ~TXGBE_RXCFG_RSCMAX_MASK; 4168 srrctl |= txgbe_get_rscctl_maxdesc(rxq->mb_pool); 4169 psrtype |= TXGBE_POOLRSS_L4HDR; 4170 4171 /* 4172 * RSC: Set ITR interval corresponding to 2K ints/s. 4173 * 4174 * Full-sized RSC aggregations for a 10Gb/s link will 4175 * arrive at about 20K aggregation/s rate. 4176 * 4177 * 2K inst/s rate will make only 10% of the 4178 * aggregations to be closed due to the interrupt timer 4179 * expiration for a streaming at wire-speed case. 4180 * 4181 * For a sparse streaming case this setting will yield 4182 * at most 500us latency for a single RSC aggregation. 4183 */ 4184 eitr &= ~TXGBE_ITR_IVAL_MASK; 4185 eitr |= TXGBE_ITR_IVAL_10G(TXGBE_QUEUE_ITR_INTERVAL_DEFAULT); 4186 eitr |= TXGBE_ITR_WRDSA; 4187 4188 wr32(hw, TXGBE_RXCFG(rxq->reg_idx), srrctl); 4189 wr32(hw, TXGBE_POOLRSS(rxq->reg_idx), psrtype); 4190 wr32(hw, TXGBE_ITR(rxq->reg_idx), eitr); 4191 4192 /* 4193 * RSC requires the mapping of the queue to the 4194 * interrupt vector. 4195 */ 4196 txgbe_set_ivar_map(hw, 0, rxq->reg_idx, i); 4197 } 4198 4199 dev->data->lro = 1; 4200 4201 PMD_INIT_LOG(DEBUG, "enabling LRO mode"); 4202 4203 return 0; 4204 } 4205 4206 void __rte_cold 4207 txgbe_set_rx_function(struct rte_eth_dev *dev) 4208 { 4209 uint16_t i; 4210 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev); 4211 4212 /* 4213 * Initialize the appropriate LRO callback. 4214 * 4215 * If all queues satisfy the bulk allocation preconditions 4216 * (adapter->rx_bulk_alloc_allowed is TRUE) then we may use 4217 * bulk allocation. Otherwise use a single allocation version. 4218 */ 4219 if (dev->data->lro) { 4220 if (adapter->rx_bulk_alloc_allowed) { 4221 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a bulk " 4222 "allocation version"); 4223 dev->rx_pkt_burst = txgbe_recv_pkts_lro_bulk_alloc; 4224 } else { 4225 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a single " 4226 "allocation version"); 4227 dev->rx_pkt_burst = txgbe_recv_pkts_lro_single_alloc; 4228 } 4229 } else if (dev->data->scattered_rx) { 4230 /* 4231 * Set the non-LRO scattered callback: there are bulk and 4232 * single allocation versions. 4233 */ 4234 if (adapter->rx_bulk_alloc_allowed) { 4235 PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk " 4236 "allocation callback (port=%d).", 4237 dev->data->port_id); 4238 dev->rx_pkt_burst = txgbe_recv_pkts_lro_bulk_alloc; 4239 } else { 4240 PMD_INIT_LOG(DEBUG, "Using Regular (non-vector, " 4241 "single allocation) " 4242 "Scattered Rx callback " 4243 "(port=%d).", 4244 dev->data->port_id); 4245 4246 dev->rx_pkt_burst = txgbe_recv_pkts_lro_single_alloc; 4247 } 4248 /* 4249 * Below we set "simple" callbacks according to port/queues parameters. 4250 * If parameters allow we are going to choose between the following 4251 * callbacks: 4252 * - Bulk Allocation 4253 * - Single buffer allocation (the simplest one) 4254 */ 4255 } else if (adapter->rx_bulk_alloc_allowed) { 4256 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " 4257 "satisfied. Rx Burst Bulk Alloc function " 4258 "will be used on port=%d.", 4259 dev->data->port_id); 4260 4261 dev->rx_pkt_burst = txgbe_recv_pkts_bulk_alloc; 4262 } else { 4263 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not " 4264 "satisfied, or Scattered Rx is requested " 4265 "(port=%d).", 4266 dev->data->port_id); 4267 4268 dev->rx_pkt_burst = txgbe_recv_pkts; 4269 } 4270 4271 #ifdef RTE_LIB_SECURITY 4272 for (i = 0; i < dev->data->nb_rx_queues; i++) { 4273 struct txgbe_rx_queue *rxq = dev->data->rx_queues[i]; 4274 4275 rxq->using_ipsec = !!(dev->data->dev_conf.rxmode.offloads & 4276 RTE_ETH_RX_OFFLOAD_SECURITY); 4277 } 4278 #endif 4279 } 4280 4281 /* 4282 * Initializes Receive Unit. 4283 */ 4284 int __rte_cold 4285 txgbe_dev_rx_init(struct rte_eth_dev *dev) 4286 { 4287 struct txgbe_hw *hw; 4288 struct txgbe_rx_queue *rxq; 4289 uint64_t bus_addr; 4290 uint32_t fctrl; 4291 uint32_t hlreg0; 4292 uint32_t srrctl; 4293 uint32_t rdrxctl; 4294 uint32_t rxcsum; 4295 uint16_t buf_size; 4296 uint16_t i; 4297 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; 4298 int rc; 4299 4300 PMD_INIT_FUNC_TRACE(); 4301 hw = TXGBE_DEV_HW(dev); 4302 4303 /* 4304 * Make sure receives are disabled while setting 4305 * up the RX context (registers, descriptor rings, etc.). 4306 */ 4307 wr32m(hw, TXGBE_MACRXCFG, TXGBE_MACRXCFG_ENA, 0); 4308 wr32m(hw, TXGBE_PBRXCTL, TXGBE_PBRXCTL_ENA, 0); 4309 4310 /* Enable receipt of broadcasted frames */ 4311 fctrl = rd32(hw, TXGBE_PSRCTL); 4312 fctrl |= TXGBE_PSRCTL_BCA; 4313 wr32(hw, TXGBE_PSRCTL, fctrl); 4314 4315 /* 4316 * Configure CRC stripping, if any. 4317 */ 4318 hlreg0 = rd32(hw, TXGBE_SECRXCTL); 4319 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 4320 hlreg0 &= ~TXGBE_SECRXCTL_CRCSTRIP; 4321 else 4322 hlreg0 |= TXGBE_SECRXCTL_CRCSTRIP; 4323 wr32(hw, TXGBE_SECRXCTL, hlreg0); 4324 4325 /* 4326 * Configure jumbo frame support, if any. 4327 */ 4328 wr32m(hw, TXGBE_FRMSZ, TXGBE_FRMSZ_MAX_MASK, 4329 TXGBE_FRMSZ_MAX(dev->data->mtu + TXGBE_ETH_OVERHEAD)); 4330 4331 /* 4332 * If loopback mode is configured, set LPBK bit. 4333 */ 4334 hlreg0 = rd32(hw, TXGBE_PSRCTL); 4335 if (hw->mac.type == txgbe_mac_raptor && 4336 dev->data->dev_conf.lpbk_mode) 4337 hlreg0 |= TXGBE_PSRCTL_LBENA; 4338 else 4339 hlreg0 &= ~TXGBE_PSRCTL_LBENA; 4340 4341 wr32(hw, TXGBE_PSRCTL, hlreg0); 4342 4343 /* 4344 * Assume no header split and no VLAN strip support 4345 * on any Rx queue first . 4346 */ 4347 rx_conf->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 4348 4349 /* Setup RX queues */ 4350 for (i = 0; i < dev->data->nb_rx_queues; i++) { 4351 rxq = dev->data->rx_queues[i]; 4352 4353 /* 4354 * Reset crc_len in case it was changed after queue setup by a 4355 * call to configure. 4356 */ 4357 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 4358 rxq->crc_len = RTE_ETHER_CRC_LEN; 4359 else 4360 rxq->crc_len = 0; 4361 4362 /* Setup the Base and Length of the Rx Descriptor Rings */ 4363 bus_addr = rxq->rx_ring_phys_addr; 4364 wr32(hw, TXGBE_RXBAL(rxq->reg_idx), 4365 (uint32_t)(bus_addr & BIT_MASK32)); 4366 wr32(hw, TXGBE_RXBAH(rxq->reg_idx), 4367 (uint32_t)(bus_addr >> 32)); 4368 wr32(hw, TXGBE_RXRP(rxq->reg_idx), 0); 4369 wr32(hw, TXGBE_RXWP(rxq->reg_idx), 0); 4370 4371 srrctl = TXGBE_RXCFG_RNGLEN(rxq->nb_rx_desc); 4372 4373 /* Set if packets are dropped when no descriptors available */ 4374 if (rxq->drop_en) 4375 srrctl |= TXGBE_RXCFG_DROP; 4376 4377 /* 4378 * Configure the RX buffer size in the PKTLEN field of 4379 * the RXCFG register of the queue. 4380 * The value is in 1 KB resolution. Valid values can be from 4381 * 1 KB to 16 KB. 4382 */ 4383 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - 4384 RTE_PKTMBUF_HEADROOM); 4385 buf_size = ROUND_UP(buf_size, 0x1 << 10); 4386 srrctl |= TXGBE_RXCFG_PKTLEN(buf_size); 4387 4388 wr32(hw, TXGBE_RXCFG(rxq->reg_idx), srrctl); 4389 4390 /* It adds dual VLAN length for supporting dual VLAN */ 4391 if (dev->data->mtu + TXGBE_ETH_OVERHEAD + 4392 2 * RTE_VLAN_HLEN > buf_size) 4393 dev->data->scattered_rx = 1; 4394 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) 4395 rx_conf->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 4396 } 4397 4398 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_SCATTER) 4399 dev->data->scattered_rx = 1; 4400 4401 /* 4402 * Device configured with multiple RX queues. 4403 */ 4404 txgbe_dev_mq_rx_configure(dev); 4405 4406 /* 4407 * Setup the Checksum Register. 4408 * Disable Full-Packet Checksum which is mutually exclusive with RSS. 4409 * Enable IP/L4 checksum computation by hardware if requested to do so. 4410 */ 4411 rxcsum = rd32(hw, TXGBE_PSRCTL); 4412 rxcsum |= TXGBE_PSRCTL_PCSD; 4413 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_CHECKSUM) 4414 rxcsum |= TXGBE_PSRCTL_L4CSUM; 4415 else 4416 rxcsum &= ~TXGBE_PSRCTL_L4CSUM; 4417 4418 wr32(hw, TXGBE_PSRCTL, rxcsum); 4419 4420 if (hw->mac.type == txgbe_mac_raptor) { 4421 rdrxctl = rd32(hw, TXGBE_SECRXCTL); 4422 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) 4423 rdrxctl &= ~TXGBE_SECRXCTL_CRCSTRIP; 4424 else 4425 rdrxctl |= TXGBE_SECRXCTL_CRCSTRIP; 4426 wr32(hw, TXGBE_SECRXCTL, rdrxctl); 4427 } 4428 4429 rc = txgbe_set_rsc(dev); 4430 if (rc) 4431 return rc; 4432 4433 txgbe_set_rx_function(dev); 4434 4435 return 0; 4436 } 4437 4438 /* 4439 * Initializes Transmit Unit. 4440 */ 4441 void __rte_cold 4442 txgbe_dev_tx_init(struct rte_eth_dev *dev) 4443 { 4444 struct txgbe_hw *hw; 4445 struct txgbe_tx_queue *txq; 4446 uint64_t bus_addr; 4447 uint16_t i; 4448 4449 PMD_INIT_FUNC_TRACE(); 4450 hw = TXGBE_DEV_HW(dev); 4451 4452 /* Setup the Base and Length of the Tx Descriptor Rings */ 4453 for (i = 0; i < dev->data->nb_tx_queues; i++) { 4454 txq = dev->data->tx_queues[i]; 4455 4456 bus_addr = txq->tx_ring_phys_addr; 4457 wr32(hw, TXGBE_TXBAL(txq->reg_idx), 4458 (uint32_t)(bus_addr & BIT_MASK32)); 4459 wr32(hw, TXGBE_TXBAH(txq->reg_idx), 4460 (uint32_t)(bus_addr >> 32)); 4461 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), TXGBE_TXCFG_BUFLEN_MASK, 4462 TXGBE_TXCFG_BUFLEN(txq->nb_tx_desc)); 4463 /* Setup the HW Tx Head and TX Tail descriptor pointers */ 4464 wr32(hw, TXGBE_TXRP(txq->reg_idx), 0); 4465 wr32(hw, TXGBE_TXWP(txq->reg_idx), 0); 4466 } 4467 4468 /* Device configured with multiple TX queues. */ 4469 txgbe_dev_mq_tx_configure(dev); 4470 } 4471 4472 /* 4473 * Set up link loopback mode Tx->Rx. 4474 */ 4475 static inline void __rte_cold 4476 txgbe_setup_loopback_link_raptor(struct txgbe_hw *hw) 4477 { 4478 PMD_INIT_FUNC_TRACE(); 4479 4480 wr32m(hw, TXGBE_MACRXCFG, TXGBE_MACRXCFG_LB, TXGBE_MACRXCFG_LB); 4481 4482 msec_delay(50); 4483 } 4484 4485 /* 4486 * Start Transmit and Receive Units. 4487 */ 4488 int __rte_cold 4489 txgbe_dev_rxtx_start(struct rte_eth_dev *dev) 4490 { 4491 struct txgbe_hw *hw; 4492 struct txgbe_tx_queue *txq; 4493 struct txgbe_rx_queue *rxq; 4494 uint32_t dmatxctl; 4495 uint32_t rxctrl; 4496 uint16_t i; 4497 int ret = 0; 4498 4499 PMD_INIT_FUNC_TRACE(); 4500 hw = TXGBE_DEV_HW(dev); 4501 4502 for (i = 0; i < dev->data->nb_tx_queues; i++) { 4503 txq = dev->data->tx_queues[i]; 4504 /* Setup Transmit Threshold Registers */ 4505 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), 4506 TXGBE_TXCFG_HTHRESH_MASK | 4507 TXGBE_TXCFG_WTHRESH_MASK, 4508 TXGBE_TXCFG_HTHRESH(txq->hthresh) | 4509 TXGBE_TXCFG_WTHRESH(txq->wthresh)); 4510 } 4511 4512 dmatxctl = rd32(hw, TXGBE_DMATXCTRL); 4513 dmatxctl |= TXGBE_DMATXCTRL_ENA; 4514 wr32(hw, TXGBE_DMATXCTRL, dmatxctl); 4515 4516 for (i = 0; i < dev->data->nb_tx_queues; i++) { 4517 txq = dev->data->tx_queues[i]; 4518 if (!txq->tx_deferred_start) { 4519 ret = txgbe_dev_tx_queue_start(dev, i); 4520 if (ret < 0) 4521 return ret; 4522 } 4523 } 4524 4525 for (i = 0; i < dev->data->nb_rx_queues; i++) { 4526 rxq = dev->data->rx_queues[i]; 4527 if (!rxq->rx_deferred_start) { 4528 ret = txgbe_dev_rx_queue_start(dev, i); 4529 if (ret < 0) 4530 return ret; 4531 } 4532 } 4533 4534 /* Enable Receive engine */ 4535 rxctrl = rd32(hw, TXGBE_PBRXCTL); 4536 rxctrl |= TXGBE_PBRXCTL_ENA; 4537 hw->mac.enable_rx_dma(hw, rxctrl); 4538 4539 /* If loopback mode is enabled, set up the link accordingly */ 4540 if (hw->mac.type == txgbe_mac_raptor && 4541 dev->data->dev_conf.lpbk_mode) 4542 txgbe_setup_loopback_link_raptor(hw); 4543 4544 #ifdef RTE_LIB_SECURITY 4545 if ((dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_SECURITY) || 4546 (dev->data->dev_conf.txmode.offloads & RTE_ETH_TX_OFFLOAD_SECURITY)) { 4547 ret = txgbe_crypto_enable_ipsec(dev); 4548 if (ret != 0) { 4549 PMD_DRV_LOG(ERR, 4550 "txgbe_crypto_enable_ipsec fails with %d.", 4551 ret); 4552 return ret; 4553 } 4554 } 4555 #endif 4556 4557 return 0; 4558 } 4559 4560 void 4561 txgbe_dev_save_rx_queue(struct txgbe_hw *hw, uint16_t rx_queue_id) 4562 { 4563 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8]; 4564 *(reg++) = rd32(hw, TXGBE_RXBAL(rx_queue_id)); 4565 *(reg++) = rd32(hw, TXGBE_RXBAH(rx_queue_id)); 4566 *(reg++) = rd32(hw, TXGBE_RXCFG(rx_queue_id)); 4567 } 4568 4569 void 4570 txgbe_dev_store_rx_queue(struct txgbe_hw *hw, uint16_t rx_queue_id) 4571 { 4572 u32 *reg = &hw->q_rx_regs[rx_queue_id * 8]; 4573 wr32(hw, TXGBE_RXBAL(rx_queue_id), *(reg++)); 4574 wr32(hw, TXGBE_RXBAH(rx_queue_id), *(reg++)); 4575 wr32(hw, TXGBE_RXCFG(rx_queue_id), *(reg++) & ~TXGBE_RXCFG_ENA); 4576 } 4577 4578 void 4579 txgbe_dev_save_tx_queue(struct txgbe_hw *hw, uint16_t tx_queue_id) 4580 { 4581 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8]; 4582 *(reg++) = rd32(hw, TXGBE_TXBAL(tx_queue_id)); 4583 *(reg++) = rd32(hw, TXGBE_TXBAH(tx_queue_id)); 4584 *(reg++) = rd32(hw, TXGBE_TXCFG(tx_queue_id)); 4585 } 4586 4587 void 4588 txgbe_dev_store_tx_queue(struct txgbe_hw *hw, uint16_t tx_queue_id) 4589 { 4590 u32 *reg = &hw->q_tx_regs[tx_queue_id * 8]; 4591 wr32(hw, TXGBE_TXBAL(tx_queue_id), *(reg++)); 4592 wr32(hw, TXGBE_TXBAH(tx_queue_id), *(reg++)); 4593 wr32(hw, TXGBE_TXCFG(tx_queue_id), *(reg++) & ~TXGBE_TXCFG_ENA); 4594 } 4595 4596 /* 4597 * Start Receive Units for specified queue. 4598 */ 4599 int __rte_cold 4600 txgbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) 4601 { 4602 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 4603 struct txgbe_rx_queue *rxq; 4604 uint32_t rxdctl; 4605 int poll_ms; 4606 4607 PMD_INIT_FUNC_TRACE(); 4608 4609 rxq = dev->data->rx_queues[rx_queue_id]; 4610 4611 /* Allocate buffers for descriptor rings */ 4612 if (txgbe_alloc_rx_queue_mbufs(rxq) != 0) { 4613 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d", 4614 rx_queue_id); 4615 return -1; 4616 } 4617 rxdctl = rd32(hw, TXGBE_RXCFG(rxq->reg_idx)); 4618 rxdctl |= TXGBE_RXCFG_ENA; 4619 wr32(hw, TXGBE_RXCFG(rxq->reg_idx), rxdctl); 4620 4621 /* Wait until RX Enable ready */ 4622 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS; 4623 do { 4624 rte_delay_ms(1); 4625 rxdctl = rd32(hw, TXGBE_RXCFG(rxq->reg_idx)); 4626 } while (--poll_ms && !(rxdctl & TXGBE_RXCFG_ENA)); 4627 if (!poll_ms) 4628 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id); 4629 rte_wmb(); 4630 wr32(hw, TXGBE_RXRP(rxq->reg_idx), 0); 4631 wr32(hw, TXGBE_RXWP(rxq->reg_idx), rxq->nb_rx_desc - 1); 4632 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 4633 4634 return 0; 4635 } 4636 4637 /* 4638 * Stop Receive Units for specified queue. 4639 */ 4640 int __rte_cold 4641 txgbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) 4642 { 4643 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 4644 struct txgbe_adapter *adapter = TXGBE_DEV_ADAPTER(dev); 4645 struct txgbe_rx_queue *rxq; 4646 uint32_t rxdctl; 4647 int poll_ms; 4648 4649 PMD_INIT_FUNC_TRACE(); 4650 4651 rxq = dev->data->rx_queues[rx_queue_id]; 4652 4653 txgbe_dev_save_rx_queue(hw, rxq->reg_idx); 4654 wr32m(hw, TXGBE_RXCFG(rxq->reg_idx), TXGBE_RXCFG_ENA, 0); 4655 4656 /* Wait until RX Enable bit clear */ 4657 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS; 4658 do { 4659 rte_delay_ms(1); 4660 rxdctl = rd32(hw, TXGBE_RXCFG(rxq->reg_idx)); 4661 } while (--poll_ms && (rxdctl & TXGBE_RXCFG_ENA)); 4662 if (!poll_ms) 4663 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id); 4664 4665 rte_delay_us(RTE_TXGBE_WAIT_100_US); 4666 txgbe_dev_store_rx_queue(hw, rxq->reg_idx); 4667 4668 txgbe_rx_queue_release_mbufs(rxq); 4669 txgbe_reset_rx_queue(adapter, rxq); 4670 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 4671 4672 return 0; 4673 } 4674 4675 /* 4676 * Start Transmit Units for specified queue. 4677 */ 4678 int __rte_cold 4679 txgbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) 4680 { 4681 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 4682 struct txgbe_tx_queue *txq; 4683 uint32_t txdctl; 4684 int poll_ms; 4685 4686 PMD_INIT_FUNC_TRACE(); 4687 4688 txq = dev->data->tx_queues[tx_queue_id]; 4689 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), TXGBE_TXCFG_ENA, TXGBE_TXCFG_ENA); 4690 4691 /* Wait until TX Enable ready */ 4692 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS; 4693 do { 4694 rte_delay_ms(1); 4695 txdctl = rd32(hw, TXGBE_TXCFG(txq->reg_idx)); 4696 } while (--poll_ms && !(txdctl & TXGBE_TXCFG_ENA)); 4697 if (!poll_ms) 4698 PMD_INIT_LOG(ERR, "Could not enable " 4699 "Tx Queue %d", tx_queue_id); 4700 4701 rte_wmb(); 4702 wr32(hw, TXGBE_TXWP(txq->reg_idx), txq->tx_tail); 4703 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 4704 4705 return 0; 4706 } 4707 4708 /* 4709 * Stop Transmit Units for specified queue. 4710 */ 4711 int __rte_cold 4712 txgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) 4713 { 4714 struct txgbe_hw *hw = TXGBE_DEV_HW(dev); 4715 struct txgbe_tx_queue *txq; 4716 uint32_t txdctl; 4717 uint32_t txtdh, txtdt; 4718 int poll_ms; 4719 4720 PMD_INIT_FUNC_TRACE(); 4721 4722 txq = dev->data->tx_queues[tx_queue_id]; 4723 4724 /* Wait until TX queue is empty */ 4725 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS; 4726 do { 4727 rte_delay_us(RTE_TXGBE_WAIT_100_US); 4728 txtdh = rd32(hw, TXGBE_TXRP(txq->reg_idx)); 4729 txtdt = rd32(hw, TXGBE_TXWP(txq->reg_idx)); 4730 } while (--poll_ms && (txtdh != txtdt)); 4731 if (!poll_ms) 4732 PMD_INIT_LOG(ERR, 4733 "Tx Queue %d is not empty when stopping.", 4734 tx_queue_id); 4735 4736 txgbe_dev_save_tx_queue(hw, txq->reg_idx); 4737 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), TXGBE_TXCFG_ENA, 0); 4738 4739 /* Wait until TX Enable bit clear */ 4740 poll_ms = RTE_TXGBE_REGISTER_POLL_WAIT_10_MS; 4741 do { 4742 rte_delay_ms(1); 4743 txdctl = rd32(hw, TXGBE_TXCFG(txq->reg_idx)); 4744 } while (--poll_ms && (txdctl & TXGBE_TXCFG_ENA)); 4745 if (!poll_ms) 4746 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d", 4747 tx_queue_id); 4748 4749 rte_delay_us(RTE_TXGBE_WAIT_100_US); 4750 txgbe_dev_store_tx_queue(hw, txq->reg_idx); 4751 4752 if (txq->ops != NULL) { 4753 txq->ops->release_mbufs(txq); 4754 txq->ops->reset(txq); 4755 } 4756 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 4757 4758 return 0; 4759 } 4760 4761 void 4762 txgbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 4763 struct rte_eth_rxq_info *qinfo) 4764 { 4765 struct txgbe_rx_queue *rxq; 4766 4767 rxq = dev->data->rx_queues[queue_id]; 4768 4769 qinfo->mp = rxq->mb_pool; 4770 qinfo->scattered_rx = dev->data->scattered_rx; 4771 qinfo->nb_desc = rxq->nb_rx_desc; 4772 4773 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh; 4774 qinfo->conf.rx_drop_en = rxq->drop_en; 4775 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start; 4776 qinfo->conf.offloads = rxq->offloads; 4777 } 4778 4779 void 4780 txgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 4781 struct rte_eth_txq_info *qinfo) 4782 { 4783 struct txgbe_tx_queue *txq; 4784 4785 txq = dev->data->tx_queues[queue_id]; 4786 4787 qinfo->nb_desc = txq->nb_tx_desc; 4788 4789 qinfo->conf.tx_thresh.pthresh = txq->pthresh; 4790 qinfo->conf.tx_thresh.hthresh = txq->hthresh; 4791 qinfo->conf.tx_thresh.wthresh = txq->wthresh; 4792 4793 qinfo->conf.tx_free_thresh = txq->tx_free_thresh; 4794 qinfo->conf.offloads = txq->offloads; 4795 qinfo->conf.tx_deferred_start = txq->tx_deferred_start; 4796 } 4797 4798 /* 4799 * [VF] Initializes Receive Unit. 4800 */ 4801 int __rte_cold 4802 txgbevf_dev_rx_init(struct rte_eth_dev *dev) 4803 { 4804 struct txgbe_hw *hw; 4805 struct txgbe_rx_queue *rxq; 4806 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 4807 uint64_t bus_addr; 4808 uint32_t srrctl, psrtype; 4809 uint16_t buf_size; 4810 uint16_t i; 4811 int ret; 4812 4813 PMD_INIT_FUNC_TRACE(); 4814 hw = TXGBE_DEV_HW(dev); 4815 4816 if (rte_is_power_of_2(dev->data->nb_rx_queues) == 0) { 4817 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, " 4818 "it should be power of 2"); 4819 return -1; 4820 } 4821 4822 if (dev->data->nb_rx_queues > hw->mac.max_rx_queues) { 4823 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, " 4824 "it should be equal to or less than %d", 4825 hw->mac.max_rx_queues); 4826 return -1; 4827 } 4828 4829 /* 4830 * When the VF driver issues a TXGBE_VF_RESET request, the PF driver 4831 * disables the VF receipt of packets if the PF MTU is > 1500. 4832 * This is done to deal with limitations that imposes 4833 * the PF and all VFs to share the same MTU. 4834 * Then, the PF driver enables again the VF receipt of packet when 4835 * the VF driver issues a TXGBE_VF_SET_LPE request. 4836 * In the meantime, the VF device cannot be used, even if the VF driver 4837 * and the Guest VM network stack are ready to accept packets with a 4838 * size up to the PF MTU. 4839 * As a work-around to this PF behaviour, force the call to 4840 * txgbevf_rlpml_set_vf even if jumbo frames are not used. This way, 4841 * VF packets received can work in all cases. 4842 */ 4843 if (txgbevf_rlpml_set_vf(hw, 4844 (uint16_t)dev->data->mtu + TXGBE_ETH_OVERHEAD)) { 4845 PMD_INIT_LOG(ERR, "Set max packet length to %d failed.", 4846 dev->data->mtu + TXGBE_ETH_OVERHEAD); 4847 return -EINVAL; 4848 } 4849 4850 /* 4851 * Assume no header split and no VLAN strip support 4852 * on any Rx queue first . 4853 */ 4854 rxmode->offloads &= ~RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 4855 4856 /* Set PSR type for VF RSS according to max Rx queue */ 4857 psrtype = TXGBE_VFPLCFG_PSRL4HDR | 4858 TXGBE_VFPLCFG_PSRL4HDR | 4859 TXGBE_VFPLCFG_PSRL2HDR | 4860 TXGBE_VFPLCFG_PSRTUNHDR | 4861 TXGBE_VFPLCFG_PSRTUNMAC; 4862 wr32(hw, TXGBE_VFPLCFG, TXGBE_VFPLCFG_PSR(psrtype)); 4863 4864 /* Setup RX queues */ 4865 for (i = 0; i < dev->data->nb_rx_queues; i++) { 4866 rxq = dev->data->rx_queues[i]; 4867 4868 /* Allocate buffers for descriptor rings */ 4869 ret = txgbe_alloc_rx_queue_mbufs(rxq); 4870 if (ret) 4871 return ret; 4872 4873 /* Setup the Base and Length of the Rx Descriptor Rings */ 4874 bus_addr = rxq->rx_ring_phys_addr; 4875 4876 wr32(hw, TXGBE_RXBAL(i), 4877 (uint32_t)(bus_addr & BIT_MASK32)); 4878 wr32(hw, TXGBE_RXBAH(i), 4879 (uint32_t)(bus_addr >> 32)); 4880 wr32(hw, TXGBE_RXRP(i), 0); 4881 wr32(hw, TXGBE_RXWP(i), 0); 4882 4883 /* Configure the RXCFG register */ 4884 srrctl = TXGBE_RXCFG_RNGLEN(rxq->nb_rx_desc); 4885 4886 /* Set if packets are dropped when no descriptors available */ 4887 if (rxq->drop_en) 4888 srrctl |= TXGBE_RXCFG_DROP; 4889 4890 /* 4891 * Configure the RX buffer size in the PKTLEN field of 4892 * the RXCFG register of the queue. 4893 * The value is in 1 KB resolution. Valid values can be from 4894 * 1 KB to 16 KB. 4895 */ 4896 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - 4897 RTE_PKTMBUF_HEADROOM); 4898 buf_size = ROUND_UP(buf_size, 1 << 10); 4899 srrctl |= TXGBE_RXCFG_PKTLEN(buf_size); 4900 4901 /* 4902 * VF modification to write virtual function RXCFG register 4903 */ 4904 wr32(hw, TXGBE_RXCFG(i), srrctl); 4905 4906 if (rxmode->offloads & RTE_ETH_RX_OFFLOAD_SCATTER || 4907 /* It adds dual VLAN length for supporting dual VLAN */ 4908 (dev->data->mtu + TXGBE_ETH_OVERHEAD + 4909 2 * RTE_VLAN_HLEN) > buf_size) { 4910 if (!dev->data->scattered_rx) 4911 PMD_INIT_LOG(DEBUG, "forcing scatter mode"); 4912 dev->data->scattered_rx = 1; 4913 } 4914 4915 if (rxq->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP) 4916 rxmode->offloads |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP; 4917 } 4918 4919 /* 4920 * Device configured with multiple RX queues. 4921 */ 4922 txgbe_dev_mq_rx_configure(dev); 4923 4924 txgbe_set_rx_function(dev); 4925 4926 return 0; 4927 } 4928 4929 /* 4930 * [VF] Initializes Transmit Unit. 4931 */ 4932 void __rte_cold 4933 txgbevf_dev_tx_init(struct rte_eth_dev *dev) 4934 { 4935 struct txgbe_hw *hw; 4936 struct txgbe_tx_queue *txq; 4937 uint64_t bus_addr; 4938 uint16_t i; 4939 4940 PMD_INIT_FUNC_TRACE(); 4941 hw = TXGBE_DEV_HW(dev); 4942 4943 /* Setup the Base and Length of the Tx Descriptor Rings */ 4944 for (i = 0; i < dev->data->nb_tx_queues; i++) { 4945 txq = dev->data->tx_queues[i]; 4946 bus_addr = txq->tx_ring_phys_addr; 4947 wr32(hw, TXGBE_TXBAL(i), 4948 (uint32_t)(bus_addr & BIT_MASK32)); 4949 wr32(hw, TXGBE_TXBAH(i), 4950 (uint32_t)(bus_addr >> 32)); 4951 wr32m(hw, TXGBE_TXCFG(i), TXGBE_TXCFG_BUFLEN_MASK, 4952 TXGBE_TXCFG_BUFLEN(txq->nb_tx_desc)); 4953 /* Setup the HW Tx Head and TX Tail descriptor pointers */ 4954 wr32(hw, TXGBE_TXRP(i), 0); 4955 wr32(hw, TXGBE_TXWP(i), 0); 4956 } 4957 } 4958 4959 /* 4960 * [VF] Start Transmit and Receive Units. 4961 */ 4962 void __rte_cold 4963 txgbevf_dev_rxtx_start(struct rte_eth_dev *dev) 4964 { 4965 struct txgbe_hw *hw; 4966 struct txgbe_tx_queue *txq; 4967 struct txgbe_rx_queue *rxq; 4968 uint32_t txdctl; 4969 uint32_t rxdctl; 4970 uint16_t i; 4971 int poll_ms; 4972 4973 PMD_INIT_FUNC_TRACE(); 4974 hw = TXGBE_DEV_HW(dev); 4975 4976 for (i = 0; i < dev->data->nb_tx_queues; i++) { 4977 txq = dev->data->tx_queues[i]; 4978 /* Setup Transmit Threshold Registers */ 4979 wr32m(hw, TXGBE_TXCFG(txq->reg_idx), 4980 TXGBE_TXCFG_HTHRESH_MASK | 4981 TXGBE_TXCFG_WTHRESH_MASK, 4982 TXGBE_TXCFG_HTHRESH(txq->hthresh) | 4983 TXGBE_TXCFG_WTHRESH(txq->wthresh)); 4984 } 4985 4986 for (i = 0; i < dev->data->nb_tx_queues; i++) { 4987 wr32m(hw, TXGBE_TXCFG(i), TXGBE_TXCFG_ENA, TXGBE_TXCFG_ENA); 4988 4989 poll_ms = 10; 4990 /* Wait until TX Enable ready */ 4991 do { 4992 rte_delay_ms(1); 4993 txdctl = rd32(hw, TXGBE_TXCFG(i)); 4994 } while (--poll_ms && !(txdctl & TXGBE_TXCFG_ENA)); 4995 if (!poll_ms) 4996 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", i); 4997 } 4998 for (i = 0; i < dev->data->nb_rx_queues; i++) { 4999 rxq = dev->data->rx_queues[i]; 5000 5001 wr32m(hw, TXGBE_RXCFG(i), TXGBE_RXCFG_ENA, TXGBE_RXCFG_ENA); 5002 5003 /* Wait until RX Enable ready */ 5004 poll_ms = 10; 5005 do { 5006 rte_delay_ms(1); 5007 rxdctl = rd32(hw, TXGBE_RXCFG(i)); 5008 } while (--poll_ms && !(rxdctl & TXGBE_RXCFG_ENA)); 5009 if (!poll_ms) 5010 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", i); 5011 rte_wmb(); 5012 wr32(hw, TXGBE_RXWP(i), rxq->nb_rx_desc - 1); 5013 } 5014 } 5015 5016 int 5017 txgbe_rss_conf_init(struct txgbe_rte_flow_rss_conf *out, 5018 const struct rte_flow_action_rss *in) 5019 { 5020 if (in->key_len > RTE_DIM(out->key) || 5021 in->queue_num > RTE_DIM(out->queue)) 5022 return -EINVAL; 5023 out->conf = (struct rte_flow_action_rss){ 5024 .func = in->func, 5025 .level = in->level, 5026 .types = in->types, 5027 .key_len = in->key_len, 5028 .queue_num = in->queue_num, 5029 .key = memcpy(out->key, in->key, in->key_len), 5030 .queue = memcpy(out->queue, in->queue, 5031 sizeof(*in->queue) * in->queue_num), 5032 }; 5033 return 0; 5034 } 5035 5036 int 5037 txgbe_action_rss_same(const struct rte_flow_action_rss *comp, 5038 const struct rte_flow_action_rss *with) 5039 { 5040 return (comp->func == with->func && 5041 comp->level == with->level && 5042 comp->types == with->types && 5043 comp->key_len == with->key_len && 5044 comp->queue_num == with->queue_num && 5045 !memcmp(comp->key, with->key, with->key_len) && 5046 !memcmp(comp->queue, with->queue, 5047 sizeof(*with->queue) * with->queue_num)); 5048 } 5049 5050 int 5051 txgbe_config_rss_filter(struct rte_eth_dev *dev, 5052 struct txgbe_rte_flow_rss_conf *conf, bool add) 5053 { 5054 struct txgbe_hw *hw; 5055 uint32_t reta; 5056 uint16_t i; 5057 uint16_t j; 5058 struct rte_eth_rss_conf rss_conf = { 5059 .rss_key = conf->conf.key_len ? 5060 (void *)(uintptr_t)conf->conf.key : NULL, 5061 .rss_key_len = conf->conf.key_len, 5062 .rss_hf = conf->conf.types, 5063 }; 5064 struct txgbe_filter_info *filter_info = TXGBE_DEV_FILTER(dev); 5065 5066 PMD_INIT_FUNC_TRACE(); 5067 hw = TXGBE_DEV_HW(dev); 5068 5069 if (!add) { 5070 if (txgbe_action_rss_same(&filter_info->rss_info.conf, 5071 &conf->conf)) { 5072 txgbe_rss_disable(dev); 5073 memset(&filter_info->rss_info, 0, 5074 sizeof(struct txgbe_rte_flow_rss_conf)); 5075 return 0; 5076 } 5077 return -EINVAL; 5078 } 5079 5080 if (filter_info->rss_info.conf.queue_num) 5081 return -EINVAL; 5082 /* Fill in redirection table 5083 * The byte-swap is needed because NIC registers are in 5084 * little-endian order. 5085 */ 5086 reta = 0; 5087 for (i = 0, j = 0; i < RTE_ETH_RSS_RETA_SIZE_128; i++, j++) { 5088 if (j == conf->conf.queue_num) 5089 j = 0; 5090 reta = (reta >> 8) | LS32(conf->conf.queue[j], 24, 0xFF); 5091 if ((i & 3) == 3) 5092 wr32at(hw, TXGBE_REG_RSSTBL, i >> 2, reta); 5093 } 5094 5095 /* Configure the RSS key and the RSS protocols used to compute 5096 * the RSS hash of input packets. 5097 */ 5098 if ((rss_conf.rss_hf & TXGBE_RSS_OFFLOAD_ALL) == 0) { 5099 txgbe_rss_disable(dev); 5100 return 0; 5101 } 5102 if (rss_conf.rss_key == NULL) 5103 rss_conf.rss_key = rss_intel_key; /* Default hash key */ 5104 txgbe_dev_rss_hash_update(dev, &rss_conf); 5105 5106 if (txgbe_rss_conf_init(&filter_info->rss_info, &conf->conf)) 5107 return -EINVAL; 5108 5109 return 0; 5110 } 5111 5112