1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation. 3 * Copyright 2014 6WIND S.A. 4 */ 5 6 #include <sys/queue.h> 7 8 #include <stdio.h> 9 #include <stdlib.h> 10 #include <string.h> 11 #include <errno.h> 12 #include <stdint.h> 13 #include <stdarg.h> 14 #include <unistd.h> 15 #include <inttypes.h> 16 17 #include <rte_byteorder.h> 18 #include <rte_common.h> 19 #include <rte_cycles.h> 20 #include <rte_log.h> 21 #include <rte_debug.h> 22 #include <rte_interrupts.h> 23 #include <rte_pci.h> 24 #include <rte_memory.h> 25 #include <rte_memzone.h> 26 #include <rte_launch.h> 27 #include <rte_eal.h> 28 #include <rte_per_lcore.h> 29 #include <rte_lcore.h> 30 #include <rte_atomic.h> 31 #include <rte_branch_prediction.h> 32 #include <rte_mempool.h> 33 #include <rte_malloc.h> 34 #include <rte_mbuf.h> 35 #include <rte_ether.h> 36 #include <ethdev_driver.h> 37 #include <rte_security_driver.h> 38 #include <rte_prefetch.h> 39 #include <rte_udp.h> 40 #include <rte_tcp.h> 41 #include <rte_sctp.h> 42 #include <rte_string_fns.h> 43 #include <rte_errno.h> 44 #include <rte_ip.h> 45 #include <rte_net.h> 46 #include <rte_vect.h> 47 48 #include "ixgbe_logs.h" 49 #include "base/ixgbe_api.h" 50 #include "base/ixgbe_vf.h" 51 #include "ixgbe_ethdev.h" 52 #include "base/ixgbe_dcb.h" 53 #include "base/ixgbe_common.h" 54 #include "ixgbe_rxtx.h" 55 56 #ifdef RTE_LIBRTE_IEEE1588 57 #define IXGBE_TX_IEEE1588_TMST PKT_TX_IEEE1588_TMST 58 #else 59 #define IXGBE_TX_IEEE1588_TMST 0 60 #endif 61 /* Bit Mask to indicate what bits required for building TX context */ 62 #define IXGBE_TX_OFFLOAD_MASK ( \ 63 PKT_TX_OUTER_IPV6 | \ 64 PKT_TX_OUTER_IPV4 | \ 65 PKT_TX_IPV6 | \ 66 PKT_TX_IPV4 | \ 67 PKT_TX_VLAN_PKT | \ 68 PKT_TX_IP_CKSUM | \ 69 PKT_TX_L4_MASK | \ 70 PKT_TX_TCP_SEG | \ 71 PKT_TX_MACSEC | \ 72 PKT_TX_OUTER_IP_CKSUM | \ 73 PKT_TX_SEC_OFFLOAD | \ 74 IXGBE_TX_IEEE1588_TMST) 75 76 #define IXGBE_TX_OFFLOAD_NOTSUP_MASK \ 77 (PKT_TX_OFFLOAD_MASK ^ IXGBE_TX_OFFLOAD_MASK) 78 79 #if 1 80 #define RTE_PMD_USE_PREFETCH 81 #endif 82 83 #ifdef RTE_PMD_USE_PREFETCH 84 /* 85 * Prefetch a cache line into all cache levels. 86 */ 87 #define rte_ixgbe_prefetch(p) rte_prefetch0(p) 88 #else 89 #define rte_ixgbe_prefetch(p) do {} while (0) 90 #endif 91 92 /********************************************************************* 93 * 94 * TX functions 95 * 96 **********************************************************************/ 97 98 /* 99 * Check for descriptors with their DD bit set and free mbufs. 100 * Return the total number of buffers freed. 101 */ 102 static __rte_always_inline int 103 ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq) 104 { 105 struct ixgbe_tx_entry *txep; 106 uint32_t status; 107 int i, nb_free = 0; 108 struct rte_mbuf *m, *free[RTE_IXGBE_TX_MAX_FREE_BUF_SZ]; 109 110 /* check DD bit on threshold descriptor */ 111 status = txq->tx_ring[txq->tx_next_dd].wb.status; 112 if (!(status & rte_cpu_to_le_32(IXGBE_ADVTXD_STAT_DD))) 113 return 0; 114 115 /* 116 * first buffer to free from S/W ring is at index 117 * tx_next_dd - (tx_rs_thresh-1) 118 */ 119 txep = &(txq->sw_ring[txq->tx_next_dd - (txq->tx_rs_thresh - 1)]); 120 121 for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) { 122 /* free buffers one at a time */ 123 m = rte_pktmbuf_prefree_seg(txep->mbuf); 124 txep->mbuf = NULL; 125 126 if (unlikely(m == NULL)) 127 continue; 128 129 if (nb_free >= RTE_IXGBE_TX_MAX_FREE_BUF_SZ || 130 (nb_free > 0 && m->pool != free[0]->pool)) { 131 rte_mempool_put_bulk(free[0]->pool, 132 (void **)free, nb_free); 133 nb_free = 0; 134 } 135 136 free[nb_free++] = m; 137 } 138 139 if (nb_free > 0) 140 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); 141 142 /* buffers were freed, update counters */ 143 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh); 144 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh); 145 if (txq->tx_next_dd >= txq->nb_tx_desc) 146 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); 147 148 return txq->tx_rs_thresh; 149 } 150 151 /* Populate 4 descriptors with data from 4 mbufs */ 152 static inline void 153 tx4(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts) 154 { 155 uint64_t buf_dma_addr; 156 uint32_t pkt_len; 157 int i; 158 159 for (i = 0; i < 4; ++i, ++txdp, ++pkts) { 160 buf_dma_addr = rte_mbuf_data_iova(*pkts); 161 pkt_len = (*pkts)->data_len; 162 163 /* write data to descriptor */ 164 txdp->read.buffer_addr = rte_cpu_to_le_64(buf_dma_addr); 165 166 txdp->read.cmd_type_len = 167 rte_cpu_to_le_32((uint32_t)DCMD_DTYP_FLAGS | pkt_len); 168 169 txdp->read.olinfo_status = 170 rte_cpu_to_le_32(pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT); 171 172 rte_prefetch0(&(*pkts)->pool); 173 } 174 } 175 176 /* Populate 1 descriptor with data from 1 mbuf */ 177 static inline void 178 tx1(volatile union ixgbe_adv_tx_desc *txdp, struct rte_mbuf **pkts) 179 { 180 uint64_t buf_dma_addr; 181 uint32_t pkt_len; 182 183 buf_dma_addr = rte_mbuf_data_iova(*pkts); 184 pkt_len = (*pkts)->data_len; 185 186 /* write data to descriptor */ 187 txdp->read.buffer_addr = rte_cpu_to_le_64(buf_dma_addr); 188 txdp->read.cmd_type_len = 189 rte_cpu_to_le_32((uint32_t)DCMD_DTYP_FLAGS | pkt_len); 190 txdp->read.olinfo_status = 191 rte_cpu_to_le_32(pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT); 192 rte_prefetch0(&(*pkts)->pool); 193 } 194 195 /* 196 * Fill H/W descriptor ring with mbuf data. 197 * Copy mbuf pointers to the S/W ring. 198 */ 199 static inline void 200 ixgbe_tx_fill_hw_ring(struct ixgbe_tx_queue *txq, struct rte_mbuf **pkts, 201 uint16_t nb_pkts) 202 { 203 volatile union ixgbe_adv_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]); 204 struct ixgbe_tx_entry *txep = &(txq->sw_ring[txq->tx_tail]); 205 const int N_PER_LOOP = 4; 206 const int N_PER_LOOP_MASK = N_PER_LOOP-1; 207 int mainpart, leftover; 208 int i, j; 209 210 /* 211 * Process most of the packets in chunks of N pkts. Any 212 * leftover packets will get processed one at a time. 213 */ 214 mainpart = (nb_pkts & ((uint32_t) ~N_PER_LOOP_MASK)); 215 leftover = (nb_pkts & ((uint32_t) N_PER_LOOP_MASK)); 216 for (i = 0; i < mainpart; i += N_PER_LOOP) { 217 /* Copy N mbuf pointers to the S/W ring */ 218 for (j = 0; j < N_PER_LOOP; ++j) { 219 (txep + i + j)->mbuf = *(pkts + i + j); 220 } 221 tx4(txdp + i, pkts + i); 222 } 223 224 if (unlikely(leftover > 0)) { 225 for (i = 0; i < leftover; ++i) { 226 (txep + mainpart + i)->mbuf = *(pkts + mainpart + i); 227 tx1(txdp + mainpart + i, pkts + mainpart + i); 228 } 229 } 230 } 231 232 static inline uint16_t 233 tx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 234 uint16_t nb_pkts) 235 { 236 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue; 237 volatile union ixgbe_adv_tx_desc *tx_r = txq->tx_ring; 238 uint16_t n = 0; 239 240 /* 241 * Begin scanning the H/W ring for done descriptors when the 242 * number of available descriptors drops below tx_free_thresh. For 243 * each done descriptor, free the associated buffer. 244 */ 245 if (txq->nb_tx_free < txq->tx_free_thresh) 246 ixgbe_tx_free_bufs(txq); 247 248 /* Only use descriptors that are available */ 249 nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts); 250 if (unlikely(nb_pkts == 0)) 251 return 0; 252 253 /* Use exactly nb_pkts descriptors */ 254 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); 255 256 /* 257 * At this point, we know there are enough descriptors in the 258 * ring to transmit all the packets. This assumes that each 259 * mbuf contains a single segment, and that no new offloads 260 * are expected, which would require a new context descriptor. 261 */ 262 263 /* 264 * See if we're going to wrap-around. If so, handle the top 265 * of the descriptor ring first, then do the bottom. If not, 266 * the processing looks just like the "bottom" part anyway... 267 */ 268 if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) { 269 n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail); 270 ixgbe_tx_fill_hw_ring(txq, tx_pkts, n); 271 272 /* 273 * We know that the last descriptor in the ring will need to 274 * have its RS bit set because tx_rs_thresh has to be 275 * a divisor of the ring size 276 */ 277 tx_r[txq->tx_next_rs].read.cmd_type_len |= 278 rte_cpu_to_le_32(IXGBE_ADVTXD_DCMD_RS); 279 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); 280 281 txq->tx_tail = 0; 282 } 283 284 /* Fill H/W descriptor ring with mbuf data */ 285 ixgbe_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n)); 286 txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n)); 287 288 /* 289 * Determine if RS bit should be set 290 * This is what we actually want: 291 * if ((txq->tx_tail - 1) >= txq->tx_next_rs) 292 * but instead of subtracting 1 and doing >=, we can just do 293 * greater than without subtracting. 294 */ 295 if (txq->tx_tail > txq->tx_next_rs) { 296 tx_r[txq->tx_next_rs].read.cmd_type_len |= 297 rte_cpu_to_le_32(IXGBE_ADVTXD_DCMD_RS); 298 txq->tx_next_rs = (uint16_t)(txq->tx_next_rs + 299 txq->tx_rs_thresh); 300 if (txq->tx_next_rs >= txq->nb_tx_desc) 301 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); 302 } 303 304 /* 305 * Check for wrap-around. This would only happen if we used 306 * up to the last descriptor in the ring, no more, no less. 307 */ 308 if (txq->tx_tail >= txq->nb_tx_desc) 309 txq->tx_tail = 0; 310 311 /* update tail pointer */ 312 rte_wmb(); 313 IXGBE_PCI_REG_WC_WRITE_RELAXED(txq->tdt_reg_addr, txq->tx_tail); 314 315 return nb_pkts; 316 } 317 318 uint16_t 319 ixgbe_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, 320 uint16_t nb_pkts) 321 { 322 uint16_t nb_tx; 323 324 /* Try to transmit at least chunks of TX_MAX_BURST pkts */ 325 if (likely(nb_pkts <= RTE_PMD_IXGBE_TX_MAX_BURST)) 326 return tx_xmit_pkts(tx_queue, tx_pkts, nb_pkts); 327 328 /* transmit more than the max burst, in chunks of TX_MAX_BURST */ 329 nb_tx = 0; 330 while (nb_pkts) { 331 uint16_t ret, n; 332 333 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_IXGBE_TX_MAX_BURST); 334 ret = tx_xmit_pkts(tx_queue, &(tx_pkts[nb_tx]), n); 335 nb_tx = (uint16_t)(nb_tx + ret); 336 nb_pkts = (uint16_t)(nb_pkts - ret); 337 if (ret < n) 338 break; 339 } 340 341 return nb_tx; 342 } 343 344 static uint16_t 345 ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, 346 uint16_t nb_pkts) 347 { 348 uint16_t nb_tx = 0; 349 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue; 350 351 while (nb_pkts) { 352 uint16_t ret, num; 353 354 num = (uint16_t)RTE_MIN(nb_pkts, txq->tx_rs_thresh); 355 ret = ixgbe_xmit_fixed_burst_vec(tx_queue, &tx_pkts[nb_tx], 356 num); 357 nb_tx += ret; 358 nb_pkts -= ret; 359 if (ret < num) 360 break; 361 } 362 363 return nb_tx; 364 } 365 366 static inline void 367 ixgbe_set_xmit_ctx(struct ixgbe_tx_queue *txq, 368 volatile struct ixgbe_adv_tx_context_desc *ctx_txd, 369 uint64_t ol_flags, union ixgbe_tx_offload tx_offload, 370 __rte_unused uint64_t *mdata) 371 { 372 uint32_t type_tucmd_mlhl; 373 uint32_t mss_l4len_idx = 0; 374 uint32_t ctx_idx; 375 uint32_t vlan_macip_lens; 376 union ixgbe_tx_offload tx_offload_mask; 377 uint32_t seqnum_seed = 0; 378 379 ctx_idx = txq->ctx_curr; 380 tx_offload_mask.data[0] = 0; 381 tx_offload_mask.data[1] = 0; 382 type_tucmd_mlhl = 0; 383 384 /* Specify which HW CTX to upload. */ 385 mss_l4len_idx |= (ctx_idx << IXGBE_ADVTXD_IDX_SHIFT); 386 387 if (ol_flags & PKT_TX_VLAN_PKT) { 388 tx_offload_mask.vlan_tci |= ~0; 389 } 390 391 /* check if TCP segmentation required for this packet */ 392 if (ol_flags & PKT_TX_TCP_SEG) { 393 /* implies IP cksum in IPv4 */ 394 if (ol_flags & PKT_TX_IP_CKSUM) 395 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4 | 396 IXGBE_ADVTXD_TUCMD_L4T_TCP | 397 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT; 398 else 399 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV6 | 400 IXGBE_ADVTXD_TUCMD_L4T_TCP | 401 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT; 402 403 tx_offload_mask.l2_len |= ~0; 404 tx_offload_mask.l3_len |= ~0; 405 tx_offload_mask.l4_len |= ~0; 406 tx_offload_mask.tso_segsz |= ~0; 407 mss_l4len_idx |= tx_offload.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT; 408 mss_l4len_idx |= tx_offload.l4_len << IXGBE_ADVTXD_L4LEN_SHIFT; 409 } else { /* no TSO, check if hardware checksum is needed */ 410 if (ol_flags & PKT_TX_IP_CKSUM) { 411 type_tucmd_mlhl = IXGBE_ADVTXD_TUCMD_IPV4; 412 tx_offload_mask.l2_len |= ~0; 413 tx_offload_mask.l3_len |= ~0; 414 } 415 416 switch (ol_flags & PKT_TX_L4_MASK) { 417 case PKT_TX_UDP_CKSUM: 418 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP | 419 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT; 420 mss_l4len_idx |= sizeof(struct rte_udp_hdr) 421 << IXGBE_ADVTXD_L4LEN_SHIFT; 422 tx_offload_mask.l2_len |= ~0; 423 tx_offload_mask.l3_len |= ~0; 424 break; 425 case PKT_TX_TCP_CKSUM: 426 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP | 427 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT; 428 mss_l4len_idx |= sizeof(struct rte_tcp_hdr) 429 << IXGBE_ADVTXD_L4LEN_SHIFT; 430 tx_offload_mask.l2_len |= ~0; 431 tx_offload_mask.l3_len |= ~0; 432 break; 433 case PKT_TX_SCTP_CKSUM: 434 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP | 435 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT; 436 mss_l4len_idx |= sizeof(struct rte_sctp_hdr) 437 << IXGBE_ADVTXD_L4LEN_SHIFT; 438 tx_offload_mask.l2_len |= ~0; 439 tx_offload_mask.l3_len |= ~0; 440 break; 441 default: 442 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_RSV | 443 IXGBE_ADVTXD_DTYP_CTXT | IXGBE_ADVTXD_DCMD_DEXT; 444 break; 445 } 446 } 447 448 if (ol_flags & PKT_TX_OUTER_IP_CKSUM) { 449 tx_offload_mask.outer_l2_len |= ~0; 450 tx_offload_mask.outer_l3_len |= ~0; 451 tx_offload_mask.l2_len |= ~0; 452 seqnum_seed |= tx_offload.outer_l3_len 453 << IXGBE_ADVTXD_OUTER_IPLEN; 454 seqnum_seed |= tx_offload.l2_len 455 << IXGBE_ADVTXD_TUNNEL_LEN; 456 } 457 #ifdef RTE_LIB_SECURITY 458 if (ol_flags & PKT_TX_SEC_OFFLOAD) { 459 union ixgbe_crypto_tx_desc_md *md = 460 (union ixgbe_crypto_tx_desc_md *)mdata; 461 seqnum_seed |= 462 (IXGBE_ADVTXD_IPSEC_SA_INDEX_MASK & md->sa_idx); 463 type_tucmd_mlhl |= md->enc ? 464 (IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP | 465 IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN) : 0; 466 type_tucmd_mlhl |= 467 (md->pad_len & IXGBE_ADVTXD_IPSEC_ESP_LEN_MASK); 468 tx_offload_mask.sa_idx |= ~0; 469 tx_offload_mask.sec_pad_len |= ~0; 470 } 471 #endif 472 473 txq->ctx_cache[ctx_idx].flags = ol_flags; 474 txq->ctx_cache[ctx_idx].tx_offload.data[0] = 475 tx_offload_mask.data[0] & tx_offload.data[0]; 476 txq->ctx_cache[ctx_idx].tx_offload.data[1] = 477 tx_offload_mask.data[1] & tx_offload.data[1]; 478 txq->ctx_cache[ctx_idx].tx_offload_mask = tx_offload_mask; 479 480 ctx_txd->type_tucmd_mlhl = rte_cpu_to_le_32(type_tucmd_mlhl); 481 vlan_macip_lens = tx_offload.l3_len; 482 if (ol_flags & PKT_TX_OUTER_IP_CKSUM) 483 vlan_macip_lens |= (tx_offload.outer_l2_len << 484 IXGBE_ADVTXD_MACLEN_SHIFT); 485 else 486 vlan_macip_lens |= (tx_offload.l2_len << 487 IXGBE_ADVTXD_MACLEN_SHIFT); 488 vlan_macip_lens |= ((uint32_t)tx_offload.vlan_tci << IXGBE_ADVTXD_VLAN_SHIFT); 489 ctx_txd->vlan_macip_lens = rte_cpu_to_le_32(vlan_macip_lens); 490 ctx_txd->mss_l4len_idx = rte_cpu_to_le_32(mss_l4len_idx); 491 ctx_txd->seqnum_seed = seqnum_seed; 492 } 493 494 /* 495 * Check which hardware context can be used. Use the existing match 496 * or create a new context descriptor. 497 */ 498 static inline uint32_t 499 what_advctx_update(struct ixgbe_tx_queue *txq, uint64_t flags, 500 union ixgbe_tx_offload tx_offload) 501 { 502 /* If match with the current used context */ 503 if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) && 504 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] == 505 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0] 506 & tx_offload.data[0])) && 507 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] == 508 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1] 509 & tx_offload.data[1])))) 510 return txq->ctx_curr; 511 512 /* What if match with the next context */ 513 txq->ctx_curr ^= 1; 514 if (likely((txq->ctx_cache[txq->ctx_curr].flags == flags) && 515 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[0] == 516 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[0] 517 & tx_offload.data[0])) && 518 (txq->ctx_cache[txq->ctx_curr].tx_offload.data[1] == 519 (txq->ctx_cache[txq->ctx_curr].tx_offload_mask.data[1] 520 & tx_offload.data[1])))) 521 return txq->ctx_curr; 522 523 /* Mismatch, use the previous context */ 524 return IXGBE_CTX_NUM; 525 } 526 527 static inline uint32_t 528 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags) 529 { 530 uint32_t tmp = 0; 531 532 if ((ol_flags & PKT_TX_L4_MASK) != PKT_TX_L4_NO_CKSUM) 533 tmp |= IXGBE_ADVTXD_POPTS_TXSM; 534 if (ol_flags & PKT_TX_IP_CKSUM) 535 tmp |= IXGBE_ADVTXD_POPTS_IXSM; 536 if (ol_flags & PKT_TX_TCP_SEG) 537 tmp |= IXGBE_ADVTXD_POPTS_TXSM; 538 return tmp; 539 } 540 541 static inline uint32_t 542 tx_desc_ol_flags_to_cmdtype(uint64_t ol_flags) 543 { 544 uint32_t cmdtype = 0; 545 546 if (ol_flags & PKT_TX_VLAN_PKT) 547 cmdtype |= IXGBE_ADVTXD_DCMD_VLE; 548 if (ol_flags & PKT_TX_TCP_SEG) 549 cmdtype |= IXGBE_ADVTXD_DCMD_TSE; 550 if (ol_flags & PKT_TX_OUTER_IP_CKSUM) 551 cmdtype |= (1 << IXGBE_ADVTXD_OUTERIPCS_SHIFT); 552 if (ol_flags & PKT_TX_MACSEC) 553 cmdtype |= IXGBE_ADVTXD_MAC_LINKSEC; 554 return cmdtype; 555 } 556 557 /* Default RS bit threshold values */ 558 #ifndef DEFAULT_TX_RS_THRESH 559 #define DEFAULT_TX_RS_THRESH 32 560 #endif 561 #ifndef DEFAULT_TX_FREE_THRESH 562 #define DEFAULT_TX_FREE_THRESH 32 563 #endif 564 565 /* Reset transmit descriptors after they have been used */ 566 static inline int 567 ixgbe_xmit_cleanup(struct ixgbe_tx_queue *txq) 568 { 569 struct ixgbe_tx_entry *sw_ring = txq->sw_ring; 570 volatile union ixgbe_adv_tx_desc *txr = txq->tx_ring; 571 uint16_t last_desc_cleaned = txq->last_desc_cleaned; 572 uint16_t nb_tx_desc = txq->nb_tx_desc; 573 uint16_t desc_to_clean_to; 574 uint16_t nb_tx_to_clean; 575 uint32_t status; 576 577 /* Determine the last descriptor needing to be cleaned */ 578 desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_rs_thresh); 579 if (desc_to_clean_to >= nb_tx_desc) 580 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc); 581 582 /* Check to make sure the last descriptor to clean is done */ 583 desc_to_clean_to = sw_ring[desc_to_clean_to].last_id; 584 status = txr[desc_to_clean_to].wb.status; 585 if (!(status & rte_cpu_to_le_32(IXGBE_TXD_STAT_DD))) { 586 PMD_TX_FREE_LOG(DEBUG, 587 "TX descriptor %4u is not done" 588 "(port=%d queue=%d)", 589 desc_to_clean_to, 590 txq->port_id, txq->queue_id); 591 /* Failed to clean any descriptors, better luck next time */ 592 return -(1); 593 } 594 595 /* Figure out how many descriptors will be cleaned */ 596 if (last_desc_cleaned > desc_to_clean_to) 597 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) + 598 desc_to_clean_to); 599 else 600 nb_tx_to_clean = (uint16_t)(desc_to_clean_to - 601 last_desc_cleaned); 602 603 PMD_TX_FREE_LOG(DEBUG, 604 "Cleaning %4u TX descriptors: %4u to %4u " 605 "(port=%d queue=%d)", 606 nb_tx_to_clean, last_desc_cleaned, desc_to_clean_to, 607 txq->port_id, txq->queue_id); 608 609 /* 610 * The last descriptor to clean is done, so that means all the 611 * descriptors from the last descriptor that was cleaned 612 * up to the last descriptor with the RS bit set 613 * are done. Only reset the threshold descriptor. 614 */ 615 txr[desc_to_clean_to].wb.status = 0; 616 617 /* Update the txq to reflect the last descriptor that was cleaned */ 618 txq->last_desc_cleaned = desc_to_clean_to; 619 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean); 620 621 /* No Error */ 622 return 0; 623 } 624 625 uint16_t 626 ixgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 627 uint16_t nb_pkts) 628 { 629 struct ixgbe_tx_queue *txq; 630 struct ixgbe_tx_entry *sw_ring; 631 struct ixgbe_tx_entry *txe, *txn; 632 volatile union ixgbe_adv_tx_desc *txr; 633 volatile union ixgbe_adv_tx_desc *txd, *txp; 634 struct rte_mbuf *tx_pkt; 635 struct rte_mbuf *m_seg; 636 uint64_t buf_dma_addr; 637 uint32_t olinfo_status; 638 uint32_t cmd_type_len; 639 uint32_t pkt_len; 640 uint16_t slen; 641 uint64_t ol_flags; 642 uint16_t tx_id; 643 uint16_t tx_last; 644 uint16_t nb_tx; 645 uint16_t nb_used; 646 uint64_t tx_ol_req; 647 uint32_t ctx = 0; 648 uint32_t new_ctx; 649 union ixgbe_tx_offload tx_offload; 650 #ifdef RTE_LIB_SECURITY 651 uint8_t use_ipsec; 652 #endif 653 654 tx_offload.data[0] = 0; 655 tx_offload.data[1] = 0; 656 txq = tx_queue; 657 sw_ring = txq->sw_ring; 658 txr = txq->tx_ring; 659 tx_id = txq->tx_tail; 660 txe = &sw_ring[tx_id]; 661 txp = NULL; 662 663 /* Determine if the descriptor ring needs to be cleaned. */ 664 if (txq->nb_tx_free < txq->tx_free_thresh) 665 ixgbe_xmit_cleanup(txq); 666 667 rte_prefetch0(&txe->mbuf->pool); 668 669 /* TX loop */ 670 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 671 new_ctx = 0; 672 tx_pkt = *tx_pkts++; 673 pkt_len = tx_pkt->pkt_len; 674 675 /* 676 * Determine how many (if any) context descriptors 677 * are needed for offload functionality. 678 */ 679 ol_flags = tx_pkt->ol_flags; 680 #ifdef RTE_LIB_SECURITY 681 use_ipsec = txq->using_ipsec && (ol_flags & PKT_TX_SEC_OFFLOAD); 682 #endif 683 684 /* If hardware offload required */ 685 tx_ol_req = ol_flags & IXGBE_TX_OFFLOAD_MASK; 686 if (tx_ol_req) { 687 tx_offload.l2_len = tx_pkt->l2_len; 688 tx_offload.l3_len = tx_pkt->l3_len; 689 tx_offload.l4_len = tx_pkt->l4_len; 690 tx_offload.vlan_tci = tx_pkt->vlan_tci; 691 tx_offload.tso_segsz = tx_pkt->tso_segsz; 692 tx_offload.outer_l2_len = tx_pkt->outer_l2_len; 693 tx_offload.outer_l3_len = tx_pkt->outer_l3_len; 694 #ifdef RTE_LIB_SECURITY 695 if (use_ipsec) { 696 union ixgbe_crypto_tx_desc_md *ipsec_mdata = 697 (union ixgbe_crypto_tx_desc_md *) 698 rte_security_dynfield(tx_pkt); 699 tx_offload.sa_idx = ipsec_mdata->sa_idx; 700 tx_offload.sec_pad_len = ipsec_mdata->pad_len; 701 } 702 #endif 703 704 /* If new context need be built or reuse the exist ctx. */ 705 ctx = what_advctx_update(txq, tx_ol_req, 706 tx_offload); 707 /* Only allocate context descriptor if required*/ 708 new_ctx = (ctx == IXGBE_CTX_NUM); 709 ctx = txq->ctx_curr; 710 } 711 712 /* 713 * Keep track of how many descriptors are used this loop 714 * This will always be the number of segments + the number of 715 * Context descriptors required to transmit the packet 716 */ 717 nb_used = (uint16_t)(tx_pkt->nb_segs + new_ctx); 718 719 if (txp != NULL && 720 nb_used + txq->nb_tx_used >= txq->tx_rs_thresh) 721 /* set RS on the previous packet in the burst */ 722 txp->read.cmd_type_len |= 723 rte_cpu_to_le_32(IXGBE_TXD_CMD_RS); 724 725 /* 726 * The number of descriptors that must be allocated for a 727 * packet is the number of segments of that packet, plus 1 728 * Context Descriptor for the hardware offload, if any. 729 * Determine the last TX descriptor to allocate in the TX ring 730 * for the packet, starting from the current position (tx_id) 731 * in the ring. 732 */ 733 tx_last = (uint16_t) (tx_id + nb_used - 1); 734 735 /* Circular ring */ 736 if (tx_last >= txq->nb_tx_desc) 737 tx_last = (uint16_t) (tx_last - txq->nb_tx_desc); 738 739 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u pktlen=%u" 740 " tx_first=%u tx_last=%u", 741 (unsigned) txq->port_id, 742 (unsigned) txq->queue_id, 743 (unsigned) pkt_len, 744 (unsigned) tx_id, 745 (unsigned) tx_last); 746 747 /* 748 * Make sure there are enough TX descriptors available to 749 * transmit the entire packet. 750 * nb_used better be less than or equal to txq->tx_rs_thresh 751 */ 752 if (nb_used > txq->nb_tx_free) { 753 PMD_TX_FREE_LOG(DEBUG, 754 "Not enough free TX descriptors " 755 "nb_used=%4u nb_free=%4u " 756 "(port=%d queue=%d)", 757 nb_used, txq->nb_tx_free, 758 txq->port_id, txq->queue_id); 759 760 if (ixgbe_xmit_cleanup(txq) != 0) { 761 /* Could not clean any descriptors */ 762 if (nb_tx == 0) 763 return 0; 764 goto end_of_tx; 765 } 766 767 /* nb_used better be <= txq->tx_rs_thresh */ 768 if (unlikely(nb_used > txq->tx_rs_thresh)) { 769 PMD_TX_FREE_LOG(DEBUG, 770 "The number of descriptors needed to " 771 "transmit the packet exceeds the " 772 "RS bit threshold. This will impact " 773 "performance." 774 "nb_used=%4u nb_free=%4u " 775 "tx_rs_thresh=%4u. " 776 "(port=%d queue=%d)", 777 nb_used, txq->nb_tx_free, 778 txq->tx_rs_thresh, 779 txq->port_id, txq->queue_id); 780 /* 781 * Loop here until there are enough TX 782 * descriptors or until the ring cannot be 783 * cleaned. 784 */ 785 while (nb_used > txq->nb_tx_free) { 786 if (ixgbe_xmit_cleanup(txq) != 0) { 787 /* 788 * Could not clean any 789 * descriptors 790 */ 791 if (nb_tx == 0) 792 return 0; 793 goto end_of_tx; 794 } 795 } 796 } 797 } 798 799 /* 800 * By now there are enough free TX descriptors to transmit 801 * the packet. 802 */ 803 804 /* 805 * Set common flags of all TX Data Descriptors. 806 * 807 * The following bits must be set in all Data Descriptors: 808 * - IXGBE_ADVTXD_DTYP_DATA 809 * - IXGBE_ADVTXD_DCMD_DEXT 810 * 811 * The following bits must be set in the first Data Descriptor 812 * and are ignored in the other ones: 813 * - IXGBE_ADVTXD_DCMD_IFCS 814 * - IXGBE_ADVTXD_MAC_1588 815 * - IXGBE_ADVTXD_DCMD_VLE 816 * 817 * The following bits must only be set in the last Data 818 * Descriptor: 819 * - IXGBE_TXD_CMD_EOP 820 * 821 * The following bits can be set in any Data Descriptor, but 822 * are only set in the last Data Descriptor: 823 * - IXGBE_TXD_CMD_RS 824 */ 825 cmd_type_len = IXGBE_ADVTXD_DTYP_DATA | 826 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT; 827 828 #ifdef RTE_LIBRTE_IEEE1588 829 if (ol_flags & PKT_TX_IEEE1588_TMST) 830 cmd_type_len |= IXGBE_ADVTXD_MAC_1588; 831 #endif 832 833 olinfo_status = 0; 834 if (tx_ol_req) { 835 836 if (ol_flags & PKT_TX_TCP_SEG) { 837 /* when TSO is on, paylen in descriptor is the 838 * not the packet len but the tcp payload len */ 839 pkt_len -= (tx_offload.l2_len + 840 tx_offload.l3_len + tx_offload.l4_len); 841 } 842 843 /* 844 * Setup the TX Advanced Context Descriptor if required 845 */ 846 if (new_ctx) { 847 volatile struct ixgbe_adv_tx_context_desc * 848 ctx_txd; 849 850 ctx_txd = (volatile struct 851 ixgbe_adv_tx_context_desc *) 852 &txr[tx_id]; 853 854 txn = &sw_ring[txe->next_id]; 855 rte_prefetch0(&txn->mbuf->pool); 856 857 if (txe->mbuf != NULL) { 858 rte_pktmbuf_free_seg(txe->mbuf); 859 txe->mbuf = NULL; 860 } 861 862 ixgbe_set_xmit_ctx(txq, ctx_txd, tx_ol_req, 863 tx_offload, 864 rte_security_dynfield(tx_pkt)); 865 866 txe->last_id = tx_last; 867 tx_id = txe->next_id; 868 txe = txn; 869 } 870 871 /* 872 * Setup the TX Advanced Data Descriptor, 873 * This path will go through 874 * whatever new/reuse the context descriptor 875 */ 876 cmd_type_len |= tx_desc_ol_flags_to_cmdtype(ol_flags); 877 olinfo_status |= tx_desc_cksum_flags_to_olinfo(ol_flags); 878 olinfo_status |= ctx << IXGBE_ADVTXD_IDX_SHIFT; 879 } 880 881 olinfo_status |= (pkt_len << IXGBE_ADVTXD_PAYLEN_SHIFT); 882 #ifdef RTE_LIB_SECURITY 883 if (use_ipsec) 884 olinfo_status |= IXGBE_ADVTXD_POPTS_IPSEC; 885 #endif 886 887 m_seg = tx_pkt; 888 do { 889 txd = &txr[tx_id]; 890 txn = &sw_ring[txe->next_id]; 891 rte_prefetch0(&txn->mbuf->pool); 892 893 if (txe->mbuf != NULL) 894 rte_pktmbuf_free_seg(txe->mbuf); 895 txe->mbuf = m_seg; 896 897 /* 898 * Set up Transmit Data Descriptor. 899 */ 900 slen = m_seg->data_len; 901 buf_dma_addr = rte_mbuf_data_iova(m_seg); 902 txd->read.buffer_addr = 903 rte_cpu_to_le_64(buf_dma_addr); 904 txd->read.cmd_type_len = 905 rte_cpu_to_le_32(cmd_type_len | slen); 906 txd->read.olinfo_status = 907 rte_cpu_to_le_32(olinfo_status); 908 txe->last_id = tx_last; 909 tx_id = txe->next_id; 910 txe = txn; 911 m_seg = m_seg->next; 912 } while (m_seg != NULL); 913 914 /* 915 * The last packet data descriptor needs End Of Packet (EOP) 916 */ 917 cmd_type_len |= IXGBE_TXD_CMD_EOP; 918 txq->nb_tx_used = (uint16_t)(txq->nb_tx_used + nb_used); 919 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used); 920 921 /* Set RS bit only on threshold packets' last descriptor */ 922 if (txq->nb_tx_used >= txq->tx_rs_thresh) { 923 PMD_TX_FREE_LOG(DEBUG, 924 "Setting RS bit on TXD id=" 925 "%4u (port=%d queue=%d)", 926 tx_last, txq->port_id, txq->queue_id); 927 928 cmd_type_len |= IXGBE_TXD_CMD_RS; 929 930 /* Update txq RS bit counters */ 931 txq->nb_tx_used = 0; 932 txp = NULL; 933 } else 934 txp = txd; 935 936 txd->read.cmd_type_len |= rte_cpu_to_le_32(cmd_type_len); 937 } 938 939 end_of_tx: 940 /* set RS on last packet in the burst */ 941 if (txp != NULL) 942 txp->read.cmd_type_len |= rte_cpu_to_le_32(IXGBE_TXD_CMD_RS); 943 944 rte_wmb(); 945 946 /* 947 * Set the Transmit Descriptor Tail (TDT) 948 */ 949 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", 950 (unsigned) txq->port_id, (unsigned) txq->queue_id, 951 (unsigned) tx_id, (unsigned) nb_tx); 952 IXGBE_PCI_REG_WC_WRITE_RELAXED(txq->tdt_reg_addr, tx_id); 953 txq->tx_tail = tx_id; 954 955 return nb_tx; 956 } 957 958 /********************************************************************* 959 * 960 * TX prep functions 961 * 962 **********************************************************************/ 963 uint16_t 964 ixgbe_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 965 { 966 int i, ret; 967 uint64_t ol_flags; 968 struct rte_mbuf *m; 969 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue; 970 971 for (i = 0; i < nb_pkts; i++) { 972 m = tx_pkts[i]; 973 ol_flags = m->ol_flags; 974 975 /** 976 * Check if packet meets requirements for number of segments 977 * 978 * NOTE: for ixgbe it's always (40 - WTHRESH) for both TSO and 979 * non-TSO 980 */ 981 982 if (m->nb_segs > IXGBE_TX_MAX_SEG - txq->wthresh) { 983 rte_errno = EINVAL; 984 return i; 985 } 986 987 if (ol_flags & IXGBE_TX_OFFLOAD_NOTSUP_MASK) { 988 rte_errno = ENOTSUP; 989 return i; 990 } 991 992 /* check the size of packet */ 993 if (m->pkt_len < IXGBE_TX_MIN_PKT_LEN) { 994 rte_errno = EINVAL; 995 return i; 996 } 997 998 #ifdef RTE_LIBRTE_ETHDEV_DEBUG 999 ret = rte_validate_tx_offload(m); 1000 if (ret != 0) { 1001 rte_errno = -ret; 1002 return i; 1003 } 1004 #endif 1005 ret = rte_net_intel_cksum_prepare(m); 1006 if (ret != 0) { 1007 rte_errno = -ret; 1008 return i; 1009 } 1010 } 1011 1012 return i; 1013 } 1014 1015 /********************************************************************* 1016 * 1017 * RX functions 1018 * 1019 **********************************************************************/ 1020 1021 #define IXGBE_PACKET_TYPE_ETHER 0X00 1022 #define IXGBE_PACKET_TYPE_IPV4 0X01 1023 #define IXGBE_PACKET_TYPE_IPV4_TCP 0X11 1024 #define IXGBE_PACKET_TYPE_IPV4_UDP 0X21 1025 #define IXGBE_PACKET_TYPE_IPV4_SCTP 0X41 1026 #define IXGBE_PACKET_TYPE_IPV4_EXT 0X03 1027 #define IXGBE_PACKET_TYPE_IPV4_EXT_TCP 0X13 1028 #define IXGBE_PACKET_TYPE_IPV4_EXT_UDP 0X23 1029 #define IXGBE_PACKET_TYPE_IPV4_EXT_SCTP 0X43 1030 #define IXGBE_PACKET_TYPE_IPV6 0X04 1031 #define IXGBE_PACKET_TYPE_IPV6_TCP 0X14 1032 #define IXGBE_PACKET_TYPE_IPV6_UDP 0X24 1033 #define IXGBE_PACKET_TYPE_IPV6_SCTP 0X44 1034 #define IXGBE_PACKET_TYPE_IPV6_EXT 0X0C 1035 #define IXGBE_PACKET_TYPE_IPV6_EXT_TCP 0X1C 1036 #define IXGBE_PACKET_TYPE_IPV6_EXT_UDP 0X2C 1037 #define IXGBE_PACKET_TYPE_IPV6_EXT_SCTP 0X4C 1038 #define IXGBE_PACKET_TYPE_IPV4_IPV6 0X05 1039 #define IXGBE_PACKET_TYPE_IPV4_IPV6_TCP 0X15 1040 #define IXGBE_PACKET_TYPE_IPV4_IPV6_UDP 0X25 1041 #define IXGBE_PACKET_TYPE_IPV4_IPV6_SCTP 0X45 1042 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6 0X07 1043 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_TCP 0X17 1044 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_UDP 0X27 1045 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_SCTP 0X47 1046 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT 0X0D 1047 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_TCP 0X1D 1048 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_UDP 0X2D 1049 #define IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_SCTP 0X4D 1050 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT 0X0F 1051 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_TCP 0X1F 1052 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_UDP 0X2F 1053 #define IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_SCTP 0X4F 1054 1055 #define IXGBE_PACKET_TYPE_NVGRE 0X00 1056 #define IXGBE_PACKET_TYPE_NVGRE_IPV4 0X01 1057 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_TCP 0X11 1058 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_UDP 0X21 1059 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_SCTP 0X41 1060 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT 0X03 1061 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_TCP 0X13 1062 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_UDP 0X23 1063 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_SCTP 0X43 1064 #define IXGBE_PACKET_TYPE_NVGRE_IPV6 0X04 1065 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_TCP 0X14 1066 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_UDP 0X24 1067 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_SCTP 0X44 1068 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT 0X0C 1069 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_TCP 0X1C 1070 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_UDP 0X2C 1071 #define IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_SCTP 0X4C 1072 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6 0X05 1073 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_TCP 0X15 1074 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_UDP 0X25 1075 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT 0X0D 1076 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_TCP 0X1D 1077 #define IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_UDP 0X2D 1078 1079 #define IXGBE_PACKET_TYPE_VXLAN 0X80 1080 #define IXGBE_PACKET_TYPE_VXLAN_IPV4 0X81 1081 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_TCP 0x91 1082 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_UDP 0xA1 1083 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_SCTP 0xC1 1084 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT 0x83 1085 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_TCP 0X93 1086 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_UDP 0XA3 1087 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_SCTP 0XC3 1088 #define IXGBE_PACKET_TYPE_VXLAN_IPV6 0X84 1089 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_TCP 0X94 1090 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_UDP 0XA4 1091 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_SCTP 0XC4 1092 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT 0X8C 1093 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_TCP 0X9C 1094 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_UDP 0XAC 1095 #define IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_SCTP 0XCC 1096 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6 0X85 1097 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_TCP 0X95 1098 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_UDP 0XA5 1099 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT 0X8D 1100 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP 0X9D 1101 #define IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP 0XAD 1102 1103 /** 1104 * Use 2 different table for normal packet and tunnel packet 1105 * to save the space. 1106 */ 1107 const uint32_t 1108 ptype_table[IXGBE_PACKET_TYPE_MAX] __rte_cache_aligned = { 1109 [IXGBE_PACKET_TYPE_ETHER] = RTE_PTYPE_L2_ETHER, 1110 [IXGBE_PACKET_TYPE_IPV4] = RTE_PTYPE_L2_ETHER | 1111 RTE_PTYPE_L3_IPV4, 1112 [IXGBE_PACKET_TYPE_IPV4_TCP] = RTE_PTYPE_L2_ETHER | 1113 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP, 1114 [IXGBE_PACKET_TYPE_IPV4_UDP] = RTE_PTYPE_L2_ETHER | 1115 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP, 1116 [IXGBE_PACKET_TYPE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER | 1117 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP, 1118 [IXGBE_PACKET_TYPE_IPV4_EXT] = RTE_PTYPE_L2_ETHER | 1119 RTE_PTYPE_L3_IPV4_EXT, 1120 [IXGBE_PACKET_TYPE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER | 1121 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP, 1122 [IXGBE_PACKET_TYPE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER | 1123 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP, 1124 [IXGBE_PACKET_TYPE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER | 1125 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_SCTP, 1126 [IXGBE_PACKET_TYPE_IPV6] = RTE_PTYPE_L2_ETHER | 1127 RTE_PTYPE_L3_IPV6, 1128 [IXGBE_PACKET_TYPE_IPV6_TCP] = RTE_PTYPE_L2_ETHER | 1129 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP, 1130 [IXGBE_PACKET_TYPE_IPV6_UDP] = RTE_PTYPE_L2_ETHER | 1131 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP, 1132 [IXGBE_PACKET_TYPE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | 1133 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP, 1134 [IXGBE_PACKET_TYPE_IPV6_EXT] = RTE_PTYPE_L2_ETHER | 1135 RTE_PTYPE_L3_IPV6_EXT, 1136 [IXGBE_PACKET_TYPE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | 1137 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP, 1138 [IXGBE_PACKET_TYPE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | 1139 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP, 1140 [IXGBE_PACKET_TYPE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | 1141 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_SCTP, 1142 [IXGBE_PACKET_TYPE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER | 1143 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 1144 RTE_PTYPE_INNER_L3_IPV6, 1145 [IXGBE_PACKET_TYPE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER | 1146 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 1147 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP, 1148 [IXGBE_PACKET_TYPE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER | 1149 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 1150 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP, 1151 [IXGBE_PACKET_TYPE_IPV4_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | 1152 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 1153 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP, 1154 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6] = RTE_PTYPE_L2_ETHER | 1155 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | 1156 RTE_PTYPE_INNER_L3_IPV6, 1157 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_TCP] = RTE_PTYPE_L2_ETHER | 1158 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | 1159 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP, 1160 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_UDP] = RTE_PTYPE_L2_ETHER | 1161 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | 1162 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP, 1163 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | 1164 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | 1165 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP, 1166 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER | 1167 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 1168 RTE_PTYPE_INNER_L3_IPV6_EXT, 1169 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | 1170 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 1171 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP, 1172 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | 1173 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 1174 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP, 1175 [IXGBE_PACKET_TYPE_IPV4_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | 1176 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 1177 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP, 1178 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT] = RTE_PTYPE_L2_ETHER | 1179 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | 1180 RTE_PTYPE_INNER_L3_IPV6_EXT, 1181 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | 1182 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | 1183 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP, 1184 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | 1185 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | 1186 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP, 1187 [IXGBE_PACKET_TYPE_IPV4_EXT_IPV6_EXT_SCTP] = 1188 RTE_PTYPE_L2_ETHER | 1189 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_IP | 1190 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP, 1191 }; 1192 1193 const uint32_t 1194 ptype_table_tn[IXGBE_PACKET_TYPE_TN_MAX] __rte_cache_aligned = { 1195 [IXGBE_PACKET_TYPE_NVGRE] = RTE_PTYPE_L2_ETHER | 1196 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1197 RTE_PTYPE_INNER_L2_ETHER, 1198 [IXGBE_PACKET_TYPE_NVGRE_IPV4] = RTE_PTYPE_L2_ETHER | 1199 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1200 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, 1201 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT] = RTE_PTYPE_L2_ETHER | 1202 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1203 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT, 1204 [IXGBE_PACKET_TYPE_NVGRE_IPV6] = RTE_PTYPE_L2_ETHER | 1205 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1206 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6, 1207 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER | 1208 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1209 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, 1210 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT] = RTE_PTYPE_L2_ETHER | 1211 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1212 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT, 1213 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER | 1214 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1215 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, 1216 [IXGBE_PACKET_TYPE_NVGRE_IPV4_TCP] = RTE_PTYPE_L2_ETHER | 1217 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1218 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 | 1219 RTE_PTYPE_INNER_L4_TCP, 1220 [IXGBE_PACKET_TYPE_NVGRE_IPV6_TCP] = RTE_PTYPE_L2_ETHER | 1221 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1222 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 | 1223 RTE_PTYPE_INNER_L4_TCP, 1224 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER | 1225 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1226 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, 1227 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | 1228 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1229 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT | 1230 RTE_PTYPE_INNER_L4_TCP, 1231 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_TCP] = 1232 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 1233 RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER | 1234 RTE_PTYPE_INNER_L3_IPV4, 1235 [IXGBE_PACKET_TYPE_NVGRE_IPV4_UDP] = RTE_PTYPE_L2_ETHER | 1236 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1237 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 | 1238 RTE_PTYPE_INNER_L4_UDP, 1239 [IXGBE_PACKET_TYPE_NVGRE_IPV6_UDP] = RTE_PTYPE_L2_ETHER | 1240 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1241 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 | 1242 RTE_PTYPE_INNER_L4_UDP, 1243 [IXGBE_PACKET_TYPE_NVGRE_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | 1244 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1245 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6 | 1246 RTE_PTYPE_INNER_L4_SCTP, 1247 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER | 1248 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1249 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, 1250 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | 1251 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1252 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT | 1253 RTE_PTYPE_INNER_L4_UDP, 1254 [IXGBE_PACKET_TYPE_NVGRE_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | 1255 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1256 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV6_EXT | 1257 RTE_PTYPE_INNER_L4_SCTP, 1258 [IXGBE_PACKET_TYPE_NVGRE_IPV4_IPV6_EXT_UDP] = 1259 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 1260 RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_INNER_L2_ETHER | 1261 RTE_PTYPE_INNER_L3_IPV4, 1262 [IXGBE_PACKET_TYPE_NVGRE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER | 1263 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1264 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4 | 1265 RTE_PTYPE_INNER_L4_SCTP, 1266 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER | 1267 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1268 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT | 1269 RTE_PTYPE_INNER_L4_SCTP, 1270 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER | 1271 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1272 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT | 1273 RTE_PTYPE_INNER_L4_TCP, 1274 [IXGBE_PACKET_TYPE_NVGRE_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER | 1275 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_TUNNEL_GRE | 1276 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4_EXT | 1277 RTE_PTYPE_INNER_L4_UDP, 1278 1279 [IXGBE_PACKET_TYPE_VXLAN] = RTE_PTYPE_L2_ETHER | 1280 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1281 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER, 1282 [IXGBE_PACKET_TYPE_VXLAN_IPV4] = RTE_PTYPE_L2_ETHER | 1283 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1284 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1285 RTE_PTYPE_INNER_L3_IPV4, 1286 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT] = RTE_PTYPE_L2_ETHER | 1287 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1288 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1289 RTE_PTYPE_INNER_L3_IPV4_EXT, 1290 [IXGBE_PACKET_TYPE_VXLAN_IPV6] = RTE_PTYPE_L2_ETHER | 1291 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1292 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1293 RTE_PTYPE_INNER_L3_IPV6, 1294 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6] = RTE_PTYPE_L2_ETHER | 1295 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1296 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1297 RTE_PTYPE_INNER_L3_IPV4, 1298 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT] = RTE_PTYPE_L2_ETHER | 1299 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1300 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1301 RTE_PTYPE_INNER_L3_IPV6_EXT, 1302 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER | 1303 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1304 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1305 RTE_PTYPE_INNER_L3_IPV4, 1306 [IXGBE_PACKET_TYPE_VXLAN_IPV4_TCP] = RTE_PTYPE_L2_ETHER | 1307 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1308 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1309 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_TCP, 1310 [IXGBE_PACKET_TYPE_VXLAN_IPV6_TCP] = RTE_PTYPE_L2_ETHER | 1311 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1312 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1313 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP, 1314 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER | 1315 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1316 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1317 RTE_PTYPE_INNER_L3_IPV4, 1318 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | 1319 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1320 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1321 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP, 1322 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_TCP] = 1323 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 1324 RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN | 1325 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, 1326 [IXGBE_PACKET_TYPE_VXLAN_IPV4_UDP] = RTE_PTYPE_L2_ETHER | 1327 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1328 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1329 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_UDP, 1330 [IXGBE_PACKET_TYPE_VXLAN_IPV6_UDP] = RTE_PTYPE_L2_ETHER | 1331 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1332 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1333 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP, 1334 [IXGBE_PACKET_TYPE_VXLAN_IPV6_SCTP] = RTE_PTYPE_L2_ETHER | 1335 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1336 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1337 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_SCTP, 1338 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER | 1339 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1340 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1341 RTE_PTYPE_INNER_L3_IPV4, 1342 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | 1343 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1344 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1345 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP, 1346 [IXGBE_PACKET_TYPE_VXLAN_IPV6_EXT_SCTP] = RTE_PTYPE_L2_ETHER | 1347 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1348 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1349 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_SCTP, 1350 [IXGBE_PACKET_TYPE_VXLAN_IPV4_IPV6_EXT_UDP] = 1351 RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | 1352 RTE_PTYPE_L4_UDP | RTE_PTYPE_TUNNEL_VXLAN | 1353 RTE_PTYPE_INNER_L2_ETHER | RTE_PTYPE_INNER_L3_IPV4, 1354 [IXGBE_PACKET_TYPE_VXLAN_IPV4_SCTP] = RTE_PTYPE_L2_ETHER | 1355 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1356 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1357 RTE_PTYPE_INNER_L3_IPV4 | RTE_PTYPE_INNER_L4_SCTP, 1358 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER | 1359 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1360 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1361 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_SCTP, 1362 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_TCP] = RTE_PTYPE_L2_ETHER | 1363 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1364 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1365 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_TCP, 1366 [IXGBE_PACKET_TYPE_VXLAN_IPV4_EXT_UDP] = RTE_PTYPE_L2_ETHER | 1367 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP | 1368 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_INNER_L2_ETHER | 1369 RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_UDP, 1370 }; 1371 1372 int 1373 ixgbe_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc) 1374 { 1375 volatile union ixgbe_adv_rx_desc *rxdp; 1376 struct ixgbe_rx_queue *rxq = rx_queue; 1377 uint16_t desc; 1378 1379 desc = rxq->rx_tail; 1380 rxdp = &rxq->rx_ring[desc]; 1381 /* watch for changes in status bit */ 1382 pmc->addr = &rxdp->wb.upper.status_error; 1383 1384 /* 1385 * we expect the DD bit to be set to 1 if this descriptor was already 1386 * written to. 1387 */ 1388 pmc->val = rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD); 1389 pmc->mask = rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD); 1390 1391 /* the registers are 32-bit */ 1392 pmc->size = sizeof(uint32_t); 1393 1394 return 0; 1395 } 1396 1397 /* @note: fix ixgbe_dev_supported_ptypes_get() if any change here. */ 1398 static inline uint32_t 1399 ixgbe_rxd_pkt_info_to_pkt_type(uint32_t pkt_info, uint16_t ptype_mask) 1400 { 1401 1402 if (unlikely(pkt_info & IXGBE_RXDADV_PKTTYPE_ETQF)) 1403 return RTE_PTYPE_UNKNOWN; 1404 1405 pkt_info = (pkt_info >> IXGBE_PACKET_TYPE_SHIFT) & ptype_mask; 1406 1407 /* For tunnel packet */ 1408 if (pkt_info & IXGBE_PACKET_TYPE_TUNNEL_BIT) { 1409 /* Remove the tunnel bit to save the space. */ 1410 pkt_info &= IXGBE_PACKET_TYPE_MASK_TUNNEL; 1411 return ptype_table_tn[pkt_info]; 1412 } 1413 1414 /** 1415 * For x550, if it's not tunnel, 1416 * tunnel type bit should be set to 0. 1417 * Reuse 82599's mask. 1418 */ 1419 pkt_info &= IXGBE_PACKET_TYPE_MASK_82599; 1420 1421 return ptype_table[pkt_info]; 1422 } 1423 1424 static inline uint64_t 1425 ixgbe_rxd_pkt_info_to_pkt_flags(uint16_t pkt_info) 1426 { 1427 static uint64_t ip_rss_types_map[16] __rte_cache_aligned = { 1428 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, 1429 0, PKT_RX_RSS_HASH, 0, PKT_RX_RSS_HASH, 1430 PKT_RX_RSS_HASH, 0, 0, 0, 1431 0, 0, 0, PKT_RX_FDIR, 1432 }; 1433 #ifdef RTE_LIBRTE_IEEE1588 1434 static uint64_t ip_pkt_etqf_map[8] = { 1435 0, 0, 0, PKT_RX_IEEE1588_PTP, 1436 0, 0, 0, 0, 1437 }; 1438 1439 if (likely(pkt_info & IXGBE_RXDADV_PKTTYPE_ETQF)) 1440 return ip_pkt_etqf_map[(pkt_info >> 4) & 0X07] | 1441 ip_rss_types_map[pkt_info & 0XF]; 1442 else 1443 return ip_rss_types_map[pkt_info & 0XF]; 1444 #else 1445 return ip_rss_types_map[pkt_info & 0XF]; 1446 #endif 1447 } 1448 1449 static inline uint64_t 1450 rx_desc_status_to_pkt_flags(uint32_t rx_status, uint64_t vlan_flags) 1451 { 1452 uint64_t pkt_flags; 1453 1454 /* 1455 * Check if VLAN present only. 1456 * Do not check whether L3/L4 rx checksum done by NIC or not, 1457 * That can be found from rte_eth_rxmode.offloads flag 1458 */ 1459 pkt_flags = (rx_status & IXGBE_RXD_STAT_VP) ? vlan_flags : 0; 1460 1461 #ifdef RTE_LIBRTE_IEEE1588 1462 if (rx_status & IXGBE_RXD_STAT_TMST) 1463 pkt_flags = pkt_flags | PKT_RX_IEEE1588_TMST; 1464 #endif 1465 return pkt_flags; 1466 } 1467 1468 static inline uint64_t 1469 rx_desc_error_to_pkt_flags(uint32_t rx_status, uint16_t pkt_info, 1470 uint8_t rx_udp_csum_zero_err) 1471 { 1472 uint64_t pkt_flags; 1473 1474 /* 1475 * Bit 31: IPE, IPv4 checksum error 1476 * Bit 30: L4I, L4I integrity error 1477 */ 1478 static uint64_t error_to_pkt_flags_map[4] = { 1479 PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD, 1480 PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD, 1481 PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD, 1482 PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD 1483 }; 1484 pkt_flags = error_to_pkt_flags_map[(rx_status >> 1485 IXGBE_RXDADV_ERR_CKSUM_BIT) & IXGBE_RXDADV_ERR_CKSUM_MSK]; 1486 1487 /* Mask out the bad UDP checksum error if the hardware has UDP zero 1488 * checksum error issue, so that the software application will then 1489 * have to recompute the checksum itself if needed. 1490 */ 1491 if ((rx_status & IXGBE_RXDADV_ERR_TCPE) && 1492 (pkt_info & IXGBE_RXDADV_PKTTYPE_UDP) && 1493 rx_udp_csum_zero_err) 1494 pkt_flags &= ~PKT_RX_L4_CKSUM_BAD; 1495 1496 if ((rx_status & IXGBE_RXD_STAT_OUTERIPCS) && 1497 (rx_status & IXGBE_RXDADV_ERR_OUTERIPER)) { 1498 pkt_flags |= PKT_RX_OUTER_IP_CKSUM_BAD; 1499 } 1500 1501 #ifdef RTE_LIB_SECURITY 1502 if (rx_status & IXGBE_RXD_STAT_SECP) { 1503 pkt_flags |= PKT_RX_SEC_OFFLOAD; 1504 if (rx_status & IXGBE_RXDADV_LNKSEC_ERROR_BAD_SIG) 1505 pkt_flags |= PKT_RX_SEC_OFFLOAD_FAILED; 1506 } 1507 #endif 1508 1509 return pkt_flags; 1510 } 1511 1512 /* 1513 * LOOK_AHEAD defines how many desc statuses to check beyond the 1514 * current descriptor. 1515 * It must be a pound define for optimal performance. 1516 * Do not change the value of LOOK_AHEAD, as the ixgbe_rx_scan_hw_ring 1517 * function only works with LOOK_AHEAD=8. 1518 */ 1519 #define LOOK_AHEAD 8 1520 #if (LOOK_AHEAD != 8) 1521 #error "PMD IXGBE: LOOK_AHEAD must be 8\n" 1522 #endif 1523 static inline int 1524 ixgbe_rx_scan_hw_ring(struct ixgbe_rx_queue *rxq) 1525 { 1526 volatile union ixgbe_adv_rx_desc *rxdp; 1527 struct ixgbe_rx_entry *rxep; 1528 struct rte_mbuf *mb; 1529 uint16_t pkt_len; 1530 uint64_t pkt_flags; 1531 int nb_dd; 1532 uint32_t s[LOOK_AHEAD]; 1533 uint32_t pkt_info[LOOK_AHEAD]; 1534 int i, j, nb_rx = 0; 1535 uint32_t status; 1536 uint64_t vlan_flags = rxq->vlan_flags; 1537 1538 /* get references to current descriptor and S/W ring entry */ 1539 rxdp = &rxq->rx_ring[rxq->rx_tail]; 1540 rxep = &rxq->sw_ring[rxq->rx_tail]; 1541 1542 status = rxdp->wb.upper.status_error; 1543 /* check to make sure there is at least 1 packet to receive */ 1544 if (!(status & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))) 1545 return 0; 1546 1547 /* 1548 * Scan LOOK_AHEAD descriptors at a time to determine which descriptors 1549 * reference packets that are ready to be received. 1550 */ 1551 for (i = 0; i < RTE_PMD_IXGBE_RX_MAX_BURST; 1552 i += LOOK_AHEAD, rxdp += LOOK_AHEAD, rxep += LOOK_AHEAD) { 1553 /* Read desc statuses backwards to avoid race condition */ 1554 for (j = 0; j < LOOK_AHEAD; j++) 1555 s[j] = rte_le_to_cpu_32(rxdp[j].wb.upper.status_error); 1556 1557 rte_smp_rmb(); 1558 1559 /* Compute how many status bits were set */ 1560 for (nb_dd = 0; nb_dd < LOOK_AHEAD && 1561 (s[nb_dd] & IXGBE_RXDADV_STAT_DD); nb_dd++) 1562 ; 1563 1564 for (j = 0; j < nb_dd; j++) 1565 pkt_info[j] = rte_le_to_cpu_32(rxdp[j].wb.lower. 1566 lo_dword.data); 1567 1568 nb_rx += nb_dd; 1569 1570 /* Translate descriptor info to mbuf format */ 1571 for (j = 0; j < nb_dd; ++j) { 1572 mb = rxep[j].mbuf; 1573 pkt_len = rte_le_to_cpu_16(rxdp[j].wb.upper.length) - 1574 rxq->crc_len; 1575 mb->data_len = pkt_len; 1576 mb->pkt_len = pkt_len; 1577 mb->vlan_tci = rte_le_to_cpu_16(rxdp[j].wb.upper.vlan); 1578 1579 /* convert descriptor fields to rte mbuf flags */ 1580 pkt_flags = rx_desc_status_to_pkt_flags(s[j], 1581 vlan_flags); 1582 pkt_flags |= rx_desc_error_to_pkt_flags(s[j], 1583 (uint16_t)pkt_info[j], 1584 rxq->rx_udp_csum_zero_err); 1585 pkt_flags |= ixgbe_rxd_pkt_info_to_pkt_flags 1586 ((uint16_t)pkt_info[j]); 1587 mb->ol_flags = pkt_flags; 1588 mb->packet_type = 1589 ixgbe_rxd_pkt_info_to_pkt_type 1590 (pkt_info[j], rxq->pkt_type_mask); 1591 1592 if (likely(pkt_flags & PKT_RX_RSS_HASH)) 1593 mb->hash.rss = rte_le_to_cpu_32( 1594 rxdp[j].wb.lower.hi_dword.rss); 1595 else if (pkt_flags & PKT_RX_FDIR) { 1596 mb->hash.fdir.hash = rte_le_to_cpu_16( 1597 rxdp[j].wb.lower.hi_dword.csum_ip.csum) & 1598 IXGBE_ATR_HASH_MASK; 1599 mb->hash.fdir.id = rte_le_to_cpu_16( 1600 rxdp[j].wb.lower.hi_dword.csum_ip.ip_id); 1601 } 1602 } 1603 1604 /* Move mbuf pointers from the S/W ring to the stage */ 1605 for (j = 0; j < LOOK_AHEAD; ++j) { 1606 rxq->rx_stage[i + j] = rxep[j].mbuf; 1607 } 1608 1609 /* stop if all requested packets could not be received */ 1610 if (nb_dd != LOOK_AHEAD) 1611 break; 1612 } 1613 1614 /* clear software ring entries so we can cleanup correctly */ 1615 for (i = 0; i < nb_rx; ++i) { 1616 rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL; 1617 } 1618 1619 1620 return nb_rx; 1621 } 1622 1623 static inline int 1624 ixgbe_rx_alloc_bufs(struct ixgbe_rx_queue *rxq, bool reset_mbuf) 1625 { 1626 volatile union ixgbe_adv_rx_desc *rxdp; 1627 struct ixgbe_rx_entry *rxep; 1628 struct rte_mbuf *mb; 1629 uint16_t alloc_idx; 1630 __le64 dma_addr; 1631 int diag, i; 1632 1633 /* allocate buffers in bulk directly into the S/W ring */ 1634 alloc_idx = rxq->rx_free_trigger - (rxq->rx_free_thresh - 1); 1635 rxep = &rxq->sw_ring[alloc_idx]; 1636 diag = rte_mempool_get_bulk(rxq->mb_pool, (void *)rxep, 1637 rxq->rx_free_thresh); 1638 if (unlikely(diag != 0)) 1639 return -ENOMEM; 1640 1641 rxdp = &rxq->rx_ring[alloc_idx]; 1642 for (i = 0; i < rxq->rx_free_thresh; ++i) { 1643 /* populate the static rte mbuf fields */ 1644 mb = rxep[i].mbuf; 1645 if (reset_mbuf) { 1646 mb->port = rxq->port_id; 1647 } 1648 1649 rte_mbuf_refcnt_set(mb, 1); 1650 mb->data_off = RTE_PKTMBUF_HEADROOM; 1651 1652 /* populate the descriptors */ 1653 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb)); 1654 rxdp[i].read.hdr_addr = 0; 1655 rxdp[i].read.pkt_addr = dma_addr; 1656 } 1657 1658 /* update state of internal queue structure */ 1659 rxq->rx_free_trigger = rxq->rx_free_trigger + rxq->rx_free_thresh; 1660 if (rxq->rx_free_trigger >= rxq->nb_rx_desc) 1661 rxq->rx_free_trigger = rxq->rx_free_thresh - 1; 1662 1663 /* no errors */ 1664 return 0; 1665 } 1666 1667 static inline uint16_t 1668 ixgbe_rx_fill_from_stage(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, 1669 uint16_t nb_pkts) 1670 { 1671 struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail]; 1672 int i; 1673 1674 /* how many packets are ready to return? */ 1675 nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail); 1676 1677 /* copy mbuf pointers to the application's packet list */ 1678 for (i = 0; i < nb_pkts; ++i) 1679 rx_pkts[i] = stage[i]; 1680 1681 /* update internal queue state */ 1682 rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts); 1683 rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts); 1684 1685 return nb_pkts; 1686 } 1687 1688 static inline uint16_t 1689 rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1690 uint16_t nb_pkts) 1691 { 1692 struct ixgbe_rx_queue *rxq = (struct ixgbe_rx_queue *)rx_queue; 1693 uint16_t nb_rx = 0; 1694 1695 /* Any previously recv'd pkts will be returned from the Rx stage */ 1696 if (rxq->rx_nb_avail) 1697 return ixgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 1698 1699 /* Scan the H/W ring for packets to receive */ 1700 nb_rx = (uint16_t)ixgbe_rx_scan_hw_ring(rxq); 1701 1702 /* update internal queue state */ 1703 rxq->rx_next_avail = 0; 1704 rxq->rx_nb_avail = nb_rx; 1705 rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx); 1706 1707 /* if required, allocate new buffers to replenish descriptors */ 1708 if (rxq->rx_tail > rxq->rx_free_trigger) { 1709 uint16_t cur_free_trigger = rxq->rx_free_trigger; 1710 1711 if (ixgbe_rx_alloc_bufs(rxq, true) != 0) { 1712 int i, j; 1713 1714 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " 1715 "queue_id=%u", (unsigned) rxq->port_id, 1716 (unsigned) rxq->queue_id); 1717 1718 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += 1719 rxq->rx_free_thresh; 1720 1721 /* 1722 * Need to rewind any previous receives if we cannot 1723 * allocate new buffers to replenish the old ones. 1724 */ 1725 rxq->rx_nb_avail = 0; 1726 rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx); 1727 for (i = 0, j = rxq->rx_tail; i < nb_rx; ++i, ++j) 1728 rxq->sw_ring[j].mbuf = rxq->rx_stage[i]; 1729 1730 return 0; 1731 } 1732 1733 /* update tail pointer */ 1734 rte_wmb(); 1735 IXGBE_PCI_REG_WC_WRITE_RELAXED(rxq->rdt_reg_addr, 1736 cur_free_trigger); 1737 } 1738 1739 if (rxq->rx_tail >= rxq->nb_rx_desc) 1740 rxq->rx_tail = 0; 1741 1742 /* received any packets this loop? */ 1743 if (rxq->rx_nb_avail) 1744 return ixgbe_rx_fill_from_stage(rxq, rx_pkts, nb_pkts); 1745 1746 return 0; 1747 } 1748 1749 /* split requests into chunks of size RTE_PMD_IXGBE_RX_MAX_BURST */ 1750 uint16_t 1751 ixgbe_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 1752 uint16_t nb_pkts) 1753 { 1754 uint16_t nb_rx; 1755 1756 if (unlikely(nb_pkts == 0)) 1757 return 0; 1758 1759 if (likely(nb_pkts <= RTE_PMD_IXGBE_RX_MAX_BURST)) 1760 return rx_recv_pkts(rx_queue, rx_pkts, nb_pkts); 1761 1762 /* request is relatively large, chunk it up */ 1763 nb_rx = 0; 1764 while (nb_pkts) { 1765 uint16_t ret, n; 1766 1767 n = (uint16_t)RTE_MIN(nb_pkts, RTE_PMD_IXGBE_RX_MAX_BURST); 1768 ret = rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n); 1769 nb_rx = (uint16_t)(nb_rx + ret); 1770 nb_pkts = (uint16_t)(nb_pkts - ret); 1771 if (ret < n) 1772 break; 1773 } 1774 1775 return nb_rx; 1776 } 1777 1778 uint16_t 1779 ixgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1780 uint16_t nb_pkts) 1781 { 1782 struct ixgbe_rx_queue *rxq; 1783 volatile union ixgbe_adv_rx_desc *rx_ring; 1784 volatile union ixgbe_adv_rx_desc *rxdp; 1785 struct ixgbe_rx_entry *sw_ring; 1786 struct ixgbe_rx_entry *rxe; 1787 struct rte_mbuf *rxm; 1788 struct rte_mbuf *nmb; 1789 union ixgbe_adv_rx_desc rxd; 1790 uint64_t dma_addr; 1791 uint32_t staterr; 1792 uint32_t pkt_info; 1793 uint16_t pkt_len; 1794 uint16_t rx_id; 1795 uint16_t nb_rx; 1796 uint16_t nb_hold; 1797 uint64_t pkt_flags; 1798 uint64_t vlan_flags; 1799 1800 nb_rx = 0; 1801 nb_hold = 0; 1802 rxq = rx_queue; 1803 rx_id = rxq->rx_tail; 1804 rx_ring = rxq->rx_ring; 1805 sw_ring = rxq->sw_ring; 1806 vlan_flags = rxq->vlan_flags; 1807 while (nb_rx < nb_pkts) { 1808 /* 1809 * The order of operations here is important as the DD status 1810 * bit must not be read after any other descriptor fields. 1811 * rx_ring and rxdp are pointing to volatile data so the order 1812 * of accesses cannot be reordered by the compiler. If they were 1813 * not volatile, they could be reordered which could lead to 1814 * using invalid descriptor fields when read from rxd. 1815 */ 1816 rxdp = &rx_ring[rx_id]; 1817 staterr = rxdp->wb.upper.status_error; 1818 if (!(staterr & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))) 1819 break; 1820 rxd = *rxdp; 1821 1822 /* 1823 * End of packet. 1824 * 1825 * If the IXGBE_RXDADV_STAT_EOP flag is not set, the RX packet 1826 * is likely to be invalid and to be dropped by the various 1827 * validation checks performed by the network stack. 1828 * 1829 * Allocate a new mbuf to replenish the RX ring descriptor. 1830 * If the allocation fails: 1831 * - arrange for that RX descriptor to be the first one 1832 * being parsed the next time the receive function is 1833 * invoked [on the same queue]. 1834 * 1835 * - Stop parsing the RX ring and return immediately. 1836 * 1837 * This policy do not drop the packet received in the RX 1838 * descriptor for which the allocation of a new mbuf failed. 1839 * Thus, it allows that packet to be later retrieved if 1840 * mbuf have been freed in the mean time. 1841 * As a side effect, holding RX descriptors instead of 1842 * systematically giving them back to the NIC may lead to 1843 * RX ring exhaustion situations. 1844 * However, the NIC can gracefully prevent such situations 1845 * to happen by sending specific "back-pressure" flow control 1846 * frames to its peer(s). 1847 */ 1848 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u " 1849 "ext_err_stat=0x%08x pkt_len=%u", 1850 (unsigned) rxq->port_id, (unsigned) rxq->queue_id, 1851 (unsigned) rx_id, (unsigned) staterr, 1852 (unsigned) rte_le_to_cpu_16(rxd.wb.upper.length)); 1853 1854 nmb = rte_mbuf_raw_alloc(rxq->mb_pool); 1855 if (nmb == NULL) { 1856 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u " 1857 "queue_id=%u", (unsigned) rxq->port_id, 1858 (unsigned) rxq->queue_id); 1859 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++; 1860 break; 1861 } 1862 1863 nb_hold++; 1864 rxe = &sw_ring[rx_id]; 1865 rx_id++; 1866 if (rx_id == rxq->nb_rx_desc) 1867 rx_id = 0; 1868 1869 /* Prefetch next mbuf while processing current one. */ 1870 rte_ixgbe_prefetch(sw_ring[rx_id].mbuf); 1871 1872 /* 1873 * When next RX descriptor is on a cache-line boundary, 1874 * prefetch the next 4 RX descriptors and the next 8 pointers 1875 * to mbufs. 1876 */ 1877 if ((rx_id & 0x3) == 0) { 1878 rte_ixgbe_prefetch(&rx_ring[rx_id]); 1879 rte_ixgbe_prefetch(&sw_ring[rx_id]); 1880 } 1881 1882 rxm = rxe->mbuf; 1883 rxe->mbuf = nmb; 1884 dma_addr = 1885 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 1886 rxdp->read.hdr_addr = 0; 1887 rxdp->read.pkt_addr = dma_addr; 1888 1889 /* 1890 * Initialize the returned mbuf. 1891 * 1) setup generic mbuf fields: 1892 * - number of segments, 1893 * - next segment, 1894 * - packet length, 1895 * - RX port identifier. 1896 * 2) integrate hardware offload data, if any: 1897 * - RSS flag & hash, 1898 * - IP checksum flag, 1899 * - VLAN TCI, if any, 1900 * - error flags. 1901 */ 1902 pkt_len = (uint16_t) (rte_le_to_cpu_16(rxd.wb.upper.length) - 1903 rxq->crc_len); 1904 rxm->data_off = RTE_PKTMBUF_HEADROOM; 1905 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off); 1906 rxm->nb_segs = 1; 1907 rxm->next = NULL; 1908 rxm->pkt_len = pkt_len; 1909 rxm->data_len = pkt_len; 1910 rxm->port = rxq->port_id; 1911 1912 pkt_info = rte_le_to_cpu_32(rxd.wb.lower.lo_dword.data); 1913 /* Only valid if PKT_RX_VLAN set in pkt_flags */ 1914 rxm->vlan_tci = rte_le_to_cpu_16(rxd.wb.upper.vlan); 1915 1916 pkt_flags = rx_desc_status_to_pkt_flags(staterr, vlan_flags); 1917 pkt_flags = pkt_flags | 1918 rx_desc_error_to_pkt_flags(staterr, (uint16_t)pkt_info, 1919 rxq->rx_udp_csum_zero_err); 1920 pkt_flags = pkt_flags | 1921 ixgbe_rxd_pkt_info_to_pkt_flags((uint16_t)pkt_info); 1922 rxm->ol_flags = pkt_flags; 1923 rxm->packet_type = 1924 ixgbe_rxd_pkt_info_to_pkt_type(pkt_info, 1925 rxq->pkt_type_mask); 1926 1927 if (likely(pkt_flags & PKT_RX_RSS_HASH)) 1928 rxm->hash.rss = rte_le_to_cpu_32( 1929 rxd.wb.lower.hi_dword.rss); 1930 else if (pkt_flags & PKT_RX_FDIR) { 1931 rxm->hash.fdir.hash = rte_le_to_cpu_16( 1932 rxd.wb.lower.hi_dword.csum_ip.csum) & 1933 IXGBE_ATR_HASH_MASK; 1934 rxm->hash.fdir.id = rte_le_to_cpu_16( 1935 rxd.wb.lower.hi_dword.csum_ip.ip_id); 1936 } 1937 /* 1938 * Store the mbuf address into the next entry of the array 1939 * of returned packets. 1940 */ 1941 rx_pkts[nb_rx++] = rxm; 1942 } 1943 rxq->rx_tail = rx_id; 1944 1945 /* 1946 * If the number of free RX descriptors is greater than the RX free 1947 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 1948 * register. 1949 * Update the RDT with the value of the last processed RX descriptor 1950 * minus 1, to guarantee that the RDT register is never equal to the 1951 * RDH register, which creates a "full" ring situtation from the 1952 * hardware point of view... 1953 */ 1954 nb_hold = (uint16_t) (nb_hold + rxq->nb_rx_hold); 1955 if (nb_hold > rxq->rx_free_thresh) { 1956 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u " 1957 "nb_hold=%u nb_rx=%u", 1958 (unsigned) rxq->port_id, (unsigned) rxq->queue_id, 1959 (unsigned) rx_id, (unsigned) nb_hold, 1960 (unsigned) nb_rx); 1961 rx_id = (uint16_t) ((rx_id == 0) ? 1962 (rxq->nb_rx_desc - 1) : (rx_id - 1)); 1963 IXGBE_PCI_REG_WC_WRITE(rxq->rdt_reg_addr, rx_id); 1964 nb_hold = 0; 1965 } 1966 rxq->nb_rx_hold = nb_hold; 1967 return nb_rx; 1968 } 1969 1970 /** 1971 * Detect an RSC descriptor. 1972 */ 1973 static inline uint32_t 1974 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx) 1975 { 1976 return (rte_le_to_cpu_32(rx->wb.lower.lo_dword.data) & 1977 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT; 1978 } 1979 1980 /** 1981 * ixgbe_fill_cluster_head_buf - fill the first mbuf of the returned packet 1982 * 1983 * Fill the following info in the HEAD buffer of the Rx cluster: 1984 * - RX port identifier 1985 * - hardware offload data, if any: 1986 * - RSS flag & hash 1987 * - IP checksum flag 1988 * - VLAN TCI, if any 1989 * - error flags 1990 * @head HEAD of the packet cluster 1991 * @desc HW descriptor to get data from 1992 * @rxq Pointer to the Rx queue 1993 */ 1994 static inline void 1995 ixgbe_fill_cluster_head_buf( 1996 struct rte_mbuf *head, 1997 union ixgbe_adv_rx_desc *desc, 1998 struct ixgbe_rx_queue *rxq, 1999 uint32_t staterr) 2000 { 2001 uint32_t pkt_info; 2002 uint64_t pkt_flags; 2003 2004 head->port = rxq->port_id; 2005 2006 /* The vlan_tci field is only valid when PKT_RX_VLAN is 2007 * set in the pkt_flags field. 2008 */ 2009 head->vlan_tci = rte_le_to_cpu_16(desc->wb.upper.vlan); 2010 pkt_info = rte_le_to_cpu_32(desc->wb.lower.lo_dword.data); 2011 pkt_flags = rx_desc_status_to_pkt_flags(staterr, rxq->vlan_flags); 2012 pkt_flags |= rx_desc_error_to_pkt_flags(staterr, (uint16_t)pkt_info, 2013 rxq->rx_udp_csum_zero_err); 2014 pkt_flags |= ixgbe_rxd_pkt_info_to_pkt_flags((uint16_t)pkt_info); 2015 head->ol_flags = pkt_flags; 2016 head->packet_type = 2017 ixgbe_rxd_pkt_info_to_pkt_type(pkt_info, rxq->pkt_type_mask); 2018 2019 if (likely(pkt_flags & PKT_RX_RSS_HASH)) 2020 head->hash.rss = rte_le_to_cpu_32(desc->wb.lower.hi_dword.rss); 2021 else if (pkt_flags & PKT_RX_FDIR) { 2022 head->hash.fdir.hash = 2023 rte_le_to_cpu_16(desc->wb.lower.hi_dword.csum_ip.csum) 2024 & IXGBE_ATR_HASH_MASK; 2025 head->hash.fdir.id = 2026 rte_le_to_cpu_16(desc->wb.lower.hi_dword.csum_ip.ip_id); 2027 } 2028 } 2029 2030 /** 2031 * ixgbe_recv_pkts_lro - receive handler for and LRO case. 2032 * 2033 * @rx_queue Rx queue handle 2034 * @rx_pkts table of received packets 2035 * @nb_pkts size of rx_pkts table 2036 * @bulk_alloc if TRUE bulk allocation is used for a HW ring refilling 2037 * 2038 * Handles the Rx HW ring completions when RSC feature is configured. Uses an 2039 * additional ring of ixgbe_rsc_entry's that will hold the relevant RSC info. 2040 * 2041 * We use the same logic as in Linux and in FreeBSD ixgbe drivers: 2042 * 1) When non-EOP RSC completion arrives: 2043 * a) Update the HEAD of the current RSC aggregation cluster with the new 2044 * segment's data length. 2045 * b) Set the "next" pointer of the current segment to point to the segment 2046 * at the NEXTP index. 2047 * c) Pass the HEAD of RSC aggregation cluster on to the next NEXTP entry 2048 * in the sw_rsc_ring. 2049 * 2) When EOP arrives we just update the cluster's total length and offload 2050 * flags and deliver the cluster up to the upper layers. In our case - put it 2051 * in the rx_pkts table. 2052 * 2053 * Returns the number of received packets/clusters (according to the "bulk 2054 * receive" interface). 2055 */ 2056 static inline uint16_t 2057 ixgbe_recv_pkts_lro(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts, 2058 bool bulk_alloc) 2059 { 2060 struct ixgbe_rx_queue *rxq = rx_queue; 2061 volatile union ixgbe_adv_rx_desc *rx_ring = rxq->rx_ring; 2062 struct ixgbe_rx_entry *sw_ring = rxq->sw_ring; 2063 struct ixgbe_scattered_rx_entry *sw_sc_ring = rxq->sw_sc_ring; 2064 uint16_t rx_id = rxq->rx_tail; 2065 uint16_t nb_rx = 0; 2066 uint16_t nb_hold = rxq->nb_rx_hold; 2067 uint16_t prev_id = rxq->rx_tail; 2068 2069 while (nb_rx < nb_pkts) { 2070 bool eop; 2071 struct ixgbe_rx_entry *rxe; 2072 struct ixgbe_scattered_rx_entry *sc_entry; 2073 struct ixgbe_scattered_rx_entry *next_sc_entry = NULL; 2074 struct ixgbe_rx_entry *next_rxe = NULL; 2075 struct rte_mbuf *first_seg; 2076 struct rte_mbuf *rxm; 2077 struct rte_mbuf *nmb = NULL; 2078 union ixgbe_adv_rx_desc rxd; 2079 uint16_t data_len; 2080 uint16_t next_id; 2081 volatile union ixgbe_adv_rx_desc *rxdp; 2082 uint32_t staterr; 2083 2084 next_desc: 2085 /* 2086 * The code in this whole file uses the volatile pointer to 2087 * ensure the read ordering of the status and the rest of the 2088 * descriptor fields (on the compiler level only!!!). This is so 2089 * UGLY - why not to just use the compiler barrier instead? DPDK 2090 * even has the rte_compiler_barrier() for that. 2091 * 2092 * But most importantly this is just wrong because this doesn't 2093 * ensure memory ordering in a general case at all. For 2094 * instance, DPDK is supposed to work on Power CPUs where 2095 * compiler barrier may just not be enough! 2096 * 2097 * I tried to write only this function properly to have a 2098 * starting point (as a part of an LRO/RSC series) but the 2099 * compiler cursed at me when I tried to cast away the 2100 * "volatile" from rx_ring (yes, it's volatile too!!!). So, I'm 2101 * keeping it the way it is for now. 2102 * 2103 * The code in this file is broken in so many other places and 2104 * will just not work on a big endian CPU anyway therefore the 2105 * lines below will have to be revisited together with the rest 2106 * of the ixgbe PMD. 2107 * 2108 * TODO: 2109 * - Get rid of "volatile" and let the compiler do its job. 2110 * - Use the proper memory barrier (rte_rmb()) to ensure the 2111 * memory ordering below. 2112 */ 2113 rxdp = &rx_ring[rx_id]; 2114 staterr = rte_le_to_cpu_32(rxdp->wb.upper.status_error); 2115 2116 if (!(staterr & IXGBE_RXDADV_STAT_DD)) 2117 break; 2118 2119 rxd = *rxdp; 2120 2121 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_id=%u " 2122 "staterr=0x%x data_len=%u", 2123 rxq->port_id, rxq->queue_id, rx_id, staterr, 2124 rte_le_to_cpu_16(rxd.wb.upper.length)); 2125 2126 if (!bulk_alloc) { 2127 nmb = rte_mbuf_raw_alloc(rxq->mb_pool); 2128 if (nmb == NULL) { 2129 PMD_RX_LOG(DEBUG, "RX mbuf alloc failed " 2130 "port_id=%u queue_id=%u", 2131 rxq->port_id, rxq->queue_id); 2132 2133 rte_eth_devices[rxq->port_id].data-> 2134 rx_mbuf_alloc_failed++; 2135 break; 2136 } 2137 } else if (nb_hold > rxq->rx_free_thresh) { 2138 uint16_t next_rdt = rxq->rx_free_trigger; 2139 2140 if (!ixgbe_rx_alloc_bufs(rxq, false)) { 2141 rte_wmb(); 2142 IXGBE_PCI_REG_WC_WRITE_RELAXED( 2143 rxq->rdt_reg_addr, 2144 next_rdt); 2145 nb_hold -= rxq->rx_free_thresh; 2146 } else { 2147 PMD_RX_LOG(DEBUG, "RX bulk alloc failed " 2148 "port_id=%u queue_id=%u", 2149 rxq->port_id, rxq->queue_id); 2150 2151 rte_eth_devices[rxq->port_id].data-> 2152 rx_mbuf_alloc_failed++; 2153 break; 2154 } 2155 } 2156 2157 nb_hold++; 2158 rxe = &sw_ring[rx_id]; 2159 eop = staterr & IXGBE_RXDADV_STAT_EOP; 2160 2161 next_id = rx_id + 1; 2162 if (next_id == rxq->nb_rx_desc) 2163 next_id = 0; 2164 2165 /* Prefetch next mbuf while processing current one. */ 2166 rte_ixgbe_prefetch(sw_ring[next_id].mbuf); 2167 2168 /* 2169 * When next RX descriptor is on a cache-line boundary, 2170 * prefetch the next 4 RX descriptors and the next 4 pointers 2171 * to mbufs. 2172 */ 2173 if ((next_id & 0x3) == 0) { 2174 rte_ixgbe_prefetch(&rx_ring[next_id]); 2175 rte_ixgbe_prefetch(&sw_ring[next_id]); 2176 } 2177 2178 rxm = rxe->mbuf; 2179 2180 if (!bulk_alloc) { 2181 __le64 dma = 2182 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 2183 /* 2184 * Update RX descriptor with the physical address of the 2185 * new data buffer of the new allocated mbuf. 2186 */ 2187 rxe->mbuf = nmb; 2188 2189 rxm->data_off = RTE_PKTMBUF_HEADROOM; 2190 rxdp->read.hdr_addr = 0; 2191 rxdp->read.pkt_addr = dma; 2192 } else 2193 rxe->mbuf = NULL; 2194 2195 /* 2196 * Set data length & data buffer address of mbuf. 2197 */ 2198 data_len = rte_le_to_cpu_16(rxd.wb.upper.length); 2199 rxm->data_len = data_len; 2200 2201 if (!eop) { 2202 uint16_t nextp_id; 2203 /* 2204 * Get next descriptor index: 2205 * - For RSC it's in the NEXTP field. 2206 * - For a scattered packet - it's just a following 2207 * descriptor. 2208 */ 2209 if (ixgbe_rsc_count(&rxd)) 2210 nextp_id = 2211 (staterr & IXGBE_RXDADV_NEXTP_MASK) >> 2212 IXGBE_RXDADV_NEXTP_SHIFT; 2213 else 2214 nextp_id = next_id; 2215 2216 next_sc_entry = &sw_sc_ring[nextp_id]; 2217 next_rxe = &sw_ring[nextp_id]; 2218 rte_ixgbe_prefetch(next_rxe); 2219 } 2220 2221 sc_entry = &sw_sc_ring[rx_id]; 2222 first_seg = sc_entry->fbuf; 2223 sc_entry->fbuf = NULL; 2224 2225 /* 2226 * If this is the first buffer of the received packet, 2227 * set the pointer to the first mbuf of the packet and 2228 * initialize its context. 2229 * Otherwise, update the total length and the number of segments 2230 * of the current scattered packet, and update the pointer to 2231 * the last mbuf of the current packet. 2232 */ 2233 if (first_seg == NULL) { 2234 first_seg = rxm; 2235 first_seg->pkt_len = data_len; 2236 first_seg->nb_segs = 1; 2237 } else { 2238 first_seg->pkt_len += data_len; 2239 first_seg->nb_segs++; 2240 } 2241 2242 prev_id = rx_id; 2243 rx_id = next_id; 2244 2245 /* 2246 * If this is not the last buffer of the received packet, update 2247 * the pointer to the first mbuf at the NEXTP entry in the 2248 * sw_sc_ring and continue to parse the RX ring. 2249 */ 2250 if (!eop && next_rxe) { 2251 rxm->next = next_rxe->mbuf; 2252 next_sc_entry->fbuf = first_seg; 2253 goto next_desc; 2254 } 2255 2256 /* Initialize the first mbuf of the returned packet */ 2257 ixgbe_fill_cluster_head_buf(first_seg, &rxd, rxq, staterr); 2258 2259 /* 2260 * Deal with the case, when HW CRC srip is disabled. 2261 * That can't happen when LRO is enabled, but still could 2262 * happen for scattered RX mode. 2263 */ 2264 first_seg->pkt_len -= rxq->crc_len; 2265 if (unlikely(rxm->data_len <= rxq->crc_len)) { 2266 struct rte_mbuf *lp; 2267 2268 for (lp = first_seg; lp->next != rxm; lp = lp->next) 2269 ; 2270 2271 first_seg->nb_segs--; 2272 lp->data_len -= rxq->crc_len - rxm->data_len; 2273 lp->next = NULL; 2274 rte_pktmbuf_free_seg(rxm); 2275 } else 2276 rxm->data_len -= rxq->crc_len; 2277 2278 /* Prefetch data of first segment, if configured to do so. */ 2279 rte_packet_prefetch((char *)first_seg->buf_addr + 2280 first_seg->data_off); 2281 2282 /* 2283 * Store the mbuf address into the next entry of the array 2284 * of returned packets. 2285 */ 2286 rx_pkts[nb_rx++] = first_seg; 2287 } 2288 2289 /* 2290 * Record index of the next RX descriptor to probe. 2291 */ 2292 rxq->rx_tail = rx_id; 2293 2294 /* 2295 * If the number of free RX descriptors is greater than the RX free 2296 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 2297 * register. 2298 * Update the RDT with the value of the last processed RX descriptor 2299 * minus 1, to guarantee that the RDT register is never equal to the 2300 * RDH register, which creates a "full" ring situtation from the 2301 * hardware point of view... 2302 */ 2303 if (!bulk_alloc && nb_hold > rxq->rx_free_thresh) { 2304 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u " 2305 "nb_hold=%u nb_rx=%u", 2306 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx); 2307 2308 rte_wmb(); 2309 IXGBE_PCI_REG_WC_WRITE_RELAXED(rxq->rdt_reg_addr, prev_id); 2310 nb_hold = 0; 2311 } 2312 2313 rxq->nb_rx_hold = nb_hold; 2314 return nb_rx; 2315 } 2316 2317 uint16_t 2318 ixgbe_recv_pkts_lro_single_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 2319 uint16_t nb_pkts) 2320 { 2321 return ixgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, false); 2322 } 2323 2324 uint16_t 2325 ixgbe_recv_pkts_lro_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts, 2326 uint16_t nb_pkts) 2327 { 2328 return ixgbe_recv_pkts_lro(rx_queue, rx_pkts, nb_pkts, true); 2329 } 2330 2331 /********************************************************************* 2332 * 2333 * Queue management functions 2334 * 2335 **********************************************************************/ 2336 2337 static void __rte_cold 2338 ixgbe_tx_queue_release_mbufs(struct ixgbe_tx_queue *txq) 2339 { 2340 unsigned i; 2341 2342 if (txq->sw_ring != NULL) { 2343 for (i = 0; i < txq->nb_tx_desc; i++) { 2344 if (txq->sw_ring[i].mbuf != NULL) { 2345 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 2346 txq->sw_ring[i].mbuf = NULL; 2347 } 2348 } 2349 } 2350 } 2351 2352 static int 2353 ixgbe_tx_done_cleanup_full(struct ixgbe_tx_queue *txq, uint32_t free_cnt) 2354 { 2355 struct ixgbe_tx_entry *swr_ring = txq->sw_ring; 2356 uint16_t i, tx_last, tx_id; 2357 uint16_t nb_tx_free_last; 2358 uint16_t nb_tx_to_clean; 2359 uint32_t pkt_cnt; 2360 2361 /* Start free mbuf from the next of tx_tail */ 2362 tx_last = txq->tx_tail; 2363 tx_id = swr_ring[tx_last].next_id; 2364 2365 if (txq->nb_tx_free == 0 && ixgbe_xmit_cleanup(txq)) 2366 return 0; 2367 2368 nb_tx_to_clean = txq->nb_tx_free; 2369 nb_tx_free_last = txq->nb_tx_free; 2370 if (!free_cnt) 2371 free_cnt = txq->nb_tx_desc; 2372 2373 /* Loop through swr_ring to count the amount of 2374 * freeable mubfs and packets. 2375 */ 2376 for (pkt_cnt = 0; pkt_cnt < free_cnt; ) { 2377 for (i = 0; i < nb_tx_to_clean && 2378 pkt_cnt < free_cnt && 2379 tx_id != tx_last; i++) { 2380 if (swr_ring[tx_id].mbuf != NULL) { 2381 rte_pktmbuf_free_seg(swr_ring[tx_id].mbuf); 2382 swr_ring[tx_id].mbuf = NULL; 2383 2384 /* 2385 * last segment in the packet, 2386 * increment packet count 2387 */ 2388 pkt_cnt += (swr_ring[tx_id].last_id == tx_id); 2389 } 2390 2391 tx_id = swr_ring[tx_id].next_id; 2392 } 2393 2394 if (txq->tx_rs_thresh > txq->nb_tx_desc - 2395 txq->nb_tx_free || tx_id == tx_last) 2396 break; 2397 2398 if (pkt_cnt < free_cnt) { 2399 if (ixgbe_xmit_cleanup(txq)) 2400 break; 2401 2402 nb_tx_to_clean = txq->nb_tx_free - nb_tx_free_last; 2403 nb_tx_free_last = txq->nb_tx_free; 2404 } 2405 } 2406 2407 return (int)pkt_cnt; 2408 } 2409 2410 static int 2411 ixgbe_tx_done_cleanup_simple(struct ixgbe_tx_queue *txq, 2412 uint32_t free_cnt) 2413 { 2414 int i, n, cnt; 2415 2416 if (free_cnt == 0 || free_cnt > txq->nb_tx_desc) 2417 free_cnt = txq->nb_tx_desc; 2418 2419 cnt = free_cnt - free_cnt % txq->tx_rs_thresh; 2420 2421 for (i = 0; i < cnt; i += n) { 2422 if (txq->nb_tx_desc - txq->nb_tx_free < txq->tx_rs_thresh) 2423 break; 2424 2425 n = ixgbe_tx_free_bufs(txq); 2426 2427 if (n == 0) 2428 break; 2429 } 2430 2431 return i; 2432 } 2433 2434 static int 2435 ixgbe_tx_done_cleanup_vec(struct ixgbe_tx_queue *txq __rte_unused, 2436 uint32_t free_cnt __rte_unused) 2437 { 2438 return -ENOTSUP; 2439 } 2440 2441 int 2442 ixgbe_dev_tx_done_cleanup(void *tx_queue, uint32_t free_cnt) 2443 { 2444 struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue; 2445 if (txq->offloads == 0 && 2446 #ifdef RTE_LIB_SECURITY 2447 !(txq->using_ipsec) && 2448 #endif 2449 txq->tx_rs_thresh >= RTE_PMD_IXGBE_TX_MAX_BURST) { 2450 if (txq->tx_rs_thresh <= RTE_IXGBE_TX_MAX_FREE_BUF_SZ && 2451 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128 && 2452 (rte_eal_process_type() != RTE_PROC_PRIMARY || 2453 txq->sw_ring_v != NULL)) { 2454 return ixgbe_tx_done_cleanup_vec(txq, free_cnt); 2455 } else { 2456 return ixgbe_tx_done_cleanup_simple(txq, free_cnt); 2457 } 2458 } 2459 2460 return ixgbe_tx_done_cleanup_full(txq, free_cnt); 2461 } 2462 2463 static void __rte_cold 2464 ixgbe_tx_free_swring(struct ixgbe_tx_queue *txq) 2465 { 2466 if (txq != NULL && 2467 txq->sw_ring != NULL) 2468 rte_free(txq->sw_ring); 2469 } 2470 2471 static void __rte_cold 2472 ixgbe_tx_queue_release(struct ixgbe_tx_queue *txq) 2473 { 2474 if (txq != NULL && txq->ops != NULL) { 2475 txq->ops->release_mbufs(txq); 2476 txq->ops->free_swring(txq); 2477 rte_free(txq); 2478 } 2479 } 2480 2481 void __rte_cold 2482 ixgbe_dev_tx_queue_release(void *txq) 2483 { 2484 ixgbe_tx_queue_release(txq); 2485 } 2486 2487 /* (Re)set dynamic ixgbe_tx_queue fields to defaults */ 2488 static void __rte_cold 2489 ixgbe_reset_tx_queue(struct ixgbe_tx_queue *txq) 2490 { 2491 static const union ixgbe_adv_tx_desc zeroed_desc = {{0}}; 2492 struct ixgbe_tx_entry *txe = txq->sw_ring; 2493 uint16_t prev, i; 2494 2495 /* Zero out HW ring memory */ 2496 for (i = 0; i < txq->nb_tx_desc; i++) { 2497 txq->tx_ring[i] = zeroed_desc; 2498 } 2499 2500 /* Initialize SW ring entries */ 2501 prev = (uint16_t) (txq->nb_tx_desc - 1); 2502 for (i = 0; i < txq->nb_tx_desc; i++) { 2503 volatile union ixgbe_adv_tx_desc *txd = &txq->tx_ring[i]; 2504 2505 txd->wb.status = rte_cpu_to_le_32(IXGBE_TXD_STAT_DD); 2506 txe[i].mbuf = NULL; 2507 txe[i].last_id = i; 2508 txe[prev].next_id = i; 2509 prev = i; 2510 } 2511 2512 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1); 2513 txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1); 2514 2515 txq->tx_tail = 0; 2516 txq->nb_tx_used = 0; 2517 /* 2518 * Always allow 1 descriptor to be un-allocated to avoid 2519 * a H/W race condition 2520 */ 2521 txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1); 2522 txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1); 2523 txq->ctx_curr = 0; 2524 memset((void *)&txq->ctx_cache, 0, 2525 IXGBE_CTX_NUM * sizeof(struct ixgbe_advctx_info)); 2526 } 2527 2528 static const struct ixgbe_txq_ops def_txq_ops = { 2529 .release_mbufs = ixgbe_tx_queue_release_mbufs, 2530 .free_swring = ixgbe_tx_free_swring, 2531 .reset = ixgbe_reset_tx_queue, 2532 }; 2533 2534 /* Takes an ethdev and a queue and sets up the tx function to be used based on 2535 * the queue parameters. Used in tx_queue_setup by primary process and then 2536 * in dev_init by secondary process when attaching to an existing ethdev. 2537 */ 2538 void __rte_cold 2539 ixgbe_set_tx_function(struct rte_eth_dev *dev, struct ixgbe_tx_queue *txq) 2540 { 2541 /* Use a simple Tx queue (no offloads, no multi segs) if possible */ 2542 if ((txq->offloads == 0) && 2543 #ifdef RTE_LIB_SECURITY 2544 !(txq->using_ipsec) && 2545 #endif 2546 (txq->tx_rs_thresh >= RTE_PMD_IXGBE_TX_MAX_BURST)) { 2547 PMD_INIT_LOG(DEBUG, "Using simple tx code path"); 2548 dev->tx_pkt_prepare = NULL; 2549 if (txq->tx_rs_thresh <= RTE_IXGBE_TX_MAX_FREE_BUF_SZ && 2550 rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128 && 2551 (rte_eal_process_type() != RTE_PROC_PRIMARY || 2552 ixgbe_txq_vec_setup(txq) == 0)) { 2553 PMD_INIT_LOG(DEBUG, "Vector tx enabled."); 2554 dev->tx_pkt_burst = ixgbe_xmit_pkts_vec; 2555 } else 2556 dev->tx_pkt_burst = ixgbe_xmit_pkts_simple; 2557 } else { 2558 PMD_INIT_LOG(DEBUG, "Using full-featured tx code path"); 2559 PMD_INIT_LOG(DEBUG, 2560 " - offloads = 0x%" PRIx64, 2561 txq->offloads); 2562 PMD_INIT_LOG(DEBUG, 2563 " - tx_rs_thresh = %lu " "[RTE_PMD_IXGBE_TX_MAX_BURST=%lu]", 2564 (unsigned long)txq->tx_rs_thresh, 2565 (unsigned long)RTE_PMD_IXGBE_TX_MAX_BURST); 2566 dev->tx_pkt_burst = ixgbe_xmit_pkts; 2567 dev->tx_pkt_prepare = ixgbe_prep_pkts; 2568 } 2569 } 2570 2571 uint64_t 2572 ixgbe_get_tx_queue_offloads(struct rte_eth_dev *dev) 2573 { 2574 RTE_SET_USED(dev); 2575 2576 return 0; 2577 } 2578 2579 uint64_t 2580 ixgbe_get_tx_port_offloads(struct rte_eth_dev *dev) 2581 { 2582 uint64_t tx_offload_capa; 2583 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2584 2585 tx_offload_capa = 2586 DEV_TX_OFFLOAD_VLAN_INSERT | 2587 DEV_TX_OFFLOAD_IPV4_CKSUM | 2588 DEV_TX_OFFLOAD_UDP_CKSUM | 2589 DEV_TX_OFFLOAD_TCP_CKSUM | 2590 DEV_TX_OFFLOAD_SCTP_CKSUM | 2591 DEV_TX_OFFLOAD_TCP_TSO | 2592 DEV_TX_OFFLOAD_MULTI_SEGS; 2593 2594 if (hw->mac.type == ixgbe_mac_82599EB || 2595 hw->mac.type == ixgbe_mac_X540) 2596 tx_offload_capa |= DEV_TX_OFFLOAD_MACSEC_INSERT; 2597 2598 if (hw->mac.type == ixgbe_mac_X550 || 2599 hw->mac.type == ixgbe_mac_X550EM_x || 2600 hw->mac.type == ixgbe_mac_X550EM_a) 2601 tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM; 2602 2603 #ifdef RTE_LIB_SECURITY 2604 if (dev->security_ctx) 2605 tx_offload_capa |= DEV_TX_OFFLOAD_SECURITY; 2606 #endif 2607 return tx_offload_capa; 2608 } 2609 2610 int __rte_cold 2611 ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev, 2612 uint16_t queue_idx, 2613 uint16_t nb_desc, 2614 unsigned int socket_id, 2615 const struct rte_eth_txconf *tx_conf) 2616 { 2617 const struct rte_memzone *tz; 2618 struct ixgbe_tx_queue *txq; 2619 struct ixgbe_hw *hw; 2620 uint16_t tx_rs_thresh, tx_free_thresh; 2621 uint64_t offloads; 2622 2623 PMD_INIT_FUNC_TRACE(); 2624 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2625 2626 offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads; 2627 2628 /* 2629 * Validate number of transmit descriptors. 2630 * It must not exceed hardware maximum, and must be multiple 2631 * of IXGBE_ALIGN. 2632 */ 2633 if (nb_desc % IXGBE_TXD_ALIGN != 0 || 2634 (nb_desc > IXGBE_MAX_RING_DESC) || 2635 (nb_desc < IXGBE_MIN_RING_DESC)) { 2636 return -EINVAL; 2637 } 2638 2639 /* 2640 * The following two parameters control the setting of the RS bit on 2641 * transmit descriptors. 2642 * TX descriptors will have their RS bit set after txq->tx_rs_thresh 2643 * descriptors have been used. 2644 * The TX descriptor ring will be cleaned after txq->tx_free_thresh 2645 * descriptors are used or if the number of descriptors required 2646 * to transmit a packet is greater than the number of free TX 2647 * descriptors. 2648 * The following constraints must be satisfied: 2649 * tx_rs_thresh must be greater than 0. 2650 * tx_rs_thresh must be less than the size of the ring minus 2. 2651 * tx_rs_thresh must be less than or equal to tx_free_thresh. 2652 * tx_rs_thresh must be a divisor of the ring size. 2653 * tx_free_thresh must be greater than 0. 2654 * tx_free_thresh must be less than the size of the ring minus 3. 2655 * tx_free_thresh + tx_rs_thresh must not exceed nb_desc. 2656 * One descriptor in the TX ring is used as a sentinel to avoid a 2657 * H/W race condition, hence the maximum threshold constraints. 2658 * When set to zero use default values. 2659 */ 2660 tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ? 2661 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH); 2662 /* force tx_rs_thresh to adapt an aggresive tx_free_thresh */ 2663 tx_rs_thresh = (DEFAULT_TX_RS_THRESH + tx_free_thresh > nb_desc) ? 2664 nb_desc - tx_free_thresh : DEFAULT_TX_RS_THRESH; 2665 if (tx_conf->tx_rs_thresh > 0) 2666 tx_rs_thresh = tx_conf->tx_rs_thresh; 2667 if (tx_rs_thresh + tx_free_thresh > nb_desc) { 2668 PMD_INIT_LOG(ERR, "tx_rs_thresh + tx_free_thresh must not " 2669 "exceed nb_desc. (tx_rs_thresh=%u " 2670 "tx_free_thresh=%u nb_desc=%u port = %d queue=%d)", 2671 (unsigned int)tx_rs_thresh, 2672 (unsigned int)tx_free_thresh, 2673 (unsigned int)nb_desc, 2674 (int)dev->data->port_id, 2675 (int)queue_idx); 2676 return -(EINVAL); 2677 } 2678 if (tx_rs_thresh >= (nb_desc - 2)) { 2679 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the number " 2680 "of TX descriptors minus 2. (tx_rs_thresh=%u " 2681 "port=%d queue=%d)", (unsigned int)tx_rs_thresh, 2682 (int)dev->data->port_id, (int)queue_idx); 2683 return -(EINVAL); 2684 } 2685 if (tx_rs_thresh > DEFAULT_TX_RS_THRESH) { 2686 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less or equal than %u. " 2687 "(tx_rs_thresh=%u port=%d queue=%d)", 2688 DEFAULT_TX_RS_THRESH, (unsigned int)tx_rs_thresh, 2689 (int)dev->data->port_id, (int)queue_idx); 2690 return -(EINVAL); 2691 } 2692 if (tx_free_thresh >= (nb_desc - 3)) { 2693 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the " 2694 "tx_free_thresh must be less than the number of " 2695 "TX descriptors minus 3. (tx_free_thresh=%u " 2696 "port=%d queue=%d)", 2697 (unsigned int)tx_free_thresh, 2698 (int)dev->data->port_id, (int)queue_idx); 2699 return -(EINVAL); 2700 } 2701 if (tx_rs_thresh > tx_free_thresh) { 2702 PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than or equal to " 2703 "tx_free_thresh. (tx_free_thresh=%u " 2704 "tx_rs_thresh=%u port=%d queue=%d)", 2705 (unsigned int)tx_free_thresh, 2706 (unsigned int)tx_rs_thresh, 2707 (int)dev->data->port_id, 2708 (int)queue_idx); 2709 return -(EINVAL); 2710 } 2711 if ((nb_desc % tx_rs_thresh) != 0) { 2712 PMD_INIT_LOG(ERR, "tx_rs_thresh must be a divisor of the " 2713 "number of TX descriptors. (tx_rs_thresh=%u " 2714 "port=%d queue=%d)", (unsigned int)tx_rs_thresh, 2715 (int)dev->data->port_id, (int)queue_idx); 2716 return -(EINVAL); 2717 } 2718 2719 /* 2720 * If rs_bit_thresh is greater than 1, then TX WTHRESH should be 2721 * set to 0. If WTHRESH is greater than zero, the RS bit is ignored 2722 * by the NIC and all descriptors are written back after the NIC 2723 * accumulates WTHRESH descriptors. 2724 */ 2725 if ((tx_rs_thresh > 1) && (tx_conf->tx_thresh.wthresh != 0)) { 2726 PMD_INIT_LOG(ERR, "TX WTHRESH must be set to 0 if " 2727 "tx_rs_thresh is greater than 1. (tx_rs_thresh=%u " 2728 "port=%d queue=%d)", (unsigned int)tx_rs_thresh, 2729 (int)dev->data->port_id, (int)queue_idx); 2730 return -(EINVAL); 2731 } 2732 2733 /* Free memory prior to re-allocation if needed... */ 2734 if (dev->data->tx_queues[queue_idx] != NULL) { 2735 ixgbe_tx_queue_release(dev->data->tx_queues[queue_idx]); 2736 dev->data->tx_queues[queue_idx] = NULL; 2737 } 2738 2739 /* First allocate the tx queue data structure */ 2740 txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct ixgbe_tx_queue), 2741 RTE_CACHE_LINE_SIZE, socket_id); 2742 if (txq == NULL) 2743 return -ENOMEM; 2744 2745 /* 2746 * Allocate TX ring hardware descriptors. A memzone large enough to 2747 * handle the maximum ring size is allocated in order to allow for 2748 * resizing in later calls to the queue setup function. 2749 */ 2750 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx, 2751 sizeof(union ixgbe_adv_tx_desc) * IXGBE_MAX_RING_DESC, 2752 IXGBE_ALIGN, socket_id); 2753 if (tz == NULL) { 2754 ixgbe_tx_queue_release(txq); 2755 return -ENOMEM; 2756 } 2757 2758 txq->nb_tx_desc = nb_desc; 2759 txq->tx_rs_thresh = tx_rs_thresh; 2760 txq->tx_free_thresh = tx_free_thresh; 2761 txq->pthresh = tx_conf->tx_thresh.pthresh; 2762 txq->hthresh = tx_conf->tx_thresh.hthresh; 2763 txq->wthresh = tx_conf->tx_thresh.wthresh; 2764 txq->queue_id = queue_idx; 2765 txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ? 2766 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx); 2767 txq->port_id = dev->data->port_id; 2768 txq->offloads = offloads; 2769 txq->ops = &def_txq_ops; 2770 txq->tx_deferred_start = tx_conf->tx_deferred_start; 2771 #ifdef RTE_LIB_SECURITY 2772 txq->using_ipsec = !!(dev->data->dev_conf.txmode.offloads & 2773 DEV_TX_OFFLOAD_SECURITY); 2774 #endif 2775 2776 /* 2777 * Modification to set VFTDT for virtual function if vf is detected 2778 */ 2779 if (hw->mac.type == ixgbe_mac_82599_vf || 2780 hw->mac.type == ixgbe_mac_X540_vf || 2781 hw->mac.type == ixgbe_mac_X550_vf || 2782 hw->mac.type == ixgbe_mac_X550EM_x_vf || 2783 hw->mac.type == ixgbe_mac_X550EM_a_vf) 2784 txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_VFTDT(queue_idx)); 2785 else 2786 txq->tdt_reg_addr = IXGBE_PCI_REG_ADDR(hw, IXGBE_TDT(txq->reg_idx)); 2787 2788 txq->tx_ring_phys_addr = tz->iova; 2789 txq->tx_ring = (union ixgbe_adv_tx_desc *) tz->addr; 2790 2791 /* Allocate software ring */ 2792 txq->sw_ring = rte_zmalloc_socket("txq->sw_ring", 2793 sizeof(struct ixgbe_tx_entry) * nb_desc, 2794 RTE_CACHE_LINE_SIZE, socket_id); 2795 if (txq->sw_ring == NULL) { 2796 ixgbe_tx_queue_release(txq); 2797 return -ENOMEM; 2798 } 2799 PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64, 2800 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); 2801 2802 /* set up vector or scalar TX function as appropriate */ 2803 ixgbe_set_tx_function(dev, txq); 2804 2805 txq->ops->reset(txq); 2806 2807 dev->data->tx_queues[queue_idx] = txq; 2808 2809 2810 return 0; 2811 } 2812 2813 /** 2814 * ixgbe_free_sc_cluster - free the not-yet-completed scattered cluster 2815 * 2816 * The "next" pointer of the last segment of (not-yet-completed) RSC clusters 2817 * in the sw_rsc_ring is not set to NULL but rather points to the next 2818 * mbuf of this RSC aggregation (that has not been completed yet and still 2819 * resides on the HW ring). So, instead of calling for rte_pktmbuf_free() we 2820 * will just free first "nb_segs" segments of the cluster explicitly by calling 2821 * an rte_pktmbuf_free_seg(). 2822 * 2823 * @m scattered cluster head 2824 */ 2825 static void __rte_cold 2826 ixgbe_free_sc_cluster(struct rte_mbuf *m) 2827 { 2828 uint16_t i, nb_segs = m->nb_segs; 2829 struct rte_mbuf *next_seg; 2830 2831 for (i = 0; i < nb_segs; i++) { 2832 next_seg = m->next; 2833 rte_pktmbuf_free_seg(m); 2834 m = next_seg; 2835 } 2836 } 2837 2838 static void __rte_cold 2839 ixgbe_rx_queue_release_mbufs(struct ixgbe_rx_queue *rxq) 2840 { 2841 unsigned i; 2842 2843 /* SSE Vector driver has a different way of releasing mbufs. */ 2844 if (rxq->rx_using_sse) { 2845 ixgbe_rx_queue_release_mbufs_vec(rxq); 2846 return; 2847 } 2848 2849 if (rxq->sw_ring != NULL) { 2850 for (i = 0; i < rxq->nb_rx_desc; i++) { 2851 if (rxq->sw_ring[i].mbuf != NULL) { 2852 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); 2853 rxq->sw_ring[i].mbuf = NULL; 2854 } 2855 } 2856 if (rxq->rx_nb_avail) { 2857 for (i = 0; i < rxq->rx_nb_avail; ++i) { 2858 struct rte_mbuf *mb; 2859 2860 mb = rxq->rx_stage[rxq->rx_next_avail + i]; 2861 rte_pktmbuf_free_seg(mb); 2862 } 2863 rxq->rx_nb_avail = 0; 2864 } 2865 } 2866 2867 if (rxq->sw_sc_ring) 2868 for (i = 0; i < rxq->nb_rx_desc; i++) 2869 if (rxq->sw_sc_ring[i].fbuf) { 2870 ixgbe_free_sc_cluster(rxq->sw_sc_ring[i].fbuf); 2871 rxq->sw_sc_ring[i].fbuf = NULL; 2872 } 2873 } 2874 2875 static void __rte_cold 2876 ixgbe_rx_queue_release(struct ixgbe_rx_queue *rxq) 2877 { 2878 if (rxq != NULL) { 2879 ixgbe_rx_queue_release_mbufs(rxq); 2880 rte_free(rxq->sw_ring); 2881 rte_free(rxq->sw_sc_ring); 2882 rte_free(rxq); 2883 } 2884 } 2885 2886 void __rte_cold 2887 ixgbe_dev_rx_queue_release(void *rxq) 2888 { 2889 ixgbe_rx_queue_release(rxq); 2890 } 2891 2892 /* 2893 * Check if Rx Burst Bulk Alloc function can be used. 2894 * Return 2895 * 0: the preconditions are satisfied and the bulk allocation function 2896 * can be used. 2897 * -EINVAL: the preconditions are NOT satisfied and the default Rx burst 2898 * function must be used. 2899 */ 2900 static inline int __rte_cold 2901 check_rx_burst_bulk_alloc_preconditions(struct ixgbe_rx_queue *rxq) 2902 { 2903 int ret = 0; 2904 2905 /* 2906 * Make sure the following pre-conditions are satisfied: 2907 * rxq->rx_free_thresh >= RTE_PMD_IXGBE_RX_MAX_BURST 2908 * rxq->rx_free_thresh < rxq->nb_rx_desc 2909 * (rxq->nb_rx_desc % rxq->rx_free_thresh) == 0 2910 * Scattered packets are not supported. This should be checked 2911 * outside of this function. 2912 */ 2913 if (!(rxq->rx_free_thresh >= RTE_PMD_IXGBE_RX_MAX_BURST)) { 2914 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 2915 "rxq->rx_free_thresh=%d, " 2916 "RTE_PMD_IXGBE_RX_MAX_BURST=%d", 2917 rxq->rx_free_thresh, RTE_PMD_IXGBE_RX_MAX_BURST); 2918 ret = -EINVAL; 2919 } else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) { 2920 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 2921 "rxq->rx_free_thresh=%d, " 2922 "rxq->nb_rx_desc=%d", 2923 rxq->rx_free_thresh, rxq->nb_rx_desc); 2924 ret = -EINVAL; 2925 } else if (!((rxq->nb_rx_desc % rxq->rx_free_thresh) == 0)) { 2926 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: " 2927 "rxq->nb_rx_desc=%d, " 2928 "rxq->rx_free_thresh=%d", 2929 rxq->nb_rx_desc, rxq->rx_free_thresh); 2930 ret = -EINVAL; 2931 } 2932 2933 return ret; 2934 } 2935 2936 /* Reset dynamic ixgbe_rx_queue fields back to defaults */ 2937 static void __rte_cold 2938 ixgbe_reset_rx_queue(struct ixgbe_adapter *adapter, struct ixgbe_rx_queue *rxq) 2939 { 2940 static const union ixgbe_adv_rx_desc zeroed_desc = {{0}}; 2941 unsigned i; 2942 uint16_t len = rxq->nb_rx_desc; 2943 2944 /* 2945 * By default, the Rx queue setup function allocates enough memory for 2946 * IXGBE_MAX_RING_DESC. The Rx Burst bulk allocation function requires 2947 * extra memory at the end of the descriptor ring to be zero'd out. 2948 */ 2949 if (adapter->rx_bulk_alloc_allowed) 2950 /* zero out extra memory */ 2951 len += RTE_PMD_IXGBE_RX_MAX_BURST; 2952 2953 /* 2954 * Zero out HW ring memory. Zero out extra memory at the end of 2955 * the H/W ring so look-ahead logic in Rx Burst bulk alloc function 2956 * reads extra memory as zeros. 2957 */ 2958 for (i = 0; i < len; i++) { 2959 rxq->rx_ring[i] = zeroed_desc; 2960 } 2961 2962 /* 2963 * initialize extra software ring entries. Space for these extra 2964 * entries is always allocated 2965 */ 2966 memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf)); 2967 for (i = rxq->nb_rx_desc; i < len; ++i) { 2968 rxq->sw_ring[i].mbuf = &rxq->fake_mbuf; 2969 } 2970 2971 rxq->rx_nb_avail = 0; 2972 rxq->rx_next_avail = 0; 2973 rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1); 2974 rxq->rx_tail = 0; 2975 rxq->nb_rx_hold = 0; 2976 rxq->pkt_first_seg = NULL; 2977 rxq->pkt_last_seg = NULL; 2978 2979 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) 2980 rxq->rxrearm_start = 0; 2981 rxq->rxrearm_nb = 0; 2982 #endif 2983 } 2984 2985 static int 2986 ixgbe_is_vf(struct rte_eth_dev *dev) 2987 { 2988 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2989 2990 switch (hw->mac.type) { 2991 case ixgbe_mac_82599_vf: 2992 case ixgbe_mac_X540_vf: 2993 case ixgbe_mac_X550_vf: 2994 case ixgbe_mac_X550EM_x_vf: 2995 case ixgbe_mac_X550EM_a_vf: 2996 return 1; 2997 default: 2998 return 0; 2999 } 3000 } 3001 3002 uint64_t 3003 ixgbe_get_rx_queue_offloads(struct rte_eth_dev *dev) 3004 { 3005 uint64_t offloads = 0; 3006 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3007 3008 if (hw->mac.type != ixgbe_mac_82598EB) 3009 offloads |= DEV_RX_OFFLOAD_VLAN_STRIP; 3010 3011 return offloads; 3012 } 3013 3014 uint64_t 3015 ixgbe_get_rx_port_offloads(struct rte_eth_dev *dev) 3016 { 3017 uint64_t offloads; 3018 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3019 3020 offloads = DEV_RX_OFFLOAD_IPV4_CKSUM | 3021 DEV_RX_OFFLOAD_UDP_CKSUM | 3022 DEV_RX_OFFLOAD_TCP_CKSUM | 3023 DEV_RX_OFFLOAD_KEEP_CRC | 3024 DEV_RX_OFFLOAD_JUMBO_FRAME | 3025 DEV_RX_OFFLOAD_VLAN_FILTER | 3026 DEV_RX_OFFLOAD_SCATTER | 3027 DEV_RX_OFFLOAD_RSS_HASH; 3028 3029 if (hw->mac.type == ixgbe_mac_82598EB) 3030 offloads |= DEV_RX_OFFLOAD_VLAN_STRIP; 3031 3032 if (ixgbe_is_vf(dev) == 0) 3033 offloads |= DEV_RX_OFFLOAD_VLAN_EXTEND; 3034 3035 /* 3036 * RSC is only supported by 82599 and x540 PF devices in a non-SR-IOV 3037 * mode. 3038 */ 3039 if ((hw->mac.type == ixgbe_mac_82599EB || 3040 hw->mac.type == ixgbe_mac_X540 || 3041 hw->mac.type == ixgbe_mac_X550) && 3042 !RTE_ETH_DEV_SRIOV(dev).active) 3043 offloads |= DEV_RX_OFFLOAD_TCP_LRO; 3044 3045 if (hw->mac.type == ixgbe_mac_82599EB || 3046 hw->mac.type == ixgbe_mac_X540) 3047 offloads |= DEV_RX_OFFLOAD_MACSEC_STRIP; 3048 3049 if (hw->mac.type == ixgbe_mac_X550 || 3050 hw->mac.type == ixgbe_mac_X550EM_x || 3051 hw->mac.type == ixgbe_mac_X550EM_a) 3052 offloads |= DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM; 3053 3054 #ifdef RTE_LIB_SECURITY 3055 if (dev->security_ctx) 3056 offloads |= DEV_RX_OFFLOAD_SECURITY; 3057 #endif 3058 3059 return offloads; 3060 } 3061 3062 int __rte_cold 3063 ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev, 3064 uint16_t queue_idx, 3065 uint16_t nb_desc, 3066 unsigned int socket_id, 3067 const struct rte_eth_rxconf *rx_conf, 3068 struct rte_mempool *mp) 3069 { 3070 const struct rte_memzone *rz; 3071 struct ixgbe_rx_queue *rxq; 3072 struct ixgbe_hw *hw; 3073 uint16_t len; 3074 struct ixgbe_adapter *adapter = dev->data->dev_private; 3075 uint64_t offloads; 3076 3077 PMD_INIT_FUNC_TRACE(); 3078 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3079 3080 offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads; 3081 3082 /* 3083 * Validate number of receive descriptors. 3084 * It must not exceed hardware maximum, and must be multiple 3085 * of IXGBE_ALIGN. 3086 */ 3087 if (nb_desc % IXGBE_RXD_ALIGN != 0 || 3088 (nb_desc > IXGBE_MAX_RING_DESC) || 3089 (nb_desc < IXGBE_MIN_RING_DESC)) { 3090 return -EINVAL; 3091 } 3092 3093 /* Free memory prior to re-allocation if needed... */ 3094 if (dev->data->rx_queues[queue_idx] != NULL) { 3095 ixgbe_rx_queue_release(dev->data->rx_queues[queue_idx]); 3096 dev->data->rx_queues[queue_idx] = NULL; 3097 } 3098 3099 /* First allocate the rx queue data structure */ 3100 rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct ixgbe_rx_queue), 3101 RTE_CACHE_LINE_SIZE, socket_id); 3102 if (rxq == NULL) 3103 return -ENOMEM; 3104 rxq->mb_pool = mp; 3105 rxq->nb_rx_desc = nb_desc; 3106 rxq->rx_free_thresh = rx_conf->rx_free_thresh; 3107 rxq->queue_id = queue_idx; 3108 rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ? 3109 queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx); 3110 rxq->port_id = dev->data->port_id; 3111 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) 3112 rxq->crc_len = RTE_ETHER_CRC_LEN; 3113 else 3114 rxq->crc_len = 0; 3115 rxq->drop_en = rx_conf->rx_drop_en; 3116 rxq->rx_deferred_start = rx_conf->rx_deferred_start; 3117 rxq->offloads = offloads; 3118 3119 /* 3120 * The packet type in RX descriptor is different for different NICs. 3121 * Some bits are used for x550 but reserved for other NICS. 3122 * So set different masks for different NICs. 3123 */ 3124 if (hw->mac.type == ixgbe_mac_X550 || 3125 hw->mac.type == ixgbe_mac_X550EM_x || 3126 hw->mac.type == ixgbe_mac_X550EM_a || 3127 hw->mac.type == ixgbe_mac_X550_vf || 3128 hw->mac.type == ixgbe_mac_X550EM_x_vf || 3129 hw->mac.type == ixgbe_mac_X550EM_a_vf) 3130 rxq->pkt_type_mask = IXGBE_PACKET_TYPE_MASK_X550; 3131 else 3132 rxq->pkt_type_mask = IXGBE_PACKET_TYPE_MASK_82599; 3133 3134 /* 3135 * 82599 errata, UDP frames with a 0 checksum can be marked as checksum 3136 * errors. 3137 */ 3138 if (hw->mac.type == ixgbe_mac_82599EB) 3139 rxq->rx_udp_csum_zero_err = 1; 3140 3141 /* 3142 * Allocate RX ring hardware descriptors. A memzone large enough to 3143 * handle the maximum ring size is allocated in order to allow for 3144 * resizing in later calls to the queue setup function. 3145 */ 3146 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, 3147 RX_RING_SZ, IXGBE_ALIGN, socket_id); 3148 if (rz == NULL) { 3149 ixgbe_rx_queue_release(rxq); 3150 return -ENOMEM; 3151 } 3152 3153 /* 3154 * Zero init all the descriptors in the ring. 3155 */ 3156 memset(rz->addr, 0, RX_RING_SZ); 3157 3158 /* 3159 * Modified to setup VFRDT for Virtual Function 3160 */ 3161 if (hw->mac.type == ixgbe_mac_82599_vf || 3162 hw->mac.type == ixgbe_mac_X540_vf || 3163 hw->mac.type == ixgbe_mac_X550_vf || 3164 hw->mac.type == ixgbe_mac_X550EM_x_vf || 3165 hw->mac.type == ixgbe_mac_X550EM_a_vf) { 3166 rxq->rdt_reg_addr = 3167 IXGBE_PCI_REG_ADDR(hw, IXGBE_VFRDT(queue_idx)); 3168 rxq->rdh_reg_addr = 3169 IXGBE_PCI_REG_ADDR(hw, IXGBE_VFRDH(queue_idx)); 3170 } else { 3171 rxq->rdt_reg_addr = 3172 IXGBE_PCI_REG_ADDR(hw, IXGBE_RDT(rxq->reg_idx)); 3173 rxq->rdh_reg_addr = 3174 IXGBE_PCI_REG_ADDR(hw, IXGBE_RDH(rxq->reg_idx)); 3175 } 3176 3177 rxq->rx_ring_phys_addr = rz->iova; 3178 rxq->rx_ring = (union ixgbe_adv_rx_desc *) rz->addr; 3179 3180 /* 3181 * Certain constraints must be met in order to use the bulk buffer 3182 * allocation Rx burst function. If any of Rx queues doesn't meet them 3183 * the feature should be disabled for the whole port. 3184 */ 3185 if (check_rx_burst_bulk_alloc_preconditions(rxq)) { 3186 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Rx Bulk Alloc " 3187 "preconditions - canceling the feature for " 3188 "the whole port[%d]", 3189 rxq->queue_id, rxq->port_id); 3190 adapter->rx_bulk_alloc_allowed = false; 3191 } 3192 3193 /* 3194 * Allocate software ring. Allow for space at the end of the 3195 * S/W ring to make sure look-ahead logic in bulk alloc Rx burst 3196 * function does not access an invalid memory region. 3197 */ 3198 len = nb_desc; 3199 if (adapter->rx_bulk_alloc_allowed) 3200 len += RTE_PMD_IXGBE_RX_MAX_BURST; 3201 3202 rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring", 3203 sizeof(struct ixgbe_rx_entry) * len, 3204 RTE_CACHE_LINE_SIZE, socket_id); 3205 if (!rxq->sw_ring) { 3206 ixgbe_rx_queue_release(rxq); 3207 return -ENOMEM; 3208 } 3209 3210 /* 3211 * Always allocate even if it's not going to be needed in order to 3212 * simplify the code. 3213 * 3214 * This ring is used in LRO and Scattered Rx cases and Scattered Rx may 3215 * be requested in ixgbe_dev_rx_init(), which is called later from 3216 * dev_start() flow. 3217 */ 3218 rxq->sw_sc_ring = 3219 rte_zmalloc_socket("rxq->sw_sc_ring", 3220 sizeof(struct ixgbe_scattered_rx_entry) * len, 3221 RTE_CACHE_LINE_SIZE, socket_id); 3222 if (!rxq->sw_sc_ring) { 3223 ixgbe_rx_queue_release(rxq); 3224 return -ENOMEM; 3225 } 3226 3227 PMD_INIT_LOG(DEBUG, "sw_ring=%p sw_sc_ring=%p hw_ring=%p " 3228 "dma_addr=0x%"PRIx64, 3229 rxq->sw_ring, rxq->sw_sc_ring, rxq->rx_ring, 3230 rxq->rx_ring_phys_addr); 3231 3232 if (!rte_is_power_of_2(nb_desc)) { 3233 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx " 3234 "preconditions - canceling the feature for " 3235 "the whole port[%d]", 3236 rxq->queue_id, rxq->port_id); 3237 adapter->rx_vec_allowed = false; 3238 } else 3239 ixgbe_rxq_vec_setup(rxq); 3240 3241 dev->data->rx_queues[queue_idx] = rxq; 3242 3243 ixgbe_reset_rx_queue(adapter, rxq); 3244 3245 return 0; 3246 } 3247 3248 uint32_t 3249 ixgbe_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id) 3250 { 3251 #define IXGBE_RXQ_SCAN_INTERVAL 4 3252 volatile union ixgbe_adv_rx_desc *rxdp; 3253 struct ixgbe_rx_queue *rxq; 3254 uint32_t desc = 0; 3255 3256 rxq = dev->data->rx_queues[rx_queue_id]; 3257 rxdp = &(rxq->rx_ring[rxq->rx_tail]); 3258 3259 while ((desc < rxq->nb_rx_desc) && 3260 (rxdp->wb.upper.status_error & 3261 rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD))) { 3262 desc += IXGBE_RXQ_SCAN_INTERVAL; 3263 rxdp += IXGBE_RXQ_SCAN_INTERVAL; 3264 if (rxq->rx_tail + desc >= rxq->nb_rx_desc) 3265 rxdp = &(rxq->rx_ring[rxq->rx_tail + 3266 desc - rxq->nb_rx_desc]); 3267 } 3268 3269 return desc; 3270 } 3271 3272 int 3273 ixgbe_dev_rx_descriptor_done(void *rx_queue, uint16_t offset) 3274 { 3275 volatile union ixgbe_adv_rx_desc *rxdp; 3276 struct ixgbe_rx_queue *rxq = rx_queue; 3277 uint32_t desc; 3278 3279 if (unlikely(offset >= rxq->nb_rx_desc)) 3280 return 0; 3281 desc = rxq->rx_tail + offset; 3282 if (desc >= rxq->nb_rx_desc) 3283 desc -= rxq->nb_rx_desc; 3284 3285 rxdp = &rxq->rx_ring[desc]; 3286 return !!(rxdp->wb.upper.status_error & 3287 rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD)); 3288 } 3289 3290 int 3291 ixgbe_dev_rx_descriptor_status(void *rx_queue, uint16_t offset) 3292 { 3293 struct ixgbe_rx_queue *rxq = rx_queue; 3294 volatile uint32_t *status; 3295 uint32_t nb_hold, desc; 3296 3297 if (unlikely(offset >= rxq->nb_rx_desc)) 3298 return -EINVAL; 3299 3300 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) 3301 if (rxq->rx_using_sse) 3302 nb_hold = rxq->rxrearm_nb; 3303 else 3304 #endif 3305 nb_hold = rxq->nb_rx_hold; 3306 if (offset >= rxq->nb_rx_desc - nb_hold) 3307 return RTE_ETH_RX_DESC_UNAVAIL; 3308 3309 desc = rxq->rx_tail + offset; 3310 if (desc >= rxq->nb_rx_desc) 3311 desc -= rxq->nb_rx_desc; 3312 3313 status = &rxq->rx_ring[desc].wb.upper.status_error; 3314 if (*status & rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD)) 3315 return RTE_ETH_RX_DESC_DONE; 3316 3317 return RTE_ETH_RX_DESC_AVAIL; 3318 } 3319 3320 int 3321 ixgbe_dev_tx_descriptor_status(void *tx_queue, uint16_t offset) 3322 { 3323 struct ixgbe_tx_queue *txq = tx_queue; 3324 volatile uint32_t *status; 3325 uint32_t desc; 3326 3327 if (unlikely(offset >= txq->nb_tx_desc)) 3328 return -EINVAL; 3329 3330 desc = txq->tx_tail + offset; 3331 /* go to next desc that has the RS bit */ 3332 desc = ((desc + txq->tx_rs_thresh - 1) / txq->tx_rs_thresh) * 3333 txq->tx_rs_thresh; 3334 if (desc >= txq->nb_tx_desc) { 3335 desc -= txq->nb_tx_desc; 3336 if (desc >= txq->nb_tx_desc) 3337 desc -= txq->nb_tx_desc; 3338 } 3339 3340 status = &txq->tx_ring[desc].wb.status; 3341 if (*status & rte_cpu_to_le_32(IXGBE_ADVTXD_STAT_DD)) 3342 return RTE_ETH_TX_DESC_DONE; 3343 3344 return RTE_ETH_TX_DESC_FULL; 3345 } 3346 3347 /* 3348 * Set up link loopback for X540/X550 mode Tx->Rx. 3349 */ 3350 static inline void __rte_cold 3351 ixgbe_setup_loopback_link_x540_x550(struct ixgbe_hw *hw, bool enable) 3352 { 3353 uint32_t macc; 3354 PMD_INIT_FUNC_TRACE(); 3355 3356 u16 autoneg_reg = IXGBE_MII_AUTONEG_REG; 3357 3358 hw->phy.ops.read_reg(hw, IXGBE_MDIO_AUTO_NEG_CONTROL, 3359 IXGBE_MDIO_AUTO_NEG_DEV_TYPE, &autoneg_reg); 3360 macc = IXGBE_READ_REG(hw, IXGBE_MACC); 3361 3362 if (enable) { 3363 /* datasheet 15.2.1: disable AUTONEG (PHY Bit 7.0.C) */ 3364 autoneg_reg |= IXGBE_MII_AUTONEG_ENABLE; 3365 /* datasheet 15.2.1: MACC.FLU = 1 (force link up) */ 3366 macc |= IXGBE_MACC_FLU; 3367 } else { 3368 autoneg_reg &= ~IXGBE_MII_AUTONEG_ENABLE; 3369 macc &= ~IXGBE_MACC_FLU; 3370 } 3371 3372 hw->phy.ops.write_reg(hw, IXGBE_MDIO_AUTO_NEG_CONTROL, 3373 IXGBE_MDIO_AUTO_NEG_DEV_TYPE, autoneg_reg); 3374 3375 IXGBE_WRITE_REG(hw, IXGBE_MACC, macc); 3376 } 3377 3378 void __rte_cold 3379 ixgbe_dev_clear_queues(struct rte_eth_dev *dev) 3380 { 3381 unsigned i; 3382 struct ixgbe_adapter *adapter = dev->data->dev_private; 3383 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3384 3385 PMD_INIT_FUNC_TRACE(); 3386 3387 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3388 struct ixgbe_tx_queue *txq = dev->data->tx_queues[i]; 3389 3390 if (txq != NULL) { 3391 txq->ops->release_mbufs(txq); 3392 txq->ops->reset(txq); 3393 } 3394 } 3395 3396 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3397 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i]; 3398 3399 if (rxq != NULL) { 3400 ixgbe_rx_queue_release_mbufs(rxq); 3401 ixgbe_reset_rx_queue(adapter, rxq); 3402 } 3403 } 3404 /* If loopback mode was enabled, reconfigure the link accordingly */ 3405 if (dev->data->dev_conf.lpbk_mode != 0) { 3406 if (hw->mac.type == ixgbe_mac_X540 || 3407 hw->mac.type == ixgbe_mac_X550 || 3408 hw->mac.type == ixgbe_mac_X550EM_x || 3409 hw->mac.type == ixgbe_mac_X550EM_a) 3410 ixgbe_setup_loopback_link_x540_x550(hw, false); 3411 } 3412 } 3413 3414 void 3415 ixgbe_dev_free_queues(struct rte_eth_dev *dev) 3416 { 3417 unsigned i; 3418 3419 PMD_INIT_FUNC_TRACE(); 3420 3421 for (i = 0; i < dev->data->nb_rx_queues; i++) { 3422 ixgbe_dev_rx_queue_release(dev->data->rx_queues[i]); 3423 dev->data->rx_queues[i] = NULL; 3424 rte_eth_dma_zone_free(dev, "rx_ring", i); 3425 } 3426 dev->data->nb_rx_queues = 0; 3427 3428 for (i = 0; i < dev->data->nb_tx_queues; i++) { 3429 ixgbe_dev_tx_queue_release(dev->data->tx_queues[i]); 3430 dev->data->tx_queues[i] = NULL; 3431 rte_eth_dma_zone_free(dev, "tx_ring", i); 3432 } 3433 dev->data->nb_tx_queues = 0; 3434 } 3435 3436 /********************************************************************* 3437 * 3438 * Device RX/TX init functions 3439 * 3440 **********************************************************************/ 3441 3442 /** 3443 * Receive Side Scaling (RSS) 3444 * See section 7.1.2.8 in the following document: 3445 * "Intel 82599 10 GbE Controller Datasheet" - Revision 2.1 October 2009 3446 * 3447 * Principles: 3448 * The source and destination IP addresses of the IP header and the source 3449 * and destination ports of TCP/UDP headers, if any, of received packets are 3450 * hashed against a configurable random key to compute a 32-bit RSS hash result. 3451 * The seven (7) LSBs of the 32-bit hash result are used as an index into a 3452 * 128-entry redirection table (RETA). Each entry of the RETA provides a 3-bit 3453 * RSS output index which is used as the RX queue index where to store the 3454 * received packets. 3455 * The following output is supplied in the RX write-back descriptor: 3456 * - 32-bit result of the Microsoft RSS hash function, 3457 * - 4-bit RSS type field. 3458 */ 3459 3460 /* 3461 * RSS random key supplied in section 7.1.2.8.3 of the Intel 82599 datasheet. 3462 * Used as the default key. 3463 */ 3464 static uint8_t rss_intel_key[40] = { 3465 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 3466 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0, 3467 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4, 3468 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, 3469 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA, 3470 }; 3471 3472 static void 3473 ixgbe_rss_disable(struct rte_eth_dev *dev) 3474 { 3475 struct ixgbe_hw *hw; 3476 uint32_t mrqc; 3477 uint32_t mrqc_reg; 3478 3479 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3480 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type); 3481 mrqc = IXGBE_READ_REG(hw, mrqc_reg); 3482 mrqc &= ~IXGBE_MRQC_RSSEN; 3483 IXGBE_WRITE_REG(hw, mrqc_reg, mrqc); 3484 } 3485 3486 static void 3487 ixgbe_hw_rss_hash_set(struct ixgbe_hw *hw, struct rte_eth_rss_conf *rss_conf) 3488 { 3489 uint8_t *hash_key; 3490 uint32_t mrqc; 3491 uint32_t rss_key; 3492 uint64_t rss_hf; 3493 uint16_t i; 3494 uint32_t mrqc_reg; 3495 uint32_t rssrk_reg; 3496 3497 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type); 3498 rssrk_reg = ixgbe_rssrk_reg_get(hw->mac.type, 0); 3499 3500 hash_key = rss_conf->rss_key; 3501 if (hash_key != NULL) { 3502 /* Fill in RSS hash key */ 3503 for (i = 0; i < 10; i++) { 3504 rss_key = hash_key[(i * 4)]; 3505 rss_key |= hash_key[(i * 4) + 1] << 8; 3506 rss_key |= hash_key[(i * 4) + 2] << 16; 3507 rss_key |= hash_key[(i * 4) + 3] << 24; 3508 IXGBE_WRITE_REG_ARRAY(hw, rssrk_reg, i, rss_key); 3509 } 3510 } 3511 3512 /* Set configured hashing protocols in MRQC register */ 3513 rss_hf = rss_conf->rss_hf; 3514 mrqc = IXGBE_MRQC_RSSEN; /* Enable RSS */ 3515 if (rss_hf & ETH_RSS_IPV4) 3516 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 3517 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) 3518 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 3519 if (rss_hf & ETH_RSS_IPV6) 3520 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 3521 if (rss_hf & ETH_RSS_IPV6_EX) 3522 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX; 3523 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) 3524 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 3525 if (rss_hf & ETH_RSS_IPV6_TCP_EX) 3526 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP; 3527 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) 3528 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 3529 if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) 3530 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 3531 if (rss_hf & ETH_RSS_IPV6_UDP_EX) 3532 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 3533 IXGBE_WRITE_REG(hw, mrqc_reg, mrqc); 3534 } 3535 3536 int 3537 ixgbe_dev_rss_hash_update(struct rte_eth_dev *dev, 3538 struct rte_eth_rss_conf *rss_conf) 3539 { 3540 struct ixgbe_hw *hw; 3541 uint32_t mrqc; 3542 uint64_t rss_hf; 3543 uint32_t mrqc_reg; 3544 3545 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3546 3547 if (!ixgbe_rss_update_sp(hw->mac.type)) { 3548 PMD_DRV_LOG(ERR, "RSS hash update is not supported on this " 3549 "NIC."); 3550 return -ENOTSUP; 3551 } 3552 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type); 3553 3554 /* 3555 * Excerpt from section 7.1.2.8 Receive-Side Scaling (RSS): 3556 * "RSS enabling cannot be done dynamically while it must be 3557 * preceded by a software reset" 3558 * Before changing anything, first check that the update RSS operation 3559 * does not attempt to disable RSS, if RSS was enabled at 3560 * initialization time, or does not attempt to enable RSS, if RSS was 3561 * disabled at initialization time. 3562 */ 3563 rss_hf = rss_conf->rss_hf & IXGBE_RSS_OFFLOAD_ALL; 3564 mrqc = IXGBE_READ_REG(hw, mrqc_reg); 3565 if (!(mrqc & IXGBE_MRQC_RSSEN)) { /* RSS disabled */ 3566 if (rss_hf != 0) /* Enable RSS */ 3567 return -(EINVAL); 3568 return 0; /* Nothing to do */ 3569 } 3570 /* RSS enabled */ 3571 if (rss_hf == 0) /* Disable RSS */ 3572 return -(EINVAL); 3573 ixgbe_hw_rss_hash_set(hw, rss_conf); 3574 return 0; 3575 } 3576 3577 int 3578 ixgbe_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 3579 struct rte_eth_rss_conf *rss_conf) 3580 { 3581 struct ixgbe_hw *hw; 3582 uint8_t *hash_key; 3583 uint32_t mrqc; 3584 uint32_t rss_key; 3585 uint64_t rss_hf; 3586 uint16_t i; 3587 uint32_t mrqc_reg; 3588 uint32_t rssrk_reg; 3589 3590 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3591 mrqc_reg = ixgbe_mrqc_reg_get(hw->mac.type); 3592 rssrk_reg = ixgbe_rssrk_reg_get(hw->mac.type, 0); 3593 hash_key = rss_conf->rss_key; 3594 if (hash_key != NULL) { 3595 /* Return RSS hash key */ 3596 for (i = 0; i < 10; i++) { 3597 rss_key = IXGBE_READ_REG_ARRAY(hw, rssrk_reg, i); 3598 hash_key[(i * 4)] = rss_key & 0x000000FF; 3599 hash_key[(i * 4) + 1] = (rss_key >> 8) & 0x000000FF; 3600 hash_key[(i * 4) + 2] = (rss_key >> 16) & 0x000000FF; 3601 hash_key[(i * 4) + 3] = (rss_key >> 24) & 0x000000FF; 3602 } 3603 } 3604 3605 /* Get RSS functions configured in MRQC register */ 3606 mrqc = IXGBE_READ_REG(hw, mrqc_reg); 3607 if ((mrqc & IXGBE_MRQC_RSSEN) == 0) { /* RSS is disabled */ 3608 rss_conf->rss_hf = 0; 3609 return 0; 3610 } 3611 rss_hf = 0; 3612 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4) 3613 rss_hf |= ETH_RSS_IPV4; 3614 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_TCP) 3615 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP; 3616 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6) 3617 rss_hf |= ETH_RSS_IPV6; 3618 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX) 3619 rss_hf |= ETH_RSS_IPV6_EX; 3620 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_TCP) 3621 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP; 3622 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP) 3623 rss_hf |= ETH_RSS_IPV6_TCP_EX; 3624 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV4_UDP) 3625 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP; 3626 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_UDP) 3627 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP; 3628 if (mrqc & IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP) 3629 rss_hf |= ETH_RSS_IPV6_UDP_EX; 3630 rss_conf->rss_hf = rss_hf; 3631 return 0; 3632 } 3633 3634 static void 3635 ixgbe_rss_configure(struct rte_eth_dev *dev) 3636 { 3637 struct rte_eth_rss_conf rss_conf; 3638 struct ixgbe_adapter *adapter; 3639 struct ixgbe_hw *hw; 3640 uint32_t reta; 3641 uint16_t i; 3642 uint16_t j; 3643 uint16_t sp_reta_size; 3644 uint32_t reta_reg; 3645 3646 PMD_INIT_FUNC_TRACE(); 3647 adapter = dev->data->dev_private; 3648 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3649 3650 sp_reta_size = ixgbe_reta_size_get(hw->mac.type); 3651 3652 /* 3653 * Fill in redirection table 3654 * The byte-swap is needed because NIC registers are in 3655 * little-endian order. 3656 */ 3657 if (adapter->rss_reta_updated == 0) { 3658 reta = 0; 3659 for (i = 0, j = 0; i < sp_reta_size; i++, j++) { 3660 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i); 3661 3662 if (j == dev->data->nb_rx_queues) 3663 j = 0; 3664 reta = (reta << 8) | j; 3665 if ((i & 3) == 3) 3666 IXGBE_WRITE_REG(hw, reta_reg, 3667 rte_bswap32(reta)); 3668 } 3669 } 3670 3671 /* 3672 * Configure the RSS key and the RSS protocols used to compute 3673 * the RSS hash of input packets. 3674 */ 3675 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf; 3676 if ((rss_conf.rss_hf & IXGBE_RSS_OFFLOAD_ALL) == 0) { 3677 ixgbe_rss_disable(dev); 3678 return; 3679 } 3680 if (rss_conf.rss_key == NULL) 3681 rss_conf.rss_key = rss_intel_key; /* Default hash key */ 3682 ixgbe_hw_rss_hash_set(hw, &rss_conf); 3683 } 3684 3685 #define NUM_VFTA_REGISTERS 128 3686 #define NIC_RX_BUFFER_SIZE 0x200 3687 #define X550_RX_BUFFER_SIZE 0x180 3688 3689 static void 3690 ixgbe_vmdq_dcb_configure(struct rte_eth_dev *dev) 3691 { 3692 struct rte_eth_vmdq_dcb_conf *cfg; 3693 struct ixgbe_hw *hw; 3694 enum rte_eth_nb_pools num_pools; 3695 uint32_t mrqc, vt_ctl, queue_mapping, vlanctrl; 3696 uint16_t pbsize; 3697 uint8_t nb_tcs; /* number of traffic classes */ 3698 int i; 3699 3700 PMD_INIT_FUNC_TRACE(); 3701 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3702 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf; 3703 num_pools = cfg->nb_queue_pools; 3704 /* Check we have a valid number of pools */ 3705 if (num_pools != ETH_16_POOLS && num_pools != ETH_32_POOLS) { 3706 ixgbe_rss_disable(dev); 3707 return; 3708 } 3709 /* 16 pools -> 8 traffic classes, 32 pools -> 4 traffic classes */ 3710 nb_tcs = (uint8_t)(ETH_VMDQ_DCB_NUM_QUEUES / (int)num_pools); 3711 3712 /* 3713 * RXPBSIZE 3714 * split rx buffer up into sections, each for 1 traffic class 3715 */ 3716 switch (hw->mac.type) { 3717 case ixgbe_mac_X550: 3718 case ixgbe_mac_X550EM_x: 3719 case ixgbe_mac_X550EM_a: 3720 pbsize = (uint16_t)(X550_RX_BUFFER_SIZE / nb_tcs); 3721 break; 3722 default: 3723 pbsize = (uint16_t)(NIC_RX_BUFFER_SIZE / nb_tcs); 3724 break; 3725 } 3726 for (i = 0; i < nb_tcs; i++) { 3727 uint32_t rxpbsize = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(i)); 3728 3729 rxpbsize &= (~(0x3FF << IXGBE_RXPBSIZE_SHIFT)); 3730 /* clear 10 bits. */ 3731 rxpbsize |= (pbsize << IXGBE_RXPBSIZE_SHIFT); /* set value */ 3732 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize); 3733 } 3734 /* zero alloc all unused TCs */ 3735 for (i = nb_tcs; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3736 uint32_t rxpbsize = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(i)); 3737 3738 rxpbsize &= (~(0x3FF << IXGBE_RXPBSIZE_SHIFT)); 3739 /* clear 10 bits. */ 3740 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize); 3741 } 3742 3743 /* MRQC: enable vmdq and dcb */ 3744 mrqc = (num_pools == ETH_16_POOLS) ? 3745 IXGBE_MRQC_VMDQRT8TCEN : IXGBE_MRQC_VMDQRT4TCEN; 3746 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 3747 3748 /* PFVTCTL: turn on virtualisation and set the default pool */ 3749 vt_ctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN; 3750 if (cfg->enable_default_pool) { 3751 vt_ctl |= (cfg->default_pool << IXGBE_VT_CTL_POOL_SHIFT); 3752 } else { 3753 vt_ctl |= IXGBE_VT_CTL_DIS_DEFPL; 3754 } 3755 3756 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vt_ctl); 3757 3758 /* RTRUP2TC: mapping user priorities to traffic classes (TCs) */ 3759 queue_mapping = 0; 3760 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) 3761 /* 3762 * mapping is done with 3 bits per priority, 3763 * so shift by i*3 each time 3764 */ 3765 queue_mapping |= ((cfg->dcb_tc[i] & 0x07) << (i * 3)); 3766 3767 IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, queue_mapping); 3768 3769 /* RTRPCS: DCB related */ 3770 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, IXGBE_RMCS_RRM); 3771 3772 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */ 3773 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 3774 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */ 3775 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl); 3776 3777 /* VFTA - enable all vlan filters */ 3778 for (i = 0; i < NUM_VFTA_REGISTERS; i++) { 3779 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 0xFFFFFFFF); 3780 } 3781 3782 /* VFRE: pool enabling for receive - 16 or 32 */ 3783 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), 3784 num_pools == ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF); 3785 3786 /* 3787 * MPSAR - allow pools to read specific mac addresses 3788 * In this case, all pools should be able to read from mac addr 0 3789 */ 3790 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(0), 0xFFFFFFFF); 3791 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(0), 0xFFFFFFFF); 3792 3793 /* PFVLVF, PFVLVFB: set up filters for vlan tags as configured */ 3794 for (i = 0; i < cfg->nb_pool_maps; i++) { 3795 /* set vlan id in VF register and set the valid bit */ 3796 IXGBE_WRITE_REG(hw, IXGBE_VLVF(i), (IXGBE_VLVF_VIEN | 3797 (cfg->pool_map[i].vlan_id & 0xFFF))); 3798 /* 3799 * Put the allowed pools in VFB reg. As we only have 16 or 32 3800 * pools, we only need to use the first half of the register 3801 * i.e. bits 0-31 3802 */ 3803 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(i*2), cfg->pool_map[i].pools); 3804 } 3805 } 3806 3807 /** 3808 * ixgbe_dcb_config_tx_hw_config - Configure general DCB TX parameters 3809 * @dev: pointer to eth_dev structure 3810 * @dcb_config: pointer to ixgbe_dcb_config structure 3811 */ 3812 static void 3813 ixgbe_dcb_tx_hw_config(struct rte_eth_dev *dev, 3814 struct ixgbe_dcb_config *dcb_config) 3815 { 3816 uint32_t reg; 3817 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3818 3819 PMD_INIT_FUNC_TRACE(); 3820 if (hw->mac.type != ixgbe_mac_82598EB) { 3821 /* Disable the Tx desc arbiter so that MTQC can be changed */ 3822 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 3823 reg |= IXGBE_RTTDCS_ARBDIS; 3824 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg); 3825 3826 /* Enable DCB for Tx with 8 TCs */ 3827 if (dcb_config->num_tcs.pg_tcs == 8) { 3828 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ; 3829 } else { 3830 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ; 3831 } 3832 if (dcb_config->vt_mode) 3833 reg |= IXGBE_MTQC_VT_ENA; 3834 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg); 3835 3836 /* Enable the Tx desc arbiter */ 3837 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 3838 reg &= ~IXGBE_RTTDCS_ARBDIS; 3839 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg); 3840 3841 /* Enable Security TX Buffer IFG for DCB */ 3842 reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG); 3843 reg |= IXGBE_SECTX_DCB; 3844 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg); 3845 } 3846 } 3847 3848 /** 3849 * ixgbe_vmdq_dcb_hw_tx_config - Configure general VMDQ+DCB TX parameters 3850 * @dev: pointer to rte_eth_dev structure 3851 * @dcb_config: pointer to ixgbe_dcb_config structure 3852 */ 3853 static void 3854 ixgbe_vmdq_dcb_hw_tx_config(struct rte_eth_dev *dev, 3855 struct ixgbe_dcb_config *dcb_config) 3856 { 3857 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3858 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf; 3859 struct ixgbe_hw *hw = 3860 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 3861 3862 PMD_INIT_FUNC_TRACE(); 3863 if (hw->mac.type != ixgbe_mac_82598EB) 3864 /*PF VF Transmit Enable*/ 3865 IXGBE_WRITE_REG(hw, IXGBE_VFTE(0), 3866 vmdq_tx_conf->nb_queue_pools == ETH_16_POOLS ? 0xFFFF : 0xFFFFFFFF); 3867 3868 /*Configure general DCB TX parameters*/ 3869 ixgbe_dcb_tx_hw_config(dev, dcb_config); 3870 } 3871 3872 static void 3873 ixgbe_vmdq_dcb_rx_config(struct rte_eth_dev *dev, 3874 struct ixgbe_dcb_config *dcb_config) 3875 { 3876 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3877 &dev->data->dev_conf.rx_adv_conf.vmdq_dcb_conf; 3878 struct ixgbe_dcb_tc_config *tc; 3879 uint8_t i, j; 3880 3881 /* convert rte_eth_conf.rx_adv_conf to struct ixgbe_dcb_config */ 3882 if (vmdq_rx_conf->nb_queue_pools == ETH_16_POOLS) { 3883 dcb_config->num_tcs.pg_tcs = ETH_8_TCS; 3884 dcb_config->num_tcs.pfc_tcs = ETH_8_TCS; 3885 } else { 3886 dcb_config->num_tcs.pg_tcs = ETH_4_TCS; 3887 dcb_config->num_tcs.pfc_tcs = ETH_4_TCS; 3888 } 3889 3890 /* Initialize User Priority to Traffic Class mapping */ 3891 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) { 3892 tc = &dcb_config->tc_config[j]; 3893 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0; 3894 } 3895 3896 /* User Priority to Traffic Class mapping */ 3897 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3898 j = vmdq_rx_conf->dcb_tc[i]; 3899 tc = &dcb_config->tc_config[j]; 3900 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |= 3901 (uint8_t)(1 << i); 3902 } 3903 } 3904 3905 static void 3906 ixgbe_dcb_vt_tx_config(struct rte_eth_dev *dev, 3907 struct ixgbe_dcb_config *dcb_config) 3908 { 3909 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3910 &dev->data->dev_conf.tx_adv_conf.vmdq_dcb_tx_conf; 3911 struct ixgbe_dcb_tc_config *tc; 3912 uint8_t i, j; 3913 3914 /* convert rte_eth_conf.rx_adv_conf to struct ixgbe_dcb_config */ 3915 if (vmdq_tx_conf->nb_queue_pools == ETH_16_POOLS) { 3916 dcb_config->num_tcs.pg_tcs = ETH_8_TCS; 3917 dcb_config->num_tcs.pfc_tcs = ETH_8_TCS; 3918 } else { 3919 dcb_config->num_tcs.pg_tcs = ETH_4_TCS; 3920 dcb_config->num_tcs.pfc_tcs = ETH_4_TCS; 3921 } 3922 3923 /* Initialize User Priority to Traffic Class mapping */ 3924 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) { 3925 tc = &dcb_config->tc_config[j]; 3926 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0; 3927 } 3928 3929 /* User Priority to Traffic Class mapping */ 3930 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3931 j = vmdq_tx_conf->dcb_tc[i]; 3932 tc = &dcb_config->tc_config[j]; 3933 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |= 3934 (uint8_t)(1 << i); 3935 } 3936 } 3937 3938 static void 3939 ixgbe_dcb_rx_config(struct rte_eth_dev *dev, 3940 struct ixgbe_dcb_config *dcb_config) 3941 { 3942 struct rte_eth_dcb_rx_conf *rx_conf = 3943 &dev->data->dev_conf.rx_adv_conf.dcb_rx_conf; 3944 struct ixgbe_dcb_tc_config *tc; 3945 uint8_t i, j; 3946 3947 dcb_config->num_tcs.pg_tcs = (uint8_t)rx_conf->nb_tcs; 3948 dcb_config->num_tcs.pfc_tcs = (uint8_t)rx_conf->nb_tcs; 3949 3950 /* Initialize User Priority to Traffic Class mapping */ 3951 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) { 3952 tc = &dcb_config->tc_config[j]; 3953 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0; 3954 } 3955 3956 /* User Priority to Traffic Class mapping */ 3957 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3958 j = rx_conf->dcb_tc[i]; 3959 tc = &dcb_config->tc_config[j]; 3960 tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap |= 3961 (uint8_t)(1 << i); 3962 } 3963 } 3964 3965 static void 3966 ixgbe_dcb_tx_config(struct rte_eth_dev *dev, 3967 struct ixgbe_dcb_config *dcb_config) 3968 { 3969 struct rte_eth_dcb_tx_conf *tx_conf = 3970 &dev->data->dev_conf.tx_adv_conf.dcb_tx_conf; 3971 struct ixgbe_dcb_tc_config *tc; 3972 uint8_t i, j; 3973 3974 dcb_config->num_tcs.pg_tcs = (uint8_t)tx_conf->nb_tcs; 3975 dcb_config->num_tcs.pfc_tcs = (uint8_t)tx_conf->nb_tcs; 3976 3977 /* Initialize User Priority to Traffic Class mapping */ 3978 for (j = 0; j < IXGBE_DCB_MAX_TRAFFIC_CLASS; j++) { 3979 tc = &dcb_config->tc_config[j]; 3980 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0; 3981 } 3982 3983 /* User Priority to Traffic Class mapping */ 3984 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3985 j = tx_conf->dcb_tc[i]; 3986 tc = &dcb_config->tc_config[j]; 3987 tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap |= 3988 (uint8_t)(1 << i); 3989 } 3990 } 3991 3992 /** 3993 * ixgbe_dcb_rx_hw_config - Configure general DCB RX HW parameters 3994 * @dev: pointer to eth_dev structure 3995 * @dcb_config: pointer to ixgbe_dcb_config structure 3996 */ 3997 static void 3998 ixgbe_dcb_rx_hw_config(struct rte_eth_dev *dev, 3999 struct ixgbe_dcb_config *dcb_config) 4000 { 4001 uint32_t reg; 4002 uint32_t vlanctrl; 4003 uint8_t i; 4004 uint32_t q; 4005 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4006 4007 PMD_INIT_FUNC_TRACE(); 4008 /* 4009 * Disable the arbiter before changing parameters 4010 * (always enable recycle mode; WSP) 4011 */ 4012 reg = IXGBE_RTRPCS_RRM | IXGBE_RTRPCS_RAC | IXGBE_RTRPCS_ARBDIS; 4013 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, reg); 4014 4015 if (hw->mac.type != ixgbe_mac_82598EB) { 4016 reg = IXGBE_READ_REG(hw, IXGBE_MRQC); 4017 if (dcb_config->num_tcs.pg_tcs == 4) { 4018 if (dcb_config->vt_mode) 4019 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) | 4020 IXGBE_MRQC_VMDQRT4TCEN; 4021 else { 4022 /* no matter the mode is DCB or DCB_RSS, just 4023 * set the MRQE to RSSXTCEN. RSS is controlled 4024 * by RSS_FIELD 4025 */ 4026 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, 0); 4027 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) | 4028 IXGBE_MRQC_RTRSS4TCEN; 4029 } 4030 } 4031 if (dcb_config->num_tcs.pg_tcs == 8) { 4032 if (dcb_config->vt_mode) 4033 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) | 4034 IXGBE_MRQC_VMDQRT8TCEN; 4035 else { 4036 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, 0); 4037 reg = (reg & ~IXGBE_MRQC_MRQE_MASK) | 4038 IXGBE_MRQC_RTRSS8TCEN; 4039 } 4040 } 4041 4042 IXGBE_WRITE_REG(hw, IXGBE_MRQC, reg); 4043 4044 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 4045 /* Disable drop for all queues in VMDQ mode*/ 4046 for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++) 4047 IXGBE_WRITE_REG(hw, IXGBE_QDE, 4048 (IXGBE_QDE_WRITE | 4049 (q << IXGBE_QDE_IDX_SHIFT))); 4050 } else { 4051 /* Enable drop for all queues in SRIOV mode */ 4052 for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++) 4053 IXGBE_WRITE_REG(hw, IXGBE_QDE, 4054 (IXGBE_QDE_WRITE | 4055 (q << IXGBE_QDE_IDX_SHIFT) | 4056 IXGBE_QDE_ENABLE)); 4057 } 4058 } 4059 4060 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */ 4061 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 4062 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */ 4063 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl); 4064 4065 /* VFTA - enable all vlan filters */ 4066 for (i = 0; i < NUM_VFTA_REGISTERS; i++) { 4067 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 0xFFFFFFFF); 4068 } 4069 4070 /* 4071 * Configure Rx packet plane (recycle mode; WSP) and 4072 * enable arbiter 4073 */ 4074 reg = IXGBE_RTRPCS_RRM | IXGBE_RTRPCS_RAC; 4075 IXGBE_WRITE_REG(hw, IXGBE_RTRPCS, reg); 4076 } 4077 4078 static void 4079 ixgbe_dcb_hw_arbite_rx_config(struct ixgbe_hw *hw, uint16_t *refill, 4080 uint16_t *max, uint8_t *bwg_id, uint8_t *tsa, uint8_t *map) 4081 { 4082 switch (hw->mac.type) { 4083 case ixgbe_mac_82598EB: 4084 ixgbe_dcb_config_rx_arbiter_82598(hw, refill, max, tsa); 4085 break; 4086 case ixgbe_mac_82599EB: 4087 case ixgbe_mac_X540: 4088 case ixgbe_mac_X550: 4089 case ixgbe_mac_X550EM_x: 4090 case ixgbe_mac_X550EM_a: 4091 ixgbe_dcb_config_rx_arbiter_82599(hw, refill, max, bwg_id, 4092 tsa, map); 4093 break; 4094 default: 4095 break; 4096 } 4097 } 4098 4099 static void 4100 ixgbe_dcb_hw_arbite_tx_config(struct ixgbe_hw *hw, uint16_t *refill, uint16_t *max, 4101 uint8_t *bwg_id, uint8_t *tsa, uint8_t *map) 4102 { 4103 switch (hw->mac.type) { 4104 case ixgbe_mac_82598EB: 4105 ixgbe_dcb_config_tx_desc_arbiter_82598(hw, refill, max, bwg_id, tsa); 4106 ixgbe_dcb_config_tx_data_arbiter_82598(hw, refill, max, bwg_id, tsa); 4107 break; 4108 case ixgbe_mac_82599EB: 4109 case ixgbe_mac_X540: 4110 case ixgbe_mac_X550: 4111 case ixgbe_mac_X550EM_x: 4112 case ixgbe_mac_X550EM_a: 4113 ixgbe_dcb_config_tx_desc_arbiter_82599(hw, refill, max, bwg_id, tsa); 4114 ixgbe_dcb_config_tx_data_arbiter_82599(hw, refill, max, bwg_id, tsa, map); 4115 break; 4116 default: 4117 break; 4118 } 4119 } 4120 4121 #define DCB_RX_CONFIG 1 4122 #define DCB_TX_CONFIG 1 4123 #define DCB_TX_PB 1024 4124 /** 4125 * ixgbe_dcb_hw_configure - Enable DCB and configure 4126 * general DCB in VT mode and non-VT mode parameters 4127 * @dev: pointer to rte_eth_dev structure 4128 * @dcb_config: pointer to ixgbe_dcb_config structure 4129 */ 4130 static int 4131 ixgbe_dcb_hw_configure(struct rte_eth_dev *dev, 4132 struct ixgbe_dcb_config *dcb_config) 4133 { 4134 int ret = 0; 4135 uint8_t i, pfc_en, nb_tcs; 4136 uint16_t pbsize, rx_buffer_size; 4137 uint8_t config_dcb_rx = 0; 4138 uint8_t config_dcb_tx = 0; 4139 uint8_t tsa[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0}; 4140 uint8_t bwgid[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0}; 4141 uint16_t refill[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0}; 4142 uint16_t max[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0}; 4143 uint8_t map[IXGBE_DCB_MAX_TRAFFIC_CLASS] = {0}; 4144 struct ixgbe_dcb_tc_config *tc; 4145 uint32_t max_frame = dev->data->mtu + RTE_ETHER_HDR_LEN + 4146 RTE_ETHER_CRC_LEN; 4147 struct ixgbe_hw *hw = 4148 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4149 struct ixgbe_bw_conf *bw_conf = 4150 IXGBE_DEV_PRIVATE_TO_BW_CONF(dev->data->dev_private); 4151 4152 switch (dev->data->dev_conf.rxmode.mq_mode) { 4153 case ETH_MQ_RX_VMDQ_DCB: 4154 dcb_config->vt_mode = true; 4155 if (hw->mac.type != ixgbe_mac_82598EB) { 4156 config_dcb_rx = DCB_RX_CONFIG; 4157 /* 4158 *get dcb and VT rx configuration parameters 4159 *from rte_eth_conf 4160 */ 4161 ixgbe_vmdq_dcb_rx_config(dev, dcb_config); 4162 /*Configure general VMDQ and DCB RX parameters*/ 4163 ixgbe_vmdq_dcb_configure(dev); 4164 } 4165 break; 4166 case ETH_MQ_RX_DCB: 4167 case ETH_MQ_RX_DCB_RSS: 4168 dcb_config->vt_mode = false; 4169 config_dcb_rx = DCB_RX_CONFIG; 4170 /* Get dcb TX configuration parameters from rte_eth_conf */ 4171 ixgbe_dcb_rx_config(dev, dcb_config); 4172 /*Configure general DCB RX parameters*/ 4173 ixgbe_dcb_rx_hw_config(dev, dcb_config); 4174 break; 4175 default: 4176 PMD_INIT_LOG(ERR, "Incorrect DCB RX mode configuration"); 4177 break; 4178 } 4179 switch (dev->data->dev_conf.txmode.mq_mode) { 4180 case ETH_MQ_TX_VMDQ_DCB: 4181 dcb_config->vt_mode = true; 4182 config_dcb_tx = DCB_TX_CONFIG; 4183 /* get DCB and VT TX configuration parameters 4184 * from rte_eth_conf 4185 */ 4186 ixgbe_dcb_vt_tx_config(dev, dcb_config); 4187 /*Configure general VMDQ and DCB TX parameters*/ 4188 ixgbe_vmdq_dcb_hw_tx_config(dev, dcb_config); 4189 break; 4190 4191 case ETH_MQ_TX_DCB: 4192 dcb_config->vt_mode = false; 4193 config_dcb_tx = DCB_TX_CONFIG; 4194 /*get DCB TX configuration parameters from rte_eth_conf*/ 4195 ixgbe_dcb_tx_config(dev, dcb_config); 4196 /*Configure general DCB TX parameters*/ 4197 ixgbe_dcb_tx_hw_config(dev, dcb_config); 4198 break; 4199 default: 4200 PMD_INIT_LOG(ERR, "Incorrect DCB TX mode configuration"); 4201 break; 4202 } 4203 4204 nb_tcs = dcb_config->num_tcs.pfc_tcs; 4205 /* Unpack map */ 4206 ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map); 4207 if (nb_tcs == ETH_4_TCS) { 4208 /* Avoid un-configured priority mapping to TC0 */ 4209 uint8_t j = 4; 4210 uint8_t mask = 0xFF; 4211 4212 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES - 4; i++) 4213 mask = (uint8_t)(mask & (~(1 << map[i]))); 4214 for (i = 0; mask && (i < IXGBE_DCB_MAX_TRAFFIC_CLASS); i++) { 4215 if ((mask & 0x1) && (j < ETH_DCB_NUM_USER_PRIORITIES)) 4216 map[j++] = i; 4217 mask >>= 1; 4218 } 4219 /* Re-configure 4 TCs BW */ 4220 for (i = 0; i < nb_tcs; i++) { 4221 tc = &dcb_config->tc_config[i]; 4222 if (bw_conf->tc_num != nb_tcs) 4223 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 4224 (uint8_t)(100 / nb_tcs); 4225 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 4226 (uint8_t)(100 / nb_tcs); 4227 } 4228 for (; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) { 4229 tc = &dcb_config->tc_config[i]; 4230 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 0; 4231 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 0; 4232 } 4233 } else { 4234 /* Re-configure 8 TCs BW */ 4235 for (i = 0; i < nb_tcs; i++) { 4236 tc = &dcb_config->tc_config[i]; 4237 if (bw_conf->tc_num != nb_tcs) 4238 tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent = 4239 (uint8_t)(100 / nb_tcs + (i & 1)); 4240 tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent = 4241 (uint8_t)(100 / nb_tcs + (i & 1)); 4242 } 4243 } 4244 4245 switch (hw->mac.type) { 4246 case ixgbe_mac_X550: 4247 case ixgbe_mac_X550EM_x: 4248 case ixgbe_mac_X550EM_a: 4249 rx_buffer_size = X550_RX_BUFFER_SIZE; 4250 break; 4251 default: 4252 rx_buffer_size = NIC_RX_BUFFER_SIZE; 4253 break; 4254 } 4255 4256 if (config_dcb_rx) { 4257 /* Set RX buffer size */ 4258 pbsize = (uint16_t)(rx_buffer_size / nb_tcs); 4259 uint32_t rxpbsize = pbsize << IXGBE_RXPBSIZE_SHIFT; 4260 4261 for (i = 0; i < nb_tcs; i++) { 4262 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), rxpbsize); 4263 } 4264 /* zero alloc all unused TCs */ 4265 for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 4266 IXGBE_WRITE_REG(hw, IXGBE_RXPBSIZE(i), 0); 4267 } 4268 } 4269 if (config_dcb_tx) { 4270 /* Only support an equally distributed 4271 * Tx packet buffer strategy. 4272 */ 4273 uint32_t txpktsize = IXGBE_TXPBSIZE_MAX / nb_tcs; 4274 uint32_t txpbthresh = (txpktsize / DCB_TX_PB) - IXGBE_TXPKT_SIZE_MAX; 4275 4276 for (i = 0; i < nb_tcs; i++) { 4277 IXGBE_WRITE_REG(hw, IXGBE_TXPBSIZE(i), txpktsize); 4278 IXGBE_WRITE_REG(hw, IXGBE_TXPBTHRESH(i), txpbthresh); 4279 } 4280 /* Clear unused TCs, if any, to zero buffer size*/ 4281 for (; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 4282 IXGBE_WRITE_REG(hw, IXGBE_TXPBSIZE(i), 0); 4283 IXGBE_WRITE_REG(hw, IXGBE_TXPBTHRESH(i), 0); 4284 } 4285 } 4286 4287 /*Calculates traffic class credits*/ 4288 ixgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame, 4289 IXGBE_DCB_TX_CONFIG); 4290 ixgbe_dcb_calculate_tc_credits_cee(hw, dcb_config, max_frame, 4291 IXGBE_DCB_RX_CONFIG); 4292 4293 if (config_dcb_rx) { 4294 /* Unpack CEE standard containers */ 4295 ixgbe_dcb_unpack_refill_cee(dcb_config, IXGBE_DCB_RX_CONFIG, refill); 4296 ixgbe_dcb_unpack_max_cee(dcb_config, max); 4297 ixgbe_dcb_unpack_bwgid_cee(dcb_config, IXGBE_DCB_RX_CONFIG, bwgid); 4298 ixgbe_dcb_unpack_tsa_cee(dcb_config, IXGBE_DCB_RX_CONFIG, tsa); 4299 /* Configure PG(ETS) RX */ 4300 ixgbe_dcb_hw_arbite_rx_config(hw, refill, max, bwgid, tsa, map); 4301 } 4302 4303 if (config_dcb_tx) { 4304 /* Unpack CEE standard containers */ 4305 ixgbe_dcb_unpack_refill_cee(dcb_config, IXGBE_DCB_TX_CONFIG, refill); 4306 ixgbe_dcb_unpack_max_cee(dcb_config, max); 4307 ixgbe_dcb_unpack_bwgid_cee(dcb_config, IXGBE_DCB_TX_CONFIG, bwgid); 4308 ixgbe_dcb_unpack_tsa_cee(dcb_config, IXGBE_DCB_TX_CONFIG, tsa); 4309 /* Configure PG(ETS) TX */ 4310 ixgbe_dcb_hw_arbite_tx_config(hw, refill, max, bwgid, tsa, map); 4311 } 4312 4313 /*Configure queue statistics registers*/ 4314 ixgbe_dcb_config_tc_stats_82599(hw, dcb_config); 4315 4316 /* Check if the PFC is supported */ 4317 if (dev->data->dev_conf.dcb_capability_en & ETH_DCB_PFC_SUPPORT) { 4318 pbsize = (uint16_t)(rx_buffer_size / nb_tcs); 4319 for (i = 0; i < nb_tcs; i++) { 4320 /* 4321 * If the TC count is 8,and the default high_water is 48, 4322 * the low_water is 16 as default. 4323 */ 4324 hw->fc.high_water[i] = (pbsize * 3) / 4; 4325 hw->fc.low_water[i] = pbsize / 4; 4326 /* Enable pfc for this TC */ 4327 tc = &dcb_config->tc_config[i]; 4328 tc->pfc = ixgbe_dcb_pfc_enabled; 4329 } 4330 ixgbe_dcb_unpack_pfc_cee(dcb_config, map, &pfc_en); 4331 if (dcb_config->num_tcs.pfc_tcs == ETH_4_TCS) 4332 pfc_en &= 0x0F; 4333 ret = ixgbe_dcb_config_pfc(hw, pfc_en, map); 4334 } 4335 4336 return ret; 4337 } 4338 4339 /** 4340 * ixgbe_configure_dcb - Configure DCB Hardware 4341 * @dev: pointer to rte_eth_dev 4342 */ 4343 void ixgbe_configure_dcb(struct rte_eth_dev *dev) 4344 { 4345 struct ixgbe_dcb_config *dcb_cfg = 4346 IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private); 4347 struct rte_eth_conf *dev_conf = &(dev->data->dev_conf); 4348 4349 PMD_INIT_FUNC_TRACE(); 4350 4351 /* check support mq_mode for DCB */ 4352 if ((dev_conf->rxmode.mq_mode != ETH_MQ_RX_VMDQ_DCB) && 4353 (dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB) && 4354 (dev_conf->rxmode.mq_mode != ETH_MQ_RX_DCB_RSS)) 4355 return; 4356 4357 if (dev->data->nb_rx_queues > ETH_DCB_NUM_QUEUES) 4358 return; 4359 4360 /** Configure DCB hardware **/ 4361 ixgbe_dcb_hw_configure(dev, dcb_cfg); 4362 } 4363 4364 /* 4365 * VMDq only support for 10 GbE NIC. 4366 */ 4367 static void 4368 ixgbe_vmdq_rx_hw_configure(struct rte_eth_dev *dev) 4369 { 4370 struct rte_eth_vmdq_rx_conf *cfg; 4371 struct ixgbe_hw *hw; 4372 enum rte_eth_nb_pools num_pools; 4373 uint32_t mrqc, vt_ctl, vlanctrl; 4374 uint32_t vmolr = 0; 4375 int i; 4376 4377 PMD_INIT_FUNC_TRACE(); 4378 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4379 cfg = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf; 4380 num_pools = cfg->nb_queue_pools; 4381 4382 ixgbe_rss_disable(dev); 4383 4384 /* MRQC: enable vmdq */ 4385 mrqc = IXGBE_MRQC_VMDQEN; 4386 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 4387 4388 /* PFVTCTL: turn on virtualisation and set the default pool */ 4389 vt_ctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN; 4390 if (cfg->enable_default_pool) 4391 vt_ctl |= (cfg->default_pool << IXGBE_VT_CTL_POOL_SHIFT); 4392 else 4393 vt_ctl |= IXGBE_VT_CTL_DIS_DEFPL; 4394 4395 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vt_ctl); 4396 4397 for (i = 0; i < (int)num_pools; i++) { 4398 vmolr = ixgbe_convert_vm_rx_mask_to_val(cfg->rx_mode, vmolr); 4399 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(i), vmolr); 4400 } 4401 4402 /* VLNCTRL: enable vlan filtering and allow all vlan tags through */ 4403 vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 4404 vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */ 4405 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl); 4406 4407 /* VFTA - enable all vlan filters */ 4408 for (i = 0; i < NUM_VFTA_REGISTERS; i++) 4409 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), UINT32_MAX); 4410 4411 /* VFRE: pool enabling for receive - 64 */ 4412 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), UINT32_MAX); 4413 if (num_pools == ETH_64_POOLS) 4414 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), UINT32_MAX); 4415 4416 /* 4417 * MPSAR - allow pools to read specific mac addresses 4418 * In this case, all pools should be able to read from mac addr 0 4419 */ 4420 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(0), UINT32_MAX); 4421 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(0), UINT32_MAX); 4422 4423 /* PFVLVF, PFVLVFB: set up filters for vlan tags as configured */ 4424 for (i = 0; i < cfg->nb_pool_maps; i++) { 4425 /* set vlan id in VF register and set the valid bit */ 4426 IXGBE_WRITE_REG(hw, IXGBE_VLVF(i), (IXGBE_VLVF_VIEN | 4427 (cfg->pool_map[i].vlan_id & IXGBE_RXD_VLAN_ID_MASK))); 4428 /* 4429 * Put the allowed pools in VFB reg. As we only have 16 or 64 4430 * pools, we only need to use the first half of the register 4431 * i.e. bits 0-31 4432 */ 4433 if (((cfg->pool_map[i].pools >> 32) & UINT32_MAX) == 0) 4434 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(i * 2), 4435 (cfg->pool_map[i].pools & UINT32_MAX)); 4436 else 4437 IXGBE_WRITE_REG(hw, IXGBE_VLVFB((i * 2 + 1)), 4438 ((cfg->pool_map[i].pools >> 32) & UINT32_MAX)); 4439 4440 } 4441 4442 /* PFDMA Tx General Switch Control Enables VMDQ loopback */ 4443 if (cfg->enable_loop_back) { 4444 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN); 4445 for (i = 0; i < RTE_IXGBE_VMTXSW_REGISTER_COUNT; i++) 4446 IXGBE_WRITE_REG(hw, IXGBE_VMTXSW(i), UINT32_MAX); 4447 } 4448 4449 IXGBE_WRITE_FLUSH(hw); 4450 } 4451 4452 /* 4453 * ixgbe_dcb_config_tx_hw_config - Configure general VMDq TX parameters 4454 * @hw: pointer to hardware structure 4455 */ 4456 static void 4457 ixgbe_vmdq_tx_hw_configure(struct ixgbe_hw *hw) 4458 { 4459 uint32_t reg; 4460 uint32_t q; 4461 4462 PMD_INIT_FUNC_TRACE(); 4463 /*PF VF Transmit Enable*/ 4464 IXGBE_WRITE_REG(hw, IXGBE_VFTE(0), UINT32_MAX); 4465 IXGBE_WRITE_REG(hw, IXGBE_VFTE(1), UINT32_MAX); 4466 4467 /* Disable the Tx desc arbiter so that MTQC can be changed */ 4468 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 4469 reg |= IXGBE_RTTDCS_ARBDIS; 4470 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg); 4471 4472 reg = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF; 4473 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg); 4474 4475 /* Disable drop for all queues */ 4476 for (q = 0; q < IXGBE_MAX_RX_QUEUE_NUM; q++) 4477 IXGBE_WRITE_REG(hw, IXGBE_QDE, 4478 (IXGBE_QDE_WRITE | (q << IXGBE_QDE_IDX_SHIFT))); 4479 4480 /* Enable the Tx desc arbiter */ 4481 reg = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 4482 reg &= ~IXGBE_RTTDCS_ARBDIS; 4483 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg); 4484 4485 IXGBE_WRITE_FLUSH(hw); 4486 } 4487 4488 static int __rte_cold 4489 ixgbe_alloc_rx_queue_mbufs(struct ixgbe_rx_queue *rxq) 4490 { 4491 struct ixgbe_rx_entry *rxe = rxq->sw_ring; 4492 uint64_t dma_addr; 4493 unsigned int i; 4494 4495 /* Initialize software ring entries */ 4496 for (i = 0; i < rxq->nb_rx_desc; i++) { 4497 volatile union ixgbe_adv_rx_desc *rxd; 4498 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool); 4499 4500 if (mbuf == NULL) { 4501 PMD_INIT_LOG(ERR, "RX mbuf alloc failed queue_id=%u", 4502 (unsigned) rxq->queue_id); 4503 return -ENOMEM; 4504 } 4505 4506 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 4507 mbuf->port = rxq->port_id; 4508 4509 dma_addr = 4510 rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 4511 rxd = &rxq->rx_ring[i]; 4512 rxd->read.hdr_addr = 0; 4513 rxd->read.pkt_addr = dma_addr; 4514 rxe[i].mbuf = mbuf; 4515 } 4516 4517 return 0; 4518 } 4519 4520 static int 4521 ixgbe_config_vf_rss(struct rte_eth_dev *dev) 4522 { 4523 struct ixgbe_hw *hw; 4524 uint32_t mrqc; 4525 4526 ixgbe_rss_configure(dev); 4527 4528 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4529 4530 /* MRQC: enable VF RSS */ 4531 mrqc = IXGBE_READ_REG(hw, IXGBE_MRQC); 4532 mrqc &= ~IXGBE_MRQC_MRQE_MASK; 4533 switch (RTE_ETH_DEV_SRIOV(dev).active) { 4534 case ETH_64_POOLS: 4535 mrqc |= IXGBE_MRQC_VMDQRSS64EN; 4536 break; 4537 4538 case ETH_32_POOLS: 4539 mrqc |= IXGBE_MRQC_VMDQRSS32EN; 4540 break; 4541 4542 default: 4543 PMD_INIT_LOG(ERR, "Invalid pool number in IOV mode with VMDQ RSS"); 4544 return -EINVAL; 4545 } 4546 4547 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 4548 4549 return 0; 4550 } 4551 4552 static int 4553 ixgbe_config_vf_default(struct rte_eth_dev *dev) 4554 { 4555 struct ixgbe_hw *hw = 4556 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4557 4558 switch (RTE_ETH_DEV_SRIOV(dev).active) { 4559 case ETH_64_POOLS: 4560 IXGBE_WRITE_REG(hw, IXGBE_MRQC, 4561 IXGBE_MRQC_VMDQEN); 4562 break; 4563 4564 case ETH_32_POOLS: 4565 IXGBE_WRITE_REG(hw, IXGBE_MRQC, 4566 IXGBE_MRQC_VMDQRT4TCEN); 4567 break; 4568 4569 case ETH_16_POOLS: 4570 IXGBE_WRITE_REG(hw, IXGBE_MRQC, 4571 IXGBE_MRQC_VMDQRT8TCEN); 4572 break; 4573 default: 4574 PMD_INIT_LOG(ERR, 4575 "invalid pool number in IOV mode"); 4576 break; 4577 } 4578 return 0; 4579 } 4580 4581 static int 4582 ixgbe_dev_mq_rx_configure(struct rte_eth_dev *dev) 4583 { 4584 struct ixgbe_hw *hw = 4585 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4586 4587 if (hw->mac.type == ixgbe_mac_82598EB) 4588 return 0; 4589 4590 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 4591 /* 4592 * SRIOV inactive scheme 4593 * any DCB/RSS w/o VMDq multi-queue setting 4594 */ 4595 switch (dev->data->dev_conf.rxmode.mq_mode) { 4596 case ETH_MQ_RX_RSS: 4597 case ETH_MQ_RX_DCB_RSS: 4598 case ETH_MQ_RX_VMDQ_RSS: 4599 ixgbe_rss_configure(dev); 4600 break; 4601 4602 case ETH_MQ_RX_VMDQ_DCB: 4603 ixgbe_vmdq_dcb_configure(dev); 4604 break; 4605 4606 case ETH_MQ_RX_VMDQ_ONLY: 4607 ixgbe_vmdq_rx_hw_configure(dev); 4608 break; 4609 4610 case ETH_MQ_RX_NONE: 4611 default: 4612 /* if mq_mode is none, disable rss mode.*/ 4613 ixgbe_rss_disable(dev); 4614 break; 4615 } 4616 } else { 4617 /* SRIOV active scheme 4618 * Support RSS together with SRIOV. 4619 */ 4620 switch (dev->data->dev_conf.rxmode.mq_mode) { 4621 case ETH_MQ_RX_RSS: 4622 case ETH_MQ_RX_VMDQ_RSS: 4623 ixgbe_config_vf_rss(dev); 4624 break; 4625 case ETH_MQ_RX_VMDQ_DCB: 4626 case ETH_MQ_RX_DCB: 4627 /* In SRIOV, the configuration is the same as VMDq case */ 4628 ixgbe_vmdq_dcb_configure(dev); 4629 break; 4630 /* DCB/RSS together with SRIOV is not supported */ 4631 case ETH_MQ_RX_VMDQ_DCB_RSS: 4632 case ETH_MQ_RX_DCB_RSS: 4633 PMD_INIT_LOG(ERR, 4634 "Could not support DCB/RSS with VMDq & SRIOV"); 4635 return -1; 4636 default: 4637 ixgbe_config_vf_default(dev); 4638 break; 4639 } 4640 } 4641 4642 return 0; 4643 } 4644 4645 static int 4646 ixgbe_dev_mq_tx_configure(struct rte_eth_dev *dev) 4647 { 4648 struct ixgbe_hw *hw = 4649 IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4650 uint32_t mtqc; 4651 uint32_t rttdcs; 4652 4653 if (hw->mac.type == ixgbe_mac_82598EB) 4654 return 0; 4655 4656 /* disable arbiter before setting MTQC */ 4657 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 4658 rttdcs |= IXGBE_RTTDCS_ARBDIS; 4659 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 4660 4661 if (RTE_ETH_DEV_SRIOV(dev).active == 0) { 4662 /* 4663 * SRIOV inactive scheme 4664 * any DCB w/o VMDq multi-queue setting 4665 */ 4666 if (dev->data->dev_conf.txmode.mq_mode == ETH_MQ_TX_VMDQ_ONLY) 4667 ixgbe_vmdq_tx_hw_configure(hw); 4668 else { 4669 mtqc = IXGBE_MTQC_64Q_1PB; 4670 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc); 4671 } 4672 } else { 4673 switch (RTE_ETH_DEV_SRIOV(dev).active) { 4674 4675 /* 4676 * SRIOV active scheme 4677 * FIXME if support DCB together with VMDq & SRIOV 4678 */ 4679 case ETH_64_POOLS: 4680 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF; 4681 break; 4682 case ETH_32_POOLS: 4683 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_32VF; 4684 break; 4685 case ETH_16_POOLS: 4686 mtqc = IXGBE_MTQC_VT_ENA | IXGBE_MTQC_RT_ENA | 4687 IXGBE_MTQC_8TC_8TQ; 4688 break; 4689 default: 4690 mtqc = IXGBE_MTQC_64Q_1PB; 4691 PMD_INIT_LOG(ERR, "invalid pool number in IOV mode"); 4692 } 4693 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc); 4694 } 4695 4696 /* re-enable arbiter */ 4697 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 4698 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 4699 4700 return 0; 4701 } 4702 4703 /** 4704 * ixgbe_get_rscctl_maxdesc - Calculate the RSCCTL[n].MAXDESC for PF 4705 * 4706 * Return the RSCCTL[n].MAXDESC for 82599 and x540 PF devices according to the 4707 * spec rev. 3.0 chapter 8.2.3.8.13. 4708 * 4709 * @pool Memory pool of the Rx queue 4710 */ 4711 static inline uint32_t 4712 ixgbe_get_rscctl_maxdesc(struct rte_mempool *pool) 4713 { 4714 struct rte_pktmbuf_pool_private *mp_priv = rte_mempool_get_priv(pool); 4715 4716 /* MAXDESC * SRRCTL.BSIZEPKT must not exceed 64 KB minus one */ 4717 uint16_t maxdesc = 4718 RTE_IPV4_MAX_PKT_LEN / 4719 (mp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM); 4720 4721 if (maxdesc >= 16) 4722 return IXGBE_RSCCTL_MAXDESC_16; 4723 else if (maxdesc >= 8) 4724 return IXGBE_RSCCTL_MAXDESC_8; 4725 else if (maxdesc >= 4) 4726 return IXGBE_RSCCTL_MAXDESC_4; 4727 else 4728 return IXGBE_RSCCTL_MAXDESC_1; 4729 } 4730 4731 /** 4732 * ixgbe_set_ivar - Setup the correct IVAR register for a particular MSIX 4733 * interrupt 4734 * 4735 * (Taken from FreeBSD tree) 4736 * (yes this is all very magic and confusing :) 4737 * 4738 * @dev port handle 4739 * @entry the register array entry 4740 * @vector the MSIX vector for this queue 4741 * @type RX/TX/MISC 4742 */ 4743 static void 4744 ixgbe_set_ivar(struct rte_eth_dev *dev, u8 entry, u8 vector, s8 type) 4745 { 4746 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4747 u32 ivar, index; 4748 4749 vector |= IXGBE_IVAR_ALLOC_VAL; 4750 4751 switch (hw->mac.type) { 4752 4753 case ixgbe_mac_82598EB: 4754 if (type == -1) 4755 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 4756 else 4757 entry += (type * 64); 4758 index = (entry >> 2) & 0x1F; 4759 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4760 ivar &= ~(0xFF << (8 * (entry & 0x3))); 4761 ivar |= (vector << (8 * (entry & 0x3))); 4762 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 4763 break; 4764 4765 case ixgbe_mac_82599EB: 4766 case ixgbe_mac_X540: 4767 if (type == -1) { /* MISC IVAR */ 4768 index = (entry & 1) * 8; 4769 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4770 ivar &= ~(0xFF << index); 4771 ivar |= (vector << index); 4772 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4773 } else { /* RX/TX IVARS */ 4774 index = (16 * (entry & 1)) + (8 * type); 4775 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 4776 ivar &= ~(0xFF << index); 4777 ivar |= (vector << index); 4778 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 4779 } 4780 4781 break; 4782 4783 default: 4784 break; 4785 } 4786 } 4787 4788 void __rte_cold 4789 ixgbe_set_rx_function(struct rte_eth_dev *dev) 4790 { 4791 uint16_t i, rx_using_sse; 4792 struct ixgbe_adapter *adapter = dev->data->dev_private; 4793 4794 /* 4795 * In order to allow Vector Rx there are a few configuration 4796 * conditions to be met and Rx Bulk Allocation should be allowed. 4797 */ 4798 if (ixgbe_rx_vec_dev_conf_condition_check(dev) || 4799 !adapter->rx_bulk_alloc_allowed || 4800 rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) { 4801 PMD_INIT_LOG(DEBUG, "Port[%d] doesn't meet Vector Rx " 4802 "preconditions", 4803 dev->data->port_id); 4804 4805 adapter->rx_vec_allowed = false; 4806 } 4807 4808 /* 4809 * Initialize the appropriate LRO callback. 4810 * 4811 * If all queues satisfy the bulk allocation preconditions 4812 * (hw->rx_bulk_alloc_allowed is TRUE) then we may use bulk allocation. 4813 * Otherwise use a single allocation version. 4814 */ 4815 if (dev->data->lro) { 4816 if (adapter->rx_bulk_alloc_allowed) { 4817 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a bulk " 4818 "allocation version"); 4819 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_bulk_alloc; 4820 } else { 4821 PMD_INIT_LOG(DEBUG, "LRO is requested. Using a single " 4822 "allocation version"); 4823 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc; 4824 } 4825 } else if (dev->data->scattered_rx) { 4826 /* 4827 * Set the non-LRO scattered callback: there are Vector and 4828 * single allocation versions. 4829 */ 4830 if (adapter->rx_vec_allowed) { 4831 PMD_INIT_LOG(DEBUG, "Using Vector Scattered Rx " 4832 "callback (port=%d).", 4833 dev->data->port_id); 4834 4835 dev->rx_pkt_burst = ixgbe_recv_scattered_pkts_vec; 4836 } else if (adapter->rx_bulk_alloc_allowed) { 4837 PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk " 4838 "allocation callback (port=%d).", 4839 dev->data->port_id); 4840 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_bulk_alloc; 4841 } else { 4842 PMD_INIT_LOG(DEBUG, "Using Regualr (non-vector, " 4843 "single allocation) " 4844 "Scattered Rx callback " 4845 "(port=%d).", 4846 dev->data->port_id); 4847 4848 dev->rx_pkt_burst = ixgbe_recv_pkts_lro_single_alloc; 4849 } 4850 /* 4851 * Below we set "simple" callbacks according to port/queues parameters. 4852 * If parameters allow we are going to choose between the following 4853 * callbacks: 4854 * - Vector 4855 * - Bulk Allocation 4856 * - Single buffer allocation (the simplest one) 4857 */ 4858 } else if (adapter->rx_vec_allowed) { 4859 PMD_INIT_LOG(DEBUG, "Vector rx enabled, please make sure RX " 4860 "burst size no less than %d (port=%d).", 4861 RTE_IXGBE_DESCS_PER_LOOP, 4862 dev->data->port_id); 4863 4864 dev->rx_pkt_burst = ixgbe_recv_pkts_vec; 4865 } else if (adapter->rx_bulk_alloc_allowed) { 4866 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " 4867 "satisfied. Rx Burst Bulk Alloc function " 4868 "will be used on port=%d.", 4869 dev->data->port_id); 4870 4871 dev->rx_pkt_burst = ixgbe_recv_pkts_bulk_alloc; 4872 } else { 4873 PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not " 4874 "satisfied, or Scattered Rx is requested " 4875 "(port=%d).", 4876 dev->data->port_id); 4877 4878 dev->rx_pkt_burst = ixgbe_recv_pkts; 4879 } 4880 4881 /* Propagate information about RX function choice through all queues. */ 4882 4883 rx_using_sse = 4884 (dev->rx_pkt_burst == ixgbe_recv_scattered_pkts_vec || 4885 dev->rx_pkt_burst == ixgbe_recv_pkts_vec); 4886 4887 for (i = 0; i < dev->data->nb_rx_queues; i++) { 4888 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i]; 4889 4890 rxq->rx_using_sse = rx_using_sse; 4891 #ifdef RTE_LIB_SECURITY 4892 rxq->using_ipsec = !!(dev->data->dev_conf.rxmode.offloads & 4893 DEV_RX_OFFLOAD_SECURITY); 4894 #endif 4895 } 4896 } 4897 4898 /** 4899 * ixgbe_set_rsc - configure RSC related port HW registers 4900 * 4901 * Configures the port's RSC related registers according to the 4.6.7.2 chapter 4902 * of 82599 Spec (x540 configuration is virtually the same). 4903 * 4904 * @dev port handle 4905 * 4906 * Returns 0 in case of success or a non-zero error code 4907 */ 4908 static int 4909 ixgbe_set_rsc(struct rte_eth_dev *dev) 4910 { 4911 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; 4912 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 4913 struct rte_eth_dev_info dev_info = { 0 }; 4914 bool rsc_capable = false; 4915 uint16_t i; 4916 uint32_t rdrxctl; 4917 uint32_t rfctl; 4918 4919 /* Sanity check */ 4920 dev->dev_ops->dev_infos_get(dev, &dev_info); 4921 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO) 4922 rsc_capable = true; 4923 4924 if (!rsc_capable && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) { 4925 PMD_INIT_LOG(CRIT, "LRO is requested on HW that doesn't " 4926 "support it"); 4927 return -EINVAL; 4928 } 4929 4930 /* RSC global configuration (chapter 4.6.7.2.1 of 82599 Spec) */ 4931 4932 if ((rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC) && 4933 (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) { 4934 /* 4935 * According to chapter of 4.6.7.2.1 of the Spec Rev. 4936 * 3.0 RSC configuration requires HW CRC stripping being 4937 * enabled. If user requested both HW CRC stripping off 4938 * and RSC on - return an error. 4939 */ 4940 PMD_INIT_LOG(CRIT, "LRO can't be enabled when HW CRC " 4941 "is disabled"); 4942 return -EINVAL; 4943 } 4944 4945 /* RFCTL configuration */ 4946 rfctl = IXGBE_READ_REG(hw, IXGBE_RFCTL); 4947 if ((rsc_capable) && (rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) 4948 rfctl &= ~IXGBE_RFCTL_RSC_DIS; 4949 else 4950 rfctl |= IXGBE_RFCTL_RSC_DIS; 4951 /* disable NFS filtering */ 4952 rfctl |= IXGBE_RFCTL_NFSW_DIS | IXGBE_RFCTL_NFSR_DIS; 4953 IXGBE_WRITE_REG(hw, IXGBE_RFCTL, rfctl); 4954 4955 /* If LRO hasn't been requested - we are done here. */ 4956 if (!(rx_conf->offloads & DEV_RX_OFFLOAD_TCP_LRO)) 4957 return 0; 4958 4959 /* Set RDRXCTL.RSCACKC bit */ 4960 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 4961 rdrxctl |= IXGBE_RDRXCTL_RSCACKC; 4962 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 4963 4964 /* Per-queue RSC configuration (chapter 4.6.7.2.2 of 82599 Spec) */ 4965 for (i = 0; i < dev->data->nb_rx_queues; i++) { 4966 struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i]; 4967 uint32_t srrctl = 4968 IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxq->reg_idx)); 4969 uint32_t rscctl = 4970 IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxq->reg_idx)); 4971 uint32_t psrtype = 4972 IXGBE_READ_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx)); 4973 uint32_t eitr = 4974 IXGBE_READ_REG(hw, IXGBE_EITR(rxq->reg_idx)); 4975 4976 /* 4977 * ixgbe PMD doesn't support header-split at the moment. 4978 * 4979 * Following the 4.6.7.2.1 chapter of the 82599/x540 4980 * Spec if RSC is enabled the SRRCTL[n].BSIZEHEADER 4981 * should be configured even if header split is not 4982 * enabled. We will configure it 128 bytes following the 4983 * recommendation in the spec. 4984 */ 4985 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 4986 srrctl |= (128 << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) & 4987 IXGBE_SRRCTL_BSIZEHDR_MASK; 4988 4989 /* 4990 * TODO: Consider setting the Receive Descriptor Minimum 4991 * Threshold Size for an RSC case. This is not an obviously 4992 * beneficiary option but the one worth considering... 4993 */ 4994 4995 rscctl |= IXGBE_RSCCTL_RSCEN; 4996 rscctl |= ixgbe_get_rscctl_maxdesc(rxq->mb_pool); 4997 psrtype |= IXGBE_PSRTYPE_TCPHDR; 4998 4999 /* 5000 * RSC: Set ITR interval corresponding to 2K ints/s. 5001 * 5002 * Full-sized RSC aggregations for a 10Gb/s link will 5003 * arrive at about 20K aggregation/s rate. 5004 * 5005 * 2K inst/s rate will make only 10% of the 5006 * aggregations to be closed due to the interrupt timer 5007 * expiration for a streaming at wire-speed case. 5008 * 5009 * For a sparse streaming case this setting will yield 5010 * at most 500us latency for a single RSC aggregation. 5011 */ 5012 eitr &= ~IXGBE_EITR_ITR_INT_MASK; 5013 eitr |= IXGBE_EITR_INTERVAL_US(IXGBE_QUEUE_ITR_INTERVAL_DEFAULT); 5014 eitr |= IXGBE_EITR_CNT_WDIS; 5015 5016 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxq->reg_idx), srrctl); 5017 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxq->reg_idx), rscctl); 5018 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(rxq->reg_idx), psrtype); 5019 IXGBE_WRITE_REG(hw, IXGBE_EITR(rxq->reg_idx), eitr); 5020 5021 /* 5022 * RSC requires the mapping of the queue to the 5023 * interrupt vector. 5024 */ 5025 ixgbe_set_ivar(dev, rxq->reg_idx, i, 0); 5026 } 5027 5028 dev->data->lro = 1; 5029 5030 PMD_INIT_LOG(DEBUG, "enabling LRO mode"); 5031 5032 return 0; 5033 } 5034 5035 /* 5036 * Initializes Receive Unit. 5037 */ 5038 int __rte_cold 5039 ixgbe_dev_rx_init(struct rte_eth_dev *dev) 5040 { 5041 struct ixgbe_hw *hw; 5042 struct ixgbe_rx_queue *rxq; 5043 uint64_t bus_addr; 5044 uint32_t rxctrl; 5045 uint32_t fctrl; 5046 uint32_t hlreg0; 5047 uint32_t maxfrs; 5048 uint32_t srrctl; 5049 uint32_t rdrxctl; 5050 uint32_t rxcsum; 5051 uint16_t buf_size; 5052 uint16_t i; 5053 struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; 5054 int rc; 5055 5056 PMD_INIT_FUNC_TRACE(); 5057 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5058 5059 /* 5060 * Make sure receives are disabled while setting 5061 * up the RX context (registers, descriptor rings, etc.). 5062 */ 5063 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 5064 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN); 5065 5066 /* Enable receipt of broadcasted frames */ 5067 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 5068 fctrl |= IXGBE_FCTRL_BAM; 5069 fctrl |= IXGBE_FCTRL_DPF; 5070 fctrl |= IXGBE_FCTRL_PMCF; 5071 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 5072 5073 /* 5074 * Configure CRC stripping, if any. 5075 */ 5076 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); 5077 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC) 5078 hlreg0 &= ~IXGBE_HLREG0_RXCRCSTRP; 5079 else 5080 hlreg0 |= IXGBE_HLREG0_RXCRCSTRP; 5081 5082 /* 5083 * Configure jumbo frame support, if any. 5084 */ 5085 if (rx_conf->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) { 5086 hlreg0 |= IXGBE_HLREG0_JUMBOEN; 5087 maxfrs = IXGBE_READ_REG(hw, IXGBE_MAXFRS); 5088 maxfrs &= 0x0000FFFF; 5089 maxfrs |= (rx_conf->max_rx_pkt_len << 16); 5090 IXGBE_WRITE_REG(hw, IXGBE_MAXFRS, maxfrs); 5091 } else 5092 hlreg0 &= ~IXGBE_HLREG0_JUMBOEN; 5093 5094 /* 5095 * If loopback mode is configured, set LPBK bit. 5096 */ 5097 if (dev->data->dev_conf.lpbk_mode != 0) { 5098 rc = ixgbe_check_supported_loopback_mode(dev); 5099 if (rc < 0) { 5100 PMD_INIT_LOG(ERR, "Unsupported loopback mode"); 5101 return rc; 5102 } 5103 hlreg0 |= IXGBE_HLREG0_LPBK; 5104 } else { 5105 hlreg0 &= ~IXGBE_HLREG0_LPBK; 5106 } 5107 5108 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); 5109 5110 /* 5111 * Assume no header split and no VLAN strip support 5112 * on any Rx queue first . 5113 */ 5114 rx_conf->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP; 5115 /* Setup RX queues */ 5116 for (i = 0; i < dev->data->nb_rx_queues; i++) { 5117 rxq = dev->data->rx_queues[i]; 5118 5119 /* 5120 * Reset crc_len in case it was changed after queue setup by a 5121 * call to configure. 5122 */ 5123 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC) 5124 rxq->crc_len = RTE_ETHER_CRC_LEN; 5125 else 5126 rxq->crc_len = 0; 5127 5128 /* Setup the Base and Length of the Rx Descriptor Rings */ 5129 bus_addr = rxq->rx_ring_phys_addr; 5130 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rxq->reg_idx), 5131 (uint32_t)(bus_addr & 0x00000000ffffffffULL)); 5132 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rxq->reg_idx), 5133 (uint32_t)(bus_addr >> 32)); 5134 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rxq->reg_idx), 5135 rxq->nb_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 5136 IXGBE_WRITE_REG(hw, IXGBE_RDH(rxq->reg_idx), 0); 5137 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxq->reg_idx), 0); 5138 5139 /* Configure the SRRCTL register */ 5140 srrctl = IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 5141 5142 /* Set if packets are dropped when no descriptors available */ 5143 if (rxq->drop_en) 5144 srrctl |= IXGBE_SRRCTL_DROP_EN; 5145 5146 /* 5147 * Configure the RX buffer size in the BSIZEPACKET field of 5148 * the SRRCTL register of the queue. 5149 * The value is in 1 KB resolution. Valid values can be from 5150 * 1 KB to 16 KB. 5151 */ 5152 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - 5153 RTE_PKTMBUF_HEADROOM); 5154 srrctl |= ((buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) & 5155 IXGBE_SRRCTL_BSIZEPKT_MASK); 5156 5157 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxq->reg_idx), srrctl); 5158 5159 buf_size = (uint16_t) ((srrctl & IXGBE_SRRCTL_BSIZEPKT_MASK) << 5160 IXGBE_SRRCTL_BSIZEPKT_SHIFT); 5161 5162 /* It adds dual VLAN length for supporting dual VLAN */ 5163 if (dev->data->dev_conf.rxmode.max_rx_pkt_len + 5164 2 * IXGBE_VLAN_TAG_SIZE > buf_size) 5165 dev->data->scattered_rx = 1; 5166 if (rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 5167 rx_conf->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP; 5168 } 5169 5170 if (rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER) 5171 dev->data->scattered_rx = 1; 5172 5173 /* 5174 * Device configured with multiple RX queues. 5175 */ 5176 ixgbe_dev_mq_rx_configure(dev); 5177 5178 /* 5179 * Setup the Checksum Register. 5180 * Disable Full-Packet Checksum which is mutually exclusive with RSS. 5181 * Enable IP/L4 checkum computation by hardware if requested to do so. 5182 */ 5183 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 5184 rxcsum |= IXGBE_RXCSUM_PCSD; 5185 if (rx_conf->offloads & DEV_RX_OFFLOAD_CHECKSUM) 5186 rxcsum |= IXGBE_RXCSUM_IPPCSE; 5187 else 5188 rxcsum &= ~IXGBE_RXCSUM_IPPCSE; 5189 5190 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 5191 5192 if (hw->mac.type == ixgbe_mac_82599EB || 5193 hw->mac.type == ixgbe_mac_X540) { 5194 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 5195 if (rx_conf->offloads & DEV_RX_OFFLOAD_KEEP_CRC) 5196 rdrxctl &= ~IXGBE_RDRXCTL_CRCSTRIP; 5197 else 5198 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; 5199 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 5200 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 5201 } 5202 5203 rc = ixgbe_set_rsc(dev); 5204 if (rc) 5205 return rc; 5206 5207 ixgbe_set_rx_function(dev); 5208 5209 return 0; 5210 } 5211 5212 /* 5213 * Initializes Transmit Unit. 5214 */ 5215 void __rte_cold 5216 ixgbe_dev_tx_init(struct rte_eth_dev *dev) 5217 { 5218 struct ixgbe_hw *hw; 5219 struct ixgbe_tx_queue *txq; 5220 uint64_t bus_addr; 5221 uint32_t hlreg0; 5222 uint32_t txctrl; 5223 uint16_t i; 5224 5225 PMD_INIT_FUNC_TRACE(); 5226 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5227 5228 /* Enable TX CRC (checksum offload requirement) and hw padding 5229 * (TSO requirement) 5230 */ 5231 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); 5232 hlreg0 |= (IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN); 5233 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); 5234 5235 /* Setup the Base and Length of the Tx Descriptor Rings */ 5236 for (i = 0; i < dev->data->nb_tx_queues; i++) { 5237 txq = dev->data->tx_queues[i]; 5238 5239 bus_addr = txq->tx_ring_phys_addr; 5240 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(txq->reg_idx), 5241 (uint32_t)(bus_addr & 0x00000000ffffffffULL)); 5242 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(txq->reg_idx), 5243 (uint32_t)(bus_addr >> 32)); 5244 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(txq->reg_idx), 5245 txq->nb_tx_desc * sizeof(union ixgbe_adv_tx_desc)); 5246 /* Setup the HW Tx Head and TX Tail descriptor pointers */ 5247 IXGBE_WRITE_REG(hw, IXGBE_TDH(txq->reg_idx), 0); 5248 IXGBE_WRITE_REG(hw, IXGBE_TDT(txq->reg_idx), 0); 5249 5250 /* 5251 * Disable Tx Head Writeback RO bit, since this hoses 5252 * bookkeeping if things aren't delivered in order. 5253 */ 5254 switch (hw->mac.type) { 5255 case ixgbe_mac_82598EB: 5256 txctrl = IXGBE_READ_REG(hw, 5257 IXGBE_DCA_TXCTRL(txq->reg_idx)); 5258 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 5259 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(txq->reg_idx), 5260 txctrl); 5261 break; 5262 5263 case ixgbe_mac_82599EB: 5264 case ixgbe_mac_X540: 5265 case ixgbe_mac_X550: 5266 case ixgbe_mac_X550EM_x: 5267 case ixgbe_mac_X550EM_a: 5268 default: 5269 txctrl = IXGBE_READ_REG(hw, 5270 IXGBE_DCA_TXCTRL_82599(txq->reg_idx)); 5271 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 5272 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(txq->reg_idx), 5273 txctrl); 5274 break; 5275 } 5276 } 5277 5278 /* Device configured with multiple TX queues. */ 5279 ixgbe_dev_mq_tx_configure(dev); 5280 } 5281 5282 /* 5283 * Check if requested loopback mode is supported 5284 */ 5285 int 5286 ixgbe_check_supported_loopback_mode(struct rte_eth_dev *dev) 5287 { 5288 struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5289 5290 if (dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_TX_RX) 5291 if (hw->mac.type == ixgbe_mac_82599EB || 5292 hw->mac.type == ixgbe_mac_X540 || 5293 hw->mac.type == ixgbe_mac_X550 || 5294 hw->mac.type == ixgbe_mac_X550EM_x || 5295 hw->mac.type == ixgbe_mac_X550EM_a) 5296 return 0; 5297 5298 return -ENOTSUP; 5299 } 5300 5301 /* 5302 * Set up link for 82599 loopback mode Tx->Rx. 5303 */ 5304 static inline void __rte_cold 5305 ixgbe_setup_loopback_link_82599(struct ixgbe_hw *hw) 5306 { 5307 PMD_INIT_FUNC_TRACE(); 5308 5309 if (ixgbe_verify_lesm_fw_enabled_82599(hw)) { 5310 if (hw->mac.ops.acquire_swfw_sync(hw, IXGBE_GSSR_MAC_CSR_SM) != 5311 IXGBE_SUCCESS) { 5312 PMD_INIT_LOG(ERR, "Could not enable loopback mode"); 5313 /* ignore error */ 5314 return; 5315 } 5316 } 5317 5318 /* Restart link */ 5319 IXGBE_WRITE_REG(hw, 5320 IXGBE_AUTOC, 5321 IXGBE_AUTOC_LMS_10G_LINK_NO_AN | IXGBE_AUTOC_FLU); 5322 ixgbe_reset_pipeline_82599(hw); 5323 5324 hw->mac.ops.release_swfw_sync(hw, IXGBE_GSSR_MAC_CSR_SM); 5325 msec_delay(50); 5326 } 5327 5328 5329 /* 5330 * Start Transmit and Receive Units. 5331 */ 5332 int __rte_cold 5333 ixgbe_dev_rxtx_start(struct rte_eth_dev *dev) 5334 { 5335 struct ixgbe_hw *hw; 5336 struct ixgbe_tx_queue *txq; 5337 struct ixgbe_rx_queue *rxq; 5338 uint32_t txdctl; 5339 uint32_t dmatxctl; 5340 uint32_t rxctrl; 5341 uint16_t i; 5342 int ret = 0; 5343 5344 PMD_INIT_FUNC_TRACE(); 5345 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5346 5347 for (i = 0; i < dev->data->nb_tx_queues; i++) { 5348 txq = dev->data->tx_queues[i]; 5349 /* Setup Transmit Threshold Registers */ 5350 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx)); 5351 txdctl |= txq->pthresh & 0x7F; 5352 txdctl |= ((txq->hthresh & 0x7F) << 8); 5353 txdctl |= ((txq->wthresh & 0x7F) << 16); 5354 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl); 5355 } 5356 5357 if (hw->mac.type != ixgbe_mac_82598EB) { 5358 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 5359 dmatxctl |= IXGBE_DMATXCTL_TE; 5360 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 5361 } 5362 5363 for (i = 0; i < dev->data->nb_tx_queues; i++) { 5364 txq = dev->data->tx_queues[i]; 5365 if (!txq->tx_deferred_start) { 5366 ret = ixgbe_dev_tx_queue_start(dev, i); 5367 if (ret < 0) 5368 return ret; 5369 } 5370 } 5371 5372 for (i = 0; i < dev->data->nb_rx_queues; i++) { 5373 rxq = dev->data->rx_queues[i]; 5374 if (!rxq->rx_deferred_start) { 5375 ret = ixgbe_dev_rx_queue_start(dev, i); 5376 if (ret < 0) 5377 return ret; 5378 } 5379 } 5380 5381 /* Enable Receive engine */ 5382 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 5383 if (hw->mac.type == ixgbe_mac_82598EB) 5384 rxctrl |= IXGBE_RXCTRL_DMBYPS; 5385 rxctrl |= IXGBE_RXCTRL_RXEN; 5386 hw->mac.ops.enable_rx_dma(hw, rxctrl); 5387 5388 /* If loopback mode is enabled, set up the link accordingly */ 5389 if (dev->data->dev_conf.lpbk_mode != 0) { 5390 if (hw->mac.type == ixgbe_mac_82599EB) 5391 ixgbe_setup_loopback_link_82599(hw); 5392 else if (hw->mac.type == ixgbe_mac_X540 || 5393 hw->mac.type == ixgbe_mac_X550 || 5394 hw->mac.type == ixgbe_mac_X550EM_x || 5395 hw->mac.type == ixgbe_mac_X550EM_a) 5396 ixgbe_setup_loopback_link_x540_x550(hw, true); 5397 } 5398 5399 #ifdef RTE_LIB_SECURITY 5400 if ((dev->data->dev_conf.rxmode.offloads & 5401 DEV_RX_OFFLOAD_SECURITY) || 5402 (dev->data->dev_conf.txmode.offloads & 5403 DEV_TX_OFFLOAD_SECURITY)) { 5404 ret = ixgbe_crypto_enable_ipsec(dev); 5405 if (ret != 0) { 5406 PMD_DRV_LOG(ERR, 5407 "ixgbe_crypto_enable_ipsec fails with %d.", 5408 ret); 5409 return ret; 5410 } 5411 } 5412 #endif 5413 5414 return 0; 5415 } 5416 5417 /* 5418 * Start Receive Units for specified queue. 5419 */ 5420 int __rte_cold 5421 ixgbe_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) 5422 { 5423 struct ixgbe_hw *hw; 5424 struct ixgbe_rx_queue *rxq; 5425 uint32_t rxdctl; 5426 int poll_ms; 5427 5428 PMD_INIT_FUNC_TRACE(); 5429 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5430 5431 rxq = dev->data->rx_queues[rx_queue_id]; 5432 5433 /* Allocate buffers for descriptor rings */ 5434 if (ixgbe_alloc_rx_queue_mbufs(rxq) != 0) { 5435 PMD_INIT_LOG(ERR, "Could not alloc mbuf for queue:%d", 5436 rx_queue_id); 5437 return -1; 5438 } 5439 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx)); 5440 rxdctl |= IXGBE_RXDCTL_ENABLE; 5441 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl); 5442 5443 /* Wait until RX Enable ready */ 5444 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS; 5445 do { 5446 rte_delay_ms(1); 5447 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx)); 5448 } while (--poll_ms && !(rxdctl & IXGBE_RXDCTL_ENABLE)); 5449 if (!poll_ms) 5450 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", rx_queue_id); 5451 rte_wmb(); 5452 IXGBE_WRITE_REG(hw, IXGBE_RDH(rxq->reg_idx), 0); 5453 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxq->reg_idx), rxq->nb_rx_desc - 1); 5454 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 5455 5456 return 0; 5457 } 5458 5459 /* 5460 * Stop Receive Units for specified queue. 5461 */ 5462 int __rte_cold 5463 ixgbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) 5464 { 5465 struct ixgbe_hw *hw; 5466 struct ixgbe_adapter *adapter = dev->data->dev_private; 5467 struct ixgbe_rx_queue *rxq; 5468 uint32_t rxdctl; 5469 int poll_ms; 5470 5471 PMD_INIT_FUNC_TRACE(); 5472 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5473 5474 rxq = dev->data->rx_queues[rx_queue_id]; 5475 5476 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx)); 5477 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 5478 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), rxdctl); 5479 5480 /* Wait until RX Enable bit clear */ 5481 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS; 5482 do { 5483 rte_delay_ms(1); 5484 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx)); 5485 } while (--poll_ms && (rxdctl & IXGBE_RXDCTL_ENABLE)); 5486 if (!poll_ms) 5487 PMD_INIT_LOG(ERR, "Could not disable Rx Queue %d", rx_queue_id); 5488 5489 rte_delay_us(RTE_IXGBE_WAIT_100_US); 5490 5491 ixgbe_rx_queue_release_mbufs(rxq); 5492 ixgbe_reset_rx_queue(adapter, rxq); 5493 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 5494 5495 return 0; 5496 } 5497 5498 5499 /* 5500 * Start Transmit Units for specified queue. 5501 */ 5502 int __rte_cold 5503 ixgbe_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) 5504 { 5505 struct ixgbe_hw *hw; 5506 struct ixgbe_tx_queue *txq; 5507 uint32_t txdctl; 5508 int poll_ms; 5509 5510 PMD_INIT_FUNC_TRACE(); 5511 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5512 5513 txq = dev->data->tx_queues[tx_queue_id]; 5514 IXGBE_WRITE_REG(hw, IXGBE_TDH(txq->reg_idx), 0); 5515 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx)); 5516 txdctl |= IXGBE_TXDCTL_ENABLE; 5517 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl); 5518 5519 /* Wait until TX Enable ready */ 5520 if (hw->mac.type == ixgbe_mac_82599EB) { 5521 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS; 5522 do { 5523 rte_delay_ms(1); 5524 txdctl = IXGBE_READ_REG(hw, 5525 IXGBE_TXDCTL(txq->reg_idx)); 5526 } while (--poll_ms && !(txdctl & IXGBE_TXDCTL_ENABLE)); 5527 if (!poll_ms) 5528 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", 5529 tx_queue_id); 5530 } 5531 rte_wmb(); 5532 IXGBE_WRITE_REG(hw, IXGBE_TDT(txq->reg_idx), 0); 5533 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 5534 5535 return 0; 5536 } 5537 5538 /* 5539 * Stop Transmit Units for specified queue. 5540 */ 5541 int __rte_cold 5542 ixgbe_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) 5543 { 5544 struct ixgbe_hw *hw; 5545 struct ixgbe_tx_queue *txq; 5546 uint32_t txdctl; 5547 uint32_t txtdh, txtdt; 5548 int poll_ms; 5549 5550 PMD_INIT_FUNC_TRACE(); 5551 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5552 5553 txq = dev->data->tx_queues[tx_queue_id]; 5554 5555 /* Wait until TX queue is empty */ 5556 if (hw->mac.type == ixgbe_mac_82599EB) { 5557 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS; 5558 do { 5559 rte_delay_us(RTE_IXGBE_WAIT_100_US); 5560 txtdh = IXGBE_READ_REG(hw, 5561 IXGBE_TDH(txq->reg_idx)); 5562 txtdt = IXGBE_READ_REG(hw, 5563 IXGBE_TDT(txq->reg_idx)); 5564 } while (--poll_ms && (txtdh != txtdt)); 5565 if (!poll_ms) 5566 PMD_INIT_LOG(ERR, 5567 "Tx Queue %d is not empty when stopping.", 5568 tx_queue_id); 5569 } 5570 5571 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txq->reg_idx)); 5572 txdctl &= ~IXGBE_TXDCTL_ENABLE; 5573 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txq->reg_idx), txdctl); 5574 5575 /* Wait until TX Enable bit clear */ 5576 if (hw->mac.type == ixgbe_mac_82599EB) { 5577 poll_ms = RTE_IXGBE_REGISTER_POLL_WAIT_10_MS; 5578 do { 5579 rte_delay_ms(1); 5580 txdctl = IXGBE_READ_REG(hw, 5581 IXGBE_TXDCTL(txq->reg_idx)); 5582 } while (--poll_ms && (txdctl & IXGBE_TXDCTL_ENABLE)); 5583 if (!poll_ms) 5584 PMD_INIT_LOG(ERR, "Could not disable Tx Queue %d", 5585 tx_queue_id); 5586 } 5587 5588 if (txq->ops != NULL) { 5589 txq->ops->release_mbufs(txq); 5590 txq->ops->reset(txq); 5591 } 5592 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 5593 5594 return 0; 5595 } 5596 5597 void 5598 ixgbe_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 5599 struct rte_eth_rxq_info *qinfo) 5600 { 5601 struct ixgbe_rx_queue *rxq; 5602 5603 rxq = dev->data->rx_queues[queue_id]; 5604 5605 qinfo->mp = rxq->mb_pool; 5606 qinfo->scattered_rx = dev->data->scattered_rx; 5607 qinfo->nb_desc = rxq->nb_rx_desc; 5608 5609 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh; 5610 qinfo->conf.rx_drop_en = rxq->drop_en; 5611 qinfo->conf.rx_deferred_start = rxq->rx_deferred_start; 5612 qinfo->conf.offloads = rxq->offloads; 5613 } 5614 5615 void 5616 ixgbe_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 5617 struct rte_eth_txq_info *qinfo) 5618 { 5619 struct ixgbe_tx_queue *txq; 5620 5621 txq = dev->data->tx_queues[queue_id]; 5622 5623 qinfo->nb_desc = txq->nb_tx_desc; 5624 5625 qinfo->conf.tx_thresh.pthresh = txq->pthresh; 5626 qinfo->conf.tx_thresh.hthresh = txq->hthresh; 5627 qinfo->conf.tx_thresh.wthresh = txq->wthresh; 5628 5629 qinfo->conf.tx_free_thresh = txq->tx_free_thresh; 5630 qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh; 5631 qinfo->conf.offloads = txq->offloads; 5632 qinfo->conf.tx_deferred_start = txq->tx_deferred_start; 5633 } 5634 5635 /* 5636 * [VF] Initializes Receive Unit. 5637 */ 5638 int __rte_cold 5639 ixgbevf_dev_rx_init(struct rte_eth_dev *dev) 5640 { 5641 struct ixgbe_hw *hw; 5642 struct ixgbe_rx_queue *rxq; 5643 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 5644 uint64_t bus_addr; 5645 uint32_t srrctl, psrtype = 0; 5646 uint16_t buf_size; 5647 uint16_t i; 5648 int ret; 5649 5650 PMD_INIT_FUNC_TRACE(); 5651 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5652 5653 if (rte_is_power_of_2(dev->data->nb_rx_queues) == 0) { 5654 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, " 5655 "it should be power of 2"); 5656 return -1; 5657 } 5658 5659 if (dev->data->nb_rx_queues > hw->mac.max_rx_queues) { 5660 PMD_INIT_LOG(ERR, "The number of Rx queue invalid, " 5661 "it should be equal to or less than %d", 5662 hw->mac.max_rx_queues); 5663 return -1; 5664 } 5665 5666 /* 5667 * When the VF driver issues a IXGBE_VF_RESET request, the PF driver 5668 * disables the VF receipt of packets if the PF MTU is > 1500. 5669 * This is done to deal with 82599 limitations that imposes 5670 * the PF and all VFs to share the same MTU. 5671 * Then, the PF driver enables again the VF receipt of packet when 5672 * the VF driver issues a IXGBE_VF_SET_LPE request. 5673 * In the meantime, the VF device cannot be used, even if the VF driver 5674 * and the Guest VM network stack are ready to accept packets with a 5675 * size up to the PF MTU. 5676 * As a work-around to this PF behaviour, force the call to 5677 * ixgbevf_rlpml_set_vf even if jumbo frames are not used. This way, 5678 * VF packets received can work in all cases. 5679 */ 5680 if (ixgbevf_rlpml_set_vf(hw, 5681 (uint16_t)dev->data->dev_conf.rxmode.max_rx_pkt_len)) { 5682 PMD_INIT_LOG(ERR, "Set max packet length to %d failed.", 5683 dev->data->dev_conf.rxmode.max_rx_pkt_len); 5684 return -EINVAL; 5685 } 5686 5687 /* 5688 * Assume no header split and no VLAN strip support 5689 * on any Rx queue first . 5690 */ 5691 rxmode->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP; 5692 /* Setup RX queues */ 5693 for (i = 0; i < dev->data->nb_rx_queues; i++) { 5694 rxq = dev->data->rx_queues[i]; 5695 5696 /* Allocate buffers for descriptor rings */ 5697 ret = ixgbe_alloc_rx_queue_mbufs(rxq); 5698 if (ret) 5699 return ret; 5700 5701 /* Setup the Base and Length of the Rx Descriptor Rings */ 5702 bus_addr = rxq->rx_ring_phys_addr; 5703 5704 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i), 5705 (uint32_t)(bus_addr & 0x00000000ffffffffULL)); 5706 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i), 5707 (uint32_t)(bus_addr >> 32)); 5708 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i), 5709 rxq->nb_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 5710 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(i), 0); 5711 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(i), 0); 5712 5713 5714 /* Configure the SRRCTL register */ 5715 srrctl = IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 5716 5717 /* Set if packets are dropped when no descriptors available */ 5718 if (rxq->drop_en) 5719 srrctl |= IXGBE_SRRCTL_DROP_EN; 5720 5721 /* 5722 * Configure the RX buffer size in the BSIZEPACKET field of 5723 * the SRRCTL register of the queue. 5724 * The value is in 1 KB resolution. Valid values can be from 5725 * 1 KB to 16 KB. 5726 */ 5727 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - 5728 RTE_PKTMBUF_HEADROOM); 5729 srrctl |= ((buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) & 5730 IXGBE_SRRCTL_BSIZEPKT_MASK); 5731 5732 /* 5733 * VF modification to write virtual function SRRCTL register 5734 */ 5735 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), srrctl); 5736 5737 buf_size = (uint16_t) ((srrctl & IXGBE_SRRCTL_BSIZEPKT_MASK) << 5738 IXGBE_SRRCTL_BSIZEPKT_SHIFT); 5739 5740 if (rxmode->offloads & DEV_RX_OFFLOAD_SCATTER || 5741 /* It adds dual VLAN length for supporting dual VLAN */ 5742 (rxmode->max_rx_pkt_len + 5743 2 * IXGBE_VLAN_TAG_SIZE) > buf_size) { 5744 if (!dev->data->scattered_rx) 5745 PMD_INIT_LOG(DEBUG, "forcing scatter mode"); 5746 dev->data->scattered_rx = 1; 5747 } 5748 5749 if (rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 5750 rxmode->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP; 5751 } 5752 5753 /* Set RQPL for VF RSS according to max Rx queue */ 5754 psrtype |= (dev->data->nb_rx_queues >> 1) << 5755 IXGBE_PSRTYPE_RQPL_SHIFT; 5756 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 5757 5758 ixgbe_set_rx_function(dev); 5759 5760 return 0; 5761 } 5762 5763 /* 5764 * [VF] Initializes Transmit Unit. 5765 */ 5766 void __rte_cold 5767 ixgbevf_dev_tx_init(struct rte_eth_dev *dev) 5768 { 5769 struct ixgbe_hw *hw; 5770 struct ixgbe_tx_queue *txq; 5771 uint64_t bus_addr; 5772 uint32_t txctrl; 5773 uint16_t i; 5774 5775 PMD_INIT_FUNC_TRACE(); 5776 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5777 5778 /* Setup the Base and Length of the Tx Descriptor Rings */ 5779 for (i = 0; i < dev->data->nb_tx_queues; i++) { 5780 txq = dev->data->tx_queues[i]; 5781 bus_addr = txq->tx_ring_phys_addr; 5782 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i), 5783 (uint32_t)(bus_addr & 0x00000000ffffffffULL)); 5784 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i), 5785 (uint32_t)(bus_addr >> 32)); 5786 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i), 5787 txq->nb_tx_desc * sizeof(union ixgbe_adv_tx_desc)); 5788 /* Setup the HW Tx Head and TX Tail descriptor pointers */ 5789 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(i), 0); 5790 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(i), 0); 5791 5792 /* 5793 * Disable Tx Head Writeback RO bit, since this hoses 5794 * bookkeeping if things aren't delivered in order. 5795 */ 5796 txctrl = IXGBE_READ_REG(hw, 5797 IXGBE_VFDCA_TXCTRL(i)); 5798 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 5799 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i), 5800 txctrl); 5801 } 5802 } 5803 5804 /* 5805 * [VF] Start Transmit and Receive Units. 5806 */ 5807 void __rte_cold 5808 ixgbevf_dev_rxtx_start(struct rte_eth_dev *dev) 5809 { 5810 struct ixgbe_hw *hw; 5811 struct ixgbe_tx_queue *txq; 5812 struct ixgbe_rx_queue *rxq; 5813 uint32_t txdctl; 5814 uint32_t rxdctl; 5815 uint16_t i; 5816 int poll_ms; 5817 5818 PMD_INIT_FUNC_TRACE(); 5819 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5820 5821 for (i = 0; i < dev->data->nb_tx_queues; i++) { 5822 txq = dev->data->tx_queues[i]; 5823 /* Setup Transmit Threshold Registers */ 5824 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 5825 txdctl |= txq->pthresh & 0x7F; 5826 txdctl |= ((txq->hthresh & 0x7F) << 8); 5827 txdctl |= ((txq->wthresh & 0x7F) << 16); 5828 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 5829 } 5830 5831 for (i = 0; i < dev->data->nb_tx_queues; i++) { 5832 5833 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 5834 txdctl |= IXGBE_TXDCTL_ENABLE; 5835 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 5836 5837 poll_ms = 10; 5838 /* Wait until TX Enable ready */ 5839 do { 5840 rte_delay_ms(1); 5841 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 5842 } while (--poll_ms && !(txdctl & IXGBE_TXDCTL_ENABLE)); 5843 if (!poll_ms) 5844 PMD_INIT_LOG(ERR, "Could not enable Tx Queue %d", i); 5845 } 5846 for (i = 0; i < dev->data->nb_rx_queues; i++) { 5847 5848 rxq = dev->data->rx_queues[i]; 5849 5850 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 5851 rxdctl |= IXGBE_RXDCTL_ENABLE; 5852 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl); 5853 5854 /* Wait until RX Enable ready */ 5855 poll_ms = 10; 5856 do { 5857 rte_delay_ms(1); 5858 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 5859 } while (--poll_ms && !(rxdctl & IXGBE_RXDCTL_ENABLE)); 5860 if (!poll_ms) 5861 PMD_INIT_LOG(ERR, "Could not enable Rx Queue %d", i); 5862 rte_wmb(); 5863 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(i), rxq->nb_rx_desc - 1); 5864 5865 } 5866 } 5867 5868 int 5869 ixgbe_rss_conf_init(struct ixgbe_rte_flow_rss_conf *out, 5870 const struct rte_flow_action_rss *in) 5871 { 5872 if (in->key_len > RTE_DIM(out->key) || 5873 in->queue_num > RTE_DIM(out->queue)) 5874 return -EINVAL; 5875 out->conf = (struct rte_flow_action_rss){ 5876 .func = in->func, 5877 .level = in->level, 5878 .types = in->types, 5879 .key_len = in->key_len, 5880 .queue_num = in->queue_num, 5881 .key = memcpy(out->key, in->key, in->key_len), 5882 .queue = memcpy(out->queue, in->queue, 5883 sizeof(*in->queue) * in->queue_num), 5884 }; 5885 return 0; 5886 } 5887 5888 int 5889 ixgbe_action_rss_same(const struct rte_flow_action_rss *comp, 5890 const struct rte_flow_action_rss *with) 5891 { 5892 return (comp->func == with->func && 5893 comp->level == with->level && 5894 comp->types == with->types && 5895 comp->key_len == with->key_len && 5896 comp->queue_num == with->queue_num && 5897 !memcmp(comp->key, with->key, with->key_len) && 5898 !memcmp(comp->queue, with->queue, 5899 sizeof(*with->queue) * with->queue_num)); 5900 } 5901 5902 int 5903 ixgbe_config_rss_filter(struct rte_eth_dev *dev, 5904 struct ixgbe_rte_flow_rss_conf *conf, bool add) 5905 { 5906 struct ixgbe_hw *hw; 5907 uint32_t reta; 5908 uint16_t i; 5909 uint16_t j; 5910 uint16_t sp_reta_size; 5911 uint32_t reta_reg; 5912 struct rte_eth_rss_conf rss_conf = { 5913 .rss_key = conf->conf.key_len ? 5914 (void *)(uintptr_t)conf->conf.key : NULL, 5915 .rss_key_len = conf->conf.key_len, 5916 .rss_hf = conf->conf.types, 5917 }; 5918 struct ixgbe_filter_info *filter_info = 5919 IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private); 5920 5921 PMD_INIT_FUNC_TRACE(); 5922 hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); 5923 5924 sp_reta_size = ixgbe_reta_size_get(hw->mac.type); 5925 5926 if (!add) { 5927 if (ixgbe_action_rss_same(&filter_info->rss_info.conf, 5928 &conf->conf)) { 5929 ixgbe_rss_disable(dev); 5930 memset(&filter_info->rss_info, 0, 5931 sizeof(struct ixgbe_rte_flow_rss_conf)); 5932 return 0; 5933 } 5934 return -EINVAL; 5935 } 5936 5937 if (filter_info->rss_info.conf.queue_num) 5938 return -EINVAL; 5939 /* Fill in redirection table 5940 * The byte-swap is needed because NIC registers are in 5941 * little-endian order. 5942 */ 5943 reta = 0; 5944 for (i = 0, j = 0; i < sp_reta_size; i++, j++) { 5945 reta_reg = ixgbe_reta_reg_get(hw->mac.type, i); 5946 5947 if (j == conf->conf.queue_num) 5948 j = 0; 5949 reta = (reta << 8) | conf->conf.queue[j]; 5950 if ((i & 3) == 3) 5951 IXGBE_WRITE_REG(hw, reta_reg, 5952 rte_bswap32(reta)); 5953 } 5954 5955 /* Configure the RSS key and the RSS protocols used to compute 5956 * the RSS hash of input packets. 5957 */ 5958 if ((rss_conf.rss_hf & IXGBE_RSS_OFFLOAD_ALL) == 0) { 5959 ixgbe_rss_disable(dev); 5960 return 0; 5961 } 5962 if (rss_conf.rss_key == NULL) 5963 rss_conf.rss_key = rss_intel_key; /* Default hash key */ 5964 ixgbe_hw_rss_hash_set(hw, &rss_conf); 5965 5966 if (ixgbe_rss_conf_init(&filter_info->rss_info, &conf->conf)) 5967 return -EINVAL; 5968 5969 return 0; 5970 } 5971 5972 /* Stubs needed for linkage when RTE_ARCH_PPC_64 is set */ 5973 #if defined(RTE_ARCH_PPC_64) 5974 int 5975 ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev __rte_unused *dev) 5976 { 5977 return -1; 5978 } 5979 5980 uint16_t 5981 ixgbe_recv_pkts_vec( 5982 void __rte_unused *rx_queue, 5983 struct rte_mbuf __rte_unused **rx_pkts, 5984 uint16_t __rte_unused nb_pkts) 5985 { 5986 return 0; 5987 } 5988 5989 uint16_t 5990 ixgbe_recv_scattered_pkts_vec( 5991 void __rte_unused *rx_queue, 5992 struct rte_mbuf __rte_unused **rx_pkts, 5993 uint16_t __rte_unused nb_pkts) 5994 { 5995 return 0; 5996 } 5997 5998 int 5999 ixgbe_rxq_vec_setup(struct ixgbe_rx_queue __rte_unused *rxq) 6000 { 6001 return -1; 6002 } 6003 6004 uint16_t 6005 ixgbe_xmit_fixed_burst_vec(void __rte_unused *tx_queue, 6006 struct rte_mbuf __rte_unused **tx_pkts, 6007 uint16_t __rte_unused nb_pkts) 6008 { 6009 return 0; 6010 } 6011 6012 int 6013 ixgbe_txq_vec_setup(struct ixgbe_tx_queue __rte_unused *txq) 6014 { 6015 return -1; 6016 } 6017 6018 void 6019 ixgbe_rx_queue_release_mbufs_vec(struct ixgbe_rx_queue __rte_unused *rxq) 6020 { 6021 return; 6022 } 6023 #endif 6024