1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2020 Intel Corporation 3 */ 4 5 #include <rte_config.h> 6 #include <rte_flow.h> 7 #include <rte_malloc.h> 8 #include <ethdev_driver.h> 9 #include <rte_net.h> 10 11 #include "igc_logs.h" 12 #include "igc_txrx.h" 13 14 #ifdef RTE_PMD_USE_PREFETCH 15 #define rte_igc_prefetch(p) rte_prefetch0(p) 16 #else 17 #define rte_igc_prefetch(p) do {} while (0) 18 #endif 19 20 #ifdef RTE_PMD_PACKET_PREFETCH 21 #define rte_packet_prefetch(p) rte_prefetch1(p) 22 #else 23 #define rte_packet_prefetch(p) do {} while (0) 24 #endif 25 26 /* Multicast / Unicast table offset mask. */ 27 #define IGC_RCTL_MO_MSK (3u << IGC_RCTL_MO_SHIFT) 28 29 /* Loopback mode. */ 30 #define IGC_RCTL_LBM_SHIFT 6 31 #define IGC_RCTL_LBM_MSK (3u << IGC_RCTL_LBM_SHIFT) 32 33 /* Hash select for MTA */ 34 #define IGC_RCTL_HSEL_SHIFT 8 35 #define IGC_RCTL_HSEL_MSK (3u << IGC_RCTL_HSEL_SHIFT) 36 #define IGC_RCTL_PSP (1u << 21) 37 38 /* Receive buffer size for header buffer */ 39 #define IGC_SRRCTL_BSIZEHEADER_SHIFT 8 40 41 /* RX descriptor status and error flags */ 42 #define IGC_RXD_STAT_L4CS (1u << 5) 43 #define IGC_RXD_STAT_VEXT (1u << 9) 44 #define IGC_RXD_STAT_LLINT (1u << 11) 45 #define IGC_RXD_STAT_SCRC (1u << 12) 46 #define IGC_RXD_STAT_SMDT_MASK (3u << 13) 47 #define IGC_RXD_STAT_MC (1u << 19) 48 #define IGC_RXD_EXT_ERR_L4E (1u << 29) 49 #define IGC_RXD_EXT_ERR_IPE (1u << 30) 50 #define IGC_RXD_EXT_ERR_RXE (1u << 31) 51 #define IGC_RXD_RSS_TYPE_MASK 0xfu 52 #define IGC_RXD_PCTYPE_MASK (0x7fu << 4) 53 #define IGC_RXD_ETQF_SHIFT 12 54 #define IGC_RXD_ETQF_MSK (0xfu << IGC_RXD_ETQF_SHIFT) 55 #define IGC_RXD_VPKT (1u << 16) 56 57 /* TXD control bits */ 58 #define IGC_TXDCTL_PTHRESH_SHIFT 0 59 #define IGC_TXDCTL_HTHRESH_SHIFT 8 60 #define IGC_TXDCTL_WTHRESH_SHIFT 16 61 #define IGC_TXDCTL_PTHRESH_MSK (0x1fu << IGC_TXDCTL_PTHRESH_SHIFT) 62 #define IGC_TXDCTL_HTHRESH_MSK (0x1fu << IGC_TXDCTL_HTHRESH_SHIFT) 63 #define IGC_TXDCTL_WTHRESH_MSK (0x1fu << IGC_TXDCTL_WTHRESH_SHIFT) 64 65 /* RXD control bits */ 66 #define IGC_RXDCTL_PTHRESH_SHIFT 0 67 #define IGC_RXDCTL_HTHRESH_SHIFT 8 68 #define IGC_RXDCTL_WTHRESH_SHIFT 16 69 #define IGC_RXDCTL_PTHRESH_MSK (0x1fu << IGC_RXDCTL_PTHRESH_SHIFT) 70 #define IGC_RXDCTL_HTHRESH_MSK (0x1fu << IGC_RXDCTL_HTHRESH_SHIFT) 71 #define IGC_RXDCTL_WTHRESH_MSK (0x1fu << IGC_RXDCTL_WTHRESH_SHIFT) 72 73 #define IGC_TSO_MAX_HDRLEN 512 74 #define IGC_TSO_MAX_MSS 9216 75 76 /* Bit Mask to indicate what bits required for building TX context */ 77 #define IGC_TX_OFFLOAD_MASK ( \ 78 PKT_TX_OUTER_IPV4 | \ 79 PKT_TX_IPV6 | \ 80 PKT_TX_IPV4 | \ 81 PKT_TX_VLAN_PKT | \ 82 PKT_TX_IP_CKSUM | \ 83 PKT_TX_L4_MASK | \ 84 PKT_TX_TCP_SEG | \ 85 PKT_TX_UDP_SEG) 86 87 #define IGC_TX_OFFLOAD_SEG (PKT_TX_TCP_SEG | PKT_TX_UDP_SEG) 88 89 #define IGC_ADVTXD_POPTS_TXSM 0x00000200 /* L4 Checksum offload request */ 90 #define IGC_ADVTXD_POPTS_IXSM 0x00000100 /* IP Checksum offload request */ 91 92 /* L4 Packet TYPE of Reserved */ 93 #define IGC_ADVTXD_TUCMD_L4T_RSV 0x00001800 94 95 #define IGC_TX_OFFLOAD_NOTSUP_MASK (PKT_TX_OFFLOAD_MASK ^ IGC_TX_OFFLOAD_MASK) 96 97 /** 98 * Structure associated with each descriptor of the RX ring of a RX queue. 99 */ 100 struct igc_rx_entry { 101 struct rte_mbuf *mbuf; /**< mbuf associated with RX descriptor. */ 102 }; 103 104 /** 105 * Structure associated with each RX queue. 106 */ 107 struct igc_rx_queue { 108 struct rte_mempool *mb_pool; /**< mbuf pool to populate RX ring. */ 109 volatile union igc_adv_rx_desc *rx_ring; 110 /**< RX ring virtual address. */ 111 uint64_t rx_ring_phys_addr; /**< RX ring DMA address. */ 112 volatile uint32_t *rdt_reg_addr; /**< RDT register address. */ 113 volatile uint32_t *rdh_reg_addr; /**< RDH register address. */ 114 struct igc_rx_entry *sw_ring; /**< address of RX software ring. */ 115 struct rte_mbuf *pkt_first_seg; /**< First segment of current packet. */ 116 struct rte_mbuf *pkt_last_seg; /**< Last segment of current packet. */ 117 uint16_t nb_rx_desc; /**< number of RX descriptors. */ 118 uint16_t rx_tail; /**< current value of RDT register. */ 119 uint16_t nb_rx_hold; /**< number of held free RX desc. */ 120 uint16_t rx_free_thresh; /**< max free RX desc to hold. */ 121 uint16_t queue_id; /**< RX queue index. */ 122 uint16_t reg_idx; /**< RX queue register index. */ 123 uint16_t port_id; /**< Device port identifier. */ 124 uint8_t pthresh; /**< Prefetch threshold register. */ 125 uint8_t hthresh; /**< Host threshold register. */ 126 uint8_t wthresh; /**< Write-back threshold register. */ 127 uint8_t crc_len; /**< 0 if CRC stripped, 4 otherwise. */ 128 uint8_t drop_en; /**< If not 0, set SRRCTL.Drop_En. */ 129 uint32_t flags; /**< RX flags. */ 130 uint64_t offloads; /**< offloads of DEV_RX_OFFLOAD_* */ 131 }; 132 133 /** Offload features */ 134 union igc_tx_offload { 135 uint64_t data; 136 struct { 137 uint64_t l3_len:9; /**< L3 (IP) Header Length. */ 138 uint64_t l2_len:7; /**< L2 (MAC) Header Length. */ 139 uint64_t vlan_tci:16; 140 /**< VLAN Tag Control Identifier(CPU order). */ 141 uint64_t l4_len:8; /**< L4 (TCP/UDP) Header Length. */ 142 uint64_t tso_segsz:16; /**< TCP TSO segment size. */ 143 /* uint64_t unused:8; */ 144 }; 145 }; 146 147 /* 148 * Compare mask for igc_tx_offload.data, 149 * should be in sync with igc_tx_offload layout. 150 */ 151 #define TX_MACIP_LEN_CMP_MASK 0x000000000000FFFFULL /**< L2L3 header mask. */ 152 #define TX_VLAN_CMP_MASK 0x00000000FFFF0000ULL /**< Vlan mask. */ 153 #define TX_TCP_LEN_CMP_MASK 0x000000FF00000000ULL /**< TCP header mask. */ 154 #define TX_TSO_MSS_CMP_MASK 0x00FFFF0000000000ULL /**< TSO segsz mask. */ 155 /** Mac + IP + TCP + Mss mask. */ 156 #define TX_TSO_CMP_MASK \ 157 (TX_MACIP_LEN_CMP_MASK | TX_TCP_LEN_CMP_MASK | TX_TSO_MSS_CMP_MASK) 158 159 /** 160 * Structure to check if new context need be built 161 */ 162 struct igc_advctx_info { 163 uint64_t flags; /**< ol_flags related to context build. */ 164 /** tx offload: vlan, tso, l2-l3-l4 lengths. */ 165 union igc_tx_offload tx_offload; 166 /** compare mask for tx offload. */ 167 union igc_tx_offload tx_offload_mask; 168 }; 169 170 /** 171 * Hardware context number 172 */ 173 enum { 174 IGC_CTX_0 = 0, /**< CTX0 */ 175 IGC_CTX_1 = 1, /**< CTX1 */ 176 IGC_CTX_NUM = 2, /**< CTX_NUM */ 177 }; 178 179 /** 180 * Structure associated with each descriptor of the TX ring of a TX queue. 181 */ 182 struct igc_tx_entry { 183 struct rte_mbuf *mbuf; /**< mbuf associated with TX desc, if any. */ 184 uint16_t next_id; /**< Index of next descriptor in ring. */ 185 uint16_t last_id; /**< Index of last scattered descriptor. */ 186 }; 187 188 /** 189 * Structure associated with each TX queue. 190 */ 191 struct igc_tx_queue { 192 volatile union igc_adv_tx_desc *tx_ring; /**< TX ring address */ 193 uint64_t tx_ring_phys_addr; /**< TX ring DMA address. */ 194 struct igc_tx_entry *sw_ring; /**< virtual address of SW ring. */ 195 volatile uint32_t *tdt_reg_addr; /**< Address of TDT register. */ 196 uint32_t txd_type; /**< Device-specific TXD type */ 197 uint16_t nb_tx_desc; /**< number of TX descriptors. */ 198 uint16_t tx_tail; /**< Current value of TDT register. */ 199 uint16_t tx_head; 200 /**< Index of first used TX descriptor. */ 201 uint16_t queue_id; /**< TX queue index. */ 202 uint16_t reg_idx; /**< TX queue register index. */ 203 uint16_t port_id; /**< Device port identifier. */ 204 uint8_t pthresh; /**< Prefetch threshold register. */ 205 uint8_t hthresh; /**< Host threshold register. */ 206 uint8_t wthresh; /**< Write-back threshold register. */ 207 uint8_t ctx_curr; 208 209 /**< Start context position for transmit queue. */ 210 struct igc_advctx_info ctx_cache[IGC_CTX_NUM]; 211 /**< Hardware context history.*/ 212 uint64_t offloads; /**< offloads of DEV_TX_OFFLOAD_* */ 213 }; 214 215 static inline uint64_t 216 rx_desc_statuserr_to_pkt_flags(uint32_t statuserr) 217 { 218 static uint64_t l4_chksum_flags[] = {0, 0, PKT_RX_L4_CKSUM_GOOD, 219 PKT_RX_L4_CKSUM_BAD}; 220 221 static uint64_t l3_chksum_flags[] = {0, 0, PKT_RX_IP_CKSUM_GOOD, 222 PKT_RX_IP_CKSUM_BAD}; 223 uint64_t pkt_flags = 0; 224 uint32_t tmp; 225 226 if (statuserr & IGC_RXD_STAT_VP) 227 pkt_flags |= PKT_RX_VLAN_STRIPPED; 228 229 tmp = !!(statuserr & (IGC_RXD_STAT_L4CS | IGC_RXD_STAT_UDPCS)); 230 tmp = (tmp << 1) | (uint32_t)!!(statuserr & IGC_RXD_EXT_ERR_L4E); 231 pkt_flags |= l4_chksum_flags[tmp]; 232 233 tmp = !!(statuserr & IGC_RXD_STAT_IPCS); 234 tmp = (tmp << 1) | (uint32_t)!!(statuserr & IGC_RXD_EXT_ERR_IPE); 235 pkt_flags |= l3_chksum_flags[tmp]; 236 237 return pkt_flags; 238 } 239 240 #define IGC_PACKET_TYPE_IPV4 0X01 241 #define IGC_PACKET_TYPE_IPV4_TCP 0X11 242 #define IGC_PACKET_TYPE_IPV4_UDP 0X21 243 #define IGC_PACKET_TYPE_IPV4_SCTP 0X41 244 #define IGC_PACKET_TYPE_IPV4_EXT 0X03 245 #define IGC_PACKET_TYPE_IPV4_EXT_SCTP 0X43 246 #define IGC_PACKET_TYPE_IPV6 0X04 247 #define IGC_PACKET_TYPE_IPV6_TCP 0X14 248 #define IGC_PACKET_TYPE_IPV6_UDP 0X24 249 #define IGC_PACKET_TYPE_IPV6_EXT 0X0C 250 #define IGC_PACKET_TYPE_IPV6_EXT_TCP 0X1C 251 #define IGC_PACKET_TYPE_IPV6_EXT_UDP 0X2C 252 #define IGC_PACKET_TYPE_IPV4_IPV6 0X05 253 #define IGC_PACKET_TYPE_IPV4_IPV6_TCP 0X15 254 #define IGC_PACKET_TYPE_IPV4_IPV6_UDP 0X25 255 #define IGC_PACKET_TYPE_IPV4_IPV6_EXT 0X0D 256 #define IGC_PACKET_TYPE_IPV4_IPV6_EXT_TCP 0X1D 257 #define IGC_PACKET_TYPE_IPV4_IPV6_EXT_UDP 0X2D 258 #define IGC_PACKET_TYPE_MAX 0X80 259 #define IGC_PACKET_TYPE_MASK 0X7F 260 #define IGC_PACKET_TYPE_SHIFT 0X04 261 262 static inline uint32_t 263 rx_desc_pkt_info_to_pkt_type(uint32_t pkt_info) 264 { 265 static const uint32_t 266 ptype_table[IGC_PACKET_TYPE_MAX] __rte_cache_aligned = { 267 [IGC_PACKET_TYPE_IPV4] = RTE_PTYPE_L2_ETHER | 268 RTE_PTYPE_L3_IPV4, 269 [IGC_PACKET_TYPE_IPV4_EXT] = RTE_PTYPE_L2_ETHER | 270 RTE_PTYPE_L3_IPV4_EXT, 271 [IGC_PACKET_TYPE_IPV6] = RTE_PTYPE_L2_ETHER | 272 RTE_PTYPE_L3_IPV6, 273 [IGC_PACKET_TYPE_IPV4_IPV6] = RTE_PTYPE_L2_ETHER | 274 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 275 RTE_PTYPE_INNER_L3_IPV6, 276 [IGC_PACKET_TYPE_IPV6_EXT] = RTE_PTYPE_L2_ETHER | 277 RTE_PTYPE_L3_IPV6_EXT, 278 [IGC_PACKET_TYPE_IPV4_IPV6_EXT] = RTE_PTYPE_L2_ETHER | 279 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 280 RTE_PTYPE_INNER_L3_IPV6_EXT, 281 [IGC_PACKET_TYPE_IPV4_TCP] = RTE_PTYPE_L2_ETHER | 282 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP, 283 [IGC_PACKET_TYPE_IPV6_TCP] = RTE_PTYPE_L2_ETHER | 284 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP, 285 [IGC_PACKET_TYPE_IPV4_IPV6_TCP] = RTE_PTYPE_L2_ETHER | 286 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 287 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_TCP, 288 [IGC_PACKET_TYPE_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | 289 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP, 290 [IGC_PACKET_TYPE_IPV4_IPV6_EXT_TCP] = RTE_PTYPE_L2_ETHER | 291 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 292 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_TCP, 293 [IGC_PACKET_TYPE_IPV4_UDP] = RTE_PTYPE_L2_ETHER | 294 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP, 295 [IGC_PACKET_TYPE_IPV6_UDP] = RTE_PTYPE_L2_ETHER | 296 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP, 297 [IGC_PACKET_TYPE_IPV4_IPV6_UDP] = RTE_PTYPE_L2_ETHER | 298 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 299 RTE_PTYPE_INNER_L3_IPV6 | RTE_PTYPE_INNER_L4_UDP, 300 [IGC_PACKET_TYPE_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | 301 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP, 302 [IGC_PACKET_TYPE_IPV4_IPV6_EXT_UDP] = RTE_PTYPE_L2_ETHER | 303 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_IP | 304 RTE_PTYPE_INNER_L3_IPV6_EXT | RTE_PTYPE_INNER_L4_UDP, 305 [IGC_PACKET_TYPE_IPV4_SCTP] = RTE_PTYPE_L2_ETHER | 306 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP, 307 [IGC_PACKET_TYPE_IPV4_EXT_SCTP] = RTE_PTYPE_L2_ETHER | 308 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_SCTP, 309 }; 310 if (unlikely(pkt_info & IGC_RXDADV_PKTTYPE_ETQF)) 311 return RTE_PTYPE_UNKNOWN; 312 313 pkt_info = (pkt_info >> IGC_PACKET_TYPE_SHIFT) & IGC_PACKET_TYPE_MASK; 314 315 return ptype_table[pkt_info]; 316 } 317 318 static inline void 319 rx_desc_get_pkt_info(struct igc_rx_queue *rxq, struct rte_mbuf *rxm, 320 union igc_adv_rx_desc *rxd, uint32_t staterr) 321 { 322 uint64_t pkt_flags; 323 uint32_t hlen_type_rss; 324 uint16_t pkt_info; 325 326 /* Prefetch data of first segment, if configured to do so. */ 327 rte_packet_prefetch((char *)rxm->buf_addr + rxm->data_off); 328 329 rxm->port = rxq->port_id; 330 hlen_type_rss = rte_le_to_cpu_32(rxd->wb.lower.lo_dword.data); 331 rxm->hash.rss = rte_le_to_cpu_32(rxd->wb.lower.hi_dword.rss); 332 rxm->vlan_tci = rte_le_to_cpu_16(rxd->wb.upper.vlan); 333 334 pkt_flags = (hlen_type_rss & IGC_RXD_RSS_TYPE_MASK) ? 335 PKT_RX_RSS_HASH : 0; 336 337 if (hlen_type_rss & IGC_RXD_VPKT) 338 pkt_flags |= PKT_RX_VLAN; 339 340 pkt_flags |= rx_desc_statuserr_to_pkt_flags(staterr); 341 342 rxm->ol_flags = pkt_flags; 343 pkt_info = rte_le_to_cpu_16(rxd->wb.lower.lo_dword.hs_rss.pkt_info); 344 rxm->packet_type = rx_desc_pkt_info_to_pkt_type(pkt_info); 345 } 346 347 static uint16_t 348 igc_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) 349 { 350 struct igc_rx_queue * const rxq = rx_queue; 351 volatile union igc_adv_rx_desc * const rx_ring = rxq->rx_ring; 352 struct igc_rx_entry * const sw_ring = rxq->sw_ring; 353 uint16_t rx_id = rxq->rx_tail; 354 uint16_t nb_rx = 0; 355 uint16_t nb_hold = 0; 356 357 while (nb_rx < nb_pkts) { 358 volatile union igc_adv_rx_desc *rxdp; 359 struct igc_rx_entry *rxe; 360 struct rte_mbuf *rxm; 361 struct rte_mbuf *nmb; 362 union igc_adv_rx_desc rxd; 363 uint32_t staterr; 364 uint16_t data_len; 365 366 /* 367 * The order of operations here is important as the DD status 368 * bit must not be read after any other descriptor fields. 369 * rx_ring and rxdp are pointing to volatile data so the order 370 * of accesses cannot be reordered by the compiler. If they were 371 * not volatile, they could be reordered which could lead to 372 * using invalid descriptor fields when read from rxd. 373 */ 374 rxdp = &rx_ring[rx_id]; 375 staterr = rte_cpu_to_le_32(rxdp->wb.upper.status_error); 376 if (!(staterr & IGC_RXD_STAT_DD)) 377 break; 378 rxd = *rxdp; 379 380 /* 381 * End of packet. 382 * 383 * If the IGC_RXD_STAT_EOP flag is not set, the RX packet is 384 * likely to be invalid and to be dropped by the various 385 * validation checks performed by the network stack. 386 * 387 * Allocate a new mbuf to replenish the RX ring descriptor. 388 * If the allocation fails: 389 * - arrange for that RX descriptor to be the first one 390 * being parsed the next time the receive function is 391 * invoked [on the same queue]. 392 * 393 * - Stop parsing the RX ring and return immediately. 394 * 395 * This policy does not drop the packet received in the RX 396 * descriptor for which the allocation of a new mbuf failed. 397 * Thus, it allows that packet to be later retrieved if 398 * mbuf have been freed in the mean time. 399 * As a side effect, holding RX descriptors instead of 400 * systematically giving them back to the NIC may lead to 401 * RX ring exhaustion situations. 402 * However, the NIC can gracefully prevent such situations 403 * to happen by sending specific "back-pressure" flow control 404 * frames to its peer(s). 405 */ 406 PMD_RX_LOG(DEBUG, 407 "port_id=%u queue_id=%u rx_id=%u staterr=0x%x data_len=%u", 408 rxq->port_id, rxq->queue_id, rx_id, staterr, 409 rte_le_to_cpu_16(rxd.wb.upper.length)); 410 411 nmb = rte_mbuf_raw_alloc(rxq->mb_pool); 412 if (nmb == NULL) { 413 unsigned int id; 414 PMD_RX_LOG(DEBUG, 415 "RX mbuf alloc failed, port_id=%u queue_id=%u", 416 rxq->port_id, rxq->queue_id); 417 id = rxq->port_id; 418 rte_eth_devices[id].data->rx_mbuf_alloc_failed++; 419 break; 420 } 421 422 nb_hold++; 423 rxe = &sw_ring[rx_id]; 424 rx_id++; 425 if (rx_id >= rxq->nb_rx_desc) 426 rx_id = 0; 427 428 /* Prefetch next mbuf while processing current one. */ 429 rte_igc_prefetch(sw_ring[rx_id].mbuf); 430 431 /* 432 * When next RX descriptor is on a cache-line boundary, 433 * prefetch the next 4 RX descriptors and the next 8 pointers 434 * to mbufs. 435 */ 436 if ((rx_id & 0x3) == 0) { 437 rte_igc_prefetch(&rx_ring[rx_id]); 438 rte_igc_prefetch(&sw_ring[rx_id]); 439 } 440 441 /* 442 * Update RX descriptor with the physical address of the new 443 * data buffer of the new allocated mbuf. 444 */ 445 rxm = rxe->mbuf; 446 rxe->mbuf = nmb; 447 rxdp->read.hdr_addr = 0; 448 rxdp->read.pkt_addr = 449 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 450 rxm->next = NULL; 451 452 rxm->data_off = RTE_PKTMBUF_HEADROOM; 453 data_len = rte_le_to_cpu_16(rxd.wb.upper.length) - rxq->crc_len; 454 rxm->data_len = data_len; 455 rxm->pkt_len = data_len; 456 rxm->nb_segs = 1; 457 458 rx_desc_get_pkt_info(rxq, rxm, &rxd, staterr); 459 460 /* 461 * Store the mbuf address into the next entry of the array 462 * of returned packets. 463 */ 464 rx_pkts[nb_rx++] = rxm; 465 } 466 rxq->rx_tail = rx_id; 467 468 /* 469 * If the number of free RX descriptors is greater than the RX free 470 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 471 * register. 472 * Update the RDT with the value of the last processed RX descriptor 473 * minus 1, to guarantee that the RDT register is never equal to the 474 * RDH register, which creates a "full" ring situation from the 475 * hardware point of view... 476 */ 477 nb_hold = nb_hold + rxq->nb_rx_hold; 478 if (nb_hold > rxq->rx_free_thresh) { 479 PMD_RX_LOG(DEBUG, 480 "port_id=%u queue_id=%u rx_tail=%u nb_hold=%u nb_rx=%u", 481 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx); 482 rx_id = (rx_id == 0) ? (rxq->nb_rx_desc - 1) : (rx_id - 1); 483 IGC_PCI_REG_WRITE(rxq->rdt_reg_addr, rx_id); 484 nb_hold = 0; 485 } 486 rxq->nb_rx_hold = nb_hold; 487 return nb_rx; 488 } 489 490 static uint16_t 491 igc_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 492 uint16_t nb_pkts) 493 { 494 struct igc_rx_queue * const rxq = rx_queue; 495 volatile union igc_adv_rx_desc * const rx_ring = rxq->rx_ring; 496 struct igc_rx_entry * const sw_ring = rxq->sw_ring; 497 struct rte_mbuf *first_seg = rxq->pkt_first_seg; 498 struct rte_mbuf *last_seg = rxq->pkt_last_seg; 499 500 uint16_t rx_id = rxq->rx_tail; 501 uint16_t nb_rx = 0; 502 uint16_t nb_hold = 0; 503 504 while (nb_rx < nb_pkts) { 505 volatile union igc_adv_rx_desc *rxdp; 506 struct igc_rx_entry *rxe; 507 struct rte_mbuf *rxm; 508 struct rte_mbuf *nmb; 509 union igc_adv_rx_desc rxd; 510 uint32_t staterr; 511 uint16_t data_len; 512 513 next_desc: 514 /* 515 * The order of operations here is important as the DD status 516 * bit must not be read after any other descriptor fields. 517 * rx_ring and rxdp are pointing to volatile data so the order 518 * of accesses cannot be reordered by the compiler. If they were 519 * not volatile, they could be reordered which could lead to 520 * using invalid descriptor fields when read from rxd. 521 */ 522 rxdp = &rx_ring[rx_id]; 523 staterr = rte_cpu_to_le_32(rxdp->wb.upper.status_error); 524 if (!(staterr & IGC_RXD_STAT_DD)) 525 break; 526 rxd = *rxdp; 527 528 /* 529 * Descriptor done. 530 * 531 * Allocate a new mbuf to replenish the RX ring descriptor. 532 * If the allocation fails: 533 * - arrange for that RX descriptor to be the first one 534 * being parsed the next time the receive function is 535 * invoked [on the same queue]. 536 * 537 * - Stop parsing the RX ring and return immediately. 538 * 539 * This policy does not drop the packet received in the RX 540 * descriptor for which the allocation of a new mbuf failed. 541 * Thus, it allows that packet to be later retrieved if 542 * mbuf have been freed in the mean time. 543 * As a side effect, holding RX descriptors instead of 544 * systematically giving them back to the NIC may lead to 545 * RX ring exhaustion situations. 546 * However, the NIC can gracefully prevent such situations 547 * to happen by sending specific "back-pressure" flow control 548 * frames to its peer(s). 549 */ 550 PMD_RX_LOG(DEBUG, 551 "port_id=%u queue_id=%u rx_id=%u staterr=0x%x data_len=%u", 552 rxq->port_id, rxq->queue_id, rx_id, staterr, 553 rte_le_to_cpu_16(rxd.wb.upper.length)); 554 555 nmb = rte_mbuf_raw_alloc(rxq->mb_pool); 556 if (nmb == NULL) { 557 unsigned int id; 558 PMD_RX_LOG(DEBUG, 559 "RX mbuf alloc failed, port_id=%u queue_id=%u", 560 rxq->port_id, rxq->queue_id); 561 id = rxq->port_id; 562 rte_eth_devices[id].data->rx_mbuf_alloc_failed++; 563 break; 564 } 565 566 nb_hold++; 567 rxe = &sw_ring[rx_id]; 568 rx_id++; 569 if (rx_id >= rxq->nb_rx_desc) 570 rx_id = 0; 571 572 /* Prefetch next mbuf while processing current one. */ 573 rte_igc_prefetch(sw_ring[rx_id].mbuf); 574 575 /* 576 * When next RX descriptor is on a cache-line boundary, 577 * prefetch the next 4 RX descriptors and the next 8 pointers 578 * to mbufs. 579 */ 580 if ((rx_id & 0x3) == 0) { 581 rte_igc_prefetch(&rx_ring[rx_id]); 582 rte_igc_prefetch(&sw_ring[rx_id]); 583 } 584 585 /* 586 * Update RX descriptor with the physical address of the new 587 * data buffer of the new allocated mbuf. 588 */ 589 rxm = rxe->mbuf; 590 rxe->mbuf = nmb; 591 rxdp->read.hdr_addr = 0; 592 rxdp->read.pkt_addr = 593 rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb)); 594 rxm->next = NULL; 595 596 /* 597 * Set data length & data buffer address of mbuf. 598 */ 599 rxm->data_off = RTE_PKTMBUF_HEADROOM; 600 data_len = rte_le_to_cpu_16(rxd.wb.upper.length); 601 rxm->data_len = data_len; 602 603 /* 604 * If this is the first buffer of the received packet, 605 * set the pointer to the first mbuf of the packet and 606 * initialize its context. 607 * Otherwise, update the total length and the number of segments 608 * of the current scattered packet, and update the pointer to 609 * the last mbuf of the current packet. 610 */ 611 if (first_seg == NULL) { 612 first_seg = rxm; 613 first_seg->pkt_len = data_len; 614 first_seg->nb_segs = 1; 615 } else { 616 first_seg->pkt_len += data_len; 617 first_seg->nb_segs++; 618 last_seg->next = rxm; 619 } 620 621 /* 622 * If this is not the last buffer of the received packet, 623 * update the pointer to the last mbuf of the current scattered 624 * packet and continue to parse the RX ring. 625 */ 626 if (!(staterr & IGC_RXD_STAT_EOP)) { 627 last_seg = rxm; 628 goto next_desc; 629 } 630 631 /* 632 * This is the last buffer of the received packet. 633 * If the CRC is not stripped by the hardware: 634 * - Subtract the CRC length from the total packet length. 635 * - If the last buffer only contains the whole CRC or a part 636 * of it, free the mbuf associated to the last buffer. 637 * If part of the CRC is also contained in the previous 638 * mbuf, subtract the length of that CRC part from the 639 * data length of the previous mbuf. 640 */ 641 if (unlikely(rxq->crc_len > 0)) { 642 first_seg->pkt_len -= RTE_ETHER_CRC_LEN; 643 if (data_len <= RTE_ETHER_CRC_LEN) { 644 rte_pktmbuf_free_seg(rxm); 645 first_seg->nb_segs--; 646 last_seg->data_len = last_seg->data_len - 647 (RTE_ETHER_CRC_LEN - data_len); 648 last_seg->next = NULL; 649 } else { 650 rxm->data_len = (uint16_t) 651 (data_len - RTE_ETHER_CRC_LEN); 652 } 653 } 654 655 rx_desc_get_pkt_info(rxq, first_seg, &rxd, staterr); 656 657 /* 658 * Store the mbuf address into the next entry of the array 659 * of returned packets. 660 */ 661 rx_pkts[nb_rx++] = first_seg; 662 663 /* Setup receipt context for a new packet. */ 664 first_seg = NULL; 665 } 666 rxq->rx_tail = rx_id; 667 668 /* 669 * Save receive context. 670 */ 671 rxq->pkt_first_seg = first_seg; 672 rxq->pkt_last_seg = last_seg; 673 674 /* 675 * If the number of free RX descriptors is greater than the RX free 676 * threshold of the queue, advance the Receive Descriptor Tail (RDT) 677 * register. 678 * Update the RDT with the value of the last processed RX descriptor 679 * minus 1, to guarantee that the RDT register is never equal to the 680 * RDH register, which creates a "full" ring situation from the 681 * hardware point of view... 682 */ 683 nb_hold = nb_hold + rxq->nb_rx_hold; 684 if (nb_hold > rxq->rx_free_thresh) { 685 PMD_RX_LOG(DEBUG, 686 "port_id=%u queue_id=%u rx_tail=%u nb_hold=%u nb_rx=%u", 687 rxq->port_id, rxq->queue_id, rx_id, nb_hold, nb_rx); 688 rx_id = (rx_id == 0) ? (rxq->nb_rx_desc - 1) : (rx_id - 1); 689 IGC_PCI_REG_WRITE(rxq->rdt_reg_addr, rx_id); 690 nb_hold = 0; 691 } 692 rxq->nb_rx_hold = nb_hold; 693 return nb_rx; 694 } 695 696 static void 697 igc_rx_queue_release_mbufs(struct igc_rx_queue *rxq) 698 { 699 unsigned int i; 700 701 if (rxq->sw_ring != NULL) { 702 for (i = 0; i < rxq->nb_rx_desc; i++) { 703 if (rxq->sw_ring[i].mbuf != NULL) { 704 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); 705 rxq->sw_ring[i].mbuf = NULL; 706 } 707 } 708 } 709 } 710 711 static void 712 igc_rx_queue_release(struct igc_rx_queue *rxq) 713 { 714 igc_rx_queue_release_mbufs(rxq); 715 rte_free(rxq->sw_ring); 716 rte_free(rxq); 717 } 718 719 void eth_igc_rx_queue_release(void *rxq) 720 { 721 if (rxq) 722 igc_rx_queue_release(rxq); 723 } 724 725 uint32_t eth_igc_rx_queue_count(struct rte_eth_dev *dev, 726 uint16_t rx_queue_id) 727 { 728 /** 729 * Check the DD bit of a rx descriptor of each 4 in a group, 730 * to avoid checking too frequently and downgrading performance 731 * too much. 732 */ 733 #define IGC_RXQ_SCAN_INTERVAL 4 734 735 volatile union igc_adv_rx_desc *rxdp; 736 struct igc_rx_queue *rxq; 737 uint16_t desc = 0; 738 739 rxq = dev->data->rx_queues[rx_queue_id]; 740 rxdp = &rxq->rx_ring[rxq->rx_tail]; 741 742 while (desc < rxq->nb_rx_desc - rxq->rx_tail) { 743 if (unlikely(!(rxdp->wb.upper.status_error & 744 IGC_RXD_STAT_DD))) 745 return desc; 746 desc += IGC_RXQ_SCAN_INTERVAL; 747 rxdp += IGC_RXQ_SCAN_INTERVAL; 748 } 749 rxdp = &rxq->rx_ring[rxq->rx_tail + desc - rxq->nb_rx_desc]; 750 751 while (desc < rxq->nb_rx_desc && 752 (rxdp->wb.upper.status_error & IGC_RXD_STAT_DD)) { 753 desc += IGC_RXQ_SCAN_INTERVAL; 754 rxdp += IGC_RXQ_SCAN_INTERVAL; 755 } 756 757 return desc; 758 } 759 760 int eth_igc_rx_descriptor_done(void *rx_queue, uint16_t offset) 761 { 762 volatile union igc_adv_rx_desc *rxdp; 763 struct igc_rx_queue *rxq = rx_queue; 764 uint32_t desc; 765 766 if (unlikely(!rxq || offset >= rxq->nb_rx_desc)) 767 return 0; 768 769 desc = rxq->rx_tail + offset; 770 if (desc >= rxq->nb_rx_desc) 771 desc -= rxq->nb_rx_desc; 772 773 rxdp = &rxq->rx_ring[desc]; 774 return !!(rxdp->wb.upper.status_error & 775 rte_cpu_to_le_32(IGC_RXD_STAT_DD)); 776 } 777 778 int eth_igc_rx_descriptor_status(void *rx_queue, uint16_t offset) 779 { 780 struct igc_rx_queue *rxq = rx_queue; 781 volatile uint32_t *status; 782 uint32_t desc; 783 784 if (unlikely(!rxq || offset >= rxq->nb_rx_desc)) 785 return -EINVAL; 786 787 if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold) 788 return RTE_ETH_RX_DESC_UNAVAIL; 789 790 desc = rxq->rx_tail + offset; 791 if (desc >= rxq->nb_rx_desc) 792 desc -= rxq->nb_rx_desc; 793 794 status = &rxq->rx_ring[desc].wb.upper.status_error; 795 if (*status & rte_cpu_to_le_32(IGC_RXD_STAT_DD)) 796 return RTE_ETH_RX_DESC_DONE; 797 798 return RTE_ETH_RX_DESC_AVAIL; 799 } 800 801 static int 802 igc_alloc_rx_queue_mbufs(struct igc_rx_queue *rxq) 803 { 804 struct igc_rx_entry *rxe = rxq->sw_ring; 805 uint64_t dma_addr; 806 unsigned int i; 807 808 /* Initialize software ring entries. */ 809 for (i = 0; i < rxq->nb_rx_desc; i++) { 810 volatile union igc_adv_rx_desc *rxd; 811 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool); 812 813 if (mbuf == NULL) { 814 PMD_DRV_LOG(ERR, "RX mbuf alloc failed, queue_id=%hu", 815 rxq->queue_id); 816 return -ENOMEM; 817 } 818 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 819 rxd = &rxq->rx_ring[i]; 820 rxd->read.hdr_addr = 0; 821 rxd->read.pkt_addr = dma_addr; 822 rxe[i].mbuf = mbuf; 823 } 824 825 return 0; 826 } 827 828 /* 829 * RSS random key supplied in section 7.1.2.9.3 of the Intel I225 datasheet. 830 * Used as the default key. 831 */ 832 static uint8_t default_rss_key[40] = { 833 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 834 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0, 835 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4, 836 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, 837 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA, 838 }; 839 840 void 841 igc_rss_disable(struct rte_eth_dev *dev) 842 { 843 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev); 844 uint32_t mrqc; 845 846 mrqc = IGC_READ_REG(hw, IGC_MRQC); 847 mrqc &= ~IGC_MRQC_ENABLE_MASK; 848 IGC_WRITE_REG(hw, IGC_MRQC, mrqc); 849 } 850 851 void 852 igc_hw_rss_hash_set(struct igc_hw *hw, struct rte_eth_rss_conf *rss_conf) 853 { 854 uint32_t *hash_key = (uint32_t *)rss_conf->rss_key; 855 uint32_t mrqc; 856 uint64_t rss_hf; 857 858 if (hash_key != NULL) { 859 uint8_t i; 860 861 /* Fill in RSS hash key */ 862 for (i = 0; i < IGC_HKEY_MAX_INDEX; i++) 863 IGC_WRITE_REG_LE_VALUE(hw, IGC_RSSRK(i), hash_key[i]); 864 } 865 866 /* Set configured hashing protocols in MRQC register */ 867 rss_hf = rss_conf->rss_hf; 868 mrqc = IGC_MRQC_ENABLE_RSS_4Q; /* RSS enabled. */ 869 if (rss_hf & ETH_RSS_IPV4) 870 mrqc |= IGC_MRQC_RSS_FIELD_IPV4; 871 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) 872 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_TCP; 873 if (rss_hf & ETH_RSS_IPV6) 874 mrqc |= IGC_MRQC_RSS_FIELD_IPV6; 875 if (rss_hf & ETH_RSS_IPV6_EX) 876 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_EX; 877 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) 878 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_TCP; 879 if (rss_hf & ETH_RSS_IPV6_TCP_EX) 880 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 881 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) 882 mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP; 883 if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) 884 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP; 885 if (rss_hf & ETH_RSS_IPV6_UDP_EX) 886 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP_EX; 887 IGC_WRITE_REG(hw, IGC_MRQC, mrqc); 888 } 889 890 static void 891 igc_rss_configure(struct rte_eth_dev *dev) 892 { 893 struct rte_eth_rss_conf rss_conf; 894 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev); 895 uint16_t i; 896 897 /* Fill in redirection table. */ 898 for (i = 0; i < IGC_RSS_RDT_SIZD; i++) { 899 union igc_rss_reta_reg reta; 900 uint16_t q_idx, reta_idx; 901 902 q_idx = (uint8_t)((dev->data->nb_rx_queues > 1) ? 903 i % dev->data->nb_rx_queues : 0); 904 reta_idx = i % sizeof(reta); 905 reta.bytes[reta_idx] = q_idx; 906 if (reta_idx == sizeof(reta) - 1) 907 IGC_WRITE_REG_LE_VALUE(hw, 908 IGC_RETA(i / sizeof(reta)), reta.dword); 909 } 910 911 /* 912 * Configure the RSS key and the RSS protocols used to compute 913 * the RSS hash of input packets. 914 */ 915 rss_conf = dev->data->dev_conf.rx_adv_conf.rss_conf; 916 if (rss_conf.rss_key == NULL) 917 rss_conf.rss_key = default_rss_key; 918 igc_hw_rss_hash_set(hw, &rss_conf); 919 } 920 921 int 922 igc_del_rss_filter(struct rte_eth_dev *dev) 923 { 924 struct igc_rss_filter *rss_filter = IGC_DEV_PRIVATE_RSS_FILTER(dev); 925 926 if (rss_filter->enable) { 927 /* recover default RSS configuration */ 928 igc_rss_configure(dev); 929 930 /* disable RSS logic and clear filter data */ 931 igc_rss_disable(dev); 932 memset(rss_filter, 0, sizeof(*rss_filter)); 933 return 0; 934 } 935 PMD_DRV_LOG(ERR, "filter not exist!"); 936 return -ENOENT; 937 } 938 939 /* Initiate the filter structure by the structure of rte_flow_action_rss */ 940 void 941 igc_rss_conf_set(struct igc_rss_filter *out, 942 const struct rte_flow_action_rss *rss) 943 { 944 out->conf.func = rss->func; 945 out->conf.level = rss->level; 946 out->conf.types = rss->types; 947 948 if (rss->key_len == sizeof(out->key)) { 949 memcpy(out->key, rss->key, rss->key_len); 950 out->conf.key = out->key; 951 out->conf.key_len = rss->key_len; 952 } else { 953 out->conf.key = NULL; 954 out->conf.key_len = 0; 955 } 956 957 if (rss->queue_num <= IGC_RSS_RDT_SIZD) { 958 memcpy(out->queue, rss->queue, 959 sizeof(*out->queue) * rss->queue_num); 960 out->conf.queue = out->queue; 961 out->conf.queue_num = rss->queue_num; 962 } else { 963 out->conf.queue = NULL; 964 out->conf.queue_num = 0; 965 } 966 } 967 968 int 969 igc_add_rss_filter(struct rte_eth_dev *dev, struct igc_rss_filter *rss) 970 { 971 struct rte_eth_rss_conf rss_conf = { 972 .rss_key = rss->conf.key_len ? 973 (void *)(uintptr_t)rss->conf.key : NULL, 974 .rss_key_len = rss->conf.key_len, 975 .rss_hf = rss->conf.types, 976 }; 977 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev); 978 struct igc_rss_filter *rss_filter = IGC_DEV_PRIVATE_RSS_FILTER(dev); 979 uint32_t i, j; 980 981 /* check RSS type is valid */ 982 if ((rss_conf.rss_hf & IGC_RSS_OFFLOAD_ALL) == 0) { 983 PMD_DRV_LOG(ERR, 984 "RSS type(0x%" PRIx64 ") error!, only 0x%" PRIx64 985 " been supported", rss_conf.rss_hf, 986 (uint64_t)IGC_RSS_OFFLOAD_ALL); 987 return -EINVAL; 988 } 989 990 /* check queue count is not zero */ 991 if (!rss->conf.queue_num) { 992 PMD_DRV_LOG(ERR, "Queue number should not be 0!"); 993 return -EINVAL; 994 } 995 996 /* check queue id is valid */ 997 for (i = 0; i < rss->conf.queue_num; i++) 998 if (rss->conf.queue[i] >= dev->data->nb_rx_queues) { 999 PMD_DRV_LOG(ERR, "Queue id %u is invalid!", 1000 rss->conf.queue[i]); 1001 return -EINVAL; 1002 } 1003 1004 /* only support one filter */ 1005 if (rss_filter->enable) { 1006 PMD_DRV_LOG(ERR, "Only support one RSS filter!"); 1007 return -ENOTSUP; 1008 } 1009 rss_filter->enable = 1; 1010 1011 igc_rss_conf_set(rss_filter, &rss->conf); 1012 1013 /* Fill in redirection table. */ 1014 for (i = 0, j = 0; i < IGC_RSS_RDT_SIZD; i++, j++) { 1015 union igc_rss_reta_reg reta; 1016 uint16_t q_idx, reta_idx; 1017 1018 if (j == rss->conf.queue_num) 1019 j = 0; 1020 q_idx = rss->conf.queue[j]; 1021 reta_idx = i % sizeof(reta); 1022 reta.bytes[reta_idx] = q_idx; 1023 if (reta_idx == sizeof(reta) - 1) 1024 IGC_WRITE_REG_LE_VALUE(hw, 1025 IGC_RETA(i / sizeof(reta)), reta.dword); 1026 } 1027 1028 if (rss_conf.rss_key == NULL) 1029 rss_conf.rss_key = default_rss_key; 1030 igc_hw_rss_hash_set(hw, &rss_conf); 1031 return 0; 1032 } 1033 1034 void 1035 igc_clear_rss_filter(struct rte_eth_dev *dev) 1036 { 1037 struct igc_rss_filter *rss_filter = IGC_DEV_PRIVATE_RSS_FILTER(dev); 1038 1039 if (!rss_filter->enable) 1040 return; 1041 1042 /* recover default RSS configuration */ 1043 igc_rss_configure(dev); 1044 1045 /* disable RSS logic and clear filter data */ 1046 igc_rss_disable(dev); 1047 memset(rss_filter, 0, sizeof(*rss_filter)); 1048 } 1049 1050 static int 1051 igc_dev_mq_rx_configure(struct rte_eth_dev *dev) 1052 { 1053 if (RTE_ETH_DEV_SRIOV(dev).active) { 1054 PMD_DRV_LOG(ERR, "SRIOV unsupported!"); 1055 return -EINVAL; 1056 } 1057 1058 switch (dev->data->dev_conf.rxmode.mq_mode) { 1059 case ETH_MQ_RX_RSS: 1060 igc_rss_configure(dev); 1061 break; 1062 case ETH_MQ_RX_NONE: 1063 /* 1064 * configure RSS register for following, 1065 * then disable the RSS logic 1066 */ 1067 igc_rss_configure(dev); 1068 igc_rss_disable(dev); 1069 break; 1070 default: 1071 PMD_DRV_LOG(ERR, "rx mode(%d) not supported!", 1072 dev->data->dev_conf.rxmode.mq_mode); 1073 return -EINVAL; 1074 } 1075 return 0; 1076 } 1077 1078 int 1079 igc_rx_init(struct rte_eth_dev *dev) 1080 { 1081 struct igc_rx_queue *rxq; 1082 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev); 1083 uint64_t offloads = dev->data->dev_conf.rxmode.offloads; 1084 uint32_t max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len; 1085 uint32_t rctl; 1086 uint32_t rxcsum; 1087 uint16_t buf_size; 1088 uint16_t rctl_bsize; 1089 uint16_t i; 1090 int ret; 1091 1092 dev->rx_pkt_burst = igc_recv_pkts; 1093 1094 /* 1095 * Make sure receives are disabled while setting 1096 * up the descriptor ring. 1097 */ 1098 rctl = IGC_READ_REG(hw, IGC_RCTL); 1099 IGC_WRITE_REG(hw, IGC_RCTL, rctl & ~IGC_RCTL_EN); 1100 1101 /* Configure support of jumbo frames, if any. */ 1102 if (offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) { 1103 rctl |= IGC_RCTL_LPE; 1104 1105 /* 1106 * Set maximum packet length by default, and might be updated 1107 * together with enabling/disabling dual VLAN. 1108 */ 1109 IGC_WRITE_REG(hw, IGC_RLPML, max_rx_pkt_len); 1110 } else { 1111 rctl &= ~IGC_RCTL_LPE; 1112 } 1113 1114 /* Configure and enable each RX queue. */ 1115 rctl_bsize = 0; 1116 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1117 uint64_t bus_addr; 1118 uint32_t rxdctl; 1119 uint32_t srrctl; 1120 1121 rxq = dev->data->rx_queues[i]; 1122 rxq->flags = 0; 1123 1124 /* Allocate buffers for descriptor rings and set up queue */ 1125 ret = igc_alloc_rx_queue_mbufs(rxq); 1126 if (ret) 1127 return ret; 1128 1129 /* 1130 * Reset crc_len in case it was changed after queue setup by a 1131 * call to configure 1132 */ 1133 rxq->crc_len = (offloads & DEV_RX_OFFLOAD_KEEP_CRC) ? 1134 RTE_ETHER_CRC_LEN : 0; 1135 1136 bus_addr = rxq->rx_ring_phys_addr; 1137 IGC_WRITE_REG(hw, IGC_RDLEN(rxq->reg_idx), 1138 rxq->nb_rx_desc * 1139 sizeof(union igc_adv_rx_desc)); 1140 IGC_WRITE_REG(hw, IGC_RDBAH(rxq->reg_idx), 1141 (uint32_t)(bus_addr >> 32)); 1142 IGC_WRITE_REG(hw, IGC_RDBAL(rxq->reg_idx), 1143 (uint32_t)bus_addr); 1144 1145 /* set descriptor configuration */ 1146 srrctl = IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 1147 1148 srrctl |= (uint32_t)(RTE_PKTMBUF_HEADROOM / 64) << 1149 IGC_SRRCTL_BSIZEHEADER_SHIFT; 1150 /* 1151 * Configure RX buffer size. 1152 */ 1153 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - 1154 RTE_PKTMBUF_HEADROOM); 1155 if (buf_size >= 1024) { 1156 /* 1157 * Configure the BSIZEPACKET field of the SRRCTL 1158 * register of the queue. 1159 * Value is in 1 KB resolution, from 1 KB to 16 KB. 1160 * If this field is equal to 0b, then RCTL.BSIZE 1161 * determines the RX packet buffer size. 1162 */ 1163 1164 srrctl |= ((buf_size >> IGC_SRRCTL_BSIZEPKT_SHIFT) & 1165 IGC_SRRCTL_BSIZEPKT_MASK); 1166 buf_size = (uint16_t)((srrctl & 1167 IGC_SRRCTL_BSIZEPKT_MASK) << 1168 IGC_SRRCTL_BSIZEPKT_SHIFT); 1169 1170 /* It adds dual VLAN length for supporting dual VLAN */ 1171 if (max_rx_pkt_len + 2 * VLAN_TAG_SIZE > buf_size) 1172 dev->data->scattered_rx = 1; 1173 } else { 1174 /* 1175 * Use BSIZE field of the device RCTL register. 1176 */ 1177 if (rctl_bsize == 0 || rctl_bsize > buf_size) 1178 rctl_bsize = buf_size; 1179 dev->data->scattered_rx = 1; 1180 } 1181 1182 /* Set if packets are dropped when no descriptors available */ 1183 if (rxq->drop_en) 1184 srrctl |= IGC_SRRCTL_DROP_EN; 1185 1186 IGC_WRITE_REG(hw, IGC_SRRCTL(rxq->reg_idx), srrctl); 1187 1188 /* Enable this RX queue. */ 1189 rxdctl = IGC_RXDCTL_QUEUE_ENABLE; 1190 rxdctl |= ((uint32_t)rxq->pthresh << IGC_RXDCTL_PTHRESH_SHIFT) & 1191 IGC_RXDCTL_PTHRESH_MSK; 1192 rxdctl |= ((uint32_t)rxq->hthresh << IGC_RXDCTL_HTHRESH_SHIFT) & 1193 IGC_RXDCTL_HTHRESH_MSK; 1194 rxdctl |= ((uint32_t)rxq->wthresh << IGC_RXDCTL_WTHRESH_SHIFT) & 1195 IGC_RXDCTL_WTHRESH_MSK; 1196 IGC_WRITE_REG(hw, IGC_RXDCTL(rxq->reg_idx), rxdctl); 1197 } 1198 1199 if (offloads & DEV_RX_OFFLOAD_SCATTER) 1200 dev->data->scattered_rx = 1; 1201 1202 if (dev->data->scattered_rx) { 1203 PMD_DRV_LOG(DEBUG, "forcing scatter mode"); 1204 dev->rx_pkt_burst = igc_recv_scattered_pkts; 1205 } 1206 /* 1207 * Setup BSIZE field of RCTL register, if needed. 1208 * Buffer sizes >= 1024 are not [supposed to be] setup in the RCTL 1209 * register, since the code above configures the SRRCTL register of 1210 * the RX queue in such a case. 1211 * All configurable sizes are: 1212 * 16384: rctl |= (IGC_RCTL_SZ_16384 | IGC_RCTL_BSEX); 1213 * 8192: rctl |= (IGC_RCTL_SZ_8192 | IGC_RCTL_BSEX); 1214 * 4096: rctl |= (IGC_RCTL_SZ_4096 | IGC_RCTL_BSEX); 1215 * 2048: rctl |= IGC_RCTL_SZ_2048; 1216 * 1024: rctl |= IGC_RCTL_SZ_1024; 1217 * 512: rctl |= IGC_RCTL_SZ_512; 1218 * 256: rctl |= IGC_RCTL_SZ_256; 1219 */ 1220 if (rctl_bsize > 0) { 1221 if (rctl_bsize >= 512) /* 512 <= buf_size < 1024 - use 512 */ 1222 rctl |= IGC_RCTL_SZ_512; 1223 else /* 256 <= buf_size < 512 - use 256 */ 1224 rctl |= IGC_RCTL_SZ_256; 1225 } 1226 1227 /* 1228 * Configure RSS if device configured with multiple RX queues. 1229 */ 1230 igc_dev_mq_rx_configure(dev); 1231 1232 /* Update the rctl since igc_dev_mq_rx_configure may change its value */ 1233 rctl |= IGC_READ_REG(hw, IGC_RCTL); 1234 1235 /* 1236 * Setup the Checksum Register. 1237 * Receive Full-Packet Checksum Offload is mutually exclusive with RSS. 1238 */ 1239 rxcsum = IGC_READ_REG(hw, IGC_RXCSUM); 1240 rxcsum |= IGC_RXCSUM_PCSD; 1241 1242 /* Enable both L3/L4 rx checksum offload */ 1243 if (offloads & DEV_RX_OFFLOAD_IPV4_CKSUM) 1244 rxcsum |= IGC_RXCSUM_IPOFL; 1245 else 1246 rxcsum &= ~IGC_RXCSUM_IPOFL; 1247 1248 if (offloads & 1249 (DEV_RX_OFFLOAD_TCP_CKSUM | DEV_RX_OFFLOAD_UDP_CKSUM)) { 1250 rxcsum |= IGC_RXCSUM_TUOFL; 1251 offloads |= DEV_RX_OFFLOAD_SCTP_CKSUM; 1252 } else { 1253 rxcsum &= ~IGC_RXCSUM_TUOFL; 1254 } 1255 1256 if (offloads & DEV_RX_OFFLOAD_SCTP_CKSUM) 1257 rxcsum |= IGC_RXCSUM_CRCOFL; 1258 else 1259 rxcsum &= ~IGC_RXCSUM_CRCOFL; 1260 1261 IGC_WRITE_REG(hw, IGC_RXCSUM, rxcsum); 1262 1263 /* Setup the Receive Control Register. */ 1264 if (offloads & DEV_RX_OFFLOAD_KEEP_CRC) 1265 rctl &= ~IGC_RCTL_SECRC; /* Do not Strip Ethernet CRC. */ 1266 else 1267 rctl |= IGC_RCTL_SECRC; /* Strip Ethernet CRC. */ 1268 1269 rctl &= ~IGC_RCTL_MO_MSK; 1270 rctl &= ~IGC_RCTL_LBM_MSK; 1271 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_LBM_NO | 1272 IGC_RCTL_DPF | 1273 (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 1274 1275 if (dev->data->dev_conf.lpbk_mode == 1) 1276 rctl |= IGC_RCTL_LBM_MAC; 1277 1278 rctl &= ~(IGC_RCTL_HSEL_MSK | IGC_RCTL_CFIEN | IGC_RCTL_CFI | 1279 IGC_RCTL_PSP | IGC_RCTL_PMCF); 1280 1281 /* Make sure VLAN Filters are off. */ 1282 rctl &= ~IGC_RCTL_VFE; 1283 /* Don't store bad packets. */ 1284 rctl &= ~IGC_RCTL_SBP; 1285 1286 /* Enable Receives. */ 1287 IGC_WRITE_REG(hw, IGC_RCTL, rctl); 1288 1289 /* 1290 * Setup the HW Rx Head and Tail Descriptor Pointers. 1291 * This needs to be done after enable. 1292 */ 1293 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1294 uint32_t dvmolr; 1295 1296 rxq = dev->data->rx_queues[i]; 1297 IGC_WRITE_REG(hw, IGC_RDH(rxq->reg_idx), 0); 1298 IGC_WRITE_REG(hw, IGC_RDT(rxq->reg_idx), rxq->nb_rx_desc - 1); 1299 1300 dvmolr = IGC_READ_REG(hw, IGC_DVMOLR(rxq->reg_idx)); 1301 if (rxq->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 1302 dvmolr |= IGC_DVMOLR_STRVLAN; 1303 else 1304 dvmolr &= ~IGC_DVMOLR_STRVLAN; 1305 1306 if (offloads & DEV_RX_OFFLOAD_KEEP_CRC) 1307 dvmolr &= ~IGC_DVMOLR_STRCRC; 1308 else 1309 dvmolr |= IGC_DVMOLR_STRCRC; 1310 1311 IGC_WRITE_REG(hw, IGC_DVMOLR(rxq->reg_idx), dvmolr); 1312 } 1313 1314 return 0; 1315 } 1316 1317 static void 1318 igc_reset_rx_queue(struct igc_rx_queue *rxq) 1319 { 1320 static const union igc_adv_rx_desc zeroed_desc = { {0} }; 1321 unsigned int i; 1322 1323 /* Zero out HW ring memory */ 1324 for (i = 0; i < rxq->nb_rx_desc; i++) 1325 rxq->rx_ring[i] = zeroed_desc; 1326 1327 rxq->rx_tail = 0; 1328 rxq->pkt_first_seg = NULL; 1329 rxq->pkt_last_seg = NULL; 1330 } 1331 1332 int 1333 eth_igc_rx_queue_setup(struct rte_eth_dev *dev, 1334 uint16_t queue_idx, 1335 uint16_t nb_desc, 1336 unsigned int socket_id, 1337 const struct rte_eth_rxconf *rx_conf, 1338 struct rte_mempool *mp) 1339 { 1340 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev); 1341 const struct rte_memzone *rz; 1342 struct igc_rx_queue *rxq; 1343 unsigned int size; 1344 1345 /* 1346 * Validate number of receive descriptors. 1347 * It must not exceed hardware maximum, and must be multiple 1348 * of IGC_RX_DESCRIPTOR_MULTIPLE. 1349 */ 1350 if (nb_desc % IGC_RX_DESCRIPTOR_MULTIPLE != 0 || 1351 nb_desc > IGC_MAX_RXD || nb_desc < IGC_MIN_RXD) { 1352 PMD_DRV_LOG(ERR, 1353 "RX descriptor must be multiple of %u(cur: %u) and between %u and %u", 1354 IGC_RX_DESCRIPTOR_MULTIPLE, nb_desc, 1355 IGC_MIN_RXD, IGC_MAX_RXD); 1356 return -EINVAL; 1357 } 1358 1359 /* Free memory prior to re-allocation if needed */ 1360 if (dev->data->rx_queues[queue_idx] != NULL) { 1361 igc_rx_queue_release(dev->data->rx_queues[queue_idx]); 1362 dev->data->rx_queues[queue_idx] = NULL; 1363 } 1364 1365 /* First allocate the RX queue data structure. */ 1366 rxq = rte_zmalloc("ethdev RX queue", sizeof(struct igc_rx_queue), 1367 RTE_CACHE_LINE_SIZE); 1368 if (rxq == NULL) 1369 return -ENOMEM; 1370 rxq->offloads = rx_conf->offloads; 1371 rxq->mb_pool = mp; 1372 rxq->nb_rx_desc = nb_desc; 1373 rxq->pthresh = rx_conf->rx_thresh.pthresh; 1374 rxq->hthresh = rx_conf->rx_thresh.hthresh; 1375 rxq->wthresh = rx_conf->rx_thresh.wthresh; 1376 rxq->drop_en = rx_conf->rx_drop_en; 1377 rxq->rx_free_thresh = rx_conf->rx_free_thresh; 1378 rxq->queue_id = queue_idx; 1379 rxq->reg_idx = queue_idx; 1380 rxq->port_id = dev->data->port_id; 1381 1382 /* 1383 * Allocate RX ring hardware descriptors. A memzone large enough to 1384 * handle the maximum ring size is allocated in order to allow for 1385 * resizing in later calls to the queue setup function. 1386 */ 1387 size = sizeof(union igc_adv_rx_desc) * IGC_MAX_RXD; 1388 rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx, size, 1389 IGC_ALIGN, socket_id); 1390 if (rz == NULL) { 1391 igc_rx_queue_release(rxq); 1392 return -ENOMEM; 1393 } 1394 rxq->rdt_reg_addr = IGC_PCI_REG_ADDR(hw, IGC_RDT(rxq->reg_idx)); 1395 rxq->rdh_reg_addr = IGC_PCI_REG_ADDR(hw, IGC_RDH(rxq->reg_idx)); 1396 rxq->rx_ring_phys_addr = rz->iova; 1397 rxq->rx_ring = (union igc_adv_rx_desc *)rz->addr; 1398 1399 /* Allocate software ring. */ 1400 rxq->sw_ring = rte_zmalloc("rxq->sw_ring", 1401 sizeof(struct igc_rx_entry) * nb_desc, 1402 RTE_CACHE_LINE_SIZE); 1403 if (rxq->sw_ring == NULL) { 1404 igc_rx_queue_release(rxq); 1405 return -ENOMEM; 1406 } 1407 1408 PMD_DRV_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64, 1409 rxq->sw_ring, rxq->rx_ring, rxq->rx_ring_phys_addr); 1410 1411 dev->data->rx_queues[queue_idx] = rxq; 1412 igc_reset_rx_queue(rxq); 1413 1414 return 0; 1415 } 1416 1417 /* prepare packets for transmit */ 1418 static uint16_t 1419 eth_igc_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, 1420 uint16_t nb_pkts) 1421 { 1422 int i, ret; 1423 struct rte_mbuf *m; 1424 1425 for (i = 0; i < nb_pkts; i++) { 1426 m = tx_pkts[i]; 1427 1428 /* Check some limitations for TSO in hardware */ 1429 if (m->ol_flags & IGC_TX_OFFLOAD_SEG) 1430 if (m->tso_segsz > IGC_TSO_MAX_MSS || 1431 m->l2_len + m->l3_len + m->l4_len > 1432 IGC_TSO_MAX_HDRLEN) { 1433 rte_errno = EINVAL; 1434 return i; 1435 } 1436 1437 if (m->ol_flags & IGC_TX_OFFLOAD_NOTSUP_MASK) { 1438 rte_errno = ENOTSUP; 1439 return i; 1440 } 1441 1442 #ifdef RTE_ETHDEV_DEBUG_TX 1443 ret = rte_validate_tx_offload(m); 1444 if (ret != 0) { 1445 rte_errno = -ret; 1446 return i; 1447 } 1448 #endif 1449 ret = rte_net_intel_cksum_prepare(m); 1450 if (ret != 0) { 1451 rte_errno = -ret; 1452 return i; 1453 } 1454 } 1455 1456 return i; 1457 } 1458 1459 /* 1460 *There're some limitations in hardware for TCP segmentation offload. We 1461 *should check whether the parameters are valid. 1462 */ 1463 static inline uint64_t 1464 check_tso_para(uint64_t ol_req, union igc_tx_offload ol_para) 1465 { 1466 if (!(ol_req & IGC_TX_OFFLOAD_SEG)) 1467 return ol_req; 1468 if (ol_para.tso_segsz > IGC_TSO_MAX_MSS || ol_para.l2_len + 1469 ol_para.l3_len + ol_para.l4_len > IGC_TSO_MAX_HDRLEN) { 1470 ol_req &= ~IGC_TX_OFFLOAD_SEG; 1471 ol_req |= PKT_TX_TCP_CKSUM; 1472 } 1473 return ol_req; 1474 } 1475 1476 /* 1477 * Check which hardware context can be used. Use the existing match 1478 * or create a new context descriptor. 1479 */ 1480 static inline uint32_t 1481 what_advctx_update(struct igc_tx_queue *txq, uint64_t flags, 1482 union igc_tx_offload tx_offload) 1483 { 1484 uint32_t curr = txq->ctx_curr; 1485 1486 /* If match with the current context */ 1487 if (likely(txq->ctx_cache[curr].flags == flags && 1488 txq->ctx_cache[curr].tx_offload.data == 1489 (txq->ctx_cache[curr].tx_offload_mask.data & 1490 tx_offload.data))) { 1491 return curr; 1492 } 1493 1494 /* Total two context, if match with the second context */ 1495 curr ^= 1; 1496 if (likely(txq->ctx_cache[curr].flags == flags && 1497 txq->ctx_cache[curr].tx_offload.data == 1498 (txq->ctx_cache[curr].tx_offload_mask.data & 1499 tx_offload.data))) { 1500 txq->ctx_curr = curr; 1501 return curr; 1502 } 1503 1504 /* Mismatch, create new one */ 1505 return IGC_CTX_NUM; 1506 } 1507 1508 /* 1509 * This is a separate function, looking for optimization opportunity here 1510 * Rework required to go with the pre-defined values. 1511 */ 1512 static inline void 1513 igc_set_xmit_ctx(struct igc_tx_queue *txq, 1514 volatile struct igc_adv_tx_context_desc *ctx_txd, 1515 uint64_t ol_flags, union igc_tx_offload tx_offload) 1516 { 1517 uint32_t type_tucmd_mlhl; 1518 uint32_t mss_l4len_idx; 1519 uint32_t ctx_curr; 1520 uint32_t vlan_macip_lens; 1521 union igc_tx_offload tx_offload_mask; 1522 1523 /* Use the previous context */ 1524 txq->ctx_curr ^= 1; 1525 ctx_curr = txq->ctx_curr; 1526 1527 tx_offload_mask.data = 0; 1528 type_tucmd_mlhl = 0; 1529 1530 /* Specify which HW CTX to upload. */ 1531 mss_l4len_idx = (ctx_curr << IGC_ADVTXD_IDX_SHIFT); 1532 1533 if (ol_flags & PKT_TX_VLAN_PKT) 1534 tx_offload_mask.vlan_tci = 0xffff; 1535 1536 /* check if TCP segmentation required for this packet */ 1537 if (ol_flags & IGC_TX_OFFLOAD_SEG) { 1538 /* implies IP cksum in IPv4 */ 1539 if (ol_flags & PKT_TX_IP_CKSUM) 1540 type_tucmd_mlhl = IGC_ADVTXD_TUCMD_IPV4 | 1541 IGC_ADVTXD_DTYP_CTXT | IGC_ADVTXD_DCMD_DEXT; 1542 else 1543 type_tucmd_mlhl = IGC_ADVTXD_TUCMD_IPV6 | 1544 IGC_ADVTXD_DTYP_CTXT | IGC_ADVTXD_DCMD_DEXT; 1545 1546 if (ol_flags & PKT_TX_TCP_SEG) 1547 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP; 1548 else 1549 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_UDP; 1550 1551 tx_offload_mask.data |= TX_TSO_CMP_MASK; 1552 mss_l4len_idx |= (uint32_t)tx_offload.tso_segsz << 1553 IGC_ADVTXD_MSS_SHIFT; 1554 mss_l4len_idx |= (uint32_t)tx_offload.l4_len << 1555 IGC_ADVTXD_L4LEN_SHIFT; 1556 } else { /* no TSO, check if hardware checksum is needed */ 1557 if (ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_L4_MASK)) 1558 tx_offload_mask.data |= TX_MACIP_LEN_CMP_MASK; 1559 1560 if (ol_flags & PKT_TX_IP_CKSUM) 1561 type_tucmd_mlhl = IGC_ADVTXD_TUCMD_IPV4; 1562 1563 switch (ol_flags & PKT_TX_L4_MASK) { 1564 case PKT_TX_TCP_CKSUM: 1565 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP | 1566 IGC_ADVTXD_DTYP_CTXT | IGC_ADVTXD_DCMD_DEXT; 1567 mss_l4len_idx |= (uint32_t)sizeof(struct rte_tcp_hdr) 1568 << IGC_ADVTXD_L4LEN_SHIFT; 1569 break; 1570 case PKT_TX_UDP_CKSUM: 1571 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_UDP | 1572 IGC_ADVTXD_DTYP_CTXT | IGC_ADVTXD_DCMD_DEXT; 1573 mss_l4len_idx |= (uint32_t)sizeof(struct rte_udp_hdr) 1574 << IGC_ADVTXD_L4LEN_SHIFT; 1575 break; 1576 case PKT_TX_SCTP_CKSUM: 1577 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_SCTP | 1578 IGC_ADVTXD_DTYP_CTXT | IGC_ADVTXD_DCMD_DEXT; 1579 mss_l4len_idx |= (uint32_t)sizeof(struct rte_sctp_hdr) 1580 << IGC_ADVTXD_L4LEN_SHIFT; 1581 break; 1582 default: 1583 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_RSV | 1584 IGC_ADVTXD_DTYP_CTXT | IGC_ADVTXD_DCMD_DEXT; 1585 break; 1586 } 1587 } 1588 1589 txq->ctx_cache[ctx_curr].flags = ol_flags; 1590 txq->ctx_cache[ctx_curr].tx_offload.data = 1591 tx_offload_mask.data & tx_offload.data; 1592 txq->ctx_cache[ctx_curr].tx_offload_mask = tx_offload_mask; 1593 1594 ctx_txd->type_tucmd_mlhl = rte_cpu_to_le_32(type_tucmd_mlhl); 1595 vlan_macip_lens = (uint32_t)tx_offload.data; 1596 ctx_txd->vlan_macip_lens = rte_cpu_to_le_32(vlan_macip_lens); 1597 ctx_txd->mss_l4len_idx = rte_cpu_to_le_32(mss_l4len_idx); 1598 ctx_txd->u.launch_time = 0; 1599 } 1600 1601 static inline uint32_t 1602 tx_desc_vlan_flags_to_cmdtype(uint64_t ol_flags) 1603 { 1604 uint32_t cmdtype; 1605 static uint32_t vlan_cmd[2] = {0, IGC_ADVTXD_DCMD_VLE}; 1606 static uint32_t tso_cmd[2] = {0, IGC_ADVTXD_DCMD_TSE}; 1607 cmdtype = vlan_cmd[(ol_flags & PKT_TX_VLAN_PKT) != 0]; 1608 cmdtype |= tso_cmd[(ol_flags & IGC_TX_OFFLOAD_SEG) != 0]; 1609 return cmdtype; 1610 } 1611 1612 static inline uint32_t 1613 tx_desc_cksum_flags_to_olinfo(uint64_t ol_flags) 1614 { 1615 static const uint32_t l4_olinfo[2] = {0, IGC_ADVTXD_POPTS_TXSM}; 1616 static const uint32_t l3_olinfo[2] = {0, IGC_ADVTXD_POPTS_IXSM}; 1617 uint32_t tmp; 1618 1619 tmp = l4_olinfo[(ol_flags & PKT_TX_L4_MASK) != PKT_TX_L4_NO_CKSUM]; 1620 tmp |= l3_olinfo[(ol_flags & PKT_TX_IP_CKSUM) != 0]; 1621 tmp |= l4_olinfo[(ol_flags & IGC_TX_OFFLOAD_SEG) != 0]; 1622 return tmp; 1623 } 1624 1625 static uint16_t 1626 igc_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 1627 { 1628 struct igc_tx_queue * const txq = tx_queue; 1629 struct igc_tx_entry * const sw_ring = txq->sw_ring; 1630 struct igc_tx_entry *txe, *txn; 1631 volatile union igc_adv_tx_desc * const txr = txq->tx_ring; 1632 volatile union igc_adv_tx_desc *txd; 1633 struct rte_mbuf *tx_pkt; 1634 struct rte_mbuf *m_seg; 1635 uint64_t buf_dma_addr; 1636 uint32_t olinfo_status; 1637 uint32_t cmd_type_len; 1638 uint32_t pkt_len; 1639 uint16_t slen; 1640 uint64_t ol_flags; 1641 uint16_t tx_end; 1642 uint16_t tx_id; 1643 uint16_t tx_last; 1644 uint16_t nb_tx; 1645 uint64_t tx_ol_req; 1646 uint32_t new_ctx = 0; 1647 union igc_tx_offload tx_offload = {0}; 1648 1649 tx_id = txq->tx_tail; 1650 txe = &sw_ring[tx_id]; 1651 1652 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { 1653 tx_pkt = *tx_pkts++; 1654 pkt_len = tx_pkt->pkt_len; 1655 1656 RTE_MBUF_PREFETCH_TO_FREE(txe->mbuf); 1657 1658 /* 1659 * The number of descriptors that must be allocated for a 1660 * packet is the number of segments of that packet, plus 1 1661 * Context Descriptor for the VLAN Tag Identifier, if any. 1662 * Determine the last TX descriptor to allocate in the TX ring 1663 * for the packet, starting from the current position (tx_id) 1664 * in the ring. 1665 */ 1666 tx_last = (uint16_t)(tx_id + tx_pkt->nb_segs - 1); 1667 1668 ol_flags = tx_pkt->ol_flags; 1669 tx_ol_req = ol_flags & IGC_TX_OFFLOAD_MASK; 1670 1671 /* If a Context Descriptor need be built . */ 1672 if (tx_ol_req) { 1673 tx_offload.l2_len = tx_pkt->l2_len; 1674 tx_offload.l3_len = tx_pkt->l3_len; 1675 tx_offload.l4_len = tx_pkt->l4_len; 1676 tx_offload.vlan_tci = tx_pkt->vlan_tci; 1677 tx_offload.tso_segsz = tx_pkt->tso_segsz; 1678 tx_ol_req = check_tso_para(tx_ol_req, tx_offload); 1679 1680 new_ctx = what_advctx_update(txq, tx_ol_req, 1681 tx_offload); 1682 /* Only allocate context descriptor if required*/ 1683 new_ctx = (new_ctx >= IGC_CTX_NUM); 1684 tx_last = (uint16_t)(tx_last + new_ctx); 1685 } 1686 if (tx_last >= txq->nb_tx_desc) 1687 tx_last = (uint16_t)(tx_last - txq->nb_tx_desc); 1688 1689 PMD_TX_LOG(DEBUG, 1690 "port_id=%u queue_id=%u pktlen=%u tx_first=%u tx_last=%u", 1691 txq->port_id, txq->queue_id, pkt_len, tx_id, tx_last); 1692 1693 /* 1694 * Check if there are enough free descriptors in the TX ring 1695 * to transmit the next packet. 1696 * This operation is based on the two following rules: 1697 * 1698 * 1- Only check that the last needed TX descriptor can be 1699 * allocated (by construction, if that descriptor is free, 1700 * all intermediate ones are also free). 1701 * 1702 * For this purpose, the index of the last TX descriptor 1703 * used for a packet (the "last descriptor" of a packet) 1704 * is recorded in the TX entries (the last one included) 1705 * that are associated with all TX descriptors allocated 1706 * for that packet. 1707 * 1708 * 2- Avoid to allocate the last free TX descriptor of the 1709 * ring, in order to never set the TDT register with the 1710 * same value stored in parallel by the NIC in the TDH 1711 * register, which makes the TX engine of the NIC enter 1712 * in a deadlock situation. 1713 * 1714 * By extension, avoid to allocate a free descriptor that 1715 * belongs to the last set of free descriptors allocated 1716 * to the same packet previously transmitted. 1717 */ 1718 1719 /* 1720 * The "last descriptor" of the previously sent packet, if any, 1721 * which used the last descriptor to allocate. 1722 */ 1723 tx_end = sw_ring[tx_last].last_id; 1724 1725 /* 1726 * The next descriptor following that "last descriptor" in the 1727 * ring. 1728 */ 1729 tx_end = sw_ring[tx_end].next_id; 1730 1731 /* 1732 * The "last descriptor" associated with that next descriptor. 1733 */ 1734 tx_end = sw_ring[tx_end].last_id; 1735 1736 /* 1737 * Check that this descriptor is free. 1738 */ 1739 if (!(txr[tx_end].wb.status & IGC_TXD_STAT_DD)) { 1740 if (nb_tx == 0) 1741 return 0; 1742 goto end_of_tx; 1743 } 1744 1745 /* 1746 * Set common flags of all TX Data Descriptors. 1747 * 1748 * The following bits must be set in all Data Descriptors: 1749 * - IGC_ADVTXD_DTYP_DATA 1750 * - IGC_ADVTXD_DCMD_DEXT 1751 * 1752 * The following bits must be set in the first Data Descriptor 1753 * and are ignored in the other ones: 1754 * - IGC_ADVTXD_DCMD_IFCS 1755 * - IGC_ADVTXD_MAC_1588 1756 * - IGC_ADVTXD_DCMD_VLE 1757 * 1758 * The following bits must only be set in the last Data 1759 * Descriptor: 1760 * - IGC_TXD_CMD_EOP 1761 * 1762 * The following bits can be set in any Data Descriptor, but 1763 * are only set in the last Data Descriptor: 1764 * - IGC_TXD_CMD_RS 1765 */ 1766 cmd_type_len = txq->txd_type | 1767 IGC_ADVTXD_DCMD_IFCS | IGC_ADVTXD_DCMD_DEXT; 1768 if (tx_ol_req & IGC_TX_OFFLOAD_SEG) 1769 pkt_len -= (tx_pkt->l2_len + tx_pkt->l3_len + 1770 tx_pkt->l4_len); 1771 olinfo_status = (pkt_len << IGC_ADVTXD_PAYLEN_SHIFT); 1772 1773 /* 1774 * Timer 0 should be used to for packet timestamping, 1775 * sample the packet timestamp to reg 0 1776 */ 1777 if (ol_flags & PKT_TX_IEEE1588_TMST) 1778 cmd_type_len |= IGC_ADVTXD_MAC_TSTAMP; 1779 1780 if (tx_ol_req) { 1781 /* Setup TX Advanced context descriptor if required */ 1782 if (new_ctx) { 1783 volatile struct igc_adv_tx_context_desc * 1784 ctx_txd = (volatile struct 1785 igc_adv_tx_context_desc *)&txr[tx_id]; 1786 1787 txn = &sw_ring[txe->next_id]; 1788 RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf); 1789 1790 if (txe->mbuf != NULL) { 1791 rte_pktmbuf_free_seg(txe->mbuf); 1792 txe->mbuf = NULL; 1793 } 1794 1795 igc_set_xmit_ctx(txq, ctx_txd, tx_ol_req, 1796 tx_offload); 1797 1798 txe->last_id = tx_last; 1799 tx_id = txe->next_id; 1800 txe = txn; 1801 } 1802 1803 /* Setup the TX Advanced Data Descriptor */ 1804 cmd_type_len |= 1805 tx_desc_vlan_flags_to_cmdtype(tx_ol_req); 1806 olinfo_status |= 1807 tx_desc_cksum_flags_to_olinfo(tx_ol_req); 1808 olinfo_status |= (uint32_t)txq->ctx_curr << 1809 IGC_ADVTXD_IDX_SHIFT; 1810 } 1811 1812 m_seg = tx_pkt; 1813 do { 1814 txn = &sw_ring[txe->next_id]; 1815 RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf); 1816 1817 txd = &txr[tx_id]; 1818 1819 if (txe->mbuf != NULL) 1820 rte_pktmbuf_free_seg(txe->mbuf); 1821 txe->mbuf = m_seg; 1822 1823 /* Set up transmit descriptor */ 1824 slen = (uint16_t)m_seg->data_len; 1825 buf_dma_addr = rte_mbuf_data_iova(m_seg); 1826 txd->read.buffer_addr = 1827 rte_cpu_to_le_64(buf_dma_addr); 1828 txd->read.cmd_type_len = 1829 rte_cpu_to_le_32(cmd_type_len | slen); 1830 txd->read.olinfo_status = 1831 rte_cpu_to_le_32(olinfo_status); 1832 txe->last_id = tx_last; 1833 tx_id = txe->next_id; 1834 txe = txn; 1835 m_seg = m_seg->next; 1836 } while (m_seg != NULL); 1837 1838 /* 1839 * The last packet data descriptor needs End Of Packet (EOP) 1840 * and Report Status (RS). 1841 */ 1842 txd->read.cmd_type_len |= 1843 rte_cpu_to_le_32(IGC_TXD_CMD_EOP | IGC_TXD_CMD_RS); 1844 } 1845 end_of_tx: 1846 rte_wmb(); 1847 1848 /* 1849 * Set the Transmit Descriptor Tail (TDT). 1850 */ 1851 IGC_PCI_REG_WRITE_RELAXED(txq->tdt_reg_addr, tx_id); 1852 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u", 1853 txq->port_id, txq->queue_id, tx_id, nb_tx); 1854 txq->tx_tail = tx_id; 1855 1856 return nb_tx; 1857 } 1858 1859 int eth_igc_tx_descriptor_status(void *tx_queue, uint16_t offset) 1860 { 1861 struct igc_tx_queue *txq = tx_queue; 1862 volatile uint32_t *status; 1863 uint32_t desc; 1864 1865 if (unlikely(!txq || offset >= txq->nb_tx_desc)) 1866 return -EINVAL; 1867 1868 desc = txq->tx_tail + offset; 1869 if (desc >= txq->nb_tx_desc) 1870 desc -= txq->nb_tx_desc; 1871 1872 status = &txq->tx_ring[desc].wb.status; 1873 if (*status & rte_cpu_to_le_32(IGC_TXD_STAT_DD)) 1874 return RTE_ETH_TX_DESC_DONE; 1875 1876 return RTE_ETH_TX_DESC_FULL; 1877 } 1878 1879 static void 1880 igc_tx_queue_release_mbufs(struct igc_tx_queue *txq) 1881 { 1882 unsigned int i; 1883 1884 if (txq->sw_ring != NULL) { 1885 for (i = 0; i < txq->nb_tx_desc; i++) { 1886 if (txq->sw_ring[i].mbuf != NULL) { 1887 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf); 1888 txq->sw_ring[i].mbuf = NULL; 1889 } 1890 } 1891 } 1892 } 1893 1894 static void 1895 igc_tx_queue_release(struct igc_tx_queue *txq) 1896 { 1897 igc_tx_queue_release_mbufs(txq); 1898 rte_free(txq->sw_ring); 1899 rte_free(txq); 1900 } 1901 1902 void eth_igc_tx_queue_release(void *txq) 1903 { 1904 if (txq) 1905 igc_tx_queue_release(txq); 1906 } 1907 1908 static void 1909 igc_reset_tx_queue_stat(struct igc_tx_queue *txq) 1910 { 1911 txq->tx_head = 0; 1912 txq->tx_tail = 0; 1913 txq->ctx_curr = 0; 1914 memset((void *)&txq->ctx_cache, 0, 1915 IGC_CTX_NUM * sizeof(struct igc_advctx_info)); 1916 } 1917 1918 static void 1919 igc_reset_tx_queue(struct igc_tx_queue *txq) 1920 { 1921 struct igc_tx_entry *txe = txq->sw_ring; 1922 uint16_t i, prev; 1923 1924 /* Initialize ring entries */ 1925 prev = (uint16_t)(txq->nb_tx_desc - 1); 1926 for (i = 0; i < txq->nb_tx_desc; i++) { 1927 volatile union igc_adv_tx_desc *txd = &txq->tx_ring[i]; 1928 1929 txd->wb.status = IGC_TXD_STAT_DD; 1930 txe[i].mbuf = NULL; 1931 txe[i].last_id = i; 1932 txe[prev].next_id = i; 1933 prev = i; 1934 } 1935 1936 txq->txd_type = IGC_ADVTXD_DTYP_DATA; 1937 igc_reset_tx_queue_stat(txq); 1938 } 1939 1940 /* 1941 * clear all rx/tx queue 1942 */ 1943 void 1944 igc_dev_clear_queues(struct rte_eth_dev *dev) 1945 { 1946 uint16_t i; 1947 struct igc_tx_queue *txq; 1948 struct igc_rx_queue *rxq; 1949 1950 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1951 txq = dev->data->tx_queues[i]; 1952 if (txq != NULL) { 1953 igc_tx_queue_release_mbufs(txq); 1954 igc_reset_tx_queue(txq); 1955 } 1956 } 1957 1958 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1959 rxq = dev->data->rx_queues[i]; 1960 if (rxq != NULL) { 1961 igc_rx_queue_release_mbufs(rxq); 1962 igc_reset_rx_queue(rxq); 1963 } 1964 } 1965 } 1966 1967 int eth_igc_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, 1968 uint16_t nb_desc, unsigned int socket_id, 1969 const struct rte_eth_txconf *tx_conf) 1970 { 1971 const struct rte_memzone *tz; 1972 struct igc_tx_queue *txq; 1973 struct igc_hw *hw; 1974 uint32_t size; 1975 1976 if (nb_desc % IGC_TX_DESCRIPTOR_MULTIPLE != 0 || 1977 nb_desc > IGC_MAX_TXD || nb_desc < IGC_MIN_TXD) { 1978 PMD_DRV_LOG(ERR, 1979 "TX-descriptor must be a multiple of %u and between %u and %u, cur: %u", 1980 IGC_TX_DESCRIPTOR_MULTIPLE, 1981 IGC_MAX_TXD, IGC_MIN_TXD, nb_desc); 1982 return -EINVAL; 1983 } 1984 1985 hw = IGC_DEV_PRIVATE_HW(dev); 1986 1987 /* 1988 * The tx_free_thresh and tx_rs_thresh values are not used in the 2.5G 1989 * driver. 1990 */ 1991 if (tx_conf->tx_free_thresh != 0) 1992 PMD_DRV_LOG(INFO, 1993 "The tx_free_thresh parameter is not used for the 2.5G driver"); 1994 if (tx_conf->tx_rs_thresh != 0) 1995 PMD_DRV_LOG(INFO, 1996 "The tx_rs_thresh parameter is not used for the 2.5G driver"); 1997 if (tx_conf->tx_thresh.wthresh == 0) 1998 PMD_DRV_LOG(INFO, 1999 "To improve 2.5G driver performance, consider setting the TX WTHRESH value to 4, 8, or 16."); 2000 2001 /* Free memory prior to re-allocation if needed */ 2002 if (dev->data->tx_queues[queue_idx] != NULL) { 2003 igc_tx_queue_release(dev->data->tx_queues[queue_idx]); 2004 dev->data->tx_queues[queue_idx] = NULL; 2005 } 2006 2007 /* First allocate the tx queue data structure */ 2008 txq = rte_zmalloc("ethdev TX queue", sizeof(struct igc_tx_queue), 2009 RTE_CACHE_LINE_SIZE); 2010 if (txq == NULL) 2011 return -ENOMEM; 2012 2013 /* 2014 * Allocate TX ring hardware descriptors. A memzone large enough to 2015 * handle the maximum ring size is allocated in order to allow for 2016 * resizing in later calls to the queue setup function. 2017 */ 2018 size = sizeof(union igc_adv_tx_desc) * IGC_MAX_TXD; 2019 tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx, size, 2020 IGC_ALIGN, socket_id); 2021 if (tz == NULL) { 2022 igc_tx_queue_release(txq); 2023 return -ENOMEM; 2024 } 2025 2026 txq->nb_tx_desc = nb_desc; 2027 txq->pthresh = tx_conf->tx_thresh.pthresh; 2028 txq->hthresh = tx_conf->tx_thresh.hthresh; 2029 txq->wthresh = tx_conf->tx_thresh.wthresh; 2030 2031 txq->queue_id = queue_idx; 2032 txq->reg_idx = queue_idx; 2033 txq->port_id = dev->data->port_id; 2034 2035 txq->tdt_reg_addr = IGC_PCI_REG_ADDR(hw, IGC_TDT(txq->reg_idx)); 2036 txq->tx_ring_phys_addr = tz->iova; 2037 2038 txq->tx_ring = (union igc_adv_tx_desc *)tz->addr; 2039 /* Allocate software ring */ 2040 txq->sw_ring = rte_zmalloc("txq->sw_ring", 2041 sizeof(struct igc_tx_entry) * nb_desc, 2042 RTE_CACHE_LINE_SIZE); 2043 if (txq->sw_ring == NULL) { 2044 igc_tx_queue_release(txq); 2045 return -ENOMEM; 2046 } 2047 PMD_DRV_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%" PRIx64, 2048 txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); 2049 2050 igc_reset_tx_queue(txq); 2051 dev->tx_pkt_burst = igc_xmit_pkts; 2052 dev->tx_pkt_prepare = ð_igc_prep_pkts; 2053 dev->data->tx_queues[queue_idx] = txq; 2054 txq->offloads = tx_conf->offloads; 2055 2056 return 0; 2057 } 2058 2059 int 2060 eth_igc_tx_done_cleanup(void *txqueue, uint32_t free_cnt) 2061 { 2062 struct igc_tx_queue *txq = txqueue; 2063 struct igc_tx_entry *sw_ring; 2064 volatile union igc_adv_tx_desc *txr; 2065 uint16_t tx_first; /* First segment analyzed. */ 2066 uint16_t tx_id; /* Current segment being processed. */ 2067 uint16_t tx_last; /* Last segment in the current packet. */ 2068 uint16_t tx_next; /* First segment of the next packet. */ 2069 uint32_t count; 2070 2071 if (txq == NULL) 2072 return -ENODEV; 2073 2074 count = 0; 2075 sw_ring = txq->sw_ring; 2076 txr = txq->tx_ring; 2077 2078 /* 2079 * tx_tail is the last sent packet on the sw_ring. Goto the end 2080 * of that packet (the last segment in the packet chain) and 2081 * then the next segment will be the start of the oldest segment 2082 * in the sw_ring. This is the first packet that will be 2083 * attempted to be freed. 2084 */ 2085 2086 /* Get last segment in most recently added packet. */ 2087 tx_first = sw_ring[txq->tx_tail].last_id; 2088 2089 /* Get the next segment, which is the oldest segment in ring. */ 2090 tx_first = sw_ring[tx_first].next_id; 2091 2092 /* Set the current index to the first. */ 2093 tx_id = tx_first; 2094 2095 /* 2096 * Loop through each packet. For each packet, verify that an 2097 * mbuf exists and that the last segment is free. If so, free 2098 * it and move on. 2099 */ 2100 while (1) { 2101 tx_last = sw_ring[tx_id].last_id; 2102 2103 if (sw_ring[tx_last].mbuf) { 2104 if (!(txr[tx_last].wb.status & 2105 rte_cpu_to_le_32(IGC_TXD_STAT_DD))) 2106 break; 2107 2108 /* Get the start of the next packet. */ 2109 tx_next = sw_ring[tx_last].next_id; 2110 2111 /* 2112 * Loop through all segments in a 2113 * packet. 2114 */ 2115 do { 2116 rte_pktmbuf_free_seg(sw_ring[tx_id].mbuf); 2117 sw_ring[tx_id].mbuf = NULL; 2118 sw_ring[tx_id].last_id = tx_id; 2119 2120 /* Move to next segemnt. */ 2121 tx_id = sw_ring[tx_id].next_id; 2122 } while (tx_id != tx_next); 2123 2124 /* 2125 * Increment the number of packets 2126 * freed. 2127 */ 2128 count++; 2129 if (unlikely(count == free_cnt)) 2130 break; 2131 } else { 2132 /* 2133 * There are multiple reasons to be here: 2134 * 1) All the packets on the ring have been 2135 * freed - tx_id is equal to tx_first 2136 * and some packets have been freed. 2137 * - Done, exit 2138 * 2) Interfaces has not sent a rings worth of 2139 * packets yet, so the segment after tail is 2140 * still empty. Or a previous call to this 2141 * function freed some of the segments but 2142 * not all so there is a hole in the list. 2143 * Hopefully this is a rare case. 2144 * - Walk the list and find the next mbuf. If 2145 * there isn't one, then done. 2146 */ 2147 if (likely(tx_id == tx_first && count != 0)) 2148 break; 2149 2150 /* 2151 * Walk the list and find the next mbuf, if any. 2152 */ 2153 do { 2154 /* Move to next segemnt. */ 2155 tx_id = sw_ring[tx_id].next_id; 2156 2157 if (sw_ring[tx_id].mbuf) 2158 break; 2159 2160 } while (tx_id != tx_first); 2161 2162 /* 2163 * Determine why previous loop bailed. If there 2164 * is not an mbuf, done. 2165 */ 2166 if (sw_ring[tx_id].mbuf == NULL) 2167 break; 2168 } 2169 } 2170 2171 return count; 2172 } 2173 2174 void 2175 igc_tx_init(struct rte_eth_dev *dev) 2176 { 2177 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev); 2178 uint32_t tctl; 2179 uint32_t txdctl; 2180 uint16_t i; 2181 2182 /* Setup the Base and Length of the Tx Descriptor Rings. */ 2183 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2184 struct igc_tx_queue *txq = dev->data->tx_queues[i]; 2185 uint64_t bus_addr = txq->tx_ring_phys_addr; 2186 2187 IGC_WRITE_REG(hw, IGC_TDLEN(txq->reg_idx), 2188 txq->nb_tx_desc * 2189 sizeof(union igc_adv_tx_desc)); 2190 IGC_WRITE_REG(hw, IGC_TDBAH(txq->reg_idx), 2191 (uint32_t)(bus_addr >> 32)); 2192 IGC_WRITE_REG(hw, IGC_TDBAL(txq->reg_idx), 2193 (uint32_t)bus_addr); 2194 2195 /* Setup the HW Tx Head and Tail descriptor pointers. */ 2196 IGC_WRITE_REG(hw, IGC_TDT(txq->reg_idx), 0); 2197 IGC_WRITE_REG(hw, IGC_TDH(txq->reg_idx), 0); 2198 2199 /* Setup Transmit threshold registers. */ 2200 txdctl = ((uint32_t)txq->pthresh << IGC_TXDCTL_PTHRESH_SHIFT) & 2201 IGC_TXDCTL_PTHRESH_MSK; 2202 txdctl |= ((uint32_t)txq->hthresh << IGC_TXDCTL_HTHRESH_SHIFT) & 2203 IGC_TXDCTL_HTHRESH_MSK; 2204 txdctl |= ((uint32_t)txq->wthresh << IGC_TXDCTL_WTHRESH_SHIFT) & 2205 IGC_TXDCTL_WTHRESH_MSK; 2206 txdctl |= IGC_TXDCTL_QUEUE_ENABLE; 2207 IGC_WRITE_REG(hw, IGC_TXDCTL(txq->reg_idx), txdctl); 2208 } 2209 2210 igc_config_collision_dist(hw); 2211 2212 /* Program the Transmit Control Register. */ 2213 tctl = IGC_READ_REG(hw, IGC_TCTL); 2214 tctl &= ~IGC_TCTL_CT; 2215 tctl |= (IGC_TCTL_PSP | IGC_TCTL_RTLC | IGC_TCTL_EN | 2216 ((uint32_t)IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT)); 2217 2218 /* This write will effectively turn on the transmit unit. */ 2219 IGC_WRITE_REG(hw, IGC_TCTL, tctl); 2220 } 2221 2222 void 2223 eth_igc_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 2224 struct rte_eth_rxq_info *qinfo) 2225 { 2226 struct igc_rx_queue *rxq; 2227 2228 rxq = dev->data->rx_queues[queue_id]; 2229 2230 qinfo->mp = rxq->mb_pool; 2231 qinfo->scattered_rx = dev->data->scattered_rx; 2232 qinfo->nb_desc = rxq->nb_rx_desc; 2233 2234 qinfo->conf.rx_free_thresh = rxq->rx_free_thresh; 2235 qinfo->conf.rx_drop_en = rxq->drop_en; 2236 qinfo->conf.offloads = rxq->offloads; 2237 qinfo->conf.rx_thresh.hthresh = rxq->hthresh; 2238 qinfo->conf.rx_thresh.pthresh = rxq->pthresh; 2239 qinfo->conf.rx_thresh.wthresh = rxq->wthresh; 2240 } 2241 2242 void 2243 eth_igc_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 2244 struct rte_eth_txq_info *qinfo) 2245 { 2246 struct igc_tx_queue *txq; 2247 2248 txq = dev->data->tx_queues[queue_id]; 2249 2250 qinfo->nb_desc = txq->nb_tx_desc; 2251 2252 qinfo->conf.tx_thresh.pthresh = txq->pthresh; 2253 qinfo->conf.tx_thresh.hthresh = txq->hthresh; 2254 qinfo->conf.tx_thresh.wthresh = txq->wthresh; 2255 qinfo->conf.offloads = txq->offloads; 2256 } 2257 2258 void 2259 eth_igc_vlan_strip_queue_set(struct rte_eth_dev *dev, 2260 uint16_t rx_queue_id, int on) 2261 { 2262 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev); 2263 struct igc_rx_queue *rxq = dev->data->rx_queues[rx_queue_id]; 2264 uint32_t reg_val; 2265 2266 if (rx_queue_id >= IGC_QUEUE_PAIRS_NUM) { 2267 PMD_DRV_LOG(ERR, "Queue index(%u) illegal, max is %u", 2268 rx_queue_id, IGC_QUEUE_PAIRS_NUM - 1); 2269 return; 2270 } 2271 2272 reg_val = IGC_READ_REG(hw, IGC_DVMOLR(rx_queue_id)); 2273 if (on) { 2274 reg_val |= IGC_DVMOLR_STRVLAN; 2275 rxq->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP; 2276 } else { 2277 reg_val &= ~(IGC_DVMOLR_STRVLAN | IGC_DVMOLR_HIDVLAN); 2278 rxq->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP; 2279 } 2280 2281 IGC_WRITE_REG(hw, IGC_DVMOLR(rx_queue_id), reg_val); 2282 } 2283