1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) Broadcom Limited. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Broadcom Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <inttypes.h> 35 #include <stdbool.h> 36 37 #include <rte_bitmap.h> 38 #include <rte_byteorder.h> 39 #include <rte_malloc.h> 40 #include <rte_memory.h> 41 42 #include "bnxt.h" 43 #include "bnxt_cpr.h" 44 #include "bnxt_ring.h" 45 #include "bnxt_rxr.h" 46 #include "bnxt_rxq.h" 47 #include "hsi_struct_def_dpdk.h" 48 49 /* 50 * RX Ring handling 51 */ 52 53 static inline struct rte_mbuf *__bnxt_alloc_rx_data(struct rte_mempool *mb) 54 { 55 struct rte_mbuf *data; 56 57 data = rte_mbuf_raw_alloc(mb); 58 59 return data; 60 } 61 62 static inline int bnxt_alloc_rx_data(struct bnxt_rx_queue *rxq, 63 struct bnxt_rx_ring_info *rxr, 64 uint16_t prod) 65 { 66 struct rx_prod_pkt_bd *rxbd = &rxr->rx_desc_ring[prod]; 67 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod]; 68 struct rte_mbuf *data; 69 70 data = __bnxt_alloc_rx_data(rxq->mb_pool); 71 if (!data) { 72 rte_atomic64_inc(&rxq->bp->rx_mbuf_alloc_fail); 73 return -ENOMEM; 74 } 75 76 rx_buf->mbuf = data; 77 78 rxbd->addr = rte_cpu_to_le_64(RTE_MBUF_DATA_DMA_ADDR(rx_buf->mbuf)); 79 80 return 0; 81 } 82 83 static inline int bnxt_alloc_ag_data(struct bnxt_rx_queue *rxq, 84 struct bnxt_rx_ring_info *rxr, 85 uint16_t prod) 86 { 87 struct rx_prod_pkt_bd *rxbd = &rxr->ag_desc_ring[prod]; 88 struct bnxt_sw_rx_bd *rx_buf = &rxr->ag_buf_ring[prod]; 89 struct rte_mbuf *data; 90 91 data = __bnxt_alloc_rx_data(rxq->mb_pool); 92 if (!data) { 93 rte_atomic64_inc(&rxq->bp->rx_mbuf_alloc_fail); 94 return -ENOMEM; 95 } 96 97 if (rxbd == NULL) 98 RTE_LOG(ERR, PMD, "Jumbo Frame. rxbd is NULL\n"); 99 if (rx_buf == NULL) 100 RTE_LOG(ERR, PMD, "Jumbo Frame. rx_buf is NULL\n"); 101 102 103 rx_buf->mbuf = data; 104 105 rxbd->addr = rte_cpu_to_le_64(RTE_MBUF_DATA_DMA_ADDR(rx_buf->mbuf)); 106 107 return 0; 108 } 109 110 static inline void bnxt_reuse_rx_mbuf(struct bnxt_rx_ring_info *rxr, 111 struct rte_mbuf *mbuf) 112 { 113 uint16_t prod = RING_NEXT(rxr->rx_ring_struct, rxr->rx_prod); 114 struct bnxt_sw_rx_bd *prod_rx_buf; 115 struct rx_prod_pkt_bd *prod_bd; 116 117 prod_rx_buf = &rxr->rx_buf_ring[prod]; 118 119 RTE_ASSERT(prod_rx_buf->mbuf == NULL); 120 RTE_ASSERT(mbuf != NULL); 121 122 prod_rx_buf->mbuf = mbuf; 123 124 prod_bd = &rxr->rx_desc_ring[prod]; 125 126 prod_bd->addr = rte_cpu_to_le_64(RTE_MBUF_DATA_DMA_ADDR(mbuf)); 127 128 rxr->rx_prod = prod; 129 } 130 131 #ifdef BNXT_DEBUG 132 static void bnxt_reuse_ag_mbuf(struct bnxt_rx_ring_info *rxr, uint16_t cons, 133 struct rte_mbuf *mbuf) 134 { 135 uint16_t prod = rxr->ag_prod; 136 struct bnxt_sw_rx_bd *prod_rx_buf; 137 struct rx_prod_pkt_bd *prod_bd, *cons_bd; 138 139 prod_rx_buf = &rxr->ag_buf_ring[prod]; 140 141 prod_rx_buf->mbuf = mbuf; 142 143 prod_bd = &rxr->ag_desc_ring[prod]; 144 cons_bd = &rxr->ag_desc_ring[cons]; 145 146 prod_bd->addr = cons_bd->addr; 147 } 148 #endif 149 150 static inline 151 struct rte_mbuf *bnxt_consume_rx_buf(struct bnxt_rx_ring_info *rxr, 152 uint16_t cons) 153 { 154 struct bnxt_sw_rx_bd *cons_rx_buf; 155 struct rte_mbuf *mbuf; 156 157 cons_rx_buf = &rxr->rx_buf_ring[cons]; 158 RTE_ASSERT(cons_rx_buf->mbuf != NULL); 159 mbuf = cons_rx_buf->mbuf; 160 cons_rx_buf->mbuf = NULL; 161 return mbuf; 162 } 163 164 static void bnxt_tpa_start(struct bnxt_rx_queue *rxq, 165 struct rx_tpa_start_cmpl *tpa_start, 166 struct rx_tpa_start_cmpl_hi *tpa_start1) 167 { 168 struct bnxt_rx_ring_info *rxr = rxq->rx_ring; 169 uint8_t agg_id = rte_le_to_cpu_32(tpa_start->agg_id & 170 RX_TPA_START_CMPL_AGG_ID_MASK) >> RX_TPA_START_CMPL_AGG_ID_SFT; 171 uint16_t data_cons; 172 struct bnxt_tpa_info *tpa_info; 173 struct rte_mbuf *mbuf; 174 175 data_cons = tpa_start->opaque; 176 tpa_info = &rxr->tpa_info[agg_id]; 177 178 mbuf = bnxt_consume_rx_buf(rxr, data_cons); 179 180 bnxt_reuse_rx_mbuf(rxr, tpa_info->mbuf); 181 182 tpa_info->mbuf = mbuf; 183 tpa_info->len = rte_le_to_cpu_32(tpa_start->len); 184 185 mbuf->nb_segs = 1; 186 mbuf->next = NULL; 187 mbuf->pkt_len = rte_le_to_cpu_32(tpa_start->len); 188 mbuf->data_len = mbuf->pkt_len; 189 mbuf->port = rxq->port_id; 190 mbuf->ol_flags = PKT_RX_LRO; 191 if (likely(tpa_start->flags_type & 192 rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS_RSS_VALID))) { 193 mbuf->hash.rss = rte_le_to_cpu_32(tpa_start->rss_hash); 194 mbuf->ol_flags |= PKT_RX_RSS_HASH; 195 } else { 196 mbuf->hash.fdir.id = rte_le_to_cpu_16(tpa_start1->cfa_code); 197 mbuf->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID; 198 } 199 if (tpa_start1->flags2 & 200 rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS2_META_FORMAT_VLAN)) { 201 mbuf->vlan_tci = rte_le_to_cpu_32(tpa_start1->metadata); 202 mbuf->ol_flags |= PKT_RX_VLAN; 203 } 204 if (likely(tpa_start1->flags2 & 205 rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS2_L4_CS_CALC))) 206 mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD; 207 208 /* recycle next mbuf */ 209 data_cons = RING_NEXT(rxr->rx_ring_struct, data_cons); 210 bnxt_reuse_rx_mbuf(rxr, bnxt_consume_rx_buf(rxr, data_cons)); 211 } 212 213 static int bnxt_agg_bufs_valid(struct bnxt_cp_ring_info *cpr, 214 uint8_t agg_bufs, uint32_t raw_cp_cons) 215 { 216 uint16_t last_cp_cons; 217 struct rx_pkt_cmpl *agg_cmpl; 218 219 raw_cp_cons = ADV_RAW_CMP(raw_cp_cons, agg_bufs); 220 last_cp_cons = RING_CMP(cpr->cp_ring_struct, raw_cp_cons); 221 agg_cmpl = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[last_cp_cons]; 222 cpr->valid = FLIP_VALID(raw_cp_cons, 223 cpr->cp_ring_struct->ring_mask, 224 cpr->valid); 225 return CMP_VALID(agg_cmpl, raw_cp_cons, cpr->cp_ring_struct); 226 } 227 228 /* TPA consume agg buffer out of order, allocate connected data only */ 229 static int bnxt_prod_ag_mbuf(struct bnxt_rx_queue *rxq) 230 { 231 struct bnxt_rx_ring_info *rxr = rxq->rx_ring; 232 uint16_t next = RING_NEXT(rxr->ag_ring_struct, rxr->ag_prod); 233 234 /* TODO batch allocation for better performance */ 235 while (rte_bitmap_get(rxr->ag_bitmap, next)) { 236 if (unlikely(bnxt_alloc_ag_data(rxq, rxr, next))) { 237 RTE_LOG(ERR, PMD, 238 "agg mbuf alloc failed: prod=0x%x\n", next); 239 break; 240 } 241 rte_bitmap_clear(rxr->ag_bitmap, next); 242 rxr->ag_prod = next; 243 next = RING_NEXT(rxr->ag_ring_struct, next); 244 } 245 246 return 0; 247 } 248 249 static int bnxt_rx_pages(struct bnxt_rx_queue *rxq, 250 struct rte_mbuf *mbuf, uint32_t *tmp_raw_cons, 251 uint8_t agg_buf) 252 { 253 struct bnxt_cp_ring_info *cpr = rxq->cp_ring; 254 struct bnxt_rx_ring_info *rxr = rxq->rx_ring; 255 int i; 256 uint16_t cp_cons, ag_cons; 257 struct rx_pkt_cmpl *rxcmp; 258 struct rte_mbuf *last = mbuf; 259 260 for (i = 0; i < agg_buf; i++) { 261 struct bnxt_sw_rx_bd *ag_buf; 262 struct rte_mbuf *ag_mbuf; 263 *tmp_raw_cons = NEXT_RAW_CMP(*tmp_raw_cons); 264 cp_cons = RING_CMP(cpr->cp_ring_struct, *tmp_raw_cons); 265 rxcmp = (struct rx_pkt_cmpl *) 266 &cpr->cp_desc_ring[cp_cons]; 267 268 #ifdef BNXT_DEBUG 269 bnxt_dump_cmpl(cp_cons, rxcmp); 270 #endif 271 272 ag_cons = rxcmp->opaque; 273 RTE_ASSERT(ag_cons <= rxr->ag_ring_struct->ring_mask); 274 ag_buf = &rxr->ag_buf_ring[ag_cons]; 275 ag_mbuf = ag_buf->mbuf; 276 RTE_ASSERT(ag_mbuf != NULL); 277 278 ag_mbuf->data_len = rte_le_to_cpu_16(rxcmp->len); 279 280 mbuf->nb_segs++; 281 mbuf->pkt_len += ag_mbuf->data_len; 282 283 last->next = ag_mbuf; 284 last = ag_mbuf; 285 286 ag_buf->mbuf = NULL; 287 288 /* 289 * As aggregation buffer consumed out of order in TPA module, 290 * use bitmap to track freed slots to be allocated and notified 291 * to NIC 292 */ 293 rte_bitmap_set(rxr->ag_bitmap, ag_cons); 294 } 295 bnxt_prod_ag_mbuf(rxq); 296 return 0; 297 } 298 299 static inline struct rte_mbuf *bnxt_tpa_end( 300 struct bnxt_rx_queue *rxq, 301 uint32_t *raw_cp_cons, 302 struct rx_tpa_end_cmpl *tpa_end, 303 struct rx_tpa_end_cmpl_hi *tpa_end1 __rte_unused) 304 { 305 struct bnxt_cp_ring_info *cpr = rxq->cp_ring; 306 struct bnxt_rx_ring_info *rxr = rxq->rx_ring; 307 uint8_t agg_id = (tpa_end->agg_id & RX_TPA_END_CMPL_AGG_ID_MASK) 308 >> RX_TPA_END_CMPL_AGG_ID_SFT; 309 struct rte_mbuf *mbuf; 310 uint8_t agg_bufs; 311 struct bnxt_tpa_info *tpa_info; 312 313 tpa_info = &rxr->tpa_info[agg_id]; 314 mbuf = tpa_info->mbuf; 315 RTE_ASSERT(mbuf != NULL); 316 317 rte_prefetch0(mbuf); 318 agg_bufs = (rte_le_to_cpu_32(tpa_end->agg_bufs_v1) & 319 RX_TPA_END_CMPL_AGG_BUFS_MASK) >> RX_TPA_END_CMPL_AGG_BUFS_SFT; 320 if (agg_bufs) { 321 if (!bnxt_agg_bufs_valid(cpr, agg_bufs, *raw_cp_cons)) 322 return NULL; 323 bnxt_rx_pages(rxq, mbuf, raw_cp_cons, agg_bufs); 324 } 325 mbuf->l4_len = tpa_end->payload_offset; 326 327 struct rte_mbuf *new_data = __bnxt_alloc_rx_data(rxq->mb_pool); 328 RTE_ASSERT(new_data != NULL); 329 if (!new_data) { 330 rte_atomic64_inc(&rxq->bp->rx_mbuf_alloc_fail); 331 return NULL; 332 } 333 tpa_info->mbuf = new_data; 334 335 return mbuf; 336 } 337 338 static uint32_t 339 bnxt_parse_pkt_type(struct rx_pkt_cmpl *rxcmp, struct rx_pkt_cmpl_hi *rxcmp1) 340 { 341 uint32_t pkt_type = 0; 342 uint32_t t_ipcs = 0, ip = 0, ip6 = 0; 343 uint32_t tcp = 0, udp = 0, icmp = 0; 344 uint32_t vlan = 0; 345 346 vlan = !!(rxcmp1->flags2 & 347 rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN)); 348 t_ipcs = !!(rxcmp1->flags2 & 349 rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC)); 350 ip6 = !!(rxcmp1->flags2 & 351 rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_IP_TYPE)); 352 icmp = !!(rxcmp->flags_type & 353 rte_cpu_to_le_16(RX_PKT_CMPL_FLAGS_ITYPE_ICMP)); 354 tcp = !!(rxcmp->flags_type & 355 rte_cpu_to_le_16(RX_PKT_CMPL_FLAGS_ITYPE_TCP)); 356 udp = !!(rxcmp->flags_type & 357 rte_cpu_to_le_16(RX_PKT_CMPL_FLAGS_ITYPE_UDP)); 358 ip = !!(rxcmp->flags_type & 359 rte_cpu_to_le_16(RX_PKT_CMPL_FLAGS_ITYPE_IP)); 360 361 pkt_type |= ((ip || tcp || udp || icmp) && !t_ipcs && !ip6) ? 362 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN : 0; 363 pkt_type |= ((ip || tcp || udp || icmp) && !t_ipcs && ip6) ? 364 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN : 0; 365 pkt_type |= (!t_ipcs && icmp) ? RTE_PTYPE_L4_ICMP : 0; 366 pkt_type |= (!t_ipcs && udp) ? RTE_PTYPE_L4_UDP : 0; 367 pkt_type |= (!t_ipcs && tcp) ? RTE_PTYPE_L4_TCP : 0; 368 pkt_type |= ((ip || tcp || udp || icmp) && t_ipcs && !ip6) ? 369 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN : 0; 370 pkt_type |= ((ip || tcp || udp || icmp) && t_ipcs && ip6) ? 371 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN : 0; 372 pkt_type |= (t_ipcs && icmp) ? RTE_PTYPE_INNER_L4_ICMP : 0; 373 pkt_type |= (t_ipcs && udp) ? RTE_PTYPE_INNER_L4_UDP : 0; 374 pkt_type |= (t_ipcs && tcp) ? RTE_PTYPE_INNER_L4_TCP : 0; 375 pkt_type |= vlan ? RTE_PTYPE_L2_ETHER_VLAN : 0; 376 377 return pkt_type; 378 } 379 380 static int bnxt_rx_pkt(struct rte_mbuf **rx_pkt, 381 struct bnxt_rx_queue *rxq, uint32_t *raw_cons) 382 { 383 struct bnxt_cp_ring_info *cpr = rxq->cp_ring; 384 struct bnxt_rx_ring_info *rxr = rxq->rx_ring; 385 struct rx_pkt_cmpl *rxcmp; 386 struct rx_pkt_cmpl_hi *rxcmp1; 387 uint32_t tmp_raw_cons = *raw_cons; 388 uint16_t cons, prod, cp_cons = 389 RING_CMP(cpr->cp_ring_struct, tmp_raw_cons); 390 #ifdef BNXT_DEBUG 391 uint16_t ag_cons; 392 #endif 393 struct rte_mbuf *mbuf; 394 int rc = 0; 395 uint8_t agg_buf = 0; 396 uint16_t cmp_type; 397 398 rxcmp = (struct rx_pkt_cmpl *) 399 &cpr->cp_desc_ring[cp_cons]; 400 401 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 402 cp_cons = RING_CMP(cpr->cp_ring_struct, tmp_raw_cons); 403 rxcmp1 = (struct rx_pkt_cmpl_hi *)&cpr->cp_desc_ring[cp_cons]; 404 405 if (!CMP_VALID(rxcmp1, tmp_raw_cons, cpr->cp_ring_struct)) 406 return -EBUSY; 407 408 cpr->valid = FLIP_VALID(cp_cons, 409 cpr->cp_ring_struct->ring_mask, 410 cpr->valid); 411 412 cmp_type = CMP_TYPE(rxcmp); 413 if (cmp_type == RX_TPA_START_CMPL_TYPE_RX_TPA_START) { 414 bnxt_tpa_start(rxq, (struct rx_tpa_start_cmpl *)rxcmp, 415 (struct rx_tpa_start_cmpl_hi *)rxcmp1); 416 rc = -EINVAL; /* Continue w/o new mbuf */ 417 goto next_rx; 418 } else if (cmp_type == RX_TPA_END_CMPL_TYPE_RX_TPA_END) { 419 mbuf = bnxt_tpa_end(rxq, &tmp_raw_cons, 420 (struct rx_tpa_end_cmpl *)rxcmp, 421 (struct rx_tpa_end_cmpl_hi *)rxcmp1); 422 if (unlikely(!mbuf)) 423 return -EBUSY; 424 *rx_pkt = mbuf; 425 goto next_rx; 426 } else if (cmp_type != 0x11) { 427 rc = -EINVAL; 428 goto next_rx; 429 } 430 431 agg_buf = (rxcmp->agg_bufs_v1 & RX_PKT_CMPL_AGG_BUFS_MASK) 432 >> RX_PKT_CMPL_AGG_BUFS_SFT; 433 if (agg_buf && !bnxt_agg_bufs_valid(cpr, agg_buf, tmp_raw_cons)) 434 return -EBUSY; 435 436 prod = rxr->rx_prod; 437 438 cons = rxcmp->opaque; 439 mbuf = bnxt_consume_rx_buf(rxr, cons); 440 if (mbuf == NULL) 441 return -EBUSY; 442 443 rte_prefetch0(mbuf); 444 445 mbuf->nb_segs = 1; 446 mbuf->next = NULL; 447 mbuf->pkt_len = rxcmp->len; 448 mbuf->data_len = mbuf->pkt_len; 449 mbuf->port = rxq->port_id; 450 mbuf->ol_flags = 0; 451 if (rxcmp->flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) { 452 mbuf->hash.rss = rxcmp->rss_hash; 453 mbuf->ol_flags |= PKT_RX_RSS_HASH; 454 } else { 455 mbuf->hash.fdir.id = rxcmp1->cfa_code; 456 mbuf->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID; 457 } 458 459 if (agg_buf) 460 bnxt_rx_pages(rxq, mbuf, &tmp_raw_cons, agg_buf); 461 462 if (rxcmp1->flags2 & RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) { 463 mbuf->vlan_tci = rxcmp1->metadata & 464 (RX_PKT_CMPL_METADATA_VID_MASK | 465 RX_PKT_CMPL_METADATA_DE | 466 RX_PKT_CMPL_METADATA_PRI_MASK); 467 mbuf->ol_flags |= PKT_RX_VLAN; 468 } 469 470 if (likely(RX_CMP_IP_CS_OK(rxcmp1))) 471 mbuf->ol_flags |= PKT_RX_IP_CKSUM_GOOD; 472 else if (likely(RX_CMP_IP_CS_UNKNOWN(rxcmp1))) 473 mbuf->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN; 474 else 475 mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD; 476 477 if (likely(RX_CMP_L4_CS_OK(rxcmp1))) 478 mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD; 479 else if (likely(RX_CMP_L4_CS_UNKNOWN(rxcmp1))) 480 mbuf->ol_flags |= PKT_RX_L4_CKSUM_UNKNOWN; 481 else 482 mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; 483 484 mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1); 485 486 #ifdef BNXT_DEBUG 487 if (rxcmp1->errors_v2 & RX_CMP_L2_ERRORS) { 488 /* Re-install the mbuf back to the rx ring */ 489 bnxt_reuse_rx_mbuf(rxr, cons, mbuf); 490 if (agg_buf) 491 bnxt_reuse_ag_mbuf(rxr, ag_cons, mbuf); 492 493 rc = -EIO; 494 goto next_rx; 495 } 496 #endif 497 /* 498 * TODO: Redesign this.... 499 * If the allocation fails, the packet does not get received. 500 * Simply returning this will result in slowly falling behind 501 * on the producer ring buffers. 502 * Instead, "filling up" the producer just before ringing the 503 * doorbell could be a better solution since it will let the 504 * producer ring starve until memory is available again pushing 505 * the drops into hardware and getting them out of the driver 506 * allowing recovery to a full producer ring. 507 * 508 * This could also help with cache usage by preventing per-packet 509 * calls in favour of a tight loop with the same function being called 510 * in it. 511 */ 512 prod = RING_NEXT(rxr->rx_ring_struct, prod); 513 if (bnxt_alloc_rx_data(rxq, rxr, prod)) { 514 RTE_LOG(ERR, PMD, "mbuf alloc failed with prod=0x%x\n", prod); 515 rc = -ENOMEM; 516 goto rx; 517 } 518 rxr->rx_prod = prod; 519 /* 520 * All MBUFs are allocated with the same size under DPDK, 521 * no optimization for rx_copy_thresh 522 */ 523 rx: 524 *rx_pkt = mbuf; 525 526 next_rx: 527 528 *raw_cons = tmp_raw_cons; 529 530 return rc; 531 } 532 533 uint16_t bnxt_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 534 uint16_t nb_pkts) 535 { 536 struct bnxt_rx_queue *rxq = rx_queue; 537 struct bnxt_cp_ring_info *cpr = rxq->cp_ring; 538 struct bnxt_rx_ring_info *rxr = rxq->rx_ring; 539 uint32_t raw_cons = cpr->cp_raw_cons; 540 uint32_t cons; 541 int nb_rx_pkts = 0; 542 struct rx_pkt_cmpl *rxcmp; 543 uint16_t prod = rxr->rx_prod; 544 uint16_t ag_prod = rxr->ag_prod; 545 int rc = 0; 546 547 /* Handle RX burst request */ 548 while (1) { 549 cons = RING_CMP(cpr->cp_ring_struct, raw_cons); 550 rte_prefetch0(&cpr->cp_desc_ring[cons]); 551 rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons]; 552 553 if (!CMP_VALID(rxcmp, raw_cons, cpr->cp_ring_struct)) 554 break; 555 cpr->valid = FLIP_VALID(cons, 556 cpr->cp_ring_struct->ring_mask, 557 cpr->valid); 558 559 /* TODO: Avoid magic numbers... */ 560 if ((CMP_TYPE(rxcmp) & 0x30) == 0x10) { 561 rc = bnxt_rx_pkt(&rx_pkts[nb_rx_pkts], rxq, &raw_cons); 562 if (likely(!rc) || rc == -ENOMEM) 563 nb_rx_pkts++; 564 if (rc == -EBUSY) /* partial completion */ 565 break; 566 } 567 raw_cons = NEXT_RAW_CMP(raw_cons); 568 if (nb_rx_pkts == nb_pkts) 569 break; 570 } 571 572 cpr->cp_raw_cons = raw_cons; 573 if (prod == rxr->rx_prod && ag_prod == rxr->ag_prod) { 574 /* 575 * For PMD, there is no need to keep on pushing to REARM 576 * the doorbell if there are no new completions 577 */ 578 return nb_rx_pkts; 579 } 580 581 B_CP_DIS_DB(cpr, cpr->cp_raw_cons); 582 B_RX_DB(rxr->rx_doorbell, rxr->rx_prod); 583 /* Ring the AGG ring DB */ 584 B_RX_DB(rxr->ag_doorbell, rxr->ag_prod); 585 586 /* Attempt to alloc Rx buf in case of a previous allocation failure. */ 587 if (rc == -ENOMEM) { 588 int i; 589 590 for (i = prod; i <= nb_rx_pkts; 591 i = RING_NEXT(rxr->rx_ring_struct, i)) { 592 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; 593 594 /* Buffer already allocated for this index. */ 595 if (rx_buf->mbuf != NULL) 596 continue; 597 598 /* This slot is empty. Alloc buffer for Rx */ 599 if (!bnxt_alloc_rx_data(rxq, rxr, i)) { 600 rxr->rx_prod = i; 601 B_RX_DB(rxr->rx_doorbell, rxr->rx_prod); 602 } else { 603 RTE_LOG(ERR, PMD, "Alloc mbuf failed\n"); 604 break; 605 } 606 } 607 } 608 609 return nb_rx_pkts; 610 } 611 612 void bnxt_free_rx_rings(struct bnxt *bp) 613 { 614 int i; 615 616 for (i = 0; i < (int)bp->rx_nr_rings; i++) { 617 struct bnxt_rx_queue *rxq = bp->rx_queues[i]; 618 619 if (!rxq) 620 continue; 621 622 bnxt_free_ring(rxq->rx_ring->rx_ring_struct); 623 rte_free(rxq->rx_ring->rx_ring_struct); 624 625 /* Free the Aggregator ring */ 626 bnxt_free_ring(rxq->rx_ring->ag_ring_struct); 627 rte_free(rxq->rx_ring->ag_ring_struct); 628 rxq->rx_ring->ag_ring_struct = NULL; 629 630 rte_free(rxq->rx_ring); 631 632 bnxt_free_ring(rxq->cp_ring->cp_ring_struct); 633 rte_free(rxq->cp_ring->cp_ring_struct); 634 rte_free(rxq->cp_ring); 635 636 rte_free(rxq); 637 bp->rx_queues[i] = NULL; 638 } 639 } 640 641 int bnxt_init_rx_ring_struct(struct bnxt_rx_queue *rxq, unsigned int socket_id) 642 { 643 struct bnxt_cp_ring_info *cpr; 644 struct bnxt_rx_ring_info *rxr; 645 struct bnxt_ring *ring; 646 647 rxq->rx_buf_use_size = BNXT_MAX_MTU + ETHER_HDR_LEN + ETHER_CRC_LEN + 648 (2 * VLAN_TAG_SIZE); 649 rxq->rx_buf_size = rxq->rx_buf_use_size + sizeof(struct rte_mbuf); 650 651 rxr = rte_zmalloc_socket("bnxt_rx_ring", 652 sizeof(struct bnxt_rx_ring_info), 653 RTE_CACHE_LINE_SIZE, socket_id); 654 if (rxr == NULL) 655 return -ENOMEM; 656 rxq->rx_ring = rxr; 657 658 ring = rte_zmalloc_socket("bnxt_rx_ring_struct", 659 sizeof(struct bnxt_ring), 660 RTE_CACHE_LINE_SIZE, socket_id); 661 if (ring == NULL) 662 return -ENOMEM; 663 rxr->rx_ring_struct = ring; 664 ring->ring_size = rte_align32pow2(rxq->nb_rx_desc); 665 ring->ring_mask = ring->ring_size - 1; 666 ring->bd = (void *)rxr->rx_desc_ring; 667 ring->bd_dma = rxr->rx_desc_mapping; 668 ring->vmem_size = ring->ring_size * sizeof(struct bnxt_sw_rx_bd); 669 ring->vmem = (void **)&rxr->rx_buf_ring; 670 671 cpr = rte_zmalloc_socket("bnxt_rx_ring", 672 sizeof(struct bnxt_cp_ring_info), 673 RTE_CACHE_LINE_SIZE, socket_id); 674 if (cpr == NULL) 675 return -ENOMEM; 676 rxq->cp_ring = cpr; 677 678 ring = rte_zmalloc_socket("bnxt_rx_ring_struct", 679 sizeof(struct bnxt_ring), 680 RTE_CACHE_LINE_SIZE, socket_id); 681 if (ring == NULL) 682 return -ENOMEM; 683 cpr->cp_ring_struct = ring; 684 ring->ring_size = rte_align32pow2(rxr->rx_ring_struct->ring_size * 685 (2 + AGG_RING_SIZE_FACTOR)); 686 ring->ring_mask = ring->ring_size - 1; 687 ring->bd = (void *)cpr->cp_desc_ring; 688 ring->bd_dma = cpr->cp_desc_mapping; 689 ring->vmem_size = 0; 690 ring->vmem = NULL; 691 692 /* Allocate Aggregator rings */ 693 ring = rte_zmalloc_socket("bnxt_rx_ring_struct", 694 sizeof(struct bnxt_ring), 695 RTE_CACHE_LINE_SIZE, socket_id); 696 if (ring == NULL) 697 return -ENOMEM; 698 rxr->ag_ring_struct = ring; 699 ring->ring_size = rte_align32pow2(rxq->nb_rx_desc * 700 AGG_RING_SIZE_FACTOR); 701 ring->ring_mask = ring->ring_size - 1; 702 ring->bd = (void *)rxr->ag_desc_ring; 703 ring->bd_dma = rxr->ag_desc_mapping; 704 ring->vmem_size = ring->ring_size * sizeof(struct bnxt_sw_rx_bd); 705 ring->vmem = (void **)&rxr->ag_buf_ring; 706 707 return 0; 708 } 709 710 static void bnxt_init_rxbds(struct bnxt_ring *ring, uint32_t type, 711 uint16_t len) 712 { 713 uint32_t j; 714 struct rx_prod_pkt_bd *rx_bd_ring = (struct rx_prod_pkt_bd *)ring->bd; 715 716 if (!rx_bd_ring) 717 return; 718 for (j = 0; j < ring->ring_size; j++) { 719 rx_bd_ring[j].flags_type = rte_cpu_to_le_16(type); 720 rx_bd_ring[j].len = rte_cpu_to_le_16(len); 721 rx_bd_ring[j].opaque = j; 722 } 723 } 724 725 int bnxt_init_one_rx_ring(struct bnxt_rx_queue *rxq) 726 { 727 struct bnxt_rx_ring_info *rxr; 728 struct bnxt_ring *ring; 729 uint32_t prod, type; 730 unsigned int i; 731 uint16_t size; 732 733 size = rte_pktmbuf_data_room_size(rxq->mb_pool) - RTE_PKTMBUF_HEADROOM; 734 if (rxq->rx_buf_use_size <= size) 735 size = rxq->rx_buf_use_size; 736 737 type = RX_PROD_PKT_BD_TYPE_RX_PROD_PKT | RX_PROD_PKT_BD_FLAGS_EOP_PAD; 738 739 rxr = rxq->rx_ring; 740 ring = rxr->rx_ring_struct; 741 bnxt_init_rxbds(ring, type, size); 742 743 prod = rxr->rx_prod; 744 for (i = 0; i < ring->ring_size; i++) { 745 if (bnxt_alloc_rx_data(rxq, rxr, prod) != 0) { 746 RTE_LOG(WARNING, PMD, 747 "init'ed rx ring %d with %d/%d mbufs only\n", 748 rxq->queue_id, i, ring->ring_size); 749 break; 750 } 751 rxr->rx_prod = prod; 752 prod = RING_NEXT(rxr->rx_ring_struct, prod); 753 } 754 RTE_LOG(DEBUG, PMD, "%s\n", __func__); 755 756 ring = rxr->ag_ring_struct; 757 type = RX_PROD_AGG_BD_TYPE_RX_PROD_AGG; 758 bnxt_init_rxbds(ring, type, size); 759 prod = rxr->ag_prod; 760 761 for (i = 0; i < ring->ring_size; i++) { 762 if (bnxt_alloc_ag_data(rxq, rxr, prod) != 0) { 763 RTE_LOG(WARNING, PMD, 764 "init'ed AG ring %d with %d/%d mbufs only\n", 765 rxq->queue_id, i, ring->ring_size); 766 break; 767 } 768 rxr->ag_prod = prod; 769 prod = RING_NEXT(rxr->ag_ring_struct, prod); 770 } 771 RTE_LOG(DEBUG, PMD, "%s AGG Done!\n", __func__); 772 773 if (rxr->tpa_info) { 774 for (i = 0; i < BNXT_TPA_MAX; i++) { 775 rxr->tpa_info[i].mbuf = 776 __bnxt_alloc_rx_data(rxq->mb_pool); 777 if (!rxr->tpa_info[i].mbuf) { 778 rte_atomic64_inc(&rxq->bp->rx_mbuf_alloc_fail); 779 return -ENOMEM; 780 } 781 } 782 } 783 RTE_LOG(DEBUG, PMD, "%s TPA alloc Done!\n", __func__); 784 785 return 0; 786 } 787