1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2014-2021 Broadcom 3 * All rights reserved. 4 */ 5 6 #include <inttypes.h> 7 #include <stdbool.h> 8 9 #include <rte_bitmap.h> 10 #include <rte_byteorder.h> 11 #include <rte_malloc.h> 12 #include <rte_memory.h> 13 14 #include "bnxt.h" 15 #include "bnxt_reps.h" 16 #include "bnxt_ring.h" 17 #include "bnxt_rxr.h" 18 #include "bnxt_rxq.h" 19 #include "hsi_struct_def_dpdk.h" 20 #ifdef RTE_LIBRTE_IEEE1588 21 #include "bnxt_hwrm.h" 22 #endif 23 24 #include <bnxt_tf_common.h> 25 #include <ulp_mark_mgr.h> 26 27 /* 28 * RX Ring handling 29 */ 30 31 static inline struct rte_mbuf *__bnxt_alloc_rx_data(struct rte_mempool *mb) 32 { 33 struct rte_mbuf *data; 34 35 data = rte_mbuf_raw_alloc(mb); 36 37 return data; 38 } 39 40 static inline int bnxt_alloc_rx_data(struct bnxt_rx_queue *rxq, 41 struct bnxt_rx_ring_info *rxr, 42 uint16_t raw_prod) 43 { 44 uint16_t prod = RING_IDX(rxr->rx_ring_struct, raw_prod); 45 struct rx_prod_pkt_bd *rxbd; 46 struct rte_mbuf **rx_buf; 47 struct rte_mbuf *mbuf; 48 49 rxbd = &rxr->rx_desc_ring[prod]; 50 rx_buf = &rxr->rx_buf_ring[prod]; 51 mbuf = __bnxt_alloc_rx_data(rxq->mb_pool); 52 if (!mbuf) { 53 rte_atomic64_inc(&rxq->rx_mbuf_alloc_fail); 54 return -ENOMEM; 55 } 56 57 *rx_buf = mbuf; 58 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 59 60 rxbd->address = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 61 62 return 0; 63 } 64 65 static inline int bnxt_alloc_ag_data(struct bnxt_rx_queue *rxq, 66 struct bnxt_rx_ring_info *rxr, 67 uint16_t raw_prod) 68 { 69 uint16_t prod = RING_IDX(rxr->ag_ring_struct, raw_prod); 70 struct rx_prod_pkt_bd *rxbd; 71 struct rte_mbuf **rx_buf; 72 struct rte_mbuf *mbuf; 73 74 rxbd = &rxr->ag_desc_ring[prod]; 75 rx_buf = &rxr->ag_buf_ring[prod]; 76 if (rxbd == NULL) { 77 PMD_DRV_LOG(ERR, "Jumbo Frame. rxbd is NULL\n"); 78 return -EINVAL; 79 } 80 81 if (rx_buf == NULL) { 82 PMD_DRV_LOG(ERR, "Jumbo Frame. rx_buf is NULL\n"); 83 return -EINVAL; 84 } 85 86 mbuf = __bnxt_alloc_rx_data(rxq->mb_pool); 87 if (!mbuf) { 88 rte_atomic64_inc(&rxq->rx_mbuf_alloc_fail); 89 return -ENOMEM; 90 } 91 92 *rx_buf = mbuf; 93 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 94 95 rxbd->address = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 96 97 return 0; 98 } 99 100 static inline void bnxt_reuse_rx_mbuf(struct bnxt_rx_ring_info *rxr, 101 struct rte_mbuf *mbuf) 102 { 103 uint16_t prod, raw_prod = RING_NEXT(rxr->rx_raw_prod); 104 struct rte_mbuf **prod_rx_buf; 105 struct rx_prod_pkt_bd *prod_bd; 106 107 prod = RING_IDX(rxr->rx_ring_struct, raw_prod); 108 prod_rx_buf = &rxr->rx_buf_ring[prod]; 109 110 RTE_ASSERT(*prod_rx_buf == NULL); 111 RTE_ASSERT(mbuf != NULL); 112 113 *prod_rx_buf = mbuf; 114 115 prod_bd = &rxr->rx_desc_ring[prod]; 116 117 prod_bd->address = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf)); 118 119 rxr->rx_raw_prod = raw_prod; 120 } 121 122 static inline 123 struct rte_mbuf *bnxt_consume_rx_buf(struct bnxt_rx_ring_info *rxr, 124 uint16_t cons) 125 { 126 struct rte_mbuf **cons_rx_buf; 127 struct rte_mbuf *mbuf; 128 129 cons_rx_buf = &rxr->rx_buf_ring[RING_IDX(rxr->rx_ring_struct, cons)]; 130 RTE_ASSERT(*cons_rx_buf != NULL); 131 mbuf = *cons_rx_buf; 132 *cons_rx_buf = NULL; 133 134 return mbuf; 135 } 136 137 static void bnxt_tpa_get_metadata(struct bnxt *bp, 138 struct bnxt_tpa_info *tpa_info, 139 struct rx_tpa_start_cmpl *tpa_start, 140 struct rx_tpa_start_cmpl_hi *tpa_start1) 141 { 142 tpa_info->cfa_code_valid = 0; 143 tpa_info->vlan_valid = 0; 144 tpa_info->hash_valid = 0; 145 tpa_info->l4_csum_valid = 0; 146 147 if (likely(tpa_start->flags_type & 148 rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS_RSS_VALID))) { 149 tpa_info->hash_valid = 1; 150 tpa_info->rss_hash = rte_le_to_cpu_32(tpa_start->rss_hash); 151 } 152 153 if (bp->vnic_cap_flags & BNXT_VNIC_CAP_RX_CMPL_V2) { 154 struct rx_tpa_start_v2_cmpl *v2_tpa_start = (void *)tpa_start; 155 struct rx_tpa_start_v2_cmpl_hi *v2_tpa_start1 = 156 (void *)tpa_start1; 157 158 if (v2_tpa_start->agg_id & 159 RX_TPA_START_V2_CMPL_METADATA1_VALID) { 160 tpa_info->vlan_valid = 1; 161 tpa_info->vlan = 162 rte_le_to_cpu_16(v2_tpa_start1->metadata0); 163 } 164 165 if (v2_tpa_start1->flags2 & RX_CMP_FLAGS2_L4_CSUM_ALL_OK_MASK) 166 tpa_info->l4_csum_valid = 1; 167 168 return; 169 } 170 171 tpa_info->cfa_code_valid = 1; 172 tpa_info->cfa_code = rte_le_to_cpu_16(tpa_start1->cfa_code); 173 if (tpa_start1->flags2 & 174 rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS2_META_FORMAT_VLAN)) { 175 tpa_info->vlan_valid = 1; 176 tpa_info->vlan = rte_le_to_cpu_32(tpa_start1->metadata); 177 } 178 179 if (likely(tpa_start1->flags2 & 180 rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS2_L4_CS_CALC))) 181 tpa_info->l4_csum_valid = 1; 182 } 183 184 static void bnxt_tpa_start(struct bnxt_rx_queue *rxq, 185 struct rx_tpa_start_cmpl *tpa_start, 186 struct rx_tpa_start_cmpl_hi *tpa_start1) 187 { 188 struct bnxt_rx_ring_info *rxr = rxq->rx_ring; 189 uint16_t agg_id; 190 uint16_t data_cons; 191 struct bnxt_tpa_info *tpa_info; 192 struct rte_mbuf *mbuf; 193 194 agg_id = bnxt_tpa_start_agg_id(rxq->bp, tpa_start); 195 196 data_cons = tpa_start->opaque; 197 tpa_info = &rxr->tpa_info[agg_id]; 198 199 mbuf = bnxt_consume_rx_buf(rxr, data_cons); 200 201 bnxt_reuse_rx_mbuf(rxr, tpa_info->mbuf); 202 203 tpa_info->agg_count = 0; 204 tpa_info->mbuf = mbuf; 205 tpa_info->len = rte_le_to_cpu_32(tpa_start->len); 206 207 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 208 mbuf->nb_segs = 1; 209 mbuf->next = NULL; 210 mbuf->pkt_len = rte_le_to_cpu_32(tpa_start->len); 211 mbuf->data_len = mbuf->pkt_len; 212 mbuf->port = rxq->port_id; 213 mbuf->ol_flags = PKT_RX_LRO; 214 215 bnxt_tpa_get_metadata(rxq->bp, tpa_info, tpa_start, tpa_start1); 216 217 if (likely(tpa_info->hash_valid)) { 218 mbuf->hash.rss = tpa_info->rss_hash; 219 mbuf->ol_flags |= PKT_RX_RSS_HASH; 220 } else if (tpa_info->cfa_code_valid) { 221 mbuf->hash.fdir.id = tpa_info->cfa_code; 222 mbuf->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID; 223 } 224 225 if (tpa_info->vlan_valid) { 226 mbuf->vlan_tci = tpa_info->vlan; 227 mbuf->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 228 } 229 230 if (likely(tpa_info->l4_csum_valid)) 231 mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD; 232 233 /* recycle next mbuf */ 234 data_cons = RING_NEXT(data_cons); 235 bnxt_reuse_rx_mbuf(rxr, bnxt_consume_rx_buf(rxr, data_cons)); 236 } 237 238 static int bnxt_agg_bufs_valid(struct bnxt_cp_ring_info *cpr, 239 uint8_t agg_bufs, uint32_t raw_cp_cons) 240 { 241 uint16_t last_cp_cons; 242 struct rx_pkt_cmpl *agg_cmpl; 243 244 raw_cp_cons = ADV_RAW_CMP(raw_cp_cons, agg_bufs); 245 last_cp_cons = RING_CMP(cpr->cp_ring_struct, raw_cp_cons); 246 agg_cmpl = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[last_cp_cons]; 247 cpr->valid = FLIP_VALID(raw_cp_cons, 248 cpr->cp_ring_struct->ring_mask, 249 cpr->valid); 250 return CMP_VALID(agg_cmpl, raw_cp_cons, cpr->cp_ring_struct); 251 } 252 253 /* TPA consume agg buffer out of order, allocate connected data only */ 254 static int bnxt_prod_ag_mbuf(struct bnxt_rx_queue *rxq) 255 { 256 struct bnxt_rx_ring_info *rxr = rxq->rx_ring; 257 uint16_t raw_next = RING_NEXT(rxr->ag_raw_prod); 258 uint16_t bmap_next = RING_IDX(rxr->ag_ring_struct, raw_next); 259 260 /* TODO batch allocation for better performance */ 261 while (rte_bitmap_get(rxr->ag_bitmap, bmap_next)) { 262 if (unlikely(bnxt_alloc_ag_data(rxq, rxr, raw_next))) { 263 PMD_DRV_LOG(ERR, "agg mbuf alloc failed: prod=0x%x\n", 264 raw_next); 265 break; 266 } 267 rte_bitmap_clear(rxr->ag_bitmap, bmap_next); 268 rxr->ag_raw_prod = raw_next; 269 raw_next = RING_NEXT(raw_next); 270 bmap_next = RING_IDX(rxr->ag_ring_struct, raw_next); 271 } 272 273 return 0; 274 } 275 276 static int bnxt_rx_pages(struct bnxt_rx_queue *rxq, 277 struct rte_mbuf *mbuf, uint32_t *tmp_raw_cons, 278 uint8_t agg_buf, struct bnxt_tpa_info *tpa_info) 279 { 280 struct bnxt_cp_ring_info *cpr = rxq->cp_ring; 281 struct bnxt_rx_ring_info *rxr = rxq->rx_ring; 282 int i; 283 uint16_t cp_cons, ag_cons; 284 struct rx_pkt_cmpl *rxcmp; 285 struct rte_mbuf *last = mbuf; 286 bool is_p5_tpa = tpa_info && BNXT_CHIP_P5(rxq->bp); 287 288 for (i = 0; i < agg_buf; i++) { 289 struct rte_mbuf **ag_buf; 290 struct rte_mbuf *ag_mbuf; 291 292 if (is_p5_tpa) { 293 rxcmp = (void *)&tpa_info->agg_arr[i]; 294 } else { 295 *tmp_raw_cons = NEXT_RAW_CMP(*tmp_raw_cons); 296 cp_cons = RING_CMP(cpr->cp_ring_struct, *tmp_raw_cons); 297 rxcmp = (struct rx_pkt_cmpl *) 298 &cpr->cp_desc_ring[cp_cons]; 299 } 300 301 #ifdef BNXT_DEBUG 302 bnxt_dump_cmpl(cp_cons, rxcmp); 303 #endif 304 305 ag_cons = rxcmp->opaque; 306 RTE_ASSERT(ag_cons <= rxr->ag_ring_struct->ring_mask); 307 ag_buf = &rxr->ag_buf_ring[ag_cons]; 308 ag_mbuf = *ag_buf; 309 RTE_ASSERT(ag_mbuf != NULL); 310 311 ag_mbuf->data_len = rte_le_to_cpu_16(rxcmp->len); 312 313 mbuf->nb_segs++; 314 mbuf->pkt_len += ag_mbuf->data_len; 315 316 last->next = ag_mbuf; 317 last = ag_mbuf; 318 319 *ag_buf = NULL; 320 321 /* 322 * As aggregation buffer consumed out of order in TPA module, 323 * use bitmap to track freed slots to be allocated and notified 324 * to NIC 325 */ 326 rte_bitmap_set(rxr->ag_bitmap, ag_cons); 327 } 328 last->next = NULL; 329 bnxt_prod_ag_mbuf(rxq); 330 return 0; 331 } 332 333 static inline struct rte_mbuf *bnxt_tpa_end( 334 struct bnxt_rx_queue *rxq, 335 uint32_t *raw_cp_cons, 336 struct rx_tpa_end_cmpl *tpa_end, 337 struct rx_tpa_end_cmpl_hi *tpa_end1) 338 { 339 struct bnxt_cp_ring_info *cpr = rxq->cp_ring; 340 struct bnxt_rx_ring_info *rxr = rxq->rx_ring; 341 uint16_t agg_id; 342 struct rte_mbuf *mbuf; 343 uint8_t agg_bufs; 344 uint8_t payload_offset; 345 struct bnxt_tpa_info *tpa_info; 346 347 if (BNXT_CHIP_P5(rxq->bp)) { 348 struct rx_tpa_v2_end_cmpl *th_tpa_end; 349 struct rx_tpa_v2_end_cmpl_hi *th_tpa_end1; 350 351 th_tpa_end = (void *)tpa_end; 352 th_tpa_end1 = (void *)tpa_end1; 353 agg_id = BNXT_TPA_END_AGG_ID_TH(th_tpa_end); 354 agg_bufs = BNXT_TPA_END_AGG_BUFS_TH(th_tpa_end1); 355 payload_offset = th_tpa_end1->payload_offset; 356 } else { 357 agg_id = BNXT_TPA_END_AGG_ID(tpa_end); 358 agg_bufs = BNXT_TPA_END_AGG_BUFS(tpa_end); 359 if (!bnxt_agg_bufs_valid(cpr, agg_bufs, *raw_cp_cons)) 360 return NULL; 361 payload_offset = tpa_end->payload_offset; 362 } 363 364 tpa_info = &rxr->tpa_info[agg_id]; 365 mbuf = tpa_info->mbuf; 366 RTE_ASSERT(mbuf != NULL); 367 368 if (agg_bufs) { 369 bnxt_rx_pages(rxq, mbuf, raw_cp_cons, agg_bufs, tpa_info); 370 } 371 mbuf->l4_len = payload_offset; 372 373 struct rte_mbuf *new_data = __bnxt_alloc_rx_data(rxq->mb_pool); 374 RTE_ASSERT(new_data != NULL); 375 if (!new_data) { 376 rte_atomic64_inc(&rxq->rx_mbuf_alloc_fail); 377 return NULL; 378 } 379 tpa_info->mbuf = new_data; 380 381 return mbuf; 382 } 383 384 uint32_t bnxt_ptype_table[BNXT_PTYPE_TBL_DIM] __rte_cache_aligned; 385 386 static void __rte_cold 387 bnxt_init_ptype_table(void) 388 { 389 uint32_t *pt = bnxt_ptype_table; 390 static bool initialized; 391 int ip6, tun, type; 392 uint32_t l3; 393 int i; 394 395 if (initialized) 396 return; 397 398 for (i = 0; i < BNXT_PTYPE_TBL_DIM; i++) { 399 if (i & (RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN >> 2)) 400 pt[i] = RTE_PTYPE_L2_ETHER_VLAN; 401 else 402 pt[i] = RTE_PTYPE_L2_ETHER; 403 404 ip6 = i & (RX_PKT_CMPL_FLAGS2_IP_TYPE >> 7); 405 tun = i & (RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC >> 2); 406 type = (i & 0x78) << 9; 407 408 if (!tun && !ip6) 409 l3 = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; 410 else if (!tun && ip6) 411 l3 = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; 412 else if (tun && !ip6) 413 l3 = RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN; 414 else 415 l3 = RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN; 416 417 switch (type) { 418 case RX_PKT_CMPL_FLAGS_ITYPE_ICMP: 419 if (tun) 420 pt[i] |= l3 | RTE_PTYPE_INNER_L4_ICMP; 421 else 422 pt[i] |= l3 | RTE_PTYPE_L4_ICMP; 423 break; 424 case RX_PKT_CMPL_FLAGS_ITYPE_TCP: 425 if (tun) 426 pt[i] |= l3 | RTE_PTYPE_INNER_L4_TCP; 427 else 428 pt[i] |= l3 | RTE_PTYPE_L4_TCP; 429 break; 430 case RX_PKT_CMPL_FLAGS_ITYPE_UDP: 431 if (tun) 432 pt[i] |= l3 | RTE_PTYPE_INNER_L4_UDP; 433 else 434 pt[i] |= l3 | RTE_PTYPE_L4_UDP; 435 break; 436 case RX_PKT_CMPL_FLAGS_ITYPE_IP: 437 pt[i] |= l3; 438 break; 439 } 440 } 441 initialized = true; 442 } 443 444 static uint32_t 445 bnxt_parse_pkt_type(struct rx_pkt_cmpl *rxcmp, struct rx_pkt_cmpl_hi *rxcmp1) 446 { 447 uint32_t flags_type, flags2; 448 uint8_t index; 449 450 flags_type = rte_le_to_cpu_16(rxcmp->flags_type); 451 flags2 = rte_le_to_cpu_32(rxcmp1->flags2); 452 453 /* 454 * Index format: 455 * bit 0: RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC 456 * bit 1: RX_CMPL_FLAGS2_IP_TYPE 457 * bit 2: RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN 458 * bits 3-6: RX_PKT_CMPL_FLAGS_ITYPE 459 */ 460 index = ((flags_type & RX_PKT_CMPL_FLAGS_ITYPE_MASK) >> 9) | 461 ((flags2 & (RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | 462 RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC)) >> 2) | 463 ((flags2 & RX_PKT_CMPL_FLAGS2_IP_TYPE) >> 7); 464 465 return bnxt_ptype_table[index]; 466 } 467 468 static void __rte_cold 469 bnxt_init_ol_flags_tables(struct bnxt_rx_queue *rxq) 470 { 471 struct bnxt_rx_ring_info *rxr = rxq->rx_ring; 472 struct rte_eth_conf *dev_conf; 473 bool outer_cksum_enabled; 474 uint64_t offloads; 475 uint32_t *pt; 476 int i; 477 478 dev_conf = &rxq->bp->eth_dev->data->dev_conf; 479 offloads = dev_conf->rxmode.offloads; 480 481 outer_cksum_enabled = !!(offloads & (DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | 482 DEV_RX_OFFLOAD_OUTER_UDP_CKSUM)); 483 484 /* Initialize ol_flags table. */ 485 pt = rxr->ol_flags_table; 486 for (i = 0; i < BNXT_OL_FLAGS_TBL_DIM; i++) { 487 pt[i] = 0; 488 489 if (i & RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) 490 pt[i] |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 491 492 if (i & (RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC << 3)) { 493 /* Tunnel case. */ 494 if (outer_cksum_enabled) { 495 if (i & RX_PKT_CMPL_FLAGS2_IP_CS_CALC) 496 pt[i] |= PKT_RX_IP_CKSUM_GOOD; 497 498 if (i & RX_PKT_CMPL_FLAGS2_L4_CS_CALC) 499 pt[i] |= PKT_RX_L4_CKSUM_GOOD; 500 501 if (i & RX_PKT_CMPL_FLAGS2_T_L4_CS_CALC) 502 pt[i] |= PKT_RX_OUTER_L4_CKSUM_GOOD; 503 } else { 504 if (i & RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC) 505 pt[i] |= PKT_RX_IP_CKSUM_GOOD; 506 507 if (i & RX_PKT_CMPL_FLAGS2_T_L4_CS_CALC) 508 pt[i] |= PKT_RX_L4_CKSUM_GOOD; 509 } 510 } else { 511 /* Non-tunnel case. */ 512 if (i & RX_PKT_CMPL_FLAGS2_IP_CS_CALC) 513 pt[i] |= PKT_RX_IP_CKSUM_GOOD; 514 515 if (i & RX_PKT_CMPL_FLAGS2_L4_CS_CALC) 516 pt[i] |= PKT_RX_L4_CKSUM_GOOD; 517 } 518 } 519 520 /* Initialize checksum error table. */ 521 pt = rxr->ol_flags_err_table; 522 for (i = 0; i < BNXT_OL_FLAGS_ERR_TBL_DIM; i++) { 523 pt[i] = 0; 524 525 if (i & (RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC << 2)) { 526 /* Tunnel case. */ 527 if (outer_cksum_enabled) { 528 if (i & (RX_PKT_CMPL_ERRORS_IP_CS_ERROR >> 4)) 529 pt[i] |= PKT_RX_IP_CKSUM_BAD; 530 531 if (i & (RX_PKT_CMPL_ERRORS_T_IP_CS_ERROR >> 4)) 532 pt[i] |= PKT_RX_OUTER_IP_CKSUM_BAD; 533 534 if (i & (RX_PKT_CMPL_ERRORS_L4_CS_ERROR >> 4)) 535 pt[i] |= PKT_RX_L4_CKSUM_BAD; 536 537 if (i & (RX_PKT_CMPL_ERRORS_T_L4_CS_ERROR >> 4)) 538 pt[i] |= PKT_RX_OUTER_L4_CKSUM_BAD; 539 } else { 540 if (i & (RX_PKT_CMPL_ERRORS_T_IP_CS_ERROR >> 4)) 541 pt[i] |= PKT_RX_IP_CKSUM_BAD; 542 543 if (i & (RX_PKT_CMPL_ERRORS_T_L4_CS_ERROR >> 4)) 544 pt[i] |= PKT_RX_L4_CKSUM_BAD; 545 } 546 } else { 547 /* Non-tunnel case. */ 548 if (i & (RX_PKT_CMPL_ERRORS_IP_CS_ERROR >> 4)) 549 pt[i] |= PKT_RX_IP_CKSUM_BAD; 550 551 if (i & (RX_PKT_CMPL_ERRORS_L4_CS_ERROR >> 4)) 552 pt[i] |= PKT_RX_L4_CKSUM_BAD; 553 } 554 } 555 } 556 557 static void 558 bnxt_set_ol_flags(struct bnxt_rx_ring_info *rxr, struct rx_pkt_cmpl *rxcmp, 559 struct rx_pkt_cmpl_hi *rxcmp1, struct rte_mbuf *mbuf) 560 { 561 uint16_t flags_type, errors, flags; 562 uint64_t ol_flags; 563 564 flags_type = rte_le_to_cpu_16(rxcmp->flags_type); 565 566 flags = rte_le_to_cpu_32(rxcmp1->flags2) & 567 (RX_PKT_CMPL_FLAGS2_IP_CS_CALC | 568 RX_PKT_CMPL_FLAGS2_L4_CS_CALC | 569 RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC | 570 RX_PKT_CMPL_FLAGS2_T_L4_CS_CALC | 571 RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN); 572 573 flags |= (flags & RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC) << 3; 574 errors = rte_le_to_cpu_16(rxcmp1->errors_v2) & 575 (RX_PKT_CMPL_ERRORS_IP_CS_ERROR | 576 RX_PKT_CMPL_ERRORS_L4_CS_ERROR | 577 RX_PKT_CMPL_ERRORS_T_IP_CS_ERROR | 578 RX_PKT_CMPL_ERRORS_T_L4_CS_ERROR); 579 errors = (errors >> 4) & flags; 580 581 ol_flags = rxr->ol_flags_table[flags & ~errors]; 582 583 if (unlikely(errors)) { 584 errors |= (flags & RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC) << 2; 585 ol_flags |= rxr->ol_flags_err_table[errors]; 586 } 587 588 if (flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) { 589 mbuf->hash.rss = rte_le_to_cpu_32(rxcmp->rss_hash); 590 ol_flags |= PKT_RX_RSS_HASH; 591 } 592 593 #ifdef RTE_LIBRTE_IEEE1588 594 if (unlikely((flags_type & RX_PKT_CMPL_FLAGS_MASK) == 595 RX_PKT_CMPL_FLAGS_ITYPE_PTP_W_TIMESTAMP)) 596 ol_flags |= PKT_RX_IEEE1588_PTP | PKT_RX_IEEE1588_TMST; 597 #endif 598 599 mbuf->ol_flags = ol_flags; 600 } 601 602 #ifdef RTE_LIBRTE_IEEE1588 603 static void 604 bnxt_get_rx_ts_p5(struct bnxt *bp, uint32_t rx_ts_cmpl) 605 { 606 uint64_t systime_cycles = 0; 607 608 if (!BNXT_CHIP_P5(bp)) 609 return; 610 611 /* On Thor, Rx timestamps are provided directly in the 612 * Rx completion records to the driver. Only 32 bits of 613 * the timestamp is present in the completion. Driver needs 614 * to read the current 48 bit free running timer using the 615 * HWRM_PORT_TS_QUERY command and combine the upper 16 bits 616 * from the HWRM response with the lower 32 bits in the 617 * Rx completion to produce the 48 bit timestamp for the Rx packet 618 */ 619 bnxt_hwrm_port_ts_query(bp, BNXT_PTP_FLAGS_CURRENT_TIME, 620 &systime_cycles); 621 bp->ptp_cfg->rx_timestamp = (systime_cycles & 0xFFFF00000000); 622 bp->ptp_cfg->rx_timestamp |= rx_ts_cmpl; 623 } 624 #endif 625 626 static uint32_t 627 bnxt_ulp_set_mark_in_mbuf(struct bnxt *bp, struct rx_pkt_cmpl_hi *rxcmp1, 628 struct rte_mbuf *mbuf, uint32_t *vfr_flag) 629 { 630 uint32_t cfa_code; 631 uint32_t meta_fmt; 632 uint32_t meta; 633 bool gfid = false; 634 uint32_t mark_id; 635 uint32_t flags2; 636 uint32_t gfid_support = 0; 637 int rc; 638 639 if (BNXT_GFID_ENABLED(bp)) 640 gfid_support = 1; 641 642 cfa_code = rte_le_to_cpu_16(rxcmp1->cfa_code); 643 flags2 = rte_le_to_cpu_32(rxcmp1->flags2); 644 meta = rte_le_to_cpu_32(rxcmp1->metadata); 645 646 /* 647 * The flags field holds extra bits of info from [6:4] 648 * which indicate if the flow is in TCAM or EM or EEM 649 */ 650 meta_fmt = (flags2 & BNXT_CFA_META_FMT_MASK) >> 651 BNXT_CFA_META_FMT_SHFT; 652 653 switch (meta_fmt) { 654 case 0: 655 if (gfid_support) { 656 /* Not an LFID or GFID, a flush cmd. */ 657 goto skip_mark; 658 } else { 659 /* LFID mode, no vlan scenario */ 660 gfid = false; 661 } 662 break; 663 case 4: 664 case 5: 665 /* 666 * EM/TCAM case 667 * Assume that EM doesn't support Mark due to GFID 668 * collisions with EEM. Simply return without setting the mark 669 * in the mbuf. 670 */ 671 if (BNXT_CFA_META_EM_TEST(meta)) { 672 /*This is EM hit {EM(1), GFID[27:16], 19'd0 or vtag } */ 673 gfid = true; 674 meta >>= BNXT_RX_META_CFA_CODE_SHIFT; 675 cfa_code |= meta << BNXT_CFA_CODE_META_SHIFT; 676 } else { 677 /* 678 * It is a TCAM entry, so it is an LFID. 679 * The TCAM IDX and Mode can also be determined 680 * by decoding the meta_data. We are not 681 * using these for now. 682 */ 683 } 684 break; 685 case 6: 686 case 7: 687 /* EEM Case, only using gfid in EEM for now. */ 688 gfid = true; 689 690 /* 691 * For EEM flows, The first part of cfa_code is 16 bits. 692 * The second part is embedded in the 693 * metadata field from bit 19 onwards. The driver needs to 694 * ignore the first 19 bits of metadata and use the next 12 695 * bits as higher 12 bits of cfa_code. 696 */ 697 meta >>= BNXT_RX_META_CFA_CODE_SHIFT; 698 cfa_code |= meta << BNXT_CFA_CODE_META_SHIFT; 699 break; 700 default: 701 /* For other values, the cfa_code is assumed to be an LFID. */ 702 break; 703 } 704 705 rc = ulp_mark_db_mark_get(bp->ulp_ctx, gfid, 706 cfa_code, vfr_flag, &mark_id); 707 if (!rc) { 708 /* VF to VFR Rx path. So, skip mark_id injection in mbuf */ 709 if (vfr_flag && *vfr_flag) 710 return mark_id; 711 /* Got the mark, write it to the mbuf and return */ 712 mbuf->hash.fdir.hi = mark_id; 713 *bnxt_cfa_code_dynfield(mbuf) = cfa_code & 0xffffffffull; 714 mbuf->hash.fdir.id = rxcmp1->cfa_code; 715 mbuf->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID; 716 return mark_id; 717 } 718 719 skip_mark: 720 mbuf->hash.fdir.hi = 0; 721 mbuf->hash.fdir.id = 0; 722 723 return 0; 724 } 725 726 void bnxt_set_mark_in_mbuf(struct bnxt *bp, 727 struct rx_pkt_cmpl_hi *rxcmp1, 728 struct rte_mbuf *mbuf) 729 { 730 uint32_t cfa_code = 0; 731 uint8_t meta_fmt = 0; 732 uint16_t flags2 = 0; 733 uint32_t meta = 0; 734 735 cfa_code = rte_le_to_cpu_16(rxcmp1->cfa_code); 736 if (!cfa_code) 737 return; 738 739 if (cfa_code && !bp->mark_table[cfa_code].valid) 740 return; 741 742 flags2 = rte_le_to_cpu_16(rxcmp1->flags2); 743 meta = rte_le_to_cpu_32(rxcmp1->metadata); 744 if (meta) { 745 meta >>= BNXT_RX_META_CFA_CODE_SHIFT; 746 747 /* The flags field holds extra bits of info from [6:4] 748 * which indicate if the flow is in TCAM or EM or EEM 749 */ 750 meta_fmt = (flags2 & BNXT_CFA_META_FMT_MASK) >> 751 BNXT_CFA_META_FMT_SHFT; 752 753 /* meta_fmt == 4 => 'b100 => 'b10x => EM. 754 * meta_fmt == 5 => 'b101 => 'b10x => EM + VLAN 755 * meta_fmt == 6 => 'b110 => 'b11x => EEM 756 * meta_fmt == 7 => 'b111 => 'b11x => EEM + VLAN. 757 */ 758 meta_fmt >>= BNXT_CFA_META_FMT_EM_EEM_SHFT; 759 } 760 761 mbuf->hash.fdir.hi = bp->mark_table[cfa_code].mark_id; 762 mbuf->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID; 763 } 764 765 static int bnxt_rx_pkt(struct rte_mbuf **rx_pkt, 766 struct bnxt_rx_queue *rxq, uint32_t *raw_cons) 767 { 768 struct bnxt_cp_ring_info *cpr = rxq->cp_ring; 769 struct bnxt_rx_ring_info *rxr = rxq->rx_ring; 770 struct rx_pkt_cmpl *rxcmp; 771 struct rx_pkt_cmpl_hi *rxcmp1; 772 uint32_t tmp_raw_cons = *raw_cons; 773 uint16_t cons, raw_prod, cp_cons = 774 RING_CMP(cpr->cp_ring_struct, tmp_raw_cons); 775 struct rte_mbuf *mbuf; 776 int rc = 0; 777 uint8_t agg_buf = 0; 778 uint16_t cmp_type; 779 uint32_t vfr_flag = 0, mark_id = 0; 780 struct bnxt *bp = rxq->bp; 781 782 rxcmp = (struct rx_pkt_cmpl *) 783 &cpr->cp_desc_ring[cp_cons]; 784 785 cmp_type = CMP_TYPE(rxcmp); 786 787 if (cmp_type == RX_TPA_V2_ABUF_CMPL_TYPE_RX_TPA_AGG) { 788 struct rx_tpa_v2_abuf_cmpl *rx_agg = (void *)rxcmp; 789 uint16_t agg_id = rte_cpu_to_le_16(rx_agg->agg_id); 790 struct bnxt_tpa_info *tpa_info; 791 792 tpa_info = &rxr->tpa_info[agg_id]; 793 RTE_ASSERT(tpa_info->agg_count < 16); 794 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg; 795 rc = -EINVAL; /* Continue w/o new mbuf */ 796 goto next_rx; 797 } 798 799 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 800 cp_cons = RING_CMP(cpr->cp_ring_struct, tmp_raw_cons); 801 rxcmp1 = (struct rx_pkt_cmpl_hi *)&cpr->cp_desc_ring[cp_cons]; 802 803 if (!CMP_VALID(rxcmp1, tmp_raw_cons, cpr->cp_ring_struct)) 804 return -EBUSY; 805 806 cpr->valid = FLIP_VALID(cp_cons, 807 cpr->cp_ring_struct->ring_mask, 808 cpr->valid); 809 810 if (cmp_type == RX_TPA_START_CMPL_TYPE_RX_TPA_START || 811 cmp_type == RX_TPA_START_V2_CMPL_TYPE_RX_TPA_START_V2) { 812 bnxt_tpa_start(rxq, (struct rx_tpa_start_cmpl *)rxcmp, 813 (struct rx_tpa_start_cmpl_hi *)rxcmp1); 814 rc = -EINVAL; /* Continue w/o new mbuf */ 815 goto next_rx; 816 } else if (cmp_type == RX_TPA_END_CMPL_TYPE_RX_TPA_END) { 817 mbuf = bnxt_tpa_end(rxq, &tmp_raw_cons, 818 (struct rx_tpa_end_cmpl *)rxcmp, 819 (struct rx_tpa_end_cmpl_hi *)rxcmp1); 820 if (unlikely(!mbuf)) 821 return -EBUSY; 822 *rx_pkt = mbuf; 823 goto next_rx; 824 } else if ((cmp_type != CMPL_BASE_TYPE_RX_L2) && 825 (cmp_type != CMPL_BASE_TYPE_RX_L2_V2)) { 826 rc = -EINVAL; 827 goto next_rx; 828 } 829 830 agg_buf = BNXT_RX_L2_AGG_BUFS(rxcmp); 831 if (agg_buf && !bnxt_agg_bufs_valid(cpr, agg_buf, tmp_raw_cons)) 832 return -EBUSY; 833 834 raw_prod = rxr->rx_raw_prod; 835 836 cons = rxcmp->opaque; 837 mbuf = bnxt_consume_rx_buf(rxr, cons); 838 if (mbuf == NULL) 839 return -EBUSY; 840 841 mbuf->data_off = RTE_PKTMBUF_HEADROOM; 842 mbuf->nb_segs = 1; 843 mbuf->next = NULL; 844 mbuf->pkt_len = rxcmp->len; 845 mbuf->data_len = mbuf->pkt_len; 846 mbuf->port = rxq->port_id; 847 848 #ifdef RTE_LIBRTE_IEEE1588 849 if (unlikely((rte_le_to_cpu_16(rxcmp->flags_type) & 850 RX_PKT_CMPL_FLAGS_MASK) == 851 RX_PKT_CMPL_FLAGS_ITYPE_PTP_W_TIMESTAMP)) 852 bnxt_get_rx_ts_p5(rxq->bp, rxcmp1->reorder); 853 #endif 854 855 if (cmp_type == CMPL_BASE_TYPE_RX_L2_V2) { 856 bnxt_parse_csum_v2(mbuf, rxcmp1); 857 bnxt_parse_pkt_type_v2(mbuf, rxcmp, rxcmp1); 858 bnxt_rx_vlan_v2(mbuf, rxcmp, rxcmp1); 859 /* TODO Add support for cfa_code parsing */ 860 goto reuse_rx_mbuf; 861 } 862 863 bnxt_set_ol_flags(rxr, rxcmp, rxcmp1, mbuf); 864 865 mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1); 866 867 if (BNXT_TRUFLOW_EN(bp)) 868 mark_id = bnxt_ulp_set_mark_in_mbuf(rxq->bp, rxcmp1, mbuf, 869 &vfr_flag); 870 else 871 bnxt_set_mark_in_mbuf(rxq->bp, rxcmp1, mbuf); 872 873 reuse_rx_mbuf: 874 if (agg_buf) 875 bnxt_rx_pages(rxq, mbuf, &tmp_raw_cons, agg_buf, NULL); 876 877 #ifdef BNXT_DEBUG 878 if (rxcmp1->errors_v2 & RX_CMP_L2_ERRORS) { 879 /* Re-install the mbuf back to the rx ring */ 880 bnxt_reuse_rx_mbuf(rxr, cons, mbuf); 881 882 rc = -EIO; 883 goto next_rx; 884 } 885 #endif 886 /* 887 * TODO: Redesign this.... 888 * If the allocation fails, the packet does not get received. 889 * Simply returning this will result in slowly falling behind 890 * on the producer ring buffers. 891 * Instead, "filling up" the producer just before ringing the 892 * doorbell could be a better solution since it will let the 893 * producer ring starve until memory is available again pushing 894 * the drops into hardware and getting them out of the driver 895 * allowing recovery to a full producer ring. 896 * 897 * This could also help with cache usage by preventing per-packet 898 * calls in favour of a tight loop with the same function being called 899 * in it. 900 */ 901 raw_prod = RING_NEXT(raw_prod); 902 if (bnxt_alloc_rx_data(rxq, rxr, raw_prod)) { 903 PMD_DRV_LOG(ERR, "mbuf alloc failed with prod=0x%x\n", 904 raw_prod); 905 rc = -ENOMEM; 906 goto rx; 907 } 908 rxr->rx_raw_prod = raw_prod; 909 910 if (BNXT_TRUFLOW_EN(bp) && (BNXT_VF_IS_TRUSTED(bp) || BNXT_PF(bp)) && 911 vfr_flag) { 912 bnxt_vfr_recv(mark_id, rxq->queue_id, mbuf); 913 /* Now return an error so that nb_rx_pkts is not 914 * incremented. 915 * This packet was meant to be given to the representor. 916 * So no need to account the packet and give it to 917 * parent Rx burst function. 918 */ 919 rc = -ENODEV; 920 goto next_rx; 921 } 922 /* 923 * All MBUFs are allocated with the same size under DPDK, 924 * no optimization for rx_copy_thresh 925 */ 926 rx: 927 *rx_pkt = mbuf; 928 929 next_rx: 930 931 *raw_cons = tmp_raw_cons; 932 933 return rc; 934 } 935 936 uint16_t bnxt_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 937 uint16_t nb_pkts) 938 { 939 struct bnxt_rx_queue *rxq = rx_queue; 940 struct bnxt_cp_ring_info *cpr = rxq->cp_ring; 941 struct bnxt_rx_ring_info *rxr = rxq->rx_ring; 942 uint16_t rx_raw_prod = rxr->rx_raw_prod; 943 uint16_t ag_raw_prod = rxr->ag_raw_prod; 944 uint32_t raw_cons = cpr->cp_raw_cons; 945 bool alloc_failed = false; 946 uint32_t cons; 947 int nb_rx_pkts = 0; 948 int nb_rep_rx_pkts = 0; 949 struct rx_pkt_cmpl *rxcmp; 950 int rc = 0; 951 bool evt = false; 952 953 if (unlikely(is_bnxt_in_error(rxq->bp))) 954 return 0; 955 956 /* If Rx Q was stopped return */ 957 if (unlikely(!rxq->rx_started)) 958 return 0; 959 960 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) 961 /* 962 * Replenish buffers if needed when a transition has been made from 963 * vector- to non-vector- receive processing. 964 */ 965 while (unlikely(rxq->rxrearm_nb)) { 966 if (!bnxt_alloc_rx_data(rxq, rxr, rxq->rxrearm_start)) { 967 rxr->rx_raw_prod = rxq->rxrearm_start; 968 bnxt_db_write(&rxr->rx_db, rxr->rx_raw_prod); 969 rxq->rxrearm_start++; 970 rxq->rxrearm_nb--; 971 } else { 972 /* Retry allocation on next call. */ 973 break; 974 } 975 } 976 #endif 977 978 /* Handle RX burst request */ 979 while (1) { 980 cons = RING_CMP(cpr->cp_ring_struct, raw_cons); 981 rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons]; 982 983 if (!CMP_VALID(rxcmp, raw_cons, cpr->cp_ring_struct)) 984 break; 985 cpr->valid = FLIP_VALID(cons, 986 cpr->cp_ring_struct->ring_mask, 987 cpr->valid); 988 989 if ((CMP_TYPE(rxcmp) >= CMPL_BASE_TYPE_RX_TPA_START_V2) && 990 (CMP_TYPE(rxcmp) <= RX_TPA_V2_ABUF_CMPL_TYPE_RX_TPA_AGG)) { 991 rc = bnxt_rx_pkt(&rx_pkts[nb_rx_pkts], rxq, &raw_cons); 992 if (!rc) 993 nb_rx_pkts++; 994 else if (rc == -EBUSY) /* partial completion */ 995 break; 996 else if (rc == -ENODEV) /* completion for representor */ 997 nb_rep_rx_pkts++; 998 else if (rc == -ENOMEM) { 999 nb_rx_pkts++; 1000 alloc_failed = true; 1001 } 1002 } else if (!BNXT_NUM_ASYNC_CPR(rxq->bp)) { 1003 evt = 1004 bnxt_event_hwrm_resp_handler(rxq->bp, 1005 (struct cmpl_base *)rxcmp); 1006 /* If the async event is Fatal error, return */ 1007 if (unlikely(is_bnxt_in_error(rxq->bp))) 1008 goto done; 1009 } 1010 1011 raw_cons = NEXT_RAW_CMP(raw_cons); 1012 if (nb_rx_pkts == nb_pkts || nb_rep_rx_pkts == nb_pkts || evt) 1013 break; 1014 /* Post some Rx buf early in case of larger burst processing */ 1015 if (nb_rx_pkts == BNXT_RX_POST_THRESH) 1016 bnxt_db_write(&rxr->rx_db, rxr->rx_raw_prod); 1017 } 1018 1019 cpr->cp_raw_cons = raw_cons; 1020 if (!nb_rx_pkts && !nb_rep_rx_pkts && !evt) { 1021 /* 1022 * For PMD, there is no need to keep on pushing to REARM 1023 * the doorbell if there are no new completions 1024 */ 1025 goto done; 1026 } 1027 1028 /* Ring the completion queue doorbell. */ 1029 bnxt_db_cq(cpr); 1030 1031 /* Ring the receive descriptor doorbell. */ 1032 if (rx_raw_prod != rxr->rx_raw_prod) 1033 bnxt_db_write(&rxr->rx_db, rxr->rx_raw_prod); 1034 1035 /* Ring the AGG ring DB */ 1036 if (ag_raw_prod != rxr->ag_raw_prod) 1037 bnxt_db_write(&rxr->ag_db, rxr->ag_raw_prod); 1038 1039 /* Attempt to alloc Rx buf in case of a previous allocation failure. */ 1040 if (alloc_failed) { 1041 uint16_t cnt; 1042 1043 rx_raw_prod = RING_NEXT(rx_raw_prod); 1044 for (cnt = 0; cnt < nb_rx_pkts + nb_rep_rx_pkts; cnt++) { 1045 struct rte_mbuf **rx_buf; 1046 uint16_t ndx; 1047 1048 ndx = RING_IDX(rxr->rx_ring_struct, rx_raw_prod + cnt); 1049 rx_buf = &rxr->rx_buf_ring[ndx]; 1050 1051 /* Buffer already allocated for this index. */ 1052 if (*rx_buf != NULL && *rx_buf != &rxq->fake_mbuf) 1053 continue; 1054 1055 /* This slot is empty. Alloc buffer for Rx */ 1056 if (!bnxt_alloc_rx_data(rxq, rxr, rx_raw_prod + cnt)) { 1057 rxr->rx_raw_prod = rx_raw_prod + cnt; 1058 bnxt_db_write(&rxr->rx_db, rxr->rx_raw_prod); 1059 } else { 1060 PMD_DRV_LOG(ERR, "Alloc mbuf failed\n"); 1061 break; 1062 } 1063 } 1064 } 1065 1066 done: 1067 return nb_rx_pkts; 1068 } 1069 1070 /* 1071 * Dummy DPDK callback for RX. 1072 * 1073 * This function is used to temporarily replace the real callback during 1074 * unsafe control operations on the queue, or in case of error. 1075 */ 1076 uint16_t 1077 bnxt_dummy_recv_pkts(void *rx_queue __rte_unused, 1078 struct rte_mbuf **rx_pkts __rte_unused, 1079 uint16_t nb_pkts __rte_unused) 1080 { 1081 return 0; 1082 } 1083 1084 void bnxt_free_rx_rings(struct bnxt *bp) 1085 { 1086 int i; 1087 struct bnxt_rx_queue *rxq; 1088 1089 if (!bp->rx_queues) 1090 return; 1091 1092 for (i = 0; i < (int)bp->rx_nr_rings; i++) { 1093 rxq = bp->rx_queues[i]; 1094 if (!rxq) 1095 continue; 1096 1097 bnxt_free_ring(rxq->rx_ring->rx_ring_struct); 1098 rte_free(rxq->rx_ring->rx_ring_struct); 1099 1100 /* Free the Aggregator ring */ 1101 bnxt_free_ring(rxq->rx_ring->ag_ring_struct); 1102 rte_free(rxq->rx_ring->ag_ring_struct); 1103 rxq->rx_ring->ag_ring_struct = NULL; 1104 1105 rte_free(rxq->rx_ring); 1106 1107 bnxt_free_ring(rxq->cp_ring->cp_ring_struct); 1108 rte_free(rxq->cp_ring->cp_ring_struct); 1109 rte_free(rxq->cp_ring); 1110 1111 rte_free(rxq); 1112 bp->rx_queues[i] = NULL; 1113 } 1114 } 1115 1116 int bnxt_init_rx_ring_struct(struct bnxt_rx_queue *rxq, unsigned int socket_id) 1117 { 1118 struct bnxt_cp_ring_info *cpr; 1119 struct bnxt_rx_ring_info *rxr; 1120 struct bnxt_ring *ring; 1121 1122 rxq->rx_buf_size = BNXT_MAX_PKT_LEN + sizeof(struct rte_mbuf); 1123 1124 rxr = rte_zmalloc_socket("bnxt_rx_ring", 1125 sizeof(struct bnxt_rx_ring_info), 1126 RTE_CACHE_LINE_SIZE, socket_id); 1127 if (rxr == NULL) 1128 return -ENOMEM; 1129 rxq->rx_ring = rxr; 1130 1131 ring = rte_zmalloc_socket("bnxt_rx_ring_struct", 1132 sizeof(struct bnxt_ring), 1133 RTE_CACHE_LINE_SIZE, socket_id); 1134 if (ring == NULL) 1135 return -ENOMEM; 1136 rxr->rx_ring_struct = ring; 1137 ring->ring_size = rte_align32pow2(rxq->nb_rx_desc); 1138 ring->ring_mask = ring->ring_size - 1; 1139 ring->bd = (void *)rxr->rx_desc_ring; 1140 ring->bd_dma = rxr->rx_desc_mapping; 1141 1142 /* Allocate extra rx ring entries for vector rx. */ 1143 ring->vmem_size = sizeof(struct rte_mbuf *) * 1144 (ring->ring_size + RTE_BNXT_DESCS_PER_LOOP); 1145 1146 ring->vmem = (void **)&rxr->rx_buf_ring; 1147 ring->fw_ring_id = INVALID_HW_RING_ID; 1148 1149 cpr = rte_zmalloc_socket("bnxt_rx_ring", 1150 sizeof(struct bnxt_cp_ring_info), 1151 RTE_CACHE_LINE_SIZE, socket_id); 1152 if (cpr == NULL) 1153 return -ENOMEM; 1154 rxq->cp_ring = cpr; 1155 1156 ring = rte_zmalloc_socket("bnxt_rx_ring_struct", 1157 sizeof(struct bnxt_ring), 1158 RTE_CACHE_LINE_SIZE, socket_id); 1159 if (ring == NULL) 1160 return -ENOMEM; 1161 cpr->cp_ring_struct = ring; 1162 1163 /* Allocate two completion slots per entry in desc ring. */ 1164 ring->ring_size = rxr->rx_ring_struct->ring_size * 2; 1165 ring->ring_size *= AGG_RING_SIZE_FACTOR; 1166 1167 ring->ring_size = rte_align32pow2(ring->ring_size); 1168 ring->ring_mask = ring->ring_size - 1; 1169 ring->bd = (void *)cpr->cp_desc_ring; 1170 ring->bd_dma = cpr->cp_desc_mapping; 1171 ring->vmem_size = 0; 1172 ring->vmem = NULL; 1173 ring->fw_ring_id = INVALID_HW_RING_ID; 1174 1175 /* Allocate Aggregator rings */ 1176 ring = rte_zmalloc_socket("bnxt_rx_ring_struct", 1177 sizeof(struct bnxt_ring), 1178 RTE_CACHE_LINE_SIZE, socket_id); 1179 if (ring == NULL) 1180 return -ENOMEM; 1181 rxr->ag_ring_struct = ring; 1182 ring->ring_size = rte_align32pow2(rxq->nb_rx_desc * 1183 AGG_RING_SIZE_FACTOR); 1184 ring->ring_mask = ring->ring_size - 1; 1185 ring->bd = (void *)rxr->ag_desc_ring; 1186 ring->bd_dma = rxr->ag_desc_mapping; 1187 ring->vmem_size = ring->ring_size * sizeof(struct rte_mbuf *); 1188 ring->vmem = (void **)&rxr->ag_buf_ring; 1189 ring->fw_ring_id = INVALID_HW_RING_ID; 1190 1191 return 0; 1192 } 1193 1194 static void bnxt_init_rxbds(struct bnxt_ring *ring, uint32_t type, 1195 uint16_t len) 1196 { 1197 uint32_t j; 1198 struct rx_prod_pkt_bd *rx_bd_ring = (struct rx_prod_pkt_bd *)ring->bd; 1199 1200 if (!rx_bd_ring) 1201 return; 1202 for (j = 0; j < ring->ring_size; j++) { 1203 rx_bd_ring[j].flags_type = rte_cpu_to_le_16(type); 1204 rx_bd_ring[j].len = rte_cpu_to_le_16(len); 1205 rx_bd_ring[j].opaque = j; 1206 } 1207 } 1208 1209 int bnxt_init_one_rx_ring(struct bnxt_rx_queue *rxq) 1210 { 1211 struct bnxt_rx_ring_info *rxr; 1212 struct bnxt_ring *ring; 1213 uint32_t raw_prod, type; 1214 unsigned int i; 1215 uint16_t size; 1216 1217 /* Initialize packet type table. */ 1218 bnxt_init_ptype_table(); 1219 1220 size = rte_pktmbuf_data_room_size(rxq->mb_pool) - RTE_PKTMBUF_HEADROOM; 1221 size = RTE_MIN(BNXT_MAX_PKT_LEN, size); 1222 1223 type = RX_PROD_PKT_BD_TYPE_RX_PROD_PKT; 1224 1225 rxr = rxq->rx_ring; 1226 ring = rxr->rx_ring_struct; 1227 bnxt_init_rxbds(ring, type, size); 1228 1229 /* Initialize offload flags parsing table. */ 1230 bnxt_init_ol_flags_tables(rxq); 1231 1232 raw_prod = rxr->rx_raw_prod; 1233 for (i = 0; i < ring->ring_size; i++) { 1234 if (unlikely(!rxr->rx_buf_ring[i])) { 1235 if (bnxt_alloc_rx_data(rxq, rxr, raw_prod) != 0) { 1236 PMD_DRV_LOG(WARNING, 1237 "init'ed rx ring %d with %d/%d mbufs only\n", 1238 rxq->queue_id, i, ring->ring_size); 1239 break; 1240 } 1241 } 1242 rxr->rx_raw_prod = raw_prod; 1243 raw_prod = RING_NEXT(raw_prod); 1244 } 1245 1246 /* Initialize dummy mbuf pointers for vector mode rx. */ 1247 for (i = ring->ring_size; 1248 i < ring->ring_size + RTE_BNXT_DESCS_PER_LOOP; i++) { 1249 rxr->rx_buf_ring[i] = &rxq->fake_mbuf; 1250 } 1251 1252 ring = rxr->ag_ring_struct; 1253 type = RX_PROD_AGG_BD_TYPE_RX_PROD_AGG; 1254 bnxt_init_rxbds(ring, type, size); 1255 raw_prod = rxr->ag_raw_prod; 1256 1257 for (i = 0; i < ring->ring_size; i++) { 1258 if (unlikely(!rxr->ag_buf_ring[i])) { 1259 if (bnxt_alloc_ag_data(rxq, rxr, raw_prod) != 0) { 1260 PMD_DRV_LOG(WARNING, 1261 "init'ed AG ring %d with %d/%d mbufs only\n", 1262 rxq->queue_id, i, ring->ring_size); 1263 break; 1264 } 1265 } 1266 rxr->ag_raw_prod = raw_prod; 1267 raw_prod = RING_NEXT(raw_prod); 1268 } 1269 PMD_DRV_LOG(DEBUG, "AGG Done!\n"); 1270 1271 if (rxr->tpa_info) { 1272 unsigned int max_aggs = BNXT_TPA_MAX_AGGS(rxq->bp); 1273 1274 for (i = 0; i < max_aggs; i++) { 1275 if (unlikely(!rxr->tpa_info[i].mbuf)) { 1276 rxr->tpa_info[i].mbuf = 1277 __bnxt_alloc_rx_data(rxq->mb_pool); 1278 if (!rxr->tpa_info[i].mbuf) { 1279 rte_atomic64_inc(&rxq->rx_mbuf_alloc_fail); 1280 return -ENOMEM; 1281 } 1282 } 1283 } 1284 } 1285 PMD_DRV_LOG(DEBUG, "TPA alloc Done!\n"); 1286 1287 return 0; 1288 } 1289