1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2014-2018 Broadcom 3 * All rights reserved. 4 */ 5 6 #include <inttypes.h> 7 8 #include <rte_malloc.h> 9 10 #include "bnxt.h" 11 #include "bnxt_filter.h" 12 #include "bnxt_hwrm.h" 13 #include "bnxt_ring.h" 14 #include "bnxt_rxq.h" 15 #include "bnxt_rxr.h" 16 #include "bnxt_vnic.h" 17 #include "hsi_struct_def_dpdk.h" 18 19 /* 20 * RX Queues 21 */ 22 23 void bnxt_free_rxq_stats(struct bnxt_rx_queue *rxq) 24 { 25 if (rxq && rxq->cp_ring && rxq->cp_ring->hw_stats) 26 rxq->cp_ring->hw_stats = NULL; 27 } 28 29 int bnxt_mq_rx_configure(struct bnxt *bp) 30 { 31 struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf; 32 const struct rte_eth_vmdq_rx_conf *conf = 33 &dev_conf->rx_adv_conf.vmdq_rx_conf; 34 unsigned int i, j, nb_q_per_grp = 1, ring_idx = 0; 35 int start_grp_id, end_grp_id = 1, rc = 0; 36 struct bnxt_vnic_info *vnic; 37 struct bnxt_filter_info *filter; 38 enum rte_eth_nb_pools pools = 1, max_pools = 0; 39 struct bnxt_rx_queue *rxq; 40 41 bp->nr_vnics = 0; 42 43 /* Single queue mode */ 44 if (bp->rx_cp_nr_rings < 2) { 45 vnic = &bp->vnic_info[0]; 46 if (!vnic) { 47 PMD_DRV_LOG(ERR, "VNIC alloc failed\n"); 48 rc = -ENOMEM; 49 goto err_out; 50 } 51 vnic->flags |= BNXT_VNIC_INFO_BCAST; 52 bp->nr_vnics++; 53 54 rxq = bp->eth_dev->data->rx_queues[0]; 55 rxq->vnic = vnic; 56 57 vnic->func_default = true; 58 vnic->start_grp_id = 0; 59 vnic->end_grp_id = vnic->start_grp_id; 60 filter = bnxt_alloc_filter(bp); 61 if (!filter) { 62 PMD_DRV_LOG(ERR, "L2 filter alloc failed\n"); 63 rc = -ENOMEM; 64 goto err_out; 65 } 66 filter->mac_index = 0; 67 filter->flags |= HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST; 68 STAILQ_INSERT_TAIL(&vnic->filter, filter, next); 69 goto out; 70 } 71 72 /* Multi-queue mode */ 73 if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_DCB_RSS) { 74 /* VMDq ONLY, VMDq+RSS, VMDq+DCB, VMDq+DCB+RSS */ 75 76 switch (dev_conf->rxmode.mq_mode) { 77 case ETH_MQ_RX_VMDQ_RSS: 78 case ETH_MQ_RX_VMDQ_ONLY: 79 case ETH_MQ_RX_VMDQ_DCB_RSS: 80 /* FALLTHROUGH */ 81 /* ETH_8/64_POOLs */ 82 pools = conf->nb_queue_pools; 83 /* For each pool, allocate MACVLAN CFA rule & VNIC */ 84 max_pools = RTE_MIN(bp->max_vnics, 85 RTE_MIN(bp->max_l2_ctx, 86 RTE_MIN(bp->max_rsscos_ctx, 87 ETH_64_POOLS))); 88 PMD_DRV_LOG(DEBUG, 89 "pools = %u max_pools = %u\n", 90 pools, max_pools); 91 if (pools > max_pools) 92 pools = max_pools; 93 break; 94 case ETH_MQ_RX_RSS: 95 pools = bp->rx_cosq_cnt ? bp->rx_cosq_cnt : 1; 96 break; 97 default: 98 PMD_DRV_LOG(ERR, "Unsupported mq_mod %d\n", 99 dev_conf->rxmode.mq_mode); 100 rc = -EINVAL; 101 goto err_out; 102 } 103 } else if (!dev_conf->rxmode.mq_mode) { 104 pools = bp->rx_cosq_cnt ? bp->rx_cosq_cnt : pools; 105 } 106 107 pools = RTE_MIN(pools, bp->rx_cp_nr_rings); 108 nb_q_per_grp = bp->rx_cp_nr_rings / pools; 109 bp->rx_num_qs_per_vnic = nb_q_per_grp; 110 PMD_DRV_LOG(DEBUG, "pools = %u nb_q_per_grp = %u\n", 111 pools, nb_q_per_grp); 112 start_grp_id = 0; 113 end_grp_id = nb_q_per_grp; 114 115 for (i = 0; i < pools; i++) { 116 vnic = &bp->vnic_info[i]; 117 if (!vnic) { 118 PMD_DRV_LOG(ERR, "VNIC alloc failed\n"); 119 rc = -ENOMEM; 120 goto err_out; 121 } 122 vnic->flags |= BNXT_VNIC_INFO_BCAST; 123 bp->nr_vnics++; 124 125 for (j = 0; j < nb_q_per_grp; j++, ring_idx++) { 126 rxq = bp->eth_dev->data->rx_queues[ring_idx]; 127 rxq->vnic = vnic; 128 PMD_DRV_LOG(DEBUG, 129 "rxq[%d] = %p vnic[%d] = %p\n", 130 ring_idx, rxq, i, vnic); 131 } 132 if (i == 0) { 133 if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_DCB) { 134 bp->eth_dev->data->promiscuous = 1; 135 vnic->flags |= BNXT_VNIC_INFO_PROMISC; 136 } 137 vnic->func_default = true; 138 } 139 vnic->start_grp_id = start_grp_id; 140 vnic->end_grp_id = end_grp_id; 141 142 if (i) { 143 if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_DCB || 144 !(dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS)) 145 vnic->rss_dflt_cr = true; 146 goto skip_filter_allocation; 147 } 148 filter = bnxt_alloc_filter(bp); 149 if (!filter) { 150 PMD_DRV_LOG(ERR, "L2 filter alloc failed\n"); 151 rc = -ENOMEM; 152 goto err_out; 153 } 154 filter->mac_index = 0; 155 filter->flags |= HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST; 156 /* 157 * TODO: Configure & associate CFA rule for 158 * each VNIC for each VMDq with MACVLAN, MACVLAN+TC 159 */ 160 STAILQ_INSERT_TAIL(&vnic->filter, filter, next); 161 162 skip_filter_allocation: 163 start_grp_id = end_grp_id; 164 end_grp_id += nb_q_per_grp; 165 } 166 167 out: 168 if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) { 169 struct rte_eth_rss_conf *rss = &dev_conf->rx_adv_conf.rss_conf; 170 171 if (bp->flags & BNXT_FLAG_UPDATE_HASH) 172 bp->flags &= ~BNXT_FLAG_UPDATE_HASH; 173 174 for (i = 0; i < bp->nr_vnics; i++) { 175 vnic = &bp->vnic_info[i]; 176 vnic->hash_type = 177 bnxt_rte_to_hwrm_hash_types(rss->rss_hf); 178 179 /* 180 * Use the supplied key if the key length is 181 * acceptable and the rss_key is not NULL 182 */ 183 if (rss->rss_key && 184 rss->rss_key_len <= HW_HASH_KEY_SIZE) 185 memcpy(vnic->rss_hash_key, 186 rss->rss_key, rss->rss_key_len); 187 } 188 } 189 190 return rc; 191 192 err_out: 193 /* Free allocated vnic/filters */ 194 195 return rc; 196 } 197 198 void bnxt_rx_queue_release_mbufs(struct bnxt_rx_queue *rxq) 199 { 200 struct rte_mbuf **sw_ring; 201 struct bnxt_tpa_info *tpa_info; 202 uint16_t i; 203 204 if (!rxq) 205 return; 206 207 rte_spinlock_lock(&rxq->lock); 208 209 sw_ring = rxq->rx_ring->rx_buf_ring; 210 if (sw_ring) { 211 for (i = 0; 212 i < rxq->rx_ring->rx_ring_struct->ring_size; i++) { 213 if (sw_ring[i]) { 214 if (sw_ring[i] != &rxq->fake_mbuf) 215 rte_pktmbuf_free_seg(sw_ring[i]); 216 sw_ring[i] = NULL; 217 } 218 } 219 } 220 /* Free up mbufs in Agg ring */ 221 sw_ring = rxq->rx_ring->ag_buf_ring; 222 if (sw_ring) { 223 for (i = 0; 224 i < rxq->rx_ring->ag_ring_struct->ring_size; i++) { 225 if (sw_ring[i]) { 226 rte_pktmbuf_free_seg(sw_ring[i]); 227 sw_ring[i] = NULL; 228 } 229 } 230 } 231 232 /* Free up mbufs in TPA */ 233 tpa_info = rxq->rx_ring->tpa_info; 234 if (tpa_info) { 235 int max_aggs = BNXT_TPA_MAX_AGGS(rxq->bp); 236 237 for (i = 0; i < max_aggs; i++) { 238 if (tpa_info[i].mbuf) { 239 rte_pktmbuf_free_seg(tpa_info[i].mbuf); 240 tpa_info[i].mbuf = NULL; 241 } 242 } 243 } 244 245 rte_spinlock_unlock(&rxq->lock); 246 } 247 248 void bnxt_free_rx_mbufs(struct bnxt *bp) 249 { 250 struct bnxt_rx_queue *rxq; 251 int i; 252 253 for (i = 0; i < (int)bp->rx_nr_rings; i++) { 254 rxq = bp->rx_queues[i]; 255 bnxt_rx_queue_release_mbufs(rxq); 256 } 257 } 258 259 void bnxt_rx_queue_release_op(void *rx_queue) 260 { 261 struct bnxt_rx_queue *rxq = (struct bnxt_rx_queue *)rx_queue; 262 263 if (rxq) { 264 if (is_bnxt_in_error(rxq->bp)) 265 return; 266 267 bnxt_rx_queue_release_mbufs(rxq); 268 269 /* Free RX ring hardware descriptors */ 270 bnxt_free_ring(rxq->rx_ring->rx_ring_struct); 271 /* Free RX Agg ring hardware descriptors */ 272 bnxt_free_ring(rxq->rx_ring->ag_ring_struct); 273 274 /* Free RX completion ring hardware descriptors */ 275 bnxt_free_ring(rxq->cp_ring->cp_ring_struct); 276 277 bnxt_free_rxq_stats(rxq); 278 rte_memzone_free(rxq->mz); 279 rxq->mz = NULL; 280 281 rte_free(rxq); 282 } 283 } 284 285 int bnxt_rx_queue_setup_op(struct rte_eth_dev *eth_dev, 286 uint16_t queue_idx, 287 uint16_t nb_desc, 288 unsigned int socket_id, 289 const struct rte_eth_rxconf *rx_conf, 290 struct rte_mempool *mp) 291 { 292 struct bnxt *bp = eth_dev->data->dev_private; 293 uint64_t rx_offloads = eth_dev->data->dev_conf.rxmode.offloads; 294 struct bnxt_rx_queue *rxq; 295 int rc = 0; 296 uint8_t queue_state; 297 298 rc = is_bnxt_in_error(bp); 299 if (rc) 300 return rc; 301 302 if (queue_idx >= BNXT_MAX_RINGS(bp)) { 303 PMD_DRV_LOG(ERR, 304 "Cannot create Rx ring %d. Only %d rings available\n", 305 queue_idx, bp->max_rx_rings); 306 return -EINVAL; 307 } 308 309 if (nb_desc < BNXT_MIN_RING_DESC || nb_desc > MAX_RX_DESC_CNT) { 310 PMD_DRV_LOG(ERR, "nb_desc %d is invalid\n", nb_desc); 311 rc = -EINVAL; 312 goto out; 313 } 314 315 if (eth_dev->data->rx_queues) { 316 rxq = eth_dev->data->rx_queues[queue_idx]; 317 if (rxq) 318 bnxt_rx_queue_release_op(rxq); 319 } 320 rxq = rte_zmalloc_socket("bnxt_rx_queue", sizeof(struct bnxt_rx_queue), 321 RTE_CACHE_LINE_SIZE, socket_id); 322 if (!rxq) { 323 PMD_DRV_LOG(ERR, "bnxt_rx_queue allocation failed!\n"); 324 rc = -ENOMEM; 325 goto out; 326 } 327 rxq->bp = bp; 328 rxq->mb_pool = mp; 329 rxq->nb_rx_desc = nb_desc; 330 rxq->rx_free_thresh = 331 RTE_MIN(rte_align32pow2(nb_desc) / 4, RTE_BNXT_MAX_RX_BURST); 332 333 PMD_DRV_LOG(DEBUG, "RX Buf MTU %d\n", eth_dev->data->mtu); 334 335 rc = bnxt_init_rx_ring_struct(rxq, socket_id); 336 if (rc) 337 goto out; 338 339 PMD_DRV_LOG(DEBUG, "RX Buf size is %d\n", rxq->rx_buf_size); 340 rxq->queue_id = queue_idx; 341 rxq->port_id = eth_dev->data->port_id; 342 if (rx_offloads & DEV_RX_OFFLOAD_KEEP_CRC) 343 rxq->crc_len = RTE_ETHER_CRC_LEN; 344 else 345 rxq->crc_len = 0; 346 347 eth_dev->data->rx_queues[queue_idx] = rxq; 348 /* Allocate RX ring hardware descriptors */ 349 if (bnxt_alloc_rings(bp, queue_idx, NULL, rxq, rxq->cp_ring, NULL, 350 "rxr")) { 351 PMD_DRV_LOG(ERR, 352 "ring_dma_zone_reserve for rx_ring failed!\n"); 353 bnxt_rx_queue_release_op(rxq); 354 rc = -ENOMEM; 355 goto out; 356 } 357 rte_atomic64_init(&rxq->rx_mbuf_alloc_fail); 358 359 /* rxq 0 must not be stopped when used as async CPR */ 360 if (!BNXT_NUM_ASYNC_CPR(bp) && queue_idx == 0) 361 rxq->rx_deferred_start = false; 362 else 363 rxq->rx_deferred_start = rx_conf->rx_deferred_start; 364 365 if (rxq->rx_deferred_start) { 366 queue_state = RTE_ETH_QUEUE_STATE_STOPPED; 367 rxq->rx_started = false; 368 } else { 369 queue_state = RTE_ETH_QUEUE_STATE_STARTED; 370 rxq->rx_started = true; 371 } 372 eth_dev->data->rx_queue_state[queue_idx] = queue_state; 373 rte_spinlock_init(&rxq->lock); 374 375 /* Configure mtu if it is different from what was configured before */ 376 if (!queue_idx) 377 bnxt_mtu_set_op(eth_dev, eth_dev->data->mtu); 378 379 out: 380 return rc; 381 } 382 383 int 384 bnxt_rx_queue_intr_enable_op(struct rte_eth_dev *eth_dev, uint16_t queue_id) 385 { 386 struct bnxt *bp = eth_dev->data->dev_private; 387 struct bnxt_rx_queue *rxq; 388 struct bnxt_cp_ring_info *cpr; 389 int rc = 0; 390 391 rc = is_bnxt_in_error(bp); 392 if (rc) 393 return rc; 394 395 if (eth_dev->data->rx_queues) { 396 rxq = eth_dev->data->rx_queues[queue_id]; 397 if (!rxq) 398 return -EINVAL; 399 400 cpr = rxq->cp_ring; 401 B_CP_DB_REARM(cpr, cpr->cp_raw_cons); 402 } 403 return rc; 404 } 405 406 int 407 bnxt_rx_queue_intr_disable_op(struct rte_eth_dev *eth_dev, uint16_t queue_id) 408 { 409 struct bnxt *bp = eth_dev->data->dev_private; 410 struct bnxt_rx_queue *rxq; 411 struct bnxt_cp_ring_info *cpr; 412 int rc = 0; 413 414 rc = is_bnxt_in_error(bp); 415 if (rc) 416 return rc; 417 418 if (eth_dev->data->rx_queues) { 419 rxq = eth_dev->data->rx_queues[queue_id]; 420 if (!rxq) 421 return -EINVAL; 422 423 cpr = rxq->cp_ring; 424 B_CP_DB_DISARM(cpr); 425 } 426 return rc; 427 } 428 429 int bnxt_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) 430 { 431 struct bnxt *bp = dev->data->dev_private; 432 struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf; 433 struct bnxt_rx_queue *rxq = bp->rx_queues[rx_queue_id]; 434 struct bnxt_vnic_info *vnic = NULL; 435 int rc = 0; 436 437 rc = is_bnxt_in_error(bp); 438 if (rc) 439 return rc; 440 441 if (rxq == NULL) { 442 PMD_DRV_LOG(ERR, "Invalid Rx queue %d\n", rx_queue_id); 443 return -EINVAL; 444 } 445 446 /* Set the queue state to started here. 447 * We check the status of the queue while posting buffer. 448 * If queue is it started, we do not post buffers for Rx. 449 */ 450 rxq->rx_started = true; 451 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 452 453 bnxt_free_hwrm_rx_ring(bp, rx_queue_id); 454 rc = bnxt_alloc_hwrm_rx_ring(bp, rx_queue_id); 455 if (rc) 456 return rc; 457 458 if (BNXT_CHIP_THOR(bp)) { 459 /* Reconfigure default receive ring and MRU. */ 460 bnxt_hwrm_vnic_cfg(bp, rxq->vnic); 461 } 462 PMD_DRV_LOG(INFO, "Rx queue started %d\n", rx_queue_id); 463 464 if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) { 465 vnic = rxq->vnic; 466 467 if (BNXT_HAS_RING_GRPS(bp)) { 468 if (vnic->fw_grp_ids[rx_queue_id] != INVALID_HW_RING_ID) 469 return 0; 470 471 vnic->fw_grp_ids[rx_queue_id] = 472 bp->grp_info[rx_queue_id].fw_grp_id; 473 PMD_DRV_LOG(DEBUG, 474 "vnic = %p fw_grp_id = %d\n", 475 vnic, bp->grp_info[rx_queue_id].fw_grp_id); 476 } 477 478 PMD_DRV_LOG(DEBUG, "Rx Queue Count %d\n", vnic->rx_queue_cnt); 479 rc = bnxt_vnic_rss_configure(bp, vnic); 480 } 481 482 if (rc != 0) { 483 dev->data->rx_queue_state[rx_queue_id] = 484 RTE_ETH_QUEUE_STATE_STOPPED; 485 rxq->rx_started = false; 486 } 487 488 PMD_DRV_LOG(INFO, 489 "queue %d, rx_deferred_start %d, state %d!\n", 490 rx_queue_id, rxq->rx_deferred_start, 491 bp->eth_dev->data->rx_queue_state[rx_queue_id]); 492 493 return rc; 494 } 495 496 int bnxt_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) 497 { 498 struct bnxt *bp = dev->data->dev_private; 499 struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf; 500 struct bnxt_vnic_info *vnic = NULL; 501 struct bnxt_rx_queue *rxq = NULL; 502 int active_queue_cnt = 0; 503 int i, rc = 0; 504 505 rc = is_bnxt_in_error(bp); 506 if (rc) 507 return rc; 508 509 /* For the stingray platform and other platforms needing tighter 510 * control of resource utilization, Rx CQ 0 also works as 511 * Default CQ for async notifications 512 */ 513 if (!BNXT_NUM_ASYNC_CPR(bp) && !rx_queue_id) { 514 PMD_DRV_LOG(ERR, "Cannot stop Rx queue id %d\n", rx_queue_id); 515 return -EINVAL; 516 } 517 518 rxq = bp->rx_queues[rx_queue_id]; 519 if (!rxq) { 520 PMD_DRV_LOG(ERR, "Invalid Rx queue %d\n", rx_queue_id); 521 return -EINVAL; 522 } 523 524 vnic = rxq->vnic; 525 if (!vnic) { 526 PMD_DRV_LOG(ERR, "VNIC not initialized for RxQ %d\n", 527 rx_queue_id); 528 return -EINVAL; 529 } 530 531 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 532 rxq->rx_started = false; 533 PMD_DRV_LOG(DEBUG, "Rx queue stopped\n"); 534 535 if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) { 536 if (BNXT_HAS_RING_GRPS(bp)) 537 vnic->fw_grp_ids[rx_queue_id] = INVALID_HW_RING_ID; 538 539 PMD_DRV_LOG(DEBUG, "Rx Queue Count %d\n", vnic->rx_queue_cnt); 540 rc = bnxt_vnic_rss_configure(bp, vnic); 541 } 542 543 if (BNXT_CHIP_THOR(bp)) { 544 /* Compute current number of active receive queues. */ 545 for (i = vnic->start_grp_id; i < vnic->end_grp_id; i++) 546 if (bp->rx_queues[i]->rx_started) 547 active_queue_cnt++; 548 549 /* 550 * For Thor, we need to ensure that the VNIC default receive 551 * ring corresponds to an active receive queue. When no queue 552 * is active, we need to temporarily set the MRU to zero so 553 * that packets are dropped early in the receive pipeline in 554 * order to prevent the VNIC default receive ring from being 555 * accessed. 556 */ 557 if (active_queue_cnt == 0) { 558 uint16_t saved_mru = vnic->mru; 559 560 vnic->mru = 0; 561 /* Reconfigure default receive ring and MRU. */ 562 bnxt_hwrm_vnic_cfg(bp, vnic); 563 vnic->mru = saved_mru; 564 } else { 565 /* Reconfigure default receive ring. */ 566 bnxt_hwrm_vnic_cfg(bp, vnic); 567 } 568 } 569 570 if (rc == 0) 571 bnxt_rx_queue_release_mbufs(rxq); 572 573 return rc; 574 } 575