1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 #include <cnxk_ethdev.h> 5 6 static inline uint64_t 7 nix_get_rx_offload_capa(struct cnxk_eth_dev *dev) 8 { 9 uint64_t capa = CNXK_NIX_RX_OFFLOAD_CAPA; 10 11 if (roc_nix_is_vf_or_sdp(&dev->nix) || 12 dev->npc.switch_header_type == ROC_PRIV_FLAGS_HIGIG) 13 capa &= ~DEV_RX_OFFLOAD_TIMESTAMP; 14 15 return capa; 16 } 17 18 static inline uint64_t 19 nix_get_tx_offload_capa(struct cnxk_eth_dev *dev) 20 { 21 RTE_SET_USED(dev); 22 return CNXK_NIX_TX_OFFLOAD_CAPA; 23 } 24 25 static inline uint32_t 26 nix_get_speed_capa(struct cnxk_eth_dev *dev) 27 { 28 uint32_t speed_capa; 29 30 /* Auto negotiation disabled */ 31 speed_capa = ETH_LINK_SPEED_FIXED; 32 if (!roc_nix_is_vf_or_sdp(&dev->nix) && !roc_nix_is_lbk(&dev->nix)) { 33 speed_capa |= ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G | 34 ETH_LINK_SPEED_25G | ETH_LINK_SPEED_40G | 35 ETH_LINK_SPEED_50G | ETH_LINK_SPEED_100G; 36 } 37 38 return speed_capa; 39 } 40 41 static void 42 nix_enable_mseg_on_jumbo(struct cnxk_eth_rxq_sp *rxq) 43 { 44 struct rte_pktmbuf_pool_private *mbp_priv; 45 struct rte_eth_dev *eth_dev; 46 struct cnxk_eth_dev *dev; 47 uint32_t buffsz; 48 49 dev = rxq->dev; 50 eth_dev = dev->eth_dev; 51 52 /* Get rx buffer size */ 53 mbp_priv = rte_mempool_get_priv(rxq->qconf.mp); 54 buffsz = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM; 55 56 if (eth_dev->data->dev_conf.rxmode.max_rx_pkt_len > buffsz) { 57 dev->rx_offloads |= DEV_RX_OFFLOAD_SCATTER; 58 dev->tx_offloads |= DEV_TX_OFFLOAD_MULTI_SEGS; 59 } 60 } 61 62 int 63 nix_recalc_mtu(struct rte_eth_dev *eth_dev) 64 { 65 struct rte_eth_dev_data *data = eth_dev->data; 66 struct cnxk_eth_rxq_sp *rxq; 67 uint16_t mtu; 68 int rc; 69 70 rxq = ((struct cnxk_eth_rxq_sp *)data->rx_queues[0]) - 1; 71 /* Setup scatter mode if needed by jumbo */ 72 nix_enable_mseg_on_jumbo(rxq); 73 74 /* Setup MTU based on max_rx_pkt_len */ 75 mtu = data->dev_conf.rxmode.max_rx_pkt_len - CNXK_NIX_L2_OVERHEAD + 76 CNXK_NIX_MAX_VTAG_ACT_SIZE; 77 78 rc = cnxk_nix_mtu_set(eth_dev, mtu); 79 if (rc) 80 plt_err("Failed to set default MTU size, rc=%d", rc); 81 82 return rc; 83 } 84 85 static int 86 nix_init_flow_ctrl_config(struct rte_eth_dev *eth_dev) 87 { 88 struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); 89 struct cnxk_fc_cfg *fc = &dev->fc_cfg; 90 struct rte_eth_fc_conf fc_conf = {0}; 91 int rc; 92 93 /* Both Rx & Tx flow ctrl get enabled(RTE_FC_FULL) in HW 94 * by AF driver, update those info in PMD structure. 95 */ 96 rc = cnxk_nix_flow_ctrl_get(eth_dev, &fc_conf); 97 if (rc) 98 goto exit; 99 100 fc->mode = fc_conf.mode; 101 fc->rx_pause = (fc_conf.mode == RTE_FC_FULL) || 102 (fc_conf.mode == RTE_FC_RX_PAUSE); 103 fc->tx_pause = (fc_conf.mode == RTE_FC_FULL) || 104 (fc_conf.mode == RTE_FC_TX_PAUSE); 105 106 exit: 107 return rc; 108 } 109 110 static int 111 nix_update_flow_ctrl_config(struct rte_eth_dev *eth_dev) 112 { 113 struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); 114 struct cnxk_fc_cfg *fc = &dev->fc_cfg; 115 struct rte_eth_fc_conf fc_cfg = {0}; 116 117 if (roc_nix_is_vf_or_sdp(&dev->nix)) 118 return 0; 119 120 fc_cfg.mode = fc->mode; 121 122 /* To avoid Link credit deadlock on Ax, disable Tx FC if it's enabled */ 123 if (roc_model_is_cn96_ax() && 124 dev->npc.switch_header_type != ROC_PRIV_FLAGS_HIGIG && 125 (fc_cfg.mode == RTE_FC_FULL || fc_cfg.mode == RTE_FC_RX_PAUSE)) { 126 fc_cfg.mode = 127 (fc_cfg.mode == RTE_FC_FULL || 128 fc_cfg.mode == RTE_FC_TX_PAUSE) ? 129 RTE_FC_TX_PAUSE : RTE_FC_NONE; 130 } 131 132 return cnxk_nix_flow_ctrl_set(eth_dev, &fc_cfg); 133 } 134 135 uint64_t 136 cnxk_nix_rxq_mbuf_setup(struct cnxk_eth_dev *dev) 137 { 138 uint16_t port_id = dev->eth_dev->data->port_id; 139 struct rte_mbuf mb_def; 140 uint64_t *tmp; 141 142 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) % 8 != 0); 143 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, refcnt) - 144 offsetof(struct rte_mbuf, data_off) != 145 2); 146 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, nb_segs) - 147 offsetof(struct rte_mbuf, data_off) != 148 4); 149 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, port) - 150 offsetof(struct rte_mbuf, data_off) != 151 6); 152 mb_def.nb_segs = 1; 153 mb_def.data_off = RTE_PKTMBUF_HEADROOM + 154 (dev->ptp_en * CNXK_NIX_TIMESYNC_RX_OFFSET); 155 mb_def.port = port_id; 156 rte_mbuf_refcnt_set(&mb_def, 1); 157 158 /* Prevent compiler reordering: rearm_data covers previous fields */ 159 rte_compiler_barrier(); 160 tmp = (uint64_t *)&mb_def.rearm_data; 161 162 return *tmp; 163 } 164 165 static inline uint8_t 166 nix_sq_max_sqe_sz(struct cnxk_eth_dev *dev) 167 { 168 /* 169 * Maximum three segments can be supported with W8, Choose 170 * NIX_MAXSQESZ_W16 for multi segment offload. 171 */ 172 if (dev->tx_offloads & DEV_TX_OFFLOAD_MULTI_SEGS) 173 return NIX_MAXSQESZ_W16; 174 else 175 return NIX_MAXSQESZ_W8; 176 } 177 178 int 179 cnxk_nix_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t qid, 180 uint16_t nb_desc, uint16_t fp_tx_q_sz, 181 const struct rte_eth_txconf *tx_conf) 182 { 183 struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); 184 const struct eth_dev_ops *dev_ops = eth_dev->dev_ops; 185 struct cnxk_eth_txq_sp *txq_sp; 186 struct roc_nix_sq *sq; 187 size_t txq_sz; 188 int rc; 189 190 /* Free memory prior to re-allocation if needed. */ 191 if (eth_dev->data->tx_queues[qid] != NULL) { 192 plt_nix_dbg("Freeing memory prior to re-allocation %d", qid); 193 dev_ops->tx_queue_release(eth_dev->data->tx_queues[qid]); 194 eth_dev->data->tx_queues[qid] = NULL; 195 } 196 197 /* Setup ROC SQ */ 198 sq = &dev->sqs[qid]; 199 sq->qid = qid; 200 sq->nb_desc = nb_desc; 201 sq->max_sqe_sz = nix_sq_max_sqe_sz(dev); 202 203 rc = roc_nix_sq_init(&dev->nix, sq); 204 if (rc) { 205 plt_err("Failed to init sq=%d, rc=%d", qid, rc); 206 return rc; 207 } 208 209 rc = -ENOMEM; 210 txq_sz = sizeof(struct cnxk_eth_txq_sp) + fp_tx_q_sz; 211 txq_sp = plt_zmalloc(txq_sz, PLT_CACHE_LINE_SIZE); 212 if (!txq_sp) { 213 plt_err("Failed to alloc tx queue mem"); 214 rc |= roc_nix_sq_fini(sq); 215 return rc; 216 } 217 218 txq_sp->dev = dev; 219 txq_sp->qid = qid; 220 txq_sp->qconf.conf.tx = *tx_conf; 221 txq_sp->qconf.nb_desc = nb_desc; 222 223 plt_nix_dbg("sq=%d fc=%p offload=0x%" PRIx64 " lmt_addr=%p" 224 " nb_sqb_bufs=%d sqes_per_sqb_log2=%d", 225 qid, sq->fc, dev->tx_offloads, sq->lmt_addr, 226 sq->nb_sqb_bufs, sq->sqes_per_sqb_log2); 227 228 /* Store start of fast path area */ 229 eth_dev->data->tx_queues[qid] = txq_sp + 1; 230 eth_dev->data->tx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STOPPED; 231 return 0; 232 } 233 234 static void 235 cnxk_nix_tx_queue_release(void *txq) 236 { 237 struct cnxk_eth_txq_sp *txq_sp; 238 struct cnxk_eth_dev *dev; 239 struct roc_nix_sq *sq; 240 uint16_t qid; 241 int rc; 242 243 if (!txq) 244 return; 245 246 txq_sp = cnxk_eth_txq_to_sp(txq); 247 dev = txq_sp->dev; 248 qid = txq_sp->qid; 249 250 plt_nix_dbg("Releasing txq %u", qid); 251 252 /* Cleanup ROC SQ */ 253 sq = &dev->sqs[qid]; 254 rc = roc_nix_sq_fini(sq); 255 if (rc) 256 plt_err("Failed to cleanup sq, rc=%d", rc); 257 258 /* Finally free */ 259 plt_free(txq_sp); 260 } 261 262 int 263 cnxk_nix_rx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t qid, 264 uint16_t nb_desc, uint16_t fp_rx_q_sz, 265 const struct rte_eth_rxconf *rx_conf, 266 struct rte_mempool *mp) 267 { 268 struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); 269 struct cnxk_eth_rxq_sp *rxq_sp; 270 struct rte_mempool_ops *ops; 271 const char *platform_ops; 272 struct roc_nix_rq *rq; 273 struct roc_nix_cq *cq; 274 uint16_t first_skip; 275 int rc = -EINVAL; 276 size_t rxq_sz; 277 278 /* Sanity checks */ 279 if (rx_conf->rx_deferred_start == 1) { 280 plt_err("Deferred Rx start is not supported"); 281 goto fail; 282 } 283 284 platform_ops = rte_mbuf_platform_mempool_ops(); 285 /* This driver needs cnxk_npa mempool ops to work */ 286 ops = rte_mempool_get_ops(mp->ops_index); 287 if (strncmp(ops->name, platform_ops, RTE_MEMPOOL_OPS_NAMESIZE)) { 288 plt_err("mempool ops should be of cnxk_npa type"); 289 goto fail; 290 } 291 292 if (mp->pool_id == 0) { 293 plt_err("Invalid pool_id"); 294 goto fail; 295 } 296 297 /* Free memory prior to re-allocation if needed */ 298 if (eth_dev->data->rx_queues[qid] != NULL) { 299 const struct eth_dev_ops *dev_ops = eth_dev->dev_ops; 300 301 plt_nix_dbg("Freeing memory prior to re-allocation %d", qid); 302 dev_ops->rx_queue_release(eth_dev->data->rx_queues[qid]); 303 eth_dev->data->rx_queues[qid] = NULL; 304 } 305 306 /* Setup ROC CQ */ 307 cq = &dev->cqs[qid]; 308 cq->qid = qid; 309 cq->nb_desc = nb_desc; 310 rc = roc_nix_cq_init(&dev->nix, cq); 311 if (rc) { 312 plt_err("Failed to init roc cq for rq=%d, rc=%d", qid, rc); 313 goto fail; 314 } 315 316 /* Setup ROC RQ */ 317 rq = &dev->rqs[qid]; 318 rq->qid = qid; 319 rq->aura_handle = mp->pool_id; 320 rq->flow_tag_width = 32; 321 rq->sso_ena = false; 322 323 /* Calculate first mbuf skip */ 324 first_skip = (sizeof(struct rte_mbuf)); 325 first_skip += RTE_PKTMBUF_HEADROOM; 326 first_skip += rte_pktmbuf_priv_size(mp); 327 rq->first_skip = first_skip; 328 rq->later_skip = sizeof(struct rte_mbuf); 329 rq->lpb_size = mp->elt_size; 330 331 rc = roc_nix_rq_init(&dev->nix, rq, !!eth_dev->data->dev_started); 332 if (rc) { 333 plt_err("Failed to init roc rq for rq=%d, rc=%d", qid, rc); 334 goto cq_fini; 335 } 336 337 /* Allocate and setup fast path rx queue */ 338 rc = -ENOMEM; 339 rxq_sz = sizeof(struct cnxk_eth_rxq_sp) + fp_rx_q_sz; 340 rxq_sp = plt_zmalloc(rxq_sz, PLT_CACHE_LINE_SIZE); 341 if (!rxq_sp) { 342 plt_err("Failed to alloc rx queue for rq=%d", qid); 343 goto rq_fini; 344 } 345 346 /* Setup slow path fields */ 347 rxq_sp->dev = dev; 348 rxq_sp->qid = qid; 349 rxq_sp->qconf.conf.rx = *rx_conf; 350 rxq_sp->qconf.nb_desc = nb_desc; 351 rxq_sp->qconf.mp = mp; 352 353 plt_nix_dbg("rq=%d pool=%s nb_desc=%d->%d", qid, mp->name, nb_desc, 354 cq->nb_desc); 355 356 /* Store start of fast path area */ 357 eth_dev->data->rx_queues[qid] = rxq_sp + 1; 358 eth_dev->data->rx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STOPPED; 359 360 /* Calculating delta and freq mult between PTP HI clock and tsc. 361 * These are needed in deriving raw clock value from tsc counter. 362 * read_clock eth op returns raw clock value. 363 */ 364 if ((dev->rx_offloads & DEV_RX_OFFLOAD_TIMESTAMP) || dev->ptp_en) { 365 rc = cnxk_nix_tsc_convert(dev); 366 if (rc) { 367 plt_err("Failed to calculate delta and freq mult"); 368 goto rq_fini; 369 } 370 } 371 372 return 0; 373 rq_fini: 374 rc |= roc_nix_rq_fini(rq); 375 cq_fini: 376 rc |= roc_nix_cq_fini(cq); 377 fail: 378 return rc; 379 } 380 381 static void 382 cnxk_nix_rx_queue_release(void *rxq) 383 { 384 struct cnxk_eth_rxq_sp *rxq_sp; 385 struct cnxk_eth_dev *dev; 386 struct roc_nix_rq *rq; 387 struct roc_nix_cq *cq; 388 uint16_t qid; 389 int rc; 390 391 if (!rxq) 392 return; 393 394 rxq_sp = cnxk_eth_rxq_to_sp(rxq); 395 dev = rxq_sp->dev; 396 qid = rxq_sp->qid; 397 398 plt_nix_dbg("Releasing rxq %u", qid); 399 400 /* Cleanup ROC RQ */ 401 rq = &dev->rqs[qid]; 402 rc = roc_nix_rq_fini(rq); 403 if (rc) 404 plt_err("Failed to cleanup rq, rc=%d", rc); 405 406 /* Cleanup ROC CQ */ 407 cq = &dev->cqs[qid]; 408 rc = roc_nix_cq_fini(cq); 409 if (rc) 410 plt_err("Failed to cleanup cq, rc=%d", rc); 411 412 /* Finally free fast path area */ 413 plt_free(rxq_sp); 414 } 415 416 uint32_t 417 cnxk_rss_ethdev_to_nix(struct cnxk_eth_dev *dev, uint64_t ethdev_rss, 418 uint8_t rss_level) 419 { 420 uint32_t flow_key_type[RSS_MAX_LEVELS][6] = { 421 {FLOW_KEY_TYPE_IPV4, FLOW_KEY_TYPE_IPV6, FLOW_KEY_TYPE_TCP, 422 FLOW_KEY_TYPE_UDP, FLOW_KEY_TYPE_SCTP, FLOW_KEY_TYPE_ETH_DMAC}, 423 {FLOW_KEY_TYPE_INNR_IPV4, FLOW_KEY_TYPE_INNR_IPV6, 424 FLOW_KEY_TYPE_INNR_TCP, FLOW_KEY_TYPE_INNR_UDP, 425 FLOW_KEY_TYPE_INNR_SCTP, FLOW_KEY_TYPE_INNR_ETH_DMAC}, 426 {FLOW_KEY_TYPE_IPV4 | FLOW_KEY_TYPE_INNR_IPV4, 427 FLOW_KEY_TYPE_IPV6 | FLOW_KEY_TYPE_INNR_IPV6, 428 FLOW_KEY_TYPE_TCP | FLOW_KEY_TYPE_INNR_TCP, 429 FLOW_KEY_TYPE_UDP | FLOW_KEY_TYPE_INNR_UDP, 430 FLOW_KEY_TYPE_SCTP | FLOW_KEY_TYPE_INNR_SCTP, 431 FLOW_KEY_TYPE_ETH_DMAC | FLOW_KEY_TYPE_INNR_ETH_DMAC} 432 }; 433 uint32_t flowkey_cfg = 0; 434 435 dev->ethdev_rss_hf = ethdev_rss; 436 437 if (ethdev_rss & ETH_RSS_L2_PAYLOAD && 438 dev->npc.switch_header_type == ROC_PRIV_FLAGS_LEN_90B) { 439 flowkey_cfg |= FLOW_KEY_TYPE_CH_LEN_90B; 440 } 441 442 if (ethdev_rss & ETH_RSS_C_VLAN) 443 flowkey_cfg |= FLOW_KEY_TYPE_VLAN; 444 445 if (ethdev_rss & ETH_RSS_L3_SRC_ONLY) 446 flowkey_cfg |= FLOW_KEY_TYPE_L3_SRC; 447 448 if (ethdev_rss & ETH_RSS_L3_DST_ONLY) 449 flowkey_cfg |= FLOW_KEY_TYPE_L3_DST; 450 451 if (ethdev_rss & ETH_RSS_L4_SRC_ONLY) 452 flowkey_cfg |= FLOW_KEY_TYPE_L4_SRC; 453 454 if (ethdev_rss & ETH_RSS_L4_DST_ONLY) 455 flowkey_cfg |= FLOW_KEY_TYPE_L4_DST; 456 457 if (ethdev_rss & RSS_IPV4_ENABLE) 458 flowkey_cfg |= flow_key_type[rss_level][RSS_IPV4_INDEX]; 459 460 if (ethdev_rss & RSS_IPV6_ENABLE) 461 flowkey_cfg |= flow_key_type[rss_level][RSS_IPV6_INDEX]; 462 463 if (ethdev_rss & ETH_RSS_TCP) 464 flowkey_cfg |= flow_key_type[rss_level][RSS_TCP_INDEX]; 465 466 if (ethdev_rss & ETH_RSS_UDP) 467 flowkey_cfg |= flow_key_type[rss_level][RSS_UDP_INDEX]; 468 469 if (ethdev_rss & ETH_RSS_SCTP) 470 flowkey_cfg |= flow_key_type[rss_level][RSS_SCTP_INDEX]; 471 472 if (ethdev_rss & ETH_RSS_L2_PAYLOAD) 473 flowkey_cfg |= flow_key_type[rss_level][RSS_DMAC_INDEX]; 474 475 if (ethdev_rss & RSS_IPV6_EX_ENABLE) 476 flowkey_cfg |= FLOW_KEY_TYPE_IPV6_EXT; 477 478 if (ethdev_rss & ETH_RSS_PORT) 479 flowkey_cfg |= FLOW_KEY_TYPE_PORT; 480 481 if (ethdev_rss & ETH_RSS_NVGRE) 482 flowkey_cfg |= FLOW_KEY_TYPE_NVGRE; 483 484 if (ethdev_rss & ETH_RSS_VXLAN) 485 flowkey_cfg |= FLOW_KEY_TYPE_VXLAN; 486 487 if (ethdev_rss & ETH_RSS_GENEVE) 488 flowkey_cfg |= FLOW_KEY_TYPE_GENEVE; 489 490 if (ethdev_rss & ETH_RSS_GTPU) 491 flowkey_cfg |= FLOW_KEY_TYPE_GTPU; 492 493 return flowkey_cfg; 494 } 495 496 static void 497 nix_free_queue_mem(struct cnxk_eth_dev *dev) 498 { 499 plt_free(dev->rqs); 500 plt_free(dev->cqs); 501 plt_free(dev->sqs); 502 dev->rqs = NULL; 503 dev->cqs = NULL; 504 dev->sqs = NULL; 505 } 506 507 static int 508 nix_rss_default_setup(struct cnxk_eth_dev *dev) 509 { 510 struct rte_eth_dev *eth_dev = dev->eth_dev; 511 uint8_t rss_hash_level; 512 uint32_t flowkey_cfg; 513 uint64_t rss_hf; 514 515 rss_hf = eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf; 516 rss_hash_level = ETH_RSS_LEVEL(rss_hf); 517 if (rss_hash_level) 518 rss_hash_level -= 1; 519 520 flowkey_cfg = cnxk_rss_ethdev_to_nix(dev, rss_hf, rss_hash_level); 521 return roc_nix_rss_default_setup(&dev->nix, flowkey_cfg); 522 } 523 524 static int 525 nix_store_queue_cfg_and_then_release(struct rte_eth_dev *eth_dev) 526 { 527 struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); 528 const struct eth_dev_ops *dev_ops = eth_dev->dev_ops; 529 struct cnxk_eth_qconf *tx_qconf = NULL; 530 struct cnxk_eth_qconf *rx_qconf = NULL; 531 struct cnxk_eth_rxq_sp *rxq_sp; 532 struct cnxk_eth_txq_sp *txq_sp; 533 int i, nb_rxq, nb_txq; 534 void **txq, **rxq; 535 536 nb_rxq = RTE_MIN(dev->nb_rxq, eth_dev->data->nb_rx_queues); 537 nb_txq = RTE_MIN(dev->nb_txq, eth_dev->data->nb_tx_queues); 538 539 tx_qconf = malloc(nb_txq * sizeof(*tx_qconf)); 540 if (tx_qconf == NULL) { 541 plt_err("Failed to allocate memory for tx_qconf"); 542 goto fail; 543 } 544 545 rx_qconf = malloc(nb_rxq * sizeof(*rx_qconf)); 546 if (rx_qconf == NULL) { 547 plt_err("Failed to allocate memory for rx_qconf"); 548 goto fail; 549 } 550 551 txq = eth_dev->data->tx_queues; 552 for (i = 0; i < nb_txq; i++) { 553 if (txq[i] == NULL) { 554 tx_qconf[i].valid = false; 555 plt_info("txq[%d] is already released", i); 556 continue; 557 } 558 txq_sp = cnxk_eth_txq_to_sp(txq[i]); 559 memcpy(&tx_qconf[i], &txq_sp->qconf, sizeof(*tx_qconf)); 560 tx_qconf[i].valid = true; 561 dev_ops->tx_queue_release(txq[i]); 562 eth_dev->data->tx_queues[i] = NULL; 563 } 564 565 rxq = eth_dev->data->rx_queues; 566 for (i = 0; i < nb_rxq; i++) { 567 if (rxq[i] == NULL) { 568 rx_qconf[i].valid = false; 569 plt_info("rxq[%d] is already released", i); 570 continue; 571 } 572 rxq_sp = cnxk_eth_rxq_to_sp(rxq[i]); 573 memcpy(&rx_qconf[i], &rxq_sp->qconf, sizeof(*rx_qconf)); 574 rx_qconf[i].valid = true; 575 dev_ops->rx_queue_release(rxq[i]); 576 eth_dev->data->rx_queues[i] = NULL; 577 } 578 579 dev->tx_qconf = tx_qconf; 580 dev->rx_qconf = rx_qconf; 581 return 0; 582 583 fail: 584 free(tx_qconf); 585 free(rx_qconf); 586 return -ENOMEM; 587 } 588 589 static int 590 nix_restore_queue_cfg(struct rte_eth_dev *eth_dev) 591 { 592 struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); 593 const struct eth_dev_ops *dev_ops = eth_dev->dev_ops; 594 struct cnxk_eth_qconf *tx_qconf = dev->tx_qconf; 595 struct cnxk_eth_qconf *rx_qconf = dev->rx_qconf; 596 int rc, i, nb_rxq, nb_txq; 597 void **txq, **rxq; 598 599 nb_rxq = RTE_MIN(dev->nb_rxq, eth_dev->data->nb_rx_queues); 600 nb_txq = RTE_MIN(dev->nb_txq, eth_dev->data->nb_tx_queues); 601 602 rc = -ENOMEM; 603 /* Setup tx & rx queues with previous configuration so 604 * that the queues can be functional in cases like ports 605 * are started without re configuring queues. 606 * 607 * Usual re config sequence is like below: 608 * port_configure() { 609 * if(reconfigure) { 610 * queue_release() 611 * queue_setup() 612 * } 613 * queue_configure() { 614 * queue_release() 615 * queue_setup() 616 * } 617 * } 618 * port_start() 619 * 620 * In some application's control path, queue_configure() would 621 * NOT be invoked for TXQs/RXQs in port_configure(). 622 * In such cases, queues can be functional after start as the 623 * queues are already setup in port_configure(). 624 */ 625 for (i = 0; i < nb_txq; i++) { 626 if (!tx_qconf[i].valid) 627 continue; 628 rc = dev_ops->tx_queue_setup(eth_dev, i, tx_qconf[i].nb_desc, 0, 629 &tx_qconf[i].conf.tx); 630 if (rc) { 631 plt_err("Failed to setup tx queue rc=%d", rc); 632 txq = eth_dev->data->tx_queues; 633 for (i -= 1; i >= 0; i--) 634 dev_ops->tx_queue_release(txq[i]); 635 goto fail; 636 } 637 } 638 639 free(tx_qconf); 640 tx_qconf = NULL; 641 642 for (i = 0; i < nb_rxq; i++) { 643 if (!rx_qconf[i].valid) 644 continue; 645 rc = dev_ops->rx_queue_setup(eth_dev, i, rx_qconf[i].nb_desc, 0, 646 &rx_qconf[i].conf.rx, 647 rx_qconf[i].mp); 648 if (rc) { 649 plt_err("Failed to setup rx queue rc=%d", rc); 650 rxq = eth_dev->data->rx_queues; 651 for (i -= 1; i >= 0; i--) 652 dev_ops->rx_queue_release(rxq[i]); 653 goto tx_queue_release; 654 } 655 } 656 657 free(rx_qconf); 658 rx_qconf = NULL; 659 660 return 0; 661 662 tx_queue_release: 663 txq = eth_dev->data->tx_queues; 664 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) 665 dev_ops->tx_queue_release(txq[i]); 666 fail: 667 if (tx_qconf) 668 free(tx_qconf); 669 if (rx_qconf) 670 free(rx_qconf); 671 672 return rc; 673 } 674 675 static uint16_t 676 nix_eth_nop_burst(void *queue, struct rte_mbuf **mbufs, uint16_t pkts) 677 { 678 RTE_SET_USED(queue); 679 RTE_SET_USED(mbufs); 680 RTE_SET_USED(pkts); 681 682 return 0; 683 } 684 685 static void 686 nix_set_nop_rxtx_function(struct rte_eth_dev *eth_dev) 687 { 688 /* These dummy functions are required for supporting 689 * some applications which reconfigure queues without 690 * stopping tx burst and rx burst threads(eg kni app) 691 * When the queues context is saved, txq/rxqs are released 692 * which caused app crash since rx/tx burst is still 693 * on different lcores 694 */ 695 eth_dev->tx_pkt_burst = nix_eth_nop_burst; 696 eth_dev->rx_pkt_burst = nix_eth_nop_burst; 697 rte_mb(); 698 } 699 700 static int 701 nix_lso_tun_fmt_update(struct cnxk_eth_dev *dev) 702 { 703 uint8_t udp_tun[ROC_NIX_LSO_TUN_MAX]; 704 uint8_t tun[ROC_NIX_LSO_TUN_MAX]; 705 struct roc_nix *nix = &dev->nix; 706 int rc; 707 708 rc = roc_nix_lso_fmt_get(nix, udp_tun, tun); 709 if (rc) 710 return rc; 711 712 dev->lso_tun_fmt = ((uint64_t)tun[ROC_NIX_LSO_TUN_V4V4] | 713 (uint64_t)tun[ROC_NIX_LSO_TUN_V4V6] << 8 | 714 (uint64_t)tun[ROC_NIX_LSO_TUN_V6V4] << 16 | 715 (uint64_t)tun[ROC_NIX_LSO_TUN_V6V6] << 24); 716 717 dev->lso_tun_fmt |= ((uint64_t)udp_tun[ROC_NIX_LSO_TUN_V4V4] << 32 | 718 (uint64_t)udp_tun[ROC_NIX_LSO_TUN_V4V6] << 40 | 719 (uint64_t)udp_tun[ROC_NIX_LSO_TUN_V6V4] << 48 | 720 (uint64_t)udp_tun[ROC_NIX_LSO_TUN_V6V6] << 56); 721 return 0; 722 } 723 724 static int 725 nix_lso_fmt_setup(struct cnxk_eth_dev *dev) 726 { 727 struct roc_nix *nix = &dev->nix; 728 int rc; 729 730 /* Nothing much to do if offload is not enabled */ 731 if (!(dev->tx_offloads & 732 (DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 733 DEV_TX_OFFLOAD_GENEVE_TNL_TSO | DEV_TX_OFFLOAD_GRE_TNL_TSO))) 734 return 0; 735 736 /* Setup LSO formats in AF. Its a no-op if other ethdev has 737 * already set it up 738 */ 739 rc = roc_nix_lso_fmt_setup(nix); 740 if (rc) 741 return rc; 742 743 return nix_lso_tun_fmt_update(dev); 744 } 745 746 int 747 cnxk_nix_configure(struct rte_eth_dev *eth_dev) 748 { 749 struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); 750 struct rte_eth_dev_data *data = eth_dev->data; 751 struct rte_eth_conf *conf = &data->dev_conf; 752 struct rte_eth_rxmode *rxmode = &conf->rxmode; 753 struct rte_eth_txmode *txmode = &conf->txmode; 754 char ea_fmt[RTE_ETHER_ADDR_FMT_SIZE]; 755 struct roc_nix_fc_cfg fc_cfg = {0}; 756 struct roc_nix *nix = &dev->nix; 757 struct rte_ether_addr *ea; 758 uint8_t nb_rxq, nb_txq; 759 uint64_t rx_cfg; 760 void *qs; 761 int rc; 762 763 rc = -EINVAL; 764 765 /* Sanity checks */ 766 if (rte_eal_has_hugepages() == 0) { 767 plt_err("Huge page is not configured"); 768 goto fail_configure; 769 } 770 771 if (conf->dcb_capability_en == 1) { 772 plt_err("dcb enable is not supported"); 773 goto fail_configure; 774 } 775 776 if (conf->fdir_conf.mode != RTE_FDIR_MODE_NONE) { 777 plt_err("Flow director is not supported"); 778 goto fail_configure; 779 } 780 781 if (rxmode->mq_mode != ETH_MQ_RX_NONE && 782 rxmode->mq_mode != ETH_MQ_RX_RSS) { 783 plt_err("Unsupported mq rx mode %d", rxmode->mq_mode); 784 goto fail_configure; 785 } 786 787 if (txmode->mq_mode != ETH_MQ_TX_NONE) { 788 plt_err("Unsupported mq tx mode %d", txmode->mq_mode); 789 goto fail_configure; 790 } 791 792 /* Free the resources allocated from the previous configure */ 793 if (dev->configured == 1) { 794 /* Unregister queue irq's */ 795 roc_nix_unregister_queue_irqs(nix); 796 797 /* Unregister CQ irqs if present */ 798 if (eth_dev->data->dev_conf.intr_conf.rxq) 799 roc_nix_unregister_cq_irqs(nix); 800 801 /* Set no-op functions */ 802 nix_set_nop_rxtx_function(eth_dev); 803 /* Store queue config for later */ 804 rc = nix_store_queue_cfg_and_then_release(eth_dev); 805 if (rc) 806 goto fail_configure; 807 roc_nix_tm_fini(nix); 808 roc_nix_lf_free(nix); 809 } 810 811 dev->rx_offloads = rxmode->offloads; 812 dev->tx_offloads = txmode->offloads; 813 814 /* Prepare rx cfg */ 815 rx_cfg = ROC_NIX_LF_RX_CFG_DIS_APAD; 816 if (dev->rx_offloads & 817 (DEV_RX_OFFLOAD_TCP_CKSUM | DEV_RX_OFFLOAD_UDP_CKSUM)) { 818 rx_cfg |= ROC_NIX_LF_RX_CFG_CSUM_OL4; 819 rx_cfg |= ROC_NIX_LF_RX_CFG_CSUM_IL4; 820 } 821 rx_cfg |= (ROC_NIX_LF_RX_CFG_DROP_RE | ROC_NIX_LF_RX_CFG_L2_LEN_ERR | 822 ROC_NIX_LF_RX_CFG_LEN_IL4 | ROC_NIX_LF_RX_CFG_LEN_IL3 | 823 ROC_NIX_LF_RX_CFG_LEN_OL4 | ROC_NIX_LF_RX_CFG_LEN_OL3); 824 825 nb_rxq = RTE_MAX(data->nb_rx_queues, 1); 826 nb_txq = RTE_MAX(data->nb_tx_queues, 1); 827 828 /* Alloc a nix lf */ 829 rc = roc_nix_lf_alloc(nix, nb_rxq, nb_txq, rx_cfg); 830 if (rc) { 831 plt_err("Failed to init nix_lf rc=%d", rc); 832 goto fail_configure; 833 } 834 835 dev->npc.channel = roc_nix_get_base_chan(nix); 836 837 nb_rxq = data->nb_rx_queues; 838 nb_txq = data->nb_tx_queues; 839 rc = -ENOMEM; 840 if (nb_rxq) { 841 /* Allocate memory for roc rq's and cq's */ 842 qs = plt_zmalloc(sizeof(struct roc_nix_rq) * nb_rxq, 0); 843 if (!qs) { 844 plt_err("Failed to alloc rqs"); 845 goto free_nix_lf; 846 } 847 dev->rqs = qs; 848 849 qs = plt_zmalloc(sizeof(struct roc_nix_cq) * nb_rxq, 0); 850 if (!qs) { 851 plt_err("Failed to alloc cqs"); 852 goto free_nix_lf; 853 } 854 dev->cqs = qs; 855 } 856 857 if (nb_txq) { 858 /* Allocate memory for roc sq's */ 859 qs = plt_zmalloc(sizeof(struct roc_nix_sq) * nb_txq, 0); 860 if (!qs) { 861 plt_err("Failed to alloc sqs"); 862 goto free_nix_lf; 863 } 864 dev->sqs = qs; 865 } 866 867 /* Re-enable NIX LF error interrupts */ 868 roc_nix_err_intr_ena_dis(nix, true); 869 roc_nix_ras_intr_ena_dis(nix, true); 870 871 if (nix->rx_ptp_ena && 872 dev->npc.switch_header_type == ROC_PRIV_FLAGS_HIGIG) { 873 plt_err("Both PTP and switch header enabled"); 874 goto free_nix_lf; 875 } 876 877 rc = roc_nix_switch_hdr_set(nix, dev->npc.switch_header_type); 878 if (rc) { 879 plt_err("Failed to enable switch type nix_lf rc=%d", rc); 880 goto free_nix_lf; 881 } 882 883 /* Setup LSO if needed */ 884 rc = nix_lso_fmt_setup(dev); 885 if (rc) { 886 plt_err("Failed to setup nix lso format fields, rc=%d", rc); 887 goto free_nix_lf; 888 } 889 890 /* Configure RSS */ 891 rc = nix_rss_default_setup(dev); 892 if (rc) { 893 plt_err("Failed to configure rss rc=%d", rc); 894 goto free_nix_lf; 895 } 896 897 /* Init the default TM scheduler hierarchy */ 898 rc = roc_nix_tm_init(nix); 899 if (rc) { 900 plt_err("Failed to init traffic manager, rc=%d", rc); 901 goto free_nix_lf; 902 } 903 904 rc = roc_nix_tm_hierarchy_enable(nix, ROC_NIX_TM_DEFAULT, false); 905 if (rc) { 906 plt_err("Failed to enable default tm hierarchy, rc=%d", rc); 907 goto tm_fini; 908 } 909 910 /* Register queue IRQs */ 911 rc = roc_nix_register_queue_irqs(nix); 912 if (rc) { 913 plt_err("Failed to register queue interrupts rc=%d", rc); 914 goto tm_fini; 915 } 916 917 /* Register cq IRQs */ 918 if (eth_dev->data->dev_conf.intr_conf.rxq) { 919 if (eth_dev->data->nb_rx_queues > dev->nix.cints) { 920 plt_err("Rx interrupt cannot be enabled, rxq > %d", 921 dev->nix.cints); 922 goto q_irq_fini; 923 } 924 /* Rx interrupt feature cannot work with vector mode because, 925 * vector mode does not process packets unless min 4 pkts are 926 * received, while cq interrupts are generated even for 1 pkt 927 * in the CQ. 928 */ 929 dev->scalar_ena = true; 930 931 rc = roc_nix_register_cq_irqs(nix); 932 if (rc) { 933 plt_err("Failed to register CQ interrupts rc=%d", rc); 934 goto q_irq_fini; 935 } 936 } 937 938 /* Configure loop back mode */ 939 rc = roc_nix_mac_loopback_enable(nix, 940 eth_dev->data->dev_conf.lpbk_mode); 941 if (rc) { 942 plt_err("Failed to configure cgx loop back mode rc=%d", rc); 943 goto cq_fini; 944 } 945 946 /* Init flow control configuration */ 947 fc_cfg.cq_cfg_valid = false; 948 fc_cfg.rxchan_cfg.enable = true; 949 rc = roc_nix_fc_config_set(nix, &fc_cfg); 950 if (rc) { 951 plt_err("Failed to initialize flow control rc=%d", rc); 952 goto cq_fini; 953 } 954 955 /* Update flow control configuration to PMD */ 956 rc = nix_init_flow_ctrl_config(eth_dev); 957 if (rc) { 958 plt_err("Failed to initialize flow control rc=%d", rc); 959 goto cq_fini; 960 } 961 /* 962 * Restore queue config when reconfigure followed by 963 * reconfigure and no queue configure invoked from application case. 964 */ 965 if (dev->configured == 1) { 966 rc = nix_restore_queue_cfg(eth_dev); 967 if (rc) 968 goto cq_fini; 969 } 970 971 /* Update the mac address */ 972 ea = eth_dev->data->mac_addrs; 973 memcpy(ea, dev->mac_addr, RTE_ETHER_ADDR_LEN); 974 if (rte_is_zero_ether_addr(ea)) 975 rte_eth_random_addr((uint8_t *)ea); 976 977 rte_ether_format_addr(ea_fmt, RTE_ETHER_ADDR_FMT_SIZE, ea); 978 979 plt_nix_dbg("Configured port%d mac=%s nb_rxq=%d nb_txq=%d" 980 " rx_offloads=0x%" PRIx64 " tx_offloads=0x%" PRIx64 "", 981 eth_dev->data->port_id, ea_fmt, nb_rxq, nb_txq, 982 dev->rx_offloads, dev->tx_offloads); 983 984 /* All good */ 985 dev->configured = 1; 986 dev->nb_rxq = data->nb_rx_queues; 987 dev->nb_txq = data->nb_tx_queues; 988 return 0; 989 990 cq_fini: 991 roc_nix_unregister_cq_irqs(nix); 992 q_irq_fini: 993 roc_nix_unregister_queue_irqs(nix); 994 tm_fini: 995 roc_nix_tm_fini(nix); 996 free_nix_lf: 997 nix_free_queue_mem(dev); 998 rc |= roc_nix_lf_free(nix); 999 fail_configure: 1000 dev->configured = 0; 1001 return rc; 1002 } 1003 1004 int 1005 cnxk_nix_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t qid) 1006 { 1007 struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); 1008 struct rte_eth_dev_data *data = eth_dev->data; 1009 struct roc_nix_sq *sq = &dev->sqs[qid]; 1010 int rc = -EINVAL; 1011 1012 if (data->tx_queue_state[qid] == RTE_ETH_QUEUE_STATE_STARTED) 1013 return 0; 1014 1015 rc = roc_nix_tm_sq_aura_fc(sq, true); 1016 if (rc) { 1017 plt_err("Failed to enable sq aura fc, txq=%u, rc=%d", qid, rc); 1018 goto done; 1019 } 1020 1021 data->tx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STARTED; 1022 done: 1023 return rc; 1024 } 1025 1026 int 1027 cnxk_nix_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t qid) 1028 { 1029 struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); 1030 struct rte_eth_dev_data *data = eth_dev->data; 1031 struct roc_nix_sq *sq = &dev->sqs[qid]; 1032 int rc; 1033 1034 if (data->tx_queue_state[qid] == RTE_ETH_QUEUE_STATE_STOPPED) 1035 return 0; 1036 1037 rc = roc_nix_tm_sq_aura_fc(sq, false); 1038 if (rc) { 1039 plt_err("Failed to disable sqb aura fc, txq=%u, rc=%d", qid, 1040 rc); 1041 goto done; 1042 } 1043 1044 data->tx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STOPPED; 1045 done: 1046 return rc; 1047 } 1048 1049 static int 1050 cnxk_nix_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t qid) 1051 { 1052 struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); 1053 struct rte_eth_dev_data *data = eth_dev->data; 1054 struct roc_nix_rq *rq = &dev->rqs[qid]; 1055 int rc; 1056 1057 if (data->rx_queue_state[qid] == RTE_ETH_QUEUE_STATE_STARTED) 1058 return 0; 1059 1060 rc = roc_nix_rq_ena_dis(rq, true); 1061 if (rc) { 1062 plt_err("Failed to enable rxq=%u, rc=%d", qid, rc); 1063 goto done; 1064 } 1065 1066 data->rx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STARTED; 1067 done: 1068 return rc; 1069 } 1070 1071 static int 1072 cnxk_nix_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t qid) 1073 { 1074 struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); 1075 struct rte_eth_dev_data *data = eth_dev->data; 1076 struct roc_nix_rq *rq = &dev->rqs[qid]; 1077 int rc; 1078 1079 if (data->rx_queue_state[qid] == RTE_ETH_QUEUE_STATE_STOPPED) 1080 return 0; 1081 1082 rc = roc_nix_rq_ena_dis(rq, false); 1083 if (rc) { 1084 plt_err("Failed to disable rxq=%u, rc=%d", qid, rc); 1085 goto done; 1086 } 1087 1088 data->rx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STOPPED; 1089 done: 1090 return rc; 1091 } 1092 1093 static int 1094 cnxk_nix_dev_stop(struct rte_eth_dev *eth_dev) 1095 { 1096 struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); 1097 const struct eth_dev_ops *dev_ops = eth_dev->dev_ops; 1098 struct rte_mbuf *rx_pkts[32]; 1099 int count, i, j, rc; 1100 void *rxq; 1101 1102 /* Disable switch hdr pkind */ 1103 roc_nix_switch_hdr_set(&dev->nix, 0); 1104 1105 /* Stop link change events */ 1106 if (!roc_nix_is_vf_or_sdp(&dev->nix)) 1107 roc_nix_mac_link_event_start_stop(&dev->nix, false); 1108 1109 /* Disable Rx via NPC */ 1110 roc_nix_npc_rx_ena_dis(&dev->nix, false); 1111 1112 /* Stop rx queues and free up pkts pending */ 1113 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 1114 rc = dev_ops->rx_queue_stop(eth_dev, i); 1115 if (rc) 1116 continue; 1117 1118 rxq = eth_dev->data->rx_queues[i]; 1119 count = dev->rx_pkt_burst_no_offload(rxq, rx_pkts, 32); 1120 while (count) { 1121 for (j = 0; j < count; j++) 1122 rte_pktmbuf_free(rx_pkts[j]); 1123 count = dev->rx_pkt_burst_no_offload(rxq, rx_pkts, 32); 1124 } 1125 } 1126 1127 /* Stop tx queues */ 1128 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) 1129 dev_ops->tx_queue_stop(eth_dev, i); 1130 1131 return 0; 1132 } 1133 1134 int 1135 cnxk_nix_dev_start(struct rte_eth_dev *eth_dev) 1136 { 1137 struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); 1138 int rc, i; 1139 1140 if (eth_dev->data->nb_rx_queues != 0 && !dev->ptp_en) { 1141 rc = nix_recalc_mtu(eth_dev); 1142 if (rc) 1143 return rc; 1144 } 1145 1146 /* Start rx queues */ 1147 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 1148 rc = cnxk_nix_rx_queue_start(eth_dev, i); 1149 if (rc) 1150 return rc; 1151 } 1152 1153 /* Start tx queues */ 1154 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { 1155 rc = cnxk_nix_tx_queue_start(eth_dev, i); 1156 if (rc) 1157 return rc; 1158 } 1159 1160 /* Update Flow control configuration */ 1161 rc = nix_update_flow_ctrl_config(eth_dev); 1162 if (rc) { 1163 plt_err("Failed to enable flow control. error code(%d)", rc); 1164 return rc; 1165 } 1166 1167 /* Enable Rx in NPC */ 1168 rc = roc_nix_npc_rx_ena_dis(&dev->nix, true); 1169 if (rc) { 1170 plt_err("Failed to enable NPC rx %d", rc); 1171 return rc; 1172 } 1173 1174 cnxk_nix_toggle_flag_link_cfg(dev, true); 1175 1176 /* Start link change events */ 1177 if (!roc_nix_is_vf_or_sdp(&dev->nix)) { 1178 rc = roc_nix_mac_link_event_start_stop(&dev->nix, true); 1179 if (rc) { 1180 plt_err("Failed to start cgx link event %d", rc); 1181 goto rx_disable; 1182 } 1183 } 1184 1185 /* Enable PTP if it is requested by the user or already 1186 * enabled on PF owning this VF 1187 */ 1188 memset(&dev->tstamp, 0, sizeof(struct cnxk_timesync_info)); 1189 if ((dev->rx_offloads & DEV_RX_OFFLOAD_TIMESTAMP) || dev->ptp_en) 1190 cnxk_eth_dev_ops.timesync_enable(eth_dev); 1191 else 1192 cnxk_eth_dev_ops.timesync_disable(eth_dev); 1193 1194 if (dev->rx_offloads & DEV_RX_OFFLOAD_TIMESTAMP) { 1195 rc = rte_mbuf_dyn_rx_timestamp_register 1196 (&dev->tstamp.tstamp_dynfield_offset, 1197 &dev->tstamp.rx_tstamp_dynflag); 1198 if (rc != 0) { 1199 plt_err("Failed to register Rx timestamp field/flag"); 1200 goto rx_disable; 1201 } 1202 } 1203 1204 cnxk_nix_toggle_flag_link_cfg(dev, false); 1205 1206 return 0; 1207 1208 rx_disable: 1209 roc_nix_npc_rx_ena_dis(&dev->nix, false); 1210 cnxk_nix_toggle_flag_link_cfg(dev, false); 1211 return rc; 1212 } 1213 1214 static int cnxk_nix_dev_reset(struct rte_eth_dev *eth_dev); 1215 static int cnxk_nix_dev_close(struct rte_eth_dev *eth_dev); 1216 1217 /* CNXK platform independent eth dev ops */ 1218 struct eth_dev_ops cnxk_eth_dev_ops = { 1219 .mtu_set = cnxk_nix_mtu_set, 1220 .mac_addr_add = cnxk_nix_mac_addr_add, 1221 .mac_addr_remove = cnxk_nix_mac_addr_del, 1222 .mac_addr_set = cnxk_nix_mac_addr_set, 1223 .dev_infos_get = cnxk_nix_info_get, 1224 .link_update = cnxk_nix_link_update, 1225 .tx_queue_release = cnxk_nix_tx_queue_release, 1226 .rx_queue_release = cnxk_nix_rx_queue_release, 1227 .dev_stop = cnxk_nix_dev_stop, 1228 .dev_close = cnxk_nix_dev_close, 1229 .dev_reset = cnxk_nix_dev_reset, 1230 .tx_queue_start = cnxk_nix_tx_queue_start, 1231 .rx_queue_start = cnxk_nix_rx_queue_start, 1232 .rx_queue_stop = cnxk_nix_rx_queue_stop, 1233 .dev_supported_ptypes_get = cnxk_nix_supported_ptypes_get, 1234 .promiscuous_enable = cnxk_nix_promisc_enable, 1235 .promiscuous_disable = cnxk_nix_promisc_disable, 1236 .allmulticast_enable = cnxk_nix_allmulticast_enable, 1237 .allmulticast_disable = cnxk_nix_allmulticast_disable, 1238 .rx_burst_mode_get = cnxk_nix_rx_burst_mode_get, 1239 .tx_burst_mode_get = cnxk_nix_tx_burst_mode_get, 1240 .flow_ctrl_get = cnxk_nix_flow_ctrl_get, 1241 .flow_ctrl_set = cnxk_nix_flow_ctrl_set, 1242 .dev_set_link_up = cnxk_nix_set_link_up, 1243 .dev_set_link_down = cnxk_nix_set_link_down, 1244 .get_module_info = cnxk_nix_get_module_info, 1245 .get_module_eeprom = cnxk_nix_get_module_eeprom, 1246 .rx_queue_intr_enable = cnxk_nix_rx_queue_intr_enable, 1247 .rx_queue_intr_disable = cnxk_nix_rx_queue_intr_disable, 1248 .pool_ops_supported = cnxk_nix_pool_ops_supported, 1249 .queue_stats_mapping_set = cnxk_nix_queue_stats_mapping, 1250 .stats_get = cnxk_nix_stats_get, 1251 .stats_reset = cnxk_nix_stats_reset, 1252 .xstats_get = cnxk_nix_xstats_get, 1253 .xstats_get_names = cnxk_nix_xstats_get_names, 1254 .xstats_reset = cnxk_nix_xstats_reset, 1255 .xstats_get_by_id = cnxk_nix_xstats_get_by_id, 1256 .xstats_get_names_by_id = cnxk_nix_xstats_get_names_by_id, 1257 .fw_version_get = cnxk_nix_fw_version_get, 1258 .rxq_info_get = cnxk_nix_rxq_info_get, 1259 .txq_info_get = cnxk_nix_txq_info_get, 1260 .tx_done_cleanup = cnxk_nix_tx_done_cleanup, 1261 .flow_ops_get = cnxk_nix_flow_ops_get, 1262 .get_reg = cnxk_nix_dev_get_reg, 1263 .timesync_read_rx_timestamp = cnxk_nix_timesync_read_rx_timestamp, 1264 .timesync_read_tx_timestamp = cnxk_nix_timesync_read_tx_timestamp, 1265 .timesync_read_time = cnxk_nix_timesync_read_time, 1266 .timesync_write_time = cnxk_nix_timesync_write_time, 1267 .timesync_adjust_time = cnxk_nix_timesync_adjust_time, 1268 .read_clock = cnxk_nix_read_clock, 1269 .reta_update = cnxk_nix_reta_update, 1270 .reta_query = cnxk_nix_reta_query, 1271 .rss_hash_update = cnxk_nix_rss_hash_update, 1272 .rss_hash_conf_get = cnxk_nix_rss_hash_conf_get, 1273 .set_mc_addr_list = cnxk_nix_mc_addr_list_configure, 1274 }; 1275 1276 static int 1277 cnxk_eth_dev_init(struct rte_eth_dev *eth_dev) 1278 { 1279 struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); 1280 struct roc_nix *nix = &dev->nix; 1281 struct rte_pci_device *pci_dev; 1282 int rc, max_entries; 1283 1284 eth_dev->dev_ops = &cnxk_eth_dev_ops; 1285 1286 /* For secondary processes, the primary has done all the work */ 1287 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1288 return 0; 1289 1290 pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1291 rte_eth_copy_pci_info(eth_dev, pci_dev); 1292 1293 /* Parse devargs string */ 1294 rc = cnxk_ethdev_parse_devargs(eth_dev->device->devargs, dev); 1295 if (rc) { 1296 plt_err("Failed to parse devargs rc=%d", rc); 1297 goto error; 1298 } 1299 1300 /* Initialize base roc nix */ 1301 nix->pci_dev = pci_dev; 1302 nix->hw_vlan_ins = true; 1303 rc = roc_nix_dev_init(nix); 1304 if (rc) { 1305 plt_err("Failed to initialize roc nix rc=%d", rc); 1306 goto error; 1307 } 1308 1309 /* Register up msg callbacks */ 1310 roc_nix_mac_link_cb_register(nix, cnxk_eth_dev_link_status_cb); 1311 1312 dev->eth_dev = eth_dev; 1313 dev->configured = 0; 1314 dev->ptype_disable = 0; 1315 1316 /* For vfs, returned max_entries will be 0. but to keep default mac 1317 * address, one entry must be allocated. so setting up to 1. 1318 */ 1319 if (roc_nix_is_vf_or_sdp(nix)) 1320 max_entries = 1; 1321 else 1322 max_entries = roc_nix_mac_max_entries_get(nix); 1323 1324 if (max_entries <= 0) { 1325 plt_err("Failed to get max entries for mac addr"); 1326 rc = -ENOTSUP; 1327 goto dev_fini; 1328 } 1329 1330 eth_dev->data->mac_addrs = 1331 rte_zmalloc("mac_addr", max_entries * RTE_ETHER_ADDR_LEN, 0); 1332 if (eth_dev->data->mac_addrs == NULL) { 1333 plt_err("Failed to allocate memory for mac addr"); 1334 rc = -ENOMEM; 1335 goto dev_fini; 1336 } 1337 1338 dev->max_mac_entries = max_entries; 1339 dev->dmac_filter_count = 1; 1340 1341 /* Get mac address */ 1342 rc = roc_nix_npc_mac_addr_get(nix, dev->mac_addr); 1343 if (rc) { 1344 plt_err("Failed to get mac addr, rc=%d", rc); 1345 goto free_mac_addrs; 1346 } 1347 1348 /* Update the mac address */ 1349 memcpy(eth_dev->data->mac_addrs, dev->mac_addr, RTE_ETHER_ADDR_LEN); 1350 1351 if (!roc_nix_is_vf_or_sdp(nix)) { 1352 /* Sync same MAC address to CGX/RPM table */ 1353 rc = roc_nix_mac_addr_set(nix, dev->mac_addr); 1354 if (rc) { 1355 plt_err("Failed to set mac addr, rc=%d", rc); 1356 goto free_mac_addrs; 1357 } 1358 } 1359 1360 /* Union of all capabilities supported by CNXK. 1361 * Platform specific capabilities will be 1362 * updated later. 1363 */ 1364 dev->rx_offload_capa = nix_get_rx_offload_capa(dev); 1365 dev->tx_offload_capa = nix_get_tx_offload_capa(dev); 1366 dev->speed_capa = nix_get_speed_capa(dev); 1367 1368 /* Initialize roc npc */ 1369 dev->npc.roc_nix = nix; 1370 rc = roc_npc_init(&dev->npc); 1371 if (rc) 1372 goto free_mac_addrs; 1373 1374 plt_nix_dbg("Port=%d pf=%d vf=%d ver=%s hwcap=0x%" PRIx64 1375 " rxoffload_capa=0x%" PRIx64 " txoffload_capa=0x%" PRIx64, 1376 eth_dev->data->port_id, roc_nix_get_pf(nix), 1377 roc_nix_get_vf(nix), CNXK_ETH_DEV_PMD_VERSION, dev->hwcap, 1378 dev->rx_offload_capa, dev->tx_offload_capa); 1379 return 0; 1380 1381 free_mac_addrs: 1382 rte_free(eth_dev->data->mac_addrs); 1383 dev_fini: 1384 roc_nix_dev_fini(nix); 1385 error: 1386 plt_err("Failed to init nix eth_dev rc=%d", rc); 1387 return rc; 1388 } 1389 1390 static int 1391 cnxk_eth_dev_uninit(struct rte_eth_dev *eth_dev, bool reset) 1392 { 1393 struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev); 1394 const struct eth_dev_ops *dev_ops = eth_dev->dev_ops; 1395 struct roc_nix *nix = &dev->nix; 1396 int rc, i; 1397 1398 /* Nothing to be done for secondary processes */ 1399 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1400 return 0; 1401 1402 /* Clear the flag since we are closing down */ 1403 dev->configured = 0; 1404 1405 roc_nix_npc_rx_ena_dis(nix, false); 1406 1407 /* Disable and free rte_flow entries */ 1408 roc_npc_fini(&dev->npc); 1409 1410 /* Disable link status events */ 1411 roc_nix_mac_link_event_start_stop(nix, false); 1412 1413 /* Free up SQs */ 1414 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { 1415 dev_ops->tx_queue_release(eth_dev->data->tx_queues[i]); 1416 eth_dev->data->tx_queues[i] = NULL; 1417 } 1418 eth_dev->data->nb_tx_queues = 0; 1419 1420 /* Free up RQ's and CQ's */ 1421 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 1422 dev_ops->rx_queue_release(eth_dev->data->rx_queues[i]); 1423 eth_dev->data->rx_queues[i] = NULL; 1424 } 1425 eth_dev->data->nb_rx_queues = 0; 1426 1427 /* Free tm resources */ 1428 roc_nix_tm_fini(nix); 1429 1430 /* Unregister queue irqs */ 1431 roc_nix_unregister_queue_irqs(nix); 1432 1433 /* Unregister cq irqs */ 1434 if (eth_dev->data->dev_conf.intr_conf.rxq) 1435 roc_nix_unregister_cq_irqs(nix); 1436 1437 /* Free ROC RQ's, SQ's and CQ's memory */ 1438 nix_free_queue_mem(dev); 1439 1440 /* Free nix lf resources */ 1441 rc = roc_nix_lf_free(nix); 1442 if (rc) 1443 plt_err("Failed to free nix lf, rc=%d", rc); 1444 1445 rte_free(eth_dev->data->mac_addrs); 1446 eth_dev->data->mac_addrs = NULL; 1447 1448 rc = roc_nix_dev_fini(nix); 1449 /* Can be freed later by PMD if NPA LF is in use */ 1450 if (rc == -EAGAIN) { 1451 if (!reset) 1452 eth_dev->data->dev_private = NULL; 1453 return 0; 1454 } else if (rc) { 1455 plt_err("Failed in nix dev fini, rc=%d", rc); 1456 } 1457 1458 return rc; 1459 } 1460 1461 static int 1462 cnxk_nix_dev_close(struct rte_eth_dev *eth_dev) 1463 { 1464 cnxk_eth_dev_uninit(eth_dev, false); 1465 return 0; 1466 } 1467 1468 static int 1469 cnxk_nix_dev_reset(struct rte_eth_dev *eth_dev) 1470 { 1471 int rc; 1472 1473 rc = cnxk_eth_dev_uninit(eth_dev, true); 1474 if (rc) 1475 return rc; 1476 1477 return cnxk_eth_dev_init(eth_dev); 1478 } 1479 1480 int 1481 cnxk_nix_remove(struct rte_pci_device *pci_dev) 1482 { 1483 struct rte_eth_dev *eth_dev; 1484 struct roc_nix *nix; 1485 int rc = -EINVAL; 1486 1487 eth_dev = rte_eth_dev_allocated(pci_dev->device.name); 1488 if (eth_dev) { 1489 /* Cleanup eth dev */ 1490 rc = cnxk_eth_dev_uninit(eth_dev, false); 1491 if (rc) 1492 return rc; 1493 1494 rte_eth_dev_release_port(eth_dev); 1495 } 1496 1497 /* Nothing to be done for secondary processes */ 1498 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1499 return 0; 1500 1501 /* Check if this device is hosting common resource */ 1502 nix = roc_idev_npa_nix_get(); 1503 if (nix->pci_dev != pci_dev) 1504 return 0; 1505 1506 /* Try nix fini now */ 1507 rc = roc_nix_dev_fini(nix); 1508 if (rc == -EAGAIN) { 1509 plt_info("%s: common resource in use by other devices", 1510 pci_dev->name); 1511 goto exit; 1512 } else if (rc) { 1513 plt_err("Failed in nix dev fini, rc=%d", rc); 1514 goto exit; 1515 } 1516 1517 /* Free device pointer as rte_ethdev does not have it anymore */ 1518 rte_free(nix); 1519 exit: 1520 return rc; 1521 } 1522 1523 int 1524 cnxk_nix_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) 1525 { 1526 int rc; 1527 1528 RTE_SET_USED(pci_drv); 1529 1530 rc = rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct cnxk_eth_dev), 1531 cnxk_eth_dev_init); 1532 1533 /* On error on secondary, recheck if port exists in primary or 1534 * in mid of detach state. 1535 */ 1536 if (rte_eal_process_type() != RTE_PROC_PRIMARY && rc) 1537 if (!rte_eth_dev_allocated(pci_dev->device.name)) 1538 return 0; 1539 return rc; 1540 } 1541