1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) 2 * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved. 3 */ 4 5 #include <rte_malloc.h> 6 #include <ethdev_driver.h> 7 8 #include "ionic.h" 9 #include "ionic_logs.h" 10 #include "ionic_lif.h" 11 #include "ionic_ethdev.h" 12 #include "ionic_rx_filter.h" 13 #include "ionic_rxtx.h" 14 15 static int ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr); 16 static int ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr); 17 18 int 19 ionic_qcq_enable(struct ionic_qcq *qcq) 20 { 21 struct ionic_queue *q = &qcq->q; 22 struct ionic_lif *lif = q->lif; 23 struct ionic_admin_ctx ctx = { 24 .pending_work = true, 25 .cmd.q_control = { 26 .opcode = IONIC_CMD_Q_CONTROL, 27 .type = q->type, 28 .index = q->index, 29 .oper = IONIC_Q_ENABLE, 30 }, 31 }; 32 33 return ionic_adminq_post_wait(lif, &ctx); 34 } 35 36 int 37 ionic_qcq_disable(struct ionic_qcq *qcq) 38 { 39 struct ionic_queue *q = &qcq->q; 40 struct ionic_lif *lif = q->lif; 41 struct ionic_admin_ctx ctx = { 42 .pending_work = true, 43 .cmd.q_control = { 44 .opcode = IONIC_CMD_Q_CONTROL, 45 .type = q->type, 46 .index = q->index, 47 .oper = IONIC_Q_DISABLE, 48 }, 49 }; 50 51 return ionic_adminq_post_wait(lif, &ctx); 52 } 53 54 void 55 ionic_lif_stop(struct ionic_lif *lif) 56 { 57 uint32_t i; 58 59 IONIC_PRINT_CALL(); 60 61 lif->state &= ~IONIC_LIF_F_UP; 62 63 for (i = 0; i < lif->nrxqcqs; i++) { 64 struct ionic_qcq *rxq = lif->rxqcqs[i]; 65 if (rxq->flags & IONIC_QCQ_F_INITED) 66 (void)ionic_dev_rx_queue_stop(lif->eth_dev, i); 67 } 68 69 for (i = 0; i < lif->ntxqcqs; i++) { 70 struct ionic_qcq *txq = lif->txqcqs[i]; 71 if (txq->flags & IONIC_QCQ_F_INITED) 72 (void)ionic_dev_tx_queue_stop(lif->eth_dev, i); 73 } 74 } 75 76 void 77 ionic_lif_reset(struct ionic_lif *lif) 78 { 79 struct ionic_dev *idev = &lif->adapter->idev; 80 int err; 81 82 IONIC_PRINT_CALL(); 83 84 ionic_dev_cmd_lif_reset(idev); 85 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); 86 if (err) 87 IONIC_PRINT(WARNING, "Failed to reset %s", lif->name); 88 } 89 90 static void 91 ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats) 92 { 93 struct ionic_lif_stats *ls = &lif->info->stats; 94 uint32_t i; 95 uint32_t num_rx_q_counters = RTE_MIN(lif->nrxqcqs, (uint32_t) 96 RTE_ETHDEV_QUEUE_STAT_CNTRS); 97 uint32_t num_tx_q_counters = RTE_MIN(lif->ntxqcqs, (uint32_t) 98 RTE_ETHDEV_QUEUE_STAT_CNTRS); 99 100 memset(stats, 0, sizeof(*stats)); 101 102 if (ls == NULL) { 103 IONIC_PRINT(DEBUG, "Stats on port %u not yet initialized", 104 lif->port_id); 105 return; 106 } 107 108 /* RX */ 109 110 stats->ipackets = ls->rx_ucast_packets + 111 ls->rx_mcast_packets + 112 ls->rx_bcast_packets; 113 114 stats->ibytes = ls->rx_ucast_bytes + 115 ls->rx_mcast_bytes + 116 ls->rx_bcast_bytes; 117 118 for (i = 0; i < lif->nrxqcqs; i++) { 119 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx; 120 stats->imissed += 121 rx_stats->no_cb_arg + 122 rx_stats->bad_cq_status + 123 rx_stats->no_room + 124 rx_stats->bad_len; 125 } 126 127 stats->imissed += 128 ls->rx_ucast_drop_packets + 129 ls->rx_mcast_drop_packets + 130 ls->rx_bcast_drop_packets; 131 132 stats->imissed += 133 ls->rx_queue_empty + 134 ls->rx_dma_error + 135 ls->rx_queue_disabled + 136 ls->rx_desc_fetch_error + 137 ls->rx_desc_data_error; 138 139 for (i = 0; i < num_rx_q_counters; i++) { 140 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx; 141 stats->q_ipackets[i] = rx_stats->packets; 142 stats->q_ibytes[i] = rx_stats->bytes; 143 stats->q_errors[i] = 144 rx_stats->no_cb_arg + 145 rx_stats->bad_cq_status + 146 rx_stats->no_room + 147 rx_stats->bad_len; 148 } 149 150 /* TX */ 151 152 stats->opackets = ls->tx_ucast_packets + 153 ls->tx_mcast_packets + 154 ls->tx_bcast_packets; 155 156 stats->obytes = ls->tx_ucast_bytes + 157 ls->tx_mcast_bytes + 158 ls->tx_bcast_bytes; 159 160 for (i = 0; i < lif->ntxqcqs; i++) { 161 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx; 162 stats->oerrors += tx_stats->drop; 163 } 164 165 stats->oerrors += 166 ls->tx_ucast_drop_packets + 167 ls->tx_mcast_drop_packets + 168 ls->tx_bcast_drop_packets; 169 170 stats->oerrors += 171 ls->tx_dma_error + 172 ls->tx_queue_disabled + 173 ls->tx_desc_fetch_error + 174 ls->tx_desc_data_error; 175 176 for (i = 0; i < num_tx_q_counters; i++) { 177 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx; 178 stats->q_opackets[i] = tx_stats->packets; 179 stats->q_obytes[i] = tx_stats->bytes; 180 } 181 } 182 183 void 184 ionic_lif_get_stats(const struct ionic_lif *lif, 185 struct rte_eth_stats *stats) 186 { 187 ionic_lif_get_abs_stats(lif, stats); 188 189 stats->ipackets -= lif->stats_base.ipackets; 190 stats->opackets -= lif->stats_base.opackets; 191 stats->ibytes -= lif->stats_base.ibytes; 192 stats->obytes -= lif->stats_base.obytes; 193 stats->imissed -= lif->stats_base.imissed; 194 stats->ierrors -= lif->stats_base.ierrors; 195 stats->oerrors -= lif->stats_base.oerrors; 196 stats->rx_nombuf -= lif->stats_base.rx_nombuf; 197 } 198 199 void 200 ionic_lif_reset_stats(struct ionic_lif *lif) 201 { 202 uint32_t i; 203 204 for (i = 0; i < lif->nrxqcqs; i++) { 205 memset(&lif->rxqcqs[i]->stats.rx, 0, 206 sizeof(struct ionic_rx_stats)); 207 memset(&lif->txqcqs[i]->stats.tx, 0, 208 sizeof(struct ionic_tx_stats)); 209 } 210 211 ionic_lif_get_abs_stats(lif, &lif->stats_base); 212 } 213 214 void 215 ionic_lif_get_hw_stats(struct ionic_lif *lif, struct ionic_lif_stats *stats) 216 { 217 uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t); 218 uint64_t *stats64 = (uint64_t *)stats; 219 uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats; 220 uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base; 221 222 for (i = 0; i < count; i++) 223 stats64[i] = lif_stats64[i] - lif_stats64_base[i]; 224 } 225 226 void 227 ionic_lif_reset_hw_stats(struct ionic_lif *lif) 228 { 229 uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t); 230 uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats; 231 uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base; 232 233 for (i = 0; i < count; i++) 234 lif_stats64_base[i] = lif_stats64[i]; 235 } 236 237 static int 238 ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr) 239 { 240 struct ionic_admin_ctx ctx = { 241 .pending_work = true, 242 .cmd.rx_filter_add = { 243 .opcode = IONIC_CMD_RX_FILTER_ADD, 244 .match = IONIC_RX_FILTER_MATCH_MAC, 245 }, 246 }; 247 int err; 248 249 memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, RTE_ETHER_ADDR_LEN); 250 251 err = ionic_adminq_post_wait(lif, &ctx); 252 if (err) 253 return err; 254 255 IONIC_PRINT(INFO, "rx_filter add (id %d)", 256 ctx.comp.rx_filter_add.filter_id); 257 258 return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx); 259 } 260 261 static int 262 ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr) 263 { 264 struct ionic_admin_ctx ctx = { 265 .pending_work = true, 266 .cmd.rx_filter_del = { 267 .opcode = IONIC_CMD_RX_FILTER_DEL, 268 }, 269 }; 270 struct ionic_rx_filter *f; 271 int err; 272 273 IONIC_PRINT_CALL(); 274 275 rte_spinlock_lock(&lif->rx_filters.lock); 276 277 f = ionic_rx_filter_by_addr(lif, addr); 278 if (!f) { 279 rte_spinlock_unlock(&lif->rx_filters.lock); 280 return -ENOENT; 281 } 282 283 ctx.cmd.rx_filter_del.filter_id = f->filter_id; 284 ionic_rx_filter_free(f); 285 286 rte_spinlock_unlock(&lif->rx_filters.lock); 287 288 err = ionic_adminq_post_wait(lif, &ctx); 289 if (err) 290 return err; 291 292 IONIC_PRINT(INFO, "rx_filter del (id %d)", 293 ctx.cmd.rx_filter_del.filter_id); 294 295 return 0; 296 } 297 298 int 299 ionic_dev_add_mac(struct rte_eth_dev *eth_dev, 300 struct rte_ether_addr *mac_addr, 301 uint32_t index __rte_unused, uint32_t pool __rte_unused) 302 { 303 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 304 305 IONIC_PRINT_CALL(); 306 307 return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr); 308 } 309 310 void 311 ionic_dev_remove_mac(struct rte_eth_dev *eth_dev, uint32_t index) 312 { 313 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 314 struct ionic_adapter *adapter = lif->adapter; 315 struct rte_ether_addr *mac_addr; 316 317 IONIC_PRINT_CALL(); 318 319 if (index >= adapter->max_mac_addrs) { 320 IONIC_PRINT(WARNING, 321 "Index %u is above MAC filter limit %u", 322 index, adapter->max_mac_addrs); 323 return; 324 } 325 326 mac_addr = ð_dev->data->mac_addrs[index]; 327 328 if (!rte_is_valid_assigned_ether_addr(mac_addr)) 329 return; 330 331 ionic_lif_addr_del(lif, (const uint8_t *)mac_addr); 332 } 333 334 int 335 ionic_dev_set_mac(struct rte_eth_dev *eth_dev, struct rte_ether_addr *mac_addr) 336 { 337 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 338 339 IONIC_PRINT_CALL(); 340 341 if (mac_addr == NULL) { 342 IONIC_PRINT(NOTICE, "New mac is null"); 343 return -1; 344 } 345 346 if (!rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) { 347 IONIC_PRINT(INFO, "Deleting mac addr %pM", 348 lif->mac_addr); 349 ionic_lif_addr_del(lif, lif->mac_addr); 350 memset(lif->mac_addr, 0, RTE_ETHER_ADDR_LEN); 351 } 352 353 IONIC_PRINT(INFO, "Updating mac addr"); 354 355 rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)lif->mac_addr); 356 357 return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr); 358 } 359 360 static int 361 ionic_vlan_rx_add_vid(struct ionic_lif *lif, uint16_t vid) 362 { 363 struct ionic_admin_ctx ctx = { 364 .pending_work = true, 365 .cmd.rx_filter_add = { 366 .opcode = IONIC_CMD_RX_FILTER_ADD, 367 .match = IONIC_RX_FILTER_MATCH_VLAN, 368 .vlan.vlan = vid, 369 }, 370 }; 371 int err; 372 373 err = ionic_adminq_post_wait(lif, &ctx); 374 if (err) 375 return err; 376 377 IONIC_PRINT(INFO, "rx_filter add VLAN %d (id %d)", vid, 378 ctx.comp.rx_filter_add.filter_id); 379 380 return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx); 381 } 382 383 static int 384 ionic_vlan_rx_kill_vid(struct ionic_lif *lif, uint16_t vid) 385 { 386 struct ionic_admin_ctx ctx = { 387 .pending_work = true, 388 .cmd.rx_filter_del = { 389 .opcode = IONIC_CMD_RX_FILTER_DEL, 390 }, 391 }; 392 struct ionic_rx_filter *f; 393 int err; 394 395 IONIC_PRINT_CALL(); 396 397 rte_spinlock_lock(&lif->rx_filters.lock); 398 399 f = ionic_rx_filter_by_vlan(lif, vid); 400 if (!f) { 401 rte_spinlock_unlock(&lif->rx_filters.lock); 402 return -ENOENT; 403 } 404 405 ctx.cmd.rx_filter_del.filter_id = f->filter_id; 406 ionic_rx_filter_free(f); 407 rte_spinlock_unlock(&lif->rx_filters.lock); 408 409 err = ionic_adminq_post_wait(lif, &ctx); 410 if (err) 411 return err; 412 413 IONIC_PRINT(INFO, "rx_filter del VLAN %d (id %d)", vid, 414 ctx.cmd.rx_filter_del.filter_id); 415 416 return 0; 417 } 418 419 int 420 ionic_dev_vlan_filter_set(struct rte_eth_dev *eth_dev, uint16_t vlan_id, 421 int on) 422 { 423 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 424 int err; 425 426 if (on) 427 err = ionic_vlan_rx_add_vid(lif, vlan_id); 428 else 429 err = ionic_vlan_rx_kill_vid(lif, vlan_id); 430 431 return err; 432 } 433 434 static void 435 ionic_lif_rx_mode(struct ionic_lif *lif, uint32_t rx_mode) 436 { 437 struct ionic_admin_ctx ctx = { 438 .pending_work = true, 439 .cmd.rx_mode_set = { 440 .opcode = IONIC_CMD_RX_MODE_SET, 441 .rx_mode = rx_mode, 442 }, 443 }; 444 int err; 445 446 if (rx_mode & IONIC_RX_MODE_F_UNICAST) 447 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_UNICAST"); 448 if (rx_mode & IONIC_RX_MODE_F_MULTICAST) 449 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_MULTICAST"); 450 if (rx_mode & IONIC_RX_MODE_F_BROADCAST) 451 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_BROADCAST"); 452 if (rx_mode & IONIC_RX_MODE_F_PROMISC) 453 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_PROMISC"); 454 if (rx_mode & IONIC_RX_MODE_F_ALLMULTI) 455 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_ALLMULTI"); 456 457 err = ionic_adminq_post_wait(lif, &ctx); 458 if (err) 459 IONIC_PRINT(ERR, "Failure setting RX mode"); 460 } 461 462 static void 463 ionic_set_rx_mode(struct ionic_lif *lif, uint32_t rx_mode) 464 { 465 if (lif->rx_mode != rx_mode) { 466 lif->rx_mode = rx_mode; 467 ionic_lif_rx_mode(lif, rx_mode); 468 } 469 } 470 471 int 472 ionic_dev_promiscuous_enable(struct rte_eth_dev *eth_dev) 473 { 474 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 475 uint32_t rx_mode = lif->rx_mode; 476 477 IONIC_PRINT_CALL(); 478 479 rx_mode |= IONIC_RX_MODE_F_PROMISC; 480 481 ionic_set_rx_mode(lif, rx_mode); 482 483 return 0; 484 } 485 486 int 487 ionic_dev_promiscuous_disable(struct rte_eth_dev *eth_dev) 488 { 489 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 490 uint32_t rx_mode = lif->rx_mode; 491 492 rx_mode &= ~IONIC_RX_MODE_F_PROMISC; 493 494 ionic_set_rx_mode(lif, rx_mode); 495 496 return 0; 497 } 498 499 int 500 ionic_dev_allmulticast_enable(struct rte_eth_dev *eth_dev) 501 { 502 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 503 uint32_t rx_mode = lif->rx_mode; 504 505 rx_mode |= IONIC_RX_MODE_F_ALLMULTI; 506 507 ionic_set_rx_mode(lif, rx_mode); 508 509 return 0; 510 } 511 512 int 513 ionic_dev_allmulticast_disable(struct rte_eth_dev *eth_dev) 514 { 515 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 516 uint32_t rx_mode = lif->rx_mode; 517 518 rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI; 519 520 ionic_set_rx_mode(lif, rx_mode); 521 522 return 0; 523 } 524 525 int 526 ionic_lif_change_mtu(struct ionic_lif *lif, int new_mtu) 527 { 528 struct ionic_admin_ctx ctx = { 529 .pending_work = true, 530 .cmd.lif_setattr = { 531 .opcode = IONIC_CMD_LIF_SETATTR, 532 .attr = IONIC_LIF_ATTR_MTU, 533 .mtu = new_mtu, 534 }, 535 }; 536 int err; 537 538 err = ionic_adminq_post_wait(lif, &ctx); 539 if (err) 540 return err; 541 542 return 0; 543 } 544 545 int 546 ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr) 547 { 548 struct ionic_adapter *adapter = lif->adapter; 549 struct ionic_dev *idev = &adapter->idev; 550 unsigned long index; 551 552 /* 553 * Note: interrupt handler is called for index = 0 only 554 * (we use interrupts for the notifyq only anyway, 555 * which has index = 0) 556 */ 557 558 for (index = 0; index < adapter->nintrs; index++) 559 if (!adapter->intrs[index]) 560 break; 561 562 if (index == adapter->nintrs) 563 return -ENOSPC; 564 565 adapter->intrs[index] = true; 566 567 ionic_intr_init(idev, intr, index); 568 569 return 0; 570 } 571 572 void 573 ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr) 574 { 575 if (intr->index != IONIC_INTR_NONE) 576 lif->adapter->intrs[intr->index] = false; 577 } 578 579 static int 580 ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, 581 uint32_t index, 582 const char *base, uint32_t flags, 583 uint32_t num_descs, 584 uint32_t desc_size, 585 uint32_t cq_desc_size, 586 uint32_t sg_desc_size, 587 struct ionic_qcq **qcq) 588 { 589 struct ionic_dev *idev = &lif->adapter->idev; 590 struct ionic_qcq *new; 591 uint32_t q_size, cq_size, sg_size, total_size; 592 void *q_base, *cq_base, *sg_base; 593 rte_iova_t q_base_pa = 0; 594 rte_iova_t cq_base_pa = 0; 595 rte_iova_t sg_base_pa = 0; 596 uint32_t socket_id = rte_socket_id(); 597 int err; 598 599 *qcq = NULL; 600 601 q_size = num_descs * desc_size; 602 cq_size = num_descs * cq_desc_size; 603 sg_size = num_descs * sg_desc_size; 604 605 total_size = RTE_ALIGN(q_size, PAGE_SIZE) + 606 RTE_ALIGN(cq_size, PAGE_SIZE); 607 /* 608 * Note: aligning q_size/cq_size is not enough due to cq_base address 609 * aligning as q_base could be not aligned to the page. 610 * Adding PAGE_SIZE. 611 */ 612 total_size += PAGE_SIZE; 613 614 if (flags & IONIC_QCQ_F_SG) { 615 total_size += RTE_ALIGN(sg_size, PAGE_SIZE); 616 total_size += PAGE_SIZE; 617 } 618 619 new = rte_zmalloc("ionic", sizeof(*new), 0); 620 if (!new) { 621 IONIC_PRINT(ERR, "Cannot allocate queue structure"); 622 return -ENOMEM; 623 } 624 625 new->lif = lif; 626 new->flags = flags; 627 628 new->q.info = rte_zmalloc("ionic", sizeof(*new->q.info) * num_descs, 0); 629 if (!new->q.info) { 630 IONIC_PRINT(ERR, "Cannot allocate queue info"); 631 err = -ENOMEM; 632 goto err_out_free_qcq; 633 } 634 635 new->q.type = type; 636 637 err = ionic_q_init(lif, idev, &new->q, index, num_descs, 638 desc_size, sg_desc_size); 639 if (err) { 640 IONIC_PRINT(ERR, "Queue initialization failed"); 641 goto err_out_free_info; 642 } 643 644 err = ionic_cq_init(lif, &new->cq, num_descs, cq_desc_size); 645 if (err) { 646 IONIC_PRINT(ERR, "Completion queue initialization failed"); 647 goto err_out_free_info; 648 } 649 650 new->base_z = rte_eth_dma_zone_reserve(lif->eth_dev, 651 base /* name */, index /* queue_idx */, 652 total_size, IONIC_ALIGN, socket_id); 653 654 if (!new->base_z) { 655 IONIC_PRINT(ERR, "Cannot reserve queue DMA memory"); 656 err = -ENOMEM; 657 goto err_out_free_info; 658 } 659 660 new->base = new->base_z->addr; 661 new->base_pa = new->base_z->iova; 662 new->total_size = total_size; 663 664 q_base = new->base; 665 q_base_pa = new->base_pa; 666 667 cq_base = (void *)RTE_ALIGN((uintptr_t)q_base + q_size, PAGE_SIZE); 668 cq_base_pa = RTE_ALIGN(q_base_pa + q_size, PAGE_SIZE); 669 670 if (flags & IONIC_QCQ_F_SG) { 671 sg_base = (void *)RTE_ALIGN((uintptr_t)cq_base + cq_size, 672 PAGE_SIZE); 673 sg_base_pa = RTE_ALIGN(cq_base_pa + cq_size, PAGE_SIZE); 674 ionic_q_sg_map(&new->q, sg_base, sg_base_pa); 675 } 676 677 IONIC_PRINT(DEBUG, "Q-Base-PA = %#jx CQ-Base-PA = %#jx " 678 "SG-base-PA = %#jx", 679 q_base_pa, cq_base_pa, sg_base_pa); 680 681 ionic_q_map(&new->q, q_base, q_base_pa); 682 ionic_cq_map(&new->cq, cq_base, cq_base_pa); 683 ionic_cq_bind(&new->cq, &new->q); 684 685 *qcq = new; 686 687 return 0; 688 689 err_out_free_info: 690 rte_free(new->q.info); 691 err_out_free_qcq: 692 rte_free(new); 693 694 return err; 695 } 696 697 void 698 ionic_qcq_free(struct ionic_qcq *qcq) 699 { 700 if (qcq->base_z) { 701 qcq->base = NULL; 702 qcq->base_pa = 0; 703 rte_memzone_free(qcq->base_z); 704 qcq->base_z = NULL; 705 } 706 707 if (qcq->q.info) { 708 rte_free(qcq->q.info); 709 qcq->q.info = NULL; 710 } 711 712 rte_free(qcq); 713 } 714 715 int 716 ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs, 717 struct ionic_qcq **qcq) 718 { 719 uint32_t flags; 720 int err = -ENOMEM; 721 722 flags = IONIC_QCQ_F_SG; 723 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, index, "rx", flags, 724 nrxq_descs, 725 sizeof(struct ionic_rxq_desc), 726 sizeof(struct ionic_rxq_comp), 727 sizeof(struct ionic_rxq_sg_desc), 728 &lif->rxqcqs[index]); 729 if (err) 730 return err; 731 732 *qcq = lif->rxqcqs[index]; 733 734 return 0; 735 } 736 737 int 738 ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs, 739 struct ionic_qcq **qcq) 740 { 741 uint32_t flags; 742 int err = -ENOMEM; 743 744 flags = IONIC_QCQ_F_SG; 745 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, index, "tx", flags, 746 ntxq_descs, 747 sizeof(struct ionic_txq_desc), 748 sizeof(struct ionic_txq_comp), 749 sizeof(struct ionic_txq_sg_desc), 750 &lif->txqcqs[index]); 751 if (err) 752 return err; 753 754 *qcq = lif->txqcqs[index]; 755 756 return 0; 757 } 758 759 static int 760 ionic_admin_qcq_alloc(struct ionic_lif *lif) 761 { 762 uint32_t flags; 763 int err = -ENOMEM; 764 765 flags = 0; 766 err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags, 767 IONIC_ADMINQ_LENGTH, 768 sizeof(struct ionic_admin_cmd), 769 sizeof(struct ionic_admin_comp), 770 0, 771 &lif->adminqcq); 772 if (err) 773 return err; 774 775 return 0; 776 } 777 778 static int 779 ionic_notify_qcq_alloc(struct ionic_lif *lif) 780 { 781 struct ionic_qcq *nqcq; 782 struct ionic_dev *idev = &lif->adapter->idev; 783 uint32_t flags = 0; 784 int err = -ENOMEM; 785 786 err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notify", 787 flags, 788 IONIC_NOTIFYQ_LENGTH, 789 sizeof(struct ionic_notifyq_cmd), 790 sizeof(union ionic_notifyq_comp), 791 0, 792 &nqcq); 793 if (err) 794 return err; 795 796 err = ionic_intr_alloc(lif, &nqcq->intr); 797 if (err) { 798 ionic_qcq_free(nqcq); 799 return err; 800 } 801 802 ionic_intr_mask_assert(idev->intr_ctrl, nqcq->intr.index, 803 IONIC_INTR_MASK_SET); 804 805 lif->notifyqcq = nqcq; 806 807 return 0; 808 } 809 810 static void * 811 ionic_bus_map_dbpage(struct ionic_adapter *adapter, int page_num) 812 { 813 char *vaddr = adapter->bars[IONIC_PCI_BAR_DBELL].vaddr; 814 815 if (adapter->num_bars <= IONIC_PCI_BAR_DBELL) 816 return NULL; 817 818 return (void *)&vaddr[page_num << PAGE_SHIFT]; 819 } 820 821 int 822 ionic_lif_alloc(struct ionic_lif *lif) 823 { 824 struct ionic_adapter *adapter = lif->adapter; 825 uint32_t socket_id = rte_socket_id(); 826 int err; 827 828 /* 829 * lif->name was zeroed on allocation. 830 * Copy (sizeof() - 1) bytes to ensure that it is NULL terminated. 831 */ 832 memcpy(lif->name, lif->eth_dev->data->name, sizeof(lif->name) - 1); 833 834 IONIC_PRINT(DEBUG, "LIF: %s", lif->name); 835 836 IONIC_PRINT(DEBUG, "Allocating Lif Info"); 837 838 rte_spinlock_init(&lif->adminq_lock); 839 rte_spinlock_init(&lif->adminq_service_lock); 840 841 lif->kern_dbpage = ionic_bus_map_dbpage(adapter, 0); 842 if (!lif->kern_dbpage) { 843 IONIC_PRINT(ERR, "Cannot map dbpage, aborting"); 844 return -ENOMEM; 845 } 846 847 lif->txqcqs = rte_zmalloc("ionic", sizeof(*lif->txqcqs) * 848 adapter->max_ntxqs_per_lif, 0); 849 850 if (!lif->txqcqs) { 851 IONIC_PRINT(ERR, "Cannot allocate tx queues array"); 852 return -ENOMEM; 853 } 854 855 lif->rxqcqs = rte_zmalloc("ionic", sizeof(*lif->rxqcqs) * 856 adapter->max_nrxqs_per_lif, 0); 857 858 if (!lif->rxqcqs) { 859 IONIC_PRINT(ERR, "Cannot allocate rx queues array"); 860 return -ENOMEM; 861 } 862 863 IONIC_PRINT(DEBUG, "Allocating Notify Queue"); 864 865 err = ionic_notify_qcq_alloc(lif); 866 if (err) { 867 IONIC_PRINT(ERR, "Cannot allocate notify queue"); 868 return err; 869 } 870 871 IONIC_PRINT(DEBUG, "Allocating Admin Queue"); 872 873 err = ionic_admin_qcq_alloc(lif); 874 if (err) { 875 IONIC_PRINT(ERR, "Cannot allocate admin queue"); 876 return err; 877 } 878 879 IONIC_PRINT(DEBUG, "Allocating Lif Info"); 880 881 lif->info_sz = RTE_ALIGN(sizeof(*lif->info), PAGE_SIZE); 882 883 lif->info_z = rte_eth_dma_zone_reserve(lif->eth_dev, 884 "lif_info", 0 /* queue_idx*/, 885 lif->info_sz, IONIC_ALIGN, socket_id); 886 if (!lif->info_z) { 887 IONIC_PRINT(ERR, "Cannot allocate lif info memory"); 888 return -ENOMEM; 889 } 890 891 lif->info = lif->info_z->addr; 892 lif->info_pa = lif->info_z->iova; 893 894 return 0; 895 } 896 897 void 898 ionic_lif_free(struct ionic_lif *lif) 899 { 900 if (lif->notifyqcq) { 901 ionic_qcq_free(lif->notifyqcq); 902 lif->notifyqcq = NULL; 903 } 904 905 if (lif->adminqcq) { 906 ionic_qcq_free(lif->adminqcq); 907 lif->adminqcq = NULL; 908 } 909 910 if (lif->txqcqs) { 911 rte_free(lif->txqcqs); 912 lif->txqcqs = NULL; 913 } 914 915 if (lif->rxqcqs) { 916 rte_free(lif->rxqcqs); 917 lif->rxqcqs = NULL; 918 } 919 920 if (lif->info) { 921 rte_memzone_free(lif->info_z); 922 lif->info = NULL; 923 } 924 } 925 926 void 927 ionic_lif_free_queues(struct ionic_lif *lif) 928 { 929 uint32_t i; 930 931 for (i = 0; i < lif->ntxqcqs; i++) { 932 ionic_dev_tx_queue_release(lif->eth_dev->data->tx_queues[i]); 933 lif->eth_dev->data->tx_queues[i] = NULL; 934 } 935 for (i = 0; i < lif->nrxqcqs; i++) { 936 ionic_dev_rx_queue_release(lif->eth_dev->data->rx_queues[i]); 937 lif->eth_dev->data->rx_queues[i] = NULL; 938 } 939 } 940 941 int 942 ionic_lif_rss_config(struct ionic_lif *lif, 943 const uint16_t types, const uint8_t *key, const uint32_t *indir) 944 { 945 struct ionic_admin_ctx ctx = { 946 .pending_work = true, 947 .cmd.lif_setattr = { 948 .opcode = IONIC_CMD_LIF_SETATTR, 949 .attr = IONIC_LIF_ATTR_RSS, 950 .rss.types = types, 951 .rss.addr = lif->rss_ind_tbl_pa, 952 }, 953 }; 954 unsigned int i; 955 956 IONIC_PRINT_CALL(); 957 958 lif->rss_types = types; 959 960 if (key) 961 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE); 962 963 if (indir) 964 for (i = 0; i < lif->adapter->ident.lif.eth.rss_ind_tbl_sz; i++) 965 lif->rss_ind_tbl[i] = indir[i]; 966 967 memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key, 968 IONIC_RSS_HASH_KEY_SIZE); 969 970 return ionic_adminq_post_wait(lif, &ctx); 971 } 972 973 static int 974 ionic_lif_rss_setup(struct ionic_lif *lif) 975 { 976 static const uint8_t toeplitz_symmetric_key[] = { 977 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 978 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 979 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 980 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 981 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 982 }; 983 uint32_t i; 984 uint16_t tbl_sz = lif->adapter->ident.lif.eth.rss_ind_tbl_sz; 985 986 IONIC_PRINT_CALL(); 987 988 if (!lif->rss_ind_tbl_z) { 989 lif->rss_ind_tbl_z = rte_eth_dma_zone_reserve(lif->eth_dev, 990 "rss_ind_tbl", 0 /* queue_idx */, 991 sizeof(*lif->rss_ind_tbl) * tbl_sz, 992 IONIC_ALIGN, rte_socket_id()); 993 if (!lif->rss_ind_tbl_z) { 994 IONIC_PRINT(ERR, "OOM"); 995 return -ENOMEM; 996 } 997 998 lif->rss_ind_tbl = lif->rss_ind_tbl_z->addr; 999 lif->rss_ind_tbl_pa = lif->rss_ind_tbl_z->iova; 1000 } 1001 1002 if (lif->rss_ind_tbl_nrxqcqs != lif->nrxqcqs) { 1003 lif->rss_ind_tbl_nrxqcqs = lif->nrxqcqs; 1004 1005 /* Fill indirection table with 'default' values */ 1006 for (i = 0; i < tbl_sz; i++) 1007 lif->rss_ind_tbl[i] = i % lif->nrxqcqs; 1008 } 1009 1010 return ionic_lif_rss_config(lif, IONIC_RSS_OFFLOAD_ALL, 1011 toeplitz_symmetric_key, NULL); 1012 } 1013 1014 static void 1015 ionic_lif_rss_teardown(struct ionic_lif *lif) 1016 { 1017 if (!lif->rss_ind_tbl) 1018 return; 1019 1020 if (lif->rss_ind_tbl_z) { 1021 /* Disable RSS on the NIC */ 1022 ionic_lif_rss_config(lif, 0x0, NULL, NULL); 1023 1024 lif->rss_ind_tbl = NULL; 1025 lif->rss_ind_tbl_pa = 0; 1026 rte_memzone_free(lif->rss_ind_tbl_z); 1027 lif->rss_ind_tbl_z = NULL; 1028 } 1029 } 1030 1031 static void 1032 ionic_lif_qcq_deinit(struct ionic_qcq *qcq) 1033 { 1034 qcq->flags &= ~IONIC_QCQ_F_INITED; 1035 } 1036 1037 void 1038 ionic_lif_txq_deinit(struct ionic_qcq *qcq) 1039 { 1040 ionic_lif_qcq_deinit(qcq); 1041 } 1042 1043 void 1044 ionic_lif_rxq_deinit(struct ionic_qcq *qcq) 1045 { 1046 ionic_lif_qcq_deinit(qcq); 1047 } 1048 1049 static void 1050 ionic_lif_notifyq_deinit(struct ionic_lif *lif) 1051 { 1052 struct ionic_qcq *nqcq = lif->notifyqcq; 1053 struct ionic_dev *idev = &lif->adapter->idev; 1054 1055 if (!(nqcq->flags & IONIC_QCQ_F_INITED)) 1056 return; 1057 1058 ionic_intr_mask(idev->intr_ctrl, nqcq->intr.index, 1059 IONIC_INTR_MASK_SET); 1060 1061 nqcq->flags &= ~IONIC_QCQ_F_INITED; 1062 } 1063 1064 bool 1065 ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index, 1066 void *cb_arg __rte_unused) 1067 { 1068 struct ionic_admin_comp *cq_desc_base = cq->base; 1069 struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index]; 1070 1071 if (!color_match(cq_desc->color, cq->done_color)) 1072 return false; 1073 1074 ionic_q_service(cq->bound_q, cq_desc_index, cq_desc->comp_index, NULL); 1075 1076 return true; 1077 } 1078 1079 /* This acts like ionic_napi */ 1080 int 1081 ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb, 1082 void *cb_arg) 1083 { 1084 struct ionic_cq *cq = &qcq->cq; 1085 uint32_t work_done; 1086 1087 work_done = ionic_cq_service(cq, budget, cb, cb_arg); 1088 1089 return work_done; 1090 } 1091 1092 static void 1093 ionic_link_status_check(struct ionic_lif *lif) 1094 { 1095 struct ionic_adapter *adapter = lif->adapter; 1096 bool link_up; 1097 1098 lif->state &= ~IONIC_LIF_F_LINK_CHECK_NEEDED; 1099 1100 if (!lif->info) 1101 return; 1102 1103 link_up = (lif->info->status.link_status == IONIC_PORT_OPER_STATUS_UP); 1104 1105 if ((link_up && adapter->link_up) || 1106 (!link_up && !adapter->link_up)) 1107 return; 1108 1109 if (link_up) { 1110 adapter->link_speed = lif->info->status.link_speed; 1111 IONIC_PRINT(DEBUG, "Link up - %d Gbps", 1112 adapter->link_speed); 1113 } else { 1114 IONIC_PRINT(DEBUG, "Link down"); 1115 } 1116 1117 adapter->link_up = link_up; 1118 ionic_dev_link_update(lif->eth_dev, 0); 1119 } 1120 1121 static void 1122 ionic_lif_handle_fw_down(struct ionic_lif *lif) 1123 { 1124 if (lif->state & IONIC_LIF_F_FW_RESET) 1125 return; 1126 1127 lif->state |= IONIC_LIF_F_FW_RESET; 1128 1129 if (lif->state & IONIC_LIF_F_UP) { 1130 IONIC_PRINT(NOTICE, 1131 "Surprise FW stop, stopping %s\n", lif->name); 1132 ionic_lif_stop(lif); 1133 } 1134 1135 IONIC_PRINT(NOTICE, "FW down, %s stopped", lif->name); 1136 } 1137 1138 static bool 1139 ionic_notifyq_cb(struct ionic_cq *cq, uint32_t cq_desc_index, void *cb_arg) 1140 { 1141 union ionic_notifyq_comp *cq_desc_base = cq->base; 1142 union ionic_notifyq_comp *cq_desc = &cq_desc_base[cq_desc_index]; 1143 struct ionic_lif *lif = cb_arg; 1144 1145 IONIC_PRINT(DEBUG, "Notifyq callback eid = %jd ecode = %d", 1146 cq_desc->event.eid, cq_desc->event.ecode); 1147 1148 /* Have we run out of new completions to process? */ 1149 if (!(cq_desc->event.eid > lif->last_eid)) 1150 return false; 1151 1152 lif->last_eid = cq_desc->event.eid; 1153 1154 switch (cq_desc->event.ecode) { 1155 case IONIC_EVENT_LINK_CHANGE: 1156 IONIC_PRINT(DEBUG, 1157 "Notifyq IONIC_EVENT_LINK_CHANGE %s " 1158 "eid=%jd link_status=%d link_speed=%d", 1159 lif->name, 1160 cq_desc->event.eid, 1161 cq_desc->link_change.link_status, 1162 cq_desc->link_change.link_speed); 1163 1164 lif->state |= IONIC_LIF_F_LINK_CHECK_NEEDED; 1165 break; 1166 1167 case IONIC_EVENT_RESET: 1168 IONIC_PRINT(NOTICE, 1169 "Notifyq IONIC_EVENT_RESET %s " 1170 "eid=%jd, reset_code=%d state=%d", 1171 lif->name, 1172 cq_desc->event.eid, 1173 cq_desc->reset.reset_code, 1174 cq_desc->reset.state); 1175 ionic_lif_handle_fw_down(lif); 1176 break; 1177 1178 default: 1179 IONIC_PRINT(WARNING, "Notifyq bad event ecode=%d eid=%jd", 1180 cq_desc->event.ecode, cq_desc->event.eid); 1181 break; 1182 } 1183 1184 return true; 1185 } 1186 1187 int 1188 ionic_notifyq_handler(struct ionic_lif *lif, int budget) 1189 { 1190 struct ionic_dev *idev = &lif->adapter->idev; 1191 struct ionic_qcq *qcq = lif->notifyqcq; 1192 uint32_t work_done; 1193 1194 if (!(qcq->flags & IONIC_QCQ_F_INITED)) { 1195 IONIC_PRINT(DEBUG, "Notifyq not yet initialized"); 1196 return -1; 1197 } 1198 1199 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, 1200 IONIC_INTR_MASK_SET); 1201 1202 work_done = ionic_qcq_service(qcq, budget, ionic_notifyq_cb, lif); 1203 1204 if (lif->state & IONIC_LIF_F_LINK_CHECK_NEEDED) 1205 ionic_link_status_check(lif); 1206 1207 ionic_intr_credits(idev->intr_ctrl, qcq->intr.index, 1208 work_done, IONIC_INTR_CRED_RESET_COALESCE); 1209 1210 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, 1211 IONIC_INTR_MASK_CLEAR); 1212 1213 return 0; 1214 } 1215 1216 static int 1217 ionic_lif_adminq_init(struct ionic_lif *lif) 1218 { 1219 struct ionic_dev *idev = &lif->adapter->idev; 1220 struct ionic_qcq *qcq = lif->adminqcq; 1221 struct ionic_queue *q = &qcq->q; 1222 struct ionic_q_init_comp comp; 1223 int err; 1224 1225 ionic_dev_cmd_adminq_init(idev, qcq); 1226 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); 1227 if (err) 1228 return err; 1229 1230 ionic_dev_cmd_comp(idev, &comp); 1231 1232 q->hw_type = comp.hw_type; 1233 q->hw_index = comp.hw_index; 1234 q->db = ionic_db_map(lif, q); 1235 1236 IONIC_PRINT(DEBUG, "adminq->hw_type %d", q->hw_type); 1237 IONIC_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index); 1238 IONIC_PRINT(DEBUG, "adminq->db %p", q->db); 1239 1240 qcq->flags |= IONIC_QCQ_F_INITED; 1241 1242 return 0; 1243 } 1244 1245 static int 1246 ionic_lif_notifyq_init(struct ionic_lif *lif) 1247 { 1248 struct ionic_dev *idev = &lif->adapter->idev; 1249 struct ionic_qcq *qcq = lif->notifyqcq; 1250 struct ionic_queue *q = &qcq->q; 1251 int err; 1252 1253 struct ionic_admin_ctx ctx = { 1254 .pending_work = true, 1255 .cmd.q_init = { 1256 .opcode = IONIC_CMD_Q_INIT, 1257 .type = q->type, 1258 .index = q->index, 1259 .flags = (IONIC_QINIT_F_IRQ | IONIC_QINIT_F_ENA), 1260 .intr_index = qcq->intr.index, 1261 .ring_size = rte_log2_u32(q->num_descs), 1262 .ring_base = q->base_pa, 1263 } 1264 }; 1265 1266 IONIC_PRINT(DEBUG, "notifyq_init.index %d", 1267 ctx.cmd.q_init.index); 1268 IONIC_PRINT(DEBUG, "notifyq_init.ring_base 0x%" PRIx64 "", 1269 ctx.cmd.q_init.ring_base); 1270 IONIC_PRINT(DEBUG, "notifyq_init.ring_size %d", 1271 ctx.cmd.q_init.ring_size); 1272 IONIC_PRINT(DEBUG, "notifyq_init.ver %u", ctx.cmd.q_init.ver); 1273 1274 err = ionic_adminq_post_wait(lif, &ctx); 1275 if (err) 1276 return err; 1277 1278 q->hw_type = ctx.comp.q_init.hw_type; 1279 q->hw_index = ctx.comp.q_init.hw_index; 1280 q->db = NULL; 1281 1282 IONIC_PRINT(DEBUG, "notifyq->hw_type %d", q->hw_type); 1283 IONIC_PRINT(DEBUG, "notifyq->hw_index %d", q->hw_index); 1284 IONIC_PRINT(DEBUG, "notifyq->db %p", q->db); 1285 1286 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, 1287 IONIC_INTR_MASK_CLEAR); 1288 1289 qcq->flags |= IONIC_QCQ_F_INITED; 1290 1291 return 0; 1292 } 1293 1294 int 1295 ionic_lif_set_features(struct ionic_lif *lif) 1296 { 1297 struct ionic_admin_ctx ctx = { 1298 .pending_work = true, 1299 .cmd.lif_setattr = { 1300 .opcode = IONIC_CMD_LIF_SETATTR, 1301 .attr = IONIC_LIF_ATTR_FEATURES, 1302 .features = lif->features, 1303 }, 1304 }; 1305 int err; 1306 1307 err = ionic_adminq_post_wait(lif, &ctx); 1308 if (err) 1309 return err; 1310 1311 lif->hw_features = (ctx.cmd.lif_setattr.features & 1312 ctx.comp.lif_setattr.features); 1313 1314 if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG) 1315 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_TX_TAG"); 1316 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP) 1317 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_STRIP"); 1318 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER) 1319 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_FILTER"); 1320 if (lif->hw_features & IONIC_ETH_HW_RX_HASH) 1321 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_HASH"); 1322 if (lif->hw_features & IONIC_ETH_HW_TX_SG) 1323 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_SG"); 1324 if (lif->hw_features & IONIC_ETH_HW_RX_SG) 1325 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_SG"); 1326 if (lif->hw_features & IONIC_ETH_HW_TX_CSUM) 1327 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_CSUM"); 1328 if (lif->hw_features & IONIC_ETH_HW_RX_CSUM) 1329 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_CSUM"); 1330 if (lif->hw_features & IONIC_ETH_HW_TSO) 1331 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO"); 1332 if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6) 1333 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPV6"); 1334 if (lif->hw_features & IONIC_ETH_HW_TSO_ECN) 1335 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_ECN"); 1336 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE) 1337 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE"); 1338 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM) 1339 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE_CSUM"); 1340 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4) 1341 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP4"); 1342 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6) 1343 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP6"); 1344 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP) 1345 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP"); 1346 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM) 1347 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP_CSUM"); 1348 1349 return 0; 1350 } 1351 1352 int 1353 ionic_lif_txq_init(struct ionic_qcq *qcq) 1354 { 1355 struct ionic_queue *q = &qcq->q; 1356 struct ionic_lif *lif = qcq->lif; 1357 struct ionic_cq *cq = &qcq->cq; 1358 struct ionic_admin_ctx ctx = { 1359 .pending_work = true, 1360 .cmd.q_init = { 1361 .opcode = IONIC_CMD_Q_INIT, 1362 .type = q->type, 1363 .index = q->index, 1364 .flags = IONIC_QINIT_F_SG | IONIC_QINIT_F_ENA, 1365 .intr_index = IONIC_INTR_NONE, 1366 .ring_size = rte_log2_u32(q->num_descs), 1367 .ring_base = q->base_pa, 1368 .cq_ring_base = cq->base_pa, 1369 .sg_ring_base = q->sg_base_pa, 1370 }, 1371 }; 1372 int err; 1373 1374 IONIC_PRINT(DEBUG, "txq_init.index %d", ctx.cmd.q_init.index); 1375 IONIC_PRINT(DEBUG, "txq_init.ring_base 0x%" PRIx64 "", 1376 ctx.cmd.q_init.ring_base); 1377 IONIC_PRINT(DEBUG, "txq_init.ring_size %d", 1378 ctx.cmd.q_init.ring_size); 1379 IONIC_PRINT(DEBUG, "txq_init.ver %u", ctx.cmd.q_init.ver); 1380 1381 err = ionic_adminq_post_wait(qcq->lif, &ctx); 1382 if (err) 1383 return err; 1384 1385 q->hw_type = ctx.comp.q_init.hw_type; 1386 q->hw_index = ctx.comp.q_init.hw_index; 1387 q->db = ionic_db_map(lif, q); 1388 1389 IONIC_PRINT(DEBUG, "txq->hw_type %d", q->hw_type); 1390 IONIC_PRINT(DEBUG, "txq->hw_index %d", q->hw_index); 1391 IONIC_PRINT(DEBUG, "txq->db %p", q->db); 1392 1393 qcq->flags |= IONIC_QCQ_F_INITED; 1394 1395 return 0; 1396 } 1397 1398 int 1399 ionic_lif_rxq_init(struct ionic_qcq *qcq) 1400 { 1401 struct ionic_queue *q = &qcq->q; 1402 struct ionic_lif *lif = qcq->lif; 1403 struct ionic_cq *cq = &qcq->cq; 1404 struct ionic_admin_ctx ctx = { 1405 .pending_work = true, 1406 .cmd.q_init = { 1407 .opcode = IONIC_CMD_Q_INIT, 1408 .type = q->type, 1409 .index = q->index, 1410 .flags = IONIC_QINIT_F_SG | IONIC_QINIT_F_ENA, 1411 .intr_index = IONIC_INTR_NONE, 1412 .ring_size = rte_log2_u32(q->num_descs), 1413 .ring_base = q->base_pa, 1414 .cq_ring_base = cq->base_pa, 1415 .sg_ring_base = q->sg_base_pa, 1416 }, 1417 }; 1418 int err; 1419 1420 IONIC_PRINT(DEBUG, "rxq_init.index %d", ctx.cmd.q_init.index); 1421 IONIC_PRINT(DEBUG, "rxq_init.ring_base 0x%" PRIx64 "", 1422 ctx.cmd.q_init.ring_base); 1423 IONIC_PRINT(DEBUG, "rxq_init.ring_size %d", 1424 ctx.cmd.q_init.ring_size); 1425 IONIC_PRINT(DEBUG, "rxq_init.ver %u", ctx.cmd.q_init.ver); 1426 1427 err = ionic_adminq_post_wait(qcq->lif, &ctx); 1428 if (err) 1429 return err; 1430 1431 q->hw_type = ctx.comp.q_init.hw_type; 1432 q->hw_index = ctx.comp.q_init.hw_index; 1433 q->db = ionic_db_map(lif, q); 1434 1435 qcq->flags |= IONIC_QCQ_F_INITED; 1436 1437 IONIC_PRINT(DEBUG, "rxq->hw_type %d", q->hw_type); 1438 IONIC_PRINT(DEBUG, "rxq->hw_index %d", q->hw_index); 1439 IONIC_PRINT(DEBUG, "rxq->db %p", q->db); 1440 1441 return 0; 1442 } 1443 1444 static int 1445 ionic_station_set(struct ionic_lif *lif) 1446 { 1447 struct ionic_admin_ctx ctx = { 1448 .pending_work = true, 1449 .cmd.lif_getattr = { 1450 .opcode = IONIC_CMD_LIF_GETATTR, 1451 .attr = IONIC_LIF_ATTR_MAC, 1452 }, 1453 }; 1454 int err; 1455 1456 IONIC_PRINT_CALL(); 1457 1458 err = ionic_adminq_post_wait(lif, &ctx); 1459 if (err) 1460 return err; 1461 1462 memcpy(lif->mac_addr, ctx.comp.lif_getattr.mac, RTE_ETHER_ADDR_LEN); 1463 1464 return 0; 1465 } 1466 1467 static void 1468 ionic_lif_set_name(struct ionic_lif *lif) 1469 { 1470 struct ionic_admin_ctx ctx = { 1471 .pending_work = true, 1472 .cmd.lif_setattr = { 1473 .opcode = IONIC_CMD_LIF_SETATTR, 1474 .attr = IONIC_LIF_ATTR_NAME, 1475 }, 1476 }; 1477 1478 memcpy(ctx.cmd.lif_setattr.name, lif->name, 1479 sizeof(ctx.cmd.lif_setattr.name) - 1); 1480 1481 ionic_adminq_post_wait(lif, &ctx); 1482 } 1483 1484 int 1485 ionic_lif_init(struct ionic_lif *lif) 1486 { 1487 struct ionic_dev *idev = &lif->adapter->idev; 1488 struct ionic_q_init_comp comp; 1489 int err; 1490 1491 memset(&lif->stats_base, 0, sizeof(lif->stats_base)); 1492 1493 ionic_dev_cmd_lif_init(idev, lif->info_pa); 1494 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); 1495 ionic_dev_cmd_comp(idev, &comp); 1496 if (err) 1497 return err; 1498 1499 lif->hw_index = comp.hw_index; 1500 1501 err = ionic_lif_adminq_init(lif); 1502 if (err) 1503 return err; 1504 1505 err = ionic_lif_notifyq_init(lif); 1506 if (err) 1507 goto err_out_adminq_deinit; 1508 1509 /* 1510 * Configure initial feature set 1511 * This will be updated later by the dev_configure() step 1512 */ 1513 lif->features = IONIC_ETH_HW_RX_HASH | IONIC_ETH_HW_VLAN_RX_FILTER; 1514 1515 err = ionic_lif_set_features(lif); 1516 if (err) 1517 goto err_out_notifyq_deinit; 1518 1519 err = ionic_rx_filters_init(lif); 1520 if (err) 1521 goto err_out_notifyq_deinit; 1522 1523 err = ionic_station_set(lif); 1524 if (err) 1525 goto err_out_rx_filter_deinit; 1526 1527 ionic_lif_set_name(lif); 1528 1529 lif->state |= IONIC_LIF_F_INITED; 1530 1531 return 0; 1532 1533 err_out_rx_filter_deinit: 1534 ionic_rx_filters_deinit(lif); 1535 1536 err_out_notifyq_deinit: 1537 ionic_lif_notifyq_deinit(lif); 1538 1539 err_out_adminq_deinit: 1540 ionic_lif_qcq_deinit(lif->adminqcq); 1541 1542 return err; 1543 } 1544 1545 void 1546 ionic_lif_deinit(struct ionic_lif *lif) 1547 { 1548 if (!(lif->state & IONIC_LIF_F_INITED)) 1549 return; 1550 1551 ionic_rx_filters_deinit(lif); 1552 ionic_lif_rss_teardown(lif); 1553 ionic_lif_notifyq_deinit(lif); 1554 ionic_lif_qcq_deinit(lif->adminqcq); 1555 1556 lif->state &= ~IONIC_LIF_F_INITED; 1557 } 1558 1559 void 1560 ionic_lif_configure_vlan_offload(struct ionic_lif *lif, int mask) 1561 { 1562 struct rte_eth_dev *eth_dev = lif->eth_dev; 1563 struct rte_eth_rxmode *rxmode = ð_dev->data->dev_conf.rxmode; 1564 1565 /* 1566 * IONIC_ETH_HW_VLAN_RX_FILTER cannot be turned off, so 1567 * set DEV_RX_OFFLOAD_VLAN_FILTER and ignore ETH_VLAN_FILTER_MASK 1568 */ 1569 rxmode->offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 1570 1571 if (mask & ETH_VLAN_STRIP_MASK) { 1572 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 1573 lif->features |= IONIC_ETH_HW_VLAN_RX_STRIP; 1574 else 1575 lif->features &= ~IONIC_ETH_HW_VLAN_RX_STRIP; 1576 } 1577 } 1578 1579 void 1580 ionic_lif_configure(struct ionic_lif *lif) 1581 { 1582 struct rte_eth_rxmode *rxmode = &lif->eth_dev->data->dev_conf.rxmode; 1583 struct rte_eth_txmode *txmode = &lif->eth_dev->data->dev_conf.txmode; 1584 struct ionic_identity *ident = &lif->adapter->ident; 1585 uint32_t ntxqs_per_lif = 1586 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ]; 1587 uint32_t nrxqs_per_lif = 1588 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ]; 1589 uint32_t nrxqs = lif->eth_dev->data->nb_rx_queues; 1590 uint32_t ntxqs = lif->eth_dev->data->nb_tx_queues; 1591 1592 lif->port_id = lif->eth_dev->data->port_id; 1593 1594 IONIC_PRINT(DEBUG, "Configuring LIF on port %u", 1595 lif->port_id); 1596 1597 if (nrxqs > 0) 1598 nrxqs_per_lif = RTE_MIN(nrxqs_per_lif, nrxqs); 1599 1600 if (ntxqs > 0) 1601 ntxqs_per_lif = RTE_MIN(ntxqs_per_lif, ntxqs); 1602 1603 lif->nrxqcqs = nrxqs_per_lif; 1604 lif->ntxqcqs = ntxqs_per_lif; 1605 1606 /* Update the LIF configuration based on the eth_dev */ 1607 1608 /* 1609 * NB: While it is true that RSS_HASH is always enabled on ionic, 1610 * setting this flag unconditionally causes problems in DTS. 1611 * rxmode->offloads |= DEV_RX_OFFLOAD_RSS_HASH; 1612 */ 1613 1614 /* RX per-port */ 1615 1616 if (rxmode->offloads & DEV_RX_OFFLOAD_IPV4_CKSUM || 1617 rxmode->offloads & DEV_RX_OFFLOAD_UDP_CKSUM || 1618 rxmode->offloads & DEV_RX_OFFLOAD_TCP_CKSUM) 1619 lif->features |= IONIC_ETH_HW_RX_CSUM; 1620 else 1621 lif->features &= ~IONIC_ETH_HW_RX_CSUM; 1622 1623 if (rxmode->offloads & DEV_RX_OFFLOAD_SCATTER) { 1624 lif->features |= IONIC_ETH_HW_RX_SG; 1625 lif->eth_dev->data->scattered_rx = 1; 1626 } else { 1627 lif->features &= ~IONIC_ETH_HW_RX_SG; 1628 lif->eth_dev->data->scattered_rx = 0; 1629 } 1630 1631 /* Covers VLAN_STRIP */ 1632 ionic_lif_configure_vlan_offload(lif, ETH_VLAN_STRIP_MASK); 1633 1634 /* TX per-port */ 1635 1636 if (txmode->offloads & DEV_TX_OFFLOAD_IPV4_CKSUM || 1637 txmode->offloads & DEV_TX_OFFLOAD_UDP_CKSUM || 1638 txmode->offloads & DEV_TX_OFFLOAD_TCP_CKSUM || 1639 txmode->offloads & DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM || 1640 txmode->offloads & DEV_TX_OFFLOAD_OUTER_UDP_CKSUM) 1641 lif->features |= IONIC_ETH_HW_TX_CSUM; 1642 else 1643 lif->features &= ~IONIC_ETH_HW_TX_CSUM; 1644 1645 if (txmode->offloads & DEV_TX_OFFLOAD_VLAN_INSERT) 1646 lif->features |= IONIC_ETH_HW_VLAN_TX_TAG; 1647 else 1648 lif->features &= ~IONIC_ETH_HW_VLAN_TX_TAG; 1649 1650 if (txmode->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) 1651 lif->features |= IONIC_ETH_HW_TX_SG; 1652 else 1653 lif->features &= ~IONIC_ETH_HW_TX_SG; 1654 1655 if (txmode->offloads & DEV_TX_OFFLOAD_TCP_TSO) { 1656 lif->features |= IONIC_ETH_HW_TSO; 1657 lif->features |= IONIC_ETH_HW_TSO_IPV6; 1658 lif->features |= IONIC_ETH_HW_TSO_ECN; 1659 } else { 1660 lif->features &= ~IONIC_ETH_HW_TSO; 1661 lif->features &= ~IONIC_ETH_HW_TSO_IPV6; 1662 lif->features &= ~IONIC_ETH_HW_TSO_ECN; 1663 } 1664 } 1665 1666 int 1667 ionic_lif_start(struct ionic_lif *lif) 1668 { 1669 uint32_t rx_mode; 1670 uint32_t i; 1671 int err; 1672 1673 err = ionic_lif_rss_setup(lif); 1674 if (err) 1675 return err; 1676 1677 if (!lif->rx_mode) { 1678 IONIC_PRINT(DEBUG, "Setting RX mode on %s", 1679 lif->name); 1680 1681 rx_mode = IONIC_RX_MODE_F_UNICAST; 1682 rx_mode |= IONIC_RX_MODE_F_MULTICAST; 1683 rx_mode |= IONIC_RX_MODE_F_BROADCAST; 1684 1685 ionic_set_rx_mode(lif, rx_mode); 1686 } 1687 1688 IONIC_PRINT(DEBUG, "Starting %u RX queues and %u TX queues " 1689 "on port %u", 1690 lif->nrxqcqs, lif->ntxqcqs, lif->port_id); 1691 1692 for (i = 0; i < lif->nrxqcqs; i++) { 1693 struct ionic_qcq *rxq = lif->rxqcqs[i]; 1694 if (!(rxq->flags & IONIC_QCQ_F_DEFERRED)) { 1695 err = ionic_dev_rx_queue_start(lif->eth_dev, i); 1696 1697 if (err) 1698 return err; 1699 } 1700 } 1701 1702 for (i = 0; i < lif->ntxqcqs; i++) { 1703 struct ionic_qcq *txq = lif->txqcqs[i]; 1704 if (!(txq->flags & IONIC_QCQ_F_DEFERRED)) { 1705 err = ionic_dev_tx_queue_start(lif->eth_dev, i); 1706 1707 if (err) 1708 return err; 1709 } 1710 } 1711 1712 /* Carrier ON here */ 1713 lif->state |= IONIC_LIF_F_UP; 1714 1715 ionic_link_status_check(lif); 1716 1717 return 0; 1718 } 1719 1720 int 1721 ionic_lif_identify(struct ionic_adapter *adapter) 1722 { 1723 struct ionic_dev *idev = &adapter->idev; 1724 struct ionic_identity *ident = &adapter->ident; 1725 int err; 1726 unsigned int i; 1727 unsigned int lif_words = sizeof(ident->lif.words) / 1728 sizeof(ident->lif.words[0]); 1729 unsigned int cmd_words = sizeof(idev->dev_cmd->data) / 1730 sizeof(idev->dev_cmd->data[0]); 1731 unsigned int nwords; 1732 1733 ionic_dev_cmd_lif_identify(idev, IONIC_LIF_TYPE_CLASSIC, 1734 IONIC_IDENTITY_VERSION_1); 1735 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); 1736 if (err) 1737 return (err); 1738 1739 nwords = RTE_MIN(lif_words, cmd_words); 1740 for (i = 0; i < nwords; i++) 1741 ident->lif.words[i] = ioread32(&idev->dev_cmd->data[i]); 1742 1743 IONIC_PRINT(INFO, "capabilities 0x%" PRIx64 " ", 1744 ident->lif.capabilities); 1745 1746 IONIC_PRINT(INFO, "eth.max_ucast_filters 0x%" PRIx32 " ", 1747 ident->lif.eth.max_ucast_filters); 1748 IONIC_PRINT(INFO, "eth.max_mcast_filters 0x%" PRIx32 " ", 1749 ident->lif.eth.max_mcast_filters); 1750 1751 IONIC_PRINT(INFO, "eth.features 0x%" PRIx64 " ", 1752 ident->lif.eth.config.features); 1753 IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_ADMINQ] 0x%" PRIx32 " ", 1754 ident->lif.eth.config.queue_count[IONIC_QTYPE_ADMINQ]); 1755 IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] 0x%" PRIx32 " ", 1756 ident->lif.eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]); 1757 IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_RXQ] 0x%" PRIx32 " ", 1758 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ]); 1759 IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_TXQ] 0x%" PRIx32 " ", 1760 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ]); 1761 1762 return 0; 1763 } 1764 1765 int 1766 ionic_lifs_size(struct ionic_adapter *adapter) 1767 { 1768 struct ionic_identity *ident = &adapter->ident; 1769 uint32_t nintrs, dev_nintrs = ident->dev.nintrs; 1770 1771 adapter->max_ntxqs_per_lif = 1772 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ]; 1773 adapter->max_nrxqs_per_lif = 1774 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ]; 1775 1776 nintrs = 1 /* notifyq */; 1777 1778 if (nintrs > dev_nintrs) { 1779 IONIC_PRINT(ERR, 1780 "At most %d intr supported, minimum req'd is %u", 1781 dev_nintrs, nintrs); 1782 return -ENOSPC; 1783 } 1784 1785 adapter->nintrs = nintrs; 1786 1787 return 0; 1788 } 1789