1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) 2 * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved. 3 */ 4 5 #include <rte_malloc.h> 6 #include <ethdev_driver.h> 7 8 #include "ionic.h" 9 #include "ionic_logs.h" 10 #include "ionic_lif.h" 11 #include "ionic_ethdev.h" 12 #include "ionic_rx_filter.h" 13 #include "ionic_rxtx.h" 14 15 static int ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr); 16 static int ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr); 17 18 int 19 ionic_qcq_enable(struct ionic_qcq *qcq) 20 { 21 struct ionic_queue *q = &qcq->q; 22 struct ionic_lif *lif = q->lif; 23 struct ionic_admin_ctx ctx = { 24 .pending_work = true, 25 .cmd.q_control = { 26 .opcode = IONIC_CMD_Q_CONTROL, 27 .type = q->type, 28 .index = rte_cpu_to_le_32(q->index), 29 .oper = IONIC_Q_ENABLE, 30 }, 31 }; 32 33 return ionic_adminq_post_wait(lif, &ctx); 34 } 35 36 int 37 ionic_qcq_disable(struct ionic_qcq *qcq) 38 { 39 struct ionic_queue *q = &qcq->q; 40 struct ionic_lif *lif = q->lif; 41 struct ionic_admin_ctx ctx = { 42 .pending_work = true, 43 .cmd.q_control = { 44 .opcode = IONIC_CMD_Q_CONTROL, 45 .type = q->type, 46 .index = rte_cpu_to_le_32(q->index), 47 .oper = IONIC_Q_DISABLE, 48 }, 49 }; 50 51 return ionic_adminq_post_wait(lif, &ctx); 52 } 53 54 void 55 ionic_lif_stop(struct ionic_lif *lif) 56 { 57 uint32_t i; 58 59 IONIC_PRINT_CALL(); 60 61 lif->state &= ~IONIC_LIF_F_UP; 62 63 for (i = 0; i < lif->nrxqcqs; i++) { 64 struct ionic_qcq *rxq = lif->rxqcqs[i]; 65 if (rxq->flags & IONIC_QCQ_F_INITED) 66 (void)ionic_dev_rx_queue_stop(lif->eth_dev, i); 67 } 68 69 for (i = 0; i < lif->ntxqcqs; i++) { 70 struct ionic_qcq *txq = lif->txqcqs[i]; 71 if (txq->flags & IONIC_QCQ_F_INITED) 72 (void)ionic_dev_tx_queue_stop(lif->eth_dev, i); 73 } 74 } 75 76 void 77 ionic_lif_reset(struct ionic_lif *lif) 78 { 79 struct ionic_dev *idev = &lif->adapter->idev; 80 int err; 81 82 IONIC_PRINT_CALL(); 83 84 ionic_dev_cmd_lif_reset(idev); 85 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); 86 if (err) 87 IONIC_PRINT(WARNING, "Failed to reset %s", lif->name); 88 } 89 90 static void 91 ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats) 92 { 93 struct ionic_lif_stats *ls = &lif->info->stats; 94 uint32_t i; 95 uint32_t num_rx_q_counters = RTE_MIN(lif->nrxqcqs, (uint32_t) 96 RTE_ETHDEV_QUEUE_STAT_CNTRS); 97 uint32_t num_tx_q_counters = RTE_MIN(lif->ntxqcqs, (uint32_t) 98 RTE_ETHDEV_QUEUE_STAT_CNTRS); 99 100 memset(stats, 0, sizeof(*stats)); 101 102 if (ls == NULL) { 103 IONIC_PRINT(DEBUG, "Stats on port %u not yet initialized", 104 lif->port_id); 105 return; 106 } 107 108 /* RX */ 109 110 stats->ipackets = ls->rx_ucast_packets + 111 ls->rx_mcast_packets + 112 ls->rx_bcast_packets; 113 114 stats->ibytes = ls->rx_ucast_bytes + 115 ls->rx_mcast_bytes + 116 ls->rx_bcast_bytes; 117 118 for (i = 0; i < lif->nrxqcqs; i++) { 119 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx; 120 stats->imissed += 121 rx_stats->no_cb_arg + 122 rx_stats->bad_cq_status + 123 rx_stats->no_room + 124 rx_stats->bad_len; 125 } 126 127 stats->imissed += 128 ls->rx_ucast_drop_packets + 129 ls->rx_mcast_drop_packets + 130 ls->rx_bcast_drop_packets; 131 132 stats->imissed += 133 ls->rx_queue_empty + 134 ls->rx_dma_error + 135 ls->rx_queue_disabled + 136 ls->rx_desc_fetch_error + 137 ls->rx_desc_data_error; 138 139 for (i = 0; i < num_rx_q_counters; i++) { 140 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx; 141 stats->q_ipackets[i] = rx_stats->packets; 142 stats->q_ibytes[i] = rx_stats->bytes; 143 stats->q_errors[i] = 144 rx_stats->no_cb_arg + 145 rx_stats->bad_cq_status + 146 rx_stats->no_room + 147 rx_stats->bad_len; 148 } 149 150 /* TX */ 151 152 stats->opackets = ls->tx_ucast_packets + 153 ls->tx_mcast_packets + 154 ls->tx_bcast_packets; 155 156 stats->obytes = ls->tx_ucast_bytes + 157 ls->tx_mcast_bytes + 158 ls->tx_bcast_bytes; 159 160 for (i = 0; i < lif->ntxqcqs; i++) { 161 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx; 162 stats->oerrors += tx_stats->drop; 163 } 164 165 stats->oerrors += 166 ls->tx_ucast_drop_packets + 167 ls->tx_mcast_drop_packets + 168 ls->tx_bcast_drop_packets; 169 170 stats->oerrors += 171 ls->tx_dma_error + 172 ls->tx_queue_disabled + 173 ls->tx_desc_fetch_error + 174 ls->tx_desc_data_error; 175 176 for (i = 0; i < num_tx_q_counters; i++) { 177 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx; 178 stats->q_opackets[i] = tx_stats->packets; 179 stats->q_obytes[i] = tx_stats->bytes; 180 } 181 } 182 183 void 184 ionic_lif_get_stats(const struct ionic_lif *lif, 185 struct rte_eth_stats *stats) 186 { 187 ionic_lif_get_abs_stats(lif, stats); 188 189 stats->ipackets -= lif->stats_base.ipackets; 190 stats->opackets -= lif->stats_base.opackets; 191 stats->ibytes -= lif->stats_base.ibytes; 192 stats->obytes -= lif->stats_base.obytes; 193 stats->imissed -= lif->stats_base.imissed; 194 stats->ierrors -= lif->stats_base.ierrors; 195 stats->oerrors -= lif->stats_base.oerrors; 196 stats->rx_nombuf -= lif->stats_base.rx_nombuf; 197 } 198 199 void 200 ionic_lif_reset_stats(struct ionic_lif *lif) 201 { 202 uint32_t i; 203 204 for (i = 0; i < lif->nrxqcqs; i++) { 205 memset(&lif->rxqcqs[i]->stats.rx, 0, 206 sizeof(struct ionic_rx_stats)); 207 memset(&lif->txqcqs[i]->stats.tx, 0, 208 sizeof(struct ionic_tx_stats)); 209 } 210 211 ionic_lif_get_abs_stats(lif, &lif->stats_base); 212 } 213 214 void 215 ionic_lif_get_hw_stats(struct ionic_lif *lif, struct ionic_lif_stats *stats) 216 { 217 uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t); 218 uint64_t *stats64 = (uint64_t *)stats; 219 uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats; 220 uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base; 221 222 for (i = 0; i < count; i++) 223 stats64[i] = lif_stats64[i] - lif_stats64_base[i]; 224 } 225 226 void 227 ionic_lif_reset_hw_stats(struct ionic_lif *lif) 228 { 229 uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t); 230 uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats; 231 uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base; 232 233 for (i = 0; i < count; i++) 234 lif_stats64_base[i] = lif_stats64[i]; 235 } 236 237 static int 238 ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr) 239 { 240 struct ionic_admin_ctx ctx = { 241 .pending_work = true, 242 .cmd.rx_filter_add = { 243 .opcode = IONIC_CMD_RX_FILTER_ADD, 244 .match = rte_cpu_to_le_16(IONIC_RX_FILTER_MATCH_MAC), 245 }, 246 }; 247 int err; 248 249 memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, RTE_ETHER_ADDR_LEN); 250 251 err = ionic_adminq_post_wait(lif, &ctx); 252 if (err) 253 return err; 254 255 IONIC_PRINT(INFO, "rx_filter add (id %d)", 256 rte_le_to_cpu_32(ctx.comp.rx_filter_add.filter_id)); 257 258 return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx); 259 } 260 261 static int 262 ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr) 263 { 264 struct ionic_admin_ctx ctx = { 265 .pending_work = true, 266 .cmd.rx_filter_del = { 267 .opcode = IONIC_CMD_RX_FILTER_DEL, 268 }, 269 }; 270 struct ionic_rx_filter *f; 271 int err; 272 273 IONIC_PRINT_CALL(); 274 275 rte_spinlock_lock(&lif->rx_filters.lock); 276 277 f = ionic_rx_filter_by_addr(lif, addr); 278 if (!f) { 279 rte_spinlock_unlock(&lif->rx_filters.lock); 280 return -ENOENT; 281 } 282 283 ctx.cmd.rx_filter_del.filter_id = rte_cpu_to_le_32(f->filter_id); 284 ionic_rx_filter_free(f); 285 286 rte_spinlock_unlock(&lif->rx_filters.lock); 287 288 err = ionic_adminq_post_wait(lif, &ctx); 289 if (err) 290 return err; 291 292 IONIC_PRINT(INFO, "rx_filter del (id %d)", 293 rte_le_to_cpu_32(ctx.cmd.rx_filter_del.filter_id)); 294 295 return 0; 296 } 297 298 int 299 ionic_dev_add_mac(struct rte_eth_dev *eth_dev, 300 struct rte_ether_addr *mac_addr, 301 uint32_t index __rte_unused, uint32_t pool __rte_unused) 302 { 303 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 304 305 IONIC_PRINT_CALL(); 306 307 return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr); 308 } 309 310 void 311 ionic_dev_remove_mac(struct rte_eth_dev *eth_dev, uint32_t index) 312 { 313 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 314 struct ionic_adapter *adapter = lif->adapter; 315 struct rte_ether_addr *mac_addr; 316 317 IONIC_PRINT_CALL(); 318 319 if (index >= adapter->max_mac_addrs) { 320 IONIC_PRINT(WARNING, 321 "Index %u is above MAC filter limit %u", 322 index, adapter->max_mac_addrs); 323 return; 324 } 325 326 mac_addr = ð_dev->data->mac_addrs[index]; 327 328 if (!rte_is_valid_assigned_ether_addr(mac_addr)) 329 return; 330 331 ionic_lif_addr_del(lif, (const uint8_t *)mac_addr); 332 } 333 334 int 335 ionic_dev_set_mac(struct rte_eth_dev *eth_dev, struct rte_ether_addr *mac_addr) 336 { 337 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 338 339 IONIC_PRINT_CALL(); 340 341 if (mac_addr == NULL) { 342 IONIC_PRINT(NOTICE, "New mac is null"); 343 return -1; 344 } 345 346 if (!rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) { 347 IONIC_PRINT(INFO, "Deleting mac addr %pM", 348 lif->mac_addr); 349 ionic_lif_addr_del(lif, lif->mac_addr); 350 memset(lif->mac_addr, 0, RTE_ETHER_ADDR_LEN); 351 } 352 353 IONIC_PRINT(INFO, "Updating mac addr"); 354 355 rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)lif->mac_addr); 356 357 return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr); 358 } 359 360 static int 361 ionic_vlan_rx_add_vid(struct ionic_lif *lif, uint16_t vid) 362 { 363 struct ionic_admin_ctx ctx = { 364 .pending_work = true, 365 .cmd.rx_filter_add = { 366 .opcode = IONIC_CMD_RX_FILTER_ADD, 367 .match = rte_cpu_to_le_16(IONIC_RX_FILTER_MATCH_VLAN), 368 .vlan.vlan = rte_cpu_to_le_16(vid), 369 }, 370 }; 371 int err; 372 373 err = ionic_adminq_post_wait(lif, &ctx); 374 if (err) 375 return err; 376 377 IONIC_PRINT(INFO, "rx_filter add VLAN %d (id %d)", vid, 378 rte_le_to_cpu_32(ctx.comp.rx_filter_add.filter_id)); 379 380 return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx); 381 } 382 383 static int 384 ionic_vlan_rx_kill_vid(struct ionic_lif *lif, uint16_t vid) 385 { 386 struct ionic_admin_ctx ctx = { 387 .pending_work = true, 388 .cmd.rx_filter_del = { 389 .opcode = IONIC_CMD_RX_FILTER_DEL, 390 }, 391 }; 392 struct ionic_rx_filter *f; 393 int err; 394 395 IONIC_PRINT_CALL(); 396 397 rte_spinlock_lock(&lif->rx_filters.lock); 398 399 f = ionic_rx_filter_by_vlan(lif, vid); 400 if (!f) { 401 rte_spinlock_unlock(&lif->rx_filters.lock); 402 return -ENOENT; 403 } 404 405 ctx.cmd.rx_filter_del.filter_id = rte_cpu_to_le_32(f->filter_id); 406 ionic_rx_filter_free(f); 407 rte_spinlock_unlock(&lif->rx_filters.lock); 408 409 err = ionic_adminq_post_wait(lif, &ctx); 410 if (err) 411 return err; 412 413 IONIC_PRINT(INFO, "rx_filter del VLAN %d (id %d)", vid, 414 rte_le_to_cpu_32(ctx.cmd.rx_filter_del.filter_id)); 415 416 return 0; 417 } 418 419 int 420 ionic_dev_vlan_filter_set(struct rte_eth_dev *eth_dev, uint16_t vlan_id, 421 int on) 422 { 423 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 424 int err; 425 426 if (on) 427 err = ionic_vlan_rx_add_vid(lif, vlan_id); 428 else 429 err = ionic_vlan_rx_kill_vid(lif, vlan_id); 430 431 return err; 432 } 433 434 static void 435 ionic_lif_rx_mode(struct ionic_lif *lif, uint32_t rx_mode) 436 { 437 struct ionic_admin_ctx ctx = { 438 .pending_work = true, 439 .cmd.rx_mode_set = { 440 .opcode = IONIC_CMD_RX_MODE_SET, 441 .rx_mode = rte_cpu_to_le_16(rx_mode), 442 }, 443 }; 444 int err; 445 446 if (rx_mode & IONIC_RX_MODE_F_UNICAST) 447 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_UNICAST"); 448 if (rx_mode & IONIC_RX_MODE_F_MULTICAST) 449 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_MULTICAST"); 450 if (rx_mode & IONIC_RX_MODE_F_BROADCAST) 451 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_BROADCAST"); 452 if (rx_mode & IONIC_RX_MODE_F_PROMISC) 453 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_PROMISC"); 454 if (rx_mode & IONIC_RX_MODE_F_ALLMULTI) 455 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_ALLMULTI"); 456 457 err = ionic_adminq_post_wait(lif, &ctx); 458 if (err) 459 IONIC_PRINT(ERR, "Failure setting RX mode"); 460 } 461 462 static void 463 ionic_set_rx_mode(struct ionic_lif *lif, uint32_t rx_mode) 464 { 465 if (lif->rx_mode != rx_mode) { 466 lif->rx_mode = rx_mode; 467 ionic_lif_rx_mode(lif, rx_mode); 468 } 469 } 470 471 int 472 ionic_dev_promiscuous_enable(struct rte_eth_dev *eth_dev) 473 { 474 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 475 uint32_t rx_mode = lif->rx_mode; 476 477 IONIC_PRINT_CALL(); 478 479 rx_mode |= IONIC_RX_MODE_F_PROMISC; 480 481 ionic_set_rx_mode(lif, rx_mode); 482 483 return 0; 484 } 485 486 int 487 ionic_dev_promiscuous_disable(struct rte_eth_dev *eth_dev) 488 { 489 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 490 uint32_t rx_mode = lif->rx_mode; 491 492 rx_mode &= ~IONIC_RX_MODE_F_PROMISC; 493 494 ionic_set_rx_mode(lif, rx_mode); 495 496 return 0; 497 } 498 499 int 500 ionic_dev_allmulticast_enable(struct rte_eth_dev *eth_dev) 501 { 502 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 503 uint32_t rx_mode = lif->rx_mode; 504 505 rx_mode |= IONIC_RX_MODE_F_ALLMULTI; 506 507 ionic_set_rx_mode(lif, rx_mode); 508 509 return 0; 510 } 511 512 int 513 ionic_dev_allmulticast_disable(struct rte_eth_dev *eth_dev) 514 { 515 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 516 uint32_t rx_mode = lif->rx_mode; 517 518 rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI; 519 520 ionic_set_rx_mode(lif, rx_mode); 521 522 return 0; 523 } 524 525 int 526 ionic_lif_change_mtu(struct ionic_lif *lif, int new_mtu) 527 { 528 struct ionic_admin_ctx ctx = { 529 .pending_work = true, 530 .cmd.lif_setattr = { 531 .opcode = IONIC_CMD_LIF_SETATTR, 532 .attr = IONIC_LIF_ATTR_MTU, 533 .mtu = rte_cpu_to_le_32(new_mtu), 534 }, 535 }; 536 int err; 537 538 err = ionic_adminq_post_wait(lif, &ctx); 539 if (err) 540 return err; 541 542 return 0; 543 } 544 545 int 546 ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr) 547 { 548 struct ionic_adapter *adapter = lif->adapter; 549 struct ionic_dev *idev = &adapter->idev; 550 unsigned long index; 551 552 /* 553 * Note: interrupt handler is called for index = 0 only 554 * (we use interrupts for the notifyq only anyway, 555 * which has index = 0) 556 */ 557 558 for (index = 0; index < adapter->nintrs; index++) 559 if (!adapter->intrs[index]) 560 break; 561 562 if (index == adapter->nintrs) 563 return -ENOSPC; 564 565 adapter->intrs[index] = true; 566 567 ionic_intr_init(idev, intr, index); 568 569 return 0; 570 } 571 572 void 573 ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr) 574 { 575 if (intr->index != IONIC_INTR_NONE) 576 lif->adapter->intrs[intr->index] = false; 577 } 578 579 static int 580 ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type, 581 uint32_t index, 582 const char *base, uint32_t flags, 583 uint32_t num_descs, 584 uint32_t desc_size, 585 uint32_t cq_desc_size, 586 uint32_t sg_desc_size, 587 struct ionic_qcq **qcq) 588 { 589 struct ionic_dev *idev = &lif->adapter->idev; 590 struct ionic_qcq *new; 591 uint32_t q_size, cq_size, sg_size, total_size; 592 void *q_base, *cq_base, *sg_base; 593 rte_iova_t q_base_pa = 0; 594 rte_iova_t cq_base_pa = 0; 595 rte_iova_t sg_base_pa = 0; 596 uint32_t socket_id = rte_socket_id(); 597 int err; 598 599 *qcq = NULL; 600 601 q_size = num_descs * desc_size; 602 cq_size = num_descs * cq_desc_size; 603 sg_size = num_descs * sg_desc_size; 604 605 total_size = RTE_ALIGN(q_size, PAGE_SIZE) + 606 RTE_ALIGN(cq_size, PAGE_SIZE); 607 /* 608 * Note: aligning q_size/cq_size is not enough due to cq_base address 609 * aligning as q_base could be not aligned to the page. 610 * Adding PAGE_SIZE. 611 */ 612 total_size += PAGE_SIZE; 613 614 if (flags & IONIC_QCQ_F_SG) { 615 total_size += RTE_ALIGN(sg_size, PAGE_SIZE); 616 total_size += PAGE_SIZE; 617 } 618 619 new = rte_zmalloc("ionic", sizeof(*new), 0); 620 if (!new) { 621 IONIC_PRINT(ERR, "Cannot allocate queue structure"); 622 return -ENOMEM; 623 } 624 625 new->lif = lif; 626 new->flags = flags; 627 628 new->q.info = rte_zmalloc("ionic", sizeof(*new->q.info) * num_descs, 0); 629 if (!new->q.info) { 630 IONIC_PRINT(ERR, "Cannot allocate queue info"); 631 err = -ENOMEM; 632 goto err_out_free_qcq; 633 } 634 635 new->q.type = type; 636 637 err = ionic_q_init(lif, idev, &new->q, index, num_descs, 638 desc_size, sg_desc_size); 639 if (err) { 640 IONIC_PRINT(ERR, "Queue initialization failed"); 641 goto err_out_free_info; 642 } 643 644 err = ionic_cq_init(lif, &new->cq, num_descs, cq_desc_size); 645 if (err) { 646 IONIC_PRINT(ERR, "Completion queue initialization failed"); 647 goto err_out_free_info; 648 } 649 650 new->base_z = rte_eth_dma_zone_reserve(lif->eth_dev, 651 base /* name */, index /* queue_idx */, 652 total_size, IONIC_ALIGN, socket_id); 653 654 if (!new->base_z) { 655 IONIC_PRINT(ERR, "Cannot reserve queue DMA memory"); 656 err = -ENOMEM; 657 goto err_out_free_info; 658 } 659 660 new->base = new->base_z->addr; 661 new->base_pa = new->base_z->iova; 662 new->total_size = total_size; 663 664 q_base = new->base; 665 q_base_pa = new->base_pa; 666 667 cq_base = (void *)RTE_ALIGN((uintptr_t)q_base + q_size, PAGE_SIZE); 668 cq_base_pa = RTE_ALIGN(q_base_pa + q_size, PAGE_SIZE); 669 670 if (flags & IONIC_QCQ_F_SG) { 671 sg_base = (void *)RTE_ALIGN((uintptr_t)cq_base + cq_size, 672 PAGE_SIZE); 673 sg_base_pa = RTE_ALIGN(cq_base_pa + cq_size, PAGE_SIZE); 674 ionic_q_sg_map(&new->q, sg_base, sg_base_pa); 675 } 676 677 IONIC_PRINT(DEBUG, "Q-Base-PA = %#jx CQ-Base-PA = %#jx " 678 "SG-base-PA = %#jx", 679 q_base_pa, cq_base_pa, sg_base_pa); 680 681 ionic_q_map(&new->q, q_base, q_base_pa); 682 ionic_cq_map(&new->cq, cq_base, cq_base_pa); 683 ionic_cq_bind(&new->cq, &new->q); 684 685 *qcq = new; 686 687 return 0; 688 689 err_out_free_info: 690 rte_free(new->q.info); 691 err_out_free_qcq: 692 rte_free(new); 693 694 return err; 695 } 696 697 void 698 ionic_qcq_free(struct ionic_qcq *qcq) 699 { 700 if (qcq->base_z) { 701 qcq->base = NULL; 702 qcq->base_pa = 0; 703 rte_memzone_free(qcq->base_z); 704 qcq->base_z = NULL; 705 } 706 707 if (qcq->q.info) { 708 rte_free(qcq->q.info); 709 qcq->q.info = NULL; 710 } 711 712 rte_free(qcq); 713 } 714 715 int 716 ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs, 717 struct ionic_qcq **qcq) 718 { 719 uint32_t flags; 720 int err = -ENOMEM; 721 722 flags = IONIC_QCQ_F_SG; 723 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, index, "rx", flags, 724 nrxq_descs, 725 sizeof(struct ionic_rxq_desc), 726 sizeof(struct ionic_rxq_comp), 727 sizeof(struct ionic_rxq_sg_desc), 728 &lif->rxqcqs[index]); 729 if (err) 730 return err; 731 732 *qcq = lif->rxqcqs[index]; 733 734 return 0; 735 } 736 737 int 738 ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs, 739 struct ionic_qcq **qcq) 740 { 741 uint32_t flags; 742 int err = -ENOMEM; 743 744 flags = IONIC_QCQ_F_SG; 745 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, index, "tx", flags, 746 ntxq_descs, 747 sizeof(struct ionic_txq_desc), 748 sizeof(struct ionic_txq_comp), 749 sizeof(struct ionic_txq_sg_desc), 750 &lif->txqcqs[index]); 751 if (err) 752 return err; 753 754 *qcq = lif->txqcqs[index]; 755 756 return 0; 757 } 758 759 static int 760 ionic_admin_qcq_alloc(struct ionic_lif *lif) 761 { 762 uint32_t flags; 763 int err = -ENOMEM; 764 765 flags = 0; 766 err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags, 767 IONIC_ADMINQ_LENGTH, 768 sizeof(struct ionic_admin_cmd), 769 sizeof(struct ionic_admin_comp), 770 0, 771 &lif->adminqcq); 772 if (err) 773 return err; 774 775 return 0; 776 } 777 778 static int 779 ionic_notify_qcq_alloc(struct ionic_lif *lif) 780 { 781 struct ionic_qcq *nqcq; 782 struct ionic_dev *idev = &lif->adapter->idev; 783 uint32_t flags = 0; 784 int err = -ENOMEM; 785 786 err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notify", 787 flags, 788 IONIC_NOTIFYQ_LENGTH, 789 sizeof(struct ionic_notifyq_cmd), 790 sizeof(union ionic_notifyq_comp), 791 0, 792 &nqcq); 793 if (err) 794 return err; 795 796 err = ionic_intr_alloc(lif, &nqcq->intr); 797 if (err) { 798 ionic_qcq_free(nqcq); 799 return err; 800 } 801 802 ionic_intr_mask_assert(idev->intr_ctrl, nqcq->intr.index, 803 IONIC_INTR_MASK_SET); 804 805 lif->notifyqcq = nqcq; 806 807 return 0; 808 } 809 810 static void * 811 ionic_bus_map_dbpage(struct ionic_adapter *adapter, int page_num) 812 { 813 char *vaddr = adapter->bars[IONIC_PCI_BAR_DBELL].vaddr; 814 815 if (adapter->num_bars <= IONIC_PCI_BAR_DBELL) 816 return NULL; 817 818 return (void *)&vaddr[page_num << PAGE_SHIFT]; 819 } 820 821 int 822 ionic_lif_alloc(struct ionic_lif *lif) 823 { 824 struct ionic_adapter *adapter = lif->adapter; 825 uint32_t socket_id = rte_socket_id(); 826 int err; 827 828 /* 829 * lif->name was zeroed on allocation. 830 * Copy (sizeof() - 1) bytes to ensure that it is NULL terminated. 831 */ 832 memcpy(lif->name, lif->eth_dev->data->name, sizeof(lif->name) - 1); 833 834 IONIC_PRINT(DEBUG, "LIF: %s", lif->name); 835 836 IONIC_PRINT(DEBUG, "Allocating Lif Info"); 837 838 rte_spinlock_init(&lif->adminq_lock); 839 rte_spinlock_init(&lif->adminq_service_lock); 840 841 lif->kern_dbpage = ionic_bus_map_dbpage(adapter, 0); 842 if (!lif->kern_dbpage) { 843 IONIC_PRINT(ERR, "Cannot map dbpage, aborting"); 844 return -ENOMEM; 845 } 846 847 lif->txqcqs = rte_zmalloc("ionic", sizeof(*lif->txqcqs) * 848 adapter->max_ntxqs_per_lif, 0); 849 850 if (!lif->txqcqs) { 851 IONIC_PRINT(ERR, "Cannot allocate tx queues array"); 852 return -ENOMEM; 853 } 854 855 lif->rxqcqs = rte_zmalloc("ionic", sizeof(*lif->rxqcqs) * 856 adapter->max_nrxqs_per_lif, 0); 857 858 if (!lif->rxqcqs) { 859 IONIC_PRINT(ERR, "Cannot allocate rx queues array"); 860 return -ENOMEM; 861 } 862 863 IONIC_PRINT(DEBUG, "Allocating Notify Queue"); 864 865 err = ionic_notify_qcq_alloc(lif); 866 if (err) { 867 IONIC_PRINT(ERR, "Cannot allocate notify queue"); 868 return err; 869 } 870 871 IONIC_PRINT(DEBUG, "Allocating Admin Queue"); 872 873 err = ionic_admin_qcq_alloc(lif); 874 if (err) { 875 IONIC_PRINT(ERR, "Cannot allocate admin queue"); 876 return err; 877 } 878 879 IONIC_PRINT(DEBUG, "Allocating Lif Info"); 880 881 lif->info_sz = RTE_ALIGN(sizeof(*lif->info), PAGE_SIZE); 882 883 lif->info_z = rte_eth_dma_zone_reserve(lif->eth_dev, 884 "lif_info", 0 /* queue_idx*/, 885 lif->info_sz, IONIC_ALIGN, socket_id); 886 if (!lif->info_z) { 887 IONIC_PRINT(ERR, "Cannot allocate lif info memory"); 888 return -ENOMEM; 889 } 890 891 lif->info = lif->info_z->addr; 892 lif->info_pa = lif->info_z->iova; 893 894 return 0; 895 } 896 897 void 898 ionic_lif_free(struct ionic_lif *lif) 899 { 900 if (lif->notifyqcq) { 901 ionic_qcq_free(lif->notifyqcq); 902 lif->notifyqcq = NULL; 903 } 904 905 if (lif->adminqcq) { 906 ionic_qcq_free(lif->adminqcq); 907 lif->adminqcq = NULL; 908 } 909 910 if (lif->txqcqs) { 911 rte_free(lif->txqcqs); 912 lif->txqcqs = NULL; 913 } 914 915 if (lif->rxqcqs) { 916 rte_free(lif->rxqcqs); 917 lif->rxqcqs = NULL; 918 } 919 920 if (lif->info) { 921 rte_memzone_free(lif->info_z); 922 lif->info = NULL; 923 } 924 } 925 926 void 927 ionic_lif_free_queues(struct ionic_lif *lif) 928 { 929 uint32_t i; 930 931 for (i = 0; i < lif->ntxqcqs; i++) { 932 ionic_dev_tx_queue_release(lif->eth_dev->data->tx_queues[i]); 933 lif->eth_dev->data->tx_queues[i] = NULL; 934 } 935 for (i = 0; i < lif->nrxqcqs; i++) { 936 ionic_dev_rx_queue_release(lif->eth_dev->data->rx_queues[i]); 937 lif->eth_dev->data->rx_queues[i] = NULL; 938 } 939 } 940 941 int 942 ionic_lif_rss_config(struct ionic_lif *lif, 943 const uint16_t types, const uint8_t *key, const uint32_t *indir) 944 { 945 struct ionic_adapter *adapter = lif->adapter; 946 struct ionic_admin_ctx ctx = { 947 .pending_work = true, 948 .cmd.lif_setattr = { 949 .opcode = IONIC_CMD_LIF_SETATTR, 950 .attr = IONIC_LIF_ATTR_RSS, 951 .rss.types = rte_cpu_to_le_16(types), 952 .rss.addr = rte_cpu_to_le_64(lif->rss_ind_tbl_pa), 953 }, 954 }; 955 unsigned int i; 956 uint16_t tbl_sz = 957 rte_le_to_cpu_16(adapter->ident.lif.eth.rss_ind_tbl_sz); 958 959 IONIC_PRINT_CALL(); 960 961 lif->rss_types = types; 962 963 if (key) 964 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE); 965 966 if (indir) 967 for (i = 0; i < tbl_sz; i++) 968 lif->rss_ind_tbl[i] = indir[i]; 969 970 memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key, 971 IONIC_RSS_HASH_KEY_SIZE); 972 973 return ionic_adminq_post_wait(lif, &ctx); 974 } 975 976 static int 977 ionic_lif_rss_setup(struct ionic_lif *lif) 978 { 979 struct ionic_adapter *adapter = lif->adapter; 980 static const uint8_t toeplitz_symmetric_key[] = { 981 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 982 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 983 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 984 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 985 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 986 }; 987 uint32_t i; 988 uint16_t tbl_sz = 989 rte_le_to_cpu_16(adapter->ident.lif.eth.rss_ind_tbl_sz); 990 991 IONIC_PRINT_CALL(); 992 993 if (!lif->rss_ind_tbl_z) { 994 lif->rss_ind_tbl_z = rte_eth_dma_zone_reserve(lif->eth_dev, 995 "rss_ind_tbl", 0 /* queue_idx */, 996 sizeof(*lif->rss_ind_tbl) * tbl_sz, 997 IONIC_ALIGN, rte_socket_id()); 998 if (!lif->rss_ind_tbl_z) { 999 IONIC_PRINT(ERR, "OOM"); 1000 return -ENOMEM; 1001 } 1002 1003 lif->rss_ind_tbl = lif->rss_ind_tbl_z->addr; 1004 lif->rss_ind_tbl_pa = lif->rss_ind_tbl_z->iova; 1005 } 1006 1007 if (lif->rss_ind_tbl_nrxqcqs != lif->nrxqcqs) { 1008 lif->rss_ind_tbl_nrxqcqs = lif->nrxqcqs; 1009 1010 /* Fill indirection table with 'default' values */ 1011 for (i = 0; i < tbl_sz; i++) 1012 lif->rss_ind_tbl[i] = i % lif->nrxqcqs; 1013 } 1014 1015 return ionic_lif_rss_config(lif, IONIC_RSS_OFFLOAD_ALL, 1016 toeplitz_symmetric_key, NULL); 1017 } 1018 1019 static void 1020 ionic_lif_rss_teardown(struct ionic_lif *lif) 1021 { 1022 if (!lif->rss_ind_tbl) 1023 return; 1024 1025 if (lif->rss_ind_tbl_z) { 1026 /* Disable RSS on the NIC */ 1027 ionic_lif_rss_config(lif, 0x0, NULL, NULL); 1028 1029 lif->rss_ind_tbl = NULL; 1030 lif->rss_ind_tbl_pa = 0; 1031 rte_memzone_free(lif->rss_ind_tbl_z); 1032 lif->rss_ind_tbl_z = NULL; 1033 } 1034 } 1035 1036 static void 1037 ionic_lif_qcq_deinit(struct ionic_qcq *qcq) 1038 { 1039 qcq->flags &= ~IONIC_QCQ_F_INITED; 1040 } 1041 1042 void 1043 ionic_lif_txq_deinit(struct ionic_qcq *qcq) 1044 { 1045 ionic_lif_qcq_deinit(qcq); 1046 } 1047 1048 void 1049 ionic_lif_rxq_deinit(struct ionic_qcq *qcq) 1050 { 1051 ionic_lif_qcq_deinit(qcq); 1052 } 1053 1054 static void 1055 ionic_lif_notifyq_deinit(struct ionic_lif *lif) 1056 { 1057 struct ionic_qcq *nqcq = lif->notifyqcq; 1058 struct ionic_dev *idev = &lif->adapter->idev; 1059 1060 if (!(nqcq->flags & IONIC_QCQ_F_INITED)) 1061 return; 1062 1063 ionic_intr_mask(idev->intr_ctrl, nqcq->intr.index, 1064 IONIC_INTR_MASK_SET); 1065 1066 nqcq->flags &= ~IONIC_QCQ_F_INITED; 1067 } 1068 1069 bool 1070 ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index, 1071 void *cb_arg __rte_unused) 1072 { 1073 struct ionic_admin_comp *cq_desc_base = cq->base; 1074 struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index]; 1075 1076 if (!color_match(cq_desc->color, cq->done_color)) 1077 return false; 1078 1079 ionic_q_service(cq->bound_q, cq_desc_index, cq_desc->comp_index, NULL); 1080 1081 return true; 1082 } 1083 1084 /* This acts like ionic_napi */ 1085 int 1086 ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb, 1087 void *cb_arg) 1088 { 1089 struct ionic_cq *cq = &qcq->cq; 1090 uint32_t work_done; 1091 1092 work_done = ionic_cq_service(cq, budget, cb, cb_arg); 1093 1094 return work_done; 1095 } 1096 1097 static void 1098 ionic_link_status_check(struct ionic_lif *lif) 1099 { 1100 struct ionic_adapter *adapter = lif->adapter; 1101 bool link_up; 1102 1103 lif->state &= ~IONIC_LIF_F_LINK_CHECK_NEEDED; 1104 1105 if (!lif->info) 1106 return; 1107 1108 link_up = (lif->info->status.link_status == IONIC_PORT_OPER_STATUS_UP); 1109 1110 if ((link_up && adapter->link_up) || 1111 (!link_up && !adapter->link_up)) 1112 return; 1113 1114 if (link_up) { 1115 adapter->link_speed = 1116 rte_le_to_cpu_32(lif->info->status.link_speed); 1117 IONIC_PRINT(DEBUG, "Link up - %d Gbps", 1118 adapter->link_speed); 1119 } else { 1120 IONIC_PRINT(DEBUG, "Link down"); 1121 } 1122 1123 adapter->link_up = link_up; 1124 ionic_dev_link_update(lif->eth_dev, 0); 1125 } 1126 1127 static void 1128 ionic_lif_handle_fw_down(struct ionic_lif *lif) 1129 { 1130 if (lif->state & IONIC_LIF_F_FW_RESET) 1131 return; 1132 1133 lif->state |= IONIC_LIF_F_FW_RESET; 1134 1135 if (lif->state & IONIC_LIF_F_UP) { 1136 IONIC_PRINT(NOTICE, 1137 "Surprise FW stop, stopping %s\n", lif->name); 1138 ionic_lif_stop(lif); 1139 } 1140 1141 IONIC_PRINT(NOTICE, "FW down, %s stopped", lif->name); 1142 } 1143 1144 static bool 1145 ionic_notifyq_cb(struct ionic_cq *cq, uint32_t cq_desc_index, void *cb_arg) 1146 { 1147 union ionic_notifyq_comp *cq_desc_base = cq->base; 1148 union ionic_notifyq_comp *cq_desc = &cq_desc_base[cq_desc_index]; 1149 struct ionic_lif *lif = cb_arg; 1150 1151 IONIC_PRINT(DEBUG, "Notifyq callback eid = %jd ecode = %d", 1152 cq_desc->event.eid, cq_desc->event.ecode); 1153 1154 /* Have we run out of new completions to process? */ 1155 if (!(cq_desc->event.eid > lif->last_eid)) 1156 return false; 1157 1158 lif->last_eid = cq_desc->event.eid; 1159 1160 switch (cq_desc->event.ecode) { 1161 case IONIC_EVENT_LINK_CHANGE: 1162 IONIC_PRINT(DEBUG, 1163 "Notifyq IONIC_EVENT_LINK_CHANGE %s " 1164 "eid=%jd link_status=%d link_speed=%d", 1165 lif->name, 1166 cq_desc->event.eid, 1167 cq_desc->link_change.link_status, 1168 cq_desc->link_change.link_speed); 1169 1170 lif->state |= IONIC_LIF_F_LINK_CHECK_NEEDED; 1171 break; 1172 1173 case IONIC_EVENT_RESET: 1174 IONIC_PRINT(NOTICE, 1175 "Notifyq IONIC_EVENT_RESET %s " 1176 "eid=%jd, reset_code=%d state=%d", 1177 lif->name, 1178 cq_desc->event.eid, 1179 cq_desc->reset.reset_code, 1180 cq_desc->reset.state); 1181 ionic_lif_handle_fw_down(lif); 1182 break; 1183 1184 default: 1185 IONIC_PRINT(WARNING, "Notifyq bad event ecode=%d eid=%jd", 1186 cq_desc->event.ecode, cq_desc->event.eid); 1187 break; 1188 } 1189 1190 return true; 1191 } 1192 1193 int 1194 ionic_notifyq_handler(struct ionic_lif *lif, int budget) 1195 { 1196 struct ionic_dev *idev = &lif->adapter->idev; 1197 struct ionic_qcq *qcq = lif->notifyqcq; 1198 uint32_t work_done; 1199 1200 if (!(qcq->flags & IONIC_QCQ_F_INITED)) { 1201 IONIC_PRINT(DEBUG, "Notifyq not yet initialized"); 1202 return -1; 1203 } 1204 1205 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, 1206 IONIC_INTR_MASK_SET); 1207 1208 work_done = ionic_qcq_service(qcq, budget, ionic_notifyq_cb, lif); 1209 1210 if (lif->state & IONIC_LIF_F_LINK_CHECK_NEEDED) 1211 ionic_link_status_check(lif); 1212 1213 ionic_intr_credits(idev->intr_ctrl, qcq->intr.index, 1214 work_done, IONIC_INTR_CRED_RESET_COALESCE); 1215 1216 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, 1217 IONIC_INTR_MASK_CLEAR); 1218 1219 return 0; 1220 } 1221 1222 static int 1223 ionic_lif_adminq_init(struct ionic_lif *lif) 1224 { 1225 struct ionic_dev *idev = &lif->adapter->idev; 1226 struct ionic_qcq *qcq = lif->adminqcq; 1227 struct ionic_queue *q = &qcq->q; 1228 struct ionic_q_init_comp comp; 1229 int err; 1230 1231 ionic_dev_cmd_adminq_init(idev, qcq); 1232 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); 1233 if (err) 1234 return err; 1235 1236 ionic_dev_cmd_comp(idev, &comp); 1237 1238 q->hw_type = comp.hw_type; 1239 q->hw_index = rte_le_to_cpu_32(comp.hw_index); 1240 q->db = ionic_db_map(lif, q); 1241 1242 IONIC_PRINT(DEBUG, "adminq->hw_type %d", q->hw_type); 1243 IONIC_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index); 1244 IONIC_PRINT(DEBUG, "adminq->db %p", q->db); 1245 1246 qcq->flags |= IONIC_QCQ_F_INITED; 1247 1248 return 0; 1249 } 1250 1251 static int 1252 ionic_lif_notifyq_init(struct ionic_lif *lif) 1253 { 1254 struct ionic_dev *idev = &lif->adapter->idev; 1255 struct ionic_qcq *qcq = lif->notifyqcq; 1256 struct ionic_queue *q = &qcq->q; 1257 int err; 1258 1259 struct ionic_admin_ctx ctx = { 1260 .pending_work = true, 1261 .cmd.q_init = { 1262 .opcode = IONIC_CMD_Q_INIT, 1263 .type = q->type, 1264 .index = rte_cpu_to_le_32(q->index), 1265 .intr_index = rte_cpu_to_le_16(qcq->intr.index), 1266 .flags = rte_cpu_to_le_16(IONIC_QINIT_F_IRQ | 1267 IONIC_QINIT_F_ENA), 1268 .ring_size = rte_log2_u32(q->num_descs), 1269 .ring_base = rte_cpu_to_le_64(q->base_pa), 1270 } 1271 }; 1272 1273 IONIC_PRINT(DEBUG, "notifyq_init.index %d", q->index); 1274 IONIC_PRINT(DEBUG, "notifyq_init.ring_base 0x%" PRIx64 "", q->base_pa); 1275 IONIC_PRINT(DEBUG, "notifyq_init.ring_size %d", 1276 ctx.cmd.q_init.ring_size); 1277 IONIC_PRINT(DEBUG, "notifyq_init.ver %u", ctx.cmd.q_init.ver); 1278 1279 err = ionic_adminq_post_wait(lif, &ctx); 1280 if (err) 1281 return err; 1282 1283 q->hw_type = ctx.comp.q_init.hw_type; 1284 q->hw_index = rte_le_to_cpu_32(ctx.comp.q_init.hw_index); 1285 q->db = NULL; 1286 1287 IONIC_PRINT(DEBUG, "notifyq->hw_type %d", q->hw_type); 1288 IONIC_PRINT(DEBUG, "notifyq->hw_index %d", q->hw_index); 1289 IONIC_PRINT(DEBUG, "notifyq->db %p", q->db); 1290 1291 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, 1292 IONIC_INTR_MASK_CLEAR); 1293 1294 qcq->flags |= IONIC_QCQ_F_INITED; 1295 1296 return 0; 1297 } 1298 1299 int 1300 ionic_lif_set_features(struct ionic_lif *lif) 1301 { 1302 struct ionic_admin_ctx ctx = { 1303 .pending_work = true, 1304 .cmd.lif_setattr = { 1305 .opcode = IONIC_CMD_LIF_SETATTR, 1306 .attr = IONIC_LIF_ATTR_FEATURES, 1307 .features = rte_cpu_to_le_64(lif->features), 1308 }, 1309 }; 1310 int err; 1311 1312 err = ionic_adminq_post_wait(lif, &ctx); 1313 if (err) 1314 return err; 1315 1316 lif->hw_features = rte_le_to_cpu_64(ctx.cmd.lif_setattr.features & 1317 ctx.comp.lif_setattr.features); 1318 1319 if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG) 1320 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_TX_TAG"); 1321 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP) 1322 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_STRIP"); 1323 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER) 1324 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_FILTER"); 1325 if (lif->hw_features & IONIC_ETH_HW_RX_HASH) 1326 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_HASH"); 1327 if (lif->hw_features & IONIC_ETH_HW_TX_SG) 1328 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_SG"); 1329 if (lif->hw_features & IONIC_ETH_HW_RX_SG) 1330 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_SG"); 1331 if (lif->hw_features & IONIC_ETH_HW_TX_CSUM) 1332 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_CSUM"); 1333 if (lif->hw_features & IONIC_ETH_HW_RX_CSUM) 1334 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_CSUM"); 1335 if (lif->hw_features & IONIC_ETH_HW_TSO) 1336 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO"); 1337 if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6) 1338 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPV6"); 1339 if (lif->hw_features & IONIC_ETH_HW_TSO_ECN) 1340 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_ECN"); 1341 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE) 1342 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE"); 1343 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM) 1344 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE_CSUM"); 1345 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4) 1346 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP4"); 1347 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6) 1348 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP6"); 1349 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP) 1350 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP"); 1351 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM) 1352 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP_CSUM"); 1353 1354 return 0; 1355 } 1356 1357 int 1358 ionic_lif_txq_init(struct ionic_qcq *qcq) 1359 { 1360 struct ionic_queue *q = &qcq->q; 1361 struct ionic_lif *lif = qcq->lif; 1362 struct ionic_cq *cq = &qcq->cq; 1363 struct ionic_admin_ctx ctx = { 1364 .pending_work = true, 1365 .cmd.q_init = { 1366 .opcode = IONIC_CMD_Q_INIT, 1367 .type = q->type, 1368 .index = rte_cpu_to_le_32(q->index), 1369 .flags = rte_cpu_to_le_16(IONIC_QINIT_F_SG | 1370 IONIC_QINIT_F_ENA), 1371 .intr_index = rte_cpu_to_le_16(IONIC_INTR_NONE), 1372 .ring_size = rte_log2_u32(q->num_descs), 1373 .ring_base = rte_cpu_to_le_64(q->base_pa), 1374 .cq_ring_base = rte_cpu_to_le_64(cq->base_pa), 1375 .sg_ring_base = rte_cpu_to_le_64(q->sg_base_pa), 1376 }, 1377 }; 1378 int err; 1379 1380 IONIC_PRINT(DEBUG, "txq_init.index %d", q->index); 1381 IONIC_PRINT(DEBUG, "txq_init.ring_base 0x%" PRIx64 "", q->base_pa); 1382 IONIC_PRINT(DEBUG, "txq_init.ring_size %d", 1383 ctx.cmd.q_init.ring_size); 1384 IONIC_PRINT(DEBUG, "txq_init.ver %u", ctx.cmd.q_init.ver); 1385 1386 err = ionic_adminq_post_wait(qcq->lif, &ctx); 1387 if (err) 1388 return err; 1389 1390 q->hw_type = ctx.comp.q_init.hw_type; 1391 q->hw_index = rte_le_to_cpu_32(ctx.comp.q_init.hw_index); 1392 q->db = ionic_db_map(lif, q); 1393 1394 IONIC_PRINT(DEBUG, "txq->hw_type %d", q->hw_type); 1395 IONIC_PRINT(DEBUG, "txq->hw_index %d", q->hw_index); 1396 IONIC_PRINT(DEBUG, "txq->db %p", q->db); 1397 1398 qcq->flags |= IONIC_QCQ_F_INITED; 1399 1400 return 0; 1401 } 1402 1403 int 1404 ionic_lif_rxq_init(struct ionic_qcq *qcq) 1405 { 1406 struct ionic_queue *q = &qcq->q; 1407 struct ionic_lif *lif = qcq->lif; 1408 struct ionic_cq *cq = &qcq->cq; 1409 struct ionic_admin_ctx ctx = { 1410 .pending_work = true, 1411 .cmd.q_init = { 1412 .opcode = IONIC_CMD_Q_INIT, 1413 .type = q->type, 1414 .index = rte_cpu_to_le_32(q->index), 1415 .flags = rte_cpu_to_le_16(IONIC_QINIT_F_SG | 1416 IONIC_QINIT_F_ENA), 1417 .intr_index = rte_cpu_to_le_16(IONIC_INTR_NONE), 1418 .ring_size = rte_log2_u32(q->num_descs), 1419 .ring_base = rte_cpu_to_le_64(q->base_pa), 1420 .cq_ring_base = rte_cpu_to_le_64(cq->base_pa), 1421 .sg_ring_base = rte_cpu_to_le_64(q->sg_base_pa), 1422 }, 1423 }; 1424 int err; 1425 1426 IONIC_PRINT(DEBUG, "rxq_init.index %d", q->index); 1427 IONIC_PRINT(DEBUG, "rxq_init.ring_base 0x%" PRIx64 "", q->base_pa); 1428 IONIC_PRINT(DEBUG, "rxq_init.ring_size %d", 1429 ctx.cmd.q_init.ring_size); 1430 IONIC_PRINT(DEBUG, "rxq_init.ver %u", ctx.cmd.q_init.ver); 1431 1432 err = ionic_adminq_post_wait(qcq->lif, &ctx); 1433 if (err) 1434 return err; 1435 1436 q->hw_type = ctx.comp.q_init.hw_type; 1437 q->hw_index = rte_le_to_cpu_32(ctx.comp.q_init.hw_index); 1438 q->db = ionic_db_map(lif, q); 1439 1440 qcq->flags |= IONIC_QCQ_F_INITED; 1441 1442 IONIC_PRINT(DEBUG, "rxq->hw_type %d", q->hw_type); 1443 IONIC_PRINT(DEBUG, "rxq->hw_index %d", q->hw_index); 1444 IONIC_PRINT(DEBUG, "rxq->db %p", q->db); 1445 1446 return 0; 1447 } 1448 1449 static int 1450 ionic_station_set(struct ionic_lif *lif) 1451 { 1452 struct ionic_admin_ctx ctx = { 1453 .pending_work = true, 1454 .cmd.lif_getattr = { 1455 .opcode = IONIC_CMD_LIF_GETATTR, 1456 .attr = IONIC_LIF_ATTR_MAC, 1457 }, 1458 }; 1459 int err; 1460 1461 IONIC_PRINT_CALL(); 1462 1463 err = ionic_adminq_post_wait(lif, &ctx); 1464 if (err) 1465 return err; 1466 1467 memcpy(lif->mac_addr, ctx.comp.lif_getattr.mac, RTE_ETHER_ADDR_LEN); 1468 1469 return 0; 1470 } 1471 1472 static void 1473 ionic_lif_set_name(struct ionic_lif *lif) 1474 { 1475 struct ionic_admin_ctx ctx = { 1476 .pending_work = true, 1477 .cmd.lif_setattr = { 1478 .opcode = IONIC_CMD_LIF_SETATTR, 1479 .attr = IONIC_LIF_ATTR_NAME, 1480 }, 1481 }; 1482 1483 memcpy(ctx.cmd.lif_setattr.name, lif->name, 1484 sizeof(ctx.cmd.lif_setattr.name) - 1); 1485 1486 ionic_adminq_post_wait(lif, &ctx); 1487 } 1488 1489 int 1490 ionic_lif_init(struct ionic_lif *lif) 1491 { 1492 struct ionic_dev *idev = &lif->adapter->idev; 1493 struct ionic_q_init_comp comp; 1494 int err; 1495 1496 memset(&lif->stats_base, 0, sizeof(lif->stats_base)); 1497 1498 ionic_dev_cmd_lif_init(idev, lif->info_pa); 1499 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); 1500 ionic_dev_cmd_comp(idev, &comp); 1501 if (err) 1502 return err; 1503 1504 lif->hw_index = rte_cpu_to_le_16(comp.hw_index); 1505 1506 err = ionic_lif_adminq_init(lif); 1507 if (err) 1508 return err; 1509 1510 err = ionic_lif_notifyq_init(lif); 1511 if (err) 1512 goto err_out_adminq_deinit; 1513 1514 /* 1515 * Configure initial feature set 1516 * This will be updated later by the dev_configure() step 1517 */ 1518 lif->features = IONIC_ETH_HW_RX_HASH | IONIC_ETH_HW_VLAN_RX_FILTER; 1519 1520 err = ionic_lif_set_features(lif); 1521 if (err) 1522 goto err_out_notifyq_deinit; 1523 1524 err = ionic_rx_filters_init(lif); 1525 if (err) 1526 goto err_out_notifyq_deinit; 1527 1528 err = ionic_station_set(lif); 1529 if (err) 1530 goto err_out_rx_filter_deinit; 1531 1532 ionic_lif_set_name(lif); 1533 1534 lif->state |= IONIC_LIF_F_INITED; 1535 1536 return 0; 1537 1538 err_out_rx_filter_deinit: 1539 ionic_rx_filters_deinit(lif); 1540 1541 err_out_notifyq_deinit: 1542 ionic_lif_notifyq_deinit(lif); 1543 1544 err_out_adminq_deinit: 1545 ionic_lif_qcq_deinit(lif->adminqcq); 1546 1547 return err; 1548 } 1549 1550 void 1551 ionic_lif_deinit(struct ionic_lif *lif) 1552 { 1553 if (!(lif->state & IONIC_LIF_F_INITED)) 1554 return; 1555 1556 ionic_rx_filters_deinit(lif); 1557 ionic_lif_rss_teardown(lif); 1558 ionic_lif_notifyq_deinit(lif); 1559 ionic_lif_qcq_deinit(lif->adminqcq); 1560 1561 lif->state &= ~IONIC_LIF_F_INITED; 1562 } 1563 1564 void 1565 ionic_lif_configure_vlan_offload(struct ionic_lif *lif, int mask) 1566 { 1567 struct rte_eth_dev *eth_dev = lif->eth_dev; 1568 struct rte_eth_rxmode *rxmode = ð_dev->data->dev_conf.rxmode; 1569 1570 /* 1571 * IONIC_ETH_HW_VLAN_RX_FILTER cannot be turned off, so 1572 * set DEV_RX_OFFLOAD_VLAN_FILTER and ignore ETH_VLAN_FILTER_MASK 1573 */ 1574 rxmode->offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 1575 1576 if (mask & ETH_VLAN_STRIP_MASK) { 1577 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 1578 lif->features |= IONIC_ETH_HW_VLAN_RX_STRIP; 1579 else 1580 lif->features &= ~IONIC_ETH_HW_VLAN_RX_STRIP; 1581 } 1582 } 1583 1584 void 1585 ionic_lif_configure(struct ionic_lif *lif) 1586 { 1587 struct rte_eth_rxmode *rxmode = &lif->eth_dev->data->dev_conf.rxmode; 1588 struct rte_eth_txmode *txmode = &lif->eth_dev->data->dev_conf.txmode; 1589 struct ionic_identity *ident = &lif->adapter->ident; 1590 union ionic_lif_config *cfg = &ident->lif.eth.config; 1591 uint32_t ntxqs_per_lif = 1592 rte_le_to_cpu_32(cfg->queue_count[IONIC_QTYPE_TXQ]); 1593 uint32_t nrxqs_per_lif = 1594 rte_le_to_cpu_32(cfg->queue_count[IONIC_QTYPE_RXQ]); 1595 uint32_t nrxqs = lif->eth_dev->data->nb_rx_queues; 1596 uint32_t ntxqs = lif->eth_dev->data->nb_tx_queues; 1597 1598 lif->port_id = lif->eth_dev->data->port_id; 1599 1600 IONIC_PRINT(DEBUG, "Configuring LIF on port %u", 1601 lif->port_id); 1602 1603 if (nrxqs > 0) 1604 nrxqs_per_lif = RTE_MIN(nrxqs_per_lif, nrxqs); 1605 1606 if (ntxqs > 0) 1607 ntxqs_per_lif = RTE_MIN(ntxqs_per_lif, ntxqs); 1608 1609 lif->nrxqcqs = nrxqs_per_lif; 1610 lif->ntxqcqs = ntxqs_per_lif; 1611 1612 /* Update the LIF configuration based on the eth_dev */ 1613 1614 /* 1615 * NB: While it is true that RSS_HASH is always enabled on ionic, 1616 * setting this flag unconditionally causes problems in DTS. 1617 * rxmode->offloads |= DEV_RX_OFFLOAD_RSS_HASH; 1618 */ 1619 1620 /* RX per-port */ 1621 1622 if (rxmode->offloads & DEV_RX_OFFLOAD_IPV4_CKSUM || 1623 rxmode->offloads & DEV_RX_OFFLOAD_UDP_CKSUM || 1624 rxmode->offloads & DEV_RX_OFFLOAD_TCP_CKSUM) 1625 lif->features |= IONIC_ETH_HW_RX_CSUM; 1626 else 1627 lif->features &= ~IONIC_ETH_HW_RX_CSUM; 1628 1629 if (rxmode->offloads & DEV_RX_OFFLOAD_SCATTER) { 1630 lif->features |= IONIC_ETH_HW_RX_SG; 1631 lif->eth_dev->data->scattered_rx = 1; 1632 } else { 1633 lif->features &= ~IONIC_ETH_HW_RX_SG; 1634 lif->eth_dev->data->scattered_rx = 0; 1635 } 1636 1637 /* Covers VLAN_STRIP */ 1638 ionic_lif_configure_vlan_offload(lif, ETH_VLAN_STRIP_MASK); 1639 1640 /* TX per-port */ 1641 1642 if (txmode->offloads & DEV_TX_OFFLOAD_IPV4_CKSUM || 1643 txmode->offloads & DEV_TX_OFFLOAD_UDP_CKSUM || 1644 txmode->offloads & DEV_TX_OFFLOAD_TCP_CKSUM || 1645 txmode->offloads & DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM || 1646 txmode->offloads & DEV_TX_OFFLOAD_OUTER_UDP_CKSUM) 1647 lif->features |= IONIC_ETH_HW_TX_CSUM; 1648 else 1649 lif->features &= ~IONIC_ETH_HW_TX_CSUM; 1650 1651 if (txmode->offloads & DEV_TX_OFFLOAD_VLAN_INSERT) 1652 lif->features |= IONIC_ETH_HW_VLAN_TX_TAG; 1653 else 1654 lif->features &= ~IONIC_ETH_HW_VLAN_TX_TAG; 1655 1656 if (txmode->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) 1657 lif->features |= IONIC_ETH_HW_TX_SG; 1658 else 1659 lif->features &= ~IONIC_ETH_HW_TX_SG; 1660 1661 if (txmode->offloads & DEV_TX_OFFLOAD_TCP_TSO) { 1662 lif->features |= IONIC_ETH_HW_TSO; 1663 lif->features |= IONIC_ETH_HW_TSO_IPV6; 1664 lif->features |= IONIC_ETH_HW_TSO_ECN; 1665 } else { 1666 lif->features &= ~IONIC_ETH_HW_TSO; 1667 lif->features &= ~IONIC_ETH_HW_TSO_IPV6; 1668 lif->features &= ~IONIC_ETH_HW_TSO_ECN; 1669 } 1670 } 1671 1672 int 1673 ionic_lif_start(struct ionic_lif *lif) 1674 { 1675 uint32_t rx_mode; 1676 uint32_t i; 1677 int err; 1678 1679 err = ionic_lif_rss_setup(lif); 1680 if (err) 1681 return err; 1682 1683 if (!lif->rx_mode) { 1684 IONIC_PRINT(DEBUG, "Setting RX mode on %s", 1685 lif->name); 1686 1687 rx_mode = IONIC_RX_MODE_F_UNICAST; 1688 rx_mode |= IONIC_RX_MODE_F_MULTICAST; 1689 rx_mode |= IONIC_RX_MODE_F_BROADCAST; 1690 1691 ionic_set_rx_mode(lif, rx_mode); 1692 } 1693 1694 IONIC_PRINT(DEBUG, "Starting %u RX queues and %u TX queues " 1695 "on port %u", 1696 lif->nrxqcqs, lif->ntxqcqs, lif->port_id); 1697 1698 for (i = 0; i < lif->nrxqcqs; i++) { 1699 struct ionic_qcq *rxq = lif->rxqcqs[i]; 1700 if (!(rxq->flags & IONIC_QCQ_F_DEFERRED)) { 1701 err = ionic_dev_rx_queue_start(lif->eth_dev, i); 1702 1703 if (err) 1704 return err; 1705 } 1706 } 1707 1708 for (i = 0; i < lif->ntxqcqs; i++) { 1709 struct ionic_qcq *txq = lif->txqcqs[i]; 1710 if (!(txq->flags & IONIC_QCQ_F_DEFERRED)) { 1711 err = ionic_dev_tx_queue_start(lif->eth_dev, i); 1712 1713 if (err) 1714 return err; 1715 } 1716 } 1717 1718 /* Carrier ON here */ 1719 lif->state |= IONIC_LIF_F_UP; 1720 1721 ionic_link_status_check(lif); 1722 1723 return 0; 1724 } 1725 1726 int 1727 ionic_lif_identify(struct ionic_adapter *adapter) 1728 { 1729 struct ionic_dev *idev = &adapter->idev; 1730 struct ionic_identity *ident = &adapter->ident; 1731 union ionic_lif_config *cfg = &ident->lif.eth.config; 1732 int err; 1733 unsigned int i; 1734 unsigned int lif_words = sizeof(ident->lif.words) / 1735 sizeof(ident->lif.words[0]); 1736 unsigned int cmd_words = sizeof(idev->dev_cmd->data) / 1737 sizeof(idev->dev_cmd->data[0]); 1738 unsigned int nwords; 1739 1740 ionic_dev_cmd_lif_identify(idev, IONIC_LIF_TYPE_CLASSIC, 1741 IONIC_IDENTITY_VERSION_1); 1742 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT); 1743 if (err) 1744 return (err); 1745 1746 nwords = RTE_MIN(lif_words, cmd_words); 1747 for (i = 0; i < nwords; i++) 1748 ident->lif.words[i] = ioread32(&idev->dev_cmd->data[i]); 1749 1750 IONIC_PRINT(INFO, "capabilities 0x%" PRIx64 " ", 1751 rte_le_to_cpu_64(ident->lif.capabilities)); 1752 1753 IONIC_PRINT(INFO, "eth.max_ucast_filters 0x%" PRIx32 " ", 1754 rte_le_to_cpu_32(ident->lif.eth.max_ucast_filters)); 1755 IONIC_PRINT(INFO, "eth.max_mcast_filters 0x%" PRIx32 " ", 1756 rte_le_to_cpu_32(ident->lif.eth.max_mcast_filters)); 1757 1758 IONIC_PRINT(INFO, "eth.features 0x%" PRIx64 " ", 1759 rte_le_to_cpu_64(cfg->features)); 1760 IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_ADMINQ] 0x%" PRIx32 " ", 1761 rte_le_to_cpu_32(cfg->queue_count[IONIC_QTYPE_ADMINQ])); 1762 IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] 0x%" PRIx32 " ", 1763 rte_le_to_cpu_32(cfg->queue_count[IONIC_QTYPE_NOTIFYQ])); 1764 IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_RXQ] 0x%" PRIx32 " ", 1765 rte_le_to_cpu_32(cfg->queue_count[IONIC_QTYPE_RXQ])); 1766 IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_TXQ] 0x%" PRIx32 " ", 1767 rte_le_to_cpu_32(cfg->queue_count[IONIC_QTYPE_TXQ])); 1768 1769 return 0; 1770 } 1771 1772 int 1773 ionic_lifs_size(struct ionic_adapter *adapter) 1774 { 1775 struct ionic_identity *ident = &adapter->ident; 1776 union ionic_lif_config *cfg = &ident->lif.eth.config; 1777 uint32_t nintrs, dev_nintrs = rte_le_to_cpu_32(ident->dev.nintrs); 1778 1779 adapter->max_ntxqs_per_lif = 1780 rte_le_to_cpu_32(cfg->queue_count[IONIC_QTYPE_TXQ]); 1781 adapter->max_nrxqs_per_lif = 1782 rte_le_to_cpu_32(cfg->queue_count[IONIC_QTYPE_RXQ]); 1783 1784 nintrs = 1 /* notifyq */; 1785 1786 if (nintrs > dev_nintrs) { 1787 IONIC_PRINT(ERR, 1788 "At most %d intr supported, minimum req'd is %u", 1789 dev_nintrs, nintrs); 1790 return -ENOSPC; 1791 } 1792 1793 adapter->nintrs = nintrs; 1794 1795 return 0; 1796 } 1797