1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (C) 2013 Emulex 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Emulex Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 * 33 * Contact Information: 34 * [email protected] 35 * 36 * Emulex 37 * 3333 Susan Street 38 * Costa Mesa, CA 92626 39 */ 40 41 /* $FreeBSD$ */ 42 43 #include "opt_inet6.h" 44 #include "opt_inet.h" 45 46 #include "oce_if.h" 47 #include "oce_user.h" 48 49 #define is_tso_pkt(m) (m->m_pkthdr.csum_flags & CSUM_TSO) 50 51 /* UE Status Low CSR */ 52 static char *ue_status_low_desc[] = { 53 "CEV", 54 "CTX", 55 "DBUF", 56 "ERX", 57 "Host", 58 "MPU", 59 "NDMA", 60 "PTC ", 61 "RDMA ", 62 "RXF ", 63 "RXIPS ", 64 "RXULP0 ", 65 "RXULP1 ", 66 "RXULP2 ", 67 "TIM ", 68 "TPOST ", 69 "TPRE ", 70 "TXIPS ", 71 "TXULP0 ", 72 "TXULP1 ", 73 "UC ", 74 "WDMA ", 75 "TXULP2 ", 76 "HOST1 ", 77 "P0_OB_LINK ", 78 "P1_OB_LINK ", 79 "HOST_GPIO ", 80 "MBOX ", 81 "AXGMAC0", 82 "AXGMAC1", 83 "JTAG", 84 "MPU_INTPEND" 85 }; 86 87 /* UE Status High CSR */ 88 static char *ue_status_hi_desc[] = { 89 "LPCMEMHOST", 90 "MGMT_MAC", 91 "PCS0ONLINE", 92 "MPU_IRAM", 93 "PCS1ONLINE", 94 "PCTL0", 95 "PCTL1", 96 "PMEM", 97 "RR", 98 "TXPB", 99 "RXPP", 100 "XAUI", 101 "TXP", 102 "ARM", 103 "IPC", 104 "HOST2", 105 "HOST3", 106 "HOST4", 107 "HOST5", 108 "HOST6", 109 "HOST7", 110 "HOST8", 111 "HOST9", 112 "NETC", 113 "Unknown", 114 "Unknown", 115 "Unknown", 116 "Unknown", 117 "Unknown", 118 "Unknown", 119 "Unknown", 120 "Unknown" 121 }; 122 123 struct oce_common_cqe_info{ 124 uint8_t vtp:1; 125 uint8_t l4_cksum_pass:1; 126 uint8_t ip_cksum_pass:1; 127 uint8_t ipv6_frame:1; 128 uint8_t qnq:1; 129 uint8_t rsvd:3; 130 uint8_t num_frags; 131 uint16_t pkt_size; 132 uint16_t vtag; 133 }; 134 135 136 /* Driver entry points prototypes */ 137 static int oce_probe(device_t dev); 138 static int oce_attach(device_t dev); 139 static int oce_detach(device_t dev); 140 static int oce_shutdown(device_t dev); 141 static int oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data); 142 static void oce_init(void *xsc); 143 static int oce_multiq_start(struct ifnet *ifp, struct mbuf *m); 144 static void oce_multiq_flush(struct ifnet *ifp); 145 146 /* Driver interrupt routines protypes */ 147 static void oce_intr(void *arg, int pending); 148 static int oce_setup_intr(POCE_SOFTC sc); 149 static int oce_fast_isr(void *arg); 150 static int oce_alloc_intr(POCE_SOFTC sc, int vector, 151 void (*isr) (void *arg, int pending)); 152 153 /* Media callbacks prototypes */ 154 static void oce_media_status(struct ifnet *ifp, struct ifmediareq *req); 155 static int oce_media_change(struct ifnet *ifp); 156 157 /* Transmit routines prototypes */ 158 static int oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index); 159 static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq); 160 static void oce_process_tx_completion(struct oce_wq *wq); 161 static int oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, 162 struct oce_wq *wq); 163 164 /* Receive routines prototypes */ 165 static int oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); 166 static int oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); 167 static void oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe); 168 static void oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq); 169 static uint16_t oce_rq_handler_lro(void *arg); 170 static void oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2); 171 static void oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2); 172 static void oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m); 173 174 /* Helper function prototypes in this file */ 175 static int oce_attach_ifp(POCE_SOFTC sc); 176 static void oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag); 177 static void oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag); 178 static int oce_vid_config(POCE_SOFTC sc); 179 static void oce_mac_addr_set(POCE_SOFTC sc); 180 static int oce_handle_passthrough(struct ifnet *ifp, caddr_t data); 181 static void oce_local_timer(void *arg); 182 static void oce_if_deactivate(POCE_SOFTC sc); 183 static void oce_if_activate(POCE_SOFTC sc); 184 static void setup_max_queues_want(POCE_SOFTC sc); 185 static void update_queues_got(POCE_SOFTC sc); 186 static void process_link_state(POCE_SOFTC sc, 187 struct oce_async_cqe_link_state *acqe); 188 static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m); 189 static void oce_get_config(POCE_SOFTC sc); 190 static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete); 191 static void oce_read_env_variables(POCE_SOFTC sc); 192 193 194 /* IP specific */ 195 #if defined(INET6) || defined(INET) 196 static int oce_init_lro(POCE_SOFTC sc); 197 static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp); 198 #endif 199 200 static device_method_t oce_dispatch[] = { 201 DEVMETHOD(device_probe, oce_probe), 202 DEVMETHOD(device_attach, oce_attach), 203 DEVMETHOD(device_detach, oce_detach), 204 DEVMETHOD(device_shutdown, oce_shutdown), 205 206 DEVMETHOD_END 207 }; 208 209 static driver_t oce_driver = { 210 "oce", 211 oce_dispatch, 212 sizeof(OCE_SOFTC) 213 }; 214 static devclass_t oce_devclass; 215 216 217 /* global vars */ 218 const char component_revision[32] = {"///" COMPONENT_REVISION "///"}; 219 220 /* Module capabilites and parameters */ 221 uint32_t oce_max_rsp_handled = OCE_MAX_RSP_HANDLED; 222 uint32_t oce_enable_rss = OCE_MODCAP_RSS; 223 uint32_t oce_rq_buf_size = 2048; 224 225 TUNABLE_INT("hw.oce.max_rsp_handled", &oce_max_rsp_handled); 226 TUNABLE_INT("hw.oce.enable_rss", &oce_enable_rss); 227 228 229 /* Supported devices table */ 230 static uint32_t supportedDevices[] = { 231 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE2, 232 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE3, 233 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_BE3, 234 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201, 235 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201_VF, 236 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_SH 237 }; 238 239 240 DRIVER_MODULE(oce, pci, oce_driver, oce_devclass, 0, 0); 241 MODULE_PNP_INFO("W32:vendor/device", pci, oce, supportedDevices, 242 nitems(supportedDevices)); 243 MODULE_DEPEND(oce, pci, 1, 1, 1); 244 MODULE_DEPEND(oce, ether, 1, 1, 1); 245 MODULE_VERSION(oce, 1); 246 247 248 POCE_SOFTC softc_head = NULL; 249 POCE_SOFTC softc_tail = NULL; 250 251 struct oce_rdma_if *oce_rdma_if = NULL; 252 253 /***************************************************************************** 254 * Driver entry points functions * 255 *****************************************************************************/ 256 257 static int 258 oce_probe(device_t dev) 259 { 260 uint16_t vendor = 0; 261 uint16_t device = 0; 262 int i = 0; 263 char str[256] = {0}; 264 POCE_SOFTC sc; 265 266 sc = device_get_softc(dev); 267 bzero(sc, sizeof(OCE_SOFTC)); 268 sc->dev = dev; 269 270 vendor = pci_get_vendor(dev); 271 device = pci_get_device(dev); 272 273 for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) { 274 if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) { 275 if (device == (supportedDevices[i] & 0xffff)) { 276 sprintf(str, "%s:%s", "Emulex CNA NIC function", 277 component_revision); 278 device_set_desc_copy(dev, str); 279 280 switch (device) { 281 case PCI_PRODUCT_BE2: 282 sc->flags |= OCE_FLAGS_BE2; 283 break; 284 case PCI_PRODUCT_BE3: 285 sc->flags |= OCE_FLAGS_BE3; 286 break; 287 case PCI_PRODUCT_XE201: 288 case PCI_PRODUCT_XE201_VF: 289 sc->flags |= OCE_FLAGS_XE201; 290 break; 291 case PCI_PRODUCT_SH: 292 sc->flags |= OCE_FLAGS_SH; 293 break; 294 default: 295 return ENXIO; 296 } 297 return BUS_PROBE_DEFAULT; 298 } 299 } 300 } 301 302 return ENXIO; 303 } 304 305 306 static int 307 oce_attach(device_t dev) 308 { 309 POCE_SOFTC sc; 310 int rc = 0; 311 312 sc = device_get_softc(dev); 313 314 rc = oce_hw_pci_alloc(sc); 315 if (rc) 316 return rc; 317 318 sc->tx_ring_size = OCE_TX_RING_SIZE; 319 sc->rx_ring_size = OCE_RX_RING_SIZE; 320 /* receive fragment size should be multiple of 2K */ 321 sc->rq_frag_size = ((oce_rq_buf_size / 2048) * 2048); 322 sc->flow_control = OCE_DEFAULT_FLOW_CONTROL; 323 sc->promisc = OCE_DEFAULT_PROMISCUOUS; 324 325 LOCK_CREATE(&sc->bmbx_lock, "Mailbox_lock"); 326 LOCK_CREATE(&sc->dev_lock, "Device_lock"); 327 328 /* initialise the hardware */ 329 rc = oce_hw_init(sc); 330 if (rc) 331 goto pci_res_free; 332 333 oce_read_env_variables(sc); 334 335 oce_get_config(sc); 336 337 setup_max_queues_want(sc); 338 339 rc = oce_setup_intr(sc); 340 if (rc) 341 goto mbox_free; 342 343 rc = oce_queue_init_all(sc); 344 if (rc) 345 goto intr_free; 346 347 rc = oce_attach_ifp(sc); 348 if (rc) 349 goto queues_free; 350 351 #if defined(INET6) || defined(INET) 352 rc = oce_init_lro(sc); 353 if (rc) 354 goto ifp_free; 355 #endif 356 357 rc = oce_hw_start(sc); 358 if (rc) 359 goto lro_free; 360 361 sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 362 oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST); 363 sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 364 oce_del_vlan, sc, EVENTHANDLER_PRI_FIRST); 365 366 rc = oce_stats_init(sc); 367 if (rc) 368 goto vlan_free; 369 370 oce_add_sysctls(sc); 371 372 callout_init(&sc->timer, CALLOUT_MPSAFE); 373 rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc); 374 if (rc) 375 goto stats_free; 376 377 sc->next =NULL; 378 if (softc_tail != NULL) { 379 softc_tail->next = sc; 380 } else { 381 softc_head = sc; 382 } 383 softc_tail = sc; 384 385 return 0; 386 387 stats_free: 388 callout_drain(&sc->timer); 389 oce_stats_free(sc); 390 vlan_free: 391 if (sc->vlan_attach) 392 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); 393 if (sc->vlan_detach) 394 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); 395 oce_hw_intr_disable(sc); 396 lro_free: 397 #if defined(INET6) || defined(INET) 398 oce_free_lro(sc); 399 ifp_free: 400 #endif 401 ether_ifdetach(sc->ifp); 402 if_free(sc->ifp); 403 queues_free: 404 oce_queue_release_all(sc); 405 intr_free: 406 oce_intr_free(sc); 407 mbox_free: 408 oce_dma_free(sc, &sc->bsmbx); 409 pci_res_free: 410 oce_hw_pci_free(sc); 411 LOCK_DESTROY(&sc->dev_lock); 412 LOCK_DESTROY(&sc->bmbx_lock); 413 return rc; 414 415 } 416 417 418 static int 419 oce_detach(device_t dev) 420 { 421 POCE_SOFTC sc = device_get_softc(dev); 422 POCE_SOFTC poce_sc_tmp, *ppoce_sc_tmp1, poce_sc_tmp2 = NULL; 423 424 poce_sc_tmp = softc_head; 425 ppoce_sc_tmp1 = &softc_head; 426 while (poce_sc_tmp != NULL) { 427 if (poce_sc_tmp == sc) { 428 *ppoce_sc_tmp1 = sc->next; 429 if (sc->next == NULL) { 430 softc_tail = poce_sc_tmp2; 431 } 432 break; 433 } 434 poce_sc_tmp2 = poce_sc_tmp; 435 ppoce_sc_tmp1 = &poce_sc_tmp->next; 436 poce_sc_tmp = poce_sc_tmp->next; 437 } 438 439 LOCK(&sc->dev_lock); 440 oce_if_deactivate(sc); 441 UNLOCK(&sc->dev_lock); 442 443 callout_drain(&sc->timer); 444 445 if (sc->vlan_attach != NULL) 446 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); 447 if (sc->vlan_detach != NULL) 448 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); 449 450 ether_ifdetach(sc->ifp); 451 452 if_free(sc->ifp); 453 454 oce_hw_shutdown(sc); 455 456 bus_generic_detach(dev); 457 458 return 0; 459 } 460 461 462 static int 463 oce_shutdown(device_t dev) 464 { 465 int rc; 466 467 rc = oce_detach(dev); 468 469 return rc; 470 } 471 472 473 static int 474 oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 475 { 476 struct ifreq *ifr = (struct ifreq *)data; 477 POCE_SOFTC sc = ifp->if_softc; 478 struct ifi2creq i2c; 479 uint8_t offset = 0; 480 int rc = 0; 481 uint32_t u; 482 483 switch (command) { 484 485 case SIOCGIFMEDIA: 486 rc = ifmedia_ioctl(ifp, ifr, &sc->media, command); 487 break; 488 489 case SIOCSIFMTU: 490 if (ifr->ifr_mtu > OCE_MAX_MTU) 491 rc = EINVAL; 492 else 493 ifp->if_mtu = ifr->ifr_mtu; 494 break; 495 496 case SIOCSIFFLAGS: 497 if (ifp->if_flags & IFF_UP) { 498 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 499 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 500 oce_init(sc); 501 } 502 device_printf(sc->dev, "Interface Up\n"); 503 } else { 504 LOCK(&sc->dev_lock); 505 506 sc->ifp->if_drv_flags &= 507 ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 508 oce_if_deactivate(sc); 509 510 UNLOCK(&sc->dev_lock); 511 512 device_printf(sc->dev, "Interface Down\n"); 513 } 514 515 if ((ifp->if_flags & IFF_PROMISC) && !sc->promisc) { 516 if (!oce_rxf_set_promiscuous(sc, (1 | (1 << 1)))) 517 sc->promisc = TRUE; 518 } else if (!(ifp->if_flags & IFF_PROMISC) && sc->promisc) { 519 if (!oce_rxf_set_promiscuous(sc, 0)) 520 sc->promisc = FALSE; 521 } 522 523 break; 524 525 case SIOCADDMULTI: 526 case SIOCDELMULTI: 527 rc = oce_hw_update_multicast(sc); 528 if (rc) 529 device_printf(sc->dev, 530 "Update multicast address failed\n"); 531 break; 532 533 case SIOCSIFCAP: 534 u = ifr->ifr_reqcap ^ ifp->if_capenable; 535 536 if (u & IFCAP_TXCSUM) { 537 ifp->if_capenable ^= IFCAP_TXCSUM; 538 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 539 540 if (IFCAP_TSO & ifp->if_capenable && 541 !(IFCAP_TXCSUM & ifp->if_capenable)) { 542 ifp->if_capenable &= ~IFCAP_TSO; 543 ifp->if_hwassist &= ~CSUM_TSO; 544 if_printf(ifp, 545 "TSO disabled due to -txcsum.\n"); 546 } 547 } 548 549 if (u & IFCAP_RXCSUM) 550 ifp->if_capenable ^= IFCAP_RXCSUM; 551 552 if (u & IFCAP_TSO4) { 553 ifp->if_capenable ^= IFCAP_TSO4; 554 555 if (IFCAP_TSO & ifp->if_capenable) { 556 if (IFCAP_TXCSUM & ifp->if_capenable) 557 ifp->if_hwassist |= CSUM_TSO; 558 else { 559 ifp->if_capenable &= ~IFCAP_TSO; 560 ifp->if_hwassist &= ~CSUM_TSO; 561 if_printf(ifp, 562 "Enable txcsum first.\n"); 563 rc = EAGAIN; 564 } 565 } else 566 ifp->if_hwassist &= ~CSUM_TSO; 567 } 568 569 if (u & IFCAP_VLAN_HWTAGGING) 570 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 571 572 if (u & IFCAP_VLAN_HWFILTER) { 573 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 574 oce_vid_config(sc); 575 } 576 #if defined(INET6) || defined(INET) 577 if (u & IFCAP_LRO) { 578 ifp->if_capenable ^= IFCAP_LRO; 579 if(sc->enable_hwlro) { 580 if(ifp->if_capenable & IFCAP_LRO) { 581 rc = oce_mbox_nic_set_iface_lro_config(sc, 1); 582 }else { 583 rc = oce_mbox_nic_set_iface_lro_config(sc, 0); 584 } 585 } 586 } 587 #endif 588 589 break; 590 591 case SIOCGI2C: 592 rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c)); 593 if (rc) 594 break; 595 596 if (i2c.dev_addr != PAGE_NUM_A0 && 597 i2c.dev_addr != PAGE_NUM_A2) { 598 rc = EINVAL; 599 break; 600 } 601 602 if (i2c.len > sizeof(i2c.data)) { 603 rc = EINVAL; 604 break; 605 } 606 607 rc = oce_mbox_read_transrecv_data(sc, i2c.dev_addr); 608 if(rc) { 609 rc = -rc; 610 break; 611 } 612 613 if (i2c.dev_addr == PAGE_NUM_A0) 614 offset = i2c.offset; 615 else 616 offset = TRANSCEIVER_A0_SIZE + i2c.offset; 617 618 memcpy(&i2c.data[0], &sfp_vpd_dump_buffer[offset], i2c.len); 619 620 rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c)); 621 break; 622 623 case SIOCGPRIVATE_0: 624 rc = priv_check(curthread, PRIV_DRIVER); 625 if (rc != 0) 626 break; 627 rc = oce_handle_passthrough(ifp, data); 628 break; 629 default: 630 rc = ether_ioctl(ifp, command, data); 631 break; 632 } 633 634 return rc; 635 } 636 637 638 static void 639 oce_init(void *arg) 640 { 641 POCE_SOFTC sc = arg; 642 643 LOCK(&sc->dev_lock); 644 645 if (sc->ifp->if_flags & IFF_UP) { 646 oce_if_deactivate(sc); 647 oce_if_activate(sc); 648 } 649 650 UNLOCK(&sc->dev_lock); 651 652 } 653 654 655 static int 656 oce_multiq_start(struct ifnet *ifp, struct mbuf *m) 657 { 658 POCE_SOFTC sc = ifp->if_softc; 659 struct oce_wq *wq = NULL; 660 int queue_index = 0; 661 int status = 0; 662 663 if (!sc->link_status) 664 return ENXIO; 665 666 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 667 queue_index = m->m_pkthdr.flowid % sc->nwqs; 668 669 wq = sc->wq[queue_index]; 670 671 LOCK(&wq->tx_lock); 672 status = oce_multiq_transmit(ifp, m, wq); 673 UNLOCK(&wq->tx_lock); 674 675 return status; 676 677 } 678 679 680 static void 681 oce_multiq_flush(struct ifnet *ifp) 682 { 683 POCE_SOFTC sc = ifp->if_softc; 684 struct mbuf *m; 685 int i = 0; 686 687 for (i = 0; i < sc->nwqs; i++) { 688 while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL) 689 m_freem(m); 690 } 691 if_qflush(ifp); 692 } 693 694 695 696 /***************************************************************************** 697 * Driver interrupt routines functions * 698 *****************************************************************************/ 699 700 static void 701 oce_intr(void *arg, int pending) 702 { 703 704 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; 705 POCE_SOFTC sc = ii->sc; 706 struct oce_eq *eq = ii->eq; 707 struct oce_eqe *eqe; 708 struct oce_cq *cq = NULL; 709 int i, num_eqes = 0; 710 711 712 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, 713 BUS_DMASYNC_POSTWRITE); 714 do { 715 eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe); 716 if (eqe->evnt == 0) 717 break; 718 eqe->evnt = 0; 719 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, 720 BUS_DMASYNC_POSTWRITE); 721 RING_GET(eq->ring, 1); 722 num_eqes++; 723 724 } while (TRUE); 725 726 if (!num_eqes) 727 goto eq_arm; /* Spurious */ 728 729 /* Clear EQ entries, but dont arm */ 730 oce_arm_eq(sc, eq->eq_id, num_eqes, FALSE, FALSE); 731 732 /* Process TX, RX and MCC. But dont arm CQ*/ 733 for (i = 0; i < eq->cq_valid; i++) { 734 cq = eq->cq[i]; 735 (*cq->cq_handler)(cq->cb_arg); 736 } 737 738 /* Arm all cqs connected to this EQ */ 739 for (i = 0; i < eq->cq_valid; i++) { 740 cq = eq->cq[i]; 741 oce_arm_cq(sc, cq->cq_id, 0, TRUE); 742 } 743 744 eq_arm: 745 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); 746 747 return; 748 } 749 750 751 static int 752 oce_setup_intr(POCE_SOFTC sc) 753 { 754 int rc = 0, use_intx = 0; 755 int vector = 0, req_vectors = 0; 756 int tot_req_vectors, tot_vectors; 757 758 if (is_rss_enabled(sc)) 759 req_vectors = MAX((sc->nrqs - 1), sc->nwqs); 760 else 761 req_vectors = 1; 762 763 tot_req_vectors = req_vectors; 764 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 765 if (req_vectors > 1) { 766 tot_req_vectors += OCE_RDMA_VECTORS; 767 sc->roce_intr_count = OCE_RDMA_VECTORS; 768 } 769 } 770 771 if (sc->flags & OCE_FLAGS_MSIX_CAPABLE) { 772 sc->intr_count = req_vectors; 773 tot_vectors = tot_req_vectors; 774 rc = pci_alloc_msix(sc->dev, &tot_vectors); 775 if (rc != 0) { 776 use_intx = 1; 777 pci_release_msi(sc->dev); 778 } else { 779 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 780 if (tot_vectors < tot_req_vectors) { 781 if (sc->intr_count < (2 * OCE_RDMA_VECTORS)) { 782 sc->roce_intr_count = (tot_vectors / 2); 783 } 784 sc->intr_count = tot_vectors - sc->roce_intr_count; 785 } 786 } else { 787 sc->intr_count = tot_vectors; 788 } 789 sc->flags |= OCE_FLAGS_USING_MSIX; 790 } 791 } else 792 use_intx = 1; 793 794 if (use_intx) 795 sc->intr_count = 1; 796 797 /* Scale number of queues based on intr we got */ 798 update_queues_got(sc); 799 800 if (use_intx) { 801 device_printf(sc->dev, "Using legacy interrupt\n"); 802 rc = oce_alloc_intr(sc, vector, oce_intr); 803 if (rc) 804 goto error; 805 } else { 806 for (; vector < sc->intr_count; vector++) { 807 rc = oce_alloc_intr(sc, vector, oce_intr); 808 if (rc) 809 goto error; 810 } 811 } 812 813 return 0; 814 error: 815 oce_intr_free(sc); 816 return rc; 817 } 818 819 820 static int 821 oce_fast_isr(void *arg) 822 { 823 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; 824 POCE_SOFTC sc = ii->sc; 825 826 if (ii->eq == NULL) 827 return FILTER_STRAY; 828 829 oce_arm_eq(sc, ii->eq->eq_id, 0, FALSE, TRUE); 830 831 taskqueue_enqueue(ii->tq, &ii->task); 832 833 ii->eq->intr++; 834 835 return FILTER_HANDLED; 836 } 837 838 839 static int 840 oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending)) 841 { 842 POCE_INTR_INFO ii; 843 int rc = 0, rr; 844 845 if (vector >= OCE_MAX_EQ) 846 return (EINVAL); 847 848 ii = &sc->intrs[vector]; 849 850 /* Set the resource id for the interrupt. 851 * MSIx is vector + 1 for the resource id, 852 * INTx is 0 for the resource id. 853 */ 854 if (sc->flags & OCE_FLAGS_USING_MSIX) 855 rr = vector + 1; 856 else 857 rr = 0; 858 ii->intr_res = bus_alloc_resource_any(sc->dev, 859 SYS_RES_IRQ, 860 &rr, RF_ACTIVE|RF_SHAREABLE); 861 ii->irq_rr = rr; 862 if (ii->intr_res == NULL) { 863 device_printf(sc->dev, 864 "Could not allocate interrupt\n"); 865 rc = ENXIO; 866 return rc; 867 } 868 869 TASK_INIT(&ii->task, 0, isr, ii); 870 ii->vector = vector; 871 sprintf(ii->task_name, "oce_task[%d]", ii->vector); 872 ii->tq = taskqueue_create_fast(ii->task_name, 873 M_NOWAIT, 874 taskqueue_thread_enqueue, 875 &ii->tq); 876 taskqueue_start_threads(&ii->tq, 1, PI_NET, "%s taskq", 877 device_get_nameunit(sc->dev)); 878 879 ii->sc = sc; 880 rc = bus_setup_intr(sc->dev, 881 ii->intr_res, 882 INTR_TYPE_NET, 883 oce_fast_isr, NULL, ii, &ii->tag); 884 return rc; 885 886 } 887 888 889 void 890 oce_intr_free(POCE_SOFTC sc) 891 { 892 int i = 0; 893 894 for (i = 0; i < sc->intr_count; i++) { 895 896 if (sc->intrs[i].tag != NULL) 897 bus_teardown_intr(sc->dev, sc->intrs[i].intr_res, 898 sc->intrs[i].tag); 899 if (sc->intrs[i].tq != NULL) 900 taskqueue_free(sc->intrs[i].tq); 901 902 if (sc->intrs[i].intr_res != NULL) 903 bus_release_resource(sc->dev, SYS_RES_IRQ, 904 sc->intrs[i].irq_rr, 905 sc->intrs[i].intr_res); 906 sc->intrs[i].tag = NULL; 907 sc->intrs[i].intr_res = NULL; 908 } 909 910 if (sc->flags & OCE_FLAGS_USING_MSIX) 911 pci_release_msi(sc->dev); 912 913 } 914 915 916 917 /****************************************************************************** 918 * Media callbacks functions * 919 ******************************************************************************/ 920 921 static void 922 oce_media_status(struct ifnet *ifp, struct ifmediareq *req) 923 { 924 POCE_SOFTC sc = (POCE_SOFTC) ifp->if_softc; 925 926 927 req->ifm_status = IFM_AVALID; 928 req->ifm_active = IFM_ETHER; 929 930 if (sc->link_status == 1) 931 req->ifm_status |= IFM_ACTIVE; 932 else 933 return; 934 935 switch (sc->link_speed) { 936 case 1: /* 10 Mbps */ 937 req->ifm_active |= IFM_10_T | IFM_FDX; 938 sc->speed = 10; 939 break; 940 case 2: /* 100 Mbps */ 941 req->ifm_active |= IFM_100_TX | IFM_FDX; 942 sc->speed = 100; 943 break; 944 case 3: /* 1 Gbps */ 945 req->ifm_active |= IFM_1000_T | IFM_FDX; 946 sc->speed = 1000; 947 break; 948 case 4: /* 10 Gbps */ 949 req->ifm_active |= IFM_10G_SR | IFM_FDX; 950 sc->speed = 10000; 951 break; 952 case 5: /* 20 Gbps */ 953 req->ifm_active |= IFM_10G_SR | IFM_FDX; 954 sc->speed = 20000; 955 break; 956 case 6: /* 25 Gbps */ 957 req->ifm_active |= IFM_10G_SR | IFM_FDX; 958 sc->speed = 25000; 959 break; 960 case 7: /* 40 Gbps */ 961 req->ifm_active |= IFM_40G_SR4 | IFM_FDX; 962 sc->speed = 40000; 963 break; 964 default: 965 sc->speed = 0; 966 break; 967 } 968 969 return; 970 } 971 972 973 int 974 oce_media_change(struct ifnet *ifp) 975 { 976 return 0; 977 } 978 979 980 static void oce_is_pkt_dest_bmc(POCE_SOFTC sc, 981 struct mbuf *m, boolean_t *os2bmc, 982 struct mbuf **m_new) 983 { 984 struct ether_header *eh = NULL; 985 986 eh = mtod(m, struct ether_header *); 987 988 if (!is_os2bmc_enabled(sc) || *os2bmc) { 989 *os2bmc = FALSE; 990 goto done; 991 } 992 if (!ETHER_IS_MULTICAST(eh->ether_dhost)) 993 goto done; 994 995 if (is_mc_allowed_on_bmc(sc, eh) || 996 is_bc_allowed_on_bmc(sc, eh) || 997 is_arp_allowed_on_bmc(sc, ntohs(eh->ether_type))) { 998 *os2bmc = TRUE; 999 goto done; 1000 } 1001 1002 if (mtod(m, struct ip *)->ip_p == IPPROTO_IPV6) { 1003 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); 1004 uint8_t nexthdr = ip6->ip6_nxt; 1005 if (nexthdr == IPPROTO_ICMPV6) { 1006 struct icmp6_hdr *icmp6 = (struct icmp6_hdr *)(ip6 + 1); 1007 switch (icmp6->icmp6_type) { 1008 case ND_ROUTER_ADVERT: 1009 *os2bmc = is_ipv6_ra_filt_enabled(sc); 1010 goto done; 1011 case ND_NEIGHBOR_ADVERT: 1012 *os2bmc = is_ipv6_na_filt_enabled(sc); 1013 goto done; 1014 default: 1015 break; 1016 } 1017 } 1018 } 1019 1020 if (mtod(m, struct ip *)->ip_p == IPPROTO_UDP) { 1021 struct ip *ip = mtod(m, struct ip *); 1022 int iphlen = ip->ip_hl << 2; 1023 struct udphdr *uh = (struct udphdr *)((caddr_t)ip + iphlen); 1024 switch (uh->uh_dport) { 1025 case DHCP_CLIENT_PORT: 1026 *os2bmc = is_dhcp_client_filt_enabled(sc); 1027 goto done; 1028 case DHCP_SERVER_PORT: 1029 *os2bmc = is_dhcp_srvr_filt_enabled(sc); 1030 goto done; 1031 case NET_BIOS_PORT1: 1032 case NET_BIOS_PORT2: 1033 *os2bmc = is_nbios_filt_enabled(sc); 1034 goto done; 1035 case DHCPV6_RAS_PORT: 1036 *os2bmc = is_ipv6_ras_filt_enabled(sc); 1037 goto done; 1038 default: 1039 break; 1040 } 1041 } 1042 done: 1043 if (*os2bmc) { 1044 *m_new = m_dup(m, M_NOWAIT); 1045 if (!*m_new) { 1046 *os2bmc = FALSE; 1047 return; 1048 } 1049 *m_new = oce_insert_vlan_tag(sc, *m_new, NULL); 1050 } 1051 } 1052 1053 1054 1055 /***************************************************************************** 1056 * Transmit routines functions * 1057 *****************************************************************************/ 1058 1059 static int 1060 oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index) 1061 { 1062 int rc = 0, i, retry_cnt = 0; 1063 bus_dma_segment_t segs[OCE_MAX_TX_ELEMENTS]; 1064 struct mbuf *m, *m_temp, *m_new = NULL; 1065 struct oce_wq *wq = sc->wq[wq_index]; 1066 struct oce_packet_desc *pd; 1067 struct oce_nic_hdr_wqe *nichdr; 1068 struct oce_nic_frag_wqe *nicfrag; 1069 struct ether_header *eh = NULL; 1070 int num_wqes; 1071 uint32_t reg_value; 1072 boolean_t complete = TRUE; 1073 boolean_t os2bmc = FALSE; 1074 1075 m = *mpp; 1076 if (!m) 1077 return EINVAL; 1078 1079 if (!(m->m_flags & M_PKTHDR)) { 1080 rc = ENXIO; 1081 goto free_ret; 1082 } 1083 1084 /* Don't allow non-TSO packets longer than MTU */ 1085 if (!is_tso_pkt(m)) { 1086 eh = mtod(m, struct ether_header *); 1087 if(m->m_pkthdr.len > ETHER_MAX_FRAME(sc->ifp, eh->ether_type, FALSE)) 1088 goto free_ret; 1089 } 1090 1091 if(oce_tx_asic_stall_verify(sc, m)) { 1092 m = oce_insert_vlan_tag(sc, m, &complete); 1093 if(!m) { 1094 device_printf(sc->dev, "Insertion unsuccessful\n"); 1095 return 0; 1096 } 1097 1098 } 1099 1100 /* Lancer, SH ASIC has a bug wherein Packets that are 32 bytes or less 1101 * may cause a transmit stall on that port. So the work-around is to 1102 * pad short packets (<= 32 bytes) to a 36-byte length. 1103 */ 1104 if(IS_SH(sc) || IS_XE201(sc) ) { 1105 if(m->m_pkthdr.len <= 32) { 1106 char buf[36]; 1107 bzero((void *)buf, 36); 1108 m_append(m, (36 - m->m_pkthdr.len), buf); 1109 } 1110 } 1111 1112 tx_start: 1113 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1114 /* consolidate packet buffers for TSO/LSO segment offload */ 1115 #if defined(INET6) || defined(INET) 1116 m = oce_tso_setup(sc, mpp); 1117 #else 1118 m = NULL; 1119 #endif 1120 if (m == NULL) { 1121 rc = ENXIO; 1122 goto free_ret; 1123 } 1124 } 1125 1126 1127 pd = &wq->pckts[wq->pkt_desc_head]; 1128 1129 retry: 1130 rc = bus_dmamap_load_mbuf_sg(wq->tag, 1131 pd->map, 1132 m, segs, &pd->nsegs, BUS_DMA_NOWAIT); 1133 if (rc == 0) { 1134 num_wqes = pd->nsegs + 1; 1135 if (IS_BE(sc) || IS_SH(sc)) { 1136 /*Dummy required only for BE3.*/ 1137 if (num_wqes & 1) 1138 num_wqes++; 1139 } 1140 if (num_wqes >= RING_NUM_FREE(wq->ring)) { 1141 bus_dmamap_unload(wq->tag, pd->map); 1142 return EBUSY; 1143 } 1144 atomic_store_rel_int(&wq->pkt_desc_head, 1145 (wq->pkt_desc_head + 1) % \ 1146 OCE_WQ_PACKET_ARRAY_SIZE); 1147 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE); 1148 pd->mbuf = m; 1149 1150 nichdr = 1151 RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe); 1152 nichdr->u0.dw[0] = 0; 1153 nichdr->u0.dw[1] = 0; 1154 nichdr->u0.dw[2] = 0; 1155 nichdr->u0.dw[3] = 0; 1156 1157 nichdr->u0.s.complete = complete; 1158 nichdr->u0.s.mgmt = os2bmc; 1159 nichdr->u0.s.event = 1; 1160 nichdr->u0.s.crc = 1; 1161 nichdr->u0.s.forward = 0; 1162 nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0; 1163 nichdr->u0.s.udpcs = 1164 (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0; 1165 nichdr->u0.s.tcpcs = 1166 (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0; 1167 nichdr->u0.s.num_wqe = num_wqes; 1168 nichdr->u0.s.total_length = m->m_pkthdr.len; 1169 1170 if (m->m_flags & M_VLANTAG) { 1171 nichdr->u0.s.vlan = 1; /*Vlan present*/ 1172 nichdr->u0.s.vlan_tag = m->m_pkthdr.ether_vtag; 1173 } 1174 1175 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1176 if (m->m_pkthdr.tso_segsz) { 1177 nichdr->u0.s.lso = 1; 1178 nichdr->u0.s.lso_mss = m->m_pkthdr.tso_segsz; 1179 } 1180 if (!IS_BE(sc) || !IS_SH(sc)) 1181 nichdr->u0.s.ipcs = 1; 1182 } 1183 1184 RING_PUT(wq->ring, 1); 1185 atomic_add_int(&wq->ring->num_used, 1); 1186 1187 for (i = 0; i < pd->nsegs; i++) { 1188 nicfrag = 1189 RING_GET_PRODUCER_ITEM_VA(wq->ring, 1190 struct oce_nic_frag_wqe); 1191 nicfrag->u0.s.rsvd0 = 0; 1192 nicfrag->u0.s.frag_pa_hi = ADDR_HI(segs[i].ds_addr); 1193 nicfrag->u0.s.frag_pa_lo = ADDR_LO(segs[i].ds_addr); 1194 nicfrag->u0.s.frag_len = segs[i].ds_len; 1195 pd->wqe_idx = wq->ring->pidx; 1196 RING_PUT(wq->ring, 1); 1197 atomic_add_int(&wq->ring->num_used, 1); 1198 } 1199 if (num_wqes > (pd->nsegs + 1)) { 1200 nicfrag = 1201 RING_GET_PRODUCER_ITEM_VA(wq->ring, 1202 struct oce_nic_frag_wqe); 1203 nicfrag->u0.dw[0] = 0; 1204 nicfrag->u0.dw[1] = 0; 1205 nicfrag->u0.dw[2] = 0; 1206 nicfrag->u0.dw[3] = 0; 1207 pd->wqe_idx = wq->ring->pidx; 1208 RING_PUT(wq->ring, 1); 1209 atomic_add_int(&wq->ring->num_used, 1); 1210 pd->nsegs++; 1211 } 1212 1213 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1); 1214 wq->tx_stats.tx_reqs++; 1215 wq->tx_stats.tx_wrbs += num_wqes; 1216 wq->tx_stats.tx_bytes += m->m_pkthdr.len; 1217 wq->tx_stats.tx_pkts++; 1218 1219 bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map, 1220 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1221 reg_value = (num_wqes << 16) | wq->wq_id; 1222 1223 /* if os2bmc is not enabled or if the pkt is already tagged as 1224 bmc, do nothing 1225 */ 1226 oce_is_pkt_dest_bmc(sc, m, &os2bmc, &m_new); 1227 1228 OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value); 1229 1230 } else if (rc == EFBIG) { 1231 if (retry_cnt == 0) { 1232 m_temp = m_defrag(m, M_NOWAIT); 1233 if (m_temp == NULL) 1234 goto free_ret; 1235 m = m_temp; 1236 *mpp = m_temp; 1237 retry_cnt = retry_cnt + 1; 1238 goto retry; 1239 } else 1240 goto free_ret; 1241 } else if (rc == ENOMEM) 1242 return rc; 1243 else 1244 goto free_ret; 1245 1246 if (os2bmc) { 1247 m = m_new; 1248 goto tx_start; 1249 } 1250 1251 return 0; 1252 1253 free_ret: 1254 m_freem(*mpp); 1255 *mpp = NULL; 1256 return rc; 1257 } 1258 1259 1260 static void 1261 oce_process_tx_completion(struct oce_wq *wq) 1262 { 1263 struct oce_packet_desc *pd; 1264 POCE_SOFTC sc = (POCE_SOFTC) wq->parent; 1265 struct mbuf *m; 1266 1267 pd = &wq->pckts[wq->pkt_desc_tail]; 1268 atomic_store_rel_int(&wq->pkt_desc_tail, 1269 (wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE); 1270 atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1); 1271 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1272 bus_dmamap_unload(wq->tag, pd->map); 1273 1274 m = pd->mbuf; 1275 m_freem(m); 1276 pd->mbuf = NULL; 1277 1278 1279 if (sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) { 1280 if (wq->ring->num_used < (wq->ring->num_items / 2)) { 1281 sc->ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE); 1282 oce_tx_restart(sc, wq); 1283 } 1284 } 1285 } 1286 1287 1288 static void 1289 oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq) 1290 { 1291 1292 if ((sc->ifp->if_drv_flags & IFF_DRV_RUNNING) != IFF_DRV_RUNNING) 1293 return; 1294 1295 #if __FreeBSD_version >= 800000 1296 if (!drbr_empty(sc->ifp, wq->br)) 1297 #else 1298 if (!IFQ_DRV_IS_EMPTY(&sc->ifp->if_snd)) 1299 #endif 1300 taskqueue_enqueue(taskqueue_swi, &wq->txtask); 1301 1302 } 1303 1304 1305 #if defined(INET6) || defined(INET) 1306 static struct mbuf * 1307 oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp) 1308 { 1309 struct mbuf *m; 1310 #ifdef INET 1311 struct ip *ip; 1312 #endif 1313 #ifdef INET6 1314 struct ip6_hdr *ip6; 1315 #endif 1316 struct ether_vlan_header *eh; 1317 struct tcphdr *th; 1318 uint16_t etype; 1319 int total_len = 0, ehdrlen = 0; 1320 1321 m = *mpp; 1322 1323 if (M_WRITABLE(m) == 0) { 1324 m = m_dup(*mpp, M_NOWAIT); 1325 if (!m) 1326 return NULL; 1327 m_freem(*mpp); 1328 *mpp = m; 1329 } 1330 1331 eh = mtod(m, struct ether_vlan_header *); 1332 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 1333 etype = ntohs(eh->evl_proto); 1334 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 1335 } else { 1336 etype = ntohs(eh->evl_encap_proto); 1337 ehdrlen = ETHER_HDR_LEN; 1338 } 1339 1340 switch (etype) { 1341 #ifdef INET 1342 case ETHERTYPE_IP: 1343 ip = (struct ip *)(m->m_data + ehdrlen); 1344 if (ip->ip_p != IPPROTO_TCP) 1345 return NULL; 1346 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); 1347 1348 total_len = ehdrlen + (ip->ip_hl << 2) + (th->th_off << 2); 1349 break; 1350 #endif 1351 #ifdef INET6 1352 case ETHERTYPE_IPV6: 1353 ip6 = (struct ip6_hdr *)(m->m_data + ehdrlen); 1354 if (ip6->ip6_nxt != IPPROTO_TCP) 1355 return NULL; 1356 th = (struct tcphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr)); 1357 1358 total_len = ehdrlen + sizeof(struct ip6_hdr) + (th->th_off << 2); 1359 break; 1360 #endif 1361 default: 1362 return NULL; 1363 } 1364 1365 m = m_pullup(m, total_len); 1366 if (!m) 1367 return NULL; 1368 *mpp = m; 1369 return m; 1370 1371 } 1372 #endif /* INET6 || INET */ 1373 1374 void 1375 oce_tx_task(void *arg, int npending) 1376 { 1377 struct oce_wq *wq = arg; 1378 POCE_SOFTC sc = wq->parent; 1379 struct ifnet *ifp = sc->ifp; 1380 int rc = 0; 1381 1382 #if __FreeBSD_version >= 800000 1383 LOCK(&wq->tx_lock); 1384 rc = oce_multiq_transmit(ifp, NULL, wq); 1385 if (rc) { 1386 device_printf(sc->dev, 1387 "TX[%d] restart failed\n", wq->queue_index); 1388 } 1389 UNLOCK(&wq->tx_lock); 1390 #else 1391 oce_start(ifp); 1392 #endif 1393 1394 } 1395 1396 1397 void 1398 oce_start(struct ifnet *ifp) 1399 { 1400 POCE_SOFTC sc = ifp->if_softc; 1401 struct mbuf *m; 1402 int rc = 0; 1403 int def_q = 0; /* Defualt tx queue is 0*/ 1404 1405 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1406 IFF_DRV_RUNNING) 1407 return; 1408 1409 if (!sc->link_status) 1410 return; 1411 1412 do { 1413 IF_DEQUEUE(&sc->ifp->if_snd, m); 1414 if (m == NULL) 1415 break; 1416 1417 LOCK(&sc->wq[def_q]->tx_lock); 1418 rc = oce_tx(sc, &m, def_q); 1419 UNLOCK(&sc->wq[def_q]->tx_lock); 1420 if (rc) { 1421 if (m != NULL) { 1422 sc->wq[def_q]->tx_stats.tx_stops ++; 1423 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1424 IFQ_DRV_PREPEND(&ifp->if_snd, m); 1425 m = NULL; 1426 } 1427 break; 1428 } 1429 if (m != NULL) 1430 ETHER_BPF_MTAP(ifp, m); 1431 1432 } while (TRUE); 1433 1434 return; 1435 } 1436 1437 1438 /* Handle the Completion Queue for transmit */ 1439 uint16_t 1440 oce_wq_handler(void *arg) 1441 { 1442 struct oce_wq *wq = (struct oce_wq *)arg; 1443 POCE_SOFTC sc = wq->parent; 1444 struct oce_cq *cq = wq->cq; 1445 struct oce_nic_tx_cqe *cqe; 1446 int num_cqes = 0; 1447 1448 LOCK(&wq->tx_compl_lock); 1449 bus_dmamap_sync(cq->ring->dma.tag, 1450 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1451 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 1452 while (cqe->u0.dw[3]) { 1453 DW_SWAP((uint32_t *) cqe, sizeof(oce_wq_cqe)); 1454 1455 wq->ring->cidx = cqe->u0.s.wqe_index + 1; 1456 if (wq->ring->cidx >= wq->ring->num_items) 1457 wq->ring->cidx -= wq->ring->num_items; 1458 1459 oce_process_tx_completion(wq); 1460 wq->tx_stats.tx_compl++; 1461 cqe->u0.dw[3] = 0; 1462 RING_GET(cq->ring, 1); 1463 bus_dmamap_sync(cq->ring->dma.tag, 1464 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1465 cqe = 1466 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 1467 num_cqes++; 1468 } 1469 1470 if (num_cqes) 1471 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); 1472 1473 UNLOCK(&wq->tx_compl_lock); 1474 return num_cqes; 1475 } 1476 1477 1478 static int 1479 oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, struct oce_wq *wq) 1480 { 1481 POCE_SOFTC sc = ifp->if_softc; 1482 int status = 0, queue_index = 0; 1483 struct mbuf *next = NULL; 1484 struct buf_ring *br = NULL; 1485 1486 br = wq->br; 1487 queue_index = wq->queue_index; 1488 1489 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1490 IFF_DRV_RUNNING) { 1491 if (m != NULL) 1492 status = drbr_enqueue(ifp, br, m); 1493 return status; 1494 } 1495 1496 if (m != NULL) { 1497 if ((status = drbr_enqueue(ifp, br, m)) != 0) 1498 return status; 1499 } 1500 while ((next = drbr_peek(ifp, br)) != NULL) { 1501 if (oce_tx(sc, &next, queue_index)) { 1502 if (next == NULL) { 1503 drbr_advance(ifp, br); 1504 } else { 1505 drbr_putback(ifp, br, next); 1506 wq->tx_stats.tx_stops ++; 1507 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1508 } 1509 break; 1510 } 1511 drbr_advance(ifp, br); 1512 if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len); 1513 if (next->m_flags & M_MCAST) 1514 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); 1515 ETHER_BPF_MTAP(ifp, next); 1516 } 1517 1518 return 0; 1519 } 1520 1521 1522 1523 1524 /***************************************************************************** 1525 * Receive routines functions * 1526 *****************************************************************************/ 1527 1528 static void 1529 oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2) 1530 { 1531 uint32_t *p; 1532 struct ether_header *eh = NULL; 1533 struct tcphdr *tcp_hdr = NULL; 1534 struct ip *ip4_hdr = NULL; 1535 struct ip6_hdr *ip6 = NULL; 1536 uint32_t payload_len = 0; 1537 1538 eh = mtod(m, struct ether_header *); 1539 /* correct IP header */ 1540 if(!cqe2->ipv6_frame) { 1541 ip4_hdr = (struct ip *)((char*)eh + sizeof(struct ether_header)); 1542 ip4_hdr->ip_ttl = cqe2->frame_lifespan; 1543 ip4_hdr->ip_len = htons(cqe2->coalesced_size - sizeof(struct ether_header)); 1544 tcp_hdr = (struct tcphdr *)((char*)ip4_hdr + sizeof(struct ip)); 1545 }else { 1546 ip6 = (struct ip6_hdr *)((char*)eh + sizeof(struct ether_header)); 1547 ip6->ip6_ctlun.ip6_un1.ip6_un1_hlim = cqe2->frame_lifespan; 1548 payload_len = cqe2->coalesced_size - sizeof(struct ether_header) 1549 - sizeof(struct ip6_hdr); 1550 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = htons(payload_len); 1551 tcp_hdr = (struct tcphdr *)((char*)ip6 + sizeof(struct ip6_hdr)); 1552 } 1553 1554 /* correct tcp header */ 1555 tcp_hdr->th_ack = htonl(cqe2->tcp_ack_num); 1556 if(cqe2->push) { 1557 tcp_hdr->th_flags |= TH_PUSH; 1558 } 1559 tcp_hdr->th_win = htons(cqe2->tcp_window); 1560 tcp_hdr->th_sum = 0xffff; 1561 if(cqe2->ts_opt) { 1562 p = (uint32_t *)((char*)tcp_hdr + sizeof(struct tcphdr) + 2); 1563 *p = cqe1->tcp_timestamp_val; 1564 *(p+1) = cqe1->tcp_timestamp_ecr; 1565 } 1566 1567 return; 1568 } 1569 1570 static void 1571 oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m) 1572 { 1573 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1574 uint32_t i = 0, frag_len = 0; 1575 uint32_t len = cqe_info->pkt_size; 1576 struct oce_packet_desc *pd; 1577 struct mbuf *tail = NULL; 1578 1579 for (i = 0; i < cqe_info->num_frags; i++) { 1580 if (rq->ring->cidx == rq->ring->pidx) { 1581 device_printf(sc->dev, 1582 "oce_rx_mbuf_chain: Invalid RX completion - Queue is empty\n"); 1583 return; 1584 } 1585 pd = &rq->pckts[rq->ring->cidx]; 1586 1587 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1588 bus_dmamap_unload(rq->tag, pd->map); 1589 RING_GET(rq->ring, 1); 1590 rq->pending--; 1591 1592 frag_len = (len > rq->cfg.frag_size) ? rq->cfg.frag_size : len; 1593 pd->mbuf->m_len = frag_len; 1594 1595 if (tail != NULL) { 1596 /* additional fragments */ 1597 pd->mbuf->m_flags &= ~M_PKTHDR; 1598 tail->m_next = pd->mbuf; 1599 if(rq->islro) 1600 tail->m_nextpkt = NULL; 1601 tail = pd->mbuf; 1602 } else { 1603 /* first fragment, fill out much of the packet header */ 1604 pd->mbuf->m_pkthdr.len = len; 1605 if(rq->islro) 1606 pd->mbuf->m_nextpkt = NULL; 1607 pd->mbuf->m_pkthdr.csum_flags = 0; 1608 if (IF_CSUM_ENABLED(sc)) { 1609 if (cqe_info->l4_cksum_pass) { 1610 if(!cqe_info->ipv6_frame) { /* IPV4 */ 1611 pd->mbuf->m_pkthdr.csum_flags |= 1612 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1613 }else { /* IPV6 frame */ 1614 if(rq->islro) { 1615 pd->mbuf->m_pkthdr.csum_flags |= 1616 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1617 } 1618 } 1619 pd->mbuf->m_pkthdr.csum_data = 0xffff; 1620 } 1621 if (cqe_info->ip_cksum_pass) { 1622 pd->mbuf->m_pkthdr.csum_flags |= 1623 (CSUM_IP_CHECKED|CSUM_IP_VALID); 1624 } 1625 } 1626 *m = tail = pd->mbuf; 1627 } 1628 pd->mbuf = NULL; 1629 len -= frag_len; 1630 } 1631 1632 return; 1633 } 1634 1635 static void 1636 oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2) 1637 { 1638 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1639 struct nic_hwlro_cqe_part1 *cqe1 = NULL; 1640 struct mbuf *m = NULL; 1641 struct oce_common_cqe_info cq_info; 1642 1643 /* parse cqe */ 1644 if(cqe2 == NULL) { 1645 cq_info.pkt_size = cqe->pkt_size; 1646 cq_info.vtag = cqe->vlan_tag; 1647 cq_info.l4_cksum_pass = cqe->l4_cksum_pass; 1648 cq_info.ip_cksum_pass = cqe->ip_cksum_pass; 1649 cq_info.ipv6_frame = cqe->ipv6_frame; 1650 cq_info.vtp = cqe->vtp; 1651 cq_info.qnq = cqe->qnq; 1652 }else { 1653 cqe1 = (struct nic_hwlro_cqe_part1 *)cqe; 1654 cq_info.pkt_size = cqe2->coalesced_size; 1655 cq_info.vtag = cqe2->vlan_tag; 1656 cq_info.l4_cksum_pass = cqe2->l4_cksum_pass; 1657 cq_info.ip_cksum_pass = cqe2->ip_cksum_pass; 1658 cq_info.ipv6_frame = cqe2->ipv6_frame; 1659 cq_info.vtp = cqe2->vtp; 1660 cq_info.qnq = cqe1->qnq; 1661 } 1662 1663 cq_info.vtag = BSWAP_16(cq_info.vtag); 1664 1665 cq_info.num_frags = cq_info.pkt_size / rq->cfg.frag_size; 1666 if(cq_info.pkt_size % rq->cfg.frag_size) 1667 cq_info.num_frags++; 1668 1669 oce_rx_mbuf_chain(rq, &cq_info, &m); 1670 1671 if (m) { 1672 if(cqe2) { 1673 //assert(cqe2->valid != 0); 1674 1675 //assert(cqe2->cqe_type != 2); 1676 oce_correct_header(m, cqe1, cqe2); 1677 } 1678 1679 m->m_pkthdr.rcvif = sc->ifp; 1680 #if __FreeBSD_version >= 800000 1681 if (rq->queue_index) 1682 m->m_pkthdr.flowid = (rq->queue_index - 1); 1683 else 1684 m->m_pkthdr.flowid = rq->queue_index; 1685 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 1686 #endif 1687 /* This deternies if vlan tag is Valid */ 1688 if (cq_info.vtp) { 1689 if (sc->function_mode & FNM_FLEX10_MODE) { 1690 /* FLEX10. If QnQ is not set, neglect VLAN */ 1691 if (cq_info.qnq) { 1692 m->m_pkthdr.ether_vtag = cq_info.vtag; 1693 m->m_flags |= M_VLANTAG; 1694 } 1695 } else if (sc->pvid != (cq_info.vtag & VLAN_VID_MASK)) { 1696 /* In UMC mode generally pvid will be striped by 1697 hw. But in some cases we have seen it comes 1698 with pvid. So if pvid == vlan, neglect vlan. 1699 */ 1700 m->m_pkthdr.ether_vtag = cq_info.vtag; 1701 m->m_flags |= M_VLANTAG; 1702 } 1703 } 1704 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 1705 1706 (*sc->ifp->if_input) (sc->ifp, m); 1707 1708 /* Update rx stats per queue */ 1709 rq->rx_stats.rx_pkts++; 1710 rq->rx_stats.rx_bytes += cq_info.pkt_size; 1711 rq->rx_stats.rx_frags += cq_info.num_frags; 1712 rq->rx_stats.rx_ucast_pkts++; 1713 } 1714 return; 1715 } 1716 1717 static void 1718 oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe) 1719 { 1720 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1721 int len; 1722 struct mbuf *m = NULL; 1723 struct oce_common_cqe_info cq_info; 1724 uint16_t vtag = 0; 1725 1726 /* Is it a flush compl that has no data */ 1727 if(!cqe->u0.s.num_fragments) 1728 goto exit; 1729 1730 len = cqe->u0.s.pkt_size; 1731 if (!len) { 1732 /*partial DMA workaround for Lancer*/ 1733 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); 1734 goto exit; 1735 } 1736 1737 if (!oce_cqe_portid_valid(sc, cqe)) { 1738 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); 1739 goto exit; 1740 } 1741 1742 /* Get vlan_tag value */ 1743 if(IS_BE(sc) || IS_SH(sc)) 1744 vtag = BSWAP_16(cqe->u0.s.vlan_tag); 1745 else 1746 vtag = cqe->u0.s.vlan_tag; 1747 1748 cq_info.l4_cksum_pass = cqe->u0.s.l4_cksum_pass; 1749 cq_info.ip_cksum_pass = cqe->u0.s.ip_cksum_pass; 1750 cq_info.ipv6_frame = cqe->u0.s.ip_ver; 1751 cq_info.num_frags = cqe->u0.s.num_fragments; 1752 cq_info.pkt_size = cqe->u0.s.pkt_size; 1753 1754 oce_rx_mbuf_chain(rq, &cq_info, &m); 1755 1756 if (m) { 1757 m->m_pkthdr.rcvif = sc->ifp; 1758 #if __FreeBSD_version >= 800000 1759 if (rq->queue_index) 1760 m->m_pkthdr.flowid = (rq->queue_index - 1); 1761 else 1762 m->m_pkthdr.flowid = rq->queue_index; 1763 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 1764 #endif 1765 /* This deternies if vlan tag is Valid */ 1766 if (oce_cqe_vtp_valid(sc, cqe)) { 1767 if (sc->function_mode & FNM_FLEX10_MODE) { 1768 /* FLEX10. If QnQ is not set, neglect VLAN */ 1769 if (cqe->u0.s.qnq) { 1770 m->m_pkthdr.ether_vtag = vtag; 1771 m->m_flags |= M_VLANTAG; 1772 } 1773 } else if (sc->pvid != (vtag & VLAN_VID_MASK)) { 1774 /* In UMC mode generally pvid will be striped by 1775 hw. But in some cases we have seen it comes 1776 with pvid. So if pvid == vlan, neglect vlan. 1777 */ 1778 m->m_pkthdr.ether_vtag = vtag; 1779 m->m_flags |= M_VLANTAG; 1780 } 1781 } 1782 1783 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 1784 #if defined(INET6) || defined(INET) 1785 /* Try to queue to LRO */ 1786 if (IF_LRO_ENABLED(sc) && 1787 (cqe->u0.s.ip_cksum_pass) && 1788 (cqe->u0.s.l4_cksum_pass) && 1789 (!cqe->u0.s.ip_ver) && 1790 (rq->lro.lro_cnt != 0)) { 1791 1792 if (tcp_lro_rx(&rq->lro, m, 0) == 0) { 1793 rq->lro_pkts_queued ++; 1794 goto post_done; 1795 } 1796 /* If LRO posting fails then try to post to STACK */ 1797 } 1798 #endif 1799 1800 (*sc->ifp->if_input) (sc->ifp, m); 1801 #if defined(INET6) || defined(INET) 1802 post_done: 1803 #endif 1804 /* Update rx stats per queue */ 1805 rq->rx_stats.rx_pkts++; 1806 rq->rx_stats.rx_bytes += cqe->u0.s.pkt_size; 1807 rq->rx_stats.rx_frags += cqe->u0.s.num_fragments; 1808 if (cqe->u0.s.pkt_type == OCE_MULTICAST_PACKET) 1809 rq->rx_stats.rx_mcast_pkts++; 1810 if (cqe->u0.s.pkt_type == OCE_UNICAST_PACKET) 1811 rq->rx_stats.rx_ucast_pkts++; 1812 } 1813 exit: 1814 return; 1815 } 1816 1817 1818 void 1819 oce_discard_rx_comp(struct oce_rq *rq, int num_frags) 1820 { 1821 uint32_t i = 0; 1822 struct oce_packet_desc *pd; 1823 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1824 1825 for (i = 0; i < num_frags; i++) { 1826 if (rq->ring->cidx == rq->ring->pidx) { 1827 device_printf(sc->dev, 1828 "oce_discard_rx_comp: Invalid RX completion - Queue is empty\n"); 1829 return; 1830 } 1831 pd = &rq->pckts[rq->ring->cidx]; 1832 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1833 bus_dmamap_unload(rq->tag, pd->map); 1834 if (pd->mbuf != NULL) { 1835 m_freem(pd->mbuf); 1836 pd->mbuf = NULL; 1837 } 1838 1839 RING_GET(rq->ring, 1); 1840 rq->pending--; 1841 } 1842 } 1843 1844 1845 static int 1846 oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) 1847 { 1848 struct oce_nic_rx_cqe_v1 *cqe_v1; 1849 int vtp = 0; 1850 1851 if (sc->be3_native) { 1852 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; 1853 vtp = cqe_v1->u0.s.vlan_tag_present; 1854 } else 1855 vtp = cqe->u0.s.vlan_tag_present; 1856 1857 return vtp; 1858 1859 } 1860 1861 1862 static int 1863 oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) 1864 { 1865 struct oce_nic_rx_cqe_v1 *cqe_v1; 1866 int port_id = 0; 1867 1868 if (sc->be3_native && (IS_BE(sc) || IS_SH(sc))) { 1869 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; 1870 port_id = cqe_v1->u0.s.port; 1871 if (sc->port_id != port_id) 1872 return 0; 1873 } else 1874 ;/* For BE3 legacy and Lancer this is dummy */ 1875 1876 return 1; 1877 1878 } 1879 1880 #if defined(INET6) || defined(INET) 1881 void 1882 oce_rx_flush_lro(struct oce_rq *rq) 1883 { 1884 struct lro_ctrl *lro = &rq->lro; 1885 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1886 1887 if (!IF_LRO_ENABLED(sc)) 1888 return; 1889 1890 tcp_lro_flush_all(lro); 1891 rq->lro_pkts_queued = 0; 1892 1893 return; 1894 } 1895 1896 1897 static int 1898 oce_init_lro(POCE_SOFTC sc) 1899 { 1900 struct lro_ctrl *lro = NULL; 1901 int i = 0, rc = 0; 1902 1903 for (i = 0; i < sc->nrqs; i++) { 1904 lro = &sc->rq[i]->lro; 1905 rc = tcp_lro_init(lro); 1906 if (rc != 0) { 1907 device_printf(sc->dev, "LRO init failed\n"); 1908 return rc; 1909 } 1910 lro->ifp = sc->ifp; 1911 } 1912 1913 return rc; 1914 } 1915 1916 1917 void 1918 oce_free_lro(POCE_SOFTC sc) 1919 { 1920 struct lro_ctrl *lro = NULL; 1921 int i = 0; 1922 1923 for (i = 0; i < sc->nrqs; i++) { 1924 lro = &sc->rq[i]->lro; 1925 if (lro) 1926 tcp_lro_free(lro); 1927 } 1928 } 1929 #endif 1930 1931 int 1932 oce_alloc_rx_bufs(struct oce_rq *rq, int count) 1933 { 1934 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1935 int i, in, rc; 1936 struct oce_packet_desc *pd; 1937 bus_dma_segment_t segs[6]; 1938 int nsegs, added = 0; 1939 struct oce_nic_rqe *rqe; 1940 pd_rxulp_db_t rxdb_reg; 1941 uint32_t val = 0; 1942 uint32_t oce_max_rq_posts = 64; 1943 1944 bzero(&rxdb_reg, sizeof(pd_rxulp_db_t)); 1945 for (i = 0; i < count; i++) { 1946 in = (rq->ring->pidx + 1) % OCE_RQ_PACKET_ARRAY_SIZE; 1947 1948 pd = &rq->pckts[rq->ring->pidx]; 1949 pd->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, oce_rq_buf_size); 1950 if (pd->mbuf == NULL) { 1951 device_printf(sc->dev, "mbuf allocation failed, size = %d\n",oce_rq_buf_size); 1952 break; 1953 } 1954 pd->mbuf->m_nextpkt = NULL; 1955 1956 pd->mbuf->m_len = pd->mbuf->m_pkthdr.len = rq->cfg.frag_size; 1957 1958 rc = bus_dmamap_load_mbuf_sg(rq->tag, 1959 pd->map, 1960 pd->mbuf, 1961 segs, &nsegs, BUS_DMA_NOWAIT); 1962 if (rc) { 1963 m_free(pd->mbuf); 1964 device_printf(sc->dev, "bus_dmamap_load_mbuf_sg failed rc = %d\n", rc); 1965 break; 1966 } 1967 1968 if (nsegs != 1) { 1969 i--; 1970 continue; 1971 } 1972 1973 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_PREREAD); 1974 1975 rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring, struct oce_nic_rqe); 1976 rqe->u0.s.frag_pa_hi = ADDR_HI(segs[0].ds_addr); 1977 rqe->u0.s.frag_pa_lo = ADDR_LO(segs[0].ds_addr); 1978 DW_SWAP(u32ptr(rqe), sizeof(struct oce_nic_rqe)); 1979 RING_PUT(rq->ring, 1); 1980 added++; 1981 rq->pending++; 1982 } 1983 oce_max_rq_posts = sc->enable_hwlro ? OCE_HWLRO_MAX_RQ_POSTS : OCE_MAX_RQ_POSTS; 1984 if (added != 0) { 1985 for (i = added / oce_max_rq_posts; i > 0; i--) { 1986 rxdb_reg.bits.num_posted = oce_max_rq_posts; 1987 rxdb_reg.bits.qid = rq->rq_id; 1988 if(rq->islro) { 1989 val |= rq->rq_id & DB_LRO_RQ_ID_MASK; 1990 val |= oce_max_rq_posts << 16; 1991 OCE_WRITE_REG32(sc, db, DB_OFFSET, val); 1992 }else { 1993 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); 1994 } 1995 added -= oce_max_rq_posts; 1996 } 1997 if (added > 0) { 1998 rxdb_reg.bits.qid = rq->rq_id; 1999 rxdb_reg.bits.num_posted = added; 2000 if(rq->islro) { 2001 val |= rq->rq_id & DB_LRO_RQ_ID_MASK; 2002 val |= added << 16; 2003 OCE_WRITE_REG32(sc, db, DB_OFFSET, val); 2004 }else { 2005 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); 2006 } 2007 } 2008 } 2009 2010 return 0; 2011 } 2012 2013 static void 2014 oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq) 2015 { 2016 if (num_cqes) { 2017 oce_arm_cq(sc, rq->cq->cq_id, num_cqes, FALSE); 2018 if(!sc->enable_hwlro) { 2019 if((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) > 1) 2020 oce_alloc_rx_bufs(rq, ((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) - 1)); 2021 }else { 2022 if ((OCE_RQ_PACKET_ARRAY_SIZE -1 - rq->pending) > 64) 2023 oce_alloc_rx_bufs(rq, 64); 2024 } 2025 } 2026 2027 return; 2028 } 2029 2030 uint16_t 2031 oce_rq_handler_lro(void *arg) 2032 { 2033 struct oce_rq *rq = (struct oce_rq *)arg; 2034 struct oce_cq *cq = rq->cq; 2035 POCE_SOFTC sc = rq->parent; 2036 struct nic_hwlro_singleton_cqe *cqe; 2037 struct nic_hwlro_cqe_part2 *cqe2; 2038 int num_cqes = 0; 2039 2040 LOCK(&rq->rx_lock); 2041 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2042 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); 2043 while (cqe->valid) { 2044 if(cqe->cqe_type == 0) { /* singleton cqe */ 2045 /* we should not get singleton cqe after cqe1 on same rq */ 2046 if(rq->cqe_firstpart != NULL) { 2047 device_printf(sc->dev, "Got singleton cqe after cqe1 \n"); 2048 goto exit_rq_handler_lro; 2049 } 2050 if(cqe->error != 0) { 2051 rq->rx_stats.rxcp_err++; 2052 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2053 } 2054 oce_rx_lro(rq, cqe, NULL); 2055 rq->rx_stats.rx_compl++; 2056 cqe->valid = 0; 2057 RING_GET(cq->ring, 1); 2058 num_cqes++; 2059 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2060 break; 2061 }else if(cqe->cqe_type == 0x1) { /* first part */ 2062 /* we should not get cqe1 after cqe1 on same rq */ 2063 if(rq->cqe_firstpart != NULL) { 2064 device_printf(sc->dev, "Got cqe1 after cqe1 \n"); 2065 goto exit_rq_handler_lro; 2066 } 2067 rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe; 2068 RING_GET(cq->ring, 1); 2069 }else if(cqe->cqe_type == 0x2) { /* second part */ 2070 cqe2 = (struct nic_hwlro_cqe_part2 *)cqe; 2071 if(cqe2->error != 0) { 2072 rq->rx_stats.rxcp_err++; 2073 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2074 } 2075 /* We should not get cqe2 without cqe1 */ 2076 if(rq->cqe_firstpart == NULL) { 2077 device_printf(sc->dev, "Got cqe2 without cqe1 \n"); 2078 goto exit_rq_handler_lro; 2079 } 2080 oce_rx_lro(rq, (struct nic_hwlro_singleton_cqe *)rq->cqe_firstpart, cqe2); 2081 2082 rq->rx_stats.rx_compl++; 2083 rq->cqe_firstpart->valid = 0; 2084 cqe2->valid = 0; 2085 rq->cqe_firstpart = NULL; 2086 2087 RING_GET(cq->ring, 1); 2088 num_cqes += 2; 2089 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2090 break; 2091 } 2092 2093 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2094 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); 2095 } 2096 oce_check_rx_bufs(sc, num_cqes, rq); 2097 exit_rq_handler_lro: 2098 UNLOCK(&rq->rx_lock); 2099 return 0; 2100 } 2101 2102 /* Handle the Completion Queue for receive */ 2103 uint16_t 2104 oce_rq_handler(void *arg) 2105 { 2106 struct oce_rq *rq = (struct oce_rq *)arg; 2107 struct oce_cq *cq = rq->cq; 2108 POCE_SOFTC sc = rq->parent; 2109 struct oce_nic_rx_cqe *cqe; 2110 int num_cqes = 0; 2111 2112 if(rq->islro) { 2113 oce_rq_handler_lro(arg); 2114 return 0; 2115 } 2116 LOCK(&rq->rx_lock); 2117 bus_dmamap_sync(cq->ring->dma.tag, 2118 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2119 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 2120 while (cqe->u0.dw[2]) { 2121 DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe)); 2122 2123 if (cqe->u0.s.error == 0) { 2124 oce_rx(rq, cqe); 2125 } else { 2126 rq->rx_stats.rxcp_err++; 2127 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2128 /* Post L3/L4 errors to stack.*/ 2129 oce_rx(rq, cqe); 2130 } 2131 rq->rx_stats.rx_compl++; 2132 cqe->u0.dw[2] = 0; 2133 2134 #if defined(INET6) || defined(INET) 2135 if (IF_LRO_ENABLED(sc) && rq->lro_pkts_queued >= 16) { 2136 oce_rx_flush_lro(rq); 2137 } 2138 #endif 2139 2140 RING_GET(cq->ring, 1); 2141 bus_dmamap_sync(cq->ring->dma.tag, 2142 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2143 cqe = 2144 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 2145 num_cqes++; 2146 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2147 break; 2148 } 2149 2150 #if defined(INET6) || defined(INET) 2151 if (IF_LRO_ENABLED(sc)) 2152 oce_rx_flush_lro(rq); 2153 #endif 2154 2155 oce_check_rx_bufs(sc, num_cqes, rq); 2156 UNLOCK(&rq->rx_lock); 2157 return 0; 2158 2159 } 2160 2161 2162 2163 2164 /***************************************************************************** 2165 * Helper function prototypes in this file * 2166 *****************************************************************************/ 2167 2168 static int 2169 oce_attach_ifp(POCE_SOFTC sc) 2170 { 2171 2172 sc->ifp = if_alloc(IFT_ETHER); 2173 if (!sc->ifp) 2174 return ENOMEM; 2175 2176 ifmedia_init(&sc->media, IFM_IMASK, oce_media_change, oce_media_status); 2177 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2178 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 2179 2180 sc->ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST; 2181 sc->ifp->if_ioctl = oce_ioctl; 2182 sc->ifp->if_start = oce_start; 2183 sc->ifp->if_init = oce_init; 2184 sc->ifp->if_mtu = ETHERMTU; 2185 sc->ifp->if_softc = sc; 2186 #if __FreeBSD_version >= 800000 2187 sc->ifp->if_transmit = oce_multiq_start; 2188 sc->ifp->if_qflush = oce_multiq_flush; 2189 #endif 2190 2191 if_initname(sc->ifp, 2192 device_get_name(sc->dev), device_get_unit(sc->dev)); 2193 2194 sc->ifp->if_snd.ifq_drv_maxlen = OCE_MAX_TX_DESC - 1; 2195 IFQ_SET_MAXLEN(&sc->ifp->if_snd, sc->ifp->if_snd.ifq_drv_maxlen); 2196 IFQ_SET_READY(&sc->ifp->if_snd); 2197 2198 sc->ifp->if_hwassist = OCE_IF_HWASSIST; 2199 sc->ifp->if_hwassist |= CSUM_TSO; 2200 sc->ifp->if_hwassist |= (CSUM_IP | CSUM_TCP | CSUM_UDP); 2201 2202 sc->ifp->if_capabilities = OCE_IF_CAPABILITIES; 2203 sc->ifp->if_capabilities |= IFCAP_HWCSUM; 2204 sc->ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 2205 2206 #if defined(INET6) || defined(INET) 2207 sc->ifp->if_capabilities |= IFCAP_TSO; 2208 sc->ifp->if_capabilities |= IFCAP_LRO; 2209 sc->ifp->if_capabilities |= IFCAP_VLAN_HWTSO; 2210 #endif 2211 2212 sc->ifp->if_capenable = sc->ifp->if_capabilities; 2213 sc->ifp->if_baudrate = IF_Gbps(10); 2214 2215 #if __FreeBSD_version >= 1000000 2216 sc->ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 2217 sc->ifp->if_hw_tsomaxsegcount = OCE_MAX_TX_ELEMENTS; 2218 sc->ifp->if_hw_tsomaxsegsize = 4096; 2219 #endif 2220 2221 ether_ifattach(sc->ifp, sc->macaddr.mac_addr); 2222 2223 return 0; 2224 } 2225 2226 2227 static void 2228 oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag) 2229 { 2230 POCE_SOFTC sc = ifp->if_softc; 2231 2232 if (ifp->if_softc != arg) 2233 return; 2234 if ((vtag == 0) || (vtag > 4095)) 2235 return; 2236 2237 sc->vlan_tag[vtag] = 1; 2238 sc->vlans_added++; 2239 if (sc->vlans_added <= (sc->max_vlans + 1)) 2240 oce_vid_config(sc); 2241 } 2242 2243 2244 static void 2245 oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag) 2246 { 2247 POCE_SOFTC sc = ifp->if_softc; 2248 2249 if (ifp->if_softc != arg) 2250 return; 2251 if ((vtag == 0) || (vtag > 4095)) 2252 return; 2253 2254 sc->vlan_tag[vtag] = 0; 2255 sc->vlans_added--; 2256 oce_vid_config(sc); 2257 } 2258 2259 2260 /* 2261 * A max of 64 vlans can be configured in BE. If the user configures 2262 * more, place the card in vlan promiscuous mode. 2263 */ 2264 static int 2265 oce_vid_config(POCE_SOFTC sc) 2266 { 2267 struct normal_vlan vtags[MAX_VLANFILTER_SIZE]; 2268 uint16_t ntags = 0, i; 2269 int status = 0; 2270 2271 if ((sc->vlans_added <= MAX_VLANFILTER_SIZE) && 2272 (sc->ifp->if_capenable & IFCAP_VLAN_HWFILTER)) { 2273 for (i = 0; i < MAX_VLANS; i++) { 2274 if (sc->vlan_tag[i]) { 2275 vtags[ntags].vtag = i; 2276 ntags++; 2277 } 2278 } 2279 if (ntags) 2280 status = oce_config_vlan(sc, (uint8_t) sc->if_id, 2281 vtags, ntags, 1, 0); 2282 } else 2283 status = oce_config_vlan(sc, (uint8_t) sc->if_id, 2284 NULL, 0, 1, 1); 2285 return status; 2286 } 2287 2288 2289 static void 2290 oce_mac_addr_set(POCE_SOFTC sc) 2291 { 2292 uint32_t old_pmac_id = sc->pmac_id; 2293 int status = 0; 2294 2295 2296 status = bcmp((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr, 2297 sc->macaddr.size_of_struct); 2298 if (!status) 2299 return; 2300 2301 status = oce_mbox_macaddr_add(sc, (uint8_t *)(IF_LLADDR(sc->ifp)), 2302 sc->if_id, &sc->pmac_id); 2303 if (!status) { 2304 status = oce_mbox_macaddr_del(sc, sc->if_id, old_pmac_id); 2305 bcopy((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr, 2306 sc->macaddr.size_of_struct); 2307 } 2308 if (status) 2309 device_printf(sc->dev, "Failed update macaddress\n"); 2310 2311 } 2312 2313 2314 static int 2315 oce_handle_passthrough(struct ifnet *ifp, caddr_t data) 2316 { 2317 POCE_SOFTC sc = ifp->if_softc; 2318 struct ifreq *ifr = (struct ifreq *)data; 2319 int rc = ENXIO; 2320 char cookie[32] = {0}; 2321 void *priv_data = ifr_data_get_ptr(ifr); 2322 void *ioctl_ptr; 2323 uint32_t req_size; 2324 struct mbx_hdr req; 2325 OCE_DMA_MEM dma_mem; 2326 struct mbx_common_get_cntl_attr *fw_cmd; 2327 2328 if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE))) 2329 return EFAULT; 2330 2331 if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE))) 2332 return EINVAL; 2333 2334 ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE); 2335 if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr))) 2336 return EFAULT; 2337 2338 req_size = le32toh(req.u0.req.request_length); 2339 if (req_size > 65536) 2340 return EINVAL; 2341 2342 req_size += sizeof(struct mbx_hdr); 2343 rc = oce_dma_alloc(sc, req_size, &dma_mem, 0); 2344 if (rc) 2345 return ENOMEM; 2346 2347 if (copyin(ioctl_ptr, OCE_DMAPTR(&dma_mem,char), req_size)) { 2348 rc = EFAULT; 2349 goto dma_free; 2350 } 2351 2352 rc = oce_pass_through_mbox(sc, &dma_mem, req_size); 2353 if (rc) { 2354 rc = EIO; 2355 goto dma_free; 2356 } 2357 2358 if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size)) 2359 rc = EFAULT; 2360 2361 /* 2362 firmware is filling all the attributes for this ioctl except 2363 the driver version..so fill it 2364 */ 2365 if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) { 2366 fw_cmd = (struct mbx_common_get_cntl_attr *) ioctl_ptr; 2367 strncpy(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str, 2368 COMPONENT_REVISION, strlen(COMPONENT_REVISION)); 2369 } 2370 2371 dma_free: 2372 oce_dma_free(sc, &dma_mem); 2373 return rc; 2374 2375 } 2376 2377 static void 2378 oce_eqd_set_periodic(POCE_SOFTC sc) 2379 { 2380 struct oce_set_eqd set_eqd[OCE_MAX_EQ]; 2381 struct oce_aic_obj *aic; 2382 struct oce_eq *eqo; 2383 uint64_t now = 0, delta; 2384 int eqd, i, num = 0; 2385 uint32_t tx_reqs = 0, rxpkts = 0, pps; 2386 struct oce_wq *wq; 2387 struct oce_rq *rq; 2388 2389 #define ticks_to_msecs(t) (1000 * (t) / hz) 2390 2391 for (i = 0 ; i < sc->neqs; i++) { 2392 eqo = sc->eq[i]; 2393 aic = &sc->aic_obj[i]; 2394 /* When setting the static eq delay from the user space */ 2395 if (!aic->enable) { 2396 if (aic->ticks) 2397 aic->ticks = 0; 2398 eqd = aic->et_eqd; 2399 goto modify_eqd; 2400 } 2401 2402 if (i == 0) { 2403 rq = sc->rq[0]; 2404 rxpkts = rq->rx_stats.rx_pkts; 2405 } else 2406 rxpkts = 0; 2407 if (i + 1 < sc->nrqs) { 2408 rq = sc->rq[i + 1]; 2409 rxpkts += rq->rx_stats.rx_pkts; 2410 } 2411 if (i < sc->nwqs) { 2412 wq = sc->wq[i]; 2413 tx_reqs = wq->tx_stats.tx_reqs; 2414 } else 2415 tx_reqs = 0; 2416 now = ticks; 2417 2418 if (!aic->ticks || now < aic->ticks || 2419 rxpkts < aic->prev_rxpkts || tx_reqs < aic->prev_txreqs) { 2420 aic->prev_rxpkts = rxpkts; 2421 aic->prev_txreqs = tx_reqs; 2422 aic->ticks = now; 2423 continue; 2424 } 2425 2426 delta = ticks_to_msecs(now - aic->ticks); 2427 2428 pps = (((uint32_t)(rxpkts - aic->prev_rxpkts) * 1000) / delta) + 2429 (((uint32_t)(tx_reqs - aic->prev_txreqs) * 1000) / delta); 2430 eqd = (pps / 15000) << 2; 2431 if (eqd < 8) 2432 eqd = 0; 2433 2434 /* Make sure that the eq delay is in the known range */ 2435 eqd = min(eqd, aic->max_eqd); 2436 eqd = max(eqd, aic->min_eqd); 2437 2438 aic->prev_rxpkts = rxpkts; 2439 aic->prev_txreqs = tx_reqs; 2440 aic->ticks = now; 2441 2442 modify_eqd: 2443 if (eqd != aic->cur_eqd) { 2444 set_eqd[num].delay_multiplier = (eqd * 65)/100; 2445 set_eqd[num].eq_id = eqo->eq_id; 2446 aic->cur_eqd = eqd; 2447 num++; 2448 } 2449 } 2450 2451 /* Is there atleast one eq that needs to be modified? */ 2452 for(i = 0; i < num; i += 8) { 2453 if((num - i) >=8 ) 2454 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], 8); 2455 else 2456 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], (num - i)); 2457 } 2458 2459 } 2460 2461 static void oce_detect_hw_error(POCE_SOFTC sc) 2462 { 2463 2464 uint32_t ue_low = 0, ue_high = 0, ue_low_mask = 0, ue_high_mask = 0; 2465 uint32_t sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0; 2466 uint32_t i; 2467 2468 if (sc->hw_error) 2469 return; 2470 2471 if (IS_XE201(sc)) { 2472 sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET); 2473 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2474 sliport_err1 = OCE_READ_REG32(sc, db, SLIPORT_ERROR1_OFFSET); 2475 sliport_err2 = OCE_READ_REG32(sc, db, SLIPORT_ERROR2_OFFSET); 2476 } 2477 } else { 2478 ue_low = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW); 2479 ue_high = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HIGH); 2480 ue_low_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW_MASK); 2481 ue_high_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HI_MASK); 2482 2483 ue_low = (ue_low & ~ue_low_mask); 2484 ue_high = (ue_high & ~ue_high_mask); 2485 } 2486 2487 /* On certain platforms BE hardware can indicate spurious UEs. 2488 * Allow the h/w to stop working completely in case of a real UE. 2489 * Hence not setting the hw_error for UE detection. 2490 */ 2491 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2492 sc->hw_error = TRUE; 2493 device_printf(sc->dev, "Error detected in the card\n"); 2494 } 2495 2496 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2497 device_printf(sc->dev, 2498 "ERR: sliport status 0x%x\n", sliport_status); 2499 device_printf(sc->dev, 2500 "ERR: sliport error1 0x%x\n", sliport_err1); 2501 device_printf(sc->dev, 2502 "ERR: sliport error2 0x%x\n", sliport_err2); 2503 } 2504 2505 if (ue_low) { 2506 for (i = 0; ue_low; ue_low >>= 1, i++) { 2507 if (ue_low & 1) 2508 device_printf(sc->dev, "UE: %s bit set\n", 2509 ue_status_low_desc[i]); 2510 } 2511 } 2512 2513 if (ue_high) { 2514 for (i = 0; ue_high; ue_high >>= 1, i++) { 2515 if (ue_high & 1) 2516 device_printf(sc->dev, "UE: %s bit set\n", 2517 ue_status_hi_desc[i]); 2518 } 2519 } 2520 2521 } 2522 2523 2524 static void 2525 oce_local_timer(void *arg) 2526 { 2527 POCE_SOFTC sc = arg; 2528 int i = 0; 2529 2530 oce_detect_hw_error(sc); 2531 oce_refresh_nic_stats(sc); 2532 oce_refresh_queue_stats(sc); 2533 oce_mac_addr_set(sc); 2534 2535 /* TX Watch Dog*/ 2536 for (i = 0; i < sc->nwqs; i++) 2537 oce_tx_restart(sc, sc->wq[i]); 2538 2539 /* calculate and set the eq delay for optimal interrupt rate */ 2540 if (IS_BE(sc) || IS_SH(sc)) 2541 oce_eqd_set_periodic(sc); 2542 2543 callout_reset(&sc->timer, hz, oce_local_timer, sc); 2544 } 2545 2546 static void 2547 oce_tx_compl_clean(POCE_SOFTC sc) 2548 { 2549 struct oce_wq *wq; 2550 int i = 0, timeo = 0, num_wqes = 0; 2551 int pending_txqs = sc->nwqs; 2552 2553 /* Stop polling for compls when HW has been silent for 10ms or 2554 * hw_error or no outstanding completions expected 2555 */ 2556 do { 2557 pending_txqs = sc->nwqs; 2558 2559 for_all_wq_queues(sc, wq, i) { 2560 num_wqes = oce_wq_handler(wq); 2561 2562 if(num_wqes) 2563 timeo = 0; 2564 2565 if(!wq->ring->num_used) 2566 pending_txqs--; 2567 } 2568 2569 if (pending_txqs == 0 || ++timeo > 10 || sc->hw_error) 2570 break; 2571 2572 DELAY(1000); 2573 } while (TRUE); 2574 2575 for_all_wq_queues(sc, wq, i) { 2576 while(wq->ring->num_used) { 2577 LOCK(&wq->tx_compl_lock); 2578 oce_process_tx_completion(wq); 2579 UNLOCK(&wq->tx_compl_lock); 2580 } 2581 } 2582 2583 } 2584 2585 /* NOTE : This should only be called holding 2586 * DEVICE_LOCK. 2587 */ 2588 static void 2589 oce_if_deactivate(POCE_SOFTC sc) 2590 { 2591 int i; 2592 struct oce_rq *rq; 2593 struct oce_wq *wq; 2594 struct oce_eq *eq; 2595 2596 sc->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2597 2598 oce_tx_compl_clean(sc); 2599 2600 /* Stop intrs and finish any bottom halves pending */ 2601 oce_hw_intr_disable(sc); 2602 2603 /* Since taskqueue_drain takes a Gaint Lock, We should not acquire 2604 any other lock. So unlock device lock and require after 2605 completing taskqueue_drain. 2606 */ 2607 UNLOCK(&sc->dev_lock); 2608 for (i = 0; i < sc->intr_count; i++) { 2609 if (sc->intrs[i].tq != NULL) { 2610 taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task); 2611 } 2612 } 2613 LOCK(&sc->dev_lock); 2614 2615 /* Delete RX queue in card with flush param */ 2616 oce_stop_rx(sc); 2617 2618 /* Invalidate any pending cq and eq entries*/ 2619 for_all_evnt_queues(sc, eq, i) 2620 oce_drain_eq(eq); 2621 for_all_rq_queues(sc, rq, i) 2622 oce_drain_rq_cq(rq); 2623 for_all_wq_queues(sc, wq, i) 2624 oce_drain_wq_cq(wq); 2625 2626 /* But still we need to get MCC aync events. 2627 So enable intrs and also arm first EQ 2628 */ 2629 oce_hw_intr_enable(sc); 2630 oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE); 2631 2632 DELAY(10); 2633 } 2634 2635 2636 static void 2637 oce_if_activate(POCE_SOFTC sc) 2638 { 2639 struct oce_eq *eq; 2640 struct oce_rq *rq; 2641 struct oce_wq *wq; 2642 int i, rc = 0; 2643 2644 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 2645 2646 oce_hw_intr_disable(sc); 2647 2648 oce_start_rx(sc); 2649 2650 for_all_rq_queues(sc, rq, i) { 2651 rc = oce_start_rq(rq); 2652 if (rc) 2653 device_printf(sc->dev, "Unable to start RX\n"); 2654 } 2655 2656 for_all_wq_queues(sc, wq, i) { 2657 rc = oce_start_wq(wq); 2658 if (rc) 2659 device_printf(sc->dev, "Unable to start TX\n"); 2660 } 2661 2662 2663 for_all_evnt_queues(sc, eq, i) 2664 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); 2665 2666 oce_hw_intr_enable(sc); 2667 2668 } 2669 2670 static void 2671 process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe) 2672 { 2673 /* Update Link status */ 2674 if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) == 2675 ASYNC_EVENT_LINK_UP) { 2676 sc->link_status = ASYNC_EVENT_LINK_UP; 2677 if_link_state_change(sc->ifp, LINK_STATE_UP); 2678 } else { 2679 sc->link_status = ASYNC_EVENT_LINK_DOWN; 2680 if_link_state_change(sc->ifp, LINK_STATE_DOWN); 2681 } 2682 } 2683 2684 2685 static void oce_async_grp5_osbmc_process(POCE_SOFTC sc, 2686 struct oce_async_evt_grp5_os2bmc *evt) 2687 { 2688 DW_SWAP(evt, sizeof(struct oce_async_evt_grp5_os2bmc)); 2689 if (evt->u.s.mgmt_enable) 2690 sc->flags |= OCE_FLAGS_OS2BMC; 2691 else 2692 return; 2693 2694 sc->bmc_filt_mask = evt->u.s.arp_filter; 2695 sc->bmc_filt_mask |= (evt->u.s.dhcp_client_filt << 1); 2696 sc->bmc_filt_mask |= (evt->u.s.dhcp_server_filt << 2); 2697 sc->bmc_filt_mask |= (evt->u.s.net_bios_filt << 3); 2698 sc->bmc_filt_mask |= (evt->u.s.bcast_filt << 4); 2699 sc->bmc_filt_mask |= (evt->u.s.ipv6_nbr_filt << 5); 2700 sc->bmc_filt_mask |= (evt->u.s.ipv6_ra_filt << 6); 2701 sc->bmc_filt_mask |= (evt->u.s.ipv6_ras_filt << 7); 2702 sc->bmc_filt_mask |= (evt->u.s.mcast_filt << 8); 2703 } 2704 2705 2706 static void oce_process_grp5_events(POCE_SOFTC sc, struct oce_mq_cqe *cqe) 2707 { 2708 struct oce_async_event_grp5_pvid_state *gcqe; 2709 struct oce_async_evt_grp5_os2bmc *bmccqe; 2710 2711 switch (cqe->u0.s.async_type) { 2712 case ASYNC_EVENT_PVID_STATE: 2713 /* GRP5 PVID */ 2714 gcqe = (struct oce_async_event_grp5_pvid_state *)cqe; 2715 if (gcqe->enabled) 2716 sc->pvid = gcqe->tag & VLAN_VID_MASK; 2717 else 2718 sc->pvid = 0; 2719 break; 2720 case ASYNC_EVENT_OS2BMC: 2721 bmccqe = (struct oce_async_evt_grp5_os2bmc *)cqe; 2722 oce_async_grp5_osbmc_process(sc, bmccqe); 2723 break; 2724 default: 2725 break; 2726 } 2727 } 2728 2729 /* Handle the Completion Queue for the Mailbox/Async notifications */ 2730 uint16_t 2731 oce_mq_handler(void *arg) 2732 { 2733 struct oce_mq *mq = (struct oce_mq *)arg; 2734 POCE_SOFTC sc = mq->parent; 2735 struct oce_cq *cq = mq->cq; 2736 int num_cqes = 0, evt_type = 0, optype = 0; 2737 struct oce_mq_cqe *cqe; 2738 struct oce_async_cqe_link_state *acqe; 2739 struct oce_async_event_qnq *dbgcqe; 2740 2741 2742 bus_dmamap_sync(cq->ring->dma.tag, 2743 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2744 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); 2745 2746 while (cqe->u0.dw[3]) { 2747 DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe)); 2748 if (cqe->u0.s.async_event) { 2749 evt_type = cqe->u0.s.event_type; 2750 optype = cqe->u0.s.async_type; 2751 if (evt_type == ASYNC_EVENT_CODE_LINK_STATE) { 2752 /* Link status evt */ 2753 acqe = (struct oce_async_cqe_link_state *)cqe; 2754 process_link_state(sc, acqe); 2755 } else if (evt_type == ASYNC_EVENT_GRP5) { 2756 oce_process_grp5_events(sc, cqe); 2757 } else if (evt_type == ASYNC_EVENT_CODE_DEBUG && 2758 optype == ASYNC_EVENT_DEBUG_QNQ) { 2759 dbgcqe = (struct oce_async_event_qnq *)cqe; 2760 if(dbgcqe->valid) 2761 sc->qnqid = dbgcqe->vlan_tag; 2762 sc->qnq_debug_event = TRUE; 2763 } 2764 } 2765 cqe->u0.dw[3] = 0; 2766 RING_GET(cq->ring, 1); 2767 bus_dmamap_sync(cq->ring->dma.tag, 2768 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2769 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); 2770 num_cqes++; 2771 } 2772 2773 if (num_cqes) 2774 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); 2775 2776 return 0; 2777 } 2778 2779 2780 static void 2781 setup_max_queues_want(POCE_SOFTC sc) 2782 { 2783 /* Check if it is FLEX machine. Is so dont use RSS */ 2784 if ((sc->function_mode & FNM_FLEX10_MODE) || 2785 (sc->function_mode & FNM_UMC_MODE) || 2786 (sc->function_mode & FNM_VNIC_MODE) || 2787 (!is_rss_enabled(sc)) || 2788 IS_BE2(sc)) { 2789 sc->nrqs = 1; 2790 sc->nwqs = 1; 2791 } else { 2792 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; 2793 sc->nwqs = MIN(OCE_NCPUS, sc->nrssqs); 2794 } 2795 2796 if (IS_BE2(sc) && is_rss_enabled(sc)) 2797 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; 2798 } 2799 2800 2801 static void 2802 update_queues_got(POCE_SOFTC sc) 2803 { 2804 if (is_rss_enabled(sc)) { 2805 sc->nrqs = sc->intr_count + 1; 2806 sc->nwqs = sc->intr_count; 2807 } else { 2808 sc->nrqs = 1; 2809 sc->nwqs = 1; 2810 } 2811 2812 if (IS_BE2(sc)) 2813 sc->nwqs = 1; 2814 } 2815 2816 static int 2817 oce_check_ipv6_ext_hdr(struct mbuf *m) 2818 { 2819 struct ether_header *eh = mtod(m, struct ether_header *); 2820 caddr_t m_datatemp = m->m_data; 2821 2822 if (eh->ether_type == htons(ETHERTYPE_IPV6)) { 2823 m->m_data += sizeof(struct ether_header); 2824 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); 2825 2826 if((ip6->ip6_nxt != IPPROTO_TCP) && \ 2827 (ip6->ip6_nxt != IPPROTO_UDP)){ 2828 struct ip6_ext *ip6e = NULL; 2829 m->m_data += sizeof(struct ip6_hdr); 2830 2831 ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *); 2832 if(ip6e->ip6e_len == 0xff) { 2833 m->m_data = m_datatemp; 2834 return TRUE; 2835 } 2836 } 2837 m->m_data = m_datatemp; 2838 } 2839 return FALSE; 2840 } 2841 2842 static int 2843 is_be3_a1(POCE_SOFTC sc) 2844 { 2845 if((sc->flags & OCE_FLAGS_BE3) && ((sc->asic_revision & 0xFF) < 2)) { 2846 return TRUE; 2847 } 2848 return FALSE; 2849 } 2850 2851 static struct mbuf * 2852 oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete) 2853 { 2854 uint16_t vlan_tag = 0; 2855 2856 if(!M_WRITABLE(m)) 2857 return NULL; 2858 2859 /* Embed vlan tag in the packet if it is not part of it */ 2860 if(m->m_flags & M_VLANTAG) { 2861 vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag); 2862 m->m_flags &= ~M_VLANTAG; 2863 } 2864 2865 /* if UMC, ignore vlan tag insertion and instead insert pvid */ 2866 if(sc->pvid) { 2867 if(!vlan_tag) 2868 vlan_tag = sc->pvid; 2869 if (complete) 2870 *complete = FALSE; 2871 } 2872 2873 if(vlan_tag) { 2874 m = ether_vlanencap(m, vlan_tag); 2875 } 2876 2877 if(sc->qnqid) { 2878 m = ether_vlanencap(m, sc->qnqid); 2879 2880 if (complete) 2881 *complete = FALSE; 2882 } 2883 return m; 2884 } 2885 2886 static int 2887 oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m) 2888 { 2889 if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \ 2890 oce_check_ipv6_ext_hdr(m)) { 2891 return TRUE; 2892 } 2893 return FALSE; 2894 } 2895 2896 static void 2897 oce_get_config(POCE_SOFTC sc) 2898 { 2899 int rc = 0; 2900 uint32_t max_rss = 0; 2901 2902 if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native)) 2903 max_rss = OCE_LEGACY_MODE_RSS; 2904 else 2905 max_rss = OCE_MAX_RSS; 2906 2907 if (!IS_BE(sc)) { 2908 rc = oce_get_profile_config(sc, max_rss); 2909 if (rc) { 2910 sc->nwqs = OCE_MAX_WQ; 2911 sc->nrssqs = max_rss; 2912 sc->nrqs = sc->nrssqs + 1; 2913 } 2914 } 2915 else { /* For BE3 don't rely on fw for determining the resources */ 2916 sc->nrssqs = max_rss; 2917 sc->nrqs = sc->nrssqs + 1; 2918 sc->nwqs = OCE_MAX_WQ; 2919 sc->max_vlans = MAX_VLANFILTER_SIZE; 2920 } 2921 } 2922 2923 static void 2924 oce_rdma_close(void) 2925 { 2926 if (oce_rdma_if != NULL) { 2927 oce_rdma_if = NULL; 2928 } 2929 } 2930 2931 static void 2932 oce_get_mac_addr(POCE_SOFTC sc, uint8_t *macaddr) 2933 { 2934 memcpy(macaddr, sc->macaddr.mac_addr, 6); 2935 } 2936 2937 int 2938 oce_register_rdma(POCE_RDMA_INFO rdma_info, POCE_RDMA_IF rdma_if) 2939 { 2940 POCE_SOFTC sc; 2941 struct oce_dev_info di; 2942 int i; 2943 2944 if ((rdma_info == NULL) || (rdma_if == NULL)) { 2945 return -EINVAL; 2946 } 2947 2948 if ((rdma_info->size != OCE_RDMA_INFO_SIZE) || 2949 (rdma_if->size != OCE_RDMA_IF_SIZE)) { 2950 return -ENXIO; 2951 } 2952 2953 rdma_info->close = oce_rdma_close; 2954 rdma_info->mbox_post = oce_mbox_post; 2955 rdma_info->common_req_hdr_init = mbx_common_req_hdr_init; 2956 rdma_info->get_mac_addr = oce_get_mac_addr; 2957 2958 oce_rdma_if = rdma_if; 2959 2960 sc = softc_head; 2961 while (sc != NULL) { 2962 if (oce_rdma_if->announce != NULL) { 2963 memset(&di, 0, sizeof(di)); 2964 di.dev = sc->dev; 2965 di.softc = sc; 2966 di.ifp = sc->ifp; 2967 di.db_bhandle = sc->db_bhandle; 2968 di.db_btag = sc->db_btag; 2969 di.db_page_size = 4096; 2970 if (sc->flags & OCE_FLAGS_USING_MSIX) { 2971 di.intr_mode = OCE_INTERRUPT_MODE_MSIX; 2972 } else if (sc->flags & OCE_FLAGS_USING_MSI) { 2973 di.intr_mode = OCE_INTERRUPT_MODE_MSI; 2974 } else { 2975 di.intr_mode = OCE_INTERRUPT_MODE_INTX; 2976 } 2977 di.dev_family = OCE_GEN2_FAMILY; // fixme: must detect skyhawk 2978 if (di.intr_mode != OCE_INTERRUPT_MODE_INTX) { 2979 di.msix.num_vectors = sc->intr_count + sc->roce_intr_count; 2980 di.msix.start_vector = sc->intr_count; 2981 for (i=0; i<di.msix.num_vectors; i++) { 2982 di.msix.vector_list[i] = sc->intrs[i].vector; 2983 } 2984 } else { 2985 } 2986 memcpy(di.mac_addr, sc->macaddr.mac_addr, 6); 2987 di.vendor_id = pci_get_vendor(sc->dev); 2988 di.dev_id = pci_get_device(sc->dev); 2989 2990 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 2991 di.flags |= OCE_RDMA_INFO_RDMA_SUPPORTED; 2992 } 2993 2994 rdma_if->announce(&di); 2995 sc = sc->next; 2996 } 2997 } 2998 2999 return 0; 3000 } 3001 3002 static void 3003 oce_read_env_variables( POCE_SOFTC sc ) 3004 { 3005 char *value = NULL; 3006 int rc = 0; 3007 3008 /* read if user wants to enable hwlro or swlro */ 3009 //value = getenv("oce_enable_hwlro"); 3010 if(value && IS_SH(sc)) { 3011 sc->enable_hwlro = strtol(value, NULL, 10); 3012 if(sc->enable_hwlro) { 3013 rc = oce_mbox_nic_query_lro_capabilities(sc, NULL, NULL); 3014 if(rc) { 3015 device_printf(sc->dev, "no hardware lro support\n"); 3016 device_printf(sc->dev, "software lro enabled\n"); 3017 sc->enable_hwlro = 0; 3018 }else { 3019 device_printf(sc->dev, "hardware lro enabled\n"); 3020 oce_max_rsp_handled = 32; 3021 } 3022 }else { 3023 device_printf(sc->dev, "software lro enabled\n"); 3024 } 3025 }else { 3026 sc->enable_hwlro = 0; 3027 } 3028 3029 /* read mbuf size */ 3030 //value = getenv("oce_rq_buf_size"); 3031 if(value && IS_SH(sc)) { 3032 oce_rq_buf_size = strtol(value, NULL, 10); 3033 switch(oce_rq_buf_size) { 3034 case 2048: 3035 case 4096: 3036 case 9216: 3037 case 16384: 3038 break; 3039 3040 default: 3041 device_printf(sc->dev, " Supported oce_rq_buf_size values are 2K, 4K, 9K, 16K \n"); 3042 oce_rq_buf_size = 2048; 3043 } 3044 } 3045 3046 return; 3047 } 3048