1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (C) 2013 Emulex
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright notice,
11 * this list of conditions and the following disclaimer.
12 *
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * 3. Neither the name of the Emulex Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived from
19 * this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
32 *
33 * Contact Information:
34 * [email protected]
35 *
36 * Emulex
37 * 3333 Susan Street
38 * Costa Mesa, CA 92626
39 */
40
41 /* $FreeBSD$ */
42
43 #include "opt_inet6.h"
44 #include "opt_inet.h"
45
46 #include "oce_if.h"
47 #include "oce_user.h"
48
49 #define is_tso_pkt(m) (m->m_pkthdr.csum_flags & CSUM_TSO)
50
51 /* UE Status Low CSR */
52 static char *ue_status_low_desc[] = {
53 "CEV",
54 "CTX",
55 "DBUF",
56 "ERX",
57 "Host",
58 "MPU",
59 "NDMA",
60 "PTC ",
61 "RDMA ",
62 "RXF ",
63 "RXIPS ",
64 "RXULP0 ",
65 "RXULP1 ",
66 "RXULP2 ",
67 "TIM ",
68 "TPOST ",
69 "TPRE ",
70 "TXIPS ",
71 "TXULP0 ",
72 "TXULP1 ",
73 "UC ",
74 "WDMA ",
75 "TXULP2 ",
76 "HOST1 ",
77 "P0_OB_LINK ",
78 "P1_OB_LINK ",
79 "HOST_GPIO ",
80 "MBOX ",
81 "AXGMAC0",
82 "AXGMAC1",
83 "JTAG",
84 "MPU_INTPEND"
85 };
86
87 /* UE Status High CSR */
88 static char *ue_status_hi_desc[] = {
89 "LPCMEMHOST",
90 "MGMT_MAC",
91 "PCS0ONLINE",
92 "MPU_IRAM",
93 "PCS1ONLINE",
94 "PCTL0",
95 "PCTL1",
96 "PMEM",
97 "RR",
98 "TXPB",
99 "RXPP",
100 "XAUI",
101 "TXP",
102 "ARM",
103 "IPC",
104 "HOST2",
105 "HOST3",
106 "HOST4",
107 "HOST5",
108 "HOST6",
109 "HOST7",
110 "HOST8",
111 "HOST9",
112 "NETC",
113 "Unknown",
114 "Unknown",
115 "Unknown",
116 "Unknown",
117 "Unknown",
118 "Unknown",
119 "Unknown",
120 "Unknown"
121 };
122
123 struct oce_common_cqe_info{
124 uint8_t vtp:1;
125 uint8_t l4_cksum_pass:1;
126 uint8_t ip_cksum_pass:1;
127 uint8_t ipv6_frame:1;
128 uint8_t qnq:1;
129 uint8_t rsvd:3;
130 uint8_t num_frags;
131 uint16_t pkt_size;
132 uint16_t vtag;
133 };
134
135
136 /* Driver entry points prototypes */
137 static int oce_probe(device_t dev);
138 static int oce_attach(device_t dev);
139 static int oce_detach(device_t dev);
140 static int oce_shutdown(device_t dev);
141 static int oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data);
142 static void oce_init(void *xsc);
143 static int oce_multiq_start(struct ifnet *ifp, struct mbuf *m);
144 static void oce_multiq_flush(struct ifnet *ifp);
145
146 /* Driver interrupt routines protypes */
147 static void oce_intr(void *arg, int pending);
148 static int oce_setup_intr(POCE_SOFTC sc);
149 static int oce_fast_isr(void *arg);
150 static int oce_alloc_intr(POCE_SOFTC sc, int vector,
151 void (*isr) (void *arg, int pending));
152
153 /* Media callbacks prototypes */
154 static void oce_media_status(struct ifnet *ifp, struct ifmediareq *req);
155 static int oce_media_change(struct ifnet *ifp);
156
157 /* Transmit routines prototypes */
158 static int oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index);
159 static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq);
160 static void oce_process_tx_completion(struct oce_wq *wq);
161 static int oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m,
162 struct oce_wq *wq);
163
164 /* Receive routines prototypes */
165 static int oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
166 static int oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
167 static void oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe);
168 static void oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq);
169 static uint16_t oce_rq_handler_lro(void *arg);
170 static void oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2);
171 static void oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2);
172 static void oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m);
173
174 /* Helper function prototypes in this file */
175 static int oce_attach_ifp(POCE_SOFTC sc);
176 static void oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag);
177 static void oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag);
178 static int oce_vid_config(POCE_SOFTC sc);
179 static void oce_mac_addr_set(POCE_SOFTC sc);
180 static int oce_handle_passthrough(struct ifnet *ifp, caddr_t data);
181 static void oce_local_timer(void *arg);
182 static void oce_if_deactivate(POCE_SOFTC sc);
183 static void oce_if_activate(POCE_SOFTC sc);
184 static void setup_max_queues_want(POCE_SOFTC sc);
185 static void update_queues_got(POCE_SOFTC sc);
186 static void process_link_state(POCE_SOFTC sc,
187 struct oce_async_cqe_link_state *acqe);
188 static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m);
189 static void oce_get_config(POCE_SOFTC sc);
190 static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete);
191 static void oce_read_env_variables(POCE_SOFTC sc);
192
193
194 /* IP specific */
195 #if defined(INET6) || defined(INET)
196 static int oce_init_lro(POCE_SOFTC sc);
197 static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp);
198 #endif
199
200 static device_method_t oce_dispatch[] = {
201 DEVMETHOD(device_probe, oce_probe),
202 DEVMETHOD(device_attach, oce_attach),
203 DEVMETHOD(device_detach, oce_detach),
204 DEVMETHOD(device_shutdown, oce_shutdown),
205
206 DEVMETHOD_END
207 };
208
209 static driver_t oce_driver = {
210 "oce",
211 oce_dispatch,
212 sizeof(OCE_SOFTC)
213 };
214 static devclass_t oce_devclass;
215
216
217 /* global vars */
218 const char component_revision[32] = {"///" COMPONENT_REVISION "///"};
219
220 /* Module capabilites and parameters */
221 uint32_t oce_max_rsp_handled = OCE_MAX_RSP_HANDLED;
222 uint32_t oce_enable_rss = OCE_MODCAP_RSS;
223 uint32_t oce_rq_buf_size = 2048;
224
225 TUNABLE_INT("hw.oce.max_rsp_handled", &oce_max_rsp_handled);
226 TUNABLE_INT("hw.oce.enable_rss", &oce_enable_rss);
227
228
229 /* Supported devices table */
230 static uint32_t supportedDevices[] = {
231 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE2,
232 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE3,
233 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_BE3,
234 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201,
235 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201_VF,
236 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_SH
237 };
238
239
240 DRIVER_MODULE(oce, pci, oce_driver, oce_devclass, 0, 0);
241 MODULE_PNP_INFO("W32:vendor/device", pci, oce, supportedDevices,
242 nitems(supportedDevices));
243 MODULE_DEPEND(oce, pci, 1, 1, 1);
244 MODULE_DEPEND(oce, ether, 1, 1, 1);
245 MODULE_VERSION(oce, 1);
246
247
248 POCE_SOFTC softc_head = NULL;
249 POCE_SOFTC softc_tail = NULL;
250
251 struct oce_rdma_if *oce_rdma_if = NULL;
252
253 /*****************************************************************************
254 * Driver entry points functions *
255 *****************************************************************************/
256
257 static int
oce_probe(device_t dev)258 oce_probe(device_t dev)
259 {
260 uint16_t vendor = 0;
261 uint16_t device = 0;
262 int i = 0;
263 char str[256] = {0};
264 POCE_SOFTC sc;
265
266 sc = device_get_softc(dev);
267 bzero(sc, sizeof(OCE_SOFTC));
268 sc->dev = dev;
269
270 vendor = pci_get_vendor(dev);
271 device = pci_get_device(dev);
272
273 for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) {
274 if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) {
275 if (device == (supportedDevices[i] & 0xffff)) {
276 sprintf(str, "%s:%s", "Emulex CNA NIC function",
277 component_revision);
278 device_set_desc_copy(dev, str);
279
280 switch (device) {
281 case PCI_PRODUCT_BE2:
282 sc->flags |= OCE_FLAGS_BE2;
283 break;
284 case PCI_PRODUCT_BE3:
285 sc->flags |= OCE_FLAGS_BE3;
286 break;
287 case PCI_PRODUCT_XE201:
288 case PCI_PRODUCT_XE201_VF:
289 sc->flags |= OCE_FLAGS_XE201;
290 break;
291 case PCI_PRODUCT_SH:
292 sc->flags |= OCE_FLAGS_SH;
293 break;
294 default:
295 return ENXIO;
296 }
297 return BUS_PROBE_DEFAULT;
298 }
299 }
300 }
301
302 return ENXIO;
303 }
304
305
306 static int
oce_attach(device_t dev)307 oce_attach(device_t dev)
308 {
309 POCE_SOFTC sc;
310 int rc = 0;
311
312 sc = device_get_softc(dev);
313
314 rc = oce_hw_pci_alloc(sc);
315 if (rc)
316 return rc;
317
318 sc->tx_ring_size = OCE_TX_RING_SIZE;
319 sc->rx_ring_size = OCE_RX_RING_SIZE;
320 /* receive fragment size should be multiple of 2K */
321 sc->rq_frag_size = ((oce_rq_buf_size / 2048) * 2048);
322 sc->flow_control = OCE_DEFAULT_FLOW_CONTROL;
323 sc->promisc = OCE_DEFAULT_PROMISCUOUS;
324
325 LOCK_CREATE(&sc->bmbx_lock, "Mailbox_lock");
326 LOCK_CREATE(&sc->dev_lock, "Device_lock");
327
328 /* initialise the hardware */
329 rc = oce_hw_init(sc);
330 if (rc)
331 goto pci_res_free;
332
333 oce_read_env_variables(sc);
334
335 oce_get_config(sc);
336
337 setup_max_queues_want(sc);
338
339 rc = oce_setup_intr(sc);
340 if (rc)
341 goto mbox_free;
342
343 rc = oce_queue_init_all(sc);
344 if (rc)
345 goto intr_free;
346
347 rc = oce_attach_ifp(sc);
348 if (rc)
349 goto queues_free;
350
351 #if defined(INET6) || defined(INET)
352 rc = oce_init_lro(sc);
353 if (rc)
354 goto ifp_free;
355 #endif
356
357 rc = oce_hw_start(sc);
358 if (rc)
359 goto lro_free;
360
361 sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
362 oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST);
363 sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
364 oce_del_vlan, sc, EVENTHANDLER_PRI_FIRST);
365
366 rc = oce_stats_init(sc);
367 if (rc)
368 goto vlan_free;
369
370 oce_add_sysctls(sc);
371
372 callout_init(&sc->timer, CALLOUT_MPSAFE);
373 rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc);
374 if (rc)
375 goto stats_free;
376
377 sc->next =NULL;
378 if (softc_tail != NULL) {
379 softc_tail->next = sc;
380 } else {
381 softc_head = sc;
382 }
383 softc_tail = sc;
384
385 return 0;
386
387 stats_free:
388 callout_drain(&sc->timer);
389 oce_stats_free(sc);
390 vlan_free:
391 if (sc->vlan_attach)
392 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach);
393 if (sc->vlan_detach)
394 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach);
395 oce_hw_intr_disable(sc);
396 lro_free:
397 #if defined(INET6) || defined(INET)
398 oce_free_lro(sc);
399 ifp_free:
400 #endif
401 ether_ifdetach(sc->ifp);
402 if_free(sc->ifp);
403 queues_free:
404 oce_queue_release_all(sc);
405 intr_free:
406 oce_intr_free(sc);
407 mbox_free:
408 oce_dma_free(sc, &sc->bsmbx);
409 pci_res_free:
410 oce_hw_pci_free(sc);
411 LOCK_DESTROY(&sc->dev_lock);
412 LOCK_DESTROY(&sc->bmbx_lock);
413 return rc;
414
415 }
416
417
418 static int
oce_detach(device_t dev)419 oce_detach(device_t dev)
420 {
421 POCE_SOFTC sc = device_get_softc(dev);
422 POCE_SOFTC poce_sc_tmp, *ppoce_sc_tmp1, poce_sc_tmp2 = NULL;
423
424 poce_sc_tmp = softc_head;
425 ppoce_sc_tmp1 = &softc_head;
426 while (poce_sc_tmp != NULL) {
427 if (poce_sc_tmp == sc) {
428 *ppoce_sc_tmp1 = sc->next;
429 if (sc->next == NULL) {
430 softc_tail = poce_sc_tmp2;
431 }
432 break;
433 }
434 poce_sc_tmp2 = poce_sc_tmp;
435 ppoce_sc_tmp1 = &poce_sc_tmp->next;
436 poce_sc_tmp = poce_sc_tmp->next;
437 }
438
439 LOCK(&sc->dev_lock);
440 oce_if_deactivate(sc);
441 UNLOCK(&sc->dev_lock);
442
443 callout_drain(&sc->timer);
444
445 if (sc->vlan_attach != NULL)
446 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach);
447 if (sc->vlan_detach != NULL)
448 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach);
449
450 ether_ifdetach(sc->ifp);
451
452 if_free(sc->ifp);
453
454 oce_hw_shutdown(sc);
455
456 bus_generic_detach(dev);
457
458 return 0;
459 }
460
461
462 static int
oce_shutdown(device_t dev)463 oce_shutdown(device_t dev)
464 {
465 int rc;
466
467 rc = oce_detach(dev);
468
469 return rc;
470 }
471
472
473 static int
oce_ioctl(struct ifnet * ifp,u_long command,caddr_t data)474 oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
475 {
476 struct ifreq *ifr = (struct ifreq *)data;
477 POCE_SOFTC sc = ifp->if_softc;
478 struct ifi2creq i2c;
479 uint8_t offset = 0;
480 int rc = 0;
481 uint32_t u;
482
483 switch (command) {
484
485 case SIOCGIFMEDIA:
486 rc = ifmedia_ioctl(ifp, ifr, &sc->media, command);
487 break;
488
489 case SIOCSIFMTU:
490 if (ifr->ifr_mtu > OCE_MAX_MTU)
491 rc = EINVAL;
492 else
493 ifp->if_mtu = ifr->ifr_mtu;
494 break;
495
496 case SIOCSIFFLAGS:
497 if (ifp->if_flags & IFF_UP) {
498 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
499 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
500 oce_init(sc);
501 }
502 device_printf(sc->dev, "Interface Up\n");
503 } else {
504 LOCK(&sc->dev_lock);
505
506 sc->ifp->if_drv_flags &=
507 ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
508 oce_if_deactivate(sc);
509
510 UNLOCK(&sc->dev_lock);
511
512 device_printf(sc->dev, "Interface Down\n");
513 }
514
515 if ((ifp->if_flags & IFF_PROMISC) && !sc->promisc) {
516 if (!oce_rxf_set_promiscuous(sc, (1 | (1 << 1))))
517 sc->promisc = TRUE;
518 } else if (!(ifp->if_flags & IFF_PROMISC) && sc->promisc) {
519 if (!oce_rxf_set_promiscuous(sc, 0))
520 sc->promisc = FALSE;
521 }
522
523 break;
524
525 case SIOCADDMULTI:
526 case SIOCDELMULTI:
527 rc = oce_hw_update_multicast(sc);
528 if (rc)
529 device_printf(sc->dev,
530 "Update multicast address failed\n");
531 break;
532
533 case SIOCSIFCAP:
534 u = ifr->ifr_reqcap ^ ifp->if_capenable;
535
536 if (u & IFCAP_TXCSUM) {
537 ifp->if_capenable ^= IFCAP_TXCSUM;
538 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
539
540 if (IFCAP_TSO & ifp->if_capenable &&
541 !(IFCAP_TXCSUM & ifp->if_capenable)) {
542 ifp->if_capenable &= ~IFCAP_TSO;
543 ifp->if_hwassist &= ~CSUM_TSO;
544 if_printf(ifp,
545 "TSO disabled due to -txcsum.\n");
546 }
547 }
548
549 if (u & IFCAP_RXCSUM)
550 ifp->if_capenable ^= IFCAP_RXCSUM;
551
552 if (u & IFCAP_TSO4) {
553 ifp->if_capenable ^= IFCAP_TSO4;
554
555 if (IFCAP_TSO & ifp->if_capenable) {
556 if (IFCAP_TXCSUM & ifp->if_capenable)
557 ifp->if_hwassist |= CSUM_TSO;
558 else {
559 ifp->if_capenable &= ~IFCAP_TSO;
560 ifp->if_hwassist &= ~CSUM_TSO;
561 if_printf(ifp,
562 "Enable txcsum first.\n");
563 rc = EAGAIN;
564 }
565 } else
566 ifp->if_hwassist &= ~CSUM_TSO;
567 }
568
569 if (u & IFCAP_VLAN_HWTAGGING)
570 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
571
572 if (u & IFCAP_VLAN_HWFILTER) {
573 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
574 oce_vid_config(sc);
575 }
576 #if defined(INET6) || defined(INET)
577 if (u & IFCAP_LRO) {
578 ifp->if_capenable ^= IFCAP_LRO;
579 if(sc->enable_hwlro) {
580 if(ifp->if_capenable & IFCAP_LRO) {
581 rc = oce_mbox_nic_set_iface_lro_config(sc, 1);
582 }else {
583 rc = oce_mbox_nic_set_iface_lro_config(sc, 0);
584 }
585 }
586 }
587 #endif
588
589 break;
590
591 case SIOCGI2C:
592 rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
593 if (rc)
594 break;
595
596 if (i2c.dev_addr != PAGE_NUM_A0 &&
597 i2c.dev_addr != PAGE_NUM_A2) {
598 rc = EINVAL;
599 break;
600 }
601
602 if (i2c.len > sizeof(i2c.data)) {
603 rc = EINVAL;
604 break;
605 }
606
607 rc = oce_mbox_read_transrecv_data(sc, i2c.dev_addr);
608 if(rc) {
609 rc = -rc;
610 break;
611 }
612
613 if (i2c.dev_addr == PAGE_NUM_A0)
614 offset = i2c.offset;
615 else
616 offset = TRANSCEIVER_A0_SIZE + i2c.offset;
617
618 memcpy(&i2c.data[0], &sfp_vpd_dump_buffer[offset], i2c.len);
619
620 rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c));
621 break;
622
623 case SIOCGPRIVATE_0:
624 rc = priv_check(curthread, PRIV_DRIVER);
625 if (rc != 0)
626 break;
627 rc = oce_handle_passthrough(ifp, data);
628 break;
629 default:
630 rc = ether_ioctl(ifp, command, data);
631 break;
632 }
633
634 return rc;
635 }
636
637
638 static void
oce_init(void * arg)639 oce_init(void *arg)
640 {
641 POCE_SOFTC sc = arg;
642
643 LOCK(&sc->dev_lock);
644
645 if (sc->ifp->if_flags & IFF_UP) {
646 oce_if_deactivate(sc);
647 oce_if_activate(sc);
648 }
649
650 UNLOCK(&sc->dev_lock);
651
652 }
653
654
655 static int
oce_multiq_start(struct ifnet * ifp,struct mbuf * m)656 oce_multiq_start(struct ifnet *ifp, struct mbuf *m)
657 {
658 POCE_SOFTC sc = ifp->if_softc;
659 struct oce_wq *wq = NULL;
660 int queue_index = 0;
661 int status = 0;
662
663 if (!sc->link_status)
664 return ENXIO;
665
666 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
667 queue_index = m->m_pkthdr.flowid % sc->nwqs;
668
669 wq = sc->wq[queue_index];
670
671 LOCK(&wq->tx_lock);
672 status = oce_multiq_transmit(ifp, m, wq);
673 UNLOCK(&wq->tx_lock);
674
675 return status;
676
677 }
678
679
680 static void
oce_multiq_flush(struct ifnet * ifp)681 oce_multiq_flush(struct ifnet *ifp)
682 {
683 POCE_SOFTC sc = ifp->if_softc;
684 struct mbuf *m;
685 int i = 0;
686
687 for (i = 0; i < sc->nwqs; i++) {
688 while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL)
689 m_freem(m);
690 }
691 if_qflush(ifp);
692 }
693
694
695
696 /*****************************************************************************
697 * Driver interrupt routines functions *
698 *****************************************************************************/
699
700 static void
oce_intr(void * arg,int pending)701 oce_intr(void *arg, int pending)
702 {
703
704 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg;
705 POCE_SOFTC sc = ii->sc;
706 struct oce_eq *eq = ii->eq;
707 struct oce_eqe *eqe;
708 struct oce_cq *cq = NULL;
709 int i, num_eqes = 0;
710
711
712 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map,
713 BUS_DMASYNC_POSTWRITE);
714 do {
715 eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe);
716 if (eqe->evnt == 0)
717 break;
718 eqe->evnt = 0;
719 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map,
720 BUS_DMASYNC_POSTWRITE);
721 RING_GET(eq->ring, 1);
722 num_eqes++;
723
724 } while (TRUE);
725
726 if (!num_eqes)
727 goto eq_arm; /* Spurious */
728
729 /* Clear EQ entries, but dont arm */
730 oce_arm_eq(sc, eq->eq_id, num_eqes, FALSE, FALSE);
731
732 /* Process TX, RX and MCC. But dont arm CQ*/
733 for (i = 0; i < eq->cq_valid; i++) {
734 cq = eq->cq[i];
735 (*cq->cq_handler)(cq->cb_arg);
736 }
737
738 /* Arm all cqs connected to this EQ */
739 for (i = 0; i < eq->cq_valid; i++) {
740 cq = eq->cq[i];
741 oce_arm_cq(sc, cq->cq_id, 0, TRUE);
742 }
743
744 eq_arm:
745 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE);
746
747 return;
748 }
749
750
751 static int
oce_setup_intr(POCE_SOFTC sc)752 oce_setup_intr(POCE_SOFTC sc)
753 {
754 int rc = 0, use_intx = 0;
755 int vector = 0, req_vectors = 0;
756 int tot_req_vectors, tot_vectors;
757
758 if (is_rss_enabled(sc))
759 req_vectors = MAX((sc->nrqs - 1), sc->nwqs);
760 else
761 req_vectors = 1;
762
763 tot_req_vectors = req_vectors;
764 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
765 if (req_vectors > 1) {
766 tot_req_vectors += OCE_RDMA_VECTORS;
767 sc->roce_intr_count = OCE_RDMA_VECTORS;
768 }
769 }
770
771 if (sc->flags & OCE_FLAGS_MSIX_CAPABLE) {
772 sc->intr_count = req_vectors;
773 tot_vectors = tot_req_vectors;
774 rc = pci_alloc_msix(sc->dev, &tot_vectors);
775 if (rc != 0) {
776 use_intx = 1;
777 pci_release_msi(sc->dev);
778 } else {
779 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
780 if (tot_vectors < tot_req_vectors) {
781 if (sc->intr_count < (2 * OCE_RDMA_VECTORS)) {
782 sc->roce_intr_count = (tot_vectors / 2);
783 }
784 sc->intr_count = tot_vectors - sc->roce_intr_count;
785 }
786 } else {
787 sc->intr_count = tot_vectors;
788 }
789 sc->flags |= OCE_FLAGS_USING_MSIX;
790 }
791 } else
792 use_intx = 1;
793
794 if (use_intx)
795 sc->intr_count = 1;
796
797 /* Scale number of queues based on intr we got */
798 update_queues_got(sc);
799
800 if (use_intx) {
801 device_printf(sc->dev, "Using legacy interrupt\n");
802 rc = oce_alloc_intr(sc, vector, oce_intr);
803 if (rc)
804 goto error;
805 } else {
806 for (; vector < sc->intr_count; vector++) {
807 rc = oce_alloc_intr(sc, vector, oce_intr);
808 if (rc)
809 goto error;
810 }
811 }
812
813 return 0;
814 error:
815 oce_intr_free(sc);
816 return rc;
817 }
818
819
820 static int
oce_fast_isr(void * arg)821 oce_fast_isr(void *arg)
822 {
823 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg;
824 POCE_SOFTC sc = ii->sc;
825
826 if (ii->eq == NULL)
827 return FILTER_STRAY;
828
829 oce_arm_eq(sc, ii->eq->eq_id, 0, FALSE, TRUE);
830
831 taskqueue_enqueue(ii->tq, &ii->task);
832
833 ii->eq->intr++;
834
835 return FILTER_HANDLED;
836 }
837
838
839 static int
oce_alloc_intr(POCE_SOFTC sc,int vector,void (* isr)(void * arg,int pending))840 oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending))
841 {
842 POCE_INTR_INFO ii;
843 int rc = 0, rr;
844
845 if (vector >= OCE_MAX_EQ)
846 return (EINVAL);
847
848 ii = &sc->intrs[vector];
849
850 /* Set the resource id for the interrupt.
851 * MSIx is vector + 1 for the resource id,
852 * INTx is 0 for the resource id.
853 */
854 if (sc->flags & OCE_FLAGS_USING_MSIX)
855 rr = vector + 1;
856 else
857 rr = 0;
858 ii->intr_res = bus_alloc_resource_any(sc->dev,
859 SYS_RES_IRQ,
860 &rr, RF_ACTIVE|RF_SHAREABLE);
861 ii->irq_rr = rr;
862 if (ii->intr_res == NULL) {
863 device_printf(sc->dev,
864 "Could not allocate interrupt\n");
865 rc = ENXIO;
866 return rc;
867 }
868
869 TASK_INIT(&ii->task, 0, isr, ii);
870 ii->vector = vector;
871 sprintf(ii->task_name, "oce_task[%d]", ii->vector);
872 ii->tq = taskqueue_create_fast(ii->task_name,
873 M_NOWAIT,
874 taskqueue_thread_enqueue,
875 &ii->tq);
876 taskqueue_start_threads(&ii->tq, 1, PI_NET, "%s taskq",
877 device_get_nameunit(sc->dev));
878
879 ii->sc = sc;
880 rc = bus_setup_intr(sc->dev,
881 ii->intr_res,
882 INTR_TYPE_NET,
883 oce_fast_isr, NULL, ii, &ii->tag);
884 return rc;
885
886 }
887
888
889 void
oce_intr_free(POCE_SOFTC sc)890 oce_intr_free(POCE_SOFTC sc)
891 {
892 int i = 0;
893
894 for (i = 0; i < sc->intr_count; i++) {
895
896 if (sc->intrs[i].tag != NULL)
897 bus_teardown_intr(sc->dev, sc->intrs[i].intr_res,
898 sc->intrs[i].tag);
899 if (sc->intrs[i].tq != NULL)
900 taskqueue_free(sc->intrs[i].tq);
901
902 if (sc->intrs[i].intr_res != NULL)
903 bus_release_resource(sc->dev, SYS_RES_IRQ,
904 sc->intrs[i].irq_rr,
905 sc->intrs[i].intr_res);
906 sc->intrs[i].tag = NULL;
907 sc->intrs[i].intr_res = NULL;
908 }
909
910 if (sc->flags & OCE_FLAGS_USING_MSIX)
911 pci_release_msi(sc->dev);
912
913 }
914
915
916
917 /******************************************************************************
918 * Media callbacks functions *
919 ******************************************************************************/
920
921 static void
oce_media_status(struct ifnet * ifp,struct ifmediareq * req)922 oce_media_status(struct ifnet *ifp, struct ifmediareq *req)
923 {
924 POCE_SOFTC sc = (POCE_SOFTC) ifp->if_softc;
925
926
927 req->ifm_status = IFM_AVALID;
928 req->ifm_active = IFM_ETHER;
929
930 if (sc->link_status == 1)
931 req->ifm_status |= IFM_ACTIVE;
932 else
933 return;
934
935 switch (sc->link_speed) {
936 case 1: /* 10 Mbps */
937 req->ifm_active |= IFM_10_T | IFM_FDX;
938 sc->speed = 10;
939 break;
940 case 2: /* 100 Mbps */
941 req->ifm_active |= IFM_100_TX | IFM_FDX;
942 sc->speed = 100;
943 break;
944 case 3: /* 1 Gbps */
945 req->ifm_active |= IFM_1000_T | IFM_FDX;
946 sc->speed = 1000;
947 break;
948 case 4: /* 10 Gbps */
949 req->ifm_active |= IFM_10G_SR | IFM_FDX;
950 sc->speed = 10000;
951 break;
952 case 5: /* 20 Gbps */
953 req->ifm_active |= IFM_10G_SR | IFM_FDX;
954 sc->speed = 20000;
955 break;
956 case 6: /* 25 Gbps */
957 req->ifm_active |= IFM_10G_SR | IFM_FDX;
958 sc->speed = 25000;
959 break;
960 case 7: /* 40 Gbps */
961 req->ifm_active |= IFM_40G_SR4 | IFM_FDX;
962 sc->speed = 40000;
963 break;
964 default:
965 sc->speed = 0;
966 break;
967 }
968
969 return;
970 }
971
972
973 int
oce_media_change(struct ifnet * ifp)974 oce_media_change(struct ifnet *ifp)
975 {
976 return 0;
977 }
978
979
oce_is_pkt_dest_bmc(POCE_SOFTC sc,struct mbuf * m,boolean_t * os2bmc,struct mbuf ** m_new)980 static void oce_is_pkt_dest_bmc(POCE_SOFTC sc,
981 struct mbuf *m, boolean_t *os2bmc,
982 struct mbuf **m_new)
983 {
984 struct ether_header *eh = NULL;
985
986 eh = mtod(m, struct ether_header *);
987
988 if (!is_os2bmc_enabled(sc) || *os2bmc) {
989 *os2bmc = FALSE;
990 goto done;
991 }
992 if (!ETHER_IS_MULTICAST(eh->ether_dhost))
993 goto done;
994
995 if (is_mc_allowed_on_bmc(sc, eh) ||
996 is_bc_allowed_on_bmc(sc, eh) ||
997 is_arp_allowed_on_bmc(sc, ntohs(eh->ether_type))) {
998 *os2bmc = TRUE;
999 goto done;
1000 }
1001
1002 if (mtod(m, struct ip *)->ip_p == IPPROTO_IPV6) {
1003 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
1004 uint8_t nexthdr = ip6->ip6_nxt;
1005 if (nexthdr == IPPROTO_ICMPV6) {
1006 struct icmp6_hdr *icmp6 = (struct icmp6_hdr *)(ip6 + 1);
1007 switch (icmp6->icmp6_type) {
1008 case ND_ROUTER_ADVERT:
1009 *os2bmc = is_ipv6_ra_filt_enabled(sc);
1010 goto done;
1011 case ND_NEIGHBOR_ADVERT:
1012 *os2bmc = is_ipv6_na_filt_enabled(sc);
1013 goto done;
1014 default:
1015 break;
1016 }
1017 }
1018 }
1019
1020 if (mtod(m, struct ip *)->ip_p == IPPROTO_UDP) {
1021 struct ip *ip = mtod(m, struct ip *);
1022 int iphlen = ip->ip_hl << 2;
1023 struct udphdr *uh = (struct udphdr *)((caddr_t)ip + iphlen);
1024 switch (uh->uh_dport) {
1025 case DHCP_CLIENT_PORT:
1026 *os2bmc = is_dhcp_client_filt_enabled(sc);
1027 goto done;
1028 case DHCP_SERVER_PORT:
1029 *os2bmc = is_dhcp_srvr_filt_enabled(sc);
1030 goto done;
1031 case NET_BIOS_PORT1:
1032 case NET_BIOS_PORT2:
1033 *os2bmc = is_nbios_filt_enabled(sc);
1034 goto done;
1035 case DHCPV6_RAS_PORT:
1036 *os2bmc = is_ipv6_ras_filt_enabled(sc);
1037 goto done;
1038 default:
1039 break;
1040 }
1041 }
1042 done:
1043 if (*os2bmc) {
1044 *m_new = m_dup(m, M_NOWAIT);
1045 if (!*m_new) {
1046 *os2bmc = FALSE;
1047 return;
1048 }
1049 *m_new = oce_insert_vlan_tag(sc, *m_new, NULL);
1050 }
1051 }
1052
1053
1054
1055 /*****************************************************************************
1056 * Transmit routines functions *
1057 *****************************************************************************/
1058
1059 static int
oce_tx(POCE_SOFTC sc,struct mbuf ** mpp,int wq_index)1060 oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index)
1061 {
1062 int rc = 0, i, retry_cnt = 0;
1063 bus_dma_segment_t segs[OCE_MAX_TX_ELEMENTS];
1064 struct mbuf *m, *m_temp, *m_new = NULL;
1065 struct oce_wq *wq = sc->wq[wq_index];
1066 struct oce_packet_desc *pd;
1067 struct oce_nic_hdr_wqe *nichdr;
1068 struct oce_nic_frag_wqe *nicfrag;
1069 struct ether_header *eh = NULL;
1070 int num_wqes;
1071 uint32_t reg_value;
1072 boolean_t complete = TRUE;
1073 boolean_t os2bmc = FALSE;
1074
1075 m = *mpp;
1076 if (!m)
1077 return EINVAL;
1078
1079 if (!(m->m_flags & M_PKTHDR)) {
1080 rc = ENXIO;
1081 goto free_ret;
1082 }
1083
1084 /* Don't allow non-TSO packets longer than MTU */
1085 if (!is_tso_pkt(m)) {
1086 eh = mtod(m, struct ether_header *);
1087 if(m->m_pkthdr.len > ETHER_MAX_FRAME(sc->ifp, eh->ether_type, FALSE))
1088 goto free_ret;
1089 }
1090
1091 if(oce_tx_asic_stall_verify(sc, m)) {
1092 m = oce_insert_vlan_tag(sc, m, &complete);
1093 if(!m) {
1094 device_printf(sc->dev, "Insertion unsuccessful\n");
1095 return 0;
1096 }
1097
1098 }
1099
1100 /* Lancer, SH ASIC has a bug wherein Packets that are 32 bytes or less
1101 * may cause a transmit stall on that port. So the work-around is to
1102 * pad short packets (<= 32 bytes) to a 36-byte length.
1103 */
1104 if(IS_SH(sc) || IS_XE201(sc) ) {
1105 if(m->m_pkthdr.len <= 32) {
1106 char buf[36];
1107 bzero((void *)buf, 36);
1108 m_append(m, (36 - m->m_pkthdr.len), buf);
1109 }
1110 }
1111
1112 tx_start:
1113 if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1114 /* consolidate packet buffers for TSO/LSO segment offload */
1115 #if defined(INET6) || defined(INET)
1116 m = oce_tso_setup(sc, mpp);
1117 #else
1118 m = NULL;
1119 #endif
1120 if (m == NULL) {
1121 rc = ENXIO;
1122 goto free_ret;
1123 }
1124 }
1125
1126
1127 pd = &wq->pckts[wq->pkt_desc_head];
1128
1129 retry:
1130 rc = bus_dmamap_load_mbuf_sg(wq->tag,
1131 pd->map,
1132 m, segs, &pd->nsegs, BUS_DMA_NOWAIT);
1133 if (rc == 0) {
1134 num_wqes = pd->nsegs + 1;
1135 if (IS_BE(sc) || IS_SH(sc)) {
1136 /*Dummy required only for BE3.*/
1137 if (num_wqes & 1)
1138 num_wqes++;
1139 }
1140 if (num_wqes >= RING_NUM_FREE(wq->ring)) {
1141 bus_dmamap_unload(wq->tag, pd->map);
1142 return EBUSY;
1143 }
1144 atomic_store_rel_int(&wq->pkt_desc_head,
1145 (wq->pkt_desc_head + 1) % \
1146 OCE_WQ_PACKET_ARRAY_SIZE);
1147 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE);
1148 pd->mbuf = m;
1149
1150 nichdr =
1151 RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe);
1152 nichdr->u0.dw[0] = 0;
1153 nichdr->u0.dw[1] = 0;
1154 nichdr->u0.dw[2] = 0;
1155 nichdr->u0.dw[3] = 0;
1156
1157 nichdr->u0.s.complete = complete;
1158 nichdr->u0.s.mgmt = os2bmc;
1159 nichdr->u0.s.event = 1;
1160 nichdr->u0.s.crc = 1;
1161 nichdr->u0.s.forward = 0;
1162 nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0;
1163 nichdr->u0.s.udpcs =
1164 (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0;
1165 nichdr->u0.s.tcpcs =
1166 (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0;
1167 nichdr->u0.s.num_wqe = num_wqes;
1168 nichdr->u0.s.total_length = m->m_pkthdr.len;
1169
1170 if (m->m_flags & M_VLANTAG) {
1171 nichdr->u0.s.vlan = 1; /*Vlan present*/
1172 nichdr->u0.s.vlan_tag = m->m_pkthdr.ether_vtag;
1173 }
1174
1175 if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1176 if (m->m_pkthdr.tso_segsz) {
1177 nichdr->u0.s.lso = 1;
1178 nichdr->u0.s.lso_mss = m->m_pkthdr.tso_segsz;
1179 }
1180 if (!IS_BE(sc) || !IS_SH(sc))
1181 nichdr->u0.s.ipcs = 1;
1182 }
1183
1184 RING_PUT(wq->ring, 1);
1185 atomic_add_int(&wq->ring->num_used, 1);
1186
1187 for (i = 0; i < pd->nsegs; i++) {
1188 nicfrag =
1189 RING_GET_PRODUCER_ITEM_VA(wq->ring,
1190 struct oce_nic_frag_wqe);
1191 nicfrag->u0.s.rsvd0 = 0;
1192 nicfrag->u0.s.frag_pa_hi = ADDR_HI(segs[i].ds_addr);
1193 nicfrag->u0.s.frag_pa_lo = ADDR_LO(segs[i].ds_addr);
1194 nicfrag->u0.s.frag_len = segs[i].ds_len;
1195 pd->wqe_idx = wq->ring->pidx;
1196 RING_PUT(wq->ring, 1);
1197 atomic_add_int(&wq->ring->num_used, 1);
1198 }
1199 if (num_wqes > (pd->nsegs + 1)) {
1200 nicfrag =
1201 RING_GET_PRODUCER_ITEM_VA(wq->ring,
1202 struct oce_nic_frag_wqe);
1203 nicfrag->u0.dw[0] = 0;
1204 nicfrag->u0.dw[1] = 0;
1205 nicfrag->u0.dw[2] = 0;
1206 nicfrag->u0.dw[3] = 0;
1207 pd->wqe_idx = wq->ring->pidx;
1208 RING_PUT(wq->ring, 1);
1209 atomic_add_int(&wq->ring->num_used, 1);
1210 pd->nsegs++;
1211 }
1212
1213 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1);
1214 wq->tx_stats.tx_reqs++;
1215 wq->tx_stats.tx_wrbs += num_wqes;
1216 wq->tx_stats.tx_bytes += m->m_pkthdr.len;
1217 wq->tx_stats.tx_pkts++;
1218
1219 bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map,
1220 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1221 reg_value = (num_wqes << 16) | wq->wq_id;
1222
1223 /* if os2bmc is not enabled or if the pkt is already tagged as
1224 bmc, do nothing
1225 */
1226 oce_is_pkt_dest_bmc(sc, m, &os2bmc, &m_new);
1227
1228 OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value);
1229
1230 } else if (rc == EFBIG) {
1231 if (retry_cnt == 0) {
1232 m_temp = m_defrag(m, M_NOWAIT);
1233 if (m_temp == NULL)
1234 goto free_ret;
1235 m = m_temp;
1236 *mpp = m_temp;
1237 retry_cnt = retry_cnt + 1;
1238 goto retry;
1239 } else
1240 goto free_ret;
1241 } else if (rc == ENOMEM)
1242 return rc;
1243 else
1244 goto free_ret;
1245
1246 if (os2bmc) {
1247 m = m_new;
1248 goto tx_start;
1249 }
1250
1251 return 0;
1252
1253 free_ret:
1254 m_freem(*mpp);
1255 *mpp = NULL;
1256 return rc;
1257 }
1258
1259
1260 static void
oce_process_tx_completion(struct oce_wq * wq)1261 oce_process_tx_completion(struct oce_wq *wq)
1262 {
1263 struct oce_packet_desc *pd;
1264 POCE_SOFTC sc = (POCE_SOFTC) wq->parent;
1265 struct mbuf *m;
1266
1267 pd = &wq->pckts[wq->pkt_desc_tail];
1268 atomic_store_rel_int(&wq->pkt_desc_tail,
1269 (wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE);
1270 atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1);
1271 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1272 bus_dmamap_unload(wq->tag, pd->map);
1273
1274 m = pd->mbuf;
1275 m_freem(m);
1276 pd->mbuf = NULL;
1277
1278
1279 if (sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) {
1280 if (wq->ring->num_used < (wq->ring->num_items / 2)) {
1281 sc->ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE);
1282 oce_tx_restart(sc, wq);
1283 }
1284 }
1285 }
1286
1287
1288 static void
oce_tx_restart(POCE_SOFTC sc,struct oce_wq * wq)1289 oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq)
1290 {
1291
1292 if ((sc->ifp->if_drv_flags & IFF_DRV_RUNNING) != IFF_DRV_RUNNING)
1293 return;
1294
1295 #if __FreeBSD_version >= 800000
1296 if (!drbr_empty(sc->ifp, wq->br))
1297 #else
1298 if (!IFQ_DRV_IS_EMPTY(&sc->ifp->if_snd))
1299 #endif
1300 taskqueue_enqueue(taskqueue_swi, &wq->txtask);
1301
1302 }
1303
1304
1305 #if defined(INET6) || defined(INET)
1306 static struct mbuf *
oce_tso_setup(POCE_SOFTC sc,struct mbuf ** mpp)1307 oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp)
1308 {
1309 struct mbuf *m;
1310 #ifdef INET
1311 struct ip *ip;
1312 #endif
1313 #ifdef INET6
1314 struct ip6_hdr *ip6;
1315 #endif
1316 struct ether_vlan_header *eh;
1317 struct tcphdr *th;
1318 uint16_t etype;
1319 int total_len = 0, ehdrlen = 0;
1320
1321 m = *mpp;
1322
1323 if (M_WRITABLE(m) == 0) {
1324 m = m_dup(*mpp, M_NOWAIT);
1325 if (!m)
1326 return NULL;
1327 m_freem(*mpp);
1328 *mpp = m;
1329 }
1330
1331 eh = mtod(m, struct ether_vlan_header *);
1332 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
1333 etype = ntohs(eh->evl_proto);
1334 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
1335 } else {
1336 etype = ntohs(eh->evl_encap_proto);
1337 ehdrlen = ETHER_HDR_LEN;
1338 }
1339
1340 switch (etype) {
1341 #ifdef INET
1342 case ETHERTYPE_IP:
1343 ip = (struct ip *)(m->m_data + ehdrlen);
1344 if (ip->ip_p != IPPROTO_TCP)
1345 return NULL;
1346 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
1347
1348 total_len = ehdrlen + (ip->ip_hl << 2) + (th->th_off << 2);
1349 break;
1350 #endif
1351 #ifdef INET6
1352 case ETHERTYPE_IPV6:
1353 ip6 = (struct ip6_hdr *)(m->m_data + ehdrlen);
1354 if (ip6->ip6_nxt != IPPROTO_TCP)
1355 return NULL;
1356 th = (struct tcphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr));
1357
1358 total_len = ehdrlen + sizeof(struct ip6_hdr) + (th->th_off << 2);
1359 break;
1360 #endif
1361 default:
1362 return NULL;
1363 }
1364
1365 m = m_pullup(m, total_len);
1366 if (!m)
1367 return NULL;
1368 *mpp = m;
1369 return m;
1370
1371 }
1372 #endif /* INET6 || INET */
1373
1374 void
oce_tx_task(void * arg,int npending)1375 oce_tx_task(void *arg, int npending)
1376 {
1377 struct oce_wq *wq = arg;
1378 POCE_SOFTC sc = wq->parent;
1379 struct ifnet *ifp = sc->ifp;
1380 int rc = 0;
1381
1382 #if __FreeBSD_version >= 800000
1383 LOCK(&wq->tx_lock);
1384 rc = oce_multiq_transmit(ifp, NULL, wq);
1385 if (rc) {
1386 device_printf(sc->dev,
1387 "TX[%d] restart failed\n", wq->queue_index);
1388 }
1389 UNLOCK(&wq->tx_lock);
1390 #else
1391 oce_start(ifp);
1392 #endif
1393
1394 }
1395
1396
1397 void
oce_start(struct ifnet * ifp)1398 oce_start(struct ifnet *ifp)
1399 {
1400 POCE_SOFTC sc = ifp->if_softc;
1401 struct mbuf *m;
1402 int rc = 0;
1403 int def_q = 0; /* Defualt tx queue is 0*/
1404
1405 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
1406 IFF_DRV_RUNNING)
1407 return;
1408
1409 if (!sc->link_status)
1410 return;
1411
1412 do {
1413 IF_DEQUEUE(&sc->ifp->if_snd, m);
1414 if (m == NULL)
1415 break;
1416
1417 LOCK(&sc->wq[def_q]->tx_lock);
1418 rc = oce_tx(sc, &m, def_q);
1419 UNLOCK(&sc->wq[def_q]->tx_lock);
1420 if (rc) {
1421 if (m != NULL) {
1422 sc->wq[def_q]->tx_stats.tx_stops ++;
1423 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1424 IFQ_DRV_PREPEND(&ifp->if_snd, m);
1425 m = NULL;
1426 }
1427 break;
1428 }
1429 if (m != NULL)
1430 ETHER_BPF_MTAP(ifp, m);
1431
1432 } while (TRUE);
1433
1434 return;
1435 }
1436
1437
1438 /* Handle the Completion Queue for transmit */
1439 uint16_t
oce_wq_handler(void * arg)1440 oce_wq_handler(void *arg)
1441 {
1442 struct oce_wq *wq = (struct oce_wq *)arg;
1443 POCE_SOFTC sc = wq->parent;
1444 struct oce_cq *cq = wq->cq;
1445 struct oce_nic_tx_cqe *cqe;
1446 int num_cqes = 0;
1447
1448 LOCK(&wq->tx_compl_lock);
1449 bus_dmamap_sync(cq->ring->dma.tag,
1450 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
1451 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
1452 while (cqe->u0.dw[3]) {
1453 DW_SWAP((uint32_t *) cqe, sizeof(oce_wq_cqe));
1454
1455 wq->ring->cidx = cqe->u0.s.wqe_index + 1;
1456 if (wq->ring->cidx >= wq->ring->num_items)
1457 wq->ring->cidx -= wq->ring->num_items;
1458
1459 oce_process_tx_completion(wq);
1460 wq->tx_stats.tx_compl++;
1461 cqe->u0.dw[3] = 0;
1462 RING_GET(cq->ring, 1);
1463 bus_dmamap_sync(cq->ring->dma.tag,
1464 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
1465 cqe =
1466 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
1467 num_cqes++;
1468 }
1469
1470 if (num_cqes)
1471 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE);
1472
1473 UNLOCK(&wq->tx_compl_lock);
1474 return num_cqes;
1475 }
1476
1477
1478 static int
oce_multiq_transmit(struct ifnet * ifp,struct mbuf * m,struct oce_wq * wq)1479 oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, struct oce_wq *wq)
1480 {
1481 POCE_SOFTC sc = ifp->if_softc;
1482 int status = 0, queue_index = 0;
1483 struct mbuf *next = NULL;
1484 struct buf_ring *br = NULL;
1485
1486 br = wq->br;
1487 queue_index = wq->queue_index;
1488
1489 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
1490 IFF_DRV_RUNNING) {
1491 if (m != NULL)
1492 status = drbr_enqueue(ifp, br, m);
1493 return status;
1494 }
1495
1496 if (m != NULL) {
1497 if ((status = drbr_enqueue(ifp, br, m)) != 0)
1498 return status;
1499 }
1500 while ((next = drbr_peek(ifp, br)) != NULL) {
1501 if (oce_tx(sc, &next, queue_index)) {
1502 if (next == NULL) {
1503 drbr_advance(ifp, br);
1504 } else {
1505 drbr_putback(ifp, br, next);
1506 wq->tx_stats.tx_stops ++;
1507 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1508 }
1509 break;
1510 }
1511 drbr_advance(ifp, br);
1512 if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len);
1513 if (next->m_flags & M_MCAST)
1514 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
1515 ETHER_BPF_MTAP(ifp, next);
1516 }
1517
1518 return 0;
1519 }
1520
1521
1522
1523
1524 /*****************************************************************************
1525 * Receive routines functions *
1526 *****************************************************************************/
1527
1528 static void
oce_correct_header(struct mbuf * m,struct nic_hwlro_cqe_part1 * cqe1,struct nic_hwlro_cqe_part2 * cqe2)1529 oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2)
1530 {
1531 uint32_t *p;
1532 struct ether_header *eh = NULL;
1533 struct tcphdr *tcp_hdr = NULL;
1534 struct ip *ip4_hdr = NULL;
1535 struct ip6_hdr *ip6 = NULL;
1536 uint32_t payload_len = 0;
1537
1538 eh = mtod(m, struct ether_header *);
1539 /* correct IP header */
1540 if(!cqe2->ipv6_frame) {
1541 ip4_hdr = (struct ip *)((char*)eh + sizeof(struct ether_header));
1542 ip4_hdr->ip_ttl = cqe2->frame_lifespan;
1543 ip4_hdr->ip_len = htons(cqe2->coalesced_size - sizeof(struct ether_header));
1544 tcp_hdr = (struct tcphdr *)((char*)ip4_hdr + sizeof(struct ip));
1545 }else {
1546 ip6 = (struct ip6_hdr *)((char*)eh + sizeof(struct ether_header));
1547 ip6->ip6_ctlun.ip6_un1.ip6_un1_hlim = cqe2->frame_lifespan;
1548 payload_len = cqe2->coalesced_size - sizeof(struct ether_header)
1549 - sizeof(struct ip6_hdr);
1550 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = htons(payload_len);
1551 tcp_hdr = (struct tcphdr *)((char*)ip6 + sizeof(struct ip6_hdr));
1552 }
1553
1554 /* correct tcp header */
1555 tcp_hdr->th_ack = htonl(cqe2->tcp_ack_num);
1556 if(cqe2->push) {
1557 tcp_hdr->th_flags |= TH_PUSH;
1558 }
1559 tcp_hdr->th_win = htons(cqe2->tcp_window);
1560 tcp_hdr->th_sum = 0xffff;
1561 if(cqe2->ts_opt) {
1562 p = (uint32_t *)((char*)tcp_hdr + sizeof(struct tcphdr) + 2);
1563 *p = cqe1->tcp_timestamp_val;
1564 *(p+1) = cqe1->tcp_timestamp_ecr;
1565 }
1566
1567 return;
1568 }
1569
1570 static void
oce_rx_mbuf_chain(struct oce_rq * rq,struct oce_common_cqe_info * cqe_info,struct mbuf ** m)1571 oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m)
1572 {
1573 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1574 uint32_t i = 0, frag_len = 0;
1575 uint32_t len = cqe_info->pkt_size;
1576 struct oce_packet_desc *pd;
1577 struct mbuf *tail = NULL;
1578
1579 for (i = 0; i < cqe_info->num_frags; i++) {
1580 if (rq->ring->cidx == rq->ring->pidx) {
1581 device_printf(sc->dev,
1582 "oce_rx_mbuf_chain: Invalid RX completion - Queue is empty\n");
1583 return;
1584 }
1585 pd = &rq->pckts[rq->ring->cidx];
1586
1587 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1588 bus_dmamap_unload(rq->tag, pd->map);
1589 RING_GET(rq->ring, 1);
1590 rq->pending--;
1591
1592 frag_len = (len > rq->cfg.frag_size) ? rq->cfg.frag_size : len;
1593 pd->mbuf->m_len = frag_len;
1594
1595 if (tail != NULL) {
1596 /* additional fragments */
1597 pd->mbuf->m_flags &= ~M_PKTHDR;
1598 tail->m_next = pd->mbuf;
1599 if(rq->islro)
1600 tail->m_nextpkt = NULL;
1601 tail = pd->mbuf;
1602 } else {
1603 /* first fragment, fill out much of the packet header */
1604 pd->mbuf->m_pkthdr.len = len;
1605 if(rq->islro)
1606 pd->mbuf->m_nextpkt = NULL;
1607 pd->mbuf->m_pkthdr.csum_flags = 0;
1608 if (IF_CSUM_ENABLED(sc)) {
1609 if (cqe_info->l4_cksum_pass) {
1610 if(!cqe_info->ipv6_frame) { /* IPV4 */
1611 pd->mbuf->m_pkthdr.csum_flags |=
1612 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1613 }else { /* IPV6 frame */
1614 if(rq->islro) {
1615 pd->mbuf->m_pkthdr.csum_flags |=
1616 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1617 }
1618 }
1619 pd->mbuf->m_pkthdr.csum_data = 0xffff;
1620 }
1621 if (cqe_info->ip_cksum_pass) {
1622 pd->mbuf->m_pkthdr.csum_flags |=
1623 (CSUM_IP_CHECKED|CSUM_IP_VALID);
1624 }
1625 }
1626 *m = tail = pd->mbuf;
1627 }
1628 pd->mbuf = NULL;
1629 len -= frag_len;
1630 }
1631
1632 return;
1633 }
1634
1635 static void
oce_rx_lro(struct oce_rq * rq,struct nic_hwlro_singleton_cqe * cqe,struct nic_hwlro_cqe_part2 * cqe2)1636 oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2)
1637 {
1638 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1639 struct nic_hwlro_cqe_part1 *cqe1 = NULL;
1640 struct mbuf *m = NULL;
1641 struct oce_common_cqe_info cq_info;
1642
1643 /* parse cqe */
1644 if(cqe2 == NULL) {
1645 cq_info.pkt_size = cqe->pkt_size;
1646 cq_info.vtag = cqe->vlan_tag;
1647 cq_info.l4_cksum_pass = cqe->l4_cksum_pass;
1648 cq_info.ip_cksum_pass = cqe->ip_cksum_pass;
1649 cq_info.ipv6_frame = cqe->ipv6_frame;
1650 cq_info.vtp = cqe->vtp;
1651 cq_info.qnq = cqe->qnq;
1652 }else {
1653 cqe1 = (struct nic_hwlro_cqe_part1 *)cqe;
1654 cq_info.pkt_size = cqe2->coalesced_size;
1655 cq_info.vtag = cqe2->vlan_tag;
1656 cq_info.l4_cksum_pass = cqe2->l4_cksum_pass;
1657 cq_info.ip_cksum_pass = cqe2->ip_cksum_pass;
1658 cq_info.ipv6_frame = cqe2->ipv6_frame;
1659 cq_info.vtp = cqe2->vtp;
1660 cq_info.qnq = cqe1->qnq;
1661 }
1662
1663 cq_info.vtag = BSWAP_16(cq_info.vtag);
1664
1665 cq_info.num_frags = cq_info.pkt_size / rq->cfg.frag_size;
1666 if(cq_info.pkt_size % rq->cfg.frag_size)
1667 cq_info.num_frags++;
1668
1669 oce_rx_mbuf_chain(rq, &cq_info, &m);
1670
1671 if (m) {
1672 if(cqe2) {
1673 //assert(cqe2->valid != 0);
1674
1675 //assert(cqe2->cqe_type != 2);
1676 oce_correct_header(m, cqe1, cqe2);
1677 }
1678
1679 m->m_pkthdr.rcvif = sc->ifp;
1680 #if __FreeBSD_version >= 800000
1681 if (rq->queue_index)
1682 m->m_pkthdr.flowid = (rq->queue_index - 1);
1683 else
1684 m->m_pkthdr.flowid = rq->queue_index;
1685 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
1686 #endif
1687 /* This deternies if vlan tag is Valid */
1688 if (cq_info.vtp) {
1689 if (sc->function_mode & FNM_FLEX10_MODE) {
1690 /* FLEX10. If QnQ is not set, neglect VLAN */
1691 if (cq_info.qnq) {
1692 m->m_pkthdr.ether_vtag = cq_info.vtag;
1693 m->m_flags |= M_VLANTAG;
1694 }
1695 } else if (sc->pvid != (cq_info.vtag & VLAN_VID_MASK)) {
1696 /* In UMC mode generally pvid will be striped by
1697 hw. But in some cases we have seen it comes
1698 with pvid. So if pvid == vlan, neglect vlan.
1699 */
1700 m->m_pkthdr.ether_vtag = cq_info.vtag;
1701 m->m_flags |= M_VLANTAG;
1702 }
1703 }
1704 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1);
1705
1706 (*sc->ifp->if_input) (sc->ifp, m);
1707
1708 /* Update rx stats per queue */
1709 rq->rx_stats.rx_pkts++;
1710 rq->rx_stats.rx_bytes += cq_info.pkt_size;
1711 rq->rx_stats.rx_frags += cq_info.num_frags;
1712 rq->rx_stats.rx_ucast_pkts++;
1713 }
1714 return;
1715 }
1716
1717 static void
oce_rx(struct oce_rq * rq,struct oce_nic_rx_cqe * cqe)1718 oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe)
1719 {
1720 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1721 int len;
1722 struct mbuf *m = NULL;
1723 struct oce_common_cqe_info cq_info;
1724 uint16_t vtag = 0;
1725
1726 /* Is it a flush compl that has no data */
1727 if(!cqe->u0.s.num_fragments)
1728 goto exit;
1729
1730 len = cqe->u0.s.pkt_size;
1731 if (!len) {
1732 /*partial DMA workaround for Lancer*/
1733 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments);
1734 goto exit;
1735 }
1736
1737 if (!oce_cqe_portid_valid(sc, cqe)) {
1738 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments);
1739 goto exit;
1740 }
1741
1742 /* Get vlan_tag value */
1743 if(IS_BE(sc) || IS_SH(sc))
1744 vtag = BSWAP_16(cqe->u0.s.vlan_tag);
1745 else
1746 vtag = cqe->u0.s.vlan_tag;
1747
1748 cq_info.l4_cksum_pass = cqe->u0.s.l4_cksum_pass;
1749 cq_info.ip_cksum_pass = cqe->u0.s.ip_cksum_pass;
1750 cq_info.ipv6_frame = cqe->u0.s.ip_ver;
1751 cq_info.num_frags = cqe->u0.s.num_fragments;
1752 cq_info.pkt_size = cqe->u0.s.pkt_size;
1753
1754 oce_rx_mbuf_chain(rq, &cq_info, &m);
1755
1756 if (m) {
1757 m->m_pkthdr.rcvif = sc->ifp;
1758 #if __FreeBSD_version >= 800000
1759 if (rq->queue_index)
1760 m->m_pkthdr.flowid = (rq->queue_index - 1);
1761 else
1762 m->m_pkthdr.flowid = rq->queue_index;
1763 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
1764 #endif
1765 /* This deternies if vlan tag is Valid */
1766 if (oce_cqe_vtp_valid(sc, cqe)) {
1767 if (sc->function_mode & FNM_FLEX10_MODE) {
1768 /* FLEX10. If QnQ is not set, neglect VLAN */
1769 if (cqe->u0.s.qnq) {
1770 m->m_pkthdr.ether_vtag = vtag;
1771 m->m_flags |= M_VLANTAG;
1772 }
1773 } else if (sc->pvid != (vtag & VLAN_VID_MASK)) {
1774 /* In UMC mode generally pvid will be striped by
1775 hw. But in some cases we have seen it comes
1776 with pvid. So if pvid == vlan, neglect vlan.
1777 */
1778 m->m_pkthdr.ether_vtag = vtag;
1779 m->m_flags |= M_VLANTAG;
1780 }
1781 }
1782
1783 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1);
1784 #if defined(INET6) || defined(INET)
1785 /* Try to queue to LRO */
1786 if (IF_LRO_ENABLED(sc) &&
1787 (cqe->u0.s.ip_cksum_pass) &&
1788 (cqe->u0.s.l4_cksum_pass) &&
1789 (!cqe->u0.s.ip_ver) &&
1790 (rq->lro.lro_cnt != 0)) {
1791
1792 if (tcp_lro_rx(&rq->lro, m, 0) == 0) {
1793 rq->lro_pkts_queued ++;
1794 goto post_done;
1795 }
1796 /* If LRO posting fails then try to post to STACK */
1797 }
1798 #endif
1799
1800 (*sc->ifp->if_input) (sc->ifp, m);
1801 #if defined(INET6) || defined(INET)
1802 post_done:
1803 #endif
1804 /* Update rx stats per queue */
1805 rq->rx_stats.rx_pkts++;
1806 rq->rx_stats.rx_bytes += cqe->u0.s.pkt_size;
1807 rq->rx_stats.rx_frags += cqe->u0.s.num_fragments;
1808 if (cqe->u0.s.pkt_type == OCE_MULTICAST_PACKET)
1809 rq->rx_stats.rx_mcast_pkts++;
1810 if (cqe->u0.s.pkt_type == OCE_UNICAST_PACKET)
1811 rq->rx_stats.rx_ucast_pkts++;
1812 }
1813 exit:
1814 return;
1815 }
1816
1817
1818 void
oce_discard_rx_comp(struct oce_rq * rq,int num_frags)1819 oce_discard_rx_comp(struct oce_rq *rq, int num_frags)
1820 {
1821 uint32_t i = 0;
1822 struct oce_packet_desc *pd;
1823 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1824
1825 for (i = 0; i < num_frags; i++) {
1826 if (rq->ring->cidx == rq->ring->pidx) {
1827 device_printf(sc->dev,
1828 "oce_discard_rx_comp: Invalid RX completion - Queue is empty\n");
1829 return;
1830 }
1831 pd = &rq->pckts[rq->ring->cidx];
1832 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1833 bus_dmamap_unload(rq->tag, pd->map);
1834 if (pd->mbuf != NULL) {
1835 m_freem(pd->mbuf);
1836 pd->mbuf = NULL;
1837 }
1838
1839 RING_GET(rq->ring, 1);
1840 rq->pending--;
1841 }
1842 }
1843
1844
1845 static int
oce_cqe_vtp_valid(POCE_SOFTC sc,struct oce_nic_rx_cqe * cqe)1846 oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
1847 {
1848 struct oce_nic_rx_cqe_v1 *cqe_v1;
1849 int vtp = 0;
1850
1851 if (sc->be3_native) {
1852 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe;
1853 vtp = cqe_v1->u0.s.vlan_tag_present;
1854 } else
1855 vtp = cqe->u0.s.vlan_tag_present;
1856
1857 return vtp;
1858
1859 }
1860
1861
1862 static int
oce_cqe_portid_valid(POCE_SOFTC sc,struct oce_nic_rx_cqe * cqe)1863 oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
1864 {
1865 struct oce_nic_rx_cqe_v1 *cqe_v1;
1866 int port_id = 0;
1867
1868 if (sc->be3_native && (IS_BE(sc) || IS_SH(sc))) {
1869 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe;
1870 port_id = cqe_v1->u0.s.port;
1871 if (sc->port_id != port_id)
1872 return 0;
1873 } else
1874 ;/* For BE3 legacy and Lancer this is dummy */
1875
1876 return 1;
1877
1878 }
1879
1880 #if defined(INET6) || defined(INET)
1881 void
oce_rx_flush_lro(struct oce_rq * rq)1882 oce_rx_flush_lro(struct oce_rq *rq)
1883 {
1884 struct lro_ctrl *lro = &rq->lro;
1885 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1886
1887 if (!IF_LRO_ENABLED(sc))
1888 return;
1889
1890 tcp_lro_flush_all(lro);
1891 rq->lro_pkts_queued = 0;
1892
1893 return;
1894 }
1895
1896
1897 static int
oce_init_lro(POCE_SOFTC sc)1898 oce_init_lro(POCE_SOFTC sc)
1899 {
1900 struct lro_ctrl *lro = NULL;
1901 int i = 0, rc = 0;
1902
1903 for (i = 0; i < sc->nrqs; i++) {
1904 lro = &sc->rq[i]->lro;
1905 rc = tcp_lro_init(lro);
1906 if (rc != 0) {
1907 device_printf(sc->dev, "LRO init failed\n");
1908 return rc;
1909 }
1910 lro->ifp = sc->ifp;
1911 }
1912
1913 return rc;
1914 }
1915
1916
1917 void
oce_free_lro(POCE_SOFTC sc)1918 oce_free_lro(POCE_SOFTC sc)
1919 {
1920 struct lro_ctrl *lro = NULL;
1921 int i = 0;
1922
1923 for (i = 0; i < sc->nrqs; i++) {
1924 lro = &sc->rq[i]->lro;
1925 if (lro)
1926 tcp_lro_free(lro);
1927 }
1928 }
1929 #endif
1930
1931 int
oce_alloc_rx_bufs(struct oce_rq * rq,int count)1932 oce_alloc_rx_bufs(struct oce_rq *rq, int count)
1933 {
1934 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1935 int i, in, rc;
1936 struct oce_packet_desc *pd;
1937 bus_dma_segment_t segs[6];
1938 int nsegs, added = 0;
1939 struct oce_nic_rqe *rqe;
1940 pd_rxulp_db_t rxdb_reg;
1941 uint32_t val = 0;
1942 uint32_t oce_max_rq_posts = 64;
1943
1944 bzero(&rxdb_reg, sizeof(pd_rxulp_db_t));
1945 for (i = 0; i < count; i++) {
1946 in = (rq->ring->pidx + 1) % OCE_RQ_PACKET_ARRAY_SIZE;
1947
1948 pd = &rq->pckts[rq->ring->pidx];
1949 pd->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, oce_rq_buf_size);
1950 if (pd->mbuf == NULL) {
1951 device_printf(sc->dev, "mbuf allocation failed, size = %d\n",oce_rq_buf_size);
1952 break;
1953 }
1954 pd->mbuf->m_nextpkt = NULL;
1955
1956 pd->mbuf->m_len = pd->mbuf->m_pkthdr.len = rq->cfg.frag_size;
1957
1958 rc = bus_dmamap_load_mbuf_sg(rq->tag,
1959 pd->map,
1960 pd->mbuf,
1961 segs, &nsegs, BUS_DMA_NOWAIT);
1962 if (rc) {
1963 m_free(pd->mbuf);
1964 device_printf(sc->dev, "bus_dmamap_load_mbuf_sg failed rc = %d\n", rc);
1965 break;
1966 }
1967
1968 if (nsegs != 1) {
1969 i--;
1970 continue;
1971 }
1972
1973 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_PREREAD);
1974
1975 rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring, struct oce_nic_rqe);
1976 rqe->u0.s.frag_pa_hi = ADDR_HI(segs[0].ds_addr);
1977 rqe->u0.s.frag_pa_lo = ADDR_LO(segs[0].ds_addr);
1978 DW_SWAP(u32ptr(rqe), sizeof(struct oce_nic_rqe));
1979 RING_PUT(rq->ring, 1);
1980 added++;
1981 rq->pending++;
1982 }
1983 oce_max_rq_posts = sc->enable_hwlro ? OCE_HWLRO_MAX_RQ_POSTS : OCE_MAX_RQ_POSTS;
1984 if (added != 0) {
1985 for (i = added / oce_max_rq_posts; i > 0; i--) {
1986 rxdb_reg.bits.num_posted = oce_max_rq_posts;
1987 rxdb_reg.bits.qid = rq->rq_id;
1988 if(rq->islro) {
1989 val |= rq->rq_id & DB_LRO_RQ_ID_MASK;
1990 val |= oce_max_rq_posts << 16;
1991 OCE_WRITE_REG32(sc, db, DB_OFFSET, val);
1992 }else {
1993 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0);
1994 }
1995 added -= oce_max_rq_posts;
1996 }
1997 if (added > 0) {
1998 rxdb_reg.bits.qid = rq->rq_id;
1999 rxdb_reg.bits.num_posted = added;
2000 if(rq->islro) {
2001 val |= rq->rq_id & DB_LRO_RQ_ID_MASK;
2002 val |= added << 16;
2003 OCE_WRITE_REG32(sc, db, DB_OFFSET, val);
2004 }else {
2005 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0);
2006 }
2007 }
2008 }
2009
2010 return 0;
2011 }
2012
2013 static void
oce_check_rx_bufs(POCE_SOFTC sc,uint32_t num_cqes,struct oce_rq * rq)2014 oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq)
2015 {
2016 if (num_cqes) {
2017 oce_arm_cq(sc, rq->cq->cq_id, num_cqes, FALSE);
2018 if(!sc->enable_hwlro) {
2019 if((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) > 1)
2020 oce_alloc_rx_bufs(rq, ((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) - 1));
2021 }else {
2022 if ((OCE_RQ_PACKET_ARRAY_SIZE -1 - rq->pending) > 64)
2023 oce_alloc_rx_bufs(rq, 64);
2024 }
2025 }
2026
2027 return;
2028 }
2029
2030 uint16_t
oce_rq_handler_lro(void * arg)2031 oce_rq_handler_lro(void *arg)
2032 {
2033 struct oce_rq *rq = (struct oce_rq *)arg;
2034 struct oce_cq *cq = rq->cq;
2035 POCE_SOFTC sc = rq->parent;
2036 struct nic_hwlro_singleton_cqe *cqe;
2037 struct nic_hwlro_cqe_part2 *cqe2;
2038 int num_cqes = 0;
2039
2040 LOCK(&rq->rx_lock);
2041 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2042 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe);
2043 while (cqe->valid) {
2044 if(cqe->cqe_type == 0) { /* singleton cqe */
2045 /* we should not get singleton cqe after cqe1 on same rq */
2046 if(rq->cqe_firstpart != NULL) {
2047 device_printf(sc->dev, "Got singleton cqe after cqe1 \n");
2048 goto exit_rq_handler_lro;
2049 }
2050 if(cqe->error != 0) {
2051 rq->rx_stats.rxcp_err++;
2052 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
2053 }
2054 oce_rx_lro(rq, cqe, NULL);
2055 rq->rx_stats.rx_compl++;
2056 cqe->valid = 0;
2057 RING_GET(cq->ring, 1);
2058 num_cqes++;
2059 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
2060 break;
2061 }else if(cqe->cqe_type == 0x1) { /* first part */
2062 /* we should not get cqe1 after cqe1 on same rq */
2063 if(rq->cqe_firstpart != NULL) {
2064 device_printf(sc->dev, "Got cqe1 after cqe1 \n");
2065 goto exit_rq_handler_lro;
2066 }
2067 rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe;
2068 RING_GET(cq->ring, 1);
2069 }else if(cqe->cqe_type == 0x2) { /* second part */
2070 cqe2 = (struct nic_hwlro_cqe_part2 *)cqe;
2071 if(cqe2->error != 0) {
2072 rq->rx_stats.rxcp_err++;
2073 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
2074 }
2075 /* We should not get cqe2 without cqe1 */
2076 if(rq->cqe_firstpart == NULL) {
2077 device_printf(sc->dev, "Got cqe2 without cqe1 \n");
2078 goto exit_rq_handler_lro;
2079 }
2080 oce_rx_lro(rq, (struct nic_hwlro_singleton_cqe *)rq->cqe_firstpart, cqe2);
2081
2082 rq->rx_stats.rx_compl++;
2083 rq->cqe_firstpart->valid = 0;
2084 cqe2->valid = 0;
2085 rq->cqe_firstpart = NULL;
2086
2087 RING_GET(cq->ring, 1);
2088 num_cqes += 2;
2089 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
2090 break;
2091 }
2092
2093 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2094 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe);
2095 }
2096 oce_check_rx_bufs(sc, num_cqes, rq);
2097 exit_rq_handler_lro:
2098 UNLOCK(&rq->rx_lock);
2099 return 0;
2100 }
2101
2102 /* Handle the Completion Queue for receive */
2103 uint16_t
oce_rq_handler(void * arg)2104 oce_rq_handler(void *arg)
2105 {
2106 struct oce_rq *rq = (struct oce_rq *)arg;
2107 struct oce_cq *cq = rq->cq;
2108 POCE_SOFTC sc = rq->parent;
2109 struct oce_nic_rx_cqe *cqe;
2110 int num_cqes = 0;
2111
2112 if(rq->islro) {
2113 oce_rq_handler_lro(arg);
2114 return 0;
2115 }
2116 LOCK(&rq->rx_lock);
2117 bus_dmamap_sync(cq->ring->dma.tag,
2118 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2119 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
2120 while (cqe->u0.dw[2]) {
2121 DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe));
2122
2123 if (cqe->u0.s.error == 0) {
2124 oce_rx(rq, cqe);
2125 } else {
2126 rq->rx_stats.rxcp_err++;
2127 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
2128 /* Post L3/L4 errors to stack.*/
2129 oce_rx(rq, cqe);
2130 }
2131 rq->rx_stats.rx_compl++;
2132 cqe->u0.dw[2] = 0;
2133
2134 #if defined(INET6) || defined(INET)
2135 if (IF_LRO_ENABLED(sc) && rq->lro_pkts_queued >= 16) {
2136 oce_rx_flush_lro(rq);
2137 }
2138 #endif
2139
2140 RING_GET(cq->ring, 1);
2141 bus_dmamap_sync(cq->ring->dma.tag,
2142 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2143 cqe =
2144 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
2145 num_cqes++;
2146 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
2147 break;
2148 }
2149
2150 #if defined(INET6) || defined(INET)
2151 if (IF_LRO_ENABLED(sc))
2152 oce_rx_flush_lro(rq);
2153 #endif
2154
2155 oce_check_rx_bufs(sc, num_cqes, rq);
2156 UNLOCK(&rq->rx_lock);
2157 return 0;
2158
2159 }
2160
2161
2162
2163
2164 /*****************************************************************************
2165 * Helper function prototypes in this file *
2166 *****************************************************************************/
2167
2168 static int
oce_attach_ifp(POCE_SOFTC sc)2169 oce_attach_ifp(POCE_SOFTC sc)
2170 {
2171
2172 sc->ifp = if_alloc(IFT_ETHER);
2173 if (!sc->ifp)
2174 return ENOMEM;
2175
2176 ifmedia_init(&sc->media, IFM_IMASK, oce_media_change, oce_media_status);
2177 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2178 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
2179
2180 sc->ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST;
2181 sc->ifp->if_ioctl = oce_ioctl;
2182 sc->ifp->if_start = oce_start;
2183 sc->ifp->if_init = oce_init;
2184 sc->ifp->if_mtu = ETHERMTU;
2185 sc->ifp->if_softc = sc;
2186 #if __FreeBSD_version >= 800000
2187 sc->ifp->if_transmit = oce_multiq_start;
2188 sc->ifp->if_qflush = oce_multiq_flush;
2189 #endif
2190
2191 if_initname(sc->ifp,
2192 device_get_name(sc->dev), device_get_unit(sc->dev));
2193
2194 sc->ifp->if_snd.ifq_drv_maxlen = OCE_MAX_TX_DESC - 1;
2195 IFQ_SET_MAXLEN(&sc->ifp->if_snd, sc->ifp->if_snd.ifq_drv_maxlen);
2196 IFQ_SET_READY(&sc->ifp->if_snd);
2197
2198 sc->ifp->if_hwassist = OCE_IF_HWASSIST;
2199 sc->ifp->if_hwassist |= CSUM_TSO;
2200 sc->ifp->if_hwassist |= (CSUM_IP | CSUM_TCP | CSUM_UDP);
2201
2202 sc->ifp->if_capabilities = OCE_IF_CAPABILITIES;
2203 sc->ifp->if_capabilities |= IFCAP_HWCSUM;
2204 sc->ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2205
2206 #if defined(INET6) || defined(INET)
2207 sc->ifp->if_capabilities |= IFCAP_TSO;
2208 sc->ifp->if_capabilities |= IFCAP_LRO;
2209 sc->ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
2210 #endif
2211
2212 sc->ifp->if_capenable = sc->ifp->if_capabilities;
2213 sc->ifp->if_baudrate = IF_Gbps(10);
2214
2215 #if __FreeBSD_version >= 1000000
2216 sc->ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
2217 sc->ifp->if_hw_tsomaxsegcount = OCE_MAX_TX_ELEMENTS;
2218 sc->ifp->if_hw_tsomaxsegsize = 4096;
2219 #endif
2220
2221 ether_ifattach(sc->ifp, sc->macaddr.mac_addr);
2222
2223 return 0;
2224 }
2225
2226
2227 static void
oce_add_vlan(void * arg,struct ifnet * ifp,uint16_t vtag)2228 oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag)
2229 {
2230 POCE_SOFTC sc = ifp->if_softc;
2231
2232 if (ifp->if_softc != arg)
2233 return;
2234 if ((vtag == 0) || (vtag > 4095))
2235 return;
2236
2237 sc->vlan_tag[vtag] = 1;
2238 sc->vlans_added++;
2239 if (sc->vlans_added <= (sc->max_vlans + 1))
2240 oce_vid_config(sc);
2241 }
2242
2243
2244 static void
oce_del_vlan(void * arg,struct ifnet * ifp,uint16_t vtag)2245 oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag)
2246 {
2247 POCE_SOFTC sc = ifp->if_softc;
2248
2249 if (ifp->if_softc != arg)
2250 return;
2251 if ((vtag == 0) || (vtag > 4095))
2252 return;
2253
2254 sc->vlan_tag[vtag] = 0;
2255 sc->vlans_added--;
2256 oce_vid_config(sc);
2257 }
2258
2259
2260 /*
2261 * A max of 64 vlans can be configured in BE. If the user configures
2262 * more, place the card in vlan promiscuous mode.
2263 */
2264 static int
oce_vid_config(POCE_SOFTC sc)2265 oce_vid_config(POCE_SOFTC sc)
2266 {
2267 struct normal_vlan vtags[MAX_VLANFILTER_SIZE];
2268 uint16_t ntags = 0, i;
2269 int status = 0;
2270
2271 if ((sc->vlans_added <= MAX_VLANFILTER_SIZE) &&
2272 (sc->ifp->if_capenable & IFCAP_VLAN_HWFILTER)) {
2273 for (i = 0; i < MAX_VLANS; i++) {
2274 if (sc->vlan_tag[i]) {
2275 vtags[ntags].vtag = i;
2276 ntags++;
2277 }
2278 }
2279 if (ntags)
2280 status = oce_config_vlan(sc, (uint8_t) sc->if_id,
2281 vtags, ntags, 1, 0);
2282 } else
2283 status = oce_config_vlan(sc, (uint8_t) sc->if_id,
2284 NULL, 0, 1, 1);
2285 return status;
2286 }
2287
2288
2289 static void
oce_mac_addr_set(POCE_SOFTC sc)2290 oce_mac_addr_set(POCE_SOFTC sc)
2291 {
2292 uint32_t old_pmac_id = sc->pmac_id;
2293 int status = 0;
2294
2295
2296 status = bcmp((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr,
2297 sc->macaddr.size_of_struct);
2298 if (!status)
2299 return;
2300
2301 status = oce_mbox_macaddr_add(sc, (uint8_t *)(IF_LLADDR(sc->ifp)),
2302 sc->if_id, &sc->pmac_id);
2303 if (!status) {
2304 status = oce_mbox_macaddr_del(sc, sc->if_id, old_pmac_id);
2305 bcopy((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr,
2306 sc->macaddr.size_of_struct);
2307 }
2308 if (status)
2309 device_printf(sc->dev, "Failed update macaddress\n");
2310
2311 }
2312
2313
2314 static int
oce_handle_passthrough(struct ifnet * ifp,caddr_t data)2315 oce_handle_passthrough(struct ifnet *ifp, caddr_t data)
2316 {
2317 POCE_SOFTC sc = ifp->if_softc;
2318 struct ifreq *ifr = (struct ifreq *)data;
2319 int rc = ENXIO;
2320 char cookie[32] = {0};
2321 void *priv_data = ifr_data_get_ptr(ifr);
2322 void *ioctl_ptr;
2323 uint32_t req_size;
2324 struct mbx_hdr req;
2325 OCE_DMA_MEM dma_mem;
2326 struct mbx_common_get_cntl_attr *fw_cmd;
2327
2328 if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE)))
2329 return EFAULT;
2330
2331 if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE)))
2332 return EINVAL;
2333
2334 ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE);
2335 if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr)))
2336 return EFAULT;
2337
2338 req_size = le32toh(req.u0.req.request_length);
2339 if (req_size > 65536)
2340 return EINVAL;
2341
2342 req_size += sizeof(struct mbx_hdr);
2343 rc = oce_dma_alloc(sc, req_size, &dma_mem, 0);
2344 if (rc)
2345 return ENOMEM;
2346
2347 if (copyin(ioctl_ptr, OCE_DMAPTR(&dma_mem,char), req_size)) {
2348 rc = EFAULT;
2349 goto dma_free;
2350 }
2351
2352 rc = oce_pass_through_mbox(sc, &dma_mem, req_size);
2353 if (rc) {
2354 rc = EIO;
2355 goto dma_free;
2356 }
2357
2358 if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size))
2359 rc = EFAULT;
2360
2361 /*
2362 firmware is filling all the attributes for this ioctl except
2363 the driver version..so fill it
2364 */
2365 if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) {
2366 fw_cmd = (struct mbx_common_get_cntl_attr *) ioctl_ptr;
2367 strncpy(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str,
2368 COMPONENT_REVISION, strlen(COMPONENT_REVISION));
2369 }
2370
2371 dma_free:
2372 oce_dma_free(sc, &dma_mem);
2373 return rc;
2374
2375 }
2376
2377 static void
oce_eqd_set_periodic(POCE_SOFTC sc)2378 oce_eqd_set_periodic(POCE_SOFTC sc)
2379 {
2380 struct oce_set_eqd set_eqd[OCE_MAX_EQ];
2381 struct oce_aic_obj *aic;
2382 struct oce_eq *eqo;
2383 uint64_t now = 0, delta;
2384 int eqd, i, num = 0;
2385 uint32_t tx_reqs = 0, rxpkts = 0, pps;
2386 struct oce_wq *wq;
2387 struct oce_rq *rq;
2388
2389 #define ticks_to_msecs(t) (1000 * (t) / hz)
2390
2391 for (i = 0 ; i < sc->neqs; i++) {
2392 eqo = sc->eq[i];
2393 aic = &sc->aic_obj[i];
2394 /* When setting the static eq delay from the user space */
2395 if (!aic->enable) {
2396 if (aic->ticks)
2397 aic->ticks = 0;
2398 eqd = aic->et_eqd;
2399 goto modify_eqd;
2400 }
2401
2402 if (i == 0) {
2403 rq = sc->rq[0];
2404 rxpkts = rq->rx_stats.rx_pkts;
2405 } else
2406 rxpkts = 0;
2407 if (i + 1 < sc->nrqs) {
2408 rq = sc->rq[i + 1];
2409 rxpkts += rq->rx_stats.rx_pkts;
2410 }
2411 if (i < sc->nwqs) {
2412 wq = sc->wq[i];
2413 tx_reqs = wq->tx_stats.tx_reqs;
2414 } else
2415 tx_reqs = 0;
2416 now = ticks;
2417
2418 if (!aic->ticks || now < aic->ticks ||
2419 rxpkts < aic->prev_rxpkts || tx_reqs < aic->prev_txreqs) {
2420 aic->prev_rxpkts = rxpkts;
2421 aic->prev_txreqs = tx_reqs;
2422 aic->ticks = now;
2423 continue;
2424 }
2425
2426 delta = ticks_to_msecs(now - aic->ticks);
2427
2428 pps = (((uint32_t)(rxpkts - aic->prev_rxpkts) * 1000) / delta) +
2429 (((uint32_t)(tx_reqs - aic->prev_txreqs) * 1000) / delta);
2430 eqd = (pps / 15000) << 2;
2431 if (eqd < 8)
2432 eqd = 0;
2433
2434 /* Make sure that the eq delay is in the known range */
2435 eqd = min(eqd, aic->max_eqd);
2436 eqd = max(eqd, aic->min_eqd);
2437
2438 aic->prev_rxpkts = rxpkts;
2439 aic->prev_txreqs = tx_reqs;
2440 aic->ticks = now;
2441
2442 modify_eqd:
2443 if (eqd != aic->cur_eqd) {
2444 set_eqd[num].delay_multiplier = (eqd * 65)/100;
2445 set_eqd[num].eq_id = eqo->eq_id;
2446 aic->cur_eqd = eqd;
2447 num++;
2448 }
2449 }
2450
2451 /* Is there atleast one eq that needs to be modified? */
2452 for(i = 0; i < num; i += 8) {
2453 if((num - i) >=8 )
2454 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], 8);
2455 else
2456 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], (num - i));
2457 }
2458
2459 }
2460
oce_detect_hw_error(POCE_SOFTC sc)2461 static void oce_detect_hw_error(POCE_SOFTC sc)
2462 {
2463
2464 uint32_t ue_low = 0, ue_high = 0, ue_low_mask = 0, ue_high_mask = 0;
2465 uint32_t sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2466 uint32_t i;
2467
2468 if (sc->hw_error)
2469 return;
2470
2471 if (IS_XE201(sc)) {
2472 sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET);
2473 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2474 sliport_err1 = OCE_READ_REG32(sc, db, SLIPORT_ERROR1_OFFSET);
2475 sliport_err2 = OCE_READ_REG32(sc, db, SLIPORT_ERROR2_OFFSET);
2476 }
2477 } else {
2478 ue_low = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW);
2479 ue_high = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HIGH);
2480 ue_low_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW_MASK);
2481 ue_high_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HI_MASK);
2482
2483 ue_low = (ue_low & ~ue_low_mask);
2484 ue_high = (ue_high & ~ue_high_mask);
2485 }
2486
2487 /* On certain platforms BE hardware can indicate spurious UEs.
2488 * Allow the h/w to stop working completely in case of a real UE.
2489 * Hence not setting the hw_error for UE detection.
2490 */
2491 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2492 sc->hw_error = TRUE;
2493 device_printf(sc->dev, "Error detected in the card\n");
2494 }
2495
2496 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2497 device_printf(sc->dev,
2498 "ERR: sliport status 0x%x\n", sliport_status);
2499 device_printf(sc->dev,
2500 "ERR: sliport error1 0x%x\n", sliport_err1);
2501 device_printf(sc->dev,
2502 "ERR: sliport error2 0x%x\n", sliport_err2);
2503 }
2504
2505 if (ue_low) {
2506 for (i = 0; ue_low; ue_low >>= 1, i++) {
2507 if (ue_low & 1)
2508 device_printf(sc->dev, "UE: %s bit set\n",
2509 ue_status_low_desc[i]);
2510 }
2511 }
2512
2513 if (ue_high) {
2514 for (i = 0; ue_high; ue_high >>= 1, i++) {
2515 if (ue_high & 1)
2516 device_printf(sc->dev, "UE: %s bit set\n",
2517 ue_status_hi_desc[i]);
2518 }
2519 }
2520
2521 }
2522
2523
2524 static void
oce_local_timer(void * arg)2525 oce_local_timer(void *arg)
2526 {
2527 POCE_SOFTC sc = arg;
2528 int i = 0;
2529
2530 oce_detect_hw_error(sc);
2531 oce_refresh_nic_stats(sc);
2532 oce_refresh_queue_stats(sc);
2533 oce_mac_addr_set(sc);
2534
2535 /* TX Watch Dog*/
2536 for (i = 0; i < sc->nwqs; i++)
2537 oce_tx_restart(sc, sc->wq[i]);
2538
2539 /* calculate and set the eq delay for optimal interrupt rate */
2540 if (IS_BE(sc) || IS_SH(sc))
2541 oce_eqd_set_periodic(sc);
2542
2543 callout_reset(&sc->timer, hz, oce_local_timer, sc);
2544 }
2545
2546 static void
oce_tx_compl_clean(POCE_SOFTC sc)2547 oce_tx_compl_clean(POCE_SOFTC sc)
2548 {
2549 struct oce_wq *wq;
2550 int i = 0, timeo = 0, num_wqes = 0;
2551 int pending_txqs = sc->nwqs;
2552
2553 /* Stop polling for compls when HW has been silent for 10ms or
2554 * hw_error or no outstanding completions expected
2555 */
2556 do {
2557 pending_txqs = sc->nwqs;
2558
2559 for_all_wq_queues(sc, wq, i) {
2560 num_wqes = oce_wq_handler(wq);
2561
2562 if(num_wqes)
2563 timeo = 0;
2564
2565 if(!wq->ring->num_used)
2566 pending_txqs--;
2567 }
2568
2569 if (pending_txqs == 0 || ++timeo > 10 || sc->hw_error)
2570 break;
2571
2572 DELAY(1000);
2573 } while (TRUE);
2574
2575 for_all_wq_queues(sc, wq, i) {
2576 while(wq->ring->num_used) {
2577 LOCK(&wq->tx_compl_lock);
2578 oce_process_tx_completion(wq);
2579 UNLOCK(&wq->tx_compl_lock);
2580 }
2581 }
2582
2583 }
2584
2585 /* NOTE : This should only be called holding
2586 * DEVICE_LOCK.
2587 */
2588 static void
oce_if_deactivate(POCE_SOFTC sc)2589 oce_if_deactivate(POCE_SOFTC sc)
2590 {
2591 int i;
2592 struct oce_rq *rq;
2593 struct oce_wq *wq;
2594 struct oce_eq *eq;
2595
2596 sc->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
2597
2598 oce_tx_compl_clean(sc);
2599
2600 /* Stop intrs and finish any bottom halves pending */
2601 oce_hw_intr_disable(sc);
2602
2603 /* Since taskqueue_drain takes a Gaint Lock, We should not acquire
2604 any other lock. So unlock device lock and require after
2605 completing taskqueue_drain.
2606 */
2607 UNLOCK(&sc->dev_lock);
2608 for (i = 0; i < sc->intr_count; i++) {
2609 if (sc->intrs[i].tq != NULL) {
2610 taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task);
2611 }
2612 }
2613 LOCK(&sc->dev_lock);
2614
2615 /* Delete RX queue in card with flush param */
2616 oce_stop_rx(sc);
2617
2618 /* Invalidate any pending cq and eq entries*/
2619 for_all_evnt_queues(sc, eq, i)
2620 oce_drain_eq(eq);
2621 for_all_rq_queues(sc, rq, i)
2622 oce_drain_rq_cq(rq);
2623 for_all_wq_queues(sc, wq, i)
2624 oce_drain_wq_cq(wq);
2625
2626 /* But still we need to get MCC aync events.
2627 So enable intrs and also arm first EQ
2628 */
2629 oce_hw_intr_enable(sc);
2630 oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE);
2631
2632 DELAY(10);
2633 }
2634
2635
2636 static void
oce_if_activate(POCE_SOFTC sc)2637 oce_if_activate(POCE_SOFTC sc)
2638 {
2639 struct oce_eq *eq;
2640 struct oce_rq *rq;
2641 struct oce_wq *wq;
2642 int i, rc = 0;
2643
2644 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
2645
2646 oce_hw_intr_disable(sc);
2647
2648 oce_start_rx(sc);
2649
2650 for_all_rq_queues(sc, rq, i) {
2651 rc = oce_start_rq(rq);
2652 if (rc)
2653 device_printf(sc->dev, "Unable to start RX\n");
2654 }
2655
2656 for_all_wq_queues(sc, wq, i) {
2657 rc = oce_start_wq(wq);
2658 if (rc)
2659 device_printf(sc->dev, "Unable to start TX\n");
2660 }
2661
2662
2663 for_all_evnt_queues(sc, eq, i)
2664 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE);
2665
2666 oce_hw_intr_enable(sc);
2667
2668 }
2669
2670 static void
process_link_state(POCE_SOFTC sc,struct oce_async_cqe_link_state * acqe)2671 process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe)
2672 {
2673 /* Update Link status */
2674 if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) ==
2675 ASYNC_EVENT_LINK_UP) {
2676 sc->link_status = ASYNC_EVENT_LINK_UP;
2677 if_link_state_change(sc->ifp, LINK_STATE_UP);
2678 } else {
2679 sc->link_status = ASYNC_EVENT_LINK_DOWN;
2680 if_link_state_change(sc->ifp, LINK_STATE_DOWN);
2681 }
2682 }
2683
2684
oce_async_grp5_osbmc_process(POCE_SOFTC sc,struct oce_async_evt_grp5_os2bmc * evt)2685 static void oce_async_grp5_osbmc_process(POCE_SOFTC sc,
2686 struct oce_async_evt_grp5_os2bmc *evt)
2687 {
2688 DW_SWAP(evt, sizeof(struct oce_async_evt_grp5_os2bmc));
2689 if (evt->u.s.mgmt_enable)
2690 sc->flags |= OCE_FLAGS_OS2BMC;
2691 else
2692 return;
2693
2694 sc->bmc_filt_mask = evt->u.s.arp_filter;
2695 sc->bmc_filt_mask |= (evt->u.s.dhcp_client_filt << 1);
2696 sc->bmc_filt_mask |= (evt->u.s.dhcp_server_filt << 2);
2697 sc->bmc_filt_mask |= (evt->u.s.net_bios_filt << 3);
2698 sc->bmc_filt_mask |= (evt->u.s.bcast_filt << 4);
2699 sc->bmc_filt_mask |= (evt->u.s.ipv6_nbr_filt << 5);
2700 sc->bmc_filt_mask |= (evt->u.s.ipv6_ra_filt << 6);
2701 sc->bmc_filt_mask |= (evt->u.s.ipv6_ras_filt << 7);
2702 sc->bmc_filt_mask |= (evt->u.s.mcast_filt << 8);
2703 }
2704
2705
oce_process_grp5_events(POCE_SOFTC sc,struct oce_mq_cqe * cqe)2706 static void oce_process_grp5_events(POCE_SOFTC sc, struct oce_mq_cqe *cqe)
2707 {
2708 struct oce_async_event_grp5_pvid_state *gcqe;
2709 struct oce_async_evt_grp5_os2bmc *bmccqe;
2710
2711 switch (cqe->u0.s.async_type) {
2712 case ASYNC_EVENT_PVID_STATE:
2713 /* GRP5 PVID */
2714 gcqe = (struct oce_async_event_grp5_pvid_state *)cqe;
2715 if (gcqe->enabled)
2716 sc->pvid = gcqe->tag & VLAN_VID_MASK;
2717 else
2718 sc->pvid = 0;
2719 break;
2720 case ASYNC_EVENT_OS2BMC:
2721 bmccqe = (struct oce_async_evt_grp5_os2bmc *)cqe;
2722 oce_async_grp5_osbmc_process(sc, bmccqe);
2723 break;
2724 default:
2725 break;
2726 }
2727 }
2728
2729 /* Handle the Completion Queue for the Mailbox/Async notifications */
2730 uint16_t
oce_mq_handler(void * arg)2731 oce_mq_handler(void *arg)
2732 {
2733 struct oce_mq *mq = (struct oce_mq *)arg;
2734 POCE_SOFTC sc = mq->parent;
2735 struct oce_cq *cq = mq->cq;
2736 int num_cqes = 0, evt_type = 0, optype = 0;
2737 struct oce_mq_cqe *cqe;
2738 struct oce_async_cqe_link_state *acqe;
2739 struct oce_async_event_qnq *dbgcqe;
2740
2741
2742 bus_dmamap_sync(cq->ring->dma.tag,
2743 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2744 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
2745
2746 while (cqe->u0.dw[3]) {
2747 DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe));
2748 if (cqe->u0.s.async_event) {
2749 evt_type = cqe->u0.s.event_type;
2750 optype = cqe->u0.s.async_type;
2751 if (evt_type == ASYNC_EVENT_CODE_LINK_STATE) {
2752 /* Link status evt */
2753 acqe = (struct oce_async_cqe_link_state *)cqe;
2754 process_link_state(sc, acqe);
2755 } else if (evt_type == ASYNC_EVENT_GRP5) {
2756 oce_process_grp5_events(sc, cqe);
2757 } else if (evt_type == ASYNC_EVENT_CODE_DEBUG &&
2758 optype == ASYNC_EVENT_DEBUG_QNQ) {
2759 dbgcqe = (struct oce_async_event_qnq *)cqe;
2760 if(dbgcqe->valid)
2761 sc->qnqid = dbgcqe->vlan_tag;
2762 sc->qnq_debug_event = TRUE;
2763 }
2764 }
2765 cqe->u0.dw[3] = 0;
2766 RING_GET(cq->ring, 1);
2767 bus_dmamap_sync(cq->ring->dma.tag,
2768 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2769 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
2770 num_cqes++;
2771 }
2772
2773 if (num_cqes)
2774 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE);
2775
2776 return 0;
2777 }
2778
2779
2780 static void
setup_max_queues_want(POCE_SOFTC sc)2781 setup_max_queues_want(POCE_SOFTC sc)
2782 {
2783 /* Check if it is FLEX machine. Is so dont use RSS */
2784 if ((sc->function_mode & FNM_FLEX10_MODE) ||
2785 (sc->function_mode & FNM_UMC_MODE) ||
2786 (sc->function_mode & FNM_VNIC_MODE) ||
2787 (!is_rss_enabled(sc)) ||
2788 IS_BE2(sc)) {
2789 sc->nrqs = 1;
2790 sc->nwqs = 1;
2791 } else {
2792 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1;
2793 sc->nwqs = MIN(OCE_NCPUS, sc->nrssqs);
2794 }
2795
2796 if (IS_BE2(sc) && is_rss_enabled(sc))
2797 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1;
2798 }
2799
2800
2801 static void
update_queues_got(POCE_SOFTC sc)2802 update_queues_got(POCE_SOFTC sc)
2803 {
2804 if (is_rss_enabled(sc)) {
2805 sc->nrqs = sc->intr_count + 1;
2806 sc->nwqs = sc->intr_count;
2807 } else {
2808 sc->nrqs = 1;
2809 sc->nwqs = 1;
2810 }
2811
2812 if (IS_BE2(sc))
2813 sc->nwqs = 1;
2814 }
2815
2816 static int
oce_check_ipv6_ext_hdr(struct mbuf * m)2817 oce_check_ipv6_ext_hdr(struct mbuf *m)
2818 {
2819 struct ether_header *eh = mtod(m, struct ether_header *);
2820 caddr_t m_datatemp = m->m_data;
2821
2822 if (eh->ether_type == htons(ETHERTYPE_IPV6)) {
2823 m->m_data += sizeof(struct ether_header);
2824 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
2825
2826 if((ip6->ip6_nxt != IPPROTO_TCP) && \
2827 (ip6->ip6_nxt != IPPROTO_UDP)){
2828 struct ip6_ext *ip6e = NULL;
2829 m->m_data += sizeof(struct ip6_hdr);
2830
2831 ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *);
2832 if(ip6e->ip6e_len == 0xff) {
2833 m->m_data = m_datatemp;
2834 return TRUE;
2835 }
2836 }
2837 m->m_data = m_datatemp;
2838 }
2839 return FALSE;
2840 }
2841
2842 static int
is_be3_a1(POCE_SOFTC sc)2843 is_be3_a1(POCE_SOFTC sc)
2844 {
2845 if((sc->flags & OCE_FLAGS_BE3) && ((sc->asic_revision & 0xFF) < 2)) {
2846 return TRUE;
2847 }
2848 return FALSE;
2849 }
2850
2851 static struct mbuf *
oce_insert_vlan_tag(POCE_SOFTC sc,struct mbuf * m,boolean_t * complete)2852 oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete)
2853 {
2854 uint16_t vlan_tag = 0;
2855
2856 if(!M_WRITABLE(m))
2857 return NULL;
2858
2859 /* Embed vlan tag in the packet if it is not part of it */
2860 if(m->m_flags & M_VLANTAG) {
2861 vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag);
2862 m->m_flags &= ~M_VLANTAG;
2863 }
2864
2865 /* if UMC, ignore vlan tag insertion and instead insert pvid */
2866 if(sc->pvid) {
2867 if(!vlan_tag)
2868 vlan_tag = sc->pvid;
2869 if (complete)
2870 *complete = FALSE;
2871 }
2872
2873 if(vlan_tag) {
2874 m = ether_vlanencap(m, vlan_tag);
2875 }
2876
2877 if(sc->qnqid) {
2878 m = ether_vlanencap(m, sc->qnqid);
2879
2880 if (complete)
2881 *complete = FALSE;
2882 }
2883 return m;
2884 }
2885
2886 static int
oce_tx_asic_stall_verify(POCE_SOFTC sc,struct mbuf * m)2887 oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m)
2888 {
2889 if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \
2890 oce_check_ipv6_ext_hdr(m)) {
2891 return TRUE;
2892 }
2893 return FALSE;
2894 }
2895
2896 static void
oce_get_config(POCE_SOFTC sc)2897 oce_get_config(POCE_SOFTC sc)
2898 {
2899 int rc = 0;
2900 uint32_t max_rss = 0;
2901
2902 if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native))
2903 max_rss = OCE_LEGACY_MODE_RSS;
2904 else
2905 max_rss = OCE_MAX_RSS;
2906
2907 if (!IS_BE(sc)) {
2908 rc = oce_get_profile_config(sc, max_rss);
2909 if (rc) {
2910 sc->nwqs = OCE_MAX_WQ;
2911 sc->nrssqs = max_rss;
2912 sc->nrqs = sc->nrssqs + 1;
2913 }
2914 }
2915 else { /* For BE3 don't rely on fw for determining the resources */
2916 sc->nrssqs = max_rss;
2917 sc->nrqs = sc->nrssqs + 1;
2918 sc->nwqs = OCE_MAX_WQ;
2919 sc->max_vlans = MAX_VLANFILTER_SIZE;
2920 }
2921 }
2922
2923 static void
oce_rdma_close(void)2924 oce_rdma_close(void)
2925 {
2926 if (oce_rdma_if != NULL) {
2927 oce_rdma_if = NULL;
2928 }
2929 }
2930
2931 static void
oce_get_mac_addr(POCE_SOFTC sc,uint8_t * macaddr)2932 oce_get_mac_addr(POCE_SOFTC sc, uint8_t *macaddr)
2933 {
2934 memcpy(macaddr, sc->macaddr.mac_addr, 6);
2935 }
2936
2937 int
oce_register_rdma(POCE_RDMA_INFO rdma_info,POCE_RDMA_IF rdma_if)2938 oce_register_rdma(POCE_RDMA_INFO rdma_info, POCE_RDMA_IF rdma_if)
2939 {
2940 POCE_SOFTC sc;
2941 struct oce_dev_info di;
2942 int i;
2943
2944 if ((rdma_info == NULL) || (rdma_if == NULL)) {
2945 return -EINVAL;
2946 }
2947
2948 if ((rdma_info->size != OCE_RDMA_INFO_SIZE) ||
2949 (rdma_if->size != OCE_RDMA_IF_SIZE)) {
2950 return -ENXIO;
2951 }
2952
2953 rdma_info->close = oce_rdma_close;
2954 rdma_info->mbox_post = oce_mbox_post;
2955 rdma_info->common_req_hdr_init = mbx_common_req_hdr_init;
2956 rdma_info->get_mac_addr = oce_get_mac_addr;
2957
2958 oce_rdma_if = rdma_if;
2959
2960 sc = softc_head;
2961 while (sc != NULL) {
2962 if (oce_rdma_if->announce != NULL) {
2963 memset(&di, 0, sizeof(di));
2964 di.dev = sc->dev;
2965 di.softc = sc;
2966 di.ifp = sc->ifp;
2967 di.db_bhandle = sc->db_bhandle;
2968 di.db_btag = sc->db_btag;
2969 di.db_page_size = 4096;
2970 if (sc->flags & OCE_FLAGS_USING_MSIX) {
2971 di.intr_mode = OCE_INTERRUPT_MODE_MSIX;
2972 } else if (sc->flags & OCE_FLAGS_USING_MSI) {
2973 di.intr_mode = OCE_INTERRUPT_MODE_MSI;
2974 } else {
2975 di.intr_mode = OCE_INTERRUPT_MODE_INTX;
2976 }
2977 di.dev_family = OCE_GEN2_FAMILY; // fixme: must detect skyhawk
2978 if (di.intr_mode != OCE_INTERRUPT_MODE_INTX) {
2979 di.msix.num_vectors = sc->intr_count + sc->roce_intr_count;
2980 di.msix.start_vector = sc->intr_count;
2981 for (i=0; i<di.msix.num_vectors; i++) {
2982 di.msix.vector_list[i] = sc->intrs[i].vector;
2983 }
2984 } else {
2985 }
2986 memcpy(di.mac_addr, sc->macaddr.mac_addr, 6);
2987 di.vendor_id = pci_get_vendor(sc->dev);
2988 di.dev_id = pci_get_device(sc->dev);
2989
2990 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
2991 di.flags |= OCE_RDMA_INFO_RDMA_SUPPORTED;
2992 }
2993
2994 rdma_if->announce(&di);
2995 sc = sc->next;
2996 }
2997 }
2998
2999 return 0;
3000 }
3001
3002 static void
oce_read_env_variables(POCE_SOFTC sc)3003 oce_read_env_variables( POCE_SOFTC sc )
3004 {
3005 char *value = NULL;
3006 int rc = 0;
3007
3008 /* read if user wants to enable hwlro or swlro */
3009 //value = getenv("oce_enable_hwlro");
3010 if(value && IS_SH(sc)) {
3011 sc->enable_hwlro = strtol(value, NULL, 10);
3012 if(sc->enable_hwlro) {
3013 rc = oce_mbox_nic_query_lro_capabilities(sc, NULL, NULL);
3014 if(rc) {
3015 device_printf(sc->dev, "no hardware lro support\n");
3016 device_printf(sc->dev, "software lro enabled\n");
3017 sc->enable_hwlro = 0;
3018 }else {
3019 device_printf(sc->dev, "hardware lro enabled\n");
3020 oce_max_rsp_handled = 32;
3021 }
3022 }else {
3023 device_printf(sc->dev, "software lro enabled\n");
3024 }
3025 }else {
3026 sc->enable_hwlro = 0;
3027 }
3028
3029 /* read mbuf size */
3030 //value = getenv("oce_rq_buf_size");
3031 if(value && IS_SH(sc)) {
3032 oce_rq_buf_size = strtol(value, NULL, 10);
3033 switch(oce_rq_buf_size) {
3034 case 2048:
3035 case 4096:
3036 case 9216:
3037 case 16384:
3038 break;
3039
3040 default:
3041 device_printf(sc->dev, " Supported oce_rq_buf_size values are 2K, 4K, 9K, 16K \n");
3042 oce_rq_buf_size = 2048;
3043 }
3044 }
3045
3046 return;
3047 }
3048