1 /*
2 * Copyright (c) 2017 Stormshield.
3 * Copyright (c) 2017 Semihalf.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
19 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
21 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
23 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
25 * POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include "opt_platform.h"
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/endian.h>
35 #include <sys/mbuf.h>
36 #include <sys/lock.h>
37 #include <sys/mutex.h>
38 #include <sys/kernel.h>
39 #include <sys/module.h>
40 #include <sys/socket.h>
41 #include <sys/sysctl.h>
42 #include <sys/smp.h>
43 #include <sys/taskqueue.h>
44 #ifdef MVNETA_KTR
45 #include <sys/ktr.h>
46 #endif
47
48 #include <net/ethernet.h>
49 #include <net/bpf.h>
50 #include <net/if.h>
51 #include <net/if_arp.h>
52 #include <net/if_dl.h>
53 #include <net/if_media.h>
54 #include <net/if_types.h>
55 #include <net/if_vlan_var.h>
56
57 #include <netinet/in_systm.h>
58 #include <netinet/in.h>
59 #include <netinet/ip.h>
60 #include <netinet/tcp_lro.h>
61
62 #include <sys/sockio.h>
63 #include <sys/bus.h>
64 #include <machine/bus.h>
65 #include <sys/rman.h>
66 #include <machine/resource.h>
67
68 #include <dev/mii/mii.h>
69 #include <dev/mii/miivar.h>
70
71 #include <dev/ofw/openfirm.h>
72 #include <dev/ofw/ofw_bus.h>
73 #include <dev/ofw/ofw_bus_subr.h>
74
75 #include <dev/mdio/mdio.h>
76
77 #include <arm/mv/mvvar.h>
78
79 #if !defined(__aarch64__)
80 #include <arm/mv/mvreg.h>
81 #include <arm/mv/mvwin.h>
82 #endif
83
84 #include "if_mvnetareg.h"
85 #include "if_mvnetavar.h"
86
87 #include "miibus_if.h"
88 #include "mdio_if.h"
89
90 #ifdef MVNETA_DEBUG
91 #define STATIC /* nothing */
92 #else
93 #define STATIC static
94 #endif
95
96 #define DASSERT(x) KASSERT((x), (#x))
97
98 #define A3700_TCLK_250MHZ 250000000
99
100 /* Device Register Initialization */
101 STATIC int mvneta_initreg(struct ifnet *);
102
103 /* Descriptor Ring Control for each of queues */
104 STATIC int mvneta_ring_alloc_rx_queue(struct mvneta_softc *, int);
105 STATIC int mvneta_ring_alloc_tx_queue(struct mvneta_softc *, int);
106 STATIC void mvneta_ring_dealloc_rx_queue(struct mvneta_softc *, int);
107 STATIC void mvneta_ring_dealloc_tx_queue(struct mvneta_softc *, int);
108 STATIC int mvneta_ring_init_rx_queue(struct mvneta_softc *, int);
109 STATIC int mvneta_ring_init_tx_queue(struct mvneta_softc *, int);
110 STATIC void mvneta_ring_flush_rx_queue(struct mvneta_softc *, int);
111 STATIC void mvneta_ring_flush_tx_queue(struct mvneta_softc *, int);
112 STATIC void mvneta_dmamap_cb(void *, bus_dma_segment_t *, int, int);
113 STATIC int mvneta_dma_create(struct mvneta_softc *);
114
115 /* Rx/Tx Queue Control */
116 STATIC int mvneta_rx_queue_init(struct ifnet *, int);
117 STATIC int mvneta_tx_queue_init(struct ifnet *, int);
118 STATIC int mvneta_rx_queue_enable(struct ifnet *, int);
119 STATIC int mvneta_tx_queue_enable(struct ifnet *, int);
120 STATIC void mvneta_rx_lockq(struct mvneta_softc *, int);
121 STATIC void mvneta_rx_unlockq(struct mvneta_softc *, int);
122 STATIC void mvneta_tx_lockq(struct mvneta_softc *, int);
123 STATIC void mvneta_tx_unlockq(struct mvneta_softc *, int);
124
125 /* Interrupt Handlers */
126 STATIC void mvneta_disable_intr(struct mvneta_softc *);
127 STATIC void mvneta_enable_intr(struct mvneta_softc *);
128 STATIC void mvneta_rxtxth_intr(void *);
129 STATIC int mvneta_misc_intr(struct mvneta_softc *);
130 STATIC void mvneta_tick(void *);
131 /* struct ifnet and mii callbacks*/
132 STATIC int mvneta_xmitfast_locked(struct mvneta_softc *, int, struct mbuf **);
133 STATIC int mvneta_xmit_locked(struct mvneta_softc *, int);
134 #ifdef MVNETA_MULTIQUEUE
135 STATIC int mvneta_transmit(struct ifnet *, struct mbuf *);
136 #else /* !MVNETA_MULTIQUEUE */
137 STATIC void mvneta_start(struct ifnet *);
138 #endif
139 STATIC void mvneta_qflush(struct ifnet *);
140 STATIC void mvneta_tx_task(void *, int);
141 STATIC int mvneta_ioctl(struct ifnet *, u_long, caddr_t);
142 STATIC void mvneta_init(void *);
143 STATIC void mvneta_init_locked(void *);
144 STATIC void mvneta_stop(struct mvneta_softc *);
145 STATIC void mvneta_stop_locked(struct mvneta_softc *);
146 STATIC int mvneta_mediachange(struct ifnet *);
147 STATIC void mvneta_mediastatus(struct ifnet *, struct ifmediareq *);
148 STATIC void mvneta_portup(struct mvneta_softc *);
149 STATIC void mvneta_portdown(struct mvneta_softc *);
150
151 /* Link State Notify */
152 STATIC void mvneta_update_autoneg(struct mvneta_softc *, int);
153 STATIC int mvneta_update_media(struct mvneta_softc *, int);
154 STATIC void mvneta_adjust_link(struct mvneta_softc *);
155 STATIC void mvneta_update_eee(struct mvneta_softc *);
156 STATIC void mvneta_update_fc(struct mvneta_softc *);
157 STATIC void mvneta_link_isr(struct mvneta_softc *);
158 STATIC void mvneta_linkupdate(struct mvneta_softc *, boolean_t);
159 STATIC void mvneta_linkup(struct mvneta_softc *);
160 STATIC void mvneta_linkdown(struct mvneta_softc *);
161 STATIC void mvneta_linkreset(struct mvneta_softc *);
162
163 /* Tx Subroutines */
164 STATIC int mvneta_tx_queue(struct mvneta_softc *, struct mbuf **, int);
165 STATIC void mvneta_tx_set_csumflag(struct ifnet *,
166 struct mvneta_tx_desc *, struct mbuf *);
167 STATIC void mvneta_tx_queue_complete(struct mvneta_softc *, int);
168 STATIC void mvneta_tx_drain(struct mvneta_softc *);
169
170 /* Rx Subroutines */
171 STATIC int mvneta_rx(struct mvneta_softc *, int, int);
172 STATIC void mvneta_rx_queue(struct mvneta_softc *, int, int);
173 STATIC void mvneta_rx_queue_refill(struct mvneta_softc *, int);
174 STATIC void mvneta_rx_set_csumflag(struct ifnet *,
175 struct mvneta_rx_desc *, struct mbuf *);
176 STATIC void mvneta_rx_buf_free(struct mvneta_softc *, struct mvneta_buf *);
177
178 /* MAC address filter */
179 STATIC void mvneta_filter_setup(struct mvneta_softc *);
180
181 /* sysctl(9) */
182 STATIC int sysctl_read_mib(SYSCTL_HANDLER_ARGS);
183 STATIC int sysctl_clear_mib(SYSCTL_HANDLER_ARGS);
184 STATIC int sysctl_set_queue_rxthtime(SYSCTL_HANDLER_ARGS);
185 STATIC void sysctl_mvneta_init(struct mvneta_softc *);
186
187 /* MIB */
188 STATIC void mvneta_clear_mib(struct mvneta_softc *);
189 STATIC uint64_t mvneta_read_mib(struct mvneta_softc *, int);
190 STATIC void mvneta_update_mib(struct mvneta_softc *);
191
192 /* Switch */
193 STATIC boolean_t mvneta_find_ethernet_prop_switch(phandle_t, phandle_t);
194 STATIC boolean_t mvneta_has_switch(device_t);
195
196 #define mvneta_sc_lock(sc) mtx_lock(&sc->mtx)
197 #define mvneta_sc_unlock(sc) mtx_unlock(&sc->mtx)
198
199 STATIC struct mtx mii_mutex;
200 STATIC int mii_init = 0;
201
202 /* Device */
203 STATIC int mvneta_detach(device_t);
204 /* MII */
205 STATIC int mvneta_miibus_readreg(device_t, int, int);
206 STATIC int mvneta_miibus_writereg(device_t, int, int, int);
207
208 /* Clock */
209 STATIC uint32_t mvneta_get_clk(void);
210
211 static device_method_t mvneta_methods[] = {
212 /* Device interface */
213 DEVMETHOD(device_detach, mvneta_detach),
214 /* MII interface */
215 DEVMETHOD(miibus_readreg, mvneta_miibus_readreg),
216 DEVMETHOD(miibus_writereg, mvneta_miibus_writereg),
217 /* MDIO interface */
218 DEVMETHOD(mdio_readreg, mvneta_miibus_readreg),
219 DEVMETHOD(mdio_writereg, mvneta_miibus_writereg),
220
221 /* End */
222 DEVMETHOD_END
223 };
224
225 DEFINE_CLASS_0(mvneta, mvneta_driver, mvneta_methods, sizeof(struct mvneta_softc));
226
227 DRIVER_MODULE(miibus, mvneta, miibus_driver, miibus_devclass, 0, 0);
228 DRIVER_MODULE(mdio, mvneta, mdio_driver, mdio_devclass, 0, 0);
229 MODULE_DEPEND(mvneta, mdio, 1, 1, 1);
230 MODULE_DEPEND(mvneta, ether, 1, 1, 1);
231 MODULE_DEPEND(mvneta, miibus, 1, 1, 1);
232 MODULE_DEPEND(mvneta, mvxpbm, 1, 1, 1);
233
234 /*
235 * List of MIB register and names
236 */
237 enum mvneta_mib_idx
238 {
239 MVNETA_MIB_RX_GOOD_OCT_IDX,
240 MVNETA_MIB_RX_BAD_OCT_IDX,
241 MVNETA_MIB_TX_MAC_TRNS_ERR_IDX,
242 MVNETA_MIB_RX_GOOD_FRAME_IDX,
243 MVNETA_MIB_RX_BAD_FRAME_IDX,
244 MVNETA_MIB_RX_BCAST_FRAME_IDX,
245 MVNETA_MIB_RX_MCAST_FRAME_IDX,
246 MVNETA_MIB_RX_FRAME64_OCT_IDX,
247 MVNETA_MIB_RX_FRAME127_OCT_IDX,
248 MVNETA_MIB_RX_FRAME255_OCT_IDX,
249 MVNETA_MIB_RX_FRAME511_OCT_IDX,
250 MVNETA_MIB_RX_FRAME1023_OCT_IDX,
251 MVNETA_MIB_RX_FRAMEMAX_OCT_IDX,
252 MVNETA_MIB_TX_GOOD_OCT_IDX,
253 MVNETA_MIB_TX_GOOD_FRAME_IDX,
254 MVNETA_MIB_TX_EXCES_COL_IDX,
255 MVNETA_MIB_TX_MCAST_FRAME_IDX,
256 MVNETA_MIB_TX_BCAST_FRAME_IDX,
257 MVNETA_MIB_TX_MAC_CTL_ERR_IDX,
258 MVNETA_MIB_FC_SENT_IDX,
259 MVNETA_MIB_FC_GOOD_IDX,
260 MVNETA_MIB_FC_BAD_IDX,
261 MVNETA_MIB_PKT_UNDERSIZE_IDX,
262 MVNETA_MIB_PKT_FRAGMENT_IDX,
263 MVNETA_MIB_PKT_OVERSIZE_IDX,
264 MVNETA_MIB_PKT_JABBER_IDX,
265 MVNETA_MIB_MAC_RX_ERR_IDX,
266 MVNETA_MIB_MAC_CRC_ERR_IDX,
267 MVNETA_MIB_MAC_COL_IDX,
268 MVNETA_MIB_MAC_LATE_COL_IDX,
269 };
270
271 STATIC struct mvneta_mib_def {
272 uint32_t regnum;
273 int reg64;
274 const char *sysctl_name;
275 const char *desc;
276 } mvneta_mib_list[] = {
277 [MVNETA_MIB_RX_GOOD_OCT_IDX] = {MVNETA_MIB_RX_GOOD_OCT, 1,
278 "rx_good_oct", "Good Octets Rx"},
279 [MVNETA_MIB_RX_BAD_OCT_IDX] = {MVNETA_MIB_RX_BAD_OCT, 0,
280 "rx_bad_oct", "Bad Octets Rx"},
281 [MVNETA_MIB_TX_MAC_TRNS_ERR_IDX] = {MVNETA_MIB_TX_MAC_TRNS_ERR, 0,
282 "tx_mac_err", "MAC Transmit Error"},
283 [MVNETA_MIB_RX_GOOD_FRAME_IDX] = {MVNETA_MIB_RX_GOOD_FRAME, 0,
284 "rx_good_frame", "Good Frames Rx"},
285 [MVNETA_MIB_RX_BAD_FRAME_IDX] = {MVNETA_MIB_RX_BAD_FRAME, 0,
286 "rx_bad_frame", "Bad Frames Rx"},
287 [MVNETA_MIB_RX_BCAST_FRAME_IDX] = {MVNETA_MIB_RX_BCAST_FRAME, 0,
288 "rx_bcast_frame", "Broadcast Frames Rx"},
289 [MVNETA_MIB_RX_MCAST_FRAME_IDX] = {MVNETA_MIB_RX_MCAST_FRAME, 0,
290 "rx_mcast_frame", "Multicast Frames Rx"},
291 [MVNETA_MIB_RX_FRAME64_OCT_IDX] = {MVNETA_MIB_RX_FRAME64_OCT, 0,
292 "rx_frame_1_64", "Frame Size 1 - 64"},
293 [MVNETA_MIB_RX_FRAME127_OCT_IDX] = {MVNETA_MIB_RX_FRAME127_OCT, 0,
294 "rx_frame_65_127", "Frame Size 65 - 127"},
295 [MVNETA_MIB_RX_FRAME255_OCT_IDX] = {MVNETA_MIB_RX_FRAME255_OCT, 0,
296 "rx_frame_128_255", "Frame Size 128 - 255"},
297 [MVNETA_MIB_RX_FRAME511_OCT_IDX] = {MVNETA_MIB_RX_FRAME511_OCT, 0,
298 "rx_frame_256_511", "Frame Size 256 - 511"},
299 [MVNETA_MIB_RX_FRAME1023_OCT_IDX] = {MVNETA_MIB_RX_FRAME1023_OCT, 0,
300 "rx_frame_512_1023", "Frame Size 512 - 1023"},
301 [MVNETA_MIB_RX_FRAMEMAX_OCT_IDX] = {MVNETA_MIB_RX_FRAMEMAX_OCT, 0,
302 "rx_fame_1024_max", "Frame Size 1024 - Max"},
303 [MVNETA_MIB_TX_GOOD_OCT_IDX] = {MVNETA_MIB_TX_GOOD_OCT, 1,
304 "tx_good_oct", "Good Octets Tx"},
305 [MVNETA_MIB_TX_GOOD_FRAME_IDX] = {MVNETA_MIB_TX_GOOD_FRAME, 0,
306 "tx_good_frame", "Good Frames Tx"},
307 [MVNETA_MIB_TX_EXCES_COL_IDX] = {MVNETA_MIB_TX_EXCES_COL, 0,
308 "tx_exces_collision", "Excessive Collision"},
309 [MVNETA_MIB_TX_MCAST_FRAME_IDX] = {MVNETA_MIB_TX_MCAST_FRAME, 0,
310 "tx_mcast_frame", "Multicast Frames Tx"},
311 [MVNETA_MIB_TX_BCAST_FRAME_IDX] = {MVNETA_MIB_TX_BCAST_FRAME, 0,
312 "tx_bcast_frame", "Broadcast Frames Tx"},
313 [MVNETA_MIB_TX_MAC_CTL_ERR_IDX] = {MVNETA_MIB_TX_MAC_CTL_ERR, 0,
314 "tx_mac_ctl_err", "Unknown MAC Control"},
315 [MVNETA_MIB_FC_SENT_IDX] = {MVNETA_MIB_FC_SENT, 0,
316 "fc_tx", "Flow Control Tx"},
317 [MVNETA_MIB_FC_GOOD_IDX] = {MVNETA_MIB_FC_GOOD, 0,
318 "fc_rx_good", "Good Flow Control Rx"},
319 [MVNETA_MIB_FC_BAD_IDX] = {MVNETA_MIB_FC_BAD, 0,
320 "fc_rx_bad", "Bad Flow Control Rx"},
321 [MVNETA_MIB_PKT_UNDERSIZE_IDX] = {MVNETA_MIB_PKT_UNDERSIZE, 0,
322 "pkt_undersize", "Undersized Packets Rx"},
323 [MVNETA_MIB_PKT_FRAGMENT_IDX] = {MVNETA_MIB_PKT_FRAGMENT, 0,
324 "pkt_fragment", "Fragmented Packets Rx"},
325 [MVNETA_MIB_PKT_OVERSIZE_IDX] = {MVNETA_MIB_PKT_OVERSIZE, 0,
326 "pkt_oversize", "Oversized Packets Rx"},
327 [MVNETA_MIB_PKT_JABBER_IDX] = {MVNETA_MIB_PKT_JABBER, 0,
328 "pkt_jabber", "Jabber Packets Rx"},
329 [MVNETA_MIB_MAC_RX_ERR_IDX] = {MVNETA_MIB_MAC_RX_ERR, 0,
330 "mac_rx_err", "MAC Rx Errors"},
331 [MVNETA_MIB_MAC_CRC_ERR_IDX] = {MVNETA_MIB_MAC_CRC_ERR, 0,
332 "mac_crc_err", "MAC CRC Errors"},
333 [MVNETA_MIB_MAC_COL_IDX] = {MVNETA_MIB_MAC_COL, 0,
334 "mac_collision", "MAC Collision"},
335 [MVNETA_MIB_MAC_LATE_COL_IDX] = {MVNETA_MIB_MAC_LATE_COL, 0,
336 "mac_late_collision", "MAC Late Collision"},
337 };
338
339 static struct resource_spec res_spec[] = {
340 { SYS_RES_MEMORY, 0, RF_ACTIVE },
341 { SYS_RES_IRQ, 0, RF_ACTIVE },
342 { -1, 0}
343 };
344
345 static struct {
346 driver_intr_t *handler;
347 char * description;
348 } mvneta_intrs[] = {
349 { mvneta_rxtxth_intr, "MVNETA aggregated interrupt" },
350 };
351
352 STATIC uint32_t
mvneta_get_clk()353 mvneta_get_clk()
354 {
355 #if defined(__aarch64__)
356 return (A3700_TCLK_250MHZ);
357 #else
358 return (get_tclk());
359 #endif
360 }
361
362 static int
mvneta_set_mac_address(struct mvneta_softc * sc,uint8_t * addr)363 mvneta_set_mac_address(struct mvneta_softc *sc, uint8_t *addr)
364 {
365 unsigned int mac_h;
366 unsigned int mac_l;
367
368 mac_l = (addr[4] << 8) | (addr[5]);
369 mac_h = (addr[0] << 24) | (addr[1] << 16) |
370 (addr[2] << 8) | (addr[3] << 0);
371
372 MVNETA_WRITE(sc, MVNETA_MACAL, mac_l);
373 MVNETA_WRITE(sc, MVNETA_MACAH, mac_h);
374 return (0);
375 }
376
377 static int
mvneta_get_mac_address(struct mvneta_softc * sc,uint8_t * addr)378 mvneta_get_mac_address(struct mvneta_softc *sc, uint8_t *addr)
379 {
380 uint32_t mac_l, mac_h;
381
382 #ifdef FDT
383 if (mvneta_fdt_mac_address(sc, addr) == 0)
384 return (0);
385 #endif
386 /*
387 * Fall back -- use the currently programmed address.
388 */
389 mac_l = MVNETA_READ(sc, MVNETA_MACAL);
390 mac_h = MVNETA_READ(sc, MVNETA_MACAH);
391 if (mac_l == 0 && mac_h == 0) {
392 /*
393 * Generate pseudo-random MAC.
394 * Set lower part to random number | unit number.
395 */
396 mac_l = arc4random() & ~0xff;
397 mac_l |= device_get_unit(sc->dev) & 0xff;
398 mac_h = arc4random();
399 mac_h &= ~(3 << 24); /* Clear multicast and LAA bits */
400 if (bootverbose) {
401 device_printf(sc->dev,
402 "Could not acquire MAC address. "
403 "Using randomized one.\n");
404 }
405 }
406
407 addr[0] = (mac_h & 0xff000000) >> 24;
408 addr[1] = (mac_h & 0x00ff0000) >> 16;
409 addr[2] = (mac_h & 0x0000ff00) >> 8;
410 addr[3] = (mac_h & 0x000000ff);
411 addr[4] = (mac_l & 0x0000ff00) >> 8;
412 addr[5] = (mac_l & 0x000000ff);
413 return (0);
414 }
415
416 STATIC boolean_t
mvneta_find_ethernet_prop_switch(phandle_t ethernet,phandle_t node)417 mvneta_find_ethernet_prop_switch(phandle_t ethernet, phandle_t node)
418 {
419 boolean_t ret;
420 phandle_t child, switch_eth_handle, switch_eth;
421
422 for (child = OF_child(node); child != 0; child = OF_peer(child)) {
423 if (OF_getencprop(child, "ethernet", (void*)&switch_eth_handle,
424 sizeof(switch_eth_handle)) > 0) {
425 if (switch_eth_handle > 0) {
426 switch_eth = OF_node_from_xref(
427 switch_eth_handle);
428
429 if (switch_eth == ethernet)
430 return (true);
431 }
432 }
433
434 ret = mvneta_find_ethernet_prop_switch(ethernet, child);
435 if (ret != 0)
436 return (ret);
437 }
438
439 return (false);
440 }
441
442 STATIC boolean_t
mvneta_has_switch(device_t self)443 mvneta_has_switch(device_t self)
444 {
445 phandle_t node;
446
447 node = ofw_bus_get_node(self);
448
449 return mvneta_find_ethernet_prop_switch(node, OF_finddevice("/"));
450 }
451
452 STATIC int
mvneta_dma_create(struct mvneta_softc * sc)453 mvneta_dma_create(struct mvneta_softc *sc)
454 {
455 size_t maxsize, maxsegsz;
456 size_t q;
457 int error;
458
459 /*
460 * Create Tx DMA
461 */
462 maxsize = maxsegsz = sizeof(struct mvneta_tx_desc) * MVNETA_TX_RING_CNT;
463
464 error = bus_dma_tag_create(
465 bus_get_dma_tag(sc->dev), /* parent */
466 16, 0, /* alignment, boundary */
467 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
468 BUS_SPACE_MAXADDR, /* highaddr */
469 NULL, NULL, /* filtfunc, filtfuncarg */
470 maxsize, /* maxsize */
471 1, /* nsegments */
472 maxsegsz, /* maxsegsz */
473 0, /* flags */
474 NULL, NULL, /* lockfunc, lockfuncarg */
475 &sc->tx_dtag); /* dmat */
476 if (error != 0) {
477 device_printf(sc->dev,
478 "Failed to create DMA tag for Tx descriptors.\n");
479 goto fail;
480 }
481 error = bus_dma_tag_create(
482 bus_get_dma_tag(sc->dev), /* parent */
483 1, 0, /* alignment, boundary */
484 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
485 BUS_SPACE_MAXADDR, /* highaddr */
486 NULL, NULL, /* filtfunc, filtfuncarg */
487 MVNETA_MAX_FRAME, /* maxsize */
488 MVNETA_TX_SEGLIMIT, /* nsegments */
489 MVNETA_MAX_FRAME, /* maxsegsz */
490 BUS_DMA_ALLOCNOW, /* flags */
491 NULL, NULL, /* lockfunc, lockfuncarg */
492 &sc->txmbuf_dtag);
493 if (error != 0) {
494 device_printf(sc->dev,
495 "Failed to create DMA tag for Tx mbufs.\n");
496 goto fail;
497 }
498
499 for (q = 0; q < MVNETA_TX_QNUM_MAX; q++) {
500 error = mvneta_ring_alloc_tx_queue(sc, q);
501 if (error != 0) {
502 device_printf(sc->dev,
503 "Failed to allocate DMA safe memory for TxQ: %zu\n", q);
504 goto fail;
505 }
506 }
507
508 /*
509 * Create Rx DMA.
510 */
511 /* Create tag for Rx descripors */
512 error = bus_dma_tag_create(
513 bus_get_dma_tag(sc->dev), /* parent */
514 32, 0, /* alignment, boundary */
515 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
516 BUS_SPACE_MAXADDR, /* highaddr */
517 NULL, NULL, /* filtfunc, filtfuncarg */
518 sizeof(struct mvneta_rx_desc) * MVNETA_RX_RING_CNT, /* maxsize */
519 1, /* nsegments */
520 sizeof(struct mvneta_rx_desc) * MVNETA_RX_RING_CNT, /* maxsegsz */
521 0, /* flags */
522 NULL, NULL, /* lockfunc, lockfuncarg */
523 &sc->rx_dtag); /* dmat */
524 if (error != 0) {
525 device_printf(sc->dev,
526 "Failed to create DMA tag for Rx descriptors.\n");
527 goto fail;
528 }
529
530 /* Create tag for Rx buffers */
531 error = bus_dma_tag_create(
532 bus_get_dma_tag(sc->dev), /* parent */
533 32, 0, /* alignment, boundary */
534 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
535 BUS_SPACE_MAXADDR, /* highaddr */
536 NULL, NULL, /* filtfunc, filtfuncarg */
537 MVNETA_MAX_FRAME, 1, /* maxsize, nsegments */
538 MVNETA_MAX_FRAME, /* maxsegsz */
539 0, /* flags */
540 NULL, NULL, /* lockfunc, lockfuncarg */
541 &sc->rxbuf_dtag); /* dmat */
542 if (error != 0) {
543 device_printf(sc->dev,
544 "Failed to create DMA tag for Rx buffers.\n");
545 goto fail;
546 }
547
548 for (q = 0; q < MVNETA_RX_QNUM_MAX; q++) {
549 if (mvneta_ring_alloc_rx_queue(sc, q) != 0) {
550 device_printf(sc->dev,
551 "Failed to allocate DMA safe memory for RxQ: %zu\n", q);
552 goto fail;
553 }
554 }
555
556 return (0);
557 fail:
558 mvneta_detach(sc->dev);
559
560 return (error);
561 }
562
563 /* ARGSUSED */
564 int
mvneta_attach(device_t self)565 mvneta_attach(device_t self)
566 {
567 struct mvneta_softc *sc;
568 struct ifnet *ifp;
569 device_t child;
570 int ifm_target;
571 int q, error;
572 #if !defined(__aarch64__)
573 uint32_t reg;
574 #endif
575
576 sc = device_get_softc(self);
577 sc->dev = self;
578
579 mtx_init(&sc->mtx, "mvneta_sc", NULL, MTX_DEF);
580
581 error = bus_alloc_resources(self, res_spec, sc->res);
582 if (error) {
583 device_printf(self, "could not allocate resources\n");
584 return (ENXIO);
585 }
586
587 sc->version = MVNETA_READ(sc, MVNETA_PV);
588 device_printf(self, "version is %x\n", sc->version);
589 callout_init(&sc->tick_ch, 0);
590
591 /*
592 * make sure DMA engines are in reset state
593 */
594 MVNETA_WRITE(sc, MVNETA_PRXINIT, 0x00000001);
595 MVNETA_WRITE(sc, MVNETA_PTXINIT, 0x00000001);
596
597 #if !defined(__aarch64__)
598 /*
599 * Disable port snoop for buffers and descriptors
600 * to avoid L2 caching of both without DRAM copy.
601 * Obtain coherency settings from the first MBUS
602 * window attribute.
603 */
604 if ((MVNETA_READ(sc, MV_WIN_NETA_BASE(0)) & IO_WIN_COH_ATTR_MASK) == 0) {
605 reg = MVNETA_READ(sc, MVNETA_PSNPCFG);
606 reg &= ~MVNETA_PSNPCFG_DESCSNP_MASK;
607 reg &= ~MVNETA_PSNPCFG_BUFSNP_MASK;
608 MVNETA_WRITE(sc, MVNETA_PSNPCFG, reg);
609 }
610 #endif
611
612 /*
613 * MAC address
614 */
615 if (mvneta_get_mac_address(sc, sc->enaddr)) {
616 device_printf(self, "no mac address.\n");
617 return (ENXIO);
618 }
619 mvneta_set_mac_address(sc, sc->enaddr);
620
621 mvneta_disable_intr(sc);
622
623 /* Allocate network interface */
624 ifp = sc->ifp = if_alloc(IFT_ETHER);
625 if (ifp == NULL) {
626 device_printf(self, "if_alloc() failed\n");
627 mvneta_detach(self);
628 return (ENOMEM);
629 }
630 if_initname(ifp, device_get_name(self), device_get_unit(self));
631
632 /*
633 * We can support 802.1Q VLAN-sized frames and jumbo
634 * Ethernet frames.
635 */
636 ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_JUMBO_MTU;
637
638 ifp->if_softc = sc;
639 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
640 #ifdef MVNETA_MULTIQUEUE
641 ifp->if_transmit = mvneta_transmit;
642 ifp->if_qflush = mvneta_qflush;
643 #else /* !MVNETA_MULTIQUEUE */
644 ifp->if_start = mvneta_start;
645 ifp->if_snd.ifq_drv_maxlen = MVNETA_TX_RING_CNT - 1;
646 IFQ_SET_MAXLEN(&ifp->if_snd, ifp->if_snd.ifq_drv_maxlen);
647 IFQ_SET_READY(&ifp->if_snd);
648 #endif
649 ifp->if_init = mvneta_init;
650 ifp->if_ioctl = mvneta_ioctl;
651
652 /*
653 * We can do IPv4/TCPv4/UDPv4/TCPv6/UDPv6 checksums in hardware.
654 */
655 ifp->if_capabilities |= IFCAP_HWCSUM;
656
657 /*
658 * As VLAN hardware tagging is not supported
659 * but is necessary to perform VLAN hardware checksums,
660 * it is done in the driver
661 */
662 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM;
663
664 /*
665 * Currently IPv6 HW checksum is broken, so make sure it is disabled.
666 */
667 ifp->if_capabilities &= ~IFCAP_HWCSUM_IPV6;
668 ifp->if_capenable = ifp->if_capabilities;
669
670 /*
671 * Disabled option(s):
672 * - Support for Large Receive Offload
673 */
674 ifp->if_capabilities |= IFCAP_LRO;
675
676 ifp->if_hwassist = CSUM_IP | CSUM_TCP | CSUM_UDP;
677
678 sc->rx_frame_size = MCLBYTES; /* ether_ifattach() always sets normal mtu */
679
680 /*
681 * Device DMA Buffer allocation.
682 * Handles resource deallocation in case of failure.
683 */
684 error = mvneta_dma_create(sc);
685 if (error != 0) {
686 mvneta_detach(self);
687 return (error);
688 }
689
690 /* Initialize queues */
691 for (q = 0; q < MVNETA_TX_QNUM_MAX; q++) {
692 error = mvneta_ring_init_tx_queue(sc, q);
693 if (error != 0) {
694 mvneta_detach(self);
695 return (error);
696 }
697 }
698
699 for (q = 0; q < MVNETA_RX_QNUM_MAX; q++) {
700 error = mvneta_ring_init_rx_queue(sc, q);
701 if (error != 0) {
702 mvneta_detach(self);
703 return (error);
704 }
705 }
706
707 ether_ifattach(ifp, sc->enaddr);
708
709 /*
710 * Enable DMA engines and Initialize Device Registers.
711 */
712 MVNETA_WRITE(sc, MVNETA_PRXINIT, 0x00000000);
713 MVNETA_WRITE(sc, MVNETA_PTXINIT, 0x00000000);
714 MVNETA_WRITE(sc, MVNETA_PACC, MVNETA_PACC_ACCELERATIONMODE_EDM);
715 mvneta_sc_lock(sc);
716 mvneta_filter_setup(sc);
717 mvneta_sc_unlock(sc);
718 mvneta_initreg(ifp);
719
720 /*
721 * Now MAC is working, setup MII.
722 */
723 if (mii_init == 0) {
724 /*
725 * MII bus is shared by all MACs and all PHYs in SoC.
726 * serializing the bus access should be safe.
727 */
728 mtx_init(&mii_mutex, "mvneta_mii", NULL, MTX_DEF);
729 mii_init = 1;
730 }
731
732 /* Attach PHY(s) */
733 if ((sc->phy_addr != MII_PHY_ANY) && (!sc->use_inband_status)) {
734 error = mii_attach(self, &sc->miibus, ifp, mvneta_mediachange,
735 mvneta_mediastatus, BMSR_DEFCAPMASK, sc->phy_addr,
736 MII_OFFSET_ANY, 0);
737 if (error != 0) {
738 device_printf(self, "MII attach failed, error: %d\n",
739 error);
740 ether_ifdetach(sc->ifp);
741 mvneta_detach(self);
742 return (error);
743 }
744 sc->mii = device_get_softc(sc->miibus);
745 sc->phy_attached = 1;
746
747 /* Disable auto-negotiation in MAC - rely on PHY layer */
748 mvneta_update_autoneg(sc, FALSE);
749 } else if (sc->use_inband_status == TRUE) {
750 /* In-band link status */
751 ifmedia_init(&sc->mvneta_ifmedia, 0, mvneta_mediachange,
752 mvneta_mediastatus);
753
754 /* Configure media */
755 ifmedia_add(&sc->mvneta_ifmedia, IFM_ETHER | IFM_1000_T | IFM_FDX,
756 0, NULL);
757 ifmedia_add(&sc->mvneta_ifmedia, IFM_ETHER | IFM_100_TX, 0, NULL);
758 ifmedia_add(&sc->mvneta_ifmedia, IFM_ETHER | IFM_100_TX | IFM_FDX,
759 0, NULL);
760 ifmedia_add(&sc->mvneta_ifmedia, IFM_ETHER | IFM_10_T, 0, NULL);
761 ifmedia_add(&sc->mvneta_ifmedia, IFM_ETHER | IFM_10_T | IFM_FDX,
762 0, NULL);
763 ifmedia_add(&sc->mvneta_ifmedia, IFM_ETHER | IFM_AUTO, 0, NULL);
764 ifmedia_set(&sc->mvneta_ifmedia, IFM_ETHER | IFM_AUTO);
765
766 /* Enable auto-negotiation */
767 mvneta_update_autoneg(sc, TRUE);
768
769 mvneta_sc_lock(sc);
770 if (MVNETA_IS_LINKUP(sc))
771 mvneta_linkup(sc);
772 else
773 mvneta_linkdown(sc);
774 mvneta_sc_unlock(sc);
775
776 } else {
777 /* Fixed-link, use predefined values */
778 mvneta_update_autoneg(sc, FALSE);
779 ifmedia_init(&sc->mvneta_ifmedia, 0, mvneta_mediachange,
780 mvneta_mediastatus);
781
782 ifm_target = IFM_ETHER;
783 switch (sc->phy_speed) {
784 case 2500:
785 if (sc->phy_mode != MVNETA_PHY_SGMII &&
786 sc->phy_mode != MVNETA_PHY_QSGMII) {
787 device_printf(self,
788 "2.5G speed can work only in (Q)SGMII mode\n");
789 ether_ifdetach(sc->ifp);
790 mvneta_detach(self);
791 return (ENXIO);
792 }
793 ifm_target |= IFM_2500_T;
794 break;
795 case 1000:
796 ifm_target |= IFM_1000_T;
797 break;
798 case 100:
799 ifm_target |= IFM_100_TX;
800 break;
801 case 10:
802 ifm_target |= IFM_10_T;
803 break;
804 default:
805 ether_ifdetach(sc->ifp);
806 mvneta_detach(self);
807 return (ENXIO);
808 }
809
810 if (sc->phy_fdx)
811 ifm_target |= IFM_FDX;
812 else
813 ifm_target |= IFM_HDX;
814
815 ifmedia_add(&sc->mvneta_ifmedia, ifm_target, 0, NULL);
816 ifmedia_set(&sc->mvneta_ifmedia, ifm_target);
817 if_link_state_change(sc->ifp, LINK_STATE_UP);
818
819 if (mvneta_has_switch(self)) {
820 if (bootverbose)
821 device_printf(self, "This device is attached to a switch\n");
822 child = device_add_child(sc->dev, "mdio", -1);
823 if (child == NULL) {
824 ether_ifdetach(sc->ifp);
825 mvneta_detach(self);
826 return (ENXIO);
827 }
828 bus_generic_attach(sc->dev);
829 bus_generic_attach(child);
830 }
831
832 /* Configure MAC media */
833 mvneta_update_media(sc, ifm_target);
834 }
835
836 sysctl_mvneta_init(sc);
837
838 callout_reset(&sc->tick_ch, 0, mvneta_tick, sc);
839
840 error = bus_setup_intr(self, sc->res[1],
841 INTR_TYPE_NET | INTR_MPSAFE, NULL, mvneta_intrs[0].handler, sc,
842 &sc->ih_cookie[0]);
843 if (error) {
844 device_printf(self, "could not setup %s\n",
845 mvneta_intrs[0].description);
846 ether_ifdetach(sc->ifp);
847 mvneta_detach(self);
848 return (error);
849 }
850
851 return (0);
852 }
853
854 STATIC int
mvneta_detach(device_t dev)855 mvneta_detach(device_t dev)
856 {
857 struct mvneta_softc *sc;
858 int q;
859
860 sc = device_get_softc(dev);
861
862 mvneta_stop(sc);
863 /* Detach network interface */
864 if (sc->ifp)
865 if_free(sc->ifp);
866
867 for (q = 0; q < MVNETA_RX_QNUM_MAX; q++)
868 mvneta_ring_dealloc_rx_queue(sc, q);
869 for (q = 0; q < MVNETA_TX_QNUM_MAX; q++)
870 mvneta_ring_dealloc_tx_queue(sc, q);
871
872 if (sc->tx_dtag != NULL)
873 bus_dma_tag_destroy(sc->tx_dtag);
874 if (sc->rx_dtag != NULL)
875 bus_dma_tag_destroy(sc->rx_dtag);
876 if (sc->txmbuf_dtag != NULL)
877 bus_dma_tag_destroy(sc->txmbuf_dtag);
878 if (sc->rxbuf_dtag != NULL)
879 bus_dma_tag_destroy(sc->rxbuf_dtag);
880
881 bus_release_resources(dev, res_spec, sc->res);
882 return (0);
883 }
884
885 /*
886 * MII
887 */
888 STATIC int
mvneta_miibus_readreg(device_t dev,int phy,int reg)889 mvneta_miibus_readreg(device_t dev, int phy, int reg)
890 {
891 struct mvneta_softc *sc;
892 struct ifnet *ifp;
893 uint32_t smi, val;
894 int i;
895
896 sc = device_get_softc(dev);
897 ifp = sc->ifp;
898
899 mtx_lock(&mii_mutex);
900
901 for (i = 0; i < MVNETA_PHY_TIMEOUT; i++) {
902 if ((MVNETA_READ(sc, MVNETA_SMI) & MVNETA_SMI_BUSY) == 0)
903 break;
904 DELAY(1);
905 }
906 if (i == MVNETA_PHY_TIMEOUT) {
907 if_printf(ifp, "SMI busy timeout\n");
908 mtx_unlock(&mii_mutex);
909 return (-1);
910 }
911
912 smi = MVNETA_SMI_PHYAD(phy) |
913 MVNETA_SMI_REGAD(reg) | MVNETA_SMI_OPCODE_READ;
914 MVNETA_WRITE(sc, MVNETA_SMI, smi);
915
916 for (i = 0; i < MVNETA_PHY_TIMEOUT; i++) {
917 if ((MVNETA_READ(sc, MVNETA_SMI) & MVNETA_SMI_BUSY) == 0)
918 break;
919 DELAY(1);
920 }
921
922 if (i == MVNETA_PHY_TIMEOUT) {
923 if_printf(ifp, "SMI busy timeout\n");
924 mtx_unlock(&mii_mutex);
925 return (-1);
926 }
927 for (i = 0; i < MVNETA_PHY_TIMEOUT; i++) {
928 smi = MVNETA_READ(sc, MVNETA_SMI);
929 if (smi & MVNETA_SMI_READVALID)
930 break;
931 DELAY(1);
932 }
933
934 if (i == MVNETA_PHY_TIMEOUT) {
935 if_printf(ifp, "SMI busy timeout\n");
936 mtx_unlock(&mii_mutex);
937 return (-1);
938 }
939
940 mtx_unlock(&mii_mutex);
941
942 #ifdef MVNETA_KTR
943 CTR3(KTR_SPARE2, "%s i=%d, timeout=%d\n", ifp->if_xname, i,
944 MVNETA_PHY_TIMEOUT);
945 #endif
946
947 val = smi & MVNETA_SMI_DATA_MASK;
948
949 #ifdef MVNETA_KTR
950 CTR4(KTR_SPARE2, "%s phy=%d, reg=%#x, val=%#x\n", ifp->if_xname, phy,
951 reg, val);
952 #endif
953 return (val);
954 }
955
956 STATIC int
mvneta_miibus_writereg(device_t dev,int phy,int reg,int val)957 mvneta_miibus_writereg(device_t dev, int phy, int reg, int val)
958 {
959 struct mvneta_softc *sc;
960 struct ifnet *ifp;
961 uint32_t smi;
962 int i;
963
964 sc = device_get_softc(dev);
965 ifp = sc->ifp;
966 #ifdef MVNETA_KTR
967 CTR4(KTR_SPARE2, "%s phy=%d, reg=%#x, val=%#x\n", ifp->if_xname,
968 phy, reg, val);
969 #endif
970
971 mtx_lock(&mii_mutex);
972
973 for (i = 0; i < MVNETA_PHY_TIMEOUT; i++) {
974 if ((MVNETA_READ(sc, MVNETA_SMI) & MVNETA_SMI_BUSY) == 0)
975 break;
976 DELAY(1);
977 }
978 if (i == MVNETA_PHY_TIMEOUT) {
979 if_printf(ifp, "SMI busy timeout\n");
980 mtx_unlock(&mii_mutex);
981 return (0);
982 }
983
984 smi = MVNETA_SMI_PHYAD(phy) | MVNETA_SMI_REGAD(reg) |
985 MVNETA_SMI_OPCODE_WRITE | (val & MVNETA_SMI_DATA_MASK);
986 MVNETA_WRITE(sc, MVNETA_SMI, smi);
987
988 for (i = 0; i < MVNETA_PHY_TIMEOUT; i++) {
989 if ((MVNETA_READ(sc, MVNETA_SMI) & MVNETA_SMI_BUSY) == 0)
990 break;
991 DELAY(1);
992 }
993
994 mtx_unlock(&mii_mutex);
995
996 if (i == MVNETA_PHY_TIMEOUT)
997 if_printf(ifp, "phy write timed out\n");
998
999 return (0);
1000 }
1001
1002 STATIC void
mvneta_portup(struct mvneta_softc * sc)1003 mvneta_portup(struct mvneta_softc *sc)
1004 {
1005 int q;
1006
1007 for (q = 0; q < MVNETA_RX_QNUM_MAX; q++) {
1008 mvneta_rx_lockq(sc, q);
1009 mvneta_rx_queue_enable(sc->ifp, q);
1010 mvneta_rx_unlockq(sc, q);
1011 }
1012
1013 for (q = 0; q < MVNETA_TX_QNUM_MAX; q++) {
1014 mvneta_tx_lockq(sc, q);
1015 mvneta_tx_queue_enable(sc->ifp, q);
1016 mvneta_tx_unlockq(sc, q);
1017 }
1018
1019 }
1020
1021 STATIC void
mvneta_portdown(struct mvneta_softc * sc)1022 mvneta_portdown(struct mvneta_softc *sc)
1023 {
1024 struct mvneta_rx_ring *rx;
1025 struct mvneta_tx_ring *tx;
1026 int q, cnt;
1027 uint32_t reg;
1028
1029 for (q = 0; q < MVNETA_RX_QNUM_MAX; q++) {
1030 rx = MVNETA_RX_RING(sc, q);
1031 mvneta_rx_lockq(sc, q);
1032 rx->queue_status = MVNETA_QUEUE_DISABLED;
1033 mvneta_rx_unlockq(sc, q);
1034 }
1035
1036 for (q = 0; q < MVNETA_TX_QNUM_MAX; q++) {
1037 tx = MVNETA_TX_RING(sc, q);
1038 mvneta_tx_lockq(sc, q);
1039 tx->queue_status = MVNETA_QUEUE_DISABLED;
1040 mvneta_tx_unlockq(sc, q);
1041 }
1042
1043 /* Wait for all Rx activity to terminate. */
1044 reg = MVNETA_READ(sc, MVNETA_RQC) & MVNETA_RQC_EN_MASK;
1045 reg = MVNETA_RQC_DIS(reg);
1046 MVNETA_WRITE(sc, MVNETA_RQC, reg);
1047 cnt = 0;
1048 do {
1049 if (cnt >= RX_DISABLE_TIMEOUT) {
1050 if_printf(sc->ifp,
1051 "timeout for RX stopped. rqc 0x%x\n", reg);
1052 break;
1053 }
1054 cnt++;
1055 reg = MVNETA_READ(sc, MVNETA_RQC);
1056 } while ((reg & MVNETA_RQC_EN_MASK) != 0);
1057
1058 /* Wait for all Tx activity to terminate. */
1059 reg = MVNETA_READ(sc, MVNETA_PIE);
1060 reg &= ~MVNETA_PIE_TXPKTINTRPTENB_MASK;
1061 MVNETA_WRITE(sc, MVNETA_PIE, reg);
1062
1063 reg = MVNETA_READ(sc, MVNETA_PRXTXTIM);
1064 reg &= ~MVNETA_PRXTXTI_TBTCQ_MASK;
1065 MVNETA_WRITE(sc, MVNETA_PRXTXTIM, reg);
1066
1067 reg = MVNETA_READ(sc, MVNETA_TQC) & MVNETA_TQC_EN_MASK;
1068 reg = MVNETA_TQC_DIS(reg);
1069 MVNETA_WRITE(sc, MVNETA_TQC, reg);
1070 cnt = 0;
1071 do {
1072 if (cnt >= TX_DISABLE_TIMEOUT) {
1073 if_printf(sc->ifp,
1074 "timeout for TX stopped. tqc 0x%x\n", reg);
1075 break;
1076 }
1077 cnt++;
1078 reg = MVNETA_READ(sc, MVNETA_TQC);
1079 } while ((reg & MVNETA_TQC_EN_MASK) != 0);
1080
1081 /* Wait for all Tx FIFO is empty */
1082 cnt = 0;
1083 do {
1084 if (cnt >= TX_FIFO_EMPTY_TIMEOUT) {
1085 if_printf(sc->ifp,
1086 "timeout for TX FIFO drained. ps0 0x%x\n", reg);
1087 break;
1088 }
1089 cnt++;
1090 reg = MVNETA_READ(sc, MVNETA_PS0);
1091 } while (((reg & MVNETA_PS0_TXFIFOEMP) == 0) &&
1092 ((reg & MVNETA_PS0_TXINPROG) != 0));
1093 }
1094
1095 /*
1096 * Device Register Initialization
1097 * reset device registers to device driver default value.
1098 * the device is not enabled here.
1099 */
1100 STATIC int
mvneta_initreg(struct ifnet * ifp)1101 mvneta_initreg(struct ifnet *ifp)
1102 {
1103 struct mvneta_softc *sc;
1104 int q;
1105 uint32_t reg;
1106
1107 sc = ifp->if_softc;
1108 #ifdef MVNETA_KTR
1109 CTR1(KTR_SPARE2, "%s initializing device register", ifp->if_xname);
1110 #endif
1111
1112 /* Disable Legacy WRR, Disable EJP, Release from reset. */
1113 MVNETA_WRITE(sc, MVNETA_TQC_1, 0);
1114 /* Enable mbus retry. */
1115 MVNETA_WRITE(sc, MVNETA_MBUS_CONF, MVNETA_MBUS_RETRY_EN);
1116
1117 /* Init TX/RX Queue Registers */
1118 for (q = 0; q < MVNETA_RX_QNUM_MAX; q++) {
1119 mvneta_rx_lockq(sc, q);
1120 if (mvneta_rx_queue_init(ifp, q) != 0) {
1121 device_printf(sc->dev,
1122 "initialization failed: cannot initialize queue\n");
1123 mvneta_rx_unlockq(sc, q);
1124 return (ENOBUFS);
1125 }
1126 mvneta_rx_unlockq(sc, q);
1127 }
1128 for (q = 0; q < MVNETA_TX_QNUM_MAX; q++) {
1129 mvneta_tx_lockq(sc, q);
1130 if (mvneta_tx_queue_init(ifp, q) != 0) {
1131 device_printf(sc->dev,
1132 "initialization failed: cannot initialize queue\n");
1133 mvneta_tx_unlockq(sc, q);
1134 return (ENOBUFS);
1135 }
1136 mvneta_tx_unlockq(sc, q);
1137 }
1138
1139 /*
1140 * Ethernet Unit Control - disable automatic PHY management by HW.
1141 * In case the port uses SMI-controlled PHY, poll its status with
1142 * mii_tick() and update MAC settings accordingly.
1143 */
1144 reg = MVNETA_READ(sc, MVNETA_EUC);
1145 reg &= ~MVNETA_EUC_POLLING;
1146 MVNETA_WRITE(sc, MVNETA_EUC, reg);
1147
1148 /* EEE: Low Power Idle */
1149 reg = MVNETA_LPIC0_LILIMIT(MVNETA_LPI_LI);
1150 reg |= MVNETA_LPIC0_TSLIMIT(MVNETA_LPI_TS);
1151 MVNETA_WRITE(sc, MVNETA_LPIC0, reg);
1152
1153 reg = MVNETA_LPIC1_TWLIMIT(MVNETA_LPI_TW);
1154 MVNETA_WRITE(sc, MVNETA_LPIC1, reg);
1155
1156 reg = MVNETA_LPIC2_MUSTSET;
1157 MVNETA_WRITE(sc, MVNETA_LPIC2, reg);
1158
1159 /* Port MAC Control set 0 */
1160 reg = MVNETA_PMACC0_MUSTSET; /* must write 0x1 */
1161 reg &= ~MVNETA_PMACC0_PORTEN; /* port is still disabled */
1162 reg |= MVNETA_PMACC0_FRAMESIZELIMIT(ifp->if_mtu + MVNETA_ETHER_SIZE);
1163 MVNETA_WRITE(sc, MVNETA_PMACC0, reg);
1164
1165 /* Port MAC Control set 2 */
1166 reg = MVNETA_READ(sc, MVNETA_PMACC2);
1167 switch (sc->phy_mode) {
1168 case MVNETA_PHY_QSGMII:
1169 reg |= (MVNETA_PMACC2_PCSEN | MVNETA_PMACC2_RGMIIEN);
1170 MVNETA_WRITE(sc, MVNETA_PSERDESCFG, MVNETA_PSERDESCFG_QSGMII);
1171 break;
1172 case MVNETA_PHY_SGMII:
1173 reg |= (MVNETA_PMACC2_PCSEN | MVNETA_PMACC2_RGMIIEN);
1174 MVNETA_WRITE(sc, MVNETA_PSERDESCFG, MVNETA_PSERDESCFG_SGMII);
1175 break;
1176 case MVNETA_PHY_RGMII:
1177 case MVNETA_PHY_RGMII_ID:
1178 reg |= MVNETA_PMACC2_RGMIIEN;
1179 break;
1180 }
1181 reg |= MVNETA_PMACC2_MUSTSET;
1182 reg &= ~MVNETA_PMACC2_PORTMACRESET;
1183 MVNETA_WRITE(sc, MVNETA_PMACC2, reg);
1184
1185 /* Port Configuration Extended: enable Tx CRC generation */
1186 reg = MVNETA_READ(sc, MVNETA_PXCX);
1187 reg &= ~MVNETA_PXCX_TXCRCDIS;
1188 MVNETA_WRITE(sc, MVNETA_PXCX, reg);
1189
1190 /* clear MIB counter registers(clear by read) */
1191 mvneta_sc_lock(sc);
1192 mvneta_clear_mib(sc);
1193 mvneta_sc_unlock(sc);
1194
1195 /* Set SDC register except IPGINT bits */
1196 reg = MVNETA_SDC_RXBSZ_16_64BITWORDS;
1197 reg |= MVNETA_SDC_TXBSZ_16_64BITWORDS;
1198 reg |= MVNETA_SDC_BLMR;
1199 reg |= MVNETA_SDC_BLMT;
1200 MVNETA_WRITE(sc, MVNETA_SDC, reg);
1201
1202 return (0);
1203 }
1204
1205 STATIC void
mvneta_dmamap_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)1206 mvneta_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
1207 {
1208
1209 if (error != 0)
1210 return;
1211 *(bus_addr_t *)arg = segs->ds_addr;
1212 }
1213
1214 STATIC int
mvneta_ring_alloc_rx_queue(struct mvneta_softc * sc,int q)1215 mvneta_ring_alloc_rx_queue(struct mvneta_softc *sc, int q)
1216 {
1217 struct mvneta_rx_ring *rx;
1218 struct mvneta_buf *rxbuf;
1219 bus_dmamap_t dmap;
1220 int i, error;
1221
1222 if (q >= MVNETA_RX_QNUM_MAX)
1223 return (EINVAL);
1224
1225 rx = MVNETA_RX_RING(sc, q);
1226 mtx_init(&rx->ring_mtx, "mvneta_rx", NULL, MTX_DEF);
1227 /* Allocate DMA memory for Rx descriptors */
1228 error = bus_dmamem_alloc(sc->rx_dtag,
1229 (void**)&(rx->desc),
1230 BUS_DMA_NOWAIT | BUS_DMA_ZERO,
1231 &rx->desc_map);
1232 if (error != 0 || rx->desc == NULL)
1233 goto fail;
1234 error = bus_dmamap_load(sc->rx_dtag, rx->desc_map,
1235 rx->desc,
1236 sizeof(struct mvneta_rx_desc) * MVNETA_RX_RING_CNT,
1237 mvneta_dmamap_cb, &rx->desc_pa, BUS_DMA_NOWAIT);
1238 if (error != 0)
1239 goto fail;
1240
1241 for (i = 0; i < MVNETA_RX_RING_CNT; i++) {
1242 error = bus_dmamap_create(sc->rxbuf_dtag, 0, &dmap);
1243 if (error != 0) {
1244 device_printf(sc->dev,
1245 "Failed to create DMA map for Rx buffer num: %d\n", i);
1246 goto fail;
1247 }
1248 rxbuf = &rx->rxbuf[i];
1249 rxbuf->dmap = dmap;
1250 rxbuf->m = NULL;
1251 }
1252
1253 return (0);
1254 fail:
1255 mvneta_ring_dealloc_rx_queue(sc, q);
1256 device_printf(sc->dev, "DMA Ring buffer allocation failure.\n");
1257 return (error);
1258 }
1259
1260 STATIC int
mvneta_ring_alloc_tx_queue(struct mvneta_softc * sc,int q)1261 mvneta_ring_alloc_tx_queue(struct mvneta_softc *sc, int q)
1262 {
1263 struct mvneta_tx_ring *tx;
1264 int error;
1265
1266 if (q >= MVNETA_TX_QNUM_MAX)
1267 return (EINVAL);
1268 tx = MVNETA_TX_RING(sc, q);
1269 mtx_init(&tx->ring_mtx, "mvneta_tx", NULL, MTX_DEF);
1270 error = bus_dmamem_alloc(sc->tx_dtag,
1271 (void**)&(tx->desc),
1272 BUS_DMA_NOWAIT | BUS_DMA_ZERO,
1273 &tx->desc_map);
1274 if (error != 0 || tx->desc == NULL)
1275 goto fail;
1276 error = bus_dmamap_load(sc->tx_dtag, tx->desc_map,
1277 tx->desc,
1278 sizeof(struct mvneta_tx_desc) * MVNETA_TX_RING_CNT,
1279 mvneta_dmamap_cb, &tx->desc_pa, BUS_DMA_NOWAIT);
1280 if (error != 0)
1281 goto fail;
1282
1283 #ifdef MVNETA_MULTIQUEUE
1284 tx->br = buf_ring_alloc(MVNETA_BUFRING_SIZE, M_DEVBUF, M_NOWAIT,
1285 &tx->ring_mtx);
1286 if (tx->br == NULL) {
1287 device_printf(sc->dev,
1288 "Could not setup buffer ring for TxQ(%d)\n", q);
1289 error = ENOMEM;
1290 goto fail;
1291 }
1292 #endif
1293
1294 return (0);
1295 fail:
1296 mvneta_ring_dealloc_tx_queue(sc, q);
1297 device_printf(sc->dev, "DMA Ring buffer allocation failure.\n");
1298 return (error);
1299 }
1300
1301 STATIC void
mvneta_ring_dealloc_tx_queue(struct mvneta_softc * sc,int q)1302 mvneta_ring_dealloc_tx_queue(struct mvneta_softc *sc, int q)
1303 {
1304 struct mvneta_tx_ring *tx;
1305 struct mvneta_buf *txbuf;
1306 void *kva;
1307 int error;
1308 int i;
1309
1310 if (q >= MVNETA_TX_QNUM_MAX)
1311 return;
1312 tx = MVNETA_TX_RING(sc, q);
1313
1314 if (tx->taskq != NULL) {
1315 /* Remove task */
1316 while (taskqueue_cancel(tx->taskq, &tx->task, NULL) != 0)
1317 taskqueue_drain(tx->taskq, &tx->task);
1318 }
1319 #ifdef MVNETA_MULTIQUEUE
1320 if (tx->br != NULL)
1321 drbr_free(tx->br, M_DEVBUF);
1322 #endif
1323
1324 if (sc->txmbuf_dtag != NULL) {
1325 if (mtx_name(&tx->ring_mtx) != NULL) {
1326 /*
1327 * It is assumed that maps are being loaded after mutex
1328 * is initialized. Therefore we can skip unloading maps
1329 * when mutex is empty.
1330 */
1331 mvneta_tx_lockq(sc, q);
1332 mvneta_ring_flush_tx_queue(sc, q);
1333 mvneta_tx_unlockq(sc, q);
1334 }
1335 for (i = 0; i < MVNETA_TX_RING_CNT; i++) {
1336 txbuf = &tx->txbuf[i];
1337 if (txbuf->dmap != NULL) {
1338 error = bus_dmamap_destroy(sc->txmbuf_dtag,
1339 txbuf->dmap);
1340 if (error != 0) {
1341 panic("%s: map busy for Tx descriptor (Q%d, %d)",
1342 __func__, q, i);
1343 }
1344 }
1345 }
1346 }
1347
1348 if (tx->desc_pa != 0)
1349 bus_dmamap_unload(sc->tx_dtag, tx->desc_map);
1350
1351 kva = (void *)tx->desc;
1352 if (kva != NULL)
1353 bus_dmamem_free(sc->tx_dtag, tx->desc, tx->desc_map);
1354
1355 if (mtx_name(&tx->ring_mtx) != NULL)
1356 mtx_destroy(&tx->ring_mtx);
1357
1358 memset(tx, 0, sizeof(*tx));
1359 }
1360
1361 STATIC void
mvneta_ring_dealloc_rx_queue(struct mvneta_softc * sc,int q)1362 mvneta_ring_dealloc_rx_queue(struct mvneta_softc *sc, int q)
1363 {
1364 struct mvneta_rx_ring *rx;
1365 struct lro_ctrl *lro;
1366 void *kva;
1367
1368 if (q >= MVNETA_RX_QNUM_MAX)
1369 return;
1370
1371 rx = MVNETA_RX_RING(sc, q);
1372
1373 mvneta_ring_flush_rx_queue(sc, q);
1374
1375 if (rx->desc_pa != 0)
1376 bus_dmamap_unload(sc->rx_dtag, rx->desc_map);
1377
1378 kva = (void *)rx->desc;
1379 if (kva != NULL)
1380 bus_dmamem_free(sc->rx_dtag, rx->desc, rx->desc_map);
1381
1382 lro = &rx->lro;
1383 tcp_lro_free(lro);
1384
1385 if (mtx_name(&rx->ring_mtx) != NULL)
1386 mtx_destroy(&rx->ring_mtx);
1387
1388 memset(rx, 0, sizeof(*rx));
1389 }
1390
1391 STATIC int
mvneta_ring_init_rx_queue(struct mvneta_softc * sc,int q)1392 mvneta_ring_init_rx_queue(struct mvneta_softc *sc, int q)
1393 {
1394 struct mvneta_rx_ring *rx;
1395 struct lro_ctrl *lro;
1396 int error;
1397
1398 if (q >= MVNETA_RX_QNUM_MAX)
1399 return (0);
1400
1401 rx = MVNETA_RX_RING(sc, q);
1402 rx->dma = rx->cpu = 0;
1403 rx->queue_th_received = MVNETA_RXTH_COUNT;
1404 rx->queue_th_time = (mvneta_get_clk() / 1000) / 10; /* 0.1 [ms] */
1405
1406 /* Initialize LRO */
1407 rx->lro_enabled = FALSE;
1408 if ((sc->ifp->if_capenable & IFCAP_LRO) != 0) {
1409 lro = &rx->lro;
1410 error = tcp_lro_init(lro);
1411 if (error != 0)
1412 device_printf(sc->dev, "LRO Initialization failed!\n");
1413 else {
1414 rx->lro_enabled = TRUE;
1415 lro->ifp = sc->ifp;
1416 }
1417 }
1418
1419 return (0);
1420 }
1421
1422 STATIC int
mvneta_ring_init_tx_queue(struct mvneta_softc * sc,int q)1423 mvneta_ring_init_tx_queue(struct mvneta_softc *sc, int q)
1424 {
1425 struct mvneta_tx_ring *tx;
1426 struct mvneta_buf *txbuf;
1427 int i, error;
1428
1429 if (q >= MVNETA_TX_QNUM_MAX)
1430 return (0);
1431
1432 tx = MVNETA_TX_RING(sc, q);
1433
1434 /* Tx handle */
1435 for (i = 0; i < MVNETA_TX_RING_CNT; i++) {
1436 txbuf = &tx->txbuf[i];
1437 txbuf->m = NULL;
1438 /* Tx handle needs DMA map for busdma_load_mbuf() */
1439 error = bus_dmamap_create(sc->txmbuf_dtag, 0,
1440 &txbuf->dmap);
1441 if (error != 0) {
1442 device_printf(sc->dev,
1443 "can't create dma map (tx ring %d)\n", i);
1444 return (error);
1445 }
1446 }
1447 tx->dma = tx->cpu = 0;
1448 tx->used = 0;
1449 tx->drv_error = 0;
1450 tx->queue_status = MVNETA_QUEUE_DISABLED;
1451 tx->queue_hung = FALSE;
1452
1453 tx->ifp = sc->ifp;
1454 tx->qidx = q;
1455 TASK_INIT(&tx->task, 0, mvneta_tx_task, tx);
1456 tx->taskq = taskqueue_create_fast("mvneta_tx_taskq", M_WAITOK,
1457 taskqueue_thread_enqueue, &tx->taskq);
1458 taskqueue_start_threads(&tx->taskq, 1, PI_NET, "%s: tx_taskq(%d)",
1459 device_get_nameunit(sc->dev), q);
1460
1461 return (0);
1462 }
1463
1464 STATIC void
mvneta_ring_flush_tx_queue(struct mvneta_softc * sc,int q)1465 mvneta_ring_flush_tx_queue(struct mvneta_softc *sc, int q)
1466 {
1467 struct mvneta_tx_ring *tx;
1468 struct mvneta_buf *txbuf;
1469 int i;
1470
1471 tx = MVNETA_TX_RING(sc, q);
1472 KASSERT_TX_MTX(sc, q);
1473
1474 /* Tx handle */
1475 for (i = 0; i < MVNETA_TX_RING_CNT; i++) {
1476 txbuf = &tx->txbuf[i];
1477 bus_dmamap_unload(sc->txmbuf_dtag, txbuf->dmap);
1478 if (txbuf->m != NULL) {
1479 m_freem(txbuf->m);
1480 txbuf->m = NULL;
1481 }
1482 }
1483 tx->dma = tx->cpu = 0;
1484 tx->used = 0;
1485 }
1486
1487 STATIC void
mvneta_ring_flush_rx_queue(struct mvneta_softc * sc,int q)1488 mvneta_ring_flush_rx_queue(struct mvneta_softc *sc, int q)
1489 {
1490 struct mvneta_rx_ring *rx;
1491 struct mvneta_buf *rxbuf;
1492 int i;
1493
1494 rx = MVNETA_RX_RING(sc, q);
1495 KASSERT_RX_MTX(sc, q);
1496
1497 /* Rx handle */
1498 for (i = 0; i < MVNETA_RX_RING_CNT; i++) {
1499 rxbuf = &rx->rxbuf[i];
1500 mvneta_rx_buf_free(sc, rxbuf);
1501 }
1502 rx->dma = rx->cpu = 0;
1503 }
1504
1505 /*
1506 * Rx/Tx Queue Control
1507 */
1508 STATIC int
mvneta_rx_queue_init(struct ifnet * ifp,int q)1509 mvneta_rx_queue_init(struct ifnet *ifp, int q)
1510 {
1511 struct mvneta_softc *sc;
1512 struct mvneta_rx_ring *rx;
1513 uint32_t reg;
1514
1515 sc = ifp->if_softc;
1516 KASSERT_RX_MTX(sc, q);
1517 rx = MVNETA_RX_RING(sc, q);
1518 DASSERT(rx->desc_pa != 0);
1519
1520 /* descriptor address */
1521 MVNETA_WRITE(sc, MVNETA_PRXDQA(q), rx->desc_pa);
1522
1523 /* Rx buffer size and descriptor ring size */
1524 reg = MVNETA_PRXDQS_BUFFERSIZE(sc->rx_frame_size >> 3);
1525 reg |= MVNETA_PRXDQS_DESCRIPTORSQUEUESIZE(MVNETA_RX_RING_CNT);
1526 MVNETA_WRITE(sc, MVNETA_PRXDQS(q), reg);
1527 #ifdef MVNETA_KTR
1528 CTR3(KTR_SPARE2, "%s PRXDQS(%d): %#x", ifp->if_xname, q,
1529 MVNETA_READ(sc, MVNETA_PRXDQS(q)));
1530 #endif
1531 /* Rx packet offset address */
1532 reg = MVNETA_PRXC_PACKETOFFSET(MVNETA_PACKET_OFFSET >> 3);
1533 MVNETA_WRITE(sc, MVNETA_PRXC(q), reg);
1534 #ifdef MVNETA_KTR
1535 CTR3(KTR_SPARE2, "%s PRXC(%d): %#x", ifp->if_xname, q,
1536 MVNETA_READ(sc, MVNETA_PRXC(q)));
1537 #endif
1538
1539 /* if DMA is not working, register is not updated */
1540 DASSERT(MVNETA_READ(sc, MVNETA_PRXDQA(q)) == rx->desc_pa);
1541 return (0);
1542 }
1543
1544 STATIC int
mvneta_tx_queue_init(struct ifnet * ifp,int q)1545 mvneta_tx_queue_init(struct ifnet *ifp, int q)
1546 {
1547 struct mvneta_softc *sc;
1548 struct mvneta_tx_ring *tx;
1549 uint32_t reg;
1550
1551 sc = ifp->if_softc;
1552 KASSERT_TX_MTX(sc, q);
1553 tx = MVNETA_TX_RING(sc, q);
1554 DASSERT(tx->desc_pa != 0);
1555
1556 /* descriptor address */
1557 MVNETA_WRITE(sc, MVNETA_PTXDQA(q), tx->desc_pa);
1558
1559 /* descriptor ring size */
1560 reg = MVNETA_PTXDQS_DQS(MVNETA_TX_RING_CNT);
1561 MVNETA_WRITE(sc, MVNETA_PTXDQS(q), reg);
1562
1563 /* if DMA is not working, register is not updated */
1564 DASSERT(MVNETA_READ(sc, MVNETA_PTXDQA(q)) == tx->desc_pa);
1565 return (0);
1566 }
1567
1568 STATIC int
mvneta_rx_queue_enable(struct ifnet * ifp,int q)1569 mvneta_rx_queue_enable(struct ifnet *ifp, int q)
1570 {
1571 struct mvneta_softc *sc;
1572 struct mvneta_rx_ring *rx;
1573 uint32_t reg;
1574
1575 sc = ifp->if_softc;
1576 rx = MVNETA_RX_RING(sc, q);
1577 KASSERT_RX_MTX(sc, q);
1578
1579 /* Set Rx interrupt threshold */
1580 reg = MVNETA_PRXDQTH_ODT(rx->queue_th_received);
1581 MVNETA_WRITE(sc, MVNETA_PRXDQTH(q), reg);
1582
1583 reg = MVNETA_PRXITTH_RITT(rx->queue_th_time);
1584 MVNETA_WRITE(sc, MVNETA_PRXITTH(q), reg);
1585
1586 /* Unmask RXTX_TH Intr. */
1587 reg = MVNETA_READ(sc, MVNETA_PRXTXTIM);
1588 reg |= MVNETA_PRXTXTI_RBICTAPQ(q); /* Rx Buffer Interrupt Coalese */
1589 MVNETA_WRITE(sc, MVNETA_PRXTXTIM, reg);
1590
1591 /* Enable Rx queue */
1592 reg = MVNETA_READ(sc, MVNETA_RQC) & MVNETA_RQC_EN_MASK;
1593 reg |= MVNETA_RQC_ENQ(q);
1594 MVNETA_WRITE(sc, MVNETA_RQC, reg);
1595
1596 rx->queue_status = MVNETA_QUEUE_WORKING;
1597 return (0);
1598 }
1599
1600 STATIC int
mvneta_tx_queue_enable(struct ifnet * ifp,int q)1601 mvneta_tx_queue_enable(struct ifnet *ifp, int q)
1602 {
1603 struct mvneta_softc *sc;
1604 struct mvneta_tx_ring *tx;
1605
1606 sc = ifp->if_softc;
1607 tx = MVNETA_TX_RING(sc, q);
1608 KASSERT_TX_MTX(sc, q);
1609
1610 /* Enable Tx queue */
1611 MVNETA_WRITE(sc, MVNETA_TQC, MVNETA_TQC_ENQ(q));
1612
1613 tx->queue_status = MVNETA_QUEUE_IDLE;
1614 tx->queue_hung = FALSE;
1615 return (0);
1616 }
1617
1618 STATIC __inline void
mvneta_rx_lockq(struct mvneta_softc * sc,int q)1619 mvneta_rx_lockq(struct mvneta_softc *sc, int q)
1620 {
1621
1622 DASSERT(q >= 0);
1623 DASSERT(q < MVNETA_RX_QNUM_MAX);
1624 mtx_lock(&sc->rx_ring[q].ring_mtx);
1625 }
1626
1627 STATIC __inline void
mvneta_rx_unlockq(struct mvneta_softc * sc,int q)1628 mvneta_rx_unlockq(struct mvneta_softc *sc, int q)
1629 {
1630
1631 DASSERT(q >= 0);
1632 DASSERT(q < MVNETA_RX_QNUM_MAX);
1633 mtx_unlock(&sc->rx_ring[q].ring_mtx);
1634 }
1635
1636 STATIC __inline int __unused
mvneta_tx_trylockq(struct mvneta_softc * sc,int q)1637 mvneta_tx_trylockq(struct mvneta_softc *sc, int q)
1638 {
1639
1640 DASSERT(q >= 0);
1641 DASSERT(q < MVNETA_TX_QNUM_MAX);
1642 return (mtx_trylock(&sc->tx_ring[q].ring_mtx));
1643 }
1644
1645 STATIC __inline void
mvneta_tx_lockq(struct mvneta_softc * sc,int q)1646 mvneta_tx_lockq(struct mvneta_softc *sc, int q)
1647 {
1648
1649 DASSERT(q >= 0);
1650 DASSERT(q < MVNETA_TX_QNUM_MAX);
1651 mtx_lock(&sc->tx_ring[q].ring_mtx);
1652 }
1653
1654 STATIC __inline void
mvneta_tx_unlockq(struct mvneta_softc * sc,int q)1655 mvneta_tx_unlockq(struct mvneta_softc *sc, int q)
1656 {
1657
1658 DASSERT(q >= 0);
1659 DASSERT(q < MVNETA_TX_QNUM_MAX);
1660 mtx_unlock(&sc->tx_ring[q].ring_mtx);
1661 }
1662
1663 /*
1664 * Interrupt Handlers
1665 */
1666 STATIC void
mvneta_disable_intr(struct mvneta_softc * sc)1667 mvneta_disable_intr(struct mvneta_softc *sc)
1668 {
1669
1670 MVNETA_WRITE(sc, MVNETA_EUIM, 0);
1671 MVNETA_WRITE(sc, MVNETA_EUIC, 0);
1672 MVNETA_WRITE(sc, MVNETA_PRXTXTIM, 0);
1673 MVNETA_WRITE(sc, MVNETA_PRXTXTIC, 0);
1674 MVNETA_WRITE(sc, MVNETA_PRXTXIM, 0);
1675 MVNETA_WRITE(sc, MVNETA_PRXTXIC, 0);
1676 MVNETA_WRITE(sc, MVNETA_PMIM, 0);
1677 MVNETA_WRITE(sc, MVNETA_PMIC, 0);
1678 MVNETA_WRITE(sc, MVNETA_PIE, 0);
1679 }
1680
1681 STATIC void
mvneta_enable_intr(struct mvneta_softc * sc)1682 mvneta_enable_intr(struct mvneta_softc *sc)
1683 {
1684 uint32_t reg;
1685
1686 /* Enable Summary Bit to check all interrupt cause. */
1687 reg = MVNETA_READ(sc, MVNETA_PRXTXTIM);
1688 reg |= MVNETA_PRXTXTI_PMISCICSUMMARY;
1689 MVNETA_WRITE(sc, MVNETA_PRXTXTIM, reg);
1690
1691 if (sc->use_inband_status) {
1692 /* Enable Port MISC Intr. (via RXTX_TH_Summary bit) */
1693 MVNETA_WRITE(sc, MVNETA_PMIM, MVNETA_PMI_PHYSTATUSCHNG |
1694 MVNETA_PMI_LINKCHANGE | MVNETA_PMI_PSCSYNCCHANGE);
1695 }
1696
1697 /* Enable All Queue Interrupt */
1698 reg = MVNETA_READ(sc, MVNETA_PIE);
1699 reg |= MVNETA_PIE_RXPKTINTRPTENB_MASK;
1700 reg |= MVNETA_PIE_TXPKTINTRPTENB_MASK;
1701 MVNETA_WRITE(sc, MVNETA_PIE, reg);
1702 }
1703
1704 STATIC void
mvneta_rxtxth_intr(void * arg)1705 mvneta_rxtxth_intr(void *arg)
1706 {
1707 struct mvneta_softc *sc;
1708 struct ifnet *ifp;
1709 uint32_t ic, queues;
1710
1711 sc = arg;
1712 ifp = sc->ifp;
1713 #ifdef MVNETA_KTR
1714 CTR1(KTR_SPARE2, "%s got RXTX_TH_Intr", ifp->if_xname);
1715 #endif
1716 ic = MVNETA_READ(sc, MVNETA_PRXTXTIC);
1717 if (ic == 0)
1718 return;
1719 MVNETA_WRITE(sc, MVNETA_PRXTXTIC, ~ic);
1720
1721 /* Ack maintance interrupt first */
1722 if (__predict_false((ic & MVNETA_PRXTXTI_PMISCICSUMMARY) &&
1723 sc->use_inband_status)) {
1724 mvneta_sc_lock(sc);
1725 mvneta_misc_intr(sc);
1726 mvneta_sc_unlock(sc);
1727 }
1728 if (__predict_false(!(ifp->if_drv_flags & IFF_DRV_RUNNING)))
1729 return;
1730 /* RxTxTH interrupt */
1731 queues = MVNETA_PRXTXTI_GET_RBICTAPQ(ic);
1732 if (__predict_true(queues)) {
1733 #ifdef MVNETA_KTR
1734 CTR1(KTR_SPARE2, "%s got PRXTXTIC: +RXEOF", ifp->if_xname);
1735 #endif
1736 /* At the moment the driver support only one RX queue. */
1737 DASSERT(MVNETA_IS_QUEUE_SET(queues, 0));
1738 mvneta_rx(sc, 0, 0);
1739 }
1740 }
1741
1742 STATIC int
mvneta_misc_intr(struct mvneta_softc * sc)1743 mvneta_misc_intr(struct mvneta_softc *sc)
1744 {
1745 uint32_t ic;
1746 int claimed = 0;
1747
1748 #ifdef MVNETA_KTR
1749 CTR1(KTR_SPARE2, "%s got MISC_INTR", sc->ifp->if_xname);
1750 #endif
1751 KASSERT_SC_MTX(sc);
1752
1753 for (;;) {
1754 ic = MVNETA_READ(sc, MVNETA_PMIC);
1755 ic &= MVNETA_READ(sc, MVNETA_PMIM);
1756 if (ic == 0)
1757 break;
1758 MVNETA_WRITE(sc, MVNETA_PMIC, ~ic);
1759 claimed = 1;
1760
1761 if (ic & (MVNETA_PMI_PHYSTATUSCHNG |
1762 MVNETA_PMI_LINKCHANGE | MVNETA_PMI_PSCSYNCCHANGE))
1763 mvneta_link_isr(sc);
1764 }
1765 return (claimed);
1766 }
1767
1768 STATIC void
mvneta_tick(void * arg)1769 mvneta_tick(void *arg)
1770 {
1771 struct mvneta_softc *sc;
1772 struct mvneta_tx_ring *tx;
1773 struct mvneta_rx_ring *rx;
1774 int q;
1775 uint32_t fc_prev, fc_curr;
1776
1777 sc = arg;
1778
1779 /*
1780 * This is done before mib update to get the right stats
1781 * for this tick.
1782 */
1783 mvneta_tx_drain(sc);
1784
1785 /* Extract previous flow-control frame received counter. */
1786 fc_prev = sc->sysctl_mib[MVNETA_MIB_FC_GOOD_IDX].counter;
1787 /* Read mib registers (clear by read). */
1788 mvneta_update_mib(sc);
1789 /* Extract current flow-control frame received counter. */
1790 fc_curr = sc->sysctl_mib[MVNETA_MIB_FC_GOOD_IDX].counter;
1791
1792
1793 if (sc->phy_attached && sc->ifp->if_flags & IFF_UP) {
1794 mvneta_sc_lock(sc);
1795 mii_tick(sc->mii);
1796
1797 /* Adjust MAC settings */
1798 mvneta_adjust_link(sc);
1799 mvneta_sc_unlock(sc);
1800 }
1801
1802 /*
1803 * We were unable to refill the rx queue and left the rx func, leaving
1804 * the ring without mbuf and no way to call the refill func.
1805 */
1806 for (q = 0; q < MVNETA_RX_QNUM_MAX; q++) {
1807 rx = MVNETA_RX_RING(sc, q);
1808 if (rx->needs_refill == TRUE) {
1809 mvneta_rx_lockq(sc, q);
1810 mvneta_rx_queue_refill(sc, q);
1811 mvneta_rx_unlockq(sc, q);
1812 }
1813 }
1814
1815 /*
1816 * Watchdog:
1817 * - check if queue is mark as hung.
1818 * - ignore hung status if we received some pause frame
1819 * as hardware may have paused packet transmit.
1820 */
1821 for (q = 0; q < MVNETA_TX_QNUM_MAX; q++) {
1822 /*
1823 * We should take queue lock, but as we only read
1824 * queue status we can do it without lock, we may
1825 * only missdetect queue status for one tick.
1826 */
1827 tx = MVNETA_TX_RING(sc, q);
1828
1829 if (tx->queue_hung && (fc_curr - fc_prev) == 0)
1830 goto timeout;
1831 }
1832
1833 callout_schedule(&sc->tick_ch, hz);
1834 return;
1835
1836 timeout:
1837 if_printf(sc->ifp, "watchdog timeout\n");
1838
1839 mvneta_sc_lock(sc);
1840 sc->counter_watchdog++;
1841 sc->counter_watchdog_mib++;
1842 /* Trigger reinitialize sequence. */
1843 mvneta_stop_locked(sc);
1844 mvneta_init_locked(sc);
1845 mvneta_sc_unlock(sc);
1846 }
1847
1848 STATIC void
mvneta_qflush(struct ifnet * ifp)1849 mvneta_qflush(struct ifnet *ifp)
1850 {
1851 #ifdef MVNETA_MULTIQUEUE
1852 struct mvneta_softc *sc;
1853 struct mvneta_tx_ring *tx;
1854 struct mbuf *m;
1855 size_t q;
1856
1857 sc = ifp->if_softc;
1858
1859 for (q = 0; q < MVNETA_TX_QNUM_MAX; q++) {
1860 tx = MVNETA_TX_RING(sc, q);
1861 mvneta_tx_lockq(sc, q);
1862 while ((m = buf_ring_dequeue_sc(tx->br)) != NULL)
1863 m_freem(m);
1864 mvneta_tx_unlockq(sc, q);
1865 }
1866 #endif
1867 if_qflush(ifp);
1868 }
1869
1870 STATIC void
mvneta_tx_task(void * arg,int pending)1871 mvneta_tx_task(void *arg, int pending)
1872 {
1873 struct mvneta_softc *sc;
1874 struct mvneta_tx_ring *tx;
1875 struct ifnet *ifp;
1876 int error;
1877
1878 tx = arg;
1879 ifp = tx->ifp;
1880 sc = ifp->if_softc;
1881
1882 mvneta_tx_lockq(sc, tx->qidx);
1883 error = mvneta_xmit_locked(sc, tx->qidx);
1884 mvneta_tx_unlockq(sc, tx->qidx);
1885
1886 /* Try again */
1887 if (__predict_false(error != 0 && error != ENETDOWN)) {
1888 pause("mvneta_tx_task_sleep", 1);
1889 taskqueue_enqueue(tx->taskq, &tx->task);
1890 }
1891 }
1892
1893 STATIC int
mvneta_xmitfast_locked(struct mvneta_softc * sc,int q,struct mbuf ** m)1894 mvneta_xmitfast_locked(struct mvneta_softc *sc, int q, struct mbuf **m)
1895 {
1896 struct mvneta_tx_ring *tx;
1897 struct ifnet *ifp;
1898 int error;
1899
1900 KASSERT_TX_MTX(sc, q);
1901 tx = MVNETA_TX_RING(sc, q);
1902 error = 0;
1903
1904 ifp = sc->ifp;
1905
1906 /* Dont enqueue packet if the queue is disabled. */
1907 if (__predict_false(tx->queue_status == MVNETA_QUEUE_DISABLED)) {
1908 m_freem(*m);
1909 *m = NULL;
1910 return (ENETDOWN);
1911 }
1912
1913 /* Reclaim mbuf if above threshold. */
1914 if (__predict_true(tx->used > MVNETA_TX_RECLAIM_COUNT))
1915 mvneta_tx_queue_complete(sc, q);
1916
1917 /* Do not call transmit path if queue is already too full. */
1918 if (__predict_false(tx->used >
1919 MVNETA_TX_RING_CNT - MVNETA_TX_SEGLIMIT))
1920 return (ENOBUFS);
1921
1922 error = mvneta_tx_queue(sc, m, q);
1923 if (__predict_false(error != 0))
1924 return (error);
1925
1926 /* Send a copy of the frame to the BPF listener */
1927 ETHER_BPF_MTAP(ifp, *m);
1928
1929 /* Set watchdog on */
1930 tx->watchdog_time = ticks;
1931 tx->queue_status = MVNETA_QUEUE_WORKING;
1932
1933 return (error);
1934 }
1935
1936 #ifdef MVNETA_MULTIQUEUE
1937 STATIC int
mvneta_transmit(struct ifnet * ifp,struct mbuf * m)1938 mvneta_transmit(struct ifnet *ifp, struct mbuf *m)
1939 {
1940 struct mvneta_softc *sc;
1941 struct mvneta_tx_ring *tx;
1942 int error;
1943 int q;
1944
1945 sc = ifp->if_softc;
1946
1947 /* Use default queue if there is no flow id as thread can migrate. */
1948 if (__predict_true(M_HASHTYPE_GET(m) != M_HASHTYPE_NONE))
1949 q = m->m_pkthdr.flowid % MVNETA_TX_QNUM_MAX;
1950 else
1951 q = 0;
1952
1953 tx = MVNETA_TX_RING(sc, q);
1954
1955 /* If buf_ring is full start transmit immediatly. */
1956 if (buf_ring_full(tx->br)) {
1957 mvneta_tx_lockq(sc, q);
1958 mvneta_xmit_locked(sc, q);
1959 mvneta_tx_unlockq(sc, q);
1960 }
1961
1962 /*
1963 * If the buf_ring is empty we will not reorder packets.
1964 * If the lock is available transmit without using buf_ring.
1965 */
1966 if (buf_ring_empty(tx->br) && mvneta_tx_trylockq(sc, q) != 0) {
1967 error = mvneta_xmitfast_locked(sc, q, &m);
1968 mvneta_tx_unlockq(sc, q);
1969 if (__predict_true(error == 0))
1970 return (0);
1971
1972 /* Transmit can fail in fastpath. */
1973 if (__predict_false(m == NULL))
1974 return (error);
1975 }
1976
1977 /* Enqueue then schedule taskqueue. */
1978 error = drbr_enqueue(ifp, tx->br, m);
1979 if (__predict_false(error != 0))
1980 return (error);
1981
1982 taskqueue_enqueue(tx->taskq, &tx->task);
1983 return (0);
1984 }
1985
1986 STATIC int
mvneta_xmit_locked(struct mvneta_softc * sc,int q)1987 mvneta_xmit_locked(struct mvneta_softc *sc, int q)
1988 {
1989 struct ifnet *ifp;
1990 struct mvneta_tx_ring *tx;
1991 struct mbuf *m;
1992 int error;
1993
1994 KASSERT_TX_MTX(sc, q);
1995 ifp = sc->ifp;
1996 tx = MVNETA_TX_RING(sc, q);
1997 error = 0;
1998
1999 while ((m = drbr_peek(ifp, tx->br)) != NULL) {
2000 error = mvneta_xmitfast_locked(sc, q, &m);
2001 if (__predict_false(error != 0)) {
2002 if (m != NULL)
2003 drbr_putback(ifp, tx->br, m);
2004 else
2005 drbr_advance(ifp, tx->br);
2006 break;
2007 }
2008 drbr_advance(ifp, tx->br);
2009 }
2010
2011 return (error);
2012 }
2013 #else /* !MVNETA_MULTIQUEUE */
2014 STATIC void
mvneta_start(struct ifnet * ifp)2015 mvneta_start(struct ifnet *ifp)
2016 {
2017 struct mvneta_softc *sc;
2018 struct mvneta_tx_ring *tx;
2019 int error;
2020
2021 sc = ifp->if_softc;
2022 tx = MVNETA_TX_RING(sc, 0);
2023
2024 mvneta_tx_lockq(sc, 0);
2025 error = mvneta_xmit_locked(sc, 0);
2026 mvneta_tx_unlockq(sc, 0);
2027 /* Handle retransmit in the background taskq. */
2028 if (__predict_false(error != 0 && error != ENETDOWN))
2029 taskqueue_enqueue(tx->taskq, &tx->task);
2030 }
2031
2032 STATIC int
mvneta_xmit_locked(struct mvneta_softc * sc,int q)2033 mvneta_xmit_locked(struct mvneta_softc *sc, int q)
2034 {
2035 struct ifnet *ifp;
2036 struct mvneta_tx_ring *tx;
2037 struct mbuf *m;
2038 int error;
2039
2040 KASSERT_TX_MTX(sc, q);
2041 ifp = sc->ifp;
2042 tx = MVNETA_TX_RING(sc, 0);
2043 error = 0;
2044
2045 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
2046 IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
2047 if (m == NULL)
2048 break;
2049
2050 error = mvneta_xmitfast_locked(sc, q, &m);
2051 if (__predict_false(error != 0)) {
2052 if (m != NULL)
2053 IFQ_DRV_PREPEND(&ifp->if_snd, m);
2054 break;
2055 }
2056 }
2057
2058 return (error);
2059 }
2060 #endif
2061
2062 STATIC int
mvneta_ioctl(struct ifnet * ifp,u_long cmd,caddr_t data)2063 mvneta_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
2064 {
2065 struct mvneta_softc *sc;
2066 struct mvneta_rx_ring *rx;
2067 struct ifreq *ifr;
2068 int error, mask;
2069 uint32_t flags;
2070 int q;
2071
2072 error = 0;
2073 sc = ifp->if_softc;
2074 ifr = (struct ifreq *)data;
2075 switch (cmd) {
2076 case SIOCSIFFLAGS:
2077 mvneta_sc_lock(sc);
2078 if (ifp->if_flags & IFF_UP) {
2079 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2080 flags = ifp->if_flags ^ sc->mvneta_if_flags;
2081
2082 if (flags != 0)
2083 sc->mvneta_if_flags = ifp->if_flags;
2084
2085 if ((flags & IFF_PROMISC) != 0)
2086 mvneta_filter_setup(sc);
2087 } else {
2088 mvneta_init_locked(sc);
2089 sc->mvneta_if_flags = ifp->if_flags;
2090 if (sc->phy_attached)
2091 mii_mediachg(sc->mii);
2092 mvneta_sc_unlock(sc);
2093 break;
2094 }
2095 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2096 mvneta_stop_locked(sc);
2097
2098 sc->mvneta_if_flags = ifp->if_flags;
2099 mvneta_sc_unlock(sc);
2100 break;
2101 case SIOCSIFCAP:
2102 if (ifp->if_mtu > sc->tx_csum_limit &&
2103 ifr->ifr_reqcap & IFCAP_TXCSUM)
2104 ifr->ifr_reqcap &= ~IFCAP_TXCSUM;
2105 mask = ifp->if_capenable ^ ifr->ifr_reqcap;
2106 if (mask & IFCAP_HWCSUM) {
2107 ifp->if_capenable &= ~IFCAP_HWCSUM;
2108 ifp->if_capenable |= IFCAP_HWCSUM & ifr->ifr_reqcap;
2109 if (ifp->if_capenable & IFCAP_TXCSUM)
2110 ifp->if_hwassist = CSUM_IP | CSUM_TCP |
2111 CSUM_UDP;
2112 else
2113 ifp->if_hwassist = 0;
2114 }
2115 if (mask & IFCAP_LRO) {
2116 mvneta_sc_lock(sc);
2117 ifp->if_capenable ^= IFCAP_LRO;
2118 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
2119 for (q = 0; q < MVNETA_RX_QNUM_MAX; q++) {
2120 rx = MVNETA_RX_RING(sc, q);
2121 rx->lro_enabled = !rx->lro_enabled;
2122 }
2123 }
2124 mvneta_sc_unlock(sc);
2125 }
2126 VLAN_CAPABILITIES(ifp);
2127 break;
2128 case SIOCSIFMEDIA:
2129 if ((IFM_SUBTYPE(ifr->ifr_media) == IFM_1000_T ||
2130 IFM_SUBTYPE(ifr->ifr_media) == IFM_2500_T) &&
2131 (ifr->ifr_media & IFM_FDX) == 0) {
2132 device_printf(sc->dev,
2133 "%s half-duplex unsupported\n",
2134 IFM_SUBTYPE(ifr->ifr_media) == IFM_1000_T ?
2135 "1000Base-T" :
2136 "2500Base-T");
2137 error = EINVAL;
2138 break;
2139 }
2140 case SIOCGIFMEDIA: /* FALLTHROUGH */
2141 case SIOCGIFXMEDIA:
2142 if (!sc->phy_attached)
2143 error = ifmedia_ioctl(ifp, ifr, &sc->mvneta_ifmedia,
2144 cmd);
2145 else
2146 error = ifmedia_ioctl(ifp, ifr, &sc->mii->mii_media,
2147 cmd);
2148 break;
2149 case SIOCSIFMTU:
2150 if (ifr->ifr_mtu < 68 || ifr->ifr_mtu > MVNETA_MAX_FRAME -
2151 MVNETA_ETHER_SIZE) {
2152 error = EINVAL;
2153 } else {
2154 ifp->if_mtu = ifr->ifr_mtu;
2155 mvneta_sc_lock(sc);
2156 if (ifp->if_mtu + MVNETA_ETHER_SIZE <= MCLBYTES) {
2157 sc->rx_frame_size = MCLBYTES;
2158 } else {
2159 sc->rx_frame_size = MJUM9BYTES;
2160 }
2161 if (ifp->if_mtu > sc->tx_csum_limit) {
2162 ifp->if_capenable &= ~IFCAP_TXCSUM;
2163 ifp->if_hwassist = 0;
2164 } else {
2165 ifp->if_capenable |= IFCAP_TXCSUM;
2166 ifp->if_hwassist = CSUM_IP | CSUM_TCP |
2167 CSUM_UDP;
2168 }
2169 /*
2170 * Reinitialize RX queues.
2171 * We need to update RX descriptor size.
2172 */
2173 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2174 mvneta_stop_locked(sc);
2175
2176 for (q = 0; q < MVNETA_RX_QNUM_MAX; q++) {
2177 mvneta_rx_lockq(sc, q);
2178 if (mvneta_rx_queue_init(ifp, q) != 0) {
2179 device_printf(sc->dev,
2180 "initialization failed:"
2181 " cannot initialize queue\n");
2182 mvneta_rx_unlockq(sc, q);
2183 error = ENOBUFS;
2184 break;
2185 }
2186 mvneta_rx_unlockq(sc, q);
2187 }
2188 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2189 mvneta_init_locked(sc);
2190
2191 mvneta_sc_unlock(sc);
2192 }
2193 break;
2194
2195 default:
2196 error = ether_ioctl(ifp, cmd, data);
2197 break;
2198 }
2199
2200 return (error);
2201 }
2202
2203 STATIC void
mvneta_init_locked(void * arg)2204 mvneta_init_locked(void *arg)
2205 {
2206 struct mvneta_softc *sc;
2207 struct ifnet *ifp;
2208 uint32_t reg;
2209 int q, cpu;
2210
2211 sc = arg;
2212 ifp = sc->ifp;
2213
2214 if (!device_is_attached(sc->dev) ||
2215 (ifp->if_drv_flags & IFF_DRV_RUNNING) != 0)
2216 return;
2217
2218 mvneta_disable_intr(sc);
2219 callout_stop(&sc->tick_ch);
2220
2221 /* Get the latest mac address */
2222 bcopy(IF_LLADDR(ifp), sc->enaddr, ETHER_ADDR_LEN);
2223 mvneta_set_mac_address(sc, sc->enaddr);
2224 mvneta_filter_setup(sc);
2225
2226 /* Start DMA Engine */
2227 MVNETA_WRITE(sc, MVNETA_PRXINIT, 0x00000000);
2228 MVNETA_WRITE(sc, MVNETA_PTXINIT, 0x00000000);
2229 MVNETA_WRITE(sc, MVNETA_PACC, MVNETA_PACC_ACCELERATIONMODE_EDM);
2230
2231 /* Enable port */
2232 reg = MVNETA_READ(sc, MVNETA_PMACC0);
2233 reg |= MVNETA_PMACC0_PORTEN;
2234 reg &= ~MVNETA_PMACC0_FRAMESIZELIMIT_MASK;
2235 reg |= MVNETA_PMACC0_FRAMESIZELIMIT(ifp->if_mtu + MVNETA_ETHER_SIZE);
2236 MVNETA_WRITE(sc, MVNETA_PMACC0, reg);
2237
2238 /* Allow access to each TXQ/RXQ from both CPU's */
2239 for (cpu = 0; cpu < mp_ncpus; ++cpu)
2240 MVNETA_WRITE(sc, MVNETA_PCP2Q(cpu),
2241 MVNETA_PCP2Q_TXQEN_MASK | MVNETA_PCP2Q_RXQEN_MASK);
2242
2243 for (q = 0; q < MVNETA_RX_QNUM_MAX; q++) {
2244 mvneta_rx_lockq(sc, q);
2245 mvneta_rx_queue_refill(sc, q);
2246 mvneta_rx_unlockq(sc, q);
2247 }
2248
2249 if (!sc->phy_attached)
2250 mvneta_linkup(sc);
2251
2252 /* Enable interrupt */
2253 mvneta_enable_intr(sc);
2254
2255 /* Set Counter */
2256 callout_schedule(&sc->tick_ch, hz);
2257
2258 ifp->if_drv_flags |= IFF_DRV_RUNNING;
2259 }
2260
2261 STATIC void
mvneta_init(void * arg)2262 mvneta_init(void *arg)
2263 {
2264 struct mvneta_softc *sc;
2265
2266 sc = arg;
2267 mvneta_sc_lock(sc);
2268 mvneta_init_locked(sc);
2269 if (sc->phy_attached)
2270 mii_mediachg(sc->mii);
2271 mvneta_sc_unlock(sc);
2272 }
2273
2274 /* ARGSUSED */
2275 STATIC void
mvneta_stop_locked(struct mvneta_softc * sc)2276 mvneta_stop_locked(struct mvneta_softc *sc)
2277 {
2278 struct ifnet *ifp;
2279 struct mvneta_rx_ring *rx;
2280 struct mvneta_tx_ring *tx;
2281 uint32_t reg;
2282 int q;
2283
2284 ifp = sc->ifp;
2285 if (ifp == NULL || (ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
2286 return;
2287
2288 mvneta_disable_intr(sc);
2289
2290 callout_stop(&sc->tick_ch);
2291
2292 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2293
2294 /* Link down */
2295 if (sc->linkup == TRUE)
2296 mvneta_linkdown(sc);
2297
2298 /* Reset the MAC Port Enable bit */
2299 reg = MVNETA_READ(sc, MVNETA_PMACC0);
2300 reg &= ~MVNETA_PMACC0_PORTEN;
2301 MVNETA_WRITE(sc, MVNETA_PMACC0, reg);
2302
2303 /* Disable each of queue */
2304 for (q = 0; q < MVNETA_RX_QNUM_MAX; q++) {
2305 rx = MVNETA_RX_RING(sc, q);
2306
2307 mvneta_rx_lockq(sc, q);
2308 mvneta_ring_flush_rx_queue(sc, q);
2309 mvneta_rx_unlockq(sc, q);
2310 }
2311
2312 /*
2313 * Hold Reset state of DMA Engine
2314 * (must write 0x0 to restart it)
2315 */
2316 MVNETA_WRITE(sc, MVNETA_PRXINIT, 0x00000001);
2317 MVNETA_WRITE(sc, MVNETA_PTXINIT, 0x00000001);
2318
2319 for (q = 0; q < MVNETA_TX_QNUM_MAX; q++) {
2320 tx = MVNETA_TX_RING(sc, q);
2321
2322 mvneta_tx_lockq(sc, q);
2323 mvneta_ring_flush_tx_queue(sc, q);
2324 mvneta_tx_unlockq(sc, q);
2325 }
2326 }
2327
2328 STATIC void
mvneta_stop(struct mvneta_softc * sc)2329 mvneta_stop(struct mvneta_softc *sc)
2330 {
2331
2332 mvneta_sc_lock(sc);
2333 mvneta_stop_locked(sc);
2334 mvneta_sc_unlock(sc);
2335 }
2336
2337 STATIC int
mvneta_mediachange(struct ifnet * ifp)2338 mvneta_mediachange(struct ifnet *ifp)
2339 {
2340 struct mvneta_softc *sc;
2341
2342 sc = ifp->if_softc;
2343
2344 if (!sc->phy_attached && !sc->use_inband_status) {
2345 /* We shouldn't be here */
2346 if_printf(ifp, "Cannot change media in fixed-link mode!\n");
2347 return (0);
2348 }
2349
2350 if (sc->use_inband_status) {
2351 mvneta_update_media(sc, sc->mvneta_ifmedia.ifm_media);
2352 return (0);
2353 }
2354
2355 mvneta_sc_lock(sc);
2356
2357 /* Update PHY */
2358 mii_mediachg(sc->mii);
2359
2360 mvneta_sc_unlock(sc);
2361
2362 return (0);
2363 }
2364
2365 STATIC void
mvneta_get_media(struct mvneta_softc * sc,struct ifmediareq * ifmr)2366 mvneta_get_media(struct mvneta_softc *sc, struct ifmediareq *ifmr)
2367 {
2368 uint32_t psr;
2369
2370 psr = MVNETA_READ(sc, MVNETA_PSR);
2371
2372 /* Speed */
2373 if (psr & MVNETA_PSR_GMIISPEED)
2374 ifmr->ifm_active = IFM_ETHER_SUBTYPE_SET(IFM_1000_T);
2375 else if (psr & MVNETA_PSR_MIISPEED)
2376 ifmr->ifm_active = IFM_ETHER_SUBTYPE_SET(IFM_100_TX);
2377 else if (psr & MVNETA_PSR_LINKUP)
2378 ifmr->ifm_active = IFM_ETHER_SUBTYPE_SET(IFM_10_T);
2379
2380 /* Duplex */
2381 if (psr & MVNETA_PSR_FULLDX)
2382 ifmr->ifm_active |= IFM_FDX;
2383
2384 /* Link */
2385 ifmr->ifm_status = IFM_AVALID;
2386 if (psr & MVNETA_PSR_LINKUP)
2387 ifmr->ifm_status |= IFM_ACTIVE;
2388 }
2389
2390 STATIC void
mvneta_mediastatus(struct ifnet * ifp,struct ifmediareq * ifmr)2391 mvneta_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
2392 {
2393 struct mvneta_softc *sc;
2394 struct mii_data *mii;
2395
2396 sc = ifp->if_softc;
2397
2398 if (!sc->phy_attached && !sc->use_inband_status) {
2399 ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE;
2400 return;
2401 }
2402
2403 mvneta_sc_lock(sc);
2404
2405 if (sc->use_inband_status) {
2406 mvneta_get_media(sc, ifmr);
2407 mvneta_sc_unlock(sc);
2408 return;
2409 }
2410
2411 mii = sc->mii;
2412 mii_pollstat(mii);
2413
2414 ifmr->ifm_active = mii->mii_media_active;
2415 ifmr->ifm_status = mii->mii_media_status;
2416
2417 mvneta_sc_unlock(sc);
2418 }
2419
2420 /*
2421 * Link State Notify
2422 */
2423 STATIC void
mvneta_update_autoneg(struct mvneta_softc * sc,int enable)2424 mvneta_update_autoneg(struct mvneta_softc *sc, int enable)
2425 {
2426 int reg;
2427
2428 if (enable) {
2429 reg = MVNETA_READ(sc, MVNETA_PANC);
2430 reg &= ~(MVNETA_PANC_FORCELINKFAIL | MVNETA_PANC_FORCELINKPASS |
2431 MVNETA_PANC_ANFCEN);
2432 reg |= MVNETA_PANC_ANDUPLEXEN | MVNETA_PANC_ANSPEEDEN |
2433 MVNETA_PANC_INBANDANEN;
2434 MVNETA_WRITE(sc, MVNETA_PANC, reg);
2435
2436 reg = MVNETA_READ(sc, MVNETA_PMACC2);
2437 reg |= MVNETA_PMACC2_INBANDANMODE;
2438 MVNETA_WRITE(sc, MVNETA_PMACC2, reg);
2439
2440 reg = MVNETA_READ(sc, MVNETA_PSOMSCD);
2441 reg |= MVNETA_PSOMSCD_ENABLE;
2442 MVNETA_WRITE(sc, MVNETA_PSOMSCD, reg);
2443 } else {
2444 reg = MVNETA_READ(sc, MVNETA_PANC);
2445 reg &= ~(MVNETA_PANC_FORCELINKFAIL | MVNETA_PANC_FORCELINKPASS |
2446 MVNETA_PANC_ANDUPLEXEN | MVNETA_PANC_ANSPEEDEN |
2447 MVNETA_PANC_INBANDANEN);
2448 MVNETA_WRITE(sc, MVNETA_PANC, reg);
2449
2450 reg = MVNETA_READ(sc, MVNETA_PMACC2);
2451 reg &= ~MVNETA_PMACC2_INBANDANMODE;
2452 MVNETA_WRITE(sc, MVNETA_PMACC2, reg);
2453
2454 reg = MVNETA_READ(sc, MVNETA_PSOMSCD);
2455 reg &= ~MVNETA_PSOMSCD_ENABLE;
2456 MVNETA_WRITE(sc, MVNETA_PSOMSCD, reg);
2457 }
2458 }
2459
2460 STATIC int
mvneta_update_media(struct mvneta_softc * sc,int media)2461 mvneta_update_media(struct mvneta_softc *sc, int media)
2462 {
2463 int reg, err;
2464 boolean_t running;
2465
2466 err = 0;
2467
2468 mvneta_sc_lock(sc);
2469
2470 mvneta_linkreset(sc);
2471
2472 running = (sc->ifp->if_drv_flags & IFF_DRV_RUNNING) != 0;
2473 if (running)
2474 mvneta_stop_locked(sc);
2475
2476 sc->autoneg = (IFM_SUBTYPE(media) == IFM_AUTO);
2477
2478 if (sc->use_inband_status)
2479 mvneta_update_autoneg(sc, IFM_SUBTYPE(media) == IFM_AUTO);
2480
2481 mvneta_update_eee(sc);
2482 mvneta_update_fc(sc);
2483
2484 if (IFM_SUBTYPE(media) != IFM_AUTO) {
2485 reg = MVNETA_READ(sc, MVNETA_PANC);
2486 reg &= ~(MVNETA_PANC_SETGMIISPEED |
2487 MVNETA_PANC_SETMIISPEED |
2488 MVNETA_PANC_SETFULLDX);
2489 if (IFM_SUBTYPE(media) == IFM_1000_T ||
2490 IFM_SUBTYPE(media) == IFM_2500_T) {
2491 if ((media & IFM_FDX) == 0) {
2492 device_printf(sc->dev,
2493 "%s half-duplex unsupported\n",
2494 IFM_SUBTYPE(media) == IFM_1000_T ?
2495 "1000Base-T" :
2496 "2500Base-T");
2497 err = EINVAL;
2498 goto out;
2499 }
2500 reg |= MVNETA_PANC_SETGMIISPEED;
2501 } else if (IFM_SUBTYPE(media) == IFM_100_TX)
2502 reg |= MVNETA_PANC_SETMIISPEED;
2503
2504 if (media & IFM_FDX)
2505 reg |= MVNETA_PANC_SETFULLDX;
2506
2507 MVNETA_WRITE(sc, MVNETA_PANC, reg);
2508 }
2509 out:
2510 if (running)
2511 mvneta_init_locked(sc);
2512 mvneta_sc_unlock(sc);
2513 return (err);
2514 }
2515
2516 STATIC void
mvneta_adjust_link(struct mvneta_softc * sc)2517 mvneta_adjust_link(struct mvneta_softc *sc)
2518 {
2519 boolean_t phy_linkup;
2520 int reg;
2521
2522 /* Update eee/fc */
2523 mvneta_update_eee(sc);
2524 mvneta_update_fc(sc);
2525
2526 /* Check for link change */
2527 phy_linkup = (sc->mii->mii_media_status &
2528 (IFM_AVALID | IFM_ACTIVE)) == (IFM_AVALID | IFM_ACTIVE);
2529
2530 if (sc->linkup != phy_linkup)
2531 mvneta_linkupdate(sc, phy_linkup);
2532
2533 /* Don't update media on disabled link */
2534 if (!phy_linkup)
2535 return;
2536
2537 /* Check for media type change */
2538 if (sc->mvneta_media != sc->mii->mii_media_active) {
2539 sc->mvneta_media = sc->mii->mii_media_active;
2540
2541 reg = MVNETA_READ(sc, MVNETA_PANC);
2542 reg &= ~(MVNETA_PANC_SETGMIISPEED |
2543 MVNETA_PANC_SETMIISPEED |
2544 MVNETA_PANC_SETFULLDX);
2545 if (IFM_SUBTYPE(sc->mvneta_media) == IFM_1000_T ||
2546 IFM_SUBTYPE(sc->mvneta_media) == IFM_2500_T) {
2547 reg |= MVNETA_PANC_SETGMIISPEED;
2548 } else if (IFM_SUBTYPE(sc->mvneta_media) == IFM_100_TX)
2549 reg |= MVNETA_PANC_SETMIISPEED;
2550
2551 if (sc->mvneta_media & IFM_FDX)
2552 reg |= MVNETA_PANC_SETFULLDX;
2553
2554 MVNETA_WRITE(sc, MVNETA_PANC, reg);
2555 }
2556 }
2557
2558 STATIC void
mvneta_link_isr(struct mvneta_softc * sc)2559 mvneta_link_isr(struct mvneta_softc *sc)
2560 {
2561 int linkup;
2562
2563 KASSERT_SC_MTX(sc);
2564
2565 linkup = MVNETA_IS_LINKUP(sc) ? TRUE : FALSE;
2566 if (sc->linkup == linkup)
2567 return;
2568
2569 if (linkup == TRUE)
2570 mvneta_linkup(sc);
2571 else
2572 mvneta_linkdown(sc);
2573
2574 #ifdef DEBUG
2575 log(LOG_DEBUG,
2576 "%s: link %s\n", device_xname(sc->dev), linkup ? "up" : "down");
2577 #endif
2578 }
2579
2580 STATIC void
mvneta_linkupdate(struct mvneta_softc * sc,boolean_t linkup)2581 mvneta_linkupdate(struct mvneta_softc *sc, boolean_t linkup)
2582 {
2583
2584 KASSERT_SC_MTX(sc);
2585
2586 if (linkup == TRUE)
2587 mvneta_linkup(sc);
2588 else
2589 mvneta_linkdown(sc);
2590
2591 #ifdef DEBUG
2592 log(LOG_DEBUG,
2593 "%s: link %s\n", device_xname(sc->dev), linkup ? "up" : "down");
2594 #endif
2595 }
2596
2597 STATIC void
mvneta_update_eee(struct mvneta_softc * sc)2598 mvneta_update_eee(struct mvneta_softc *sc)
2599 {
2600 uint32_t reg;
2601
2602 KASSERT_SC_MTX(sc);
2603
2604 /* set EEE parameters */
2605 reg = MVNETA_READ(sc, MVNETA_LPIC1);
2606 if (sc->cf_lpi)
2607 reg |= MVNETA_LPIC1_LPIRE;
2608 else
2609 reg &= ~MVNETA_LPIC1_LPIRE;
2610 MVNETA_WRITE(sc, MVNETA_LPIC1, reg);
2611 }
2612
2613 STATIC void
mvneta_update_fc(struct mvneta_softc * sc)2614 mvneta_update_fc(struct mvneta_softc *sc)
2615 {
2616 uint32_t reg;
2617
2618 KASSERT_SC_MTX(sc);
2619
2620 reg = MVNETA_READ(sc, MVNETA_PANC);
2621 if (sc->cf_fc) {
2622 /* Flow control negotiation */
2623 reg |= MVNETA_PANC_PAUSEADV;
2624 reg |= MVNETA_PANC_ANFCEN;
2625 } else {
2626 /* Disable flow control negotiation */
2627 reg &= ~MVNETA_PANC_PAUSEADV;
2628 reg &= ~MVNETA_PANC_ANFCEN;
2629 }
2630
2631 MVNETA_WRITE(sc, MVNETA_PANC, reg);
2632 }
2633
2634 STATIC void
mvneta_linkup(struct mvneta_softc * sc)2635 mvneta_linkup(struct mvneta_softc *sc)
2636 {
2637 uint32_t reg;
2638
2639 KASSERT_SC_MTX(sc);
2640
2641 if (!sc->use_inband_status) {
2642 reg = MVNETA_READ(sc, MVNETA_PANC);
2643 reg |= MVNETA_PANC_FORCELINKPASS;
2644 reg &= ~MVNETA_PANC_FORCELINKFAIL;
2645 MVNETA_WRITE(sc, MVNETA_PANC, reg);
2646 }
2647
2648 mvneta_qflush(sc->ifp);
2649 mvneta_portup(sc);
2650 sc->linkup = TRUE;
2651 if_link_state_change(sc->ifp, LINK_STATE_UP);
2652 }
2653
2654 STATIC void
mvneta_linkdown(struct mvneta_softc * sc)2655 mvneta_linkdown(struct mvneta_softc *sc)
2656 {
2657 uint32_t reg;
2658
2659 KASSERT_SC_MTX(sc);
2660
2661 if (!sc->use_inband_status) {
2662 reg = MVNETA_READ(sc, MVNETA_PANC);
2663 reg &= ~MVNETA_PANC_FORCELINKPASS;
2664 reg |= MVNETA_PANC_FORCELINKFAIL;
2665 MVNETA_WRITE(sc, MVNETA_PANC, reg);
2666 }
2667
2668 mvneta_portdown(sc);
2669 mvneta_qflush(sc->ifp);
2670 sc->linkup = FALSE;
2671 if_link_state_change(sc->ifp, LINK_STATE_DOWN);
2672 }
2673
2674 STATIC void
mvneta_linkreset(struct mvneta_softc * sc)2675 mvneta_linkreset(struct mvneta_softc *sc)
2676 {
2677 struct mii_softc *mii;
2678
2679 if (sc->phy_attached) {
2680 /* Force reset PHY */
2681 mii = LIST_FIRST(&sc->mii->mii_phys);
2682 if (mii)
2683 mii_phy_reset(mii);
2684 }
2685 }
2686
2687 /*
2688 * Tx Subroutines
2689 */
2690 STATIC int
mvneta_tx_queue(struct mvneta_softc * sc,struct mbuf ** mbufp,int q)2691 mvneta_tx_queue(struct mvneta_softc *sc, struct mbuf **mbufp, int q)
2692 {
2693 struct ifnet *ifp;
2694 bus_dma_segment_t txsegs[MVNETA_TX_SEGLIMIT];
2695 struct mbuf *mtmp, *mbuf;
2696 struct mvneta_tx_ring *tx;
2697 struct mvneta_buf *txbuf;
2698 struct mvneta_tx_desc *t;
2699 uint32_t ptxsu;
2700 int start, used, error, i, txnsegs;
2701
2702 mbuf = *mbufp;
2703 tx = MVNETA_TX_RING(sc, q);
2704 DASSERT(tx->used >= 0);
2705 DASSERT(tx->used <= MVNETA_TX_RING_CNT);
2706 t = NULL;
2707 ifp = sc->ifp;
2708
2709 if (__predict_false(mbuf->m_flags & M_VLANTAG)) {
2710 mbuf = ether_vlanencap(mbuf, mbuf->m_pkthdr.ether_vtag);
2711 if (mbuf == NULL) {
2712 tx->drv_error++;
2713 *mbufp = NULL;
2714 return (ENOBUFS);
2715 }
2716 mbuf->m_flags &= ~M_VLANTAG;
2717 *mbufp = mbuf;
2718 }
2719
2720 if (__predict_false(mbuf->m_next != NULL &&
2721 (mbuf->m_pkthdr.csum_flags &
2722 (CSUM_IP | CSUM_TCP | CSUM_UDP)) != 0)) {
2723 if (M_WRITABLE(mbuf) == 0) {
2724 mtmp = m_dup(mbuf, M_NOWAIT);
2725 m_freem(mbuf);
2726 if (mtmp == NULL) {
2727 tx->drv_error++;
2728 *mbufp = NULL;
2729 return (ENOBUFS);
2730 }
2731 *mbufp = mbuf = mtmp;
2732 }
2733 }
2734
2735 /* load mbuf using dmamap of 1st descriptor */
2736 txbuf = &tx->txbuf[tx->cpu];
2737 error = bus_dmamap_load_mbuf_sg(sc->txmbuf_dtag,
2738 txbuf->dmap, mbuf, txsegs, &txnsegs,
2739 BUS_DMA_NOWAIT);
2740 if (__predict_false(error != 0)) {
2741 #ifdef MVNETA_KTR
2742 CTR3(KTR_SPARE2, "%s:%u bus_dmamap_load_mbuf_sg error=%d", ifp->if_xname, q, error);
2743 #endif
2744 /* This is the only recoverable error (except EFBIG). */
2745 if (error != ENOMEM) {
2746 tx->drv_error++;
2747 m_freem(mbuf);
2748 *mbufp = NULL;
2749 return (ENOBUFS);
2750 }
2751 return (error);
2752 }
2753
2754 if (__predict_false(txnsegs <= 0
2755 || (txnsegs + tx->used) > MVNETA_TX_RING_CNT)) {
2756 /* we have no enough descriptors or mbuf is broken */
2757 #ifdef MVNETA_KTR
2758 CTR3(KTR_SPARE2, "%s:%u not enough descriptors txnsegs=%d",
2759 ifp->if_xname, q, txnsegs);
2760 #endif
2761 bus_dmamap_unload(sc->txmbuf_dtag, txbuf->dmap);
2762 return (ENOBUFS);
2763 }
2764 DASSERT(txbuf->m == NULL);
2765
2766 /* remember mbuf using 1st descriptor */
2767 txbuf->m = mbuf;
2768 bus_dmamap_sync(sc->txmbuf_dtag, txbuf->dmap,
2769 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
2770
2771 /* load to tx descriptors */
2772 start = tx->cpu;
2773 used = 0;
2774 for (i = 0; i < txnsegs; i++) {
2775 t = &tx->desc[tx->cpu];
2776 t->command = 0;
2777 t->l4ichk = 0;
2778 t->flags = 0;
2779 if (__predict_true(i == 0)) {
2780 /* 1st descriptor */
2781 t->command |= MVNETA_TX_CMD_W_PACKET_OFFSET(0);
2782 t->command |= MVNETA_TX_CMD_F;
2783 mvneta_tx_set_csumflag(ifp, t, mbuf);
2784 }
2785 t->bufptr_pa = txsegs[i].ds_addr;
2786 t->bytecnt = txsegs[i].ds_len;
2787 tx->cpu = tx_counter_adv(tx->cpu, 1);
2788
2789 tx->used++;
2790 used++;
2791 }
2792 /* t is last descriptor here */
2793 DASSERT(t != NULL);
2794 t->command |= MVNETA_TX_CMD_L|MVNETA_TX_CMD_PADDING;
2795
2796 bus_dmamap_sync(sc->tx_dtag, tx->desc_map,
2797 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
2798
2799 while (__predict_false(used > 255)) {
2800 ptxsu = MVNETA_PTXSU_NOWD(255);
2801 MVNETA_WRITE(sc, MVNETA_PTXSU(q), ptxsu);
2802 used -= 255;
2803 }
2804 if (__predict_true(used > 0)) {
2805 ptxsu = MVNETA_PTXSU_NOWD(used);
2806 MVNETA_WRITE(sc, MVNETA_PTXSU(q), ptxsu);
2807 }
2808 return (0);
2809 }
2810
2811 STATIC void
mvneta_tx_set_csumflag(struct ifnet * ifp,struct mvneta_tx_desc * t,struct mbuf * m)2812 mvneta_tx_set_csumflag(struct ifnet *ifp,
2813 struct mvneta_tx_desc *t, struct mbuf *m)
2814 {
2815 struct ether_header *eh;
2816 int csum_flags;
2817 uint32_t iphl, ipoff;
2818 struct ip *ip;
2819
2820 iphl = ipoff = 0;
2821 csum_flags = ifp->if_hwassist & m->m_pkthdr.csum_flags;
2822 eh = mtod(m, struct ether_header *);
2823
2824 switch (ntohs(eh->ether_type)) {
2825 case ETHERTYPE_IP:
2826 ipoff = ETHER_HDR_LEN;
2827 break;
2828 case ETHERTYPE_VLAN:
2829 ipoff = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
2830 break;
2831 default:
2832 csum_flags = 0;
2833 }
2834
2835 if (__predict_true(csum_flags & (CSUM_IP|CSUM_IP_TCP|CSUM_IP_UDP))) {
2836 ip = (struct ip *)(m->m_data + ipoff);
2837 iphl = ip->ip_hl<<2;
2838 t->command |= MVNETA_TX_CMD_L3_IP4;
2839 } else {
2840 t->command |= MVNETA_TX_CMD_L4_CHECKSUM_NONE;
2841 return;
2842 }
2843
2844
2845 /* L3 */
2846 if (csum_flags & CSUM_IP) {
2847 t->command |= MVNETA_TX_CMD_IP4_CHECKSUM;
2848 }
2849
2850 /* L4 */
2851 if (csum_flags & CSUM_IP_TCP) {
2852 t->command |= MVNETA_TX_CMD_L4_CHECKSUM_NOFRAG;
2853 t->command |= MVNETA_TX_CMD_L4_TCP;
2854 } else if (csum_flags & CSUM_IP_UDP) {
2855 t->command |= MVNETA_TX_CMD_L4_CHECKSUM_NOFRAG;
2856 t->command |= MVNETA_TX_CMD_L4_UDP;
2857 } else
2858 t->command |= MVNETA_TX_CMD_L4_CHECKSUM_NONE;
2859
2860 t->l4ichk = 0;
2861 t->command |= MVNETA_TX_CMD_IP_HEADER_LEN(iphl >> 2);
2862 t->command |= MVNETA_TX_CMD_L3_OFFSET(ipoff);
2863 }
2864
2865 STATIC void
mvneta_tx_queue_complete(struct mvneta_softc * sc,int q)2866 mvneta_tx_queue_complete(struct mvneta_softc *sc, int q)
2867 {
2868 struct mvneta_tx_ring *tx;
2869 struct mvneta_buf *txbuf;
2870 struct mvneta_tx_desc *t;
2871 uint32_t ptxs, ptxsu, ndesc;
2872 int i;
2873
2874 KASSERT_TX_MTX(sc, q);
2875
2876 tx = MVNETA_TX_RING(sc, q);
2877 if (__predict_false(tx->queue_status == MVNETA_QUEUE_DISABLED))
2878 return;
2879
2880 ptxs = MVNETA_READ(sc, MVNETA_PTXS(q));
2881 ndesc = MVNETA_PTXS_GET_TBC(ptxs);
2882
2883 if (__predict_false(ndesc == 0)) {
2884 if (tx->used == 0)
2885 tx->queue_status = MVNETA_QUEUE_IDLE;
2886 else if (tx->queue_status == MVNETA_QUEUE_WORKING &&
2887 ((ticks - tx->watchdog_time) > MVNETA_WATCHDOG))
2888 tx->queue_hung = TRUE;
2889 return;
2890 }
2891
2892 #ifdef MVNETA_KTR
2893 CTR3(KTR_SPARE2, "%s:%u tx_complete begin ndesc=%u",
2894 sc->ifp->if_xname, q, ndesc);
2895 #endif
2896
2897 bus_dmamap_sync(sc->tx_dtag, tx->desc_map,
2898 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
2899
2900 for (i = 0; i < ndesc; i++) {
2901 t = &tx->desc[tx->dma];
2902 #ifdef MVNETA_KTR
2903 if (t->flags & MVNETA_TX_F_ES)
2904 CTR3(KTR_SPARE2, "%s tx error queue %d desc %d",
2905 sc->ifp->if_xname, q, tx->dma);
2906 #endif
2907 txbuf = &tx->txbuf[tx->dma];
2908 if (__predict_true(txbuf->m != NULL)) {
2909 DASSERT((t->command & MVNETA_TX_CMD_F) != 0);
2910 bus_dmamap_unload(sc->txmbuf_dtag, txbuf->dmap);
2911 m_freem(txbuf->m);
2912 txbuf->m = NULL;
2913 }
2914 else
2915 DASSERT((t->flags & MVNETA_TX_CMD_F) == 0);
2916 tx->dma = tx_counter_adv(tx->dma, 1);
2917 tx->used--;
2918 }
2919 DASSERT(tx->used >= 0);
2920 DASSERT(tx->used <= MVNETA_TX_RING_CNT);
2921 while (__predict_false(ndesc > 255)) {
2922 ptxsu = MVNETA_PTXSU_NORB(255);
2923 MVNETA_WRITE(sc, MVNETA_PTXSU(q), ptxsu);
2924 ndesc -= 255;
2925 }
2926 if (__predict_true(ndesc > 0)) {
2927 ptxsu = MVNETA_PTXSU_NORB(ndesc);
2928 MVNETA_WRITE(sc, MVNETA_PTXSU(q), ptxsu);
2929 }
2930 #ifdef MVNETA_KTR
2931 CTR5(KTR_SPARE2, "%s:%u tx_complete tx_cpu=%d tx_dma=%d tx_used=%d",
2932 sc->ifp->if_xname, q, tx->cpu, tx->dma, tx->used);
2933 #endif
2934
2935 tx->watchdog_time = ticks;
2936
2937 if (tx->used == 0)
2938 tx->queue_status = MVNETA_QUEUE_IDLE;
2939 }
2940
2941 /*
2942 * Do a final TX complete when TX is idle.
2943 */
2944 STATIC void
mvneta_tx_drain(struct mvneta_softc * sc)2945 mvneta_tx_drain(struct mvneta_softc *sc)
2946 {
2947 struct mvneta_tx_ring *tx;
2948 int q;
2949
2950 /*
2951 * Handle trailing mbuf on TX queue.
2952 * Check is done lockess to avoid TX path contention.
2953 */
2954 for (q = 0; q < MVNETA_TX_QNUM_MAX; q++) {
2955 tx = MVNETA_TX_RING(sc, q);
2956 if ((ticks - tx->watchdog_time) > MVNETA_WATCHDOG_TXCOMP &&
2957 tx->used > 0) {
2958 mvneta_tx_lockq(sc, q);
2959 mvneta_tx_queue_complete(sc, q);
2960 mvneta_tx_unlockq(sc, q);
2961 }
2962 }
2963 }
2964
2965 /*
2966 * Rx Subroutines
2967 */
2968 STATIC int
mvneta_rx(struct mvneta_softc * sc,int q,int count)2969 mvneta_rx(struct mvneta_softc *sc, int q, int count)
2970 {
2971 uint32_t prxs, npkt;
2972 int more;
2973
2974 more = 0;
2975 mvneta_rx_lockq(sc, q);
2976 prxs = MVNETA_READ(sc, MVNETA_PRXS(q));
2977 npkt = MVNETA_PRXS_GET_ODC(prxs);
2978 if (__predict_false(npkt == 0))
2979 goto out;
2980
2981 if (count > 0 && npkt > count) {
2982 more = 1;
2983 npkt = count;
2984 }
2985 mvneta_rx_queue(sc, q, npkt);
2986 out:
2987 mvneta_rx_unlockq(sc, q);
2988 return more;
2989 }
2990
2991 /*
2992 * Helper routine for updating PRXSU register of a given queue.
2993 * Handles number of processed descriptors bigger than maximum acceptable value.
2994 */
2995 STATIC __inline void
mvneta_prxsu_update(struct mvneta_softc * sc,int q,int processed)2996 mvneta_prxsu_update(struct mvneta_softc *sc, int q, int processed)
2997 {
2998 uint32_t prxsu;
2999
3000 while (__predict_false(processed > 255)) {
3001 prxsu = MVNETA_PRXSU_NOOFPROCESSEDDESCRIPTORS(255);
3002 MVNETA_WRITE(sc, MVNETA_PRXSU(q), prxsu);
3003 processed -= 255;
3004 }
3005 prxsu = MVNETA_PRXSU_NOOFPROCESSEDDESCRIPTORS(processed);
3006 MVNETA_WRITE(sc, MVNETA_PRXSU(q), prxsu);
3007 }
3008
3009 static __inline void
mvneta_prefetch(void * p)3010 mvneta_prefetch(void *p)
3011 {
3012
3013 __builtin_prefetch(p);
3014 }
3015
3016 STATIC void
mvneta_rx_queue(struct mvneta_softc * sc,int q,int npkt)3017 mvneta_rx_queue(struct mvneta_softc *sc, int q, int npkt)
3018 {
3019 struct ifnet *ifp;
3020 struct mvneta_rx_ring *rx;
3021 struct mvneta_rx_desc *r;
3022 struct mvneta_buf *rxbuf;
3023 struct mbuf *m;
3024 struct lro_ctrl *lro;
3025 struct lro_entry *queued;
3026 void *pktbuf;
3027 int i, pktlen, processed, ndma;
3028
3029 KASSERT_RX_MTX(sc, q);
3030
3031 ifp = sc->ifp;
3032 rx = MVNETA_RX_RING(sc, q);
3033 processed = 0;
3034
3035 if (__predict_false(rx->queue_status == MVNETA_QUEUE_DISABLED))
3036 return;
3037
3038 bus_dmamap_sync(sc->rx_dtag, rx->desc_map,
3039 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
3040
3041 for (i = 0; i < npkt; i++) {
3042 /* Prefetch next desc, rxbuf. */
3043 ndma = rx_counter_adv(rx->dma, 1);
3044 mvneta_prefetch(&rx->desc[ndma]);
3045 mvneta_prefetch(&rx->rxbuf[ndma]);
3046
3047 /* get descriptor and packet */
3048 r = &rx->desc[rx->dma];
3049 rxbuf = &rx->rxbuf[rx->dma];
3050 m = rxbuf->m;
3051 rxbuf->m = NULL;
3052 DASSERT(m != NULL);
3053 bus_dmamap_sync(sc->rxbuf_dtag, rxbuf->dmap,
3054 BUS_DMASYNC_POSTREAD);
3055 bus_dmamap_unload(sc->rxbuf_dtag, rxbuf->dmap);
3056 /* Prefetch mbuf header. */
3057 mvneta_prefetch(m);
3058
3059 processed++;
3060 /* Drop desc with error status or not in a single buffer. */
3061 DASSERT((r->status & (MVNETA_RX_F|MVNETA_RX_L)) ==
3062 (MVNETA_RX_F|MVNETA_RX_L));
3063 if (__predict_false((r->status & MVNETA_RX_ES) ||
3064 (r->status & (MVNETA_RX_F|MVNETA_RX_L)) !=
3065 (MVNETA_RX_F|MVNETA_RX_L)))
3066 goto rx_error;
3067
3068 /*
3069 * [ OFF | MH | PKT | CRC ]
3070 * bytecnt cover MH, PKT, CRC
3071 */
3072 pktlen = r->bytecnt - ETHER_CRC_LEN - MVNETA_HWHEADER_SIZE;
3073 pktbuf = (uint8_t *)rx->rxbuf_virt_addr[rx->dma] + MVNETA_PACKET_OFFSET +
3074 MVNETA_HWHEADER_SIZE;
3075
3076 /* Prefetch mbuf data. */
3077 mvneta_prefetch(pktbuf);
3078
3079 /* Write value to mbuf (avoid read). */
3080 m->m_data = pktbuf;
3081 m->m_len = m->m_pkthdr.len = pktlen;
3082 m->m_pkthdr.rcvif = ifp;
3083 mvneta_rx_set_csumflag(ifp, r, m);
3084
3085 /* Increase rx_dma before releasing the lock. */
3086 rx->dma = ndma;
3087
3088 if (__predict_false(rx->lro_enabled &&
3089 ((r->status & MVNETA_RX_L3_IP) != 0) &&
3090 ((r->status & MVNETA_RX_L4_MASK) == MVNETA_RX_L4_TCP) &&
3091 (m->m_pkthdr.csum_flags &
3092 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
3093 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR))) {
3094 if (rx->lro.lro_cnt != 0) {
3095 if (tcp_lro_rx(&rx->lro, m, 0) == 0)
3096 goto rx_done;
3097 }
3098 }
3099
3100 mvneta_rx_unlockq(sc, q);
3101 (*ifp->if_input)(ifp, m);
3102 mvneta_rx_lockq(sc, q);
3103 /*
3104 * Check whether this queue has been disabled in the
3105 * meantime. If yes, then clear LRO and exit.
3106 */
3107 if(__predict_false(rx->queue_status == MVNETA_QUEUE_DISABLED))
3108 goto rx_lro;
3109 rx_done:
3110 /* Refresh receive ring to avoid stall and minimize jitter. */
3111 if (processed >= MVNETA_RX_REFILL_COUNT) {
3112 mvneta_prxsu_update(sc, q, processed);
3113 mvneta_rx_queue_refill(sc, q);
3114 processed = 0;
3115 }
3116 continue;
3117 rx_error:
3118 m_freem(m);
3119 rx->dma = ndma;
3120 /* Refresh receive ring to avoid stall and minimize jitter. */
3121 if (processed >= MVNETA_RX_REFILL_COUNT) {
3122 mvneta_prxsu_update(sc, q, processed);
3123 mvneta_rx_queue_refill(sc, q);
3124 processed = 0;
3125 }
3126 }
3127 #ifdef MVNETA_KTR
3128 CTR3(KTR_SPARE2, "%s:%u %u packets received", ifp->if_xname, q, npkt);
3129 #endif
3130 /* DMA status update */
3131 mvneta_prxsu_update(sc, q, processed);
3132 /* Refill the rest of buffers if there are any to refill */
3133 mvneta_rx_queue_refill(sc, q);
3134
3135 rx_lro:
3136 /*
3137 * Flush any outstanding LRO work
3138 */
3139 lro = &rx->lro;
3140 while (__predict_false((queued = LIST_FIRST(&lro->lro_active)) != NULL)) {
3141 LIST_REMOVE(LIST_FIRST((&lro->lro_active)), next);
3142 tcp_lro_flush(lro, queued);
3143 }
3144 }
3145
3146 STATIC void
mvneta_rx_buf_free(struct mvneta_softc * sc,struct mvneta_buf * rxbuf)3147 mvneta_rx_buf_free(struct mvneta_softc *sc, struct mvneta_buf *rxbuf)
3148 {
3149
3150 bus_dmamap_unload(sc->rxbuf_dtag, rxbuf->dmap);
3151 /* This will remove all data at once */
3152 m_freem(rxbuf->m);
3153 }
3154
3155 STATIC void
mvneta_rx_queue_refill(struct mvneta_softc * sc,int q)3156 mvneta_rx_queue_refill(struct mvneta_softc *sc, int q)
3157 {
3158 struct mvneta_rx_ring *rx;
3159 struct mvneta_rx_desc *r;
3160 struct mvneta_buf *rxbuf;
3161 bus_dma_segment_t segs;
3162 struct mbuf *m;
3163 uint32_t prxs, prxsu, ndesc;
3164 int npkt, refill, nsegs, error;
3165
3166 KASSERT_RX_MTX(sc, q);
3167
3168 rx = MVNETA_RX_RING(sc, q);
3169 prxs = MVNETA_READ(sc, MVNETA_PRXS(q));
3170 ndesc = MVNETA_PRXS_GET_NODC(prxs) + MVNETA_PRXS_GET_ODC(prxs);
3171 refill = MVNETA_RX_RING_CNT - ndesc;
3172 #ifdef MVNETA_KTR
3173 CTR3(KTR_SPARE2, "%s:%u refill %u packets", sc->ifp->if_xname, q,
3174 refill);
3175 #endif
3176 if (__predict_false(refill <= 0))
3177 return;
3178
3179 for (npkt = 0; npkt < refill; npkt++) {
3180 rxbuf = &rx->rxbuf[rx->cpu];
3181 m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, sc->rx_frame_size);
3182 if (__predict_false(m == NULL)) {
3183 error = ENOBUFS;
3184 break;
3185 }
3186 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
3187
3188 error = bus_dmamap_load_mbuf_sg(sc->rxbuf_dtag, rxbuf->dmap,
3189 m, &segs, &nsegs, BUS_DMA_NOWAIT);
3190 if (__predict_false(error != 0 || nsegs != 1)) {
3191 KASSERT(1, ("Failed to load Rx mbuf DMA map"));
3192 m_freem(m);
3193 break;
3194 }
3195
3196 /* Add the packet to the ring */
3197 rxbuf->m = m;
3198 r = &rx->desc[rx->cpu];
3199 r->bufptr_pa = segs.ds_addr;
3200 rx->rxbuf_virt_addr[rx->cpu] = m->m_data;
3201
3202 rx->cpu = rx_counter_adv(rx->cpu, 1);
3203 }
3204 if (npkt == 0) {
3205 if (refill == MVNETA_RX_RING_CNT)
3206 rx->needs_refill = TRUE;
3207 return;
3208 }
3209
3210 rx->needs_refill = FALSE;
3211 bus_dmamap_sync(sc->rx_dtag, rx->desc_map, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
3212
3213 while (__predict_false(npkt > 255)) {
3214 prxsu = MVNETA_PRXSU_NOOFNEWDESCRIPTORS(255);
3215 MVNETA_WRITE(sc, MVNETA_PRXSU(q), prxsu);
3216 npkt -= 255;
3217 }
3218 if (__predict_true(npkt > 0)) {
3219 prxsu = MVNETA_PRXSU_NOOFNEWDESCRIPTORS(npkt);
3220 MVNETA_WRITE(sc, MVNETA_PRXSU(q), prxsu);
3221 }
3222 }
3223
3224 STATIC __inline void
mvneta_rx_set_csumflag(struct ifnet * ifp,struct mvneta_rx_desc * r,struct mbuf * m)3225 mvneta_rx_set_csumflag(struct ifnet *ifp,
3226 struct mvneta_rx_desc *r, struct mbuf *m)
3227 {
3228 uint32_t csum_flags;
3229
3230 csum_flags = 0;
3231 if (__predict_false((r->status &
3232 (MVNETA_RX_IP_HEADER_OK|MVNETA_RX_L3_IP)) == 0))
3233 return; /* not a IP packet */
3234
3235 /* L3 */
3236 if (__predict_true((r->status & MVNETA_RX_IP_HEADER_OK) ==
3237 MVNETA_RX_IP_HEADER_OK))
3238 csum_flags |= CSUM_L3_CALC|CSUM_L3_VALID;
3239
3240 if (__predict_true((r->status & (MVNETA_RX_IP_HEADER_OK|MVNETA_RX_L3_IP)) ==
3241 (MVNETA_RX_IP_HEADER_OK|MVNETA_RX_L3_IP))) {
3242 /* L4 */
3243 switch (r->status & MVNETA_RX_L4_MASK) {
3244 case MVNETA_RX_L4_TCP:
3245 case MVNETA_RX_L4_UDP:
3246 csum_flags |= CSUM_L4_CALC;
3247 if (__predict_true((r->status &
3248 MVNETA_RX_L4_CHECKSUM_OK) == MVNETA_RX_L4_CHECKSUM_OK)) {
3249 csum_flags |= CSUM_L4_VALID;
3250 m->m_pkthdr.csum_data = htons(0xffff);
3251 }
3252 break;
3253 case MVNETA_RX_L4_OTH:
3254 default:
3255 break;
3256 }
3257 }
3258 m->m_pkthdr.csum_flags = csum_flags;
3259 }
3260
3261 /*
3262 * MAC address filter
3263 */
3264 STATIC void
mvneta_filter_setup(struct mvneta_softc * sc)3265 mvneta_filter_setup(struct mvneta_softc *sc)
3266 {
3267 struct ifnet *ifp;
3268 uint32_t dfut[MVNETA_NDFUT], dfsmt[MVNETA_NDFSMT], dfomt[MVNETA_NDFOMT];
3269 uint32_t pxc;
3270 int i;
3271
3272 KASSERT_SC_MTX(sc);
3273
3274 memset(dfut, 0, sizeof(dfut));
3275 memset(dfsmt, 0, sizeof(dfsmt));
3276 memset(dfomt, 0, sizeof(dfomt));
3277
3278 ifp = sc->ifp;
3279 ifp->if_flags |= IFF_ALLMULTI;
3280 if (ifp->if_flags & (IFF_ALLMULTI|IFF_PROMISC)) {
3281 for (i = 0; i < MVNETA_NDFSMT; i++) {
3282 dfsmt[i] = dfomt[i] =
3283 MVNETA_DF(0, MVNETA_DF_QUEUE(0) | MVNETA_DF_PASS) |
3284 MVNETA_DF(1, MVNETA_DF_QUEUE(0) | MVNETA_DF_PASS) |
3285 MVNETA_DF(2, MVNETA_DF_QUEUE(0) | MVNETA_DF_PASS) |
3286 MVNETA_DF(3, MVNETA_DF_QUEUE(0) | MVNETA_DF_PASS);
3287 }
3288 }
3289
3290 pxc = MVNETA_READ(sc, MVNETA_PXC);
3291 pxc &= ~(MVNETA_PXC_UPM | MVNETA_PXC_RXQ_MASK | MVNETA_PXC_RXQARP_MASK |
3292 MVNETA_PXC_TCPQ_MASK | MVNETA_PXC_UDPQ_MASK | MVNETA_PXC_BPDUQ_MASK);
3293 pxc |= MVNETA_PXC_RXQ(MVNETA_RX_QNUM_MAX-1);
3294 pxc |= MVNETA_PXC_RXQARP(MVNETA_RX_QNUM_MAX-1);
3295 pxc |= MVNETA_PXC_TCPQ(MVNETA_RX_QNUM_MAX-1);
3296 pxc |= MVNETA_PXC_UDPQ(MVNETA_RX_QNUM_MAX-1);
3297 pxc |= MVNETA_PXC_BPDUQ(MVNETA_RX_QNUM_MAX-1);
3298 pxc |= MVNETA_PXC_RB | MVNETA_PXC_RBIP | MVNETA_PXC_RBARP;
3299 if (ifp->if_flags & IFF_BROADCAST) {
3300 pxc &= ~(MVNETA_PXC_RB | MVNETA_PXC_RBIP | MVNETA_PXC_RBARP);
3301 }
3302 if (ifp->if_flags & IFF_PROMISC) {
3303 pxc |= MVNETA_PXC_UPM;
3304 }
3305 MVNETA_WRITE(sc, MVNETA_PXC, pxc);
3306
3307 /* Set Destination Address Filter Unicast Table */
3308 if (ifp->if_flags & IFF_PROMISC) {
3309 /* pass all unicast addresses */
3310 for (i = 0; i < MVNETA_NDFUT; i++) {
3311 dfut[i] =
3312 MVNETA_DF(0, MVNETA_DF_QUEUE(0) | MVNETA_DF_PASS) |
3313 MVNETA_DF(1, MVNETA_DF_QUEUE(0) | MVNETA_DF_PASS) |
3314 MVNETA_DF(2, MVNETA_DF_QUEUE(0) | MVNETA_DF_PASS) |
3315 MVNETA_DF(3, MVNETA_DF_QUEUE(0) | MVNETA_DF_PASS);
3316 }
3317 } else {
3318 i = sc->enaddr[5] & 0xf; /* last nibble */
3319 dfut[i>>2] = MVNETA_DF(i&3, MVNETA_DF_QUEUE(0) | MVNETA_DF_PASS);
3320 }
3321 MVNETA_WRITE_REGION(sc, MVNETA_DFUT(0), dfut, MVNETA_NDFUT);
3322
3323 /* Set Destination Address Filter Multicast Tables */
3324 MVNETA_WRITE_REGION(sc, MVNETA_DFSMT(0), dfsmt, MVNETA_NDFSMT);
3325 MVNETA_WRITE_REGION(sc, MVNETA_DFOMT(0), dfomt, MVNETA_NDFOMT);
3326 }
3327
3328 /*
3329 * sysctl(9)
3330 */
3331 STATIC int
sysctl_read_mib(SYSCTL_HANDLER_ARGS)3332 sysctl_read_mib(SYSCTL_HANDLER_ARGS)
3333 {
3334 struct mvneta_sysctl_mib *arg;
3335 struct mvneta_softc *sc;
3336 uint64_t val;
3337
3338 arg = (struct mvneta_sysctl_mib *)arg1;
3339 if (arg == NULL)
3340 return (EINVAL);
3341
3342 sc = arg->sc;
3343 if (sc == NULL)
3344 return (EINVAL);
3345 if (arg->index < 0 || arg->index > MVNETA_PORTMIB_NOCOUNTER)
3346 return (EINVAL);
3347
3348 mvneta_sc_lock(sc);
3349 val = arg->counter;
3350 mvneta_sc_unlock(sc);
3351 return sysctl_handle_64(oidp, &val, 0, req);
3352 }
3353
3354
3355 STATIC int
sysctl_clear_mib(SYSCTL_HANDLER_ARGS)3356 sysctl_clear_mib(SYSCTL_HANDLER_ARGS)
3357 {
3358 struct mvneta_softc *sc;
3359 int err, val;
3360
3361 val = 0;
3362 sc = (struct mvneta_softc *)arg1;
3363 if (sc == NULL)
3364 return (EINVAL);
3365
3366 err = sysctl_handle_int(oidp, &val, 0, req);
3367 if (err != 0)
3368 return (err);
3369
3370 if (val < 0 || val > 1)
3371 return (EINVAL);
3372
3373 if (val == 1) {
3374 mvneta_sc_lock(sc);
3375 mvneta_clear_mib(sc);
3376 mvneta_sc_unlock(sc);
3377 }
3378
3379 return (0);
3380 }
3381
3382 STATIC int
sysctl_set_queue_rxthtime(SYSCTL_HANDLER_ARGS)3383 sysctl_set_queue_rxthtime(SYSCTL_HANDLER_ARGS)
3384 {
3385 struct mvneta_sysctl_queue *arg;
3386 struct mvneta_rx_ring *rx;
3387 struct mvneta_softc *sc;
3388 uint32_t reg, time_mvtclk;
3389 int err, time_us;
3390
3391 rx = NULL;
3392 arg = (struct mvneta_sysctl_queue *)arg1;
3393 if (arg == NULL)
3394 return (EINVAL);
3395 if (arg->queue < 0 || arg->queue > MVNETA_RX_RING_CNT)
3396 return (EINVAL);
3397 if (arg->rxtx != MVNETA_SYSCTL_RX)
3398 return (EINVAL);
3399
3400 sc = arg->sc;
3401 if (sc == NULL)
3402 return (EINVAL);
3403
3404 /* read queue length */
3405 mvneta_sc_lock(sc);
3406 mvneta_rx_lockq(sc, arg->queue);
3407 rx = MVNETA_RX_RING(sc, arg->queue);
3408 time_mvtclk = rx->queue_th_time;
3409 time_us = ((uint64_t)time_mvtclk * 1000ULL * 1000ULL) / mvneta_get_clk();
3410 mvneta_rx_unlockq(sc, arg->queue);
3411 mvneta_sc_unlock(sc);
3412
3413 err = sysctl_handle_int(oidp, &time_us, 0, req);
3414 if (err != 0)
3415 return (err);
3416
3417 mvneta_sc_lock(sc);
3418 mvneta_rx_lockq(sc, arg->queue);
3419
3420 /* update queue length (0[sec] - 1[sec]) */
3421 if (time_us < 0 || time_us > (1000 * 1000)) {
3422 mvneta_rx_unlockq(sc, arg->queue);
3423 mvneta_sc_unlock(sc);
3424 return (EINVAL);
3425 }
3426 time_mvtclk =
3427 (uint64_t)mvneta_get_clk() * (uint64_t)time_us / (1000ULL * 1000ULL);
3428 rx->queue_th_time = time_mvtclk;
3429 reg = MVNETA_PRXITTH_RITT(rx->queue_th_time);
3430 MVNETA_WRITE(sc, MVNETA_PRXITTH(arg->queue), reg);
3431 mvneta_rx_unlockq(sc, arg->queue);
3432 mvneta_sc_unlock(sc);
3433
3434 return (0);
3435 }
3436
3437 STATIC void
sysctl_mvneta_init(struct mvneta_softc * sc)3438 sysctl_mvneta_init(struct mvneta_softc *sc)
3439 {
3440 struct sysctl_ctx_list *ctx;
3441 struct sysctl_oid_list *children;
3442 struct sysctl_oid_list *rxchildren;
3443 struct sysctl_oid_list *qchildren, *mchildren;
3444 struct sysctl_oid *tree;
3445 int i, q;
3446 struct mvneta_sysctl_queue *rxarg;
3447 #define MVNETA_SYSCTL_NAME(num) "queue" # num
3448 static const char *sysctl_queue_names[] = {
3449 MVNETA_SYSCTL_NAME(0), MVNETA_SYSCTL_NAME(1),
3450 MVNETA_SYSCTL_NAME(2), MVNETA_SYSCTL_NAME(3),
3451 MVNETA_SYSCTL_NAME(4), MVNETA_SYSCTL_NAME(5),
3452 MVNETA_SYSCTL_NAME(6), MVNETA_SYSCTL_NAME(7),
3453 };
3454 #undef MVNETA_SYSCTL_NAME
3455
3456 #ifndef NO_SYSCTL_DESCR
3457 #define MVNETA_SYSCTL_DESCR(num) "configuration parameters for queue " # num
3458 static const char *sysctl_queue_descrs[] = {
3459 MVNETA_SYSCTL_DESCR(0), MVNETA_SYSCTL_DESCR(1),
3460 MVNETA_SYSCTL_DESCR(2), MVNETA_SYSCTL_DESCR(3),
3461 MVNETA_SYSCTL_DESCR(4), MVNETA_SYSCTL_DESCR(5),
3462 MVNETA_SYSCTL_DESCR(6), MVNETA_SYSCTL_DESCR(7),
3463 };
3464 #undef MVNETA_SYSCTL_DESCR
3465 #endif
3466
3467
3468 ctx = device_get_sysctl_ctx(sc->dev);
3469 children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev));
3470
3471 tree = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "rx",
3472 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "NETA RX");
3473 rxchildren = SYSCTL_CHILDREN(tree);
3474 tree = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "mib",
3475 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "NETA MIB");
3476 mchildren = SYSCTL_CHILDREN(tree);
3477
3478
3479 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "flow_control",
3480 CTLFLAG_RW, &sc->cf_fc, 0, "flow control");
3481 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lpi",
3482 CTLFLAG_RW, &sc->cf_lpi, 0, "Low Power Idle");
3483
3484 /*
3485 * MIB access
3486 */
3487 /* dev.mvneta.[unit].mib.<mibs> */
3488 for (i = 0; i < MVNETA_PORTMIB_NOCOUNTER; i++) {
3489 struct mvneta_sysctl_mib *mib_arg = &sc->sysctl_mib[i];
3490
3491 mib_arg->sc = sc;
3492 mib_arg->index = i;
3493 SYSCTL_ADD_PROC(ctx, mchildren, OID_AUTO,
3494 mvneta_mib_list[i].sysctl_name,
3495 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_NEEDGIANT,
3496 (void *)mib_arg, 0, sysctl_read_mib, "I",
3497 mvneta_mib_list[i].desc);
3498 }
3499 SYSCTL_ADD_UQUAD(ctx, mchildren, OID_AUTO, "rx_discard",
3500 CTLFLAG_RD, &sc->counter_pdfc, "Port Rx Discard Frame Counter");
3501 SYSCTL_ADD_UQUAD(ctx, mchildren, OID_AUTO, "overrun",
3502 CTLFLAG_RD, &sc->counter_pofc, "Port Overrun Frame Counter");
3503 SYSCTL_ADD_UINT(ctx, mchildren, OID_AUTO, "watchdog",
3504 CTLFLAG_RD, &sc->counter_watchdog, 0, "TX Watchdog Counter");
3505
3506 SYSCTL_ADD_PROC(ctx, mchildren, OID_AUTO, "reset",
3507 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
3508 (void *)sc, 0, sysctl_clear_mib, "I", "Reset MIB counters");
3509
3510 for (q = 0; q < MVNETA_RX_QNUM_MAX; q++) {
3511 rxarg = &sc->sysctl_rx_queue[q];
3512
3513 rxarg->sc = sc;
3514 rxarg->queue = q;
3515 rxarg->rxtx = MVNETA_SYSCTL_RX;
3516
3517 /* hw.mvneta.mvneta[unit].rx.[queue] */
3518 tree = SYSCTL_ADD_NODE(ctx, rxchildren, OID_AUTO,
3519 sysctl_queue_names[q], CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
3520 sysctl_queue_descrs[q]);
3521 qchildren = SYSCTL_CHILDREN(tree);
3522
3523 /* hw.mvneta.mvneta[unit].rx.[queue].threshold_timer_us */
3524 SYSCTL_ADD_PROC(ctx, qchildren, OID_AUTO, "threshold_timer_us",
3525 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, rxarg, 0,
3526 sysctl_set_queue_rxthtime, "I",
3527 "interrupt coalescing threshold timer [us]");
3528 }
3529 }
3530
3531 /*
3532 * MIB
3533 */
3534 STATIC uint64_t
mvneta_read_mib(struct mvneta_softc * sc,int index)3535 mvneta_read_mib(struct mvneta_softc *sc, int index)
3536 {
3537 struct mvneta_mib_def *mib;
3538 uint64_t val;
3539
3540 mib = &mvneta_mib_list[index];
3541 val = MVNETA_READ_MIB(sc, mib->regnum);
3542 if (mib->reg64)
3543 val |= (uint64_t)MVNETA_READ_MIB(sc, mib->regnum + 4) << 32;
3544 return (val);
3545 }
3546
3547 STATIC void
mvneta_clear_mib(struct mvneta_softc * sc)3548 mvneta_clear_mib(struct mvneta_softc *sc)
3549 {
3550 int i;
3551
3552 KASSERT_SC_MTX(sc);
3553
3554 for (i = 0; i < nitems(mvneta_mib_list); i++) {
3555 (void)mvneta_read_mib(sc, i);
3556 sc->sysctl_mib[i].counter = 0;
3557 }
3558 MVNETA_READ(sc, MVNETA_PDFC);
3559 sc->counter_pdfc = 0;
3560 MVNETA_READ(sc, MVNETA_POFC);
3561 sc->counter_pofc = 0;
3562 sc->counter_watchdog = 0;
3563 }
3564
3565 STATIC void
mvneta_update_mib(struct mvneta_softc * sc)3566 mvneta_update_mib(struct mvneta_softc *sc)
3567 {
3568 struct mvneta_tx_ring *tx;
3569 int i;
3570 uint64_t val;
3571 uint32_t reg;
3572
3573 for (i = 0; i < nitems(mvneta_mib_list); i++) {
3574
3575 val = mvneta_read_mib(sc, i);
3576 if (val == 0)
3577 continue;
3578
3579 sc->sysctl_mib[i].counter += val;
3580 switch (mvneta_mib_list[i].regnum) {
3581 case MVNETA_MIB_RX_GOOD_OCT:
3582 if_inc_counter(sc->ifp, IFCOUNTER_IBYTES, val);
3583 break;
3584 case MVNETA_MIB_RX_BAD_FRAME:
3585 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, val);
3586 break;
3587 case MVNETA_MIB_RX_GOOD_FRAME:
3588 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, val);
3589 break;
3590 case MVNETA_MIB_RX_MCAST_FRAME:
3591 if_inc_counter(sc->ifp, IFCOUNTER_IMCASTS, val);
3592 break;
3593 case MVNETA_MIB_TX_GOOD_OCT:
3594 if_inc_counter(sc->ifp, IFCOUNTER_OBYTES, val);
3595 break;
3596 case MVNETA_MIB_TX_GOOD_FRAME:
3597 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, val);
3598 break;
3599 case MVNETA_MIB_TX_MCAST_FRAME:
3600 if_inc_counter(sc->ifp, IFCOUNTER_OMCASTS, val);
3601 break;
3602 case MVNETA_MIB_MAC_COL:
3603 if_inc_counter(sc->ifp, IFCOUNTER_COLLISIONS, val);
3604 break;
3605 case MVNETA_MIB_TX_MAC_TRNS_ERR:
3606 case MVNETA_MIB_TX_EXCES_COL:
3607 case MVNETA_MIB_MAC_LATE_COL:
3608 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, val);
3609 break;
3610 }
3611 }
3612
3613 reg = MVNETA_READ(sc, MVNETA_PDFC);
3614 sc->counter_pdfc += reg;
3615 if_inc_counter(sc->ifp, IFCOUNTER_IQDROPS, reg);
3616 reg = MVNETA_READ(sc, MVNETA_POFC);
3617 sc->counter_pofc += reg;
3618 if_inc_counter(sc->ifp, IFCOUNTER_IQDROPS, reg);
3619
3620 /* TX watchdog. */
3621 if (sc->counter_watchdog_mib > 0) {
3622 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, sc->counter_watchdog_mib);
3623 sc->counter_watchdog_mib = 0;
3624 }
3625 /*
3626 * TX driver errors:
3627 * We do not take queue locks to not disrupt TX path.
3628 * We may only miss one drv error which will be fixed at
3629 * next mib update. We may also clear counter when TX path
3630 * is incrementing it but we only do it if counter was not zero
3631 * thus we may only loose one error.
3632 */
3633 for (i = 0; i < MVNETA_TX_QNUM_MAX; i++) {
3634 tx = MVNETA_TX_RING(sc, i);
3635
3636 if (tx->drv_error > 0) {
3637 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, tx->drv_error);
3638 tx->drv_error = 0;
3639 }
3640 }
3641 }
3642