1 /*-
2 * Copyright (c) 2013 Tsubai Masanari
3 * Copyright (c) 2013 Bryan Venteicher <[email protected]>
4 * Copyright (c) 2018 Patrick Kelsey
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 *
18 * $OpenBSD: src/sys/dev/pci/if_vmx.c,v 1.11 2013/06/22 00:28:10 uebayasi Exp $
19 */
20
21 /* Driver for VMware vmxnet3 virtual ethernet devices. */
22
23 #include <sys/cdefs.h>
24 __FBSDID("$FreeBSD$");
25
26 #include "opt_rss.h"
27
28 #include <sys/param.h>
29 #include <sys/systm.h>
30 #include <sys/kernel.h>
31 #include <sys/endian.h>
32 #include <sys/sockio.h>
33 #include <sys/mbuf.h>
34 #include <sys/malloc.h>
35 #include <sys/module.h>
36 #include <sys/socket.h>
37 #include <sys/sysctl.h>
38 #include <sys/smp.h>
39 #include <vm/vm.h>
40 #include <vm/pmap.h>
41
42 #include <net/ethernet.h>
43 #include <net/if.h>
44 #include <net/if_var.h>
45 #include <net/if_arp.h>
46 #include <net/if_dl.h>
47 #include <net/if_types.h>
48 #include <net/if_media.h>
49 #include <net/if_vlan_var.h>
50 #include <net/iflib.h>
51 #ifdef RSS
52 #include <net/rss_config.h>
53 #endif
54
55 #include <netinet/in_systm.h>
56 #include <netinet/in.h>
57 #include <netinet/ip.h>
58 #include <netinet/ip6.h>
59 #include <netinet6/ip6_var.h>
60 #include <netinet/udp.h>
61 #include <netinet/tcp.h>
62
63 #include <machine/bus.h>
64 #include <machine/resource.h>
65 #include <sys/bus.h>
66 #include <sys/rman.h>
67
68 #include <dev/pci/pcireg.h>
69 #include <dev/pci/pcivar.h>
70
71 #include "ifdi_if.h"
72
73 #include "if_vmxreg.h"
74 #include "if_vmxvar.h"
75
76 #include "opt_inet.h"
77 #include "opt_inet6.h"
78
79 #define VMXNET3_VMWARE_VENDOR_ID 0x15AD
80 #define VMXNET3_VMWARE_DEVICE_ID 0x07B0
81
82 static pci_vendor_info_t vmxnet3_vendor_info_array[] =
83 {
84 PVID(VMXNET3_VMWARE_VENDOR_ID, VMXNET3_VMWARE_DEVICE_ID, "VMware VMXNET3 Ethernet Adapter"),
85 /* required last entry */
86 PVID_END
87 };
88
89 static void *vmxnet3_register(device_t);
90 static int vmxnet3_attach_pre(if_ctx_t);
91 static int vmxnet3_msix_intr_assign(if_ctx_t, int);
92 static void vmxnet3_free_irqs(struct vmxnet3_softc *);
93 static int vmxnet3_attach_post(if_ctx_t);
94 static int vmxnet3_detach(if_ctx_t);
95 static int vmxnet3_shutdown(if_ctx_t);
96 static int vmxnet3_suspend(if_ctx_t);
97 static int vmxnet3_resume(if_ctx_t);
98
99 static int vmxnet3_alloc_resources(struct vmxnet3_softc *);
100 static void vmxnet3_free_resources(struct vmxnet3_softc *);
101 static int vmxnet3_check_version(struct vmxnet3_softc *);
102 static void vmxnet3_set_interrupt_idx(struct vmxnet3_softc *);
103
104 static int vmxnet3_queues_shared_alloc(struct vmxnet3_softc *);
105 static void vmxnet3_init_txq(struct vmxnet3_softc *, int);
106 static int vmxnet3_tx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int);
107 static void vmxnet3_init_rxq(struct vmxnet3_softc *, int, int);
108 static int vmxnet3_rx_queues_alloc(if_ctx_t, caddr_t *, uint64_t *, int, int);
109 static void vmxnet3_queues_free(if_ctx_t);
110
111 static int vmxnet3_alloc_shared_data(struct vmxnet3_softc *);
112 static void vmxnet3_free_shared_data(struct vmxnet3_softc *);
113 static int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *);
114 static void vmxnet3_free_mcast_table(struct vmxnet3_softc *);
115 static void vmxnet3_init_shared_data(struct vmxnet3_softc *);
116 static void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *);
117 static void vmxnet3_reinit_shared_data(struct vmxnet3_softc *);
118 static int vmxnet3_alloc_data(struct vmxnet3_softc *);
119 static void vmxnet3_free_data(struct vmxnet3_softc *);
120
121 static void vmxnet3_evintr(struct vmxnet3_softc *);
122 static int vmxnet3_isc_txd_encap(void *, if_pkt_info_t);
123 static void vmxnet3_isc_txd_flush(void *, uint16_t, qidx_t);
124 static int vmxnet3_isc_txd_credits_update(void *, uint16_t, bool);
125 static int vmxnet3_isc_rxd_available(void *, uint16_t, qidx_t, qidx_t);
126 static int vmxnet3_isc_rxd_pkt_get(void *, if_rxd_info_t);
127 static void vmxnet3_isc_rxd_refill(void *, if_rxd_update_t);
128 static void vmxnet3_isc_rxd_flush(void *, uint16_t, uint8_t, qidx_t);
129 static int vmxnet3_legacy_intr(void *);
130 static int vmxnet3_rxq_intr(void *);
131 static int vmxnet3_event_intr(void *);
132
133 static void vmxnet3_stop(if_ctx_t);
134
135 static void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
136 static void vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
137 static void vmxnet3_reinit_queues(struct vmxnet3_softc *);
138 static int vmxnet3_enable_device(struct vmxnet3_softc *);
139 static void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *);
140 static void vmxnet3_init(if_ctx_t);
141 static void vmxnet3_multi_set(if_ctx_t);
142 static int vmxnet3_mtu_set(if_ctx_t, uint32_t);
143 static void vmxnet3_media_status(if_ctx_t, struct ifmediareq *);
144 static int vmxnet3_media_change(if_ctx_t);
145 static int vmxnet3_promisc_set(if_ctx_t, int);
146 static uint64_t vmxnet3_get_counter(if_ctx_t, ift_counter);
147 static void vmxnet3_update_admin_status(if_ctx_t);
148 static void vmxnet3_txq_timer(if_ctx_t, uint16_t);
149
150 static void vmxnet3_update_vlan_filter(struct vmxnet3_softc *, int,
151 uint16_t);
152 static void vmxnet3_vlan_register(if_ctx_t, uint16_t);
153 static void vmxnet3_vlan_unregister(if_ctx_t, uint16_t);
154 static void vmxnet3_set_rxfilter(struct vmxnet3_softc *, int);
155
156 static void vmxnet3_refresh_host_stats(struct vmxnet3_softc *);
157 static int vmxnet3_link_is_up(struct vmxnet3_softc *);
158 static void vmxnet3_link_status(struct vmxnet3_softc *);
159 static void vmxnet3_set_lladdr(struct vmxnet3_softc *);
160 static void vmxnet3_get_lladdr(struct vmxnet3_softc *);
161
162 static void vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *,
163 struct sysctl_ctx_list *, struct sysctl_oid_list *);
164 static void vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *,
165 struct sysctl_ctx_list *, struct sysctl_oid_list *);
166 static void vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *,
167 struct sysctl_ctx_list *, struct sysctl_oid_list *);
168 static void vmxnet3_setup_sysctl(struct vmxnet3_softc *);
169
170 static void vmxnet3_write_bar0(struct vmxnet3_softc *, bus_size_t,
171 uint32_t);
172 static uint32_t vmxnet3_read_bar1(struct vmxnet3_softc *, bus_size_t);
173 static void vmxnet3_write_bar1(struct vmxnet3_softc *, bus_size_t,
174 uint32_t);
175 static void vmxnet3_write_cmd(struct vmxnet3_softc *, uint32_t);
176 static uint32_t vmxnet3_read_cmd(struct vmxnet3_softc *, uint32_t);
177
178 static int vmxnet3_tx_queue_intr_enable(if_ctx_t, uint16_t);
179 static int vmxnet3_rx_queue_intr_enable(if_ctx_t, uint16_t);
180 static void vmxnet3_link_intr_enable(if_ctx_t);
181 static void vmxnet3_enable_intr(struct vmxnet3_softc *, int);
182 static void vmxnet3_disable_intr(struct vmxnet3_softc *, int);
183 static void vmxnet3_intr_enable_all(if_ctx_t);
184 static void vmxnet3_intr_disable_all(if_ctx_t);
185
186 typedef enum {
187 VMXNET3_BARRIER_RD,
188 VMXNET3_BARRIER_WR,
189 VMXNET3_BARRIER_RDWR,
190 } vmxnet3_barrier_t;
191
192 static void vmxnet3_barrier(struct vmxnet3_softc *, vmxnet3_barrier_t);
193
194 static device_method_t vmxnet3_methods[] = {
195 /* Device interface */
196 DEVMETHOD(device_register, vmxnet3_register),
197 DEVMETHOD(device_probe, iflib_device_probe),
198 DEVMETHOD(device_attach, iflib_device_attach),
199 DEVMETHOD(device_detach, iflib_device_detach),
200 DEVMETHOD(device_shutdown, iflib_device_shutdown),
201 DEVMETHOD(device_suspend, iflib_device_suspend),
202 DEVMETHOD(device_resume, iflib_device_resume),
203 DEVMETHOD_END
204 };
205
206 static driver_t vmxnet3_driver = {
207 "vmx", vmxnet3_methods, sizeof(struct vmxnet3_softc)
208 };
209
210 static devclass_t vmxnet3_devclass;
211 DRIVER_MODULE(vmx, pci, vmxnet3_driver, vmxnet3_devclass, 0, 0);
212 IFLIB_PNP_INFO(pci, vmx, vmxnet3_vendor_info_array);
213 MODULE_VERSION(vmx, 2);
214
215 MODULE_DEPEND(vmx, pci, 1, 1, 1);
216 MODULE_DEPEND(vmx, ether, 1, 1, 1);
217 MODULE_DEPEND(vmx, iflib, 1, 1, 1);
218
219 static device_method_t vmxnet3_iflib_methods[] = {
220 DEVMETHOD(ifdi_tx_queues_alloc, vmxnet3_tx_queues_alloc),
221 DEVMETHOD(ifdi_rx_queues_alloc, vmxnet3_rx_queues_alloc),
222 DEVMETHOD(ifdi_queues_free, vmxnet3_queues_free),
223
224 DEVMETHOD(ifdi_attach_pre, vmxnet3_attach_pre),
225 DEVMETHOD(ifdi_attach_post, vmxnet3_attach_post),
226 DEVMETHOD(ifdi_detach, vmxnet3_detach),
227
228 DEVMETHOD(ifdi_init, vmxnet3_init),
229 DEVMETHOD(ifdi_stop, vmxnet3_stop),
230 DEVMETHOD(ifdi_multi_set, vmxnet3_multi_set),
231 DEVMETHOD(ifdi_mtu_set, vmxnet3_mtu_set),
232 DEVMETHOD(ifdi_media_status, vmxnet3_media_status),
233 DEVMETHOD(ifdi_media_change, vmxnet3_media_change),
234 DEVMETHOD(ifdi_promisc_set, vmxnet3_promisc_set),
235 DEVMETHOD(ifdi_get_counter, vmxnet3_get_counter),
236 DEVMETHOD(ifdi_update_admin_status, vmxnet3_update_admin_status),
237 DEVMETHOD(ifdi_timer, vmxnet3_txq_timer),
238
239 DEVMETHOD(ifdi_tx_queue_intr_enable, vmxnet3_tx_queue_intr_enable),
240 DEVMETHOD(ifdi_rx_queue_intr_enable, vmxnet3_rx_queue_intr_enable),
241 DEVMETHOD(ifdi_link_intr_enable, vmxnet3_link_intr_enable),
242 DEVMETHOD(ifdi_intr_enable, vmxnet3_intr_enable_all),
243 DEVMETHOD(ifdi_intr_disable, vmxnet3_intr_disable_all),
244 DEVMETHOD(ifdi_msix_intr_assign, vmxnet3_msix_intr_assign),
245
246 DEVMETHOD(ifdi_vlan_register, vmxnet3_vlan_register),
247 DEVMETHOD(ifdi_vlan_unregister, vmxnet3_vlan_unregister),
248
249 DEVMETHOD(ifdi_shutdown, vmxnet3_shutdown),
250 DEVMETHOD(ifdi_suspend, vmxnet3_suspend),
251 DEVMETHOD(ifdi_resume, vmxnet3_resume),
252
253 DEVMETHOD_END
254 };
255
256 static driver_t vmxnet3_iflib_driver = {
257 "vmx", vmxnet3_iflib_methods, sizeof(struct vmxnet3_softc)
258 };
259
260 struct if_txrx vmxnet3_txrx = {
261 .ift_txd_encap = vmxnet3_isc_txd_encap,
262 .ift_txd_flush = vmxnet3_isc_txd_flush,
263 .ift_txd_credits_update = vmxnet3_isc_txd_credits_update,
264 .ift_rxd_available = vmxnet3_isc_rxd_available,
265 .ift_rxd_pkt_get = vmxnet3_isc_rxd_pkt_get,
266 .ift_rxd_refill = vmxnet3_isc_rxd_refill,
267 .ift_rxd_flush = vmxnet3_isc_rxd_flush,
268 .ift_legacy_intr = vmxnet3_legacy_intr
269 };
270
271 static struct if_shared_ctx vmxnet3_sctx_init = {
272 .isc_magic = IFLIB_MAGIC,
273 .isc_q_align = 512,
274
275 .isc_tx_maxsize = VMXNET3_TX_MAXSIZE,
276 .isc_tx_maxsegsize = VMXNET3_TX_MAXSEGSIZE,
277 .isc_tso_maxsize = VMXNET3_TSO_MAXSIZE + sizeof(struct ether_vlan_header),
278 .isc_tso_maxsegsize = VMXNET3_TX_MAXSEGSIZE,
279
280 /*
281 * These values are used to configure the busdma tag used for
282 * receive descriptors. Each receive descriptor only points to one
283 * buffer.
284 */
285 .isc_rx_maxsize = VMXNET3_RX_MAXSEGSIZE, /* One buf per descriptor */
286 .isc_rx_nsegments = 1, /* One mapping per descriptor */
287 .isc_rx_maxsegsize = VMXNET3_RX_MAXSEGSIZE,
288
289 .isc_admin_intrcnt = 1,
290 .isc_vendor_info = vmxnet3_vendor_info_array,
291 .isc_driver_version = "2",
292 .isc_driver = &vmxnet3_iflib_driver,
293 .isc_flags = IFLIB_HAS_RXCQ | IFLIB_HAS_TXCQ | IFLIB_SINGLE_IRQ_RX_ONLY,
294
295 /*
296 * Number of receive queues per receive queue set, with associated
297 * descriptor settings for each.
298 */
299 .isc_nrxqs = 3,
300 .isc_nfl = 2, /* one free list for each receive command queue */
301 .isc_nrxd_min = {VMXNET3_MIN_RX_NDESC, VMXNET3_MIN_RX_NDESC, VMXNET3_MIN_RX_NDESC},
302 .isc_nrxd_max = {VMXNET3_MAX_RX_NDESC, VMXNET3_MAX_RX_NDESC, VMXNET3_MAX_RX_NDESC},
303 .isc_nrxd_default = {VMXNET3_DEF_RX_NDESC, VMXNET3_DEF_RX_NDESC, VMXNET3_DEF_RX_NDESC},
304
305 /*
306 * Number of transmit queues per transmit queue set, with associated
307 * descriptor settings for each.
308 */
309 .isc_ntxqs = 2,
310 .isc_ntxd_min = {VMXNET3_MIN_TX_NDESC, VMXNET3_MIN_TX_NDESC},
311 .isc_ntxd_max = {VMXNET3_MAX_TX_NDESC, VMXNET3_MAX_TX_NDESC},
312 .isc_ntxd_default = {VMXNET3_DEF_TX_NDESC, VMXNET3_DEF_TX_NDESC},
313 };
314
315 static void *
vmxnet3_register(device_t dev)316 vmxnet3_register(device_t dev)
317 {
318 return (&vmxnet3_sctx_init);
319 }
320
321 static int
trunc_powerof2(int val)322 trunc_powerof2(int val)
323 {
324
325 return (1U << (fls(val) - 1));
326 }
327
328 static int
vmxnet3_attach_pre(if_ctx_t ctx)329 vmxnet3_attach_pre(if_ctx_t ctx)
330 {
331 device_t dev;
332 if_softc_ctx_t scctx;
333 struct vmxnet3_softc *sc;
334 uint32_t intr_config;
335 int error;
336
337 dev = iflib_get_dev(ctx);
338 sc = iflib_get_softc(ctx);
339 sc->vmx_dev = dev;
340 sc->vmx_ctx = ctx;
341 sc->vmx_sctx = iflib_get_sctx(ctx);
342 sc->vmx_scctx = iflib_get_softc_ctx(ctx);
343 sc->vmx_ifp = iflib_get_ifp(ctx);
344 sc->vmx_media = iflib_get_media(ctx);
345 scctx = sc->vmx_scctx;
346
347 scctx->isc_tx_nsegments = VMXNET3_TX_MAXSEGS;
348 scctx->isc_tx_tso_segments_max = VMXNET3_TX_MAXSEGS;
349 /* isc_tx_tso_size_max doesn't include possible vlan header */
350 scctx->isc_tx_tso_size_max = VMXNET3_TSO_MAXSIZE;
351 scctx->isc_tx_tso_segsize_max = VMXNET3_TX_MAXSEGSIZE;
352 scctx->isc_txrx = &vmxnet3_txrx;
353
354 /* If 0, the iflib tunable was not set, so set to the default */
355 if (scctx->isc_nrxqsets == 0)
356 scctx->isc_nrxqsets = VMXNET3_DEF_RX_QUEUES;
357 scctx->isc_nrxqsets = trunc_powerof2(scctx->isc_nrxqsets);
358 scctx->isc_nrxqsets_max = min(VMXNET3_MAX_RX_QUEUES, mp_ncpus);
359 scctx->isc_nrxqsets_max = trunc_powerof2(scctx->isc_nrxqsets_max);
360
361 /* If 0, the iflib tunable was not set, so set to the default */
362 if (scctx->isc_ntxqsets == 0)
363 scctx->isc_ntxqsets = VMXNET3_DEF_TX_QUEUES;
364 scctx->isc_ntxqsets = trunc_powerof2(scctx->isc_ntxqsets);
365 scctx->isc_ntxqsets_max = min(VMXNET3_MAX_TX_QUEUES, mp_ncpus);
366 scctx->isc_ntxqsets_max = trunc_powerof2(scctx->isc_ntxqsets_max);
367
368 /*
369 * Enforce that the transmit completion queue descriptor count is
370 * the same as the transmit command queue descriptor count.
371 */
372 scctx->isc_ntxd[0] = scctx->isc_ntxd[1];
373 scctx->isc_txqsizes[0] =
374 sizeof(struct vmxnet3_txcompdesc) * scctx->isc_ntxd[0];
375 scctx->isc_txqsizes[1] =
376 sizeof(struct vmxnet3_txdesc) * scctx->isc_ntxd[1];
377
378 /*
379 * Enforce that the receive completion queue descriptor count is the
380 * sum of the receive command queue descriptor counts, and that the
381 * second receive command queue descriptor count is the same as the
382 * first one.
383 */
384 scctx->isc_nrxd[2] = scctx->isc_nrxd[1];
385 scctx->isc_nrxd[0] = scctx->isc_nrxd[1] + scctx->isc_nrxd[2];
386 scctx->isc_rxqsizes[0] =
387 sizeof(struct vmxnet3_rxcompdesc) * scctx->isc_nrxd[0];
388 scctx->isc_rxqsizes[1] =
389 sizeof(struct vmxnet3_rxdesc) * scctx->isc_nrxd[1];
390 scctx->isc_rxqsizes[2] =
391 sizeof(struct vmxnet3_rxdesc) * scctx->isc_nrxd[2];
392
393 /*
394 * Initialize the max frame size and descriptor queue buffer
395 * sizes.
396 */
397 vmxnet3_mtu_set(ctx, if_getmtu(sc->vmx_ifp));
398
399 scctx->isc_rss_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE;
400
401 /* Map PCI BARs */
402 error = vmxnet3_alloc_resources(sc);
403 if (error)
404 goto fail;
405
406 /* Check device versions */
407 error = vmxnet3_check_version(sc);
408 if (error)
409 goto fail;
410
411 /*
412 * The interrupt mode can be set in the hypervisor configuration via
413 * the parameter ethernet<N>.intrMode.
414 */
415 intr_config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG);
416 sc->vmx_intr_mask_mode = (intr_config >> 2) & 0x03;
417
418 /*
419 * Configure the softc context to attempt to configure the interrupt
420 * mode now indicated by intr_config. iflib will follow the usual
421 * fallback path MSI-X -> MSI -> LEGACY, starting at the configured
422 * starting mode.
423 */
424 switch (intr_config & 0x03) {
425 case VMXNET3_IT_AUTO:
426 case VMXNET3_IT_MSIX:
427 scctx->isc_msix_bar = pci_msix_table_bar(dev);
428 break;
429 case VMXNET3_IT_MSI:
430 scctx->isc_msix_bar = -1;
431 scctx->isc_disable_msix = 1;
432 break;
433 case VMXNET3_IT_LEGACY:
434 scctx->isc_msix_bar = 0;
435 break;
436 }
437
438 scctx->isc_tx_csum_flags = VMXNET3_CSUM_ALL_OFFLOAD;
439 scctx->isc_capabilities = scctx->isc_capenable =
440 IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 |
441 IFCAP_TSO4 | IFCAP_TSO6 |
442 IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 |
443 IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING |
444 IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWTSO |
445 IFCAP_JUMBO_MTU;
446
447 /* These capabilities are not enabled by default. */
448 scctx->isc_capabilities |= IFCAP_LRO | IFCAP_VLAN_HWFILTER;
449
450 vmxnet3_get_lladdr(sc);
451 iflib_set_mac(ctx, sc->vmx_lladdr);
452
453 return (0);
454 fail:
455 /*
456 * We must completely clean up anything allocated above as iflib
457 * will not invoke any other driver entry points as a result of this
458 * failure.
459 */
460 vmxnet3_free_resources(sc);
461
462 return (error);
463 }
464
465 static int
vmxnet3_msix_intr_assign(if_ctx_t ctx,int msix)466 vmxnet3_msix_intr_assign(if_ctx_t ctx, int msix)
467 {
468 struct vmxnet3_softc *sc;
469 if_softc_ctx_t scctx;
470 struct vmxnet3_rxqueue *rxq;
471 int error;
472 int i;
473 char irq_name[16];
474
475 sc = iflib_get_softc(ctx);
476 scctx = sc->vmx_scctx;
477
478 for (i = 0; i < scctx->isc_nrxqsets; i++) {
479 snprintf(irq_name, sizeof(irq_name), "rxq%d", i);
480
481 rxq = &sc->vmx_rxq[i];
482 error = iflib_irq_alloc_generic(ctx, &rxq->vxrxq_irq, i + 1,
483 IFLIB_INTR_RXTX, vmxnet3_rxq_intr, rxq, i, irq_name);
484 if (error) {
485 device_printf(iflib_get_dev(ctx),
486 "Failed to register rxq %d interrupt handler\n", i);
487 return (error);
488 }
489 }
490
491 for (i = 0; i < scctx->isc_ntxqsets; i++) {
492 snprintf(irq_name, sizeof(irq_name), "txq%d", i);
493
494 /*
495 * Don't provide the corresponding rxq irq for reference -
496 * we want the transmit task to be attached to a task queue
497 * that is different from the one used by the corresponding
498 * rxq irq. That is because the TX doorbell writes are very
499 * expensive as virtualized MMIO operations, so we want to
500 * be able to defer them to another core when possible so
501 * that they don't steal receive processing cycles during
502 * stack turnarounds like TCP ACK generation. The other
503 * piece to this approach is enabling the iflib abdicate
504 * option (currently via an interface-specific
505 * tunable/sysctl).
506 */
507 iflib_softirq_alloc_generic(ctx, NULL, IFLIB_INTR_TX, NULL, i,
508 irq_name);
509 }
510
511 error = iflib_irq_alloc_generic(ctx, &sc->vmx_event_intr_irq,
512 scctx->isc_nrxqsets + 1, IFLIB_INTR_ADMIN, vmxnet3_event_intr, sc, 0,
513 "event");
514 if (error) {
515 device_printf(iflib_get_dev(ctx),
516 "Failed to register event interrupt handler\n");
517 return (error);
518 }
519
520 return (0);
521 }
522
523 static void
vmxnet3_free_irqs(struct vmxnet3_softc * sc)524 vmxnet3_free_irqs(struct vmxnet3_softc *sc)
525 {
526 if_softc_ctx_t scctx;
527 struct vmxnet3_rxqueue *rxq;
528 int i;
529
530 scctx = sc->vmx_scctx;
531
532 for (i = 0; i < scctx->isc_nrxqsets; i++) {
533 rxq = &sc->vmx_rxq[i];
534 iflib_irq_free(sc->vmx_ctx, &rxq->vxrxq_irq);
535 }
536
537 iflib_irq_free(sc->vmx_ctx, &sc->vmx_event_intr_irq);
538 }
539
540 static int
vmxnet3_attach_post(if_ctx_t ctx)541 vmxnet3_attach_post(if_ctx_t ctx)
542 {
543 device_t dev;
544 if_softc_ctx_t scctx;
545 struct vmxnet3_softc *sc;
546 int error;
547
548 dev = iflib_get_dev(ctx);
549 scctx = iflib_get_softc_ctx(ctx);
550 sc = iflib_get_softc(ctx);
551
552 if (scctx->isc_nrxqsets > 1)
553 sc->vmx_flags |= VMXNET3_FLAG_RSS;
554
555 error = vmxnet3_alloc_data(sc);
556 if (error)
557 goto fail;
558
559 vmxnet3_set_interrupt_idx(sc);
560 vmxnet3_setup_sysctl(sc);
561
562 ifmedia_add(sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL);
563 ifmedia_set(sc->vmx_media, IFM_ETHER | IFM_AUTO);
564
565 fail:
566 return (error);
567 }
568
569 static int
vmxnet3_detach(if_ctx_t ctx)570 vmxnet3_detach(if_ctx_t ctx)
571 {
572 struct vmxnet3_softc *sc;
573
574 sc = iflib_get_softc(ctx);
575
576 vmxnet3_free_irqs(sc);
577 vmxnet3_free_data(sc);
578 vmxnet3_free_resources(sc);
579
580 return (0);
581 }
582
583 static int
vmxnet3_shutdown(if_ctx_t ctx)584 vmxnet3_shutdown(if_ctx_t ctx)
585 {
586
587 return (0);
588 }
589
590 static int
vmxnet3_suspend(if_ctx_t ctx)591 vmxnet3_suspend(if_ctx_t ctx)
592 {
593
594 return (0);
595 }
596
597 static int
vmxnet3_resume(if_ctx_t ctx)598 vmxnet3_resume(if_ctx_t ctx)
599 {
600
601 return (0);
602 }
603
604 static int
vmxnet3_alloc_resources(struct vmxnet3_softc * sc)605 vmxnet3_alloc_resources(struct vmxnet3_softc *sc)
606 {
607 device_t dev;
608 int rid;
609
610 dev = sc->vmx_dev;
611
612 rid = PCIR_BAR(0);
613 sc->vmx_res0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
614 RF_ACTIVE);
615 if (sc->vmx_res0 == NULL) {
616 device_printf(dev,
617 "could not map BAR0 memory\n");
618 return (ENXIO);
619 }
620
621 sc->vmx_iot0 = rman_get_bustag(sc->vmx_res0);
622 sc->vmx_ioh0 = rman_get_bushandle(sc->vmx_res0);
623
624 rid = PCIR_BAR(1);
625 sc->vmx_res1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
626 RF_ACTIVE);
627 if (sc->vmx_res1 == NULL) {
628 device_printf(dev,
629 "could not map BAR1 memory\n");
630 return (ENXIO);
631 }
632
633 sc->vmx_iot1 = rman_get_bustag(sc->vmx_res1);
634 sc->vmx_ioh1 = rman_get_bushandle(sc->vmx_res1);
635
636 return (0);
637 }
638
639 static void
vmxnet3_free_resources(struct vmxnet3_softc * sc)640 vmxnet3_free_resources(struct vmxnet3_softc *sc)
641 {
642 device_t dev;
643
644 dev = sc->vmx_dev;
645
646 if (sc->vmx_res0 != NULL) {
647 bus_release_resource(dev, SYS_RES_MEMORY,
648 rman_get_rid(sc->vmx_res0), sc->vmx_res0);
649 sc->vmx_res0 = NULL;
650 }
651
652 if (sc->vmx_res1 != NULL) {
653 bus_release_resource(dev, SYS_RES_MEMORY,
654 rman_get_rid(sc->vmx_res1), sc->vmx_res1);
655 sc->vmx_res1 = NULL;
656 }
657 }
658
659 static int
vmxnet3_check_version(struct vmxnet3_softc * sc)660 vmxnet3_check_version(struct vmxnet3_softc *sc)
661 {
662 device_t dev;
663 uint32_t version;
664
665 dev = sc->vmx_dev;
666
667 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS);
668 if ((version & 0x01) == 0) {
669 device_printf(dev, "unsupported hardware version %#x\n",
670 version);
671 return (ENOTSUP);
672 }
673 vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1);
674
675 version = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS);
676 if ((version & 0x01) == 0) {
677 device_printf(dev, "unsupported UPT version %#x\n", version);
678 return (ENOTSUP);
679 }
680 vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1);
681
682 return (0);
683 }
684
685 static void
vmxnet3_set_interrupt_idx(struct vmxnet3_softc * sc)686 vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc)
687 {
688 if_softc_ctx_t scctx;
689 struct vmxnet3_txqueue *txq;
690 struct vmxnet3_txq_shared *txs;
691 struct vmxnet3_rxqueue *rxq;
692 struct vmxnet3_rxq_shared *rxs;
693 int intr_idx;
694 int i;
695
696 scctx = sc->vmx_scctx;
697
698 /*
699 * There is always one interrupt per receive queue, assigned
700 * starting with the first interrupt. When there is only one
701 * interrupt available, the event interrupt shares the receive queue
702 * interrupt, otherwise it uses the interrupt following the last
703 * receive queue interrupt. Transmit queues are not assigned
704 * interrupts, so they are given indexes beyond the indexes that
705 * correspond to the real interrupts.
706 */
707
708 /* The event interrupt is always the last vector. */
709 sc->vmx_event_intr_idx = scctx->isc_vectors - 1;
710
711 intr_idx = 0;
712 for (i = 0; i < scctx->isc_nrxqsets; i++, intr_idx++) {
713 rxq = &sc->vmx_rxq[i];
714 rxs = rxq->vxrxq_rs;
715 rxq->vxrxq_intr_idx = intr_idx;
716 rxs->intr_idx = rxq->vxrxq_intr_idx;
717 }
718
719 /*
720 * Assign the tx queues interrupt indexes above what we are actually
721 * using. These interrupts will never be enabled.
722 */
723 intr_idx = scctx->isc_vectors;
724 for (i = 0; i < scctx->isc_ntxqsets; i++, intr_idx++) {
725 txq = &sc->vmx_txq[i];
726 txs = txq->vxtxq_ts;
727 txq->vxtxq_intr_idx = intr_idx;
728 txs->intr_idx = txq->vxtxq_intr_idx;
729 }
730 }
731
732 static int
vmxnet3_queues_shared_alloc(struct vmxnet3_softc * sc)733 vmxnet3_queues_shared_alloc(struct vmxnet3_softc *sc)
734 {
735 if_softc_ctx_t scctx;
736 int size;
737 int error;
738
739 scctx = sc->vmx_scctx;
740
741 /*
742 * The txq and rxq shared data areas must be allocated contiguously
743 * as vmxnet3_driver_shared contains only a single address member
744 * for the shared queue data area.
745 */
746 size = scctx->isc_ntxqsets * sizeof(struct vmxnet3_txq_shared) +
747 scctx->isc_nrxqsets * sizeof(struct vmxnet3_rxq_shared);
748 error = iflib_dma_alloc_align(sc->vmx_ctx, size, 128, &sc->vmx_qs_dma, 0);
749 if (error) {
750 device_printf(sc->vmx_dev, "cannot alloc queue shared memory\n");
751 return (error);
752 }
753
754 return (0);
755 }
756
757 static void
vmxnet3_init_txq(struct vmxnet3_softc * sc,int q)758 vmxnet3_init_txq(struct vmxnet3_softc *sc, int q)
759 {
760 struct vmxnet3_txqueue *txq;
761 struct vmxnet3_comp_ring *txc;
762 struct vmxnet3_txring *txr;
763 if_softc_ctx_t scctx;
764
765 txq = &sc->vmx_txq[q];
766 txc = &txq->vxtxq_comp_ring;
767 txr = &txq->vxtxq_cmd_ring;
768 scctx = sc->vmx_scctx;
769
770 snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d",
771 device_get_nameunit(sc->vmx_dev), q);
772
773 txq->vxtxq_sc = sc;
774 txq->vxtxq_id = q;
775 txc->vxcr_ndesc = scctx->isc_ntxd[0];
776 txr->vxtxr_ndesc = scctx->isc_ntxd[1];
777 }
778
779 static int
vmxnet3_tx_queues_alloc(if_ctx_t ctx,caddr_t * vaddrs,uint64_t * paddrs,int ntxqs,int ntxqsets)780 vmxnet3_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
781 int ntxqs, int ntxqsets)
782 {
783 struct vmxnet3_softc *sc;
784 int q;
785 int error;
786 caddr_t kva;
787
788 sc = iflib_get_softc(ctx);
789
790 /* Allocate the array of transmit queues */
791 sc->vmx_txq = malloc(sizeof(struct vmxnet3_txqueue) *
792 ntxqsets, M_DEVBUF, M_NOWAIT | M_ZERO);
793 if (sc->vmx_txq == NULL)
794 return (ENOMEM);
795
796 /* Initialize driver state for each transmit queue */
797 for (q = 0; q < ntxqsets; q++)
798 vmxnet3_init_txq(sc, q);
799
800 /*
801 * Allocate queue state that is shared with the device. This check
802 * and call is performed in both vmxnet3_tx_queues_alloc() and
803 * vmxnet3_rx_queues_alloc() so that we don't have to care which
804 * order iflib invokes those routines in.
805 */
806 if (sc->vmx_qs_dma.idi_size == 0) {
807 error = vmxnet3_queues_shared_alloc(sc);
808 if (error)
809 return (error);
810 }
811
812 kva = sc->vmx_qs_dma.idi_vaddr;
813 for (q = 0; q < ntxqsets; q++) {
814 sc->vmx_txq[q].vxtxq_ts = (struct vmxnet3_txq_shared *) kva;
815 kva += sizeof(struct vmxnet3_txq_shared);
816 }
817
818 /* Record descriptor ring vaddrs and paddrs */
819 for (q = 0; q < ntxqsets; q++) {
820 struct vmxnet3_txqueue *txq;
821 struct vmxnet3_txring *txr;
822 struct vmxnet3_comp_ring *txc;
823
824 txq = &sc->vmx_txq[q];
825 txc = &txq->vxtxq_comp_ring;
826 txr = &txq->vxtxq_cmd_ring;
827
828 /* Completion ring */
829 txc->vxcr_u.txcd =
830 (struct vmxnet3_txcompdesc *) vaddrs[q * ntxqs + 0];
831 txc->vxcr_paddr = paddrs[q * ntxqs + 0];
832
833 /* Command ring */
834 txr->vxtxr_txd =
835 (struct vmxnet3_txdesc *) vaddrs[q * ntxqs + 1];
836 txr->vxtxr_paddr = paddrs[q * ntxqs + 1];
837 }
838
839 return (0);
840 }
841
842 static void
vmxnet3_init_rxq(struct vmxnet3_softc * sc,int q,int nrxqs)843 vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q, int nrxqs)
844 {
845 struct vmxnet3_rxqueue *rxq;
846 struct vmxnet3_comp_ring *rxc;
847 struct vmxnet3_rxring *rxr;
848 if_softc_ctx_t scctx;
849 int i;
850
851 rxq = &sc->vmx_rxq[q];
852 rxc = &rxq->vxrxq_comp_ring;
853 scctx = sc->vmx_scctx;
854
855 snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d",
856 device_get_nameunit(sc->vmx_dev), q);
857
858 rxq->vxrxq_sc = sc;
859 rxq->vxrxq_id = q;
860
861 /*
862 * First rxq is the completion queue, so there are nrxqs - 1 command
863 * rings starting at iflib queue id 1.
864 */
865 rxc->vxcr_ndesc = scctx->isc_nrxd[0];
866 for (i = 0; i < nrxqs - 1; i++) {
867 rxr = &rxq->vxrxq_cmd_ring[i];
868 rxr->vxrxr_ndesc = scctx->isc_nrxd[i + 1];
869 }
870 }
871
872 static int
vmxnet3_rx_queues_alloc(if_ctx_t ctx,caddr_t * vaddrs,uint64_t * paddrs,int nrxqs,int nrxqsets)873 vmxnet3_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs,
874 int nrxqs, int nrxqsets)
875 {
876 struct vmxnet3_softc *sc;
877 if_softc_ctx_t scctx;
878 int q;
879 int i;
880 int error;
881 caddr_t kva;
882
883 sc = iflib_get_softc(ctx);
884 scctx = sc->vmx_scctx;
885
886 /* Allocate the array of receive queues */
887 sc->vmx_rxq = malloc(sizeof(struct vmxnet3_rxqueue) *
888 nrxqsets, M_DEVBUF, M_NOWAIT | M_ZERO);
889 if (sc->vmx_rxq == NULL)
890 return (ENOMEM);
891
892 /* Initialize driver state for each receive queue */
893 for (q = 0; q < nrxqsets; q++)
894 vmxnet3_init_rxq(sc, q, nrxqs);
895
896 /*
897 * Allocate queue state that is shared with the device. This check
898 * and call is performed in both vmxnet3_tx_queues_alloc() and
899 * vmxnet3_rx_queues_alloc() so that we don't have to care which
900 * order iflib invokes those routines in.
901 */
902 if (sc->vmx_qs_dma.idi_size == 0) {
903 error = vmxnet3_queues_shared_alloc(sc);
904 if (error)
905 return (error);
906 }
907
908 kva = sc->vmx_qs_dma.idi_vaddr +
909 scctx->isc_ntxqsets * sizeof(struct vmxnet3_txq_shared);
910 for (q = 0; q < nrxqsets; q++) {
911 sc->vmx_rxq[q].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva;
912 kva += sizeof(struct vmxnet3_rxq_shared);
913 }
914
915 /* Record descriptor ring vaddrs and paddrs */
916 for (q = 0; q < nrxqsets; q++) {
917 struct vmxnet3_rxqueue *rxq;
918 struct vmxnet3_rxring *rxr;
919 struct vmxnet3_comp_ring *rxc;
920
921 rxq = &sc->vmx_rxq[q];
922 rxc = &rxq->vxrxq_comp_ring;
923
924 /* Completion ring */
925 rxc->vxcr_u.rxcd =
926 (struct vmxnet3_rxcompdesc *) vaddrs[q * nrxqs + 0];
927 rxc->vxcr_paddr = paddrs[q * nrxqs + 0];
928
929 /* Command ring(s) */
930 for (i = 0; i < nrxqs - 1; i++) {
931 rxr = &rxq->vxrxq_cmd_ring[i];
932
933 rxr->vxrxr_rxd =
934 (struct vmxnet3_rxdesc *) vaddrs[q * nrxqs + 1 + i];
935 rxr->vxrxr_paddr = paddrs[q * nrxqs + 1 + i];
936 }
937 }
938
939 return (0);
940 }
941
942 static void
vmxnet3_queues_free(if_ctx_t ctx)943 vmxnet3_queues_free(if_ctx_t ctx)
944 {
945 struct vmxnet3_softc *sc;
946
947 sc = iflib_get_softc(ctx);
948
949 /* Free queue state area that is shared with the device */
950 if (sc->vmx_qs_dma.idi_size != 0) {
951 iflib_dma_free(&sc->vmx_qs_dma);
952 sc->vmx_qs_dma.idi_size = 0;
953 }
954
955 /* Free array of receive queues */
956 if (sc->vmx_rxq != NULL) {
957 free(sc->vmx_rxq, M_DEVBUF);
958 sc->vmx_rxq = NULL;
959 }
960
961 /* Free array of transmit queues */
962 if (sc->vmx_txq != NULL) {
963 free(sc->vmx_txq, M_DEVBUF);
964 sc->vmx_txq = NULL;
965 }
966 }
967
968 static int
vmxnet3_alloc_shared_data(struct vmxnet3_softc * sc)969 vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc)
970 {
971 device_t dev;
972 size_t size;
973 int error;
974
975 dev = sc->vmx_dev;
976
977 /* Top level state structure shared with the device */
978 size = sizeof(struct vmxnet3_driver_shared);
979 error = iflib_dma_alloc_align(sc->vmx_ctx, size, 1, &sc->vmx_ds_dma, 0);
980 if (error) {
981 device_printf(dev, "cannot alloc shared memory\n");
982 return (error);
983 }
984 sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.idi_vaddr;
985
986 /* RSS table state shared with the device */
987 if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
988 size = sizeof(struct vmxnet3_rss_shared);
989 error = iflib_dma_alloc_align(sc->vmx_ctx, size, 128,
990 &sc->vmx_rss_dma, 0);
991 if (error) {
992 device_printf(dev, "cannot alloc rss shared memory\n");
993 return (error);
994 }
995 sc->vmx_rss =
996 (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.idi_vaddr;
997 }
998
999 return (0);
1000 }
1001
1002 static void
vmxnet3_free_shared_data(struct vmxnet3_softc * sc)1003 vmxnet3_free_shared_data(struct vmxnet3_softc *sc)
1004 {
1005
1006 /* Free RSS table state shared with the device */
1007 if (sc->vmx_rss != NULL) {
1008 iflib_dma_free(&sc->vmx_rss_dma);
1009 sc->vmx_rss = NULL;
1010 }
1011
1012 /* Free top level state structure shared with the device */
1013 if (sc->vmx_ds != NULL) {
1014 iflib_dma_free(&sc->vmx_ds_dma);
1015 sc->vmx_ds = NULL;
1016 }
1017 }
1018
1019 static int
vmxnet3_alloc_mcast_table(struct vmxnet3_softc * sc)1020 vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc)
1021 {
1022 int error;
1023
1024 /* Multicast table state shared with the device */
1025 error = iflib_dma_alloc_align(sc->vmx_ctx,
1026 VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN, 32, &sc->vmx_mcast_dma, 0);
1027 if (error)
1028 device_printf(sc->vmx_dev, "unable to alloc multicast table\n");
1029 else
1030 sc->vmx_mcast = sc->vmx_mcast_dma.idi_vaddr;
1031
1032 return (error);
1033 }
1034
1035 static void
vmxnet3_free_mcast_table(struct vmxnet3_softc * sc)1036 vmxnet3_free_mcast_table(struct vmxnet3_softc *sc)
1037 {
1038
1039 /* Free multicast table state shared with the device */
1040 if (sc->vmx_mcast != NULL) {
1041 iflib_dma_free(&sc->vmx_mcast_dma);
1042 sc->vmx_mcast = NULL;
1043 }
1044 }
1045
1046 static void
vmxnet3_init_shared_data(struct vmxnet3_softc * sc)1047 vmxnet3_init_shared_data(struct vmxnet3_softc *sc)
1048 {
1049 struct vmxnet3_driver_shared *ds;
1050 if_shared_ctx_t sctx;
1051 if_softc_ctx_t scctx;
1052 struct vmxnet3_txqueue *txq;
1053 struct vmxnet3_txq_shared *txs;
1054 struct vmxnet3_rxqueue *rxq;
1055 struct vmxnet3_rxq_shared *rxs;
1056 int i;
1057
1058 ds = sc->vmx_ds;
1059 sctx = sc->vmx_sctx;
1060 scctx = sc->vmx_scctx;
1061
1062 /*
1063 * Initialize fields of the shared data that remains the same across
1064 * reinits. Note the shared data is zero'd when allocated.
1065 */
1066
1067 ds->magic = VMXNET3_REV1_MAGIC;
1068
1069 /* DriverInfo */
1070 ds->version = VMXNET3_DRIVER_VERSION;
1071 ds->guest = VMXNET3_GOS_FREEBSD |
1072 #ifdef __LP64__
1073 VMXNET3_GOS_64BIT;
1074 #else
1075 VMXNET3_GOS_32BIT;
1076 #endif
1077 ds->vmxnet3_revision = 1;
1078 ds->upt_version = 1;
1079
1080 /* Misc. conf */
1081 ds->driver_data = vtophys(sc);
1082 ds->driver_data_len = sizeof(struct vmxnet3_softc);
1083 ds->queue_shared = sc->vmx_qs_dma.idi_paddr;
1084 ds->queue_shared_len = sc->vmx_qs_dma.idi_size;
1085 ds->nrxsg_max = IFLIB_MAX_RX_SEGS;
1086
1087 /* RSS conf */
1088 if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1089 ds->rss.version = 1;
1090 ds->rss.paddr = sc->vmx_rss_dma.idi_paddr;
1091 ds->rss.len = sc->vmx_rss_dma.idi_size;
1092 }
1093
1094 /* Interrupt control. */
1095 ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO;
1096 /*
1097 * Total number of interrupt indexes we are using in the shared
1098 * config data, even though we don't actually allocate interrupt
1099 * resources for the tx queues. Some versions of the device will
1100 * fail to initialize successfully if interrupt indexes are used in
1101 * the shared config that exceed the number of interrupts configured
1102 * here.
1103 */
1104 ds->nintr = (scctx->isc_vectors == 1) ?
1105 2 : (scctx->isc_nrxqsets + scctx->isc_ntxqsets + 1);
1106 ds->evintr = sc->vmx_event_intr_idx;
1107 ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL;
1108
1109 for (i = 0; i < ds->nintr; i++)
1110 ds->modlevel[i] = UPT1_IMOD_ADAPTIVE;
1111
1112 /* Receive filter. */
1113 ds->mcast_table = sc->vmx_mcast_dma.idi_paddr;
1114 ds->mcast_tablelen = sc->vmx_mcast_dma.idi_size;
1115
1116 /* Tx queues */
1117 for (i = 0; i < scctx->isc_ntxqsets; i++) {
1118 txq = &sc->vmx_txq[i];
1119 txs = txq->vxtxq_ts;
1120
1121 txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_paddr;
1122 txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc;
1123 txs->comp_ring = txq->vxtxq_comp_ring.vxcr_paddr;
1124 txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc;
1125 txs->driver_data = vtophys(txq);
1126 txs->driver_data_len = sizeof(struct vmxnet3_txqueue);
1127 }
1128
1129 /* Rx queues */
1130 for (i = 0; i < scctx->isc_nrxqsets; i++) {
1131 rxq = &sc->vmx_rxq[i];
1132 rxs = rxq->vxrxq_rs;
1133
1134 rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_paddr;
1135 rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc;
1136 rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_paddr;
1137 rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc;
1138 rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_paddr;
1139 rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc;
1140 rxs->driver_data = vtophys(rxq);
1141 rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue);
1142 }
1143 }
1144
1145 static void
vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc * sc)1146 vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc)
1147 {
1148 /*
1149 * Use the same key as the Linux driver until FreeBSD can do
1150 * RSS (presumably Toeplitz) in software.
1151 */
1152 static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {
1153 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
1154 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
1155 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
1156 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
1157 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
1158 };
1159
1160 struct vmxnet3_driver_shared *ds;
1161 if_softc_ctx_t scctx;
1162 struct vmxnet3_rss_shared *rss;
1163 #ifdef RSS
1164 uint8_t rss_algo;
1165 #endif
1166 int i;
1167
1168 ds = sc->vmx_ds;
1169 scctx = sc->vmx_scctx;
1170 rss = sc->vmx_rss;
1171
1172 rss->hash_type =
1173 UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 |
1174 UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6;
1175 rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ;
1176 rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE;
1177 rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE;
1178 #ifdef RSS
1179 /*
1180 * If the software RSS is configured to anything else other than
1181 * Toeplitz, then just do Toeplitz in "hardware" for the sake of
1182 * the packet distribution, but report the hash as opaque to
1183 * disengage from the software RSS.
1184 */
1185 rss_algo = rss_gethashalgo();
1186 if (rss_algo == RSS_HASH_TOEPLITZ) {
1187 rss_getkey(rss->hash_key);
1188 for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++) {
1189 rss->ind_table[i] = rss_get_indirection_to_bucket(i) %
1190 scctx->isc_nrxqsets;
1191 }
1192 sc->vmx_flags |= VMXNET3_FLAG_SOFT_RSS;
1193 } else
1194 #endif
1195 {
1196 memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE);
1197 for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++)
1198 rss->ind_table[i] = i % scctx->isc_nrxqsets;
1199 sc->vmx_flags &= ~VMXNET3_FLAG_SOFT_RSS;
1200 }
1201 }
1202
1203 static void
vmxnet3_reinit_shared_data(struct vmxnet3_softc * sc)1204 vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc)
1205 {
1206 struct ifnet *ifp;
1207 struct vmxnet3_driver_shared *ds;
1208 if_softc_ctx_t scctx;
1209
1210 ifp = sc->vmx_ifp;
1211 ds = sc->vmx_ds;
1212 scctx = sc->vmx_scctx;
1213
1214 ds->mtu = ifp->if_mtu;
1215 ds->ntxqueue = scctx->isc_ntxqsets;
1216 ds->nrxqueue = scctx->isc_nrxqsets;
1217
1218 ds->upt_features = 0;
1219 if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6))
1220 ds->upt_features |= UPT1_F_CSUM;
1221 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1222 ds->upt_features |= UPT1_F_VLAN;
1223 if (ifp->if_capenable & IFCAP_LRO)
1224 ds->upt_features |= UPT1_F_LRO;
1225
1226 if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
1227 ds->upt_features |= UPT1_F_RSS;
1228 vmxnet3_reinit_rss_shared_data(sc);
1229 }
1230
1231 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.idi_paddr);
1232 vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH,
1233 (uint64_t) sc->vmx_ds_dma.idi_paddr >> 32);
1234 }
1235
1236 static int
vmxnet3_alloc_data(struct vmxnet3_softc * sc)1237 vmxnet3_alloc_data(struct vmxnet3_softc *sc)
1238 {
1239 int error;
1240
1241 error = vmxnet3_alloc_shared_data(sc);
1242 if (error)
1243 return (error);
1244
1245 error = vmxnet3_alloc_mcast_table(sc);
1246 if (error)
1247 return (error);
1248
1249 vmxnet3_init_shared_data(sc);
1250
1251 return (0);
1252 }
1253
1254 static void
vmxnet3_free_data(struct vmxnet3_softc * sc)1255 vmxnet3_free_data(struct vmxnet3_softc *sc)
1256 {
1257
1258 vmxnet3_free_mcast_table(sc);
1259 vmxnet3_free_shared_data(sc);
1260 }
1261
1262 static void
vmxnet3_evintr(struct vmxnet3_softc * sc)1263 vmxnet3_evintr(struct vmxnet3_softc *sc)
1264 {
1265 device_t dev;
1266 struct vmxnet3_txq_shared *ts;
1267 struct vmxnet3_rxq_shared *rs;
1268 uint32_t event;
1269
1270 dev = sc->vmx_dev;
1271
1272 /* Clear events. */
1273 event = sc->vmx_ds->event;
1274 vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event);
1275
1276 if (event & VMXNET3_EVENT_LINK)
1277 vmxnet3_link_status(sc);
1278
1279 if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) {
1280 vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS);
1281 ts = sc->vmx_txq[0].vxtxq_ts;
1282 if (ts->stopped != 0)
1283 device_printf(dev, "Tx queue error %#x\n", ts->error);
1284 rs = sc->vmx_rxq[0].vxrxq_rs;
1285 if (rs->stopped != 0)
1286 device_printf(dev, "Rx queue error %#x\n", rs->error);
1287
1288 /* XXX - rely on liflib watchdog to reset us? */
1289 device_printf(dev, "Rx/Tx queue error event ... "
1290 "waiting for iflib watchdog reset\n");
1291 }
1292
1293 if (event & VMXNET3_EVENT_DIC)
1294 device_printf(dev, "device implementation change event\n");
1295 if (event & VMXNET3_EVENT_DEBUG)
1296 device_printf(dev, "debug event\n");
1297 }
1298
1299 static int
vmxnet3_isc_txd_encap(void * vsc,if_pkt_info_t pi)1300 vmxnet3_isc_txd_encap(void *vsc, if_pkt_info_t pi)
1301 {
1302 struct vmxnet3_softc *sc;
1303 struct vmxnet3_txqueue *txq;
1304 struct vmxnet3_txring *txr;
1305 struct vmxnet3_txdesc *txd, *sop;
1306 bus_dma_segment_t *segs;
1307 int nsegs;
1308 int pidx;
1309 int hdrlen;
1310 int i;
1311 int gen;
1312
1313 sc = vsc;
1314 txq = &sc->vmx_txq[pi->ipi_qsidx];
1315 txr = &txq->vxtxq_cmd_ring;
1316 segs = pi->ipi_segs;
1317 nsegs = pi->ipi_nsegs;
1318 pidx = pi->ipi_pidx;
1319
1320 KASSERT(nsegs <= VMXNET3_TX_MAXSEGS,
1321 ("%s: packet with too many segments %d", __func__, nsegs));
1322
1323 sop = &txr->vxtxr_txd[pidx];
1324 gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */
1325
1326 for (i = 0; i < nsegs; i++) {
1327 txd = &txr->vxtxr_txd[pidx];
1328
1329 txd->addr = segs[i].ds_addr;
1330 txd->len = segs[i].ds_len;
1331 txd->gen = gen;
1332 txd->dtype = 0;
1333 txd->offload_mode = VMXNET3_OM_NONE;
1334 txd->offload_pos = 0;
1335 txd->hlen = 0;
1336 txd->eop = 0;
1337 txd->compreq = 0;
1338 txd->vtag_mode = 0;
1339 txd->vtag = 0;
1340
1341 if (++pidx == txr->vxtxr_ndesc) {
1342 pidx = 0;
1343 txr->vxtxr_gen ^= 1;
1344 }
1345 gen = txr->vxtxr_gen;
1346 }
1347 txd->eop = 1;
1348 txd->compreq = !!(pi->ipi_flags & IPI_TX_INTR);
1349 pi->ipi_new_pidx = pidx;
1350
1351 /*
1352 * VLAN
1353 */
1354 if (pi->ipi_mflags & M_VLANTAG) {
1355 sop->vtag_mode = 1;
1356 sop->vtag = pi->ipi_vtag;
1357 }
1358
1359 /*
1360 * TSO and checksum offloads
1361 */
1362 hdrlen = pi->ipi_ehdrlen + pi->ipi_ip_hlen;
1363 if (pi->ipi_csum_flags & CSUM_TSO) {
1364 sop->offload_mode = VMXNET3_OM_TSO;
1365 sop->hlen = hdrlen + pi->ipi_tcp_hlen;
1366 sop->offload_pos = pi->ipi_tso_segsz;
1367 } else if (pi->ipi_csum_flags & (VMXNET3_CSUM_OFFLOAD |
1368 VMXNET3_CSUM_OFFLOAD_IPV6)) {
1369 sop->offload_mode = VMXNET3_OM_CSUM;
1370 sop->hlen = hdrlen;
1371 sop->offload_pos = hdrlen +
1372 ((pi->ipi_ipproto == IPPROTO_TCP) ?
1373 offsetof(struct tcphdr, th_sum) :
1374 offsetof(struct udphdr, uh_sum));
1375 }
1376
1377 /* Finally, change the ownership. */
1378 vmxnet3_barrier(sc, VMXNET3_BARRIER_WR);
1379 sop->gen ^= 1;
1380
1381 return (0);
1382 }
1383
1384 static void
vmxnet3_isc_txd_flush(void * vsc,uint16_t txqid,qidx_t pidx)1385 vmxnet3_isc_txd_flush(void *vsc, uint16_t txqid, qidx_t pidx)
1386 {
1387 struct vmxnet3_softc *sc;
1388 struct vmxnet3_txqueue *txq;
1389
1390 sc = vsc;
1391 txq = &sc->vmx_txq[txqid];
1392
1393 /*
1394 * pidx is what we last set ipi_new_pidx to in
1395 * vmxnet3_isc_txd_encap()
1396 */
1397
1398 /*
1399 * Avoid expensive register updates if the flush request is
1400 * redundant.
1401 */
1402 if (txq->vxtxq_last_flush == pidx)
1403 return;
1404 txq->vxtxq_last_flush = pidx;
1405 vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id), pidx);
1406 }
1407
1408 static int
vmxnet3_isc_txd_credits_update(void * vsc,uint16_t txqid,bool clear)1409 vmxnet3_isc_txd_credits_update(void *vsc, uint16_t txqid, bool clear)
1410 {
1411 struct vmxnet3_softc *sc;
1412 struct vmxnet3_txqueue *txq;
1413 struct vmxnet3_comp_ring *txc;
1414 struct vmxnet3_txcompdesc *txcd;
1415 struct vmxnet3_txring *txr;
1416 int processed;
1417
1418 sc = vsc;
1419 txq = &sc->vmx_txq[txqid];
1420 txc = &txq->vxtxq_comp_ring;
1421 txr = &txq->vxtxq_cmd_ring;
1422
1423 /*
1424 * If clear is true, we need to report the number of TX command ring
1425 * descriptors that have been processed by the device. If clear is
1426 * false, we just need to report whether or not at least one TX
1427 * command ring descriptor has been processed by the device.
1428 */
1429 processed = 0;
1430 for (;;) {
1431 txcd = &txc->vxcr_u.txcd[txc->vxcr_next];
1432 if (txcd->gen != txc->vxcr_gen)
1433 break;
1434 else if (!clear)
1435 return (1);
1436 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
1437
1438 if (++txc->vxcr_next == txc->vxcr_ndesc) {
1439 txc->vxcr_next = 0;
1440 txc->vxcr_gen ^= 1;
1441 }
1442
1443 if (txcd->eop_idx < txr->vxtxr_next)
1444 processed += txr->vxtxr_ndesc -
1445 (txr->vxtxr_next - txcd->eop_idx) + 1;
1446 else
1447 processed += txcd->eop_idx - txr->vxtxr_next + 1;
1448 txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc;
1449 }
1450
1451 return (processed);
1452 }
1453
1454 static int
vmxnet3_isc_rxd_available(void * vsc,uint16_t rxqid,qidx_t idx,qidx_t budget)1455 vmxnet3_isc_rxd_available(void *vsc, uint16_t rxqid, qidx_t idx, qidx_t budget)
1456 {
1457 struct vmxnet3_softc *sc;
1458 struct vmxnet3_rxqueue *rxq;
1459 struct vmxnet3_comp_ring *rxc;
1460 struct vmxnet3_rxcompdesc *rxcd;
1461 int avail;
1462 int completed_gen;
1463 #ifdef INVARIANTS
1464 int expect_sop = 1;
1465 #endif
1466 sc = vsc;
1467 rxq = &sc->vmx_rxq[rxqid];
1468 rxc = &rxq->vxrxq_comp_ring;
1469
1470 avail = 0;
1471 completed_gen = rxc->vxcr_gen;
1472 for (;;) {
1473 rxcd = &rxc->vxcr_u.rxcd[idx];
1474 if (rxcd->gen != completed_gen)
1475 break;
1476 vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
1477
1478 #ifdef INVARIANTS
1479 if (expect_sop)
1480 KASSERT(rxcd->sop, ("%s: expected sop", __func__));
1481 else
1482 KASSERT(!rxcd->sop, ("%s: unexpected sop", __func__));
1483 expect_sop = rxcd->eop;
1484 #endif
1485 if (rxcd->eop && (rxcd->len != 0))
1486 avail++;
1487 if (avail > budget)
1488 break;
1489 if (++idx == rxc->vxcr_ndesc) {
1490 idx = 0;
1491 completed_gen ^= 1;
1492 }
1493 }
1494
1495 return (avail);
1496 }
1497
1498 static int
vmxnet3_isc_rxd_pkt_get(void * vsc,if_rxd_info_t ri)1499 vmxnet3_isc_rxd_pkt_get(void *vsc, if_rxd_info_t ri)
1500 {
1501 struct vmxnet3_softc *sc;
1502 if_softc_ctx_t scctx;
1503 struct vmxnet3_rxqueue *rxq;
1504 struct vmxnet3_comp_ring *rxc;
1505 struct vmxnet3_rxcompdesc *rxcd;
1506 if_rxd_frag_t frag;
1507 int cqidx;
1508 uint16_t total_len;
1509 uint8_t nfrags;
1510 uint8_t i;
1511 uint8_t flid;
1512
1513 sc = vsc;
1514 scctx = sc->vmx_scctx;
1515 rxq = &sc->vmx_rxq[ri->iri_qsidx];
1516 rxc = &rxq->vxrxq_comp_ring;
1517
1518 /*
1519 * Get a single packet starting at the given index in the completion
1520 * queue. That we have been called indicates that
1521 * vmxnet3_isc_rxd_available() has already verified that either
1522 * there is a complete packet available starting at the given index,
1523 * or there are one or more zero length packets starting at the
1524 * given index followed by a complete packet, so no verification of
1525 * ownership of the descriptors (and no associated read barrier) is
1526 * required here.
1527 */
1528 cqidx = ri->iri_cidx;
1529 rxcd = &rxc->vxcr_u.rxcd[cqidx];
1530 while (rxcd->len == 0) {
1531 KASSERT(rxcd->sop && rxcd->eop,
1532 ("%s: zero-length packet without both sop and eop set",
1533 __func__));
1534 rxc->vxcr_zero_length++;
1535 if (++cqidx == rxc->vxcr_ndesc) {
1536 cqidx = 0;
1537 rxc->vxcr_gen ^= 1;
1538 }
1539 rxcd = &rxc->vxcr_u.rxcd[cqidx];
1540 }
1541 KASSERT(rxcd->sop, ("%s: expected sop", __func__));
1542
1543 /*
1544 * RSS and flow ID.
1545 * Types other than M_HASHTYPE_NONE and M_HASHTYPE_OPAQUE_HASH should
1546 * be used only if the software RSS is enabled and it uses the same
1547 * algorithm and the hash key as the "hardware". If the software RSS
1548 * is not enabled, then it's simply pointless to use those types.
1549 * If it's enabled but with different parameters, then hash values will
1550 * not match.
1551 */
1552 ri->iri_flowid = rxcd->rss_hash;
1553 #ifdef RSS
1554 if ((sc->vmx_flags & VMXNET3_FLAG_SOFT_RSS) != 0) {
1555 switch (rxcd->rss_type) {
1556 case VMXNET3_RCD_RSS_TYPE_NONE:
1557 ri->iri_flowid = ri->iri_qsidx;
1558 ri->iri_rsstype = M_HASHTYPE_NONE;
1559 break;
1560 case VMXNET3_RCD_RSS_TYPE_IPV4:
1561 ri->iri_rsstype = M_HASHTYPE_RSS_IPV4;
1562 break;
1563 case VMXNET3_RCD_RSS_TYPE_TCPIPV4:
1564 ri->iri_rsstype = M_HASHTYPE_RSS_TCP_IPV4;
1565 break;
1566 case VMXNET3_RCD_RSS_TYPE_IPV6:
1567 ri->iri_rsstype = M_HASHTYPE_RSS_IPV6;
1568 break;
1569 case VMXNET3_RCD_RSS_TYPE_TCPIPV6:
1570 ri->iri_rsstype = M_HASHTYPE_RSS_TCP_IPV6;
1571 break;
1572 default:
1573 ri->iri_rsstype = M_HASHTYPE_OPAQUE_HASH;
1574 break;
1575 }
1576 } else
1577 #endif
1578 {
1579 switch (rxcd->rss_type) {
1580 case VMXNET3_RCD_RSS_TYPE_NONE:
1581 ri->iri_flowid = ri->iri_qsidx;
1582 ri->iri_rsstype = M_HASHTYPE_NONE;
1583 break;
1584 default:
1585 ri->iri_rsstype = M_HASHTYPE_OPAQUE_HASH;
1586 break;
1587 }
1588 }
1589
1590 /*
1591 * The queue numbering scheme used for rxcd->qid is as follows:
1592 * - All of the command ring 0s are numbered [0, nrxqsets - 1]
1593 * - All of the command ring 1s are numbered [nrxqsets, 2*nrxqsets - 1]
1594 *
1595 * Thus, rxcd->qid less than nrxqsets indicates command ring (and
1596 * flid) 0, and rxcd->qid greater than or equal to nrxqsets
1597 * indicates command ring (and flid) 1.
1598 */
1599 nfrags = 0;
1600 total_len = 0;
1601 do {
1602 rxcd = &rxc->vxcr_u.rxcd[cqidx];
1603 KASSERT(rxcd->gen == rxc->vxcr_gen,
1604 ("%s: generation mismatch", __func__));
1605 KASSERT(nfrags < IFLIB_MAX_RX_SEGS,
1606 ("%s: too many fragments", __func__));
1607 if (__predict_true(rxcd->len != 0)) {
1608 frag = &ri->iri_frags[nfrags];
1609 flid = (rxcd->qid >= scctx->isc_nrxqsets) ? 1 : 0;
1610 frag->irf_flid = flid;
1611 frag->irf_idx = rxcd->rxd_idx;
1612 frag->irf_len = rxcd->len;
1613 total_len += rxcd->len;
1614 nfrags++;
1615 } else {
1616 rxc->vcxr_zero_length_frag++;
1617 }
1618 if (++cqidx == rxc->vxcr_ndesc) {
1619 cqidx = 0;
1620 rxc->vxcr_gen ^= 1;
1621 }
1622 } while (!rxcd->eop);
1623
1624 ri->iri_cidx = cqidx;
1625 ri->iri_nfrags = nfrags;
1626 ri->iri_len = total_len;
1627
1628 /*
1629 * If there's an error, the last descriptor in the packet will
1630 * have the error indicator set. In this case, set all
1631 * fragment lengths to zero. This will cause iflib to discard
1632 * the packet, but process all associated descriptors through
1633 * the refill mechanism.
1634 */
1635 if (__predict_false(rxcd->error)) {
1636 rxc->vxcr_pkt_errors++;
1637 for (i = 0; i < nfrags; i++) {
1638 frag = &ri->iri_frags[i];
1639 frag->irf_len = 0;
1640 }
1641 } else {
1642 /* Checksum offload information is in the last descriptor. */
1643 if (!rxcd->no_csum) {
1644 uint32_t csum_flags = 0;
1645
1646 if (rxcd->ipv4) {
1647 csum_flags |= CSUM_IP_CHECKED;
1648 if (rxcd->ipcsum_ok)
1649 csum_flags |= CSUM_IP_VALID;
1650 }
1651 if (!rxcd->fragment && (rxcd->tcp || rxcd->udp)) {
1652 csum_flags |= CSUM_L4_CALC;
1653 if (rxcd->csum_ok) {
1654 csum_flags |= CSUM_L4_VALID;
1655 ri->iri_csum_data = 0xffff;
1656 }
1657 }
1658 ri->iri_csum_flags = csum_flags;
1659 }
1660
1661 /* VLAN information is in the last descriptor. */
1662 if (rxcd->vlan) {
1663 ri->iri_flags |= M_VLANTAG;
1664 ri->iri_vtag = rxcd->vtag;
1665 }
1666 }
1667
1668 return (0);
1669 }
1670
1671 static void
vmxnet3_isc_rxd_refill(void * vsc,if_rxd_update_t iru)1672 vmxnet3_isc_rxd_refill(void *vsc, if_rxd_update_t iru)
1673 {
1674 struct vmxnet3_softc *sc;
1675 struct vmxnet3_rxqueue *rxq;
1676 struct vmxnet3_rxring *rxr;
1677 struct vmxnet3_rxdesc *rxd;
1678 uint64_t *paddrs;
1679 int count;
1680 int len;
1681 int idx;
1682 int i;
1683 uint8_t flid;
1684 uint8_t btype;
1685
1686 count = iru->iru_count;
1687 len = iru->iru_buf_size;
1688 flid = iru->iru_flidx;
1689 paddrs = iru->iru_paddrs;
1690
1691 sc = vsc;
1692 rxq = &sc->vmx_rxq[iru->iru_qsidx];
1693 rxr = &rxq->vxrxq_cmd_ring[flid];
1694 rxd = rxr->vxrxr_rxd;
1695
1696 /*
1697 * Command ring 0 is filled with BTYPE_HEAD descriptors, and
1698 * command ring 1 is filled with BTYPE_BODY descriptors.
1699 */
1700 btype = (flid == 0) ? VMXNET3_BTYPE_HEAD : VMXNET3_BTYPE_BODY;
1701 /*
1702 * The refill entries from iflib will advance monotonically,
1703 * but the refilled descriptors may not be contiguous due to
1704 * earlier skipping of descriptors by the device. The refill
1705 * entries from iflib need an entire state update, while the
1706 * descriptors previously skipped by the device only need to
1707 * have their generation numbers updated.
1708 */
1709 idx = rxr->vxrxr_refill_start;
1710 i = 0;
1711 do {
1712 if (idx == iru->iru_idxs[i]) {
1713 rxd[idx].addr = paddrs[i];
1714 rxd[idx].len = len;
1715 rxd[idx].btype = btype;
1716 i++;
1717 } else
1718 rxr->vxrxr_desc_skips++;
1719 rxd[idx].gen = rxr->vxrxr_gen;
1720
1721 if (++idx == rxr->vxrxr_ndesc) {
1722 idx = 0;
1723 rxr->vxrxr_gen ^= 1;
1724 }
1725 } while (i != count);
1726 rxr->vxrxr_refill_start = idx;
1727 }
1728
1729 static void
vmxnet3_isc_rxd_flush(void * vsc,uint16_t rxqid,uint8_t flid,qidx_t pidx)1730 vmxnet3_isc_rxd_flush(void *vsc, uint16_t rxqid, uint8_t flid, qidx_t pidx)
1731 {
1732 struct vmxnet3_softc *sc;
1733 struct vmxnet3_rxqueue *rxq;
1734 struct vmxnet3_rxring *rxr;
1735 bus_size_t r;
1736
1737 sc = vsc;
1738 rxq = &sc->vmx_rxq[rxqid];
1739 rxr = &rxq->vxrxq_cmd_ring[flid];
1740
1741 if (flid == 0)
1742 r = VMXNET3_BAR0_RXH1(rxqid);
1743 else
1744 r = VMXNET3_BAR0_RXH2(rxqid);
1745
1746 vmxnet3_write_bar0(sc, r, pidx);
1747 }
1748
1749 static int
vmxnet3_legacy_intr(void * xsc)1750 vmxnet3_legacy_intr(void *xsc)
1751 {
1752 struct vmxnet3_softc *sc;
1753 if_softc_ctx_t scctx;
1754 if_ctx_t ctx;
1755
1756 sc = xsc;
1757 scctx = sc->vmx_scctx;
1758 ctx = sc->vmx_ctx;
1759
1760 /*
1761 * When there is only a single interrupt configured, this routine
1762 * runs in fast interrupt context, following which the rxq 0 task
1763 * will be enqueued.
1764 */
1765 if (scctx->isc_intr == IFLIB_INTR_LEGACY) {
1766 if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0)
1767 return (FILTER_HANDLED);
1768 }
1769 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
1770 vmxnet3_intr_disable_all(ctx);
1771
1772 if (sc->vmx_ds->event != 0)
1773 iflib_admin_intr_deferred(ctx);
1774
1775 /*
1776 * XXX - When there is both rxq and event activity, do we care
1777 * whether the rxq 0 task or the admin task re-enables the interrupt
1778 * first?
1779 */
1780 return (FILTER_SCHEDULE_THREAD);
1781 }
1782
1783 static int
vmxnet3_rxq_intr(void * vrxq)1784 vmxnet3_rxq_intr(void *vrxq)
1785 {
1786 struct vmxnet3_softc *sc;
1787 struct vmxnet3_rxqueue *rxq;
1788
1789 rxq = vrxq;
1790 sc = rxq->vxrxq_sc;
1791
1792 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
1793 vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx);
1794
1795 return (FILTER_SCHEDULE_THREAD);
1796 }
1797
1798 static int
vmxnet3_event_intr(void * vsc)1799 vmxnet3_event_intr(void *vsc)
1800 {
1801 struct vmxnet3_softc *sc;
1802
1803 sc = vsc;
1804
1805 if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
1806 vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx);
1807
1808 /*
1809 * The work will be done via vmxnet3_update_admin_status(), and the
1810 * interrupt will be re-enabled in vmxnet3_link_intr_enable().
1811 *
1812 * The interrupt will be re-enabled by vmxnet3_link_intr_enable().
1813 */
1814 return (FILTER_SCHEDULE_THREAD);
1815 }
1816
1817 static void
vmxnet3_stop(if_ctx_t ctx)1818 vmxnet3_stop(if_ctx_t ctx)
1819 {
1820 struct vmxnet3_softc *sc;
1821
1822 sc = iflib_get_softc(ctx);
1823
1824 sc->vmx_link_active = 0;
1825 vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE);
1826 vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET);
1827 }
1828
1829 static void
vmxnet3_txinit(struct vmxnet3_softc * sc,struct vmxnet3_txqueue * txq)1830 vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
1831 {
1832 struct vmxnet3_txring *txr;
1833 struct vmxnet3_comp_ring *txc;
1834
1835 txq->vxtxq_last_flush = -1;
1836
1837 txr = &txq->vxtxq_cmd_ring;
1838 txr->vxtxr_next = 0;
1839 txr->vxtxr_gen = VMXNET3_INIT_GEN;
1840 /*
1841 * iflib has zeroed out the descriptor array during the prior attach
1842 * or stop
1843 */
1844
1845 txc = &txq->vxtxq_comp_ring;
1846 txc->vxcr_next = 0;
1847 txc->vxcr_gen = VMXNET3_INIT_GEN;
1848 /*
1849 * iflib has zeroed out the descriptor array during the prior attach
1850 * or stop
1851 */
1852 }
1853
1854 static void
vmxnet3_rxinit(struct vmxnet3_softc * sc,struct vmxnet3_rxqueue * rxq)1855 vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
1856 {
1857 struct vmxnet3_rxring *rxr;
1858 struct vmxnet3_comp_ring *rxc;
1859 int i;
1860
1861 /*
1862 * The descriptors will be populated with buffers during a
1863 * subsequent invocation of vmxnet3_isc_rxd_refill()
1864 */
1865 for (i = 0; i < sc->vmx_sctx->isc_nrxqs - 1; i++) {
1866 rxr = &rxq->vxrxq_cmd_ring[i];
1867 rxr->vxrxr_gen = VMXNET3_INIT_GEN;
1868 rxr->vxrxr_desc_skips = 0;
1869 rxr->vxrxr_refill_start = 0;
1870 /*
1871 * iflib has zeroed out the descriptor array during the
1872 * prior attach or stop
1873 */
1874 }
1875
1876 for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) {
1877 rxr = &rxq->vxrxq_cmd_ring[i];
1878 rxr->vxrxr_gen = 0;
1879 rxr->vxrxr_desc_skips = 0;
1880 rxr->vxrxr_refill_start = 0;
1881 bzero(rxr->vxrxr_rxd,
1882 rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
1883 }
1884
1885 rxc = &rxq->vxrxq_comp_ring;
1886 rxc->vxcr_next = 0;
1887 rxc->vxcr_gen = VMXNET3_INIT_GEN;
1888 rxc->vxcr_zero_length = 0;
1889 rxc->vcxr_zero_length_frag = 0;
1890 rxc->vxcr_pkt_errors = 0;
1891 /*
1892 * iflib has zeroed out the descriptor array during the prior attach
1893 * or stop
1894 */
1895 }
1896
1897 static void
vmxnet3_reinit_queues(struct vmxnet3_softc * sc)1898 vmxnet3_reinit_queues(struct vmxnet3_softc *sc)
1899 {
1900 if_softc_ctx_t scctx;
1901 int q;
1902
1903 scctx = sc->vmx_scctx;
1904
1905 for (q = 0; q < scctx->isc_ntxqsets; q++)
1906 vmxnet3_txinit(sc, &sc->vmx_txq[q]);
1907
1908 for (q = 0; q < scctx->isc_nrxqsets; q++)
1909 vmxnet3_rxinit(sc, &sc->vmx_rxq[q]);
1910 }
1911
1912 static int
vmxnet3_enable_device(struct vmxnet3_softc * sc)1913 vmxnet3_enable_device(struct vmxnet3_softc *sc)
1914 {
1915 if_softc_ctx_t scctx;
1916 int q;
1917
1918 scctx = sc->vmx_scctx;
1919
1920 if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) {
1921 device_printf(sc->vmx_dev, "device enable command failed!\n");
1922 return (1);
1923 }
1924
1925 /* Reset the Rx queue heads. */
1926 for (q = 0; q < scctx->isc_nrxqsets; q++) {
1927 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0);
1928 vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0);
1929 }
1930
1931 return (0);
1932 }
1933
1934 static void
vmxnet3_reinit_rxfilters(struct vmxnet3_softc * sc)1935 vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc)
1936 {
1937 struct ifnet *ifp;
1938
1939 ifp = sc->vmx_ifp;
1940
1941 vmxnet3_set_rxfilter(sc, if_getflags(ifp));
1942
1943 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
1944 bcopy(sc->vmx_vlan_filter, sc->vmx_ds->vlan_filter,
1945 sizeof(sc->vmx_ds->vlan_filter));
1946 else
1947 bzero(sc->vmx_ds->vlan_filter,
1948 sizeof(sc->vmx_ds->vlan_filter));
1949 vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER);
1950 }
1951
1952 static void
vmxnet3_init(if_ctx_t ctx)1953 vmxnet3_init(if_ctx_t ctx)
1954 {
1955 struct vmxnet3_softc *sc;
1956
1957 sc = iflib_get_softc(ctx);
1958
1959 /* Use the current MAC address. */
1960 bcopy(IF_LLADDR(sc->vmx_ifp), sc->vmx_lladdr, ETHER_ADDR_LEN);
1961 vmxnet3_set_lladdr(sc);
1962
1963 vmxnet3_reinit_shared_data(sc);
1964 vmxnet3_reinit_queues(sc);
1965
1966 vmxnet3_enable_device(sc);
1967
1968 vmxnet3_reinit_rxfilters(sc);
1969 vmxnet3_link_status(sc);
1970 }
1971
1972 static void
vmxnet3_multi_set(if_ctx_t ctx)1973 vmxnet3_multi_set(if_ctx_t ctx)
1974 {
1975
1976 vmxnet3_set_rxfilter(iflib_get_softc(ctx),
1977 if_getflags(iflib_get_ifp(ctx)));
1978 }
1979
1980 static int
vmxnet3_mtu_set(if_ctx_t ctx,uint32_t mtu)1981 vmxnet3_mtu_set(if_ctx_t ctx, uint32_t mtu)
1982 {
1983 struct vmxnet3_softc *sc;
1984 if_softc_ctx_t scctx;
1985
1986 sc = iflib_get_softc(ctx);
1987 scctx = sc->vmx_scctx;
1988
1989 if (mtu > VMXNET3_TX_MAXSIZE - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN +
1990 ETHER_CRC_LEN))
1991 return (EINVAL);
1992
1993 /*
1994 * Update the max frame size so that the rx mbuf size is
1995 * chosen based on the new mtu during the interface init that
1996 * will occur after this routine returns.
1997 */
1998 scctx->isc_max_frame_size = mtu +
1999 ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + ETHER_CRC_LEN;
2000 /* RX completion queue - n/a */
2001 scctx->isc_rxd_buf_size[0] = 0;
2002 /*
2003 * For header-type descriptors (used for first segment of
2004 * packet), let iflib determine the buffer size based on the
2005 * max frame size.
2006 */
2007 scctx->isc_rxd_buf_size[1] = 0;
2008 /*
2009 * For body-type descriptors (used for jumbo frames and LRO),
2010 * always use page-sized buffers.
2011 */
2012 scctx->isc_rxd_buf_size[2] = MJUMPAGESIZE;
2013
2014 return (0);
2015 }
2016
2017 static void
vmxnet3_media_status(if_ctx_t ctx,struct ifmediareq * ifmr)2018 vmxnet3_media_status(if_ctx_t ctx, struct ifmediareq * ifmr)
2019 {
2020 struct vmxnet3_softc *sc;
2021
2022 sc = iflib_get_softc(ctx);
2023
2024 ifmr->ifm_status = IFM_AVALID;
2025 ifmr->ifm_active = IFM_ETHER;
2026
2027 if (vmxnet3_link_is_up(sc) != 0) {
2028 ifmr->ifm_status |= IFM_ACTIVE;
2029 ifmr->ifm_active |= IFM_AUTO;
2030 } else
2031 ifmr->ifm_active |= IFM_NONE;
2032 }
2033
2034 static int
vmxnet3_media_change(if_ctx_t ctx)2035 vmxnet3_media_change(if_ctx_t ctx)
2036 {
2037
2038 /* Ignore. */
2039 return (0);
2040 }
2041
2042 static int
vmxnet3_promisc_set(if_ctx_t ctx,int flags)2043 vmxnet3_promisc_set(if_ctx_t ctx, int flags)
2044 {
2045
2046 vmxnet3_set_rxfilter(iflib_get_softc(ctx), flags);
2047
2048 return (0);
2049 }
2050
2051 static uint64_t
vmxnet3_get_counter(if_ctx_t ctx,ift_counter cnt)2052 vmxnet3_get_counter(if_ctx_t ctx, ift_counter cnt)
2053 {
2054 if_t ifp = iflib_get_ifp(ctx);
2055
2056 if (cnt < IFCOUNTERS)
2057 return if_get_counter_default(ifp, cnt);
2058
2059 return (0);
2060 }
2061
2062 static void
vmxnet3_update_admin_status(if_ctx_t ctx)2063 vmxnet3_update_admin_status(if_ctx_t ctx)
2064 {
2065 struct vmxnet3_softc *sc;
2066
2067 sc = iflib_get_softc(ctx);
2068 if (sc->vmx_ds->event != 0)
2069 vmxnet3_evintr(sc);
2070
2071 vmxnet3_refresh_host_stats(sc);
2072 }
2073
2074 static void
vmxnet3_txq_timer(if_ctx_t ctx,uint16_t qid)2075 vmxnet3_txq_timer(if_ctx_t ctx, uint16_t qid)
2076 {
2077 /* Host stats refresh is global, so just trigger it on txq 0 */
2078 if (qid == 0)
2079 vmxnet3_refresh_host_stats(iflib_get_softc(ctx));
2080 }
2081
2082 static void
vmxnet3_update_vlan_filter(struct vmxnet3_softc * sc,int add,uint16_t tag)2083 vmxnet3_update_vlan_filter(struct vmxnet3_softc *sc, int add, uint16_t tag)
2084 {
2085 int idx, bit;
2086
2087 if (tag == 0 || tag > 4095)
2088 return;
2089
2090 idx = (tag >> 5) & 0x7F;
2091 bit = tag & 0x1F;
2092
2093 /* Update our private VLAN bitvector. */
2094 if (add)
2095 sc->vmx_vlan_filter[idx] |= (1 << bit);
2096 else
2097 sc->vmx_vlan_filter[idx] &= ~(1 << bit);
2098 }
2099
2100 static void
vmxnet3_vlan_register(if_ctx_t ctx,uint16_t tag)2101 vmxnet3_vlan_register(if_ctx_t ctx, uint16_t tag)
2102 {
2103
2104 vmxnet3_update_vlan_filter(iflib_get_softc(ctx), 1, tag);
2105 }
2106
2107 static void
vmxnet3_vlan_unregister(if_ctx_t ctx,uint16_t tag)2108 vmxnet3_vlan_unregister(if_ctx_t ctx, uint16_t tag)
2109 {
2110
2111 vmxnet3_update_vlan_filter(iflib_get_softc(ctx), 0, tag);
2112 }
2113
2114 static u_int
vmxnet3_hash_maddr(void * arg,struct sockaddr_dl * sdl,u_int count)2115 vmxnet3_hash_maddr(void *arg, struct sockaddr_dl *sdl, u_int count)
2116 {
2117 struct vmxnet3_softc *sc = arg;
2118
2119 if (count < VMXNET3_MULTICAST_MAX)
2120 bcopy(LLADDR(sdl), &sc->vmx_mcast[count * ETHER_ADDR_LEN],
2121 ETHER_ADDR_LEN);
2122
2123 return (1);
2124 }
2125
2126 static void
vmxnet3_set_rxfilter(struct vmxnet3_softc * sc,int flags)2127 vmxnet3_set_rxfilter(struct vmxnet3_softc *sc, int flags)
2128 {
2129 struct ifnet *ifp;
2130 struct vmxnet3_driver_shared *ds;
2131 u_int mode;
2132
2133 ifp = sc->vmx_ifp;
2134 ds = sc->vmx_ds;
2135
2136 mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST;
2137 if (flags & IFF_PROMISC)
2138 mode |= VMXNET3_RXMODE_PROMISC;
2139 if (flags & IFF_ALLMULTI)
2140 mode |= VMXNET3_RXMODE_ALLMULTI;
2141 else {
2142 int cnt;
2143
2144 cnt = if_foreach_llmaddr(ifp, vmxnet3_hash_maddr, sc);
2145 if (cnt >= VMXNET3_MULTICAST_MAX) {
2146 cnt = 0;
2147 mode |= VMXNET3_RXMODE_ALLMULTI;
2148 } else if (cnt > 0)
2149 mode |= VMXNET3_RXMODE_MCAST;
2150 ds->mcast_tablelen = cnt * ETHER_ADDR_LEN;
2151 }
2152
2153 ds->rxmode = mode;
2154
2155 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER);
2156 vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE);
2157 }
2158
2159 static void
vmxnet3_refresh_host_stats(struct vmxnet3_softc * sc)2160 vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc)
2161 {
2162
2163 vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS);
2164 }
2165
2166 static int
vmxnet3_link_is_up(struct vmxnet3_softc * sc)2167 vmxnet3_link_is_up(struct vmxnet3_softc *sc)
2168 {
2169 uint32_t status;
2170
2171 status = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK);
2172 return !!(status & 0x1);
2173 }
2174
2175 static void
vmxnet3_link_status(struct vmxnet3_softc * sc)2176 vmxnet3_link_status(struct vmxnet3_softc *sc)
2177 {
2178 if_ctx_t ctx;
2179 uint64_t speed;
2180 int link;
2181
2182 ctx = sc->vmx_ctx;
2183 link = vmxnet3_link_is_up(sc);
2184 speed = IF_Gbps(10);
2185
2186 if (link != 0 && sc->vmx_link_active == 0) {
2187 sc->vmx_link_active = 1;
2188 iflib_link_state_change(ctx, LINK_STATE_UP, speed);
2189 } else if (link == 0 && sc->vmx_link_active != 0) {
2190 sc->vmx_link_active = 0;
2191 iflib_link_state_change(ctx, LINK_STATE_DOWN, speed);
2192 }
2193 }
2194
2195 static void
vmxnet3_set_lladdr(struct vmxnet3_softc * sc)2196 vmxnet3_set_lladdr(struct vmxnet3_softc *sc)
2197 {
2198 uint32_t ml, mh;
2199
2200 ml = sc->vmx_lladdr[0];
2201 ml |= sc->vmx_lladdr[1] << 8;
2202 ml |= sc->vmx_lladdr[2] << 16;
2203 ml |= sc->vmx_lladdr[3] << 24;
2204 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml);
2205
2206 mh = sc->vmx_lladdr[4];
2207 mh |= sc->vmx_lladdr[5] << 8;
2208 vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh);
2209 }
2210
2211 static void
vmxnet3_get_lladdr(struct vmxnet3_softc * sc)2212 vmxnet3_get_lladdr(struct vmxnet3_softc *sc)
2213 {
2214 uint32_t ml, mh;
2215
2216 ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL);
2217 mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH);
2218
2219 sc->vmx_lladdr[0] = ml;
2220 sc->vmx_lladdr[1] = ml >> 8;
2221 sc->vmx_lladdr[2] = ml >> 16;
2222 sc->vmx_lladdr[3] = ml >> 24;
2223 sc->vmx_lladdr[4] = mh;
2224 sc->vmx_lladdr[5] = mh >> 8;
2225 }
2226
2227 static void
vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue * txq,struct sysctl_ctx_list * ctx,struct sysctl_oid_list * child)2228 vmxnet3_setup_txq_sysctl(struct vmxnet3_txqueue *txq,
2229 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
2230 {
2231 struct sysctl_oid *node, *txsnode;
2232 struct sysctl_oid_list *list, *txslist;
2233 struct UPT1_TxStats *txstats;
2234 char namebuf[16];
2235
2236 txstats = &txq->vxtxq_ts->stats;
2237
2238 snprintf(namebuf, sizeof(namebuf), "txq%d", txq->vxtxq_id);
2239 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
2240 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Transmit Queue");
2241 txq->vxtxq_sysctl = list = SYSCTL_CHILDREN(node);
2242
2243 /*
2244 * Add statistics reported by the host. These are updated by the
2245 * iflib txq timer on txq 0.
2246 */
2247 txsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats",
2248 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics");
2249 txslist = SYSCTL_CHILDREN(txsnode);
2250 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_packets", CTLFLAG_RD,
2251 &txstats->TSO_packets, "TSO packets");
2252 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "tso_bytes", CTLFLAG_RD,
2253 &txstats->TSO_bytes, "TSO bytes");
2254 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "ucast_packets", CTLFLAG_RD,
2255 &txstats->ucast_packets, "Unicast packets");
2256 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD,
2257 &txstats->ucast_bytes, "Unicast bytes");
2258 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_packets", CTLFLAG_RD,
2259 &txstats->mcast_packets, "Multicast packets");
2260 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD,
2261 &txstats->mcast_bytes, "Multicast bytes");
2262 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "error", CTLFLAG_RD,
2263 &txstats->error, "Errors");
2264 SYSCTL_ADD_UQUAD(ctx, txslist, OID_AUTO, "discard", CTLFLAG_RD,
2265 &txstats->discard, "Discards");
2266 }
2267
2268 static void
vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue * rxq,struct sysctl_ctx_list * ctx,struct sysctl_oid_list * child)2269 vmxnet3_setup_rxq_sysctl(struct vmxnet3_rxqueue *rxq,
2270 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
2271 {
2272 struct sysctl_oid *node, *rxsnode;
2273 struct sysctl_oid_list *list, *rxslist;
2274 struct UPT1_RxStats *rxstats;
2275 char namebuf[16];
2276
2277 rxstats = &rxq->vxrxq_rs->stats;
2278
2279 snprintf(namebuf, sizeof(namebuf), "rxq%d", rxq->vxrxq_id);
2280 node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
2281 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Receive Queue");
2282 rxq->vxrxq_sysctl = list = SYSCTL_CHILDREN(node);
2283
2284 /*
2285 * Add statistics reported by the host. These are updated by the
2286 * iflib txq timer on txq 0.
2287 */
2288 rxsnode = SYSCTL_ADD_NODE(ctx, list, OID_AUTO, "hstats",
2289 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Host Statistics");
2290 rxslist = SYSCTL_CHILDREN(rxsnode);
2291 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_packets", CTLFLAG_RD,
2292 &rxstats->LRO_packets, "LRO packets");
2293 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "lro_bytes", CTLFLAG_RD,
2294 &rxstats->LRO_bytes, "LRO bytes");
2295 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "ucast_packets", CTLFLAG_RD,
2296 &rxstats->ucast_packets, "Unicast packets");
2297 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "unicast_bytes", CTLFLAG_RD,
2298 &rxstats->ucast_bytes, "Unicast bytes");
2299 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_packets", CTLFLAG_RD,
2300 &rxstats->mcast_packets, "Multicast packets");
2301 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "mcast_bytes", CTLFLAG_RD,
2302 &rxstats->mcast_bytes, "Multicast bytes");
2303 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_packets", CTLFLAG_RD,
2304 &rxstats->bcast_packets, "Broadcast packets");
2305 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "bcast_bytes", CTLFLAG_RD,
2306 &rxstats->bcast_bytes, "Broadcast bytes");
2307 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "nobuffer", CTLFLAG_RD,
2308 &rxstats->nobuffer, "No buffer");
2309 SYSCTL_ADD_UQUAD(ctx, rxslist, OID_AUTO, "error", CTLFLAG_RD,
2310 &rxstats->error, "Errors");
2311 }
2312
2313 static void
vmxnet3_setup_debug_sysctl(struct vmxnet3_softc * sc,struct sysctl_ctx_list * ctx,struct sysctl_oid_list * child)2314 vmxnet3_setup_debug_sysctl(struct vmxnet3_softc *sc,
2315 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
2316 {
2317 if_softc_ctx_t scctx;
2318 struct sysctl_oid *node;
2319 struct sysctl_oid_list *list;
2320 int i;
2321
2322 scctx = sc->vmx_scctx;
2323
2324 for (i = 0; i < scctx->isc_ntxqsets; i++) {
2325 struct vmxnet3_txqueue *txq = &sc->vmx_txq[i];
2326
2327 node = SYSCTL_ADD_NODE(ctx, txq->vxtxq_sysctl, OID_AUTO,
2328 "debug", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "");
2329 list = SYSCTL_CHILDREN(node);
2330
2331 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_next", CTLFLAG_RD,
2332 &txq->vxtxq_cmd_ring.vxtxr_next, 0, "");
2333 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd_ndesc", CTLFLAG_RD,
2334 &txq->vxtxq_cmd_ring.vxtxr_ndesc, 0, "");
2335 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd_gen", CTLFLAG_RD,
2336 &txq->vxtxq_cmd_ring.vxtxr_gen, 0, "");
2337 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_next", CTLFLAG_RD,
2338 &txq->vxtxq_comp_ring.vxcr_next, 0, "");
2339 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD,
2340 &txq->vxtxq_comp_ring.vxcr_ndesc, 0,"");
2341 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD,
2342 &txq->vxtxq_comp_ring.vxcr_gen, 0, "");
2343 }
2344
2345 for (i = 0; i < scctx->isc_nrxqsets; i++) {
2346 struct vmxnet3_rxqueue *rxq = &sc->vmx_rxq[i];
2347
2348 node = SYSCTL_ADD_NODE(ctx, rxq->vxrxq_sysctl, OID_AUTO,
2349 "debug", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "");
2350 list = SYSCTL_CHILDREN(node);
2351
2352 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd0_ndesc", CTLFLAG_RD,
2353 &rxq->vxrxq_cmd_ring[0].vxrxr_ndesc, 0, "");
2354 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd0_gen", CTLFLAG_RD,
2355 &rxq->vxrxq_cmd_ring[0].vxrxr_gen, 0, "");
2356 SYSCTL_ADD_U64(ctx, list, OID_AUTO, "cmd0_desc_skips", CTLFLAG_RD,
2357 &rxq->vxrxq_cmd_ring[0].vxrxr_desc_skips, 0, "");
2358 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "cmd1_ndesc", CTLFLAG_RD,
2359 &rxq->vxrxq_cmd_ring[1].vxrxr_ndesc, 0, "");
2360 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "cmd1_gen", CTLFLAG_RD,
2361 &rxq->vxrxq_cmd_ring[1].vxrxr_gen, 0, "");
2362 SYSCTL_ADD_U64(ctx, list, OID_AUTO, "cmd1_desc_skips", CTLFLAG_RD,
2363 &rxq->vxrxq_cmd_ring[1].vxrxr_desc_skips, 0, "");
2364 SYSCTL_ADD_UINT(ctx, list, OID_AUTO, "comp_ndesc", CTLFLAG_RD,
2365 &rxq->vxrxq_comp_ring.vxcr_ndesc, 0,"");
2366 SYSCTL_ADD_INT(ctx, list, OID_AUTO, "comp_gen", CTLFLAG_RD,
2367 &rxq->vxrxq_comp_ring.vxcr_gen, 0, "");
2368 SYSCTL_ADD_U64(ctx, list, OID_AUTO, "comp_zero_length", CTLFLAG_RD,
2369 &rxq->vxrxq_comp_ring.vxcr_zero_length, 0, "");
2370 SYSCTL_ADD_U64(ctx, list, OID_AUTO, "comp_zero_length_frag",
2371 CTLFLAG_RD, &rxq->vxrxq_comp_ring.vcxr_zero_length_frag,
2372 0, "");
2373 SYSCTL_ADD_U64(ctx, list, OID_AUTO, "comp_pkt_errors", CTLFLAG_RD,
2374 &rxq->vxrxq_comp_ring.vxcr_pkt_errors, 0, "");
2375 }
2376 }
2377
2378 static void
vmxnet3_setup_queue_sysctl(struct vmxnet3_softc * sc,struct sysctl_ctx_list * ctx,struct sysctl_oid_list * child)2379 vmxnet3_setup_queue_sysctl(struct vmxnet3_softc *sc,
2380 struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child)
2381 {
2382 if_softc_ctx_t scctx;
2383 int i;
2384
2385 scctx = sc->vmx_scctx;
2386
2387 for (i = 0; i < scctx->isc_ntxqsets; i++)
2388 vmxnet3_setup_txq_sysctl(&sc->vmx_txq[i], ctx, child);
2389 for (i = 0; i < scctx->isc_nrxqsets; i++)
2390 vmxnet3_setup_rxq_sysctl(&sc->vmx_rxq[i], ctx, child);
2391
2392 vmxnet3_setup_debug_sysctl(sc, ctx, child);
2393 }
2394
2395 static void
vmxnet3_setup_sysctl(struct vmxnet3_softc * sc)2396 vmxnet3_setup_sysctl(struct vmxnet3_softc *sc)
2397 {
2398 device_t dev;
2399 struct sysctl_ctx_list *ctx;
2400 struct sysctl_oid *tree;
2401 struct sysctl_oid_list *child;
2402
2403 dev = sc->vmx_dev;
2404 ctx = device_get_sysctl_ctx(dev);
2405 tree = device_get_sysctl_tree(dev);
2406 child = SYSCTL_CHILDREN(tree);
2407
2408 vmxnet3_setup_queue_sysctl(sc, ctx, child);
2409 }
2410
2411 static void
vmxnet3_write_bar0(struct vmxnet3_softc * sc,bus_size_t r,uint32_t v)2412 vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
2413 {
2414
2415 bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v);
2416 }
2417
2418 static uint32_t
vmxnet3_read_bar1(struct vmxnet3_softc * sc,bus_size_t r)2419 vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r)
2420 {
2421
2422 return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r));
2423 }
2424
2425 static void
vmxnet3_write_bar1(struct vmxnet3_softc * sc,bus_size_t r,uint32_t v)2426 vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
2427 {
2428
2429 bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v);
2430 }
2431
2432 static void
vmxnet3_write_cmd(struct vmxnet3_softc * sc,uint32_t cmd)2433 vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
2434 {
2435
2436 vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd);
2437 }
2438
2439 static uint32_t
vmxnet3_read_cmd(struct vmxnet3_softc * sc,uint32_t cmd)2440 vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
2441 {
2442
2443 vmxnet3_write_cmd(sc, cmd);
2444 bus_space_barrier(sc->vmx_iot1, sc->vmx_ioh1, 0, 0,
2445 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
2446 return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD));
2447 }
2448
2449 static void
vmxnet3_enable_intr(struct vmxnet3_softc * sc,int irq)2450 vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq)
2451 {
2452
2453 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0);
2454 }
2455
2456 static void
vmxnet3_disable_intr(struct vmxnet3_softc * sc,int irq)2457 vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq)
2458 {
2459
2460 vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1);
2461 }
2462
2463 static int
vmxnet3_tx_queue_intr_enable(if_ctx_t ctx,uint16_t qid)2464 vmxnet3_tx_queue_intr_enable(if_ctx_t ctx, uint16_t qid)
2465 {
2466 /* Not using interrupts for TX */
2467 return (0);
2468 }
2469
2470 static int
vmxnet3_rx_queue_intr_enable(if_ctx_t ctx,uint16_t qid)2471 vmxnet3_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid)
2472 {
2473 struct vmxnet3_softc *sc;
2474
2475 sc = iflib_get_softc(ctx);
2476 vmxnet3_enable_intr(sc, sc->vmx_rxq[qid].vxrxq_intr_idx);
2477 return (0);
2478 }
2479
2480 static void
vmxnet3_link_intr_enable(if_ctx_t ctx)2481 vmxnet3_link_intr_enable(if_ctx_t ctx)
2482 {
2483 struct vmxnet3_softc *sc;
2484
2485 sc = iflib_get_softc(ctx);
2486 vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx);
2487 }
2488
2489 static void
vmxnet3_intr_enable_all(if_ctx_t ctx)2490 vmxnet3_intr_enable_all(if_ctx_t ctx)
2491 {
2492 struct vmxnet3_softc *sc;
2493 if_softc_ctx_t scctx;
2494 int i;
2495
2496 sc = iflib_get_softc(ctx);
2497 scctx = sc->vmx_scctx;
2498 sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL;
2499 for (i = 0; i < scctx->isc_vectors; i++)
2500 vmxnet3_enable_intr(sc, i);
2501 }
2502
2503 static void
vmxnet3_intr_disable_all(if_ctx_t ctx)2504 vmxnet3_intr_disable_all(if_ctx_t ctx)
2505 {
2506 struct vmxnet3_softc *sc;
2507 int i;
2508
2509 sc = iflib_get_softc(ctx);
2510 /*
2511 * iflib may invoke this routine before vmxnet3_attach_post() has
2512 * run, which is before the top level shared data area is
2513 * initialized and the device made aware of it.
2514 */
2515 if (sc->vmx_ds != NULL)
2516 sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL;
2517 for (i = 0; i < VMXNET3_MAX_INTRS; i++)
2518 vmxnet3_disable_intr(sc, i);
2519 }
2520
2521 /*
2522 * Since this is a purely paravirtualized device, we do not have
2523 * to worry about DMA coherency. But at times, we must make sure
2524 * both the compiler and CPU do not reorder memory operations.
2525 */
2526 static inline void
vmxnet3_barrier(struct vmxnet3_softc * sc,vmxnet3_barrier_t type)2527 vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type)
2528 {
2529
2530 switch (type) {
2531 case VMXNET3_BARRIER_RD:
2532 rmb();
2533 break;
2534 case VMXNET3_BARRIER_WR:
2535 wmb();
2536 break;
2537 case VMXNET3_BARRIER_RDWR:
2538 mb();
2539 break;
2540 default:
2541 panic("%s: bad barrier type %d", __func__, type);
2542 }
2543 }
2544