1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2008 The FreeBSD Foundation
5 * Copyright (c) 2009-2010 Bjoern A. Zeeb <[email protected]>
6 * All rights reserved.
7 *
8 * This software was developed by CK Software GmbH under sponsorship
9 * from the FreeBSD Foundation.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33 /*
34 * A pair of virtual back-to-back connected ethernet like interfaces
35 * (``two interfaces with a virtual cross-over cable'').
36 *
37 * This is mostly intended to be used to provide connectivity between
38 * different virtual network stack instances.
39 */
40 /*
41 * Things to re-think once we have more experience:
42 * - ifp->if_reassign function once we can test with vimage. Depending on
43 * how if_vmove() is going to be improved.
44 * - Real random etheraddrs that are checked to be uniquish; we would need
45 * to re-do them in case we move the interface between network stacks
46 * in a private if_reassign function.
47 * In case we bridge to a real interface/network or between indepedent
48 * epairs on multiple stacks/machines, we may need this.
49 * For now let the user handle that case.
50 */
51
52 #include <sys/cdefs.h>
53 __FBSDID("$FreeBSD$");
54
55 #include <sys/param.h>
56 #include <sys/hash.h>
57 #include <sys/jail.h>
58 #include <sys/kernel.h>
59 #include <sys/libkern.h>
60 #include <sys/malloc.h>
61 #include <sys/mbuf.h>
62 #include <sys/module.h>
63 #include <sys/proc.h>
64 #include <sys/refcount.h>
65 #include <sys/queue.h>
66 #include <sys/smp.h>
67 #include <sys/socket.h>
68 #include <sys/sockio.h>
69 #include <sys/sysctl.h>
70 #include <sys/types.h>
71
72 #include <net/bpf.h>
73 #include <net/ethernet.h>
74 #include <net/if.h>
75 #include <net/if_var.h>
76 #include <net/if_clone.h>
77 #include <net/if_media.h>
78 #include <net/if_var.h>
79 #include <net/if_types.h>
80 #include <net/netisr.h>
81 #include <net/vnet.h>
82
83 SYSCTL_DECL(_net_link);
84 static SYSCTL_NODE(_net_link, OID_AUTO, epair, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
85 "epair sysctl");
86
87 #ifdef EPAIR_DEBUG
88 static int epair_debug = 0;
89 SYSCTL_INT(_net_link_epair, OID_AUTO, epair_debug, CTLFLAG_RW,
90 &epair_debug, 0, "if_epair(4) debugging.");
91 #define DPRINTF(fmt, arg...) \
92 if (epair_debug) \
93 printf("[%s:%d] " fmt, __func__, __LINE__, ##arg)
94 #else
95 #define DPRINTF(fmt, arg...)
96 #endif
97
98 static void epair_nh_sintr(struct mbuf *);
99 static struct mbuf *epair_nh_m2cpuid(struct mbuf *, uintptr_t, u_int *);
100 static void epair_nh_drainedcpu(u_int);
101
102 static void epair_start_locked(struct ifnet *);
103 static int epair_media_change(struct ifnet *);
104 static void epair_media_status(struct ifnet *, struct ifmediareq *);
105
106 static int epair_clone_match(struct if_clone *, const char *);
107 static int epair_clone_create(struct if_clone *, char *, size_t, caddr_t);
108 static int epair_clone_destroy(struct if_clone *, struct ifnet *);
109
110 static const char epairname[] = "epair";
111 static unsigned int next_index = 0;
112
113 /* Netisr related definitions and sysctl. */
114 static struct netisr_handler epair_nh = {
115 .nh_name = epairname,
116 .nh_proto = NETISR_EPAIR,
117 .nh_policy = NETISR_POLICY_CPU,
118 .nh_handler = epair_nh_sintr,
119 .nh_m2cpuid = epair_nh_m2cpuid,
120 .nh_drainedcpu = epair_nh_drainedcpu,
121 };
122
123 static int
sysctl_epair_netisr_maxqlen(SYSCTL_HANDLER_ARGS)124 sysctl_epair_netisr_maxqlen(SYSCTL_HANDLER_ARGS)
125 {
126 int error, qlimit;
127
128 netisr_getqlimit(&epair_nh, &qlimit);
129 error = sysctl_handle_int(oidp, &qlimit, 0, req);
130 if (error || !req->newptr)
131 return (error);
132 if (qlimit < 1)
133 return (EINVAL);
134 return (netisr_setqlimit(&epair_nh, qlimit));
135 }
136 SYSCTL_PROC(_net_link_epair, OID_AUTO, netisr_maxqlen,
137 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0,
138 sysctl_epair_netisr_maxqlen, "I",
139 "Maximum if_epair(4) netisr \"hw\" queue length");
140
141 struct epair_softc {
142 struct ifnet *ifp; /* This ifp. */
143 struct ifnet *oifp; /* other ifp of pair. */
144 struct ifmedia media; /* Media config (fake). */
145 u_int refcount; /* # of mbufs in flight. */
146 u_int cpuid; /* CPU ID assigned upon creation. */
147 void (*if_qflush)(struct ifnet *);
148 /* Original if_qflush routine. */
149 };
150
151 /*
152 * Per-CPU list of ifps with data in the ifq that needs to be flushed
153 * to the netisr ``hw'' queue before we allow any further direct queuing
154 * to the ``hw'' queue.
155 */
156 struct epair_ifp_drain {
157 STAILQ_ENTRY(epair_ifp_drain) ifp_next;
158 struct ifnet *ifp;
159 };
160 STAILQ_HEAD(eid_list, epair_ifp_drain);
161
162 #define EPAIR_LOCK_INIT(dpcpu) mtx_init(&(dpcpu)->if_epair_mtx, \
163 "if_epair", NULL, MTX_DEF)
164 #define EPAIR_LOCK_DESTROY(dpcpu) mtx_destroy(&(dpcpu)->if_epair_mtx)
165 #define EPAIR_LOCK_ASSERT(dpcpu) mtx_assert(&(dpcpu)->if_epair_mtx, \
166 MA_OWNED)
167 #define EPAIR_LOCK(dpcpu) mtx_lock(&(dpcpu)->if_epair_mtx)
168 #define EPAIR_UNLOCK(dpcpu) mtx_unlock(&(dpcpu)->if_epair_mtx)
169
170 #ifdef INVARIANTS
171 #define EPAIR_REFCOUNT_INIT(r, v) refcount_init((r), (v))
172 #define EPAIR_REFCOUNT_AQUIRE(r) refcount_acquire((r))
173 #define EPAIR_REFCOUNT_RELEASE(r) refcount_release((r))
174 #define EPAIR_REFCOUNT_ASSERT(a, p) KASSERT(a, p)
175 #else
176 #define EPAIR_REFCOUNT_INIT(r, v)
177 #define EPAIR_REFCOUNT_AQUIRE(r)
178 #define EPAIR_REFCOUNT_RELEASE(r)
179 #define EPAIR_REFCOUNT_ASSERT(a, p)
180 #endif
181
182 static MALLOC_DEFINE(M_EPAIR, epairname,
183 "Pair of virtual cross-over connected Ethernet-like interfaces");
184
185 VNET_DEFINE_STATIC(struct if_clone *, epair_cloner);
186 #define V_epair_cloner VNET(epair_cloner)
187
188 /*
189 * DPCPU area and functions.
190 */
191 struct epair_dpcpu {
192 struct mtx if_epair_mtx; /* Per-CPU locking. */
193 int epair_drv_flags; /* Per-CPU ``hw'' drv flags. */
194 struct eid_list epair_ifp_drain_list; /* Per-CPU list of ifps with
195 * data in the ifq. */
196 };
197 DPCPU_DEFINE(struct epair_dpcpu, epair_dpcpu);
198
199 static void
epair_dpcpu_init(void)200 epair_dpcpu_init(void)
201 {
202 struct epair_dpcpu *epair_dpcpu;
203 struct eid_list *s;
204 u_int cpuid;
205
206 CPU_FOREACH(cpuid) {
207 epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu);
208
209 /* Initialize per-cpu lock. */
210 EPAIR_LOCK_INIT(epair_dpcpu);
211
212 /* Driver flags are per-cpu as are our netisr "hw" queues. */
213 epair_dpcpu->epair_drv_flags = 0;
214
215 /*
216 * Initialize per-cpu drain list.
217 * Manually do what STAILQ_HEAD_INITIALIZER would do.
218 */
219 s = &epair_dpcpu->epair_ifp_drain_list;
220 s->stqh_first = NULL;
221 s->stqh_last = &s->stqh_first;
222 }
223 }
224
225 static void
epair_dpcpu_detach(void)226 epair_dpcpu_detach(void)
227 {
228 struct epair_dpcpu *epair_dpcpu;
229 u_int cpuid;
230
231 CPU_FOREACH(cpuid) {
232 epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu);
233
234 /* Destroy per-cpu lock. */
235 EPAIR_LOCK_DESTROY(epair_dpcpu);
236 }
237 }
238
239 /*
240 * Helper functions.
241 */
242 static u_int
cpuid_from_ifp(struct ifnet * ifp)243 cpuid_from_ifp(struct ifnet *ifp)
244 {
245 struct epair_softc *sc;
246
247 if (ifp == NULL)
248 return (0);
249 sc = ifp->if_softc;
250
251 return (sc->cpuid);
252 }
253
254 /*
255 * Netisr handler functions.
256 */
257 static void
epair_nh_sintr(struct mbuf * m)258 epair_nh_sintr(struct mbuf *m)
259 {
260 struct ifnet *ifp;
261 struct epair_softc *sc __unused;
262
263 ifp = m->m_pkthdr.rcvif;
264 (*ifp->if_input)(ifp, m);
265 sc = ifp->if_softc;
266 EPAIR_REFCOUNT_RELEASE(&sc->refcount);
267 EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1,
268 ("%s: ifp=%p sc->refcount not >= 1: %d",
269 __func__, ifp, sc->refcount));
270 DPRINTF("ifp=%p refcount=%u\n", ifp, sc->refcount);
271 }
272
273 static struct mbuf *
epair_nh_m2cpuid(struct mbuf * m,uintptr_t source,u_int * cpuid)274 epair_nh_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid)
275 {
276
277 *cpuid = cpuid_from_ifp(m->m_pkthdr.rcvif);
278
279 return (m);
280 }
281
282 static void
epair_nh_drainedcpu(u_int cpuid)283 epair_nh_drainedcpu(u_int cpuid)
284 {
285 struct epair_dpcpu *epair_dpcpu;
286 struct epair_ifp_drain *elm, *tvar;
287 struct ifnet *ifp;
288
289 epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu);
290 EPAIR_LOCK(epair_dpcpu);
291 /*
292 * Assume our "hw" queue and possibly ifq will be emptied
293 * again. In case we will overflow the "hw" queue while
294 * draining, epair_start_locked will set IFF_DRV_OACTIVE
295 * again and we will stop and return.
296 */
297 STAILQ_FOREACH_SAFE(elm, &epair_dpcpu->epair_ifp_drain_list,
298 ifp_next, tvar) {
299 ifp = elm->ifp;
300 epair_dpcpu->epair_drv_flags &= ~IFF_DRV_OACTIVE;
301 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
302 epair_start_locked(ifp);
303
304 IFQ_LOCK(&ifp->if_snd);
305 if (IFQ_IS_EMPTY(&ifp->if_snd)) {
306 struct epair_softc *sc __unused;
307
308 STAILQ_REMOVE(&epair_dpcpu->epair_ifp_drain_list,
309 elm, epair_ifp_drain, ifp_next);
310 /* The cached ifp goes off the list. */
311 sc = ifp->if_softc;
312 EPAIR_REFCOUNT_RELEASE(&sc->refcount);
313 EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1,
314 ("%s: ifp=%p sc->refcount not >= 1: %d",
315 __func__, ifp, sc->refcount));
316 free(elm, M_EPAIR);
317 }
318 IFQ_UNLOCK(&ifp->if_snd);
319
320 if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) != 0) {
321 /* Our "hw"q overflew again. */
322 epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE;
323 DPRINTF("hw queue length overflow at %u\n",
324 epair_nh.nh_qlimit);
325 break;
326 }
327 }
328 EPAIR_UNLOCK(epair_dpcpu);
329 }
330
331 /*
332 * Network interface (`if') related functions.
333 */
334 static void
epair_remove_ifp_from_draining(struct ifnet * ifp)335 epair_remove_ifp_from_draining(struct ifnet *ifp)
336 {
337 struct epair_dpcpu *epair_dpcpu;
338 struct epair_ifp_drain *elm, *tvar;
339 u_int cpuid;
340
341 CPU_FOREACH(cpuid) {
342 epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu);
343 EPAIR_LOCK(epair_dpcpu);
344 STAILQ_FOREACH_SAFE(elm, &epair_dpcpu->epair_ifp_drain_list,
345 ifp_next, tvar) {
346 if (ifp == elm->ifp) {
347 struct epair_softc *sc __unused;
348
349 STAILQ_REMOVE(
350 &epair_dpcpu->epair_ifp_drain_list, elm,
351 epair_ifp_drain, ifp_next);
352 /* The cached ifp goes off the list. */
353 sc = ifp->if_softc;
354 EPAIR_REFCOUNT_RELEASE(&sc->refcount);
355 EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1,
356 ("%s: ifp=%p sc->refcount not >= 1: %d",
357 __func__, ifp, sc->refcount));
358 free(elm, M_EPAIR);
359 }
360 }
361 EPAIR_UNLOCK(epair_dpcpu);
362 }
363 }
364
365 static int
epair_add_ifp_for_draining(struct ifnet * ifp)366 epair_add_ifp_for_draining(struct ifnet *ifp)
367 {
368 struct epair_dpcpu *epair_dpcpu;
369 struct epair_softc *sc;
370 struct epair_ifp_drain *elm = NULL;
371
372 sc = ifp->if_softc;
373 epair_dpcpu = DPCPU_ID_PTR(sc->cpuid, epair_dpcpu);
374 EPAIR_LOCK_ASSERT(epair_dpcpu);
375 STAILQ_FOREACH(elm, &epair_dpcpu->epair_ifp_drain_list, ifp_next)
376 if (elm->ifp == ifp)
377 break;
378 /* If the ifp is there already, return success. */
379 if (elm != NULL)
380 return (0);
381
382 elm = malloc(sizeof(struct epair_ifp_drain), M_EPAIR, M_NOWAIT|M_ZERO);
383 if (elm == NULL)
384 return (ENOMEM);
385
386 elm->ifp = ifp;
387 /* Add a reference for the ifp pointer on the list. */
388 EPAIR_REFCOUNT_AQUIRE(&sc->refcount);
389 STAILQ_INSERT_TAIL(&epair_dpcpu->epair_ifp_drain_list, elm, ifp_next);
390
391 return (0);
392 }
393
394 static void
epair_start_locked(struct ifnet * ifp)395 epair_start_locked(struct ifnet *ifp)
396 {
397 struct epair_dpcpu *epair_dpcpu;
398 struct mbuf *m;
399 struct epair_softc *sc;
400 struct ifnet *oifp;
401 int error;
402
403 DPRINTF("ifp=%p\n", ifp);
404 sc = ifp->if_softc;
405 epair_dpcpu = DPCPU_ID_PTR(sc->cpuid, epair_dpcpu);
406 EPAIR_LOCK_ASSERT(epair_dpcpu);
407
408 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
409 return;
410 if ((ifp->if_flags & IFF_UP) == 0)
411 return;
412
413 /*
414 * We get packets here from ether_output via if_handoff()
415 * and need to put them into the input queue of the oifp
416 * and call oifp->if_input() via netisr/epair_sintr().
417 */
418 oifp = sc->oifp;
419 sc = oifp->if_softc;
420 for (;;) {
421 IFQ_DEQUEUE(&ifp->if_snd, m);
422 if (m == NULL)
423 break;
424 BPF_MTAP(ifp, m);
425
426 /*
427 * In case the outgoing interface is not usable,
428 * drop the packet.
429 */
430 if ((oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
431 (oifp->if_flags & IFF_UP) ==0) {
432 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
433 m_freem(m);
434 continue;
435 }
436 DPRINTF("packet %s -> %s\n", ifp->if_xname, oifp->if_xname);
437
438 /*
439 * Add a reference so the interface cannot go while the
440 * packet is in transit as we rely on rcvif to stay valid.
441 */
442 EPAIR_REFCOUNT_AQUIRE(&sc->refcount);
443 m->m_pkthdr.rcvif = oifp;
444 CURVNET_SET_QUIET(oifp->if_vnet);
445 error = netisr_queue(NETISR_EPAIR, m);
446 CURVNET_RESTORE();
447 if (!error) {
448 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
449 /* Someone else received the packet. */
450 if_inc_counter(oifp, IFCOUNTER_IPACKETS, 1);
451 } else {
452 /* The packet was freed already. */
453 epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE;
454 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
455 (void) epair_add_ifp_for_draining(ifp);
456 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
457 EPAIR_REFCOUNT_RELEASE(&sc->refcount);
458 EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1,
459 ("%s: ifp=%p sc->refcount not >= 1: %d",
460 __func__, oifp, sc->refcount));
461 }
462 }
463 }
464
465 static void
epair_start(struct ifnet * ifp)466 epair_start(struct ifnet *ifp)
467 {
468 struct epair_dpcpu *epair_dpcpu;
469
470 epair_dpcpu = DPCPU_ID_PTR(cpuid_from_ifp(ifp), epair_dpcpu);
471 EPAIR_LOCK(epair_dpcpu);
472 epair_start_locked(ifp);
473 EPAIR_UNLOCK(epair_dpcpu);
474 }
475
476 static int
epair_transmit_locked(struct ifnet * ifp,struct mbuf * m)477 epair_transmit_locked(struct ifnet *ifp, struct mbuf *m)
478 {
479 struct epair_dpcpu *epair_dpcpu;
480 struct epair_softc *sc;
481 struct ifnet *oifp;
482 int error, len;
483 short mflags;
484
485 DPRINTF("ifp=%p m=%p\n", ifp, m);
486 sc = ifp->if_softc;
487 epair_dpcpu = DPCPU_ID_PTR(sc->cpuid, epair_dpcpu);
488 EPAIR_LOCK_ASSERT(epair_dpcpu);
489
490 if (m == NULL)
491 return (0);
492
493 /*
494 * We are not going to use the interface en/dequeue mechanism
495 * on the TX side. We are called from ether_output_frame()
496 * and will put the packet into the incoming queue of the
497 * other interface of our pair via the netsir.
498 */
499 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
500 m_freem(m);
501 return (ENXIO);
502 }
503 if ((ifp->if_flags & IFF_UP) == 0) {
504 m_freem(m);
505 return (ENETDOWN);
506 }
507
508 BPF_MTAP(ifp, m);
509
510 /*
511 * In case the outgoing interface is not usable,
512 * drop the packet.
513 */
514 oifp = sc->oifp;
515 if ((oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 ||
516 (oifp->if_flags & IFF_UP) ==0) {
517 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
518 m_freem(m);
519 return (0);
520 }
521 len = m->m_pkthdr.len;
522 mflags = m->m_flags;
523 DPRINTF("packet %s -> %s\n", ifp->if_xname, oifp->if_xname);
524
525 #ifdef ALTQ
526 /* Support ALTQ via the classic if_start() path. */
527 IF_LOCK(&ifp->if_snd);
528 if (ALTQ_IS_ENABLED(&ifp->if_snd)) {
529 ALTQ_ENQUEUE(&ifp->if_snd, m, NULL, error);
530 if (error)
531 if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1);
532 IF_UNLOCK(&ifp->if_snd);
533 if (!error) {
534 if_inc_counter(ifp, IFCOUNTER_OBYTES, len);
535 if (mflags & (M_BCAST|M_MCAST))
536 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
537
538 if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0)
539 epair_start_locked(ifp);
540 else
541 (void)epair_add_ifp_for_draining(ifp);
542 }
543 return (error);
544 }
545 IF_UNLOCK(&ifp->if_snd);
546 #endif
547
548 if ((epair_dpcpu->epair_drv_flags & IFF_DRV_OACTIVE) != 0) {
549 /*
550 * Our hardware queue is full, try to fall back
551 * queuing to the ifq but do not call ifp->if_start.
552 * Either we are lucky or the packet is gone.
553 */
554 IFQ_ENQUEUE(&ifp->if_snd, m, error);
555 if (!error)
556 (void)epair_add_ifp_for_draining(ifp);
557 return (error);
558 }
559 sc = oifp->if_softc;
560 /*
561 * Add a reference so the interface cannot go while the
562 * packet is in transit as we rely on rcvif to stay valid.
563 */
564 EPAIR_REFCOUNT_AQUIRE(&sc->refcount);
565 m->m_pkthdr.rcvif = oifp;
566 CURVNET_SET_QUIET(oifp->if_vnet);
567 error = netisr_queue(NETISR_EPAIR, m);
568 CURVNET_RESTORE();
569 if (!error) {
570 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1);
571 /*
572 * IFQ_HANDOFF_ADJ/ip_handoff() update statistics,
573 * but as we bypass all this we have to duplicate
574 * the logic another time.
575 */
576 if_inc_counter(ifp, IFCOUNTER_OBYTES, len);
577 if (mflags & (M_BCAST|M_MCAST))
578 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
579 /* Someone else received the packet. */
580 if_inc_counter(oifp, IFCOUNTER_IPACKETS, 1);
581 } else {
582 /* The packet was freed already. */
583 epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE;
584 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
585 if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
586 EPAIR_REFCOUNT_RELEASE(&sc->refcount);
587 EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1,
588 ("%s: ifp=%p sc->refcount not >= 1: %d",
589 __func__, oifp, sc->refcount));
590 }
591
592 return (error);
593 }
594
595 static int
epair_transmit(struct ifnet * ifp,struct mbuf * m)596 epair_transmit(struct ifnet *ifp, struct mbuf *m)
597 {
598 struct epair_dpcpu *epair_dpcpu;
599 int error;
600
601 epair_dpcpu = DPCPU_ID_PTR(cpuid_from_ifp(ifp), epair_dpcpu);
602 EPAIR_LOCK(epair_dpcpu);
603 error = epair_transmit_locked(ifp, m);
604 EPAIR_UNLOCK(epair_dpcpu);
605 return (error);
606 }
607
608 static void
epair_qflush(struct ifnet * ifp)609 epair_qflush(struct ifnet *ifp)
610 {
611 struct epair_softc *sc;
612
613 sc = ifp->if_softc;
614 KASSERT(sc != NULL, ("%s: ifp=%p, epair_softc gone? sc=%p\n",
615 __func__, ifp, sc));
616 /*
617 * Remove this ifp from all backpointer lists. The interface will not
618 * usable for flushing anyway nor should it have anything to flush
619 * after if_qflush().
620 */
621 epair_remove_ifp_from_draining(ifp);
622
623 if (sc->if_qflush)
624 sc->if_qflush(ifp);
625 }
626
627 static int
epair_media_change(struct ifnet * ifp __unused)628 epair_media_change(struct ifnet *ifp __unused)
629 {
630
631 /* Do nothing. */
632 return (0);
633 }
634
635 static void
epair_media_status(struct ifnet * ifp __unused,struct ifmediareq * imr)636 epair_media_status(struct ifnet *ifp __unused, struct ifmediareq *imr)
637 {
638
639 imr->ifm_status = IFM_AVALID | IFM_ACTIVE;
640 imr->ifm_active = IFM_ETHER | IFM_10G_T | IFM_FDX;
641 }
642
643 static int
epair_ioctl(struct ifnet * ifp,u_long cmd,caddr_t data)644 epair_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
645 {
646 struct epair_softc *sc;
647 struct ifreq *ifr;
648 int error;
649
650 ifr = (struct ifreq *)data;
651 switch (cmd) {
652 case SIOCSIFFLAGS:
653 case SIOCADDMULTI:
654 case SIOCDELMULTI:
655 error = 0;
656 break;
657
658 case SIOCSIFMEDIA:
659 case SIOCGIFMEDIA:
660 sc = ifp->if_softc;
661 error = ifmedia_ioctl(ifp, ifr, &sc->media, cmd);
662 break;
663
664 case SIOCSIFMTU:
665 /* We basically allow all kinds of MTUs. */
666 ifp->if_mtu = ifr->ifr_mtu;
667 error = 0;
668 break;
669
670 default:
671 /* Let the common ethernet handler process this. */
672 error = ether_ioctl(ifp, cmd, data);
673 break;
674 }
675
676 return (error);
677 }
678
679 static void
epair_init(void * dummy __unused)680 epair_init(void *dummy __unused)
681 {
682 }
683
684 /*
685 * Interface cloning functions.
686 * We use our private ones so that we can create/destroy our secondary
687 * device along with the primary one.
688 */
689 static int
epair_clone_match(struct if_clone * ifc,const char * name)690 epair_clone_match(struct if_clone *ifc, const char *name)
691 {
692 const char *cp;
693
694 DPRINTF("name='%s'\n", name);
695
696 /*
697 * Our base name is epair.
698 * Our interfaces will be named epair<n>[ab].
699 * So accept anything of the following list:
700 * - epair
701 * - epair<n>
702 * but not the epair<n>[ab] versions.
703 */
704 if (strncmp(epairname, name, sizeof(epairname)-1) != 0)
705 return (0);
706
707 for (cp = name + sizeof(epairname) - 1; *cp != '\0'; cp++) {
708 if (*cp < '0' || *cp > '9')
709 return (0);
710 }
711
712 return (1);
713 }
714
715 static void
epair_clone_add(struct if_clone * ifc,struct epair_softc * scb)716 epair_clone_add(struct if_clone *ifc, struct epair_softc *scb)
717 {
718 struct ifnet *ifp;
719 uint8_t eaddr[ETHER_ADDR_LEN]; /* 00:00:00:00:00:00 */
720
721 ifp = scb->ifp;
722 /* Copy epairNa etheraddr and change the last byte. */
723 memcpy(eaddr, scb->oifp->if_hw_addr, ETHER_ADDR_LEN);
724 eaddr[5] = 0x0b;
725 ether_ifattach(ifp, eaddr);
726
727 if_clone_addif(ifc, ifp);
728 }
729
730 static int
epair_clone_create(struct if_clone * ifc,char * name,size_t len,caddr_t params)731 epair_clone_create(struct if_clone *ifc, char *name, size_t len, caddr_t params)
732 {
733 struct epair_softc *sca, *scb;
734 struct ifnet *ifp;
735 char *dp;
736 int error, unit, wildcard;
737 uint64_t hostid;
738 uint32_t key[3];
739 uint32_t hash;
740 uint8_t eaddr[ETHER_ADDR_LEN]; /* 00:00:00:00:00:00 */
741
742 /* Try to see if a special unit was requested. */
743 error = ifc_name2unit(name, &unit);
744 if (error != 0)
745 return (error);
746 wildcard = (unit < 0);
747
748 error = ifc_alloc_unit(ifc, &unit);
749 if (error != 0)
750 return (error);
751
752 /*
753 * If no unit had been given, we need to adjust the ifName.
754 * Also make sure there is space for our extra [ab] suffix.
755 */
756 for (dp = name; *dp != '\0'; dp++);
757 if (wildcard) {
758 error = snprintf(dp, len - (dp - name), "%d", unit);
759 if (error > len - (dp - name) - 1) {
760 /* ifName too long. */
761 ifc_free_unit(ifc, unit);
762 return (ENOSPC);
763 }
764 dp += error;
765 }
766 if (len - (dp - name) - 1 < 1) {
767 /* No space left for our [ab] suffix. */
768 ifc_free_unit(ifc, unit);
769 return (ENOSPC);
770 }
771 *dp = 'b';
772 /* Must not change dp so we can replace 'a' by 'b' later. */
773 *(dp+1) = '\0';
774
775 /* Check if 'a' and 'b' interfaces already exist. */
776 if (ifunit(name) != NULL)
777 return (EEXIST);
778 *dp = 'a';
779 if (ifunit(name) != NULL)
780 return (EEXIST);
781
782 /* Allocate memory for both [ab] interfaces */
783 sca = malloc(sizeof(struct epair_softc), M_EPAIR, M_WAITOK | M_ZERO);
784 EPAIR_REFCOUNT_INIT(&sca->refcount, 1);
785 sca->ifp = if_alloc(IFT_ETHER);
786 if (sca->ifp == NULL) {
787 free(sca, M_EPAIR);
788 ifc_free_unit(ifc, unit);
789 return (ENOSPC);
790 }
791
792 scb = malloc(sizeof(struct epair_softc), M_EPAIR, M_WAITOK | M_ZERO);
793 EPAIR_REFCOUNT_INIT(&scb->refcount, 1);
794 scb->ifp = if_alloc(IFT_ETHER);
795 if (scb->ifp == NULL) {
796 free(scb, M_EPAIR);
797 if_free(sca->ifp);
798 free(sca, M_EPAIR);
799 ifc_free_unit(ifc, unit);
800 return (ENOSPC);
801 }
802
803 /*
804 * Cross-reference the interfaces so we will be able to free both.
805 */
806 sca->oifp = scb->ifp;
807 scb->oifp = sca->ifp;
808
809 /*
810 * Calculate the cpuid for netisr queueing based on the
811 * ifIndex of the interfaces. As long as we cannot configure
812 * this or use cpuset information easily we cannot guarantee
813 * cache locality but we can at least allow parallelism.
814 */
815 sca->cpuid =
816 netisr_get_cpuid(sca->ifp->if_index);
817 scb->cpuid =
818 netisr_get_cpuid(scb->ifp->if_index);
819
820 /* Initialise pseudo media types. */
821 ifmedia_init(&sca->media, 0, epair_media_change, epair_media_status);
822 ifmedia_add(&sca->media, IFM_ETHER | IFM_10G_T, 0, NULL);
823 ifmedia_set(&sca->media, IFM_ETHER | IFM_10G_T);
824 ifmedia_init(&scb->media, 0, epair_media_change, epair_media_status);
825 ifmedia_add(&scb->media, IFM_ETHER | IFM_10G_T, 0, NULL);
826 ifmedia_set(&scb->media, IFM_ETHER | IFM_10G_T);
827
828 /* Finish initialization of interface <n>a. */
829 ifp = sca->ifp;
830 ifp->if_softc = sca;
831 strlcpy(ifp->if_xname, name, IFNAMSIZ);
832 ifp->if_dname = epairname;
833 ifp->if_dunit = unit;
834 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
835 ifp->if_capabilities = IFCAP_VLAN_MTU;
836 ifp->if_capenable = IFCAP_VLAN_MTU;
837 ifp->if_start = epair_start;
838 ifp->if_ioctl = epair_ioctl;
839 ifp->if_init = epair_init;
840 if_setsendqlen(ifp, ifqmaxlen);
841 if_setsendqready(ifp);
842
843 /*
844 * Calculate the etheraddr hashing the hostid and the
845 * interface index. The result would be hopefully unique.
846 * Note that the "a" component of an epair instance may get moved
847 * to a different VNET after creation. In that case its index
848 * will be freed and the index can get reused by new epair instance.
849 * Make sure we do not create same etheraddr again.
850 */
851 getcredhostid(curthread->td_ucred, (unsigned long *)&hostid);
852 if (hostid == 0)
853 arc4rand(&hostid, sizeof(hostid), 0);
854
855 if (ifp->if_index > next_index)
856 next_index = ifp->if_index;
857 else
858 next_index++;
859
860 key[0] = (uint32_t)next_index;
861 key[1] = (uint32_t)(hostid & 0xffffffff);
862 key[2] = (uint32_t)((hostid >> 32) & 0xfffffffff);
863 hash = jenkins_hash32(key, 3, 0);
864
865 eaddr[0] = 0x02;
866 memcpy(&eaddr[1], &hash, 4);
867 eaddr[5] = 0x0a;
868 ether_ifattach(ifp, eaddr);
869 sca->if_qflush = ifp->if_qflush;
870 ifp->if_qflush = epair_qflush;
871 ifp->if_transmit = epair_transmit;
872 ifp->if_baudrate = IF_Gbps(10); /* arbitrary maximum */
873
874 /* Swap the name and finish initialization of interface <n>b. */
875 *dp = 'b';
876
877 ifp = scb->ifp;
878 ifp->if_softc = scb;
879 strlcpy(ifp->if_xname, name, IFNAMSIZ);
880 ifp->if_dname = epairname;
881 ifp->if_dunit = unit;
882 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
883 ifp->if_capabilities = IFCAP_VLAN_MTU;
884 ifp->if_capenable = IFCAP_VLAN_MTU;
885 ifp->if_start = epair_start;
886 ifp->if_ioctl = epair_ioctl;
887 ifp->if_init = epair_init;
888 if_setsendqlen(ifp, ifqmaxlen);
889 if_setsendqready(ifp);
890 /* We need to play some tricks here for the second interface. */
891 strlcpy(name, epairname, len);
892
893 /* Correctly set the name for the cloner list. */
894 strlcpy(name, scb->ifp->if_xname, len);
895 epair_clone_add(ifc, scb);
896
897 scb->if_qflush = ifp->if_qflush;
898 ifp->if_qflush = epair_qflush;
899 ifp->if_transmit = epair_transmit;
900 ifp->if_baudrate = IF_Gbps(10); /* arbitrary maximum */
901
902 /*
903 * Restore name to <n>a as the ifp for this will go into the
904 * cloner list for the initial call.
905 */
906 strlcpy(name, sca->ifp->if_xname, len);
907 DPRINTF("name='%s/%db' created sca=%p scb=%p\n", name, unit, sca, scb);
908
909 /* Tell the world, that we are ready to rock. */
910 sca->ifp->if_drv_flags |= IFF_DRV_RUNNING;
911 scb->ifp->if_drv_flags |= IFF_DRV_RUNNING;
912 if_link_state_change(sca->ifp, LINK_STATE_UP);
913 if_link_state_change(scb->ifp, LINK_STATE_UP);
914
915 return (0);
916 }
917
918 static int
epair_clone_destroy(struct if_clone * ifc,struct ifnet * ifp)919 epair_clone_destroy(struct if_clone *ifc, struct ifnet *ifp)
920 {
921 struct ifnet *oifp;
922 struct epair_softc *sca, *scb;
923 int unit, error;
924
925 DPRINTF("ifp=%p\n", ifp);
926
927 /*
928 * In case we called into if_clone_destroyif() ourselves
929 * again to remove the second interface, the softc will be
930 * NULL. In that case so not do anything but return success.
931 */
932 if (ifp->if_softc == NULL)
933 return (0);
934
935 unit = ifp->if_dunit;
936 sca = ifp->if_softc;
937 oifp = sca->oifp;
938 scb = oifp->if_softc;
939
940 DPRINTF("ifp=%p oifp=%p\n", ifp, oifp);
941 if_link_state_change(ifp, LINK_STATE_DOWN);
942 if_link_state_change(oifp, LINK_STATE_DOWN);
943 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
944 oifp->if_drv_flags &= ~IFF_DRV_RUNNING;
945
946 /*
947 * Get rid of our second half. As the other of the two
948 * interfaces may reside in a different vnet, we need to
949 * switch before freeing them.
950 */
951 CURVNET_SET_QUIET(oifp->if_vnet);
952 ether_ifdetach(oifp);
953 /*
954 * Wait for all packets to be dispatched to if_input.
955 * The numbers can only go down as the interface is
956 * detached so there is no need to use atomics.
957 */
958 DPRINTF("scb refcnt=%u\n", scb->refcount);
959 EPAIR_REFCOUNT_ASSERT(scb->refcount == 1,
960 ("%s: ifp=%p scb->refcount!=1: %d", __func__, oifp, scb->refcount));
961 oifp->if_softc = NULL;
962 error = if_clone_destroyif(ifc, oifp);
963 if (error)
964 panic("%s: if_clone_destroyif() for our 2nd iface failed: %d",
965 __func__, error);
966 if_free(oifp);
967 ifmedia_removeall(&scb->media);
968 free(scb, M_EPAIR);
969 CURVNET_RESTORE();
970
971 ether_ifdetach(ifp);
972 /*
973 * Wait for all packets to be dispatched to if_input.
974 */
975 DPRINTF("sca refcnt=%u\n", sca->refcount);
976 EPAIR_REFCOUNT_ASSERT(sca->refcount == 1,
977 ("%s: ifp=%p sca->refcount!=1: %d", __func__, ifp, sca->refcount));
978 if_free(ifp);
979 ifmedia_removeall(&sca->media);
980 free(sca, M_EPAIR);
981 ifc_free_unit(ifc, unit);
982
983 return (0);
984 }
985
986 static void
vnet_epair_init(const void * unused __unused)987 vnet_epair_init(const void *unused __unused)
988 {
989
990 V_epair_cloner = if_clone_advanced(epairname, 0,
991 epair_clone_match, epair_clone_create, epair_clone_destroy);
992 #ifdef VIMAGE
993 netisr_register_vnet(&epair_nh);
994 #endif
995 }
996 VNET_SYSINIT(vnet_epair_init, SI_SUB_PSEUDO, SI_ORDER_ANY,
997 vnet_epair_init, NULL);
998
999 static void
vnet_epair_uninit(const void * unused __unused)1000 vnet_epair_uninit(const void *unused __unused)
1001 {
1002
1003 #ifdef VIMAGE
1004 netisr_unregister_vnet(&epair_nh);
1005 #endif
1006 if_clone_detach(V_epair_cloner);
1007 }
1008 VNET_SYSUNINIT(vnet_epair_uninit, SI_SUB_INIT_IF, SI_ORDER_ANY,
1009 vnet_epair_uninit, NULL);
1010
1011 static void
epair_uninit(const void * unused __unused)1012 epair_uninit(const void *unused __unused)
1013 {
1014 netisr_unregister(&epair_nh);
1015 epair_dpcpu_detach();
1016 if (bootverbose)
1017 printf("%s unloaded.\n", epairname);
1018 }
1019 SYSUNINIT(epair_uninit, SI_SUB_INIT_IF, SI_ORDER_MIDDLE,
1020 epair_uninit, NULL);
1021
1022 static int
epair_modevent(module_t mod,int type,void * data)1023 epair_modevent(module_t mod, int type, void *data)
1024 {
1025 int qlimit;
1026
1027 switch (type) {
1028 case MOD_LOAD:
1029 /* For now limit us to one global mutex and one inq. */
1030 epair_dpcpu_init();
1031 epair_nh.nh_qlimit = 42 * ifqmaxlen; /* 42 shall be the number. */
1032 if (TUNABLE_INT_FETCH("net.link.epair.netisr_maxqlen", &qlimit))
1033 epair_nh.nh_qlimit = qlimit;
1034 netisr_register(&epair_nh);
1035 if (bootverbose)
1036 printf("%s initialized.\n", epairname);
1037 break;
1038 case MOD_UNLOAD:
1039 /* Handled in epair_uninit() */
1040 break;
1041 default:
1042 return (EOPNOTSUPP);
1043 }
1044 return (0);
1045 }
1046
1047 static moduledata_t epair_mod = {
1048 "if_epair",
1049 epair_modevent,
1050 0
1051 };
1052
1053 DECLARE_MODULE(if_epair, epair_mod, SI_SUB_PSEUDO, SI_ORDER_MIDDLE);
1054 MODULE_VERSION(if_epair, 1);
1055