/* * Copyright (C) 2013 Universita` di Pisa. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This module implements netmap support on top of standard, * unmodified device drivers. * * A NIOCREGIF request is handled here if the device does not * have native support. TX and RX rings are emulated as follows: * * NIOCREGIF * We preallocate a block of TX mbufs (roughly as many as * tx descriptors; the number is not critical) to speed up * operation during transmissions. The refcount on most of * these buffers is artificially bumped up so we can recycle * them more easily. Also, the destructor is intercepted * so we use it as an interrupt notification to wake up * processes blocked on a poll(). * * For each receive ring we allocate one "struct mbq" * (an mbuf tailq plus a spinlock). We intercept packets * (through if_input) * on the receive path and put them in the mbq from which * netmap receive routines can grab them. * * TX: * in the generic_txsync() routine, netmap buffers are copied * (or linked, in a future) to the preallocated mbufs * and pushed to the transmit queue. Some of these mbufs * (those with NS_REPORT, or otherwise every half ring) * have the refcount=1, others have refcount=2. * When the destructor is invoked, we take that as * a notification that all mbufs up to that one in * the specific ring have been completed, and generate * the equivalent of a transmit interrupt. * * RX: * */ #ifdef __FreeBSD__ #include /* prerequisite */ __FBSDID("$FreeBSD$"); #include #include #include #include /* PROT_EXEC */ #include #include /* sockaddrs */ #include #include #include #include /* bus_dmamap_* in netmap_kern.h */ // XXX temporary - D() defined here #include #include #include #define rtnl_lock() D("rtnl_lock called"); #define rtnl_unlock() D("rtnl_lock called"); #define MBUF_TXQ(m) ((m)->m_pkthdr.flowid) #define smp_mb() /* * mbuf wrappers */ /* * we allocate an EXT_PACKET */ #define netmap_get_mbuf(len) m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR|M_NOFREE) /* mbuf destructor, also need to change the type to EXT_EXTREF, * add an M_NOFREE flag, and then clear the flag and * chain into uma_zfree(zone_pack, mf) * (or reinstall the buffer ?) */ #define SET_MBUF_DESTRUCTOR(m, fn) do { \ (m)->m_ext.ext_free = (void *)fn; \ (m)->m_ext.ext_type = EXT_EXTREF; \ } while (0) #define GET_MBUF_REFCNT(m) ((m)->m_ext.ref_cnt ? *(m)->m_ext.ref_cnt : -1) #else /* linux */ #include "bsd_glue.h" #include /* rtnl_[un]lock() */ #include /* struct ethtool_ops, get_ringparam */ #include //#define RATE /* Enables communication statistics. */ //#define REG_RESET #endif /* linux */ /* Common headers. */ #include #include #include /* ======================== usage stats =========================== */ #ifdef RATE #define IFRATE(x) x struct rate_stats { unsigned long txpkt; unsigned long txsync; unsigned long txirq; unsigned long rxpkt; unsigned long rxirq; unsigned long rxsync; }; struct rate_context { unsigned refcount; struct timer_list timer; struct rate_stats new; struct rate_stats old; }; #define RATE_PRINTK(_NAME_) \ printk( #_NAME_ " = %lu Hz\n", (cur._NAME_ - ctx->old._NAME_)/RATE_PERIOD); #define RATE_PERIOD 2 static void rate_callback(unsigned long arg) { struct rate_context * ctx = (struct rate_context *)arg; struct rate_stats cur = ctx->new; int r; RATE_PRINTK(txpkt); RATE_PRINTK(txsync); RATE_PRINTK(txirq); RATE_PRINTK(rxpkt); RATE_PRINTK(rxsync); RATE_PRINTK(rxirq); printk("\n"); ctx->old = cur; r = mod_timer(&ctx->timer, jiffies + msecs_to_jiffies(RATE_PERIOD * 1000)); if (unlikely(r)) D("[v1000] Error: mod_timer()"); } static struct rate_context rate_ctx; #else /* !RATE */ #define IFRATE(x) #endif /* !RATE */ /* =============== GENERIC NETMAP ADAPTER SUPPORT ================= */ #define GENERIC_BUF_SIZE netmap_buf_size /* Size of the mbufs in the Tx pool. */ /* * Wrapper used by the generic adapter layer to notify * the poller threads. Differently from netmap_rx_irq(), we check * only IFCAP_NETMAP instead of NAF_NATIVE_ON to enable the irq. */ static void netmap_generic_irq(struct ifnet *ifp, u_int q, u_int *work_done) { if (unlikely(!(ifp->if_capenable & IFCAP_NETMAP))) return; netmap_common_irq(ifp, q, work_done); } /* Enable/disable netmap mode for a generic network interface. */ int generic_netmap_register(struct netmap_adapter *na, int enable) { struct ifnet *ifp = na->ifp; struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; struct mbuf *m; int error; int i, r; if (!na) return EINVAL; #ifdef REG_RESET error = ifp->netdev_ops->ndo_stop(ifp); if (error) { return error; } #endif /* REG_RESET */ if (enable) { /* Enable netmap mode. */ /* Initialize the rx queue, as generic_rx_handler() can * be called as soon as netmap_catch_rx() returns. */ for (r=0; rnum_rx_rings; r++) { mbq_safe_init(&na->rx_rings[r].rx_queue); na->rx_rings[r].nr_ntc = 0; } /* Init the mitigation timer. */ netmap_mitigation_init(gna); /* * Preallocate packet buffers for the tx rings. */ for (r=0; rnum_tx_rings; r++) { na->tx_rings[r].nr_ntc = 0; na->tx_rings[r].tx_pool = malloc(na->num_tx_desc * sizeof(struct mbuf *), M_DEVBUF, M_NOWAIT | M_ZERO); if (!na->tx_rings[r].tx_pool) { D("tx_pool allocation failed"); error = ENOMEM; goto free_tx_pool; } for (i=0; inum_tx_desc; i++) { m = netmap_get_mbuf(GENERIC_BUF_SIZE); if (!m) { D("tx_pool[%d] allocation failed", i); error = ENOMEM; goto free_mbufs; } na->tx_rings[r].tx_pool[i] = m; } } rtnl_lock(); /* Prepare to intercept incoming traffic. */ error = netmap_catch_rx(na, 1); if (error) { D("netdev_rx_handler_register() failed"); goto register_handler; } ifp->if_capenable |= IFCAP_NETMAP; /* Make netmap control the packet steering. */ netmap_catch_packet_steering(gna, 1); rtnl_unlock(); #ifdef RATE if (rate_ctx.refcount == 0) { D("setup_timer()"); memset(&rate_ctx, 0, sizeof(rate_ctx)); setup_timer(&rate_ctx.timer, &rate_callback, (unsigned long)&rate_ctx); if (mod_timer(&rate_ctx.timer, jiffies + msecs_to_jiffies(1500))) { D("Error: mod_timer()"); } } rate_ctx.refcount++; #endif /* RATE */ } else { /* Disable netmap mode. */ rtnl_lock(); ifp->if_capenable &= ~IFCAP_NETMAP; /* Release packet steering control. */ netmap_catch_packet_steering(gna, 0); /* Do not intercept packets on the rx path. */ netmap_catch_rx(na, 0); rtnl_unlock(); /* Free the mbufs going to the netmap rings */ for (r=0; rnum_rx_rings; r++) { mbq_safe_purge(&na->rx_rings[r].rx_queue); mbq_safe_destroy(&na->rx_rings[r].rx_queue); } netmap_mitigation_cleanup(gna); for (r=0; rnum_tx_rings; r++) { for (i=0; inum_tx_desc; i++) { m_freem(na->tx_rings[r].tx_pool[i]); } free(na->tx_rings[r].tx_pool, M_DEVBUF); } #ifdef RATE if (--rate_ctx.refcount == 0) { D("del_timer()"); del_timer(&rate_ctx.timer); } #endif } #ifdef REG_RESET error = ifp->netdev_ops->ndo_open(ifp); if (error) { goto alloc_tx_pool; } #endif return 0; register_handler: rtnl_unlock(); free_tx_pool: r--; i = na->num_tx_desc; /* Useless, but just to stay safe. */ free_mbufs: i--; for (; r>=0; r--) { for (; i>=0; i--) { m_freem(na->tx_rings[r].tx_pool[i]); } free(na->tx_rings[r].tx_pool, M_DEVBUF); i = na->num_tx_desc - 1; } return error; } /* * Callback invoked when the device driver frees an mbuf used * by netmap to transmit a packet. This usually happens when * the NIC notifies the driver that transmission is completed. */ static void generic_mbuf_destructor(struct mbuf *m) { if (netmap_verbose) D("Tx irq (%p) queue %d", m, MBUF_TXQ(m)); netmap_generic_irq(MBUF_IFP(m), MBUF_TXQ(m), NULL); #ifdef __FreeBSD__ m->m_ext.ext_type = EXT_PACKET; m->m_ext.ext_free = NULL; if (*(m->m_ext.ref_cnt) == 0) *(m->m_ext.ref_cnt) = 1; uma_zfree(zone_pack, m); #endif /* __FreeBSD__ */ IFRATE(rate_ctx.new.txirq++); } /* Record completed transmissions and update hwavail. * * nr_ntc is the oldest tx buffer not yet completed * (same as nr_hwavail + nr_hwcur + 1), * nr_hwcur is the first unsent buffer. * When cleaning, we try to recover buffers between nr_ntc and nr_hwcur. */ static int generic_netmap_tx_clean(struct netmap_kring *kring) { u_int num_slots = kring->nkr_num_slots; u_int ntc = kring->nr_ntc; u_int hwcur = kring->nr_hwcur; u_int n = 0; struct mbuf **tx_pool = kring->tx_pool; while (ntc != hwcur) { /* buffers not completed */ struct mbuf *m = tx_pool[ntc]; if (unlikely(m == NULL)) { /* try to replenish the entry */ tx_pool[ntc] = m = netmap_get_mbuf(GENERIC_BUF_SIZE); if (unlikely(m == NULL)) { D("mbuf allocation failed, XXX error"); // XXX how do we proceed ? break ? return -ENOMEM; } } else if (GET_MBUF_REFCNT(m) != 1) { break; /* This mbuf is still busy: its refcnt is 2. */ } if (unlikely(++ntc == num_slots)) { ntc = 0; } n++; } kring->nr_ntc = ntc; kring->nr_hwavail += n; ND("tx completed [%d] -> hwavail %d", n, kring->nr_hwavail); return n; } /* * We have pending packets in the driver between nr_ntc and j. * Compute a position in the middle, to be used to generate * a notification. */ static inline u_int generic_tx_event_middle(struct netmap_kring *kring, u_int hwcur) { u_int n = kring->nkr_num_slots; u_int ntc = kring->nr_ntc; u_int e; if (hwcur >= ntc) { e = (hwcur + ntc) / 2; } else { /* wrap around */ e = (hwcur + n + ntc) / 2; if (e >= n) { e -= n; } } if (unlikely(e >= n)) { D("This cannot happen"); e = 0; } return e; } /* * We have pending packets in the driver between nr_ntc and hwcur. * Schedule a notification approximately in the middle of the two. * There is a race but this is only called within txsync which does * a double check. */ static void generic_set_tx_event(struct netmap_kring *kring, u_int hwcur) { struct mbuf *m; u_int e; if (kring->nr_ntc == hwcur) { return; } e = generic_tx_event_middle(kring, hwcur); m = kring->tx_pool[e]; if (m == NULL) { /* This can happen if there is already an event on the netmap slot 'e': There is nothing to do. */ return; } ND("Event at %d mbuf %p refcnt %d", e, m, GET_MBUF_REFCNT(m)); kring->tx_pool[e] = NULL; SET_MBUF_DESTRUCTOR(m, generic_mbuf_destructor); // XXX wmb() ? /* Decrement the refcount an free it if we have the last one. */ m_freem(m); smp_mb(); } /* * generic_netmap_txsync() transforms netmap buffers into mbufs * and passes them to the standard device driver * (ndo_start_xmit() or ifp->if_transmit() ). * On linux this is not done directly, but using dev_queue_xmit(), * since it implements the TX flow control (and takes some locks). */ static int generic_netmap_txsync(struct netmap_adapter *na, u_int ring_nr, int flags) { struct ifnet *ifp = na->ifp; struct netmap_kring *kring = &na->tx_rings[ring_nr]; struct netmap_ring *ring = kring->ring; u_int j, k, num_slots = kring->nkr_num_slots; int new_slots, ntx; IFRATE(rate_ctx.new.txsync++); // TODO: handle the case of mbuf allocation failure /* first, reclaim completed buffers */ generic_netmap_tx_clean(kring); /* Take a copy of ring->cur now, and never read it again. */ k = ring->cur; if (unlikely(k >= num_slots)) { return netmap_ring_reinit(kring); } rmb(); j = kring->nr_hwcur; /* * 'new_slots' counts how many new slots have been added: * everything from hwcur to cur, excluding reserved ones, if any. * nr_hwreserved start from hwcur and counts how many slots were * not sent to the NIC from the previous round. */ new_slots = k - j - kring->nr_hwreserved; if (new_slots < 0) { new_slots += num_slots; } ntx = 0; if (j != k) { /* Process new packets to send: * j is the current index in the netmap ring. */ while (j != k) { struct netmap_slot *slot = &ring->slot[j]; /* Current slot in the netmap ring */ void *addr = NMB(slot); u_int len = slot->len; struct mbuf *m; int tx_ret; if (unlikely(addr == netmap_buffer_base || len > NETMAP_BUF_SIZE)) { return netmap_ring_reinit(kring); } /* Tale a mbuf from the tx pool and copy in the user packet. */ m = kring->tx_pool[j]; if (unlikely(!m)) { RD(5, "This should never happen"); kring->tx_pool[j] = m = netmap_get_mbuf(GENERIC_BUF_SIZE); if (unlikely(m == NULL)) { D("mbuf allocation failed"); break; } } /* XXX we should ask notifications when NS_REPORT is set, * or roughly every half frame. We can optimize this * by lazily requesting notifications only when a * transmission fails. Probably the best way is to * break on failures and set notifications when * ring->avail == 0 || j != k */ tx_ret = generic_xmit_frame(ifp, m, addr, len, ring_nr); if (unlikely(tx_ret)) { RD(5, "start_xmit failed: err %d [%u,%u,%u,%u]", tx_ret, kring->nr_ntc, j, k, kring->nr_hwavail); /* * No room for this mbuf in the device driver. * Request a notification FOR A PREVIOUS MBUF, * then call generic_netmap_tx_clean(kring) to do the * double check and see if we can free more buffers. * If there is space continue, else break; * NOTE: the double check is necessary if the problem * occurs in the txsync call after selrecord(). * Also, we need some way to tell the caller that not * all buffers were queued onto the device (this was * not a problem with native netmap driver where space * is preallocated). The bridge has a similar problem * and we solve it there by dropping the excess packets. */ generic_set_tx_event(kring, j); if (generic_netmap_tx_clean(kring)) { /* space now available */ continue; } else { break; } } slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); if (unlikely(++j == num_slots)) j = 0; ntx++; } /* Update hwcur to the next slot to transmit. */ kring->nr_hwcur = j; /* * Report all new slots as unavailable, even those not sent. * We account for them with with hwreserved, so that * nr_hwreserved =:= cur - nr_hwcur */ kring->nr_hwavail -= new_slots; kring->nr_hwreserved = k - j; if (kring->nr_hwreserved < 0) { kring->nr_hwreserved += num_slots; } IFRATE(rate_ctx.new.txpkt += ntx); if (!kring->nr_hwavail) { /* No more available slots? Set a notification event * on a netmap slot that will be cleaned in the future. * No doublecheck is performed, since txsync() will be * called twice by netmap_poll(). */ generic_set_tx_event(kring, j); } ND("tx #%d, hwavail = %d", n, kring->nr_hwavail); } /* Synchronize the user's view to the kernel view. */ ring->avail = kring->nr_hwavail; ring->reserved = kring->nr_hwreserved; return 0; } /* * This handler is registered (through netmap_catch_rx()) * within the attached network interface * in the RX subsystem, so that every mbuf passed up by * the driver can be stolen to the network stack. * Stolen packets are put in a queue where the * generic_netmap_rxsync() callback can extract them. */ void generic_rx_handler(struct ifnet *ifp, struct mbuf *m) { struct netmap_adapter *na = NA(ifp); struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; u_int work_done; u_int rr = 0; // receive ring number ND("called"); /* limit the size of the queue */ if (unlikely(mbq_len(&na->rx_rings[rr].rx_queue) > 1024)) { m_freem(m); } else { mbq_safe_enqueue(&na->rx_rings[rr].rx_queue, m); } if (netmap_generic_mit < 32768) { /* no rx mitigation, pass notification up */ netmap_generic_irq(na->ifp, rr, &work_done); IFRATE(rate_ctx.new.rxirq++); } else { /* same as send combining, filter notification if there is a * pending timer, otherwise pass it up and start a timer. */ if (likely(netmap_mitigation_active(gna))) { /* Record that there is some pending work. */ gna->mit_pending = 1; } else { netmap_generic_irq(na->ifp, rr, &work_done); IFRATE(rate_ctx.new.rxirq++); netmap_mitigation_start(gna); } } } /* * generic_netmap_rxsync() extracts mbufs from the queue filled by * generic_netmap_rx_handler() and puts their content in the netmap * receive ring. * Access must be protected because the rx handler is asynchronous, */ static int generic_netmap_rxsync(struct netmap_adapter *na, u_int ring_nr, int flags) { struct netmap_kring *kring = &na->rx_rings[ring_nr]; struct netmap_ring *ring = kring->ring; u_int j, n, lim = kring->nkr_num_slots - 1; int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR; u_int k, resvd = ring->reserved; if (ring->cur > lim) return netmap_ring_reinit(kring); /* Import newly received packets into the netmap ring. */ if (netmap_no_pendintr || force_update) { uint16_t slot_flags = kring->nkr_slot_flags; struct mbuf *m; n = 0; j = kring->nr_ntc; /* first empty slot in the receive ring */ /* extract buffers from the rx queue, stop at most one * slot before nr_hwcur (index k) */ k = (kring->nr_hwcur) ? kring->nr_hwcur-1 : lim; while (j != k) { int len; void *addr = NMB(&ring->slot[j]); if (addr == netmap_buffer_base) { /* Bad buffer */ return netmap_ring_reinit(kring); } /* * Call the locked version of the function. * XXX Ideally we could grab a batch of mbufs at once, * by changing rx_queue into a ring. */ m = mbq_safe_dequeue(&kring->rx_queue); if (!m) break; len = MBUF_LEN(m); m_copydata(m, 0, len, addr); ring->slot[j].len = len; ring->slot[j].flags = slot_flags; m_freem(m); if (unlikely(j++ == lim)) j = 0; n++; } if (n) { kring->nr_ntc = j; kring->nr_hwavail += n; IFRATE(rate_ctx.new.rxpkt += n); } kring->nr_kflags &= ~NKR_PENDINTR; } // XXX should we invert the order ? /* Skip past packets that userspace has released */ j = kring->nr_hwcur; k = ring->cur; if (resvd > 0) { if (resvd + ring->avail >= lim + 1) { D("XXX invalid reserve/avail %d %d", resvd, ring->avail); ring->reserved = resvd = 0; // XXX panic... } k = (k >= resvd) ? k - resvd : k + lim + 1 - resvd; } if (j != k) { /* Userspace has released some packets. */ for (n = 0; j != k; n++) { struct netmap_slot *slot = &ring->slot[j]; slot->flags &= ~NS_BUF_CHANGED; if (unlikely(j++ == lim)) j = 0; } kring->nr_hwavail -= n; kring->nr_hwcur = k; } /* Tell userspace that there are new packets. */ ring->avail = kring->nr_hwavail - resvd; IFRATE(rate_ctx.new.rxsync++); return 0; } static void generic_netmap_dtor(struct netmap_adapter *na) { struct ifnet *ifp = na->ifp; struct netmap_generic_adapter *gna = (struct netmap_generic_adapter*)na; struct netmap_adapter *prev_na = gna->prev; if (prev_na != NULL) { D("Released generic NA %p", gna); if_rele(na->ifp); netmap_adapter_put(prev_na); } if (ifp != NULL) { WNA(ifp) = prev_na; D("Restored native NA %p", prev_na); na->ifp = NULL; } } /* * generic_netmap_attach() makes it possible to use netmap on * a device without native netmap support. * This is less performant than native support but potentially * faster than raw sockets or similar schemes. * * In this "emulated" mode, netmap rings do not necessarily * have the same size as those in the NIC. We use a default * value and possibly override it if the OS has ways to fetch the * actual configuration. */ int generic_netmap_attach(struct ifnet *ifp) { struct netmap_adapter *na; struct netmap_generic_adapter *gna; int retval; u_int num_tx_desc, num_rx_desc; num_tx_desc = num_rx_desc = netmap_generic_ringsize; /* starting point */ generic_find_num_desc(ifp, &num_tx_desc, &num_rx_desc); ND("Netmap ring size: TX = %d, RX = %d", num_tx_desc, num_rx_desc); gna = malloc(sizeof(*gna), M_DEVBUF, M_NOWAIT | M_ZERO); if (gna == NULL) { D("no memory on attach, give up"); return ENOMEM; } na = (struct netmap_adapter *)gna; na->ifp = ifp; na->num_tx_desc = num_tx_desc; na->num_rx_desc = num_rx_desc; na->nm_register = &generic_netmap_register; na->nm_txsync = &generic_netmap_txsync; na->nm_rxsync = &generic_netmap_rxsync; na->nm_dtor = &generic_netmap_dtor; /* when using generic, IFCAP_NETMAP is set so we force * NAF_SKIP_INTR to use the regular interrupt handler */ na->na_flags = NAF_SKIP_INTR; ND("[GNA] num_tx_queues(%d), real_num_tx_queues(%d), len(%lu)", ifp->num_tx_queues, ifp->real_num_tx_queues, ifp->tx_queue_len); ND("[GNA] num_rx_queues(%d), real_num_rx_queues(%d)", ifp->num_rx_queues, ifp->real_num_rx_queues); generic_find_num_queues(ifp, &na->num_tx_rings, &na->num_rx_rings); retval = netmap_attach_common(na); if (retval) { free(gna, M_DEVBUF); } return retval; }